Bump version to 5.3.6

Summary:
PR to fix this issue -> https://github.com/facebook/rocksdb/issues/1926
Closes https://github.com/facebook/rocksdb/pull/1965

Differential Revision: D4682411

Pulled By: siying

fbshipit-source-id: a519be1

.travis.yml

@@ -18,12 +18,11 @@ addons:
       sources: ['ubuntu-toolchain-r-test', 'llvm-toolchain-precise-3.6']
       packages: ['clang-3.6' , 'g++-6', 'zlib1g-dev', 'libbz2-dev', 'libsnappy-dev', 'curl', 'libgflags-dev']
 env:
-  # Run all tests before db_block_cache_test (db_test, db_test2)
-  - JOB_NAME=unittests ROCKSDBTESTS_END=db_block_cache_test
-  # Run all tests starting from db_block_cache_test (db_block_cache_test, ..., plain_table_db_test)
-  - JOB_NAME=unittests ROCKSDBTESTS_START=db_block_cache_test ROCKSDBTESTS_END=comparator_db_test
-  # Run all tests starting from db_block_cache_test (comparator_db_test, ...)
-  - JOB_NAME=unittests ROCKSDBTESTS_START=comparator_db_test
+  - TEST_GROUP=platform_dependent1
+  - TEST_GROUP=platform_dependent2
+  - TEST_GROUP=platform_dependent3
+  - TEST_GROUP=1
+  - TEST_GROUP=2
   # Run java tests
   - JOB_NAME=java_test
   # Build ROCKSDB_LITE
@@ -33,6 +32,10 @@ matrix:
   exclude:
   - os: osx
     compiler: gcc
+  - os: osx
+    env: TEST_GROUP=1
+  - os: osx
+    env: TEST_GROUP=2
 
 before_script:
   - if [[ "${TRAVIS_OS_NAME}" == 'linux' && "${CXX}" == 'clang++' ]]; then CXX=clang++-3.6; fi
@@ -43,7 +46,11 @@ before_script:
 
 script:
   - ${CXX} --version
-  - if [[ "${JOB_NAME}" == 'unittests' ]]; then OPT=-DTRAVIS V=1 make -j4 check_some; fi
+  - if [[ "${TEST_GROUP}" == 'platform_dependent1' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_END=external_sst_file_basic_test make -j4 check_some; fi
+  - if [[ "${TEST_GROUP}" == 'platform_dependent2' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=external_sst_file_basic_test ROCKSDBTESTS_END=checkpoint_test make -j4 check_some; fi
+  - if [[ "${TEST_GROUP}" == 'platform_dependent3' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=checkpoint_test ROCKSDBTESTS_END=db_block_cache_test make -j4 check_some; fi
+  - if [[ "${TEST_GROUP}" == '1' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=db_block_cache_test ROCKSDBTESTS_END=comparator_db_test make -j4 check_some; fi
+  - if [[ "${TEST_GROUP}" == '2' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=comparator_db_test make -j4 check_some; fi
   - if [[ "${JOB_NAME}" == 'java_test' ]]; then OPT=-DTRAVIS V=1 make clean jclean rocksdbjava jtest; fi
   - if [[ "${JOB_NAME}" == 'lite_build' ]]; then OPT="-DTRAVIS -DROCKSDB_LITE" V=1 make -j4 static_lib; fi
 notifications:

CMakeLists.txt

@@ -314,7 +314,6 @@ set(SOURCES
         db/write_batch_base.cc
         db/write_controller.cc
         db/write_thread.cc
-        db/xfunc_test_points.cc
         memtable/hash_cuckoo_rep.cc
         memtable/hash_linklist_rep.cc
         memtable/hash_skiplist_rep.cc
@@ -337,10 +336,12 @@ set(SOURCES
         table/format.cc
         table/full_filter_block.cc
         table/get_context.cc
+        table/index_builder.cc
         table/iterator.cc
         table/merging_iterator.cc
         table/sst_file_writer.cc
         table/meta_blocks.cc
+        table/partitioned_filter_block.cc
         table/plain_table_builder.cc
         table/plain_table_factory.cc
         table/plain_table_index.cc
@@ -408,7 +409,6 @@ set(SOURCES
         util/thread_status_util.cc
         util/thread_status_util_debug.cc
         util/transaction_test_util.cc
-        util/xfunc.cc
         util/xxhash.cc
         utilities/backupable/backupable_db.cc
         utilities/blob_db/blob_db.cc
@@ -538,6 +538,7 @@ set(TESTS
         db/comparator_db_test.cc
         db/corruption_test.cc
         db/cuckoo_table_db_test.cc
+        db/db_basic_test.cc
         db/db_compaction_filter_test.cc
         db/db_compaction_test.cc
         db/db_dynamic_level_test.cc
@@ -558,6 +559,7 @@ set(TESTS
         db/db_iterator_test.cc
         db/db_range_del_test.cc
         db/db_sst_test.cc
+        db/external_sst_file_basic_test.cc
         db/external_sst_file_test.cc
         db/db_universal_compaction_test.cc
         db/db_wal_test.cc

HISTORY.md

@@ -1,21 +1,34 @@
 # Rocksdb Change Log
-## Unreleased
+## 5.3.0 (03/08/2017)
+### Public API Change
+* Remove disableDataSync option.
+* Remove timeout_hint_us option from WriteOptions. The option has been deprecated and has no effect since 3.13.0.
+* Remove option min_partial_merge_operands. Partial merge operands will always be merged in flush or compaction if there are more than one.
+* Remove option verify_checksums_in_compaction. Compaction will always verify checksum.
+
+### New Features
+* Memtable flush can be avoided during checkpoint creation if total log file size is smaller than a threshold specified by the user.
+
+### Bug Fixes
+* Fix the bug that iterator may skip keys
 
 ## 5.2.0 (02/08/2017)
 ### Public API Change
 * NewLRUCache() will determine number of shard bits automatically based on capacity, if the user doesn't pass one. This also impacts the default block cache when the user doesn't explict provide one.
 * Change the default of delayed slowdown value to 16MB/s and further increase the L0 stop condition to 36 files.
+* Options::use_direct_writes and Options::use_direct_reads are now ready to use.
 * (Experimental) Two-level indexing that partition the index and creates a 2nd level index on the partitions. The feature can be enabled by setting kTwoLevelIndexSearch as IndexType and configuring index_per_partition.
 
 ### New Features
 * Added new overloaded function GetApproximateSizes that allows to specify if memtable stats should be computed only without computing SST files' stats approximations.
 * Added new function GetApproximateMemTableStats that approximates both number of records and size of memtables.
+* Add Direct I/O mode for SST file I/O
 
 ### Bug Fixes
 * RangeSync() should work if ROCKSDB_FALLOCATE_PRESENT is not set
 * Fix wrong results in a data race case in Get()
 * Some fixes related to 2PC.
-* Fix several bugs in Direct I/O supports.
+* Fix bugs of data corruption in direct I/O
 
 ## 5.1.0 (01/13/2017)
 * Support dynamically change `delete_obsolete_files_period_micros` option via SetDBOptions().

INSTALL.md

@@ -29,8 +29,10 @@ your make commands, like this: `PORTABLE=1 make static_lib`
 * You can link RocksDB with following compression libraries:
   - [zlib](http://www.zlib.net/) - a library for data compression.
   - [bzip2](http://www.bzip.org/) - a library for data compression.
-  - [snappy](https://code.google.com/p/snappy/) - a library for fast
+  - [snappy](http://google.github.io/snappy/) - a library for fast
       data compression.
+  - [zstandard](http://www.zstd.net) - Fast real-time compression
+      algorithm.
 
 * All our tools depend on:
   - [gflags](https://gflags.github.io/gflags/) - a library that handles
@@ -48,21 +50,23 @@ your make commands, like this: `PORTABLE=1 make static_lib`
       `sudo apt-get install libsnappy-dev`.
     * Install zlib. Try: `sudo apt-get install zlib1g-dev`.
     * Install bzip2: `sudo apt-get install libbz2-dev`.
-* **Linux - CentOS**
+    * Install zstandard: `sudo apt-get install libzstd-dev`.
+
+* **Linux - CentOS / RHEL**
     * Upgrade your gcc to version at least 4.8 to get C++11 support:
       `yum install gcc48-c++`
     * Install gflags:
 
-              wget https://gflags.googlecode.com/files/gflags-2.0-no-svn-files.tar.gz
-              tar -xzvf gflags-2.0-no-svn-files.tar.gz
-              cd gflags-2.0
+              git clone https://github.com/gflags/gflags.git
+              git checkout v2.0
+              cd gflags
               ./configure && make && sudo make install
 
     * Install snappy:
 
-              wget https://snappy.googlecode.com/files/snappy-1.1.1.tar.gz
-              tar -xzvf snappy-1.1.1.tar.gz
-              cd snappy-1.1.1
+              wget https://github.com/google/snappy/releases/download/1.1.4/snappy-1.1.4.tar.gz
+              tar -xzvf snappy-1.1.4.tar.gz
+              cd snappy-1.1.4
               ./configure && make && sudo make install
 
     * Install zlib:
@@ -75,6 +79,14 @@ your make commands, like this: `PORTABLE=1 make static_lib`
               sudo yum install bzip2
               sudo yum install bzip2-devel
 
+    * Install zstandard:
+
+             wget https://github.com/facebook/zstd/archive/v1.1.3.tar.gz
+             mv v1.1.3.tar.gz zstandard-1.1.3.tar.gz
+             tar zxvf zstd-1.1.3.tar.gz
+             cd zstd-1.1.3
+             make && sudo make install
+
 * **OS X**:
     * Install latest C++ compiler that supports C++ 11:
         * Update XCode:  run `xcode-select --install` (or install it from XCode App's settting).

LANGUAGE-BINDINGS.md

@@ -10,3 +10,4 @@ This is the list of all known third-party language bindings for RocksDB. If some
 * PHP - https://github.com/Photonios/rocksdb-php
 * C# - https://github.com/warrenfalk/rocksdb-sharp
 * Rust - https://github.com/spacejam/rust-rocksdb
+* D programming language - https://github.com/b1naryth1ef/rocksdb

Makefile

@@ -223,6 +223,11 @@ default: all
 WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare -Wshadow \
   -Wno-unused-parameter
 
+ifndef DISABLE_WARNING_AS_ERROR
+	WARNING_FLAGS += -Werror
+endif
+
+
 ifdef LUA_PATH
 
 ifndef LUA_INCLUDE
@@ -294,9 +299,26 @@ BENCHTOOLOBJECTS = $(BENCH_LIB_SOURCES:.cc=.o) $(LIBOBJECTS) $(TESTUTIL)
 EXPOBJECTS = $(EXP_LIB_SOURCES:.cc=.o) $(LIBOBJECTS) $(TESTUTIL)
 
 TESTS = \
-	db_test \
+	db_basic_test \
 	db_test2 \
+	external_sst_file_basic_test \
+	auto_roll_logger_test \
+	bloom_test \
+	dynamic_bloom_test \
+	c_test \
+	checkpoint_test \
+	crc32c_test \
+	coding_test \
+	inlineskiplist_test \
+	env_basic_test \
+	env_test \
+	thread_local_test \
+	rate_limiter_test \
+	perf_context_test \
+	iostats_context_test \
+	db_wal_test \
 	db_block_cache_test \
+	db_test \
 	db_bloom_filter_test \
 	db_iter_test \
 	db_log_iter_test \
@@ -311,10 +333,8 @@ TESTS = \
 	db_options_test \
 	db_range_del_test \
 	db_sst_test \
-	external_sst_file_test \
 	db_tailing_iter_test \
 	db_universal_compaction_test \
-	db_wal_test \
 	db_io_failure_test \
 	db_properties_test \
 	db_table_properties_test \
@@ -323,20 +343,11 @@ TESTS = \
 	column_family_test \
 	table_properties_collector_test \
 	arena_test \
-	auto_roll_logger_test \
 	block_test \
-	bloom_test \
-	dynamic_bloom_test \
-	c_test \
 	cache_test \
-	checkpoint_test \
-	coding_test \
 	corruption_test \
-	crc32c_test \
 	slice_transform_test \
 	dbformat_test \
-	env_basic_test \
-	env_test \
 	fault_injection_test \
 	filelock_test \
 	filename_test \
@@ -345,7 +356,6 @@ TESTS = \
 	full_filter_block_test \
 	hash_table_test \
 	histogram_test \
-	inlineskiplist_test \
 	log_test \
 	manual_compaction_test \
 	mock_env_test \
@@ -360,6 +370,7 @@ TESTS = \
 	reduce_levels_test \
 	plain_table_db_test \
 	comparator_db_test \
+	external_sst_file_test \
 	prefix_test \
 	skiplist_test \
 	stringappend_test \
@@ -381,9 +392,7 @@ TESTS = \
 	write_controller_test\
 	deletefile_test \
 	table_test \
-	thread_local_test \
 	geodb_test \
-	rate_limiter_test \
 	delete_scheduler_test \
 	options_test \
 	options_settable_test \
@@ -401,7 +410,6 @@ TESTS = \
 	sst_dump_test \
 	column_aware_encoding_test \
 	compact_files_test \
-	perf_context_test \
 	optimistic_transaction_test \
 	write_callback_test \
 	heap_test \
@@ -410,13 +418,13 @@ TESTS = \
 	option_change_migration_test \
 	transaction_test \
 	ldb_cmd_test \
-	iostats_context_test \
 	persistent_cache_test \
 	statistics_test \
 	lua_test \
 	range_del_aggregator_test \
 	lru_cache_test \
 	object_registry_test \
+	repair_test \
 
 PARALLEL_TEST = \
 	backupable_db_test \
@@ -664,6 +672,8 @@ check_0:
 	  | grep -E '$(tests-regexp)'					\
 	  | build_tools/gnu_parallel -j$(J) --plain --joblog=LOG $$eta --gnu '{} >& t/log-{/}'
 
+valgrind-blacklist-regexp = InlineSkipTest.ConcurrentInsert|TransactionTest.DeadlockStress|DBCompactionTest.SuggestCompactRangeNoTwoLevel0Compactions|BackupableDBTest.RateLimiting|DBTest.CloseSpeedup|DBTest.ThreadStatusFlush|DBTest.RateLimitingTest|DBTest.EncodeDecompressedBlockSizeTest|FaultInjectionTest.UninstalledCompaction|HarnessTest.Randomized|ExternalSSTFileTest.CompactDuringAddFileRandom|ExternalSSTFileTest.IngestFileWithGlobalSeqnoRandomized
+
 .PHONY: valgrind_check_0
 valgrind_check_0:
 	$(AM_V_GEN)export TEST_TMPDIR=$(TMPD);				\
@@ -677,6 +687,7 @@ valgrind_check_0:
 	}								\
 	  | $(prioritize_long_running_tests)				\
 	  | grep -E '$(tests-regexp)'					\
+	  | grep -E -v '$(valgrind-blacklist-regexp)'					\
 	  | build_tools/gnu_parallel -j$(J) --plain --joblog=LOG $$eta --gnu \
 	  '(if [[ "{}" == "./"* ]] ; then $(DRIVER) {}; else {}; fi) ' \
 	  '>& t/valgrind_log-{/}'
@@ -962,6 +973,9 @@ crc32c_test: util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS)
 slice_transform_test: util/slice_transform_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
+db_basic_test: db/db_basic_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
 db_test: db/db_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -1010,6 +1024,9 @@ db_range_del_test: db/db_range_del_test.o db/db_test_util.o $(LIBOBJECTS) $(TEST
 db_sst_test: db/db_sst_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
+external_sst_file_basic_test: db/external_sst_file_basic_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
 external_sst_file_test: db/external_sst_file_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -1364,10 +1381,24 @@ ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-linu
 ROCKSDB_JAR_ALL = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH).jar
 ROCKSDB_JAVADOCS_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-javadoc.jar
 ROCKSDB_SOURCES_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-sources.jar
+SHA256_CMD = sha256sum
+
+ZLIB_VER = 1.2.11
+ZLIB_SHA256 = c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1
+BZIP2_VER = 1.0.6
+BZIP2_SHA256 = a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd
+SNAPPY_VER = 1.1.4
+SNAPPY_SHA256 = 134bfe122fd25599bb807bb8130e7ba6d9bdb851e0b16efcb83ac4f5d0b70057
+LZ4_VER = 1.7.5
+LZ4_SHA256 = 0190cacd63022ccb86f44fa5041dc6c3804407ad61550ca21c382827319e7e7e
+ZSTD_VER ?= 1.2.0
+ZSTD_SHA256 ?= 4a7e4593a3638276ca7f2a09dc4f38e674d8317bbea51626393ca73fc047cbfb
+ZSTD_DOWNLOAD_BASE ?= https://github.com/facebook/zstd/archive
 
 ifeq ($(PLATFORM), OS_MACOSX)
 	ROCKSDBJNILIB = librocksdbjni-osx.jnilib
 	ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-osx.jar
+	SHA256_CMD = openssl sha256 -r
 ifneq ("$(wildcard $(JAVA_HOME)/include/darwin)","")
 	JAVA_INCLUDE = -I$(JAVA_HOME)/include -I $(JAVA_HOME)/include/darwin
 else
@@ -1383,46 +1414,84 @@ ifeq ($(PLATFORM), OS_SOLARIS)
 	ROCKSDBJNILIB = librocksdbjni-solaris$(ARCH).so
 	ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-solaris$(ARCH).jar
 	JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/solaris
+	SHA256_CMD = digest -a sha256
 endif
 
 libz.a:
-	-rm -rf zlib-1.2.10
-	curl -O -L http://zlib.net/zlib-1.2.10.tar.gz
-	tar xvzf zlib-1.2.10.tar.gz
-	cd zlib-1.2.10 && CFLAGS='-fPIC' ./configure --static && make
-	cp zlib-1.2.10/libz.a .
+	-rm -rf zlib-$(ZLIB_VER)
+	curl -O -L http://zlib.net/zlib-$(ZLIB_VER).tar.gz
+	ZLIB_SHA256_ACTUAL=`$(SHA256_CMD) zlib-$(ZLIB_VER).tar.gz | cut -d ' ' -f 1`; \
+	if [ "$(ZLIB_SHA256)" != "$$ZLIB_SHA256_ACTUAL" ]; then \
+		echo zlib-$(ZLIB_VER).tar.gz checksum mismatch, expected=\"$(ZLIB_SHA256)\" actual=\"$$ZLIB_SHA256_ACTUAL\"; \
+		exit 1; \
+	fi
+	tar xvzf zlib-$(ZLIB_VER).tar.gz
+	cd zlib-$(ZLIB_VER) && CFLAGS='-fPIC' ./configure --static && make
+	cp zlib-$(ZLIB_VER)/libz.a .
 
 libbz2.a:
-	-rm -rf bzip2-1.0.6
-	curl -O -L http://www.bzip.org/1.0.6/bzip2-1.0.6.tar.gz
-	tar xvzf bzip2-1.0.6.tar.gz
-	cd bzip2-1.0.6 && make CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64'
-	cp bzip2-1.0.6/libbz2.a .
+	-rm -rf bzip2-$(BZIP2_VER)
+	curl -O -L http://www.bzip.org/$(BZIP2_VER)/bzip2-$(BZIP2_VER).tar.gz
+	BZIP2_SHA256_ACTUAL=`$(SHA256_CMD) bzip2-$(BZIP2_VER).tar.gz | cut -d ' ' -f 1`; \
+	if [ "$(BZIP2_SHA256)" != "$$BZIP2_SHA256_ACTUAL" ]; then \
+		echo bzip2-$(BZIP2_VER).tar.gz checksum mismatch, expected=\"$(BZIP2_SHA256)\" actual=\"$$BZIP2_SHA256_ACTUAL\"; \
+		exit 1; \
+	fi
+	tar xvzf bzip2-$(BZIP2_VER).tar.gz
+	cd bzip2-$(BZIP2_VER) && make CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64'
+	cp bzip2-$(BZIP2_VER)/libbz2.a .
 
 libsnappy.a:
-	-rm -rf snappy-1.1.3
-	curl -O -L https://github.com/google/snappy/releases/download/1.1.3/snappy-1.1.3.tar.gz
-	tar xvzf snappy-1.1.3.tar.gz
-	cd snappy-1.1.3 && ./configure --with-pic --enable-static
-	cd snappy-1.1.3 && make
-	cp snappy-1.1.3/.libs/libsnappy.a .
+	-rm -rf snappy-$(SNAPPY_VER)
+	curl -O -L https://github.com/google/snappy/releases/download/$(SNAPPY_VER)/snappy-$(SNAPPY_VER).tar.gz
+	SNAPPY_SHA256_ACTUAL=`$(SHA256_CMD) snappy-$(SNAPPY_VER).tar.gz | cut -d ' ' -f 1`; \
+	if [ "$(SNAPPY_SHA256)" != "$$SNAPPY_SHA256_ACTUAL" ]; then \
+		echo snappy-$(SNAPPY_VER).tar.gz checksum mismatch, expected=\"$(SNAPPY_SHA256)\" actual=\"$$SNAPPY_SHA256_ACTUAL\"; \
+		exit 1; \
+	fi
+	tar xvzf snappy-$(SNAPPY_VER).tar.gz
+	cd snappy-$(SNAPPY_VER) && ./configure --with-pic --enable-static
+	cd snappy-$(SNAPPY_VER) && make
+	cp snappy-$(SNAPPY_VER)/.libs/libsnappy.a .
 
 liblz4.a:
-	   -rm -rf lz4-r127
-	   curl -O -L https://codeload.github.com/Cyan4973/lz4/tar.gz/r127
-	   mv r127 lz4-r127.tar.gz
-	   tar xvzf lz4-r127.tar.gz
-	   cd lz4-r127/lib && make CFLAGS='-fPIC' all
-	   cp lz4-r127/lib/liblz4.a .
+	-rm -rf lz4-$(LZ4_VER)
+	curl -O -L https://github.com/lz4/lz4/archive/v$(LZ4_VER).tar.gz
+	mv v$(LZ4_VER).tar.gz lz4-$(LZ4_VER).tar.gz
+	LZ4_SHA256_ACTUAL=`$(SHA256_CMD) lz4-$(LZ4_VER).tar.gz | cut -d ' ' -f 1`; \
+	if [ "$(LZ4_SHA256)" != "$$LZ4_SHA256_ACTUAL" ]; then \
+		echo lz4-$(LZ4_VER).tar.gz checksum mismatch, expected=\"$(LZ4_SHA256)\" actual=\"$$LZ4_SHA256_ACTUAL\"; \
+		exit 1; \
+	fi
+	tar xvzf lz4-$(LZ4_VER).tar.gz
+	cd lz4-$(LZ4_VER)/lib && make CFLAGS='-fPIC' all
+	cp lz4-$(LZ4_VER)/lib/liblz4.a .
+
+libzstd.a:
+	-rm -rf zstd-$(ZSTD_VER)
+	curl -O -L ${ZSTD_DOWNLOAD_BASE}/v$(ZSTD_VER).tar.gz
+	mv v$(ZSTD_VER).tar.gz zstd-$(ZSTD_VER).tar.gz
+	ZSTD_SHA256_ACTUAL=`$(SHA256_CMD) zstd-$(ZSTD_VER).tar.gz | cut -d ' ' -f 1`; \
+	if [ "$(ZSTD_SHA256)" != "$$ZSTD_SHA256_ACTUAL" ]; then \
+		echo zstd-$(ZSTD_VER).tar.gz checksum mismatch, expected=\"$(ZSTD_SHA256)\" actual=\"$$ZSTD_SHA256_ACTUAL\"; \
+		exit 1; \
+	fi
+	tar xvzf zstd-$(ZSTD_VER).tar.gz
+	cd zstd-$(ZSTD_VER)/lib && make CFLAGS='-fPIC -O2 ${EXTRA_CFLAGS}' all
+	cp zstd-$(ZSTD_VER)/lib/libzstd.a .
 
 # A version of each $(LIBOBJECTS) compiled with -fPIC and a fixed set of static compression libraries
 java_static_libobjects = $(patsubst %,jls/%,$(LIBOBJECTS))
 CLEAN_FILES += jls
 
-JAVA_STATIC_FLAGS = -DZLIB -DBZIP2 -DSNAPPY -DLZ4
-JAVA_STATIC_INCLUDES = -I./zlib-1.2.10 -I./bzip2-1.0.6 -I./snappy-1.1.3 -I./lz4-r127/lib
+ifneq ($(ROCKSDB_JAVA_NO_COMPRESSION), 1)
+JAVA_COMPRESSIONS = libz.a libbz2.a libsnappy.a liblz4.a libzstd.a
+endif
 
-$(java_static_libobjects): jls/%.o: %.cc libz.a libbz2.a libsnappy.a liblz4.a
+JAVA_STATIC_FLAGS = -DZLIB -DBZIP2 -DSNAPPY -DLZ4 -DZSTD
+JAVA_STATIC_INCLUDES = -I./zlib-$(ZLIB_VER) -I./bzip2-$(BZIP2_VER) -I./snappy-$(SNAPPY_VER) -I./lz4-$(LZ4_VER)/lib -I./zstd-$(ZSTD_VER)/lib
+
+$(java_static_libobjects): jls/%.o: %.cc $(JAVA_COMPRESSIONS)
 	$(AM_V_CC)mkdir -p $(@D) && $(CXX) $(CXXFLAGS) $(JAVA_STATIC_FLAGS) $(JAVA_STATIC_INCLUDES) -fPIC -c $< -o $@ $(COVERAGEFLAGS)
 
 rocksdbjavastatic: $(java_static_libobjects)
@@ -1431,7 +1500,7 @@ rocksdbjavastatic: $(java_static_libobjects)
 	$(CXX) $(CXXFLAGS) -I./java/. $(JAVA_INCLUDE) -shared -fPIC \
 	  -o ./java/target/$(ROCKSDBJNILIB) $(JNI_NATIVE_SOURCES) \
 	  $(java_static_libobjects) $(COVERAGEFLAGS) \
-	  libz.a libbz2.a libsnappy.a liblz4.a $(JAVA_STATIC_LDFLAGS)
+	  $(JAVA_COMPRESSIONS) $(JAVA_STATIC_LDFLAGS)
 	cd java/target;strip -S -x $(ROCKSDBJNILIB)
 	cd java;jar -cf target/$(ROCKSDB_JAR) HISTORY*.md
 	cd java/target;jar -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB)
@@ -1445,8 +1514,25 @@ rocksdbjavastaticrelease: rocksdbjavastatic
 	cd java/target;jar -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib
 	cd java/target/classes;jar -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class
 
+rocksdbjavastaticreleasedocker: rocksdbjavastatic
+	DOCKER_LINUX_X64_CONTAINER=`docker ps -aqf name=rocksdb_linux_x64-be`; \
+	if [ -z "$$DOCKER_LINUX_X64_CONTAINER" ]; then \
+		docker container create --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host --name rocksdb_linux_x64-be evolvedbinary/rocksjava:centos5_x64-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh; \
+	fi
+	docker start -a rocksdb_linux_x64-be
+	DOCKER_LINUX_X86_CONTAINER=`docker ps -aqf name=rocksdb_linux_x86-be`; \
+	if [ -z "$$DOCKER_LINUX_X86_CONTAINER" ]; then \
+		docker container create --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host --name rocksdb_linux_x86-be evolvedbinary/rocksjava:centos5_x86-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh; \
+	fi
+	docker start -a rocksdb_linux_x86-be
+	cd java;jar -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md
+	cd java/target;jar -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib
+	cd java/target/classes;jar -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class
+
 rocksdbjavastaticpublish: rocksdbjavastaticrelease rocksdbjavastaticpublishcentral
 
+rocksdbjavastaticpublishdocker: rocksdbjavastaticreleasedocker rocksdbjavastaticpublishcentral
+
 rocksdbjavastaticpublishcentral:
 	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/rocksjni.pom -Dfile=java/target/rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-javadoc.jar -Dclassifier=javadoc
 	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/rocksjni.pom -Dfile=java/target/rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-sources.jar -Dclassifier=sources
@@ -1491,13 +1577,6 @@ commit_prereq: build_tools/rocksdb-lego-determinator \
 	J=$(J) build_tools/precommit_checker.py unit unit_481 clang_unit release release_481 clang_release tsan asan ubsan lite unit_non_shm
 	$(MAKE) clean && $(MAKE) jclean && $(MAKE) rocksdbjava;
 
-xfunc:
-	for xftest in $(XFUNC_TESTS); do \
-		echo "===== Running xftest $$xftest"; \
-		make check ROCKSDB_XFUNC_TEST="$$xftest" tests-regexp="DBTest" ;\
-	done
-
-
 # ---------------------------------------------------------------------------
 #  	Platform-specific compilation
 # ---------------------------------------------------------------------------

appveyor.yml

@@ -11,6 +11,7 @@ build:
   verbosity: minimal
 test:
 test_script:
+- ps: build_tools\run_ci_db_test.ps1 -Run db_basic_test -Concurrency 8
 - ps: build_tools\run_ci_db_test.ps1 -Run db_test2 -Concurrency 8
 - ps: build_tools\run_ci_db_test.ps1 -Run db_test -Concurrency 8
 - ps: build_tools\run_ci_db_test.ps1 -Run env_test,env_basic_test -Concurrency 2

build_tools/build_detect_platform

@@ -51,7 +51,12 @@ if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
     FBCODE_BUILD="true"
     # If we're compiling with TSAN we need pic build
     PIC_BUILD=$COMPILE_WITH_TSAN
-    source "$PWD/build_tools/fbcode_config.sh"
+    if [ -z "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
+      source "$PWD/build_tools/fbcode_config.sh"
+    else
+      # we need this to build with MySQL. Don't use for other purposes.
+      source "$PWD/build_tools/fbcode_config4.8.1.sh"
+    fi
 fi
 
 # Delete existing output, if it exists

build_tools/dependencies.sh

@@ -1,19 +1,18 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-GCC_BASE=/mnt/gvfs/third-party2/gcc/7331085db891a2ef4a88a48a751d834e8d68f4cb/7.x/centos7-native/b2ef2b6
-CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/963d9aeda70cc4779885b1277484fe7544a04e3e/9.0.0/platform007/9e92d53/
-LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/6ace84e956873d53638c738b6f65f3f469cca74c/7.x/platform007/5620abc
-GLIBC_BASE=/mnt/gvfs/third-party2/glibc/192b0f42d63dcf6210d6ceae387b49af049e6e0c/2.26/platform007/f259413
-SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/platform007/ca4da3d
-ZLIB_BASE=/mnt/gvfs/third-party2/zlib/2d9f0b9a4274cc21f61272a9e89bdb859bce8f1f/1.2.8/platform007/ca4da3d
-BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/platform007/ca4da3d
-LZ4_BASE=/mnt/gvfs/third-party2/lz4/0f607f8fc442ea7d6b876931b1898bb573d5e5da/1.9.1/platform007/ca4da3d
-ZSTD_BASE=/mnt/gvfs/third-party2/zstd/ca22bc441a4eb709e9e0b1f9fec9750fed7b31c5/1.4.x/platform007/15a3614
-GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/platform007/ca4da3d
-JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/c26f08f47ac35fc31da2633b7da92d6b863246eb/master/platform007/c26c002
-NUMA_BASE=/mnt/gvfs/third-party2/numa/3f3fb57a5ccc5fd21c66416c0b83e0aa76a05376/2.0.11/platform007/ca4da3d
-LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/40c73d874898b386a71847f1b99115d93822d11f/1.4/platform007/6f3e0a9
-TBB_BASE=/mnt/gvfs/third-party2/tbb/4ce8e8dba77cdbd81b75d6f0c32fd7a1b76a11ec/2018_U5/platform007/ca4da3d
-KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/fb251ecd2f5ae16f8671f7014c246e52a748fe0b/fb/platform007/da39a3e
-BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/ab9f09bba370e7066cafd4eb59752db93f2e8312/2.29.1/platform007/15a3614
-VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d42d152a15636529b0861ec493927200ebebca8e/3.15.0/platform007/ca4da3d
-LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.3.4/platform007/5007832
+GCC_BASE=/mnt/gvfs/third-party2/gcc/cf7d14c625ce30bae1a4661c2319c5a283e4dd22/4.9.x/centos6-native/108cf83
+CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/8598c375b0e94e1448182eb3df034704144a838d/stable/centos6-native/3f16ddd
+LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/d6e0a7da6faba45f5e5b1638f9edd7afc2f34e7d/4.9.x/gcc-4.9-glibc-2.20/024dbc3
+GLIBC_BASE=/mnt/gvfs/third-party2/glibc/d282e6e8f3d20f4e40a516834847bdc038e07973/2.20/gcc-4.9-glibc-2.20/500e281
+SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/8c38a4c1e52b4c2cc8a9cdc31b9c947ed7dbfcb4/1.1.3/gcc-4.9-glibc-2.20/e9936bf
+ZLIB_BASE=/mnt/gvfs/third-party2/zlib/0882df3713c7a84f15abe368dc004581f20b39d7/1.2.8/gcc-5-glibc-2.23/9bc6787
+BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/740325875f6729f42d28deaa2147b0854f3a347e/1.0.6/gcc-5-glibc-2.23/9bc6787
+LZ4_BASE=/mnt/gvfs/third-party2/lz4/0e790b441e2d9acd68d51e1d2e028f88c6a79ddf/r131/gcc-5-glibc-2.23/9bc6787
+ZSTD_BASE=/mnt/gvfs/third-party2/zstd/9455f75ff7f4831dc9fda02a6a0f8c68922fad8f/1.0.0/gcc-5-glibc-2.23/9bc6787
+GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/f001a51b2854957676d07306ef3abf67186b5c8b/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a
+JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/fc8a13ca1fffa4d0765c716c5a0b49f0c107518f/master/gcc-5-glibc-2.23/1c32b4b
+NUMA_BASE=/mnt/gvfs/third-party2/numa/17c514c4d102a25ca15f4558be564eeed76f4b6a/2.0.8/gcc-5-glibc-2.23/9bc6787
+LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/ad576de2a1ea560c4d3434304f0fc4e079bede42/trunk/gcc-5-glibc-2.23/b1847cb
+TBB_BASE=/mnt/gvfs/third-party2/tbb/9d9a554877d0c5bef330fe818ab7178806dd316a/4.0_update2/gcc-4.9-glibc-2.20/e9936bf
+KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/7c111ff27e0c466235163f00f280a9d617c3d2ec/4.0.9-36_fbk5_2933_gd092e3f/gcc-5-glibc-2.23/da39a3e
+BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/b7fd454c4b10c6a81015d4524ed06cdeab558490/2.26/centos6-native/da39a3e
+VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d7f4d4d86674a57668e3a96f76f0e17dd0eb8765/3.10.0/gcc-4.9-glibc-2.20/e9936bf
+LUA_BASE=/mnt/gvfs/third-party2/lua/61e4abf5813bbc39bc4f548757ccfcadde175a48/5.2.3/gcc-4.9-glibc-2.20/690f0d7

build_tools/fbcode_config.sh

@@ -13,7 +13,7 @@ source "$BASEDIR/dependencies.sh"
 CFLAGS=""
 
 # libgcc
-LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/7.3.0"
+LIBGCC_INCLUDE="$LIBGCC_BASE/include"
 LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
 
 # glibc
@@ -43,15 +43,11 @@ if test -z $PIC_BUILD; then
   LZ4_INCLUDE=" -I $LZ4_BASE/include/"
   LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
   CFLAGS+=" -DLZ4"
-fi
 
-ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
-if test -z $PIC_BUILD; then
+  ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
   ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
-else
-  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a"
+  CFLAGS+=" -DZSTD"
 fi
-CFLAGS+=" -DZSTD"
 
 # location of gflags headers and libraries
 GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
@@ -60,7 +56,7 @@ if test -z $PIC_BUILD; then
 else
   GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
 fi
-CFLAGS+=" -DGFLAGS=gflags"
+CFLAGS+=" -DGFLAGS=google"
 
 # location of jemalloc
 JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
@@ -108,8 +104,8 @@ if [ -z "$USE_CLANG" ]; then
   CXX="$GCC_BASE/bin/g++"
 
   CFLAGS+=" -B$BINUTILS/gold"
-  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
   CFLAGS+=" -isystem $GLIBC_INCLUDE"
+  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
   JEMALLOC=1
 else
   # clang
@@ -120,8 +116,8 @@ else
   KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
 
   CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x "
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x/x86_64-facebook-linux "
+  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.9.x "
+  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.9.x/x86_64-facebook-linux "
   CFLAGS+=" -isystem $GLIBC_INCLUDE"
   CFLAGS+=" -isystem $LIBGCC_INCLUDE"
   CFLAGS+=" -isystem $CLANG_INCLUDE"
@@ -132,14 +128,13 @@ else
 fi
 
 CFLAGS+=" $DEPS_INCLUDE"
-CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
+CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE"
 CXXFLAGS+=" $CFLAGS"
 
 EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
-EXEC_LDFLAGS+=" -B$BINUTILS/gold"
-EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform007/lib/ld.so"
+EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.9-glibc-2.20/lib/ld.so"
 EXEC_LDFLAGS+=" $LIBUNWIND"
-EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform007/lib"
+EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/gcc-4.9-glibc-2.20/lib"
 # required by libtbb
 EXEC_LDFLAGS+=" -ldl"
 
@@ -149,4 +144,12 @@ EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GF
 
 VALGRIND_VER="$VALGRIND_BASE/bin/"
 
-export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD 
+LUA_PATH="$LUA_BASE"
+
+if test -z $PIC_BUILD; then
+  LUA_LIB=" $LUA_PATH/lib/liblua.a"
+else
+  LUA_LIB=" $LUA_PATH/lib/liblua_pic.a"
+fi
+
+export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB

build_tools/regression_build_test.sh

@@ -55,7 +55,6 @@ make release
     --open_files=55000 \
     --statistics=1 \
     --histogram=1 \
-    --disable_data_sync=1 \
     --disable_wal=1 \
     --sync=0  > ${STAT_FILE}.fillseq
 
@@ -73,7 +72,6 @@ make release
     --open_files=55000 \
     --statistics=1 \
     --histogram=1 \
-    --disable_data_sync=1 \
     --disable_wal=1 \
     --sync=0 \
     --threads=8 > ${STAT_FILE}.overwrite
@@ -92,7 +90,6 @@ make release
     --open_files=55000 \
     --statistics=1 \
     --histogram=1 \
-    --disable_data_sync=1 \
     --disable_wal=1 \
     --sync=0 \
     --threads=1 > /dev/null
@@ -111,7 +108,6 @@ make release
     --open_files=55000 \
     --statistics=1 \
     --histogram=1 \
-    --disable_data_sync=1 \
     --disable_wal=1 \
     --sync=0 \
     --threads=16 > ${STAT_FILE}.readrandom
@@ -131,7 +127,6 @@ make release
     --use_tailing_iterator=1 \
     --statistics=1 \
     --histogram=1 \
-    --disable_data_sync=1 \
     --disable_wal=1 \
     --sync=0 \
     --threads=16 > ${STAT_FILE}.readrandomtailing
@@ -150,7 +145,6 @@ make release
     --open_files=55000 \
     --statistics=1 \
     --histogram=1 \
-    --disable_data_sync=1 \
     --disable_wal=1 \
     --sync=0 \
     --threads=16 > ${STAT_FILE}.readrandomsmallblockcache
@@ -171,7 +165,6 @@ make release
     --open_files=55000 \
     --statistics=1 \
     --histogram=1 \
-    --disable_data_sync=1 \
     --disable_wal=1 \
     --sync=0 \
     --threads=16 > ${STAT_FILE}.readrandom_mem_sst
@@ -191,7 +184,6 @@ make release
     --open_files=55000 \
     --statistics=1 \
     --histogram=1 \
-    --disable_data_sync=1 \
     --disable_wal=1 \
     --sync=0 \
     --threads=1 > /dev/null
@@ -210,7 +202,6 @@ make release
     --open_files=55000 \
     --statistics=1 \
     --histogram=1 \
-    --disable_data_sync=1 \
     --disable_wal=1 \
     --sync=0 \
     --threads=16 > /dev/null
@@ -230,7 +221,6 @@ make release
     --disable_auto_compactions=1 \
     --statistics=1 \
     --histogram=1 \
-    --disable_data_sync=1 \
     --disable_wal=1 \
     --sync=0 \
     --threads=16 > ${STAT_FILE}.readrandom_filluniquerandom
@@ -251,7 +241,6 @@ make release
     --open_files=55000 \
     --statistics=1 \
     --histogram=1 \
-    --disable_data_sync=1 \
     --disable_wal=1 \
     --sync=0 \
     --threads=16 > ${STAT_FILE}.readwhilewriting
@@ -270,7 +259,6 @@ make release
     --open_files=55000 \
     --statistics=1 \
     --histogram=1 \
-    --disable_data_sync=1 \
     --disable_wal=1 \
     --sync=0 \
     --value_size=10 \
@@ -295,7 +283,6 @@ common_in_mem_args="--db=/dev/shm/rocksdb \
     --disable_wal=0 \
     --wal_dir=/dev/shm/rocksdb \
     --sync=0 \
-    --disable_data_sync=1 \
     --verify_checksum=1 \
     --delete_obsolete_files_period_micros=314572800 \
     --max_grandparent_overlap_factor=10 \
@@ -356,7 +343,6 @@ common_in_mem_args="--db=/dev/shm/rocksdb \
     --open_files=55000 \
     --statistics=1 \
     --histogram=1 \
-    --disable_data_sync=1 \
     --disable_wal=1 \
     --sync=0  > ${STAT_FILE}.fillseq_lots_column_families
 
@@ -372,7 +358,6 @@ common_in_mem_args="--db=/dev/shm/rocksdb \
     --open_files=55000 \
     --statistics=1 \
     --histogram=1 \
-    --disable_data_sync=1 \
     --disable_wal=1 \
     --sync=0 \
     --threads=8 > ${STAT_FILE}.overwrite_lots_column_families

build_tools/rocksdb-lego-determinator

@@ -89,7 +89,7 @@ TSAN="COMPILE_WITH_TSAN=1"
 UBSAN="COMPILE_WITH_UBSAN=1"
 DISABLE_JEMALLOC="DISABLE_JEMALLOC=1"
 HTTP_PROXY="https_proxy=http://fwdproxy.29.prn1:8080 http_proxy=http://fwdproxy.29.prn1:8080 ftp_proxy=http://fwdproxy.29.prn1:8080"
-SETUP_JAVA_ENV="export $HTTP_PROXY; export JAVA_HOME=/usr/local/jdk-7u10-64/; export PATH=\$PATH:\$JAVA_HOME/bin"
+SETUP_JAVA_ENV="export $HTTP_PROXY; export JAVA_HOME=/usr/local/jdk-8u60-64/; export PATH=\$JAVA_HOME/bin:\$PATH"
 PARSER="'parser':'python build_tools/error_filter.py $1'"
 
 CONTRUN_NAME="ROCKSDB_CONTRUN_NAME"

build_tools/run_ci_db_test.ps1

@@ -50,7 +50,8 @@ Write-Output "Root: $RootFolder, WorkFolder: $WorkFolder"
 
 # Use JEMALLOC executables
 if($Run -ceq "db_test" -or
-   $Run -ceq "db_test2" ) {
+   $Run -ceq "db_test2" -or
+   $Run -ceq "db_basic_test") {
 
    $file_name = $Run
 
@@ -150,7 +151,7 @@ function MakeAndAdd([string]$token, $HashTable) {
 # Test executable name -> Log file
 function Discover-TestBinaries([string]$Pattern, $HashTable) {
 
-    $Exclusions = @("db_test*", "db_sanity_test*")
+    $Exclusions = @("db_test*", "db_sanity_test*", "db_basic_test*")
 
     $p = -join ($BinariesFolder, $pattern)
 
@@ -164,7 +165,8 @@ function Discover-TestBinaries([string]$Pattern, $HashTable) {
 $TestsToRun = [ordered]@{}
 
 if($Run -ceq "db_test" -or
-   $Run -ceq "db_test2") {
+   $Run -ceq "db_test2" -or
+   $Run -ceq "db_basic_test") {
     Normalize-DbTests -HashTable $TestsToRun
 } elseif($Run -ceq "tests") {
     if($EnableJE) {
@@ -237,7 +239,8 @@ function RunJobs($TestToLog, [int]$ConcurrencyVal, [bool]$AddForRerun)
             $log_path = ($TestToLog.$k)
 
             if($Run -ceq "db_test" -or
-               $Run -ceq "db_test2") {
+               $Run -ceq "db_test2" -or
+               $Run -ceq "db_basic_test") {
               $job = Start-Job -Name $k -ScriptBlock $InvokeTestCase -ArgumentList @($db_test,$k,$log_path)
             } else {
               [string]$Exe =  -Join ($BinariesFolder, $k)

db/auto_roll_logger.cc

@@ -8,6 +8,7 @@
 
 namespace rocksdb {
 
+#ifndef ROCKSDB_LITE
 // -- AutoRollLogger
 Status AutoRollLogger::ResetLogger() {
   TEST_SYNC_POINT("AutoRollLogger::ResetLogger:BeforeNewLogger");
@@ -130,6 +131,7 @@ bool AutoRollLogger::LogExpired() {
   ++cached_now_access_count;
   return cached_now >= ctime_ + kLogFileTimeToRoll;
 }
+#endif  // !ROCKSDB_LITE
 
 Status CreateLoggerFromOptions(const std::string& dbname,
                                const DBOptions& options,
@@ -147,6 +149,7 @@ Status CreateLoggerFromOptions(const std::string& dbname,
 
   env->CreateDirIfMissing(dbname);  // In case it does not exist
   // Currently we only support roll by time-to-roll and log size
+#ifndef ROCKSDB_LITE
   if (options.log_file_time_to_roll > 0 || options.max_log_file_size > 0) {
     AutoRollLogger* result = new AutoRollLogger(
         env, dbname, options.db_log_dir, options.max_log_file_size,
@@ -158,17 +161,17 @@ Status CreateLoggerFromOptions(const std::string& dbname,
       logger->reset(result);
     }
     return s;
-  } else {
-    // Open a log file in the same directory as the db
-    env->RenameFile(
-        fname, OldInfoLogFileName(dbname, env->NowMicros(), db_absolute_path,
-                                  options.db_log_dir));
-    auto s = env->NewLogger(fname, logger);
-    if (logger->get() != nullptr) {
-      (*logger)->SetInfoLogLevel(options.info_log_level);
-    }
-    return s;
   }
+#endif  // !ROCKSDB_LITE
+  // Open a log file in the same directory as the db
+  env->RenameFile(fname,
+                  OldInfoLogFileName(dbname, env->NowMicros(), db_absolute_path,
+                                     options.db_log_dir));
+  auto s = env->NewLogger(fname, logger);
+  if (logger->get() != nullptr) {
+    (*logger)->SetInfoLogLevel(options.info_log_level);
+  }
+  return s;
 }
 
 }  // namespace rocksdb

db/auto_roll_logger.h

@@ -18,6 +18,7 @@
 
 namespace rocksdb {
 
+#ifndef ROCKSDB_LITE
 // Rolls the log file by size and/or time
 class AutoRollLogger : public Logger {
  public:
@@ -120,6 +121,7 @@ class AutoRollLogger : public Logger {
   uint64_t call_NowMicros_every_N_records_;
   mutable port::Mutex mutex_;
 };
+#endif  // !ROCKSDB_LITE
 
 // Facade to craete logger automatically
 Status CreateLoggerFromOptions(const std::string& dbname,

db/auto_roll_logger_test.cc

@@ -3,6 +3,9 @@
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
 //
+
+#ifndef ROCKSDB_LITE
+
 #include <string>
 #include <thread>
 #include <vector>
@@ -469,3 +472,14 @@ int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }
+
+#else
+#include <stdio.h>
+
+int main(int argc, char** argv) {
+  fprintf(stderr,
+          "SKIPPED as AutoRollLogger is not supported in ROCKSDB_LITE\n");
+  return 0;
+}
+
+#endif  // !ROCKSDB_LITE

db/builder.cc

@@ -112,7 +112,8 @@ Status BuildTable(
       }
       file->SetIOPriority(io_priority);
 
-      file_writer.reset(new WritableFileWriter(std::move(file), env_options));
+      file_writer.reset(new WritableFileWriter(std::move(file), env_options,
+                                               ioptions.statistics));
 
       builder = NewTableBuilder(
           ioptions, internal_comparator, int_tbl_prop_collector_factories,
@@ -122,7 +123,6 @@ Status BuildTable(
 
     MergeHelper merge(env, internal_comparator.user_comparator(),
                       ioptions.merge_operator, nullptr, ioptions.info_log,
-                      mutable_cf_options.min_partial_merge_operands,
                       true /* internal key corruption is not ok */,
                       snapshots.empty() ? 0 : snapshots.back());
 
@@ -170,7 +170,7 @@ Status BuildTable(
     delete builder;
 
     // Finish and check for file errors
-    if (s.ok() && !empty && !ioptions.disable_data_sync) {
+    if (s.ok() && !empty) {
       StopWatch sw(env, ioptions.statistics, TABLE_SYNC_MICROS);
       file_writer->Sync(ioptions.use_fsync);
     }

db/c.cc

@@ -1439,11 +1439,6 @@ void rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache(
   options->rep.pin_l0_filter_and_index_blocks_in_cache = v;
 }
 
-void rocksdb_block_based_options_set_skip_table_builder_flush(
-    rocksdb_block_based_table_options_t* options, unsigned char v) {
-  options->rep.skip_table_builder_flush = v;
-}
-
 void rocksdb_options_set_block_based_table_factory(
     rocksdb_options_t *opt,
     rocksdb_block_based_table_options_t* table_options) {
@@ -1713,11 +1708,6 @@ void rocksdb_options_set_prefix_extractor(
   opt->rep.prefix_extractor.reset(prefix_extractor);
 }
 
-void rocksdb_options_set_disable_data_sync(
-    rocksdb_options_t* opt, int disable_data_sync) {
-  opt->rep.disableDataSync = disable_data_sync;
-}
-
 void rocksdb_options_set_use_fsync(
     rocksdb_options_t* opt, int use_fsync) {
   opt->rep.use_fsync = use_fsync;
@@ -1829,11 +1819,6 @@ void rocksdb_options_set_enable_write_thread_adaptive_yield(
   opt->rep.enable_write_thread_adaptive_yield = v;
 }
 
-void rocksdb_options_set_verify_checksums_in_compaction(
-    rocksdb_options_t* opt, unsigned char v) {
-  opt->rep.verify_checksums_in_compaction = v;
-}
-
 void rocksdb_options_set_max_sequential_skip_in_iterations(
     rocksdb_options_t* opt, uint64_t v) {
   opt->rep.max_sequential_skip_in_iterations = v;
@@ -1985,11 +1970,6 @@ void rocksdb_options_set_max_successive_merges(
   opt->rep.max_successive_merges = v;
 }
 
-void rocksdb_options_set_min_partial_merge_operands(
-    rocksdb_options_t* opt, uint32_t v) {
-  opt->rep.min_partial_merge_operands = v;
-}
-
 void rocksdb_options_set_bloom_locality(
     rocksdb_options_t* opt, uint32_t v) {
   opt->rep.bloom_locality = v;

db/column_family.cc

@@ -32,7 +32,6 @@
 #include "util/compression.h"
 #include "util/options_helper.h"
 #include "util/thread_status_util.h"
-#include "util/xfunc.h"
 
 namespace rocksdb {
 
@@ -182,14 +181,6 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
   } else if (result.memtable_prefix_bloom_size_ratio < 0) {
     result.memtable_prefix_bloom_size_ratio = 0;
   }
-  XFUNC_TEST("memtablelist_history", "transaction_xftest_SanitizeOptions",
-             xf_transaction_set_memtable_history1,
-             xf_transaction_set_memtable_history,
-             &result.max_write_buffer_number_to_maintain);
-  XFUNC_TEST("memtablelist_history_clear", "transaction_xftest_SanitizeOptions",
-             xf_transaction_clear_memtable_history1,
-             xf_transaction_clear_memtable_history,
-             &result.max_write_buffer_number_to_maintain);
 
   if (!result.prefix_extractor) {
     assert(result.memtable_factory);

db/column_family_test.cc

@@ -213,6 +213,30 @@ class ColumnFamilyTest : public testing::Test {
 #endif
   }
 
+  bool IsDbWriteStopped() {
+#ifndef ROCKSDB_LITE
+    uint64_t v;
+    EXPECT_TRUE(dbfull()->GetIntProperty("rocksdb.is-write-stopped", &v));
+    return (v == 1);
+#else
+    return dbfull()->TEST_write_controler().IsStopped();
+#endif  // !ROCKSDB_LITE
+  }
+
+  uint64_t GetDbDelayedWriteRate() {
+#ifndef ROCKSDB_LITE
+    uint64_t v;
+    EXPECT_TRUE(
+        dbfull()->GetIntProperty("rocksdb.actual-delayed-write-rate", &v));
+    return v;
+#else
+    if (!dbfull()->TEST_write_controler().NeedsDelay()) {
+      return 0;
+    }
+    return dbfull()->TEST_write_controler().delayed_write_rate();
+#endif  // !ROCKSDB_LITE
+  }
+
   void Destroy() {
     Close();
     ASSERT_OK(DestroyDB(dbname_, Options(db_options_, column_family_options_)));
@@ -1148,7 +1172,6 @@ TEST_F(ColumnFamilyTest, DifferentCompactionStyles) {
   CreateColumnFamilies({"one", "two"});
   ColumnFamilyOptions default_cf, one, two;
   db_options_.max_open_files = 20;  // only 10 files in file cache
-  db_options_.disableDataSync = true;
 
   default_cf.compaction_style = kCompactionStyleLevel;
   default_cf.num_levels = 3;
@@ -1221,7 +1244,6 @@ TEST_F(ColumnFamilyTest, MultipleManualCompactions) {
   CreateColumnFamilies({"one", "two"});
   ColumnFamilyOptions default_cf, one, two;
   db_options_.max_open_files = 20;  // only 10 files in file cache
-  db_options_.disableDataSync = true;
   db_options_.max_background_compactions = 3;
 
   default_cf.compaction_style = kCompactionStyleLevel;
@@ -1319,7 +1341,6 @@ TEST_F(ColumnFamilyTest, AutomaticAndManualCompactions) {
   CreateColumnFamilies({"one", "two"});
   ColumnFamilyOptions default_cf, one, two;
   db_options_.max_open_files = 20;  // only 10 files in file cache
-  db_options_.disableDataSync = true;
   db_options_.max_background_compactions = 3;
   db_options_.base_background_compactions = 3;
 
@@ -1412,7 +1433,6 @@ TEST_F(ColumnFamilyTest, ManualAndAutomaticCompactions) {
   CreateColumnFamilies({"one", "two"});
   ColumnFamilyOptions default_cf, one, two;
   db_options_.max_open_files = 20;  // only 10 files in file cache
-  db_options_.disableDataSync = true;
   db_options_.max_background_compactions = 3;
   db_options_.base_background_compactions = 3;
 
@@ -1508,7 +1528,6 @@ TEST_F(ColumnFamilyTest, SameCFManualManualCompactions) {
   CreateColumnFamilies({"one"});
   ColumnFamilyOptions default_cf, one;
   db_options_.max_open_files = 20;  // only 10 files in file cache
-  db_options_.disableDataSync = true;
   db_options_.max_background_compactions = 3;
   db_options_.base_background_compactions = 3;
 
@@ -1607,7 +1626,6 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactions) {
   CreateColumnFamilies({"one"});
   ColumnFamilyOptions default_cf, one;
   db_options_.max_open_files = 20;  // only 10 files in file cache
-  db_options_.disableDataSync = true;
   db_options_.max_background_compactions = 3;
   db_options_.base_background_compactions = 3;
 
@@ -1697,7 +1715,6 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactionsLevel) {
   CreateColumnFamilies({"one"});
   ColumnFamilyOptions default_cf, one;
   db_options_.max_open_files = 20;  // only 10 files in file cache
-  db_options_.disableDataSync = true;
   db_options_.max_background_compactions = 3;
   db_options_.base_background_compactions = 3;
 
@@ -1794,7 +1811,6 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticConflict) {
   CreateColumnFamilies({"one"});
   ColumnFamilyOptions default_cf, one;
   db_options_.max_open_files = 20;  // only 10 files in file cache
-  db_options_.disableDataSync = true;
   db_options_.max_background_compactions = 3;
   db_options_.base_background_compactions = 3;
 
@@ -1914,7 +1930,6 @@ TEST_F(ColumnFamilyTest, SameCFAutomaticManualCompactions) {
   CreateColumnFamilies({"one"});
   ColumnFamilyOptions default_cf, one;
   db_options_.max_open_files = 20;  // only 10 files in file cache
-  db_options_.disableDataSync = true;
   db_options_.max_background_compactions = 3;
   db_options_.base_background_compactions = 3;
 
@@ -2462,173 +2477,165 @@ TEST_F(ColumnFamilyTest, WriteStallSingleColumnFamily) {
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(50);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(201);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate, dbfull()->TEST_write_controler().delayed_write_rate());
+  ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate());
   ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(400);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25,
-            dbfull()->TEST_write_controler().delayed_write_rate());
+  ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate());
   ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(500);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25 / 1.25,
-            dbfull()->TEST_write_controler().delayed_write_rate());
+  ASSERT_EQ(kBaseRate / 1.25 / 1.25, GetDbDelayedWriteRate());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(450);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25,
-            dbfull()->TEST_write_controler().delayed_write_rate());
+  ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(205);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate, dbfull()->TEST_write_controler().delayed_write_rate());
+  ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(202);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate, dbfull()->TEST_write_controler().delayed_write_rate());
+  ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(201);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate, dbfull()->TEST_write_controler().delayed_write_rate());
+  ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(198);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(399);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate, dbfull()->TEST_write_controler().delayed_write_rate());
+  ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(599);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25,
-            dbfull()->TEST_write_controler().delayed_write_rate());
+  ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(2001);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(IsDbWriteStopped());
   ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
   ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(3001);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(IsDbWriteStopped());
   ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(390);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25,
-            dbfull()->TEST_write_controler().delayed_write_rate());
+  ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(100);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
 
   vstorage->set_l0_delay_trigger_count(100);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate, dbfull()->TEST_write_controler().delayed_write_rate());
+  ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate());
   ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed());
 
   vstorage->set_l0_delay_trigger_count(101);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25,
-            dbfull()->TEST_write_controler().delayed_write_rate());
+  ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate());
 
   vstorage->set_l0_delay_trigger_count(0);
   vstorage->TEST_set_estimated_compaction_needed_bytes(300);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25 / 1.25,
-            dbfull()->TEST_write_controler().delayed_write_rate());
+  ASSERT_EQ(kBaseRate / 1.25 / 1.25, GetDbDelayedWriteRate());
 
   vstorage->set_l0_delay_trigger_count(101);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25 / 1.25 / 1.25,
-            dbfull()->TEST_write_controler().delayed_write_rate());
+  ASSERT_EQ(kBaseRate / 1.25 / 1.25 / 1.25, GetDbDelayedWriteRate());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(200);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25 / 1.25,
-            dbfull()->TEST_write_controler().delayed_write_rate());
+  ASSERT_EQ(kBaseRate / 1.25 / 1.25, GetDbDelayedWriteRate());
 
   vstorage->set_l0_delay_trigger_count(0);
   vstorage->TEST_set_estimated_compaction_needed_bytes(0);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
 
   mutable_cf_options.disable_auto_compactions = true;
   dbfull()->TEST_write_controler().set_delayed_write_rate(kBaseRate);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
 
   vstorage->set_l0_delay_trigger_count(50);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
+  ASSERT_EQ(0, GetDbDelayedWriteRate());
   ASSERT_EQ(kBaseRate, dbfull()->TEST_write_controler().delayed_write_rate());
 
   vstorage->set_l0_delay_trigger_count(60);
   vstorage->TEST_set_estimated_compaction_needed_bytes(300);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
+  ASSERT_EQ(0, GetDbDelayedWriteRate());
   ASSERT_EQ(kBaseRate, dbfull()->TEST_write_controler().delayed_write_rate());
 
   mutable_cf_options.disable_auto_compactions = false;
   vstorage->set_l0_delay_trigger_count(70);
   vstorage->TEST_set_estimated_compaction_needed_bytes(500);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate, dbfull()->TEST_write_controler().delayed_write_rate());
+  ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate());
 
   vstorage->set_l0_delay_trigger_count(71);
   vstorage->TEST_set_estimated_compaction_needed_bytes(501);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25,
-            dbfull()->TEST_write_controler().delayed_write_rate());
+  ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate());
 }
 
 TEST_F(ColumnFamilyTest, CompactionSpeedupSingleColumnFamily) {
@@ -2720,60 +2727,55 @@ TEST_F(ColumnFamilyTest, WriteStallTwoColumnFamilies) {
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(50);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
 
   vstorage1->TEST_set_estimated_compaction_needed_bytes(201);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
 
   vstorage1->TEST_set_estimated_compaction_needed_bytes(600);
   cfd1->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate, dbfull()->TEST_write_controler().delayed_write_rate());
+  ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(70);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate, dbfull()->TEST_write_controler().delayed_write_rate());
+  ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate());
 
   vstorage1->TEST_set_estimated_compaction_needed_bytes(800);
   cfd1->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25,
-            dbfull()->TEST_write_controler().delayed_write_rate());
+  ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(300);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25 / 1.25,
-            dbfull()->TEST_write_controler().delayed_write_rate());
+  ASSERT_EQ(kBaseRate / 1.25 / 1.25, GetDbDelayedWriteRate());
 
   vstorage1->TEST_set_estimated_compaction_needed_bytes(700);
   cfd1->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25,
-            dbfull()->TEST_write_controler().delayed_write_rate());
+  ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(500);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25 / 1.25,
-            dbfull()->TEST_write_controler().delayed_write_rate());
+  ASSERT_EQ(kBaseRate / 1.25 / 1.25, GetDbDelayedWriteRate());
 
   vstorage1->TEST_set_estimated_compaction_needed_bytes(600);
   cfd1->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
+  ASSERT_TRUE(!IsDbWriteStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25,
-            dbfull()->TEST_write_controler().delayed_write_rate());
+  ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate());
 }
 
 TEST_F(ColumnFamilyTest, CompactionSpeedupTwoColumnFamilies) {

db/compaction_iterator_test.cc

@@ -182,7 +182,7 @@ class CompactionIteratorTest : public testing::Test {
     }
 
     merge_helper_.reset(new MergeHelper(Env::Default(), cmp_, merge_op, filter,
-                                        nullptr, 0U, false, 0, 0, nullptr,
+                                        nullptr, false, 0, 0, nullptr,
                                         &shutting_down_));
     iter_.reset(new LoggingForwardVectorIterator(ks, vs));
     iter_->SeekToFirst();

db/compaction_job.cc

@@ -538,7 +538,7 @@ Status CompactionJob::Run() {
     thread.join();
   }
 
-  if (output_directory_ && !db_options_.disable_data_sync) {
+  if (output_directory_) {
     output_directory_->Fsync();
   }
 
@@ -721,7 +721,6 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
   MergeHelper merge(
       env_, cfd->user_comparator(), cfd->ioptions()->merge_operator,
       compaction_filter, db_options_.info_log.get(),
-      mutable_cf_options->min_partial_merge_operands,
       false /* internal key corruption is expected */,
       existing_snapshots_.empty() ? 0 : existing_snapshots_.back(),
       compact_->compaction->level(), db_options_.statistics.get(),
@@ -747,6 +746,14 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
       shutting_down_));
   auto c_iter = sub_compact->c_iter.get();
   c_iter->SeekToFirst();
+  if (c_iter->Valid() &&
+      sub_compact->compaction->output_level() != 0) {
+    // ShouldStopBefore() maintains state based on keys processed so far. The
+    // compaction loop always calls it on the "next" key, thus won't tell it the
+    // first key. So we do that here.
+    sub_compact->ShouldStopBefore(
+      c_iter->key(), sub_compact->current_output_file_size);
+  }
   const auto& c_iter_stats = c_iter->iter_stats();
   auto sample_begin_offset_iter = sample_begin_offsets.cbegin();
   // data_begin_offset and compression_dict are only valid while generating
@@ -766,21 +773,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
     if (end != nullptr &&
         cfd->user_comparator()->Compare(c_iter->user_key(), *end) >= 0) {
       break;
-    } else if (sub_compact->compaction->output_level() != 0 &&
-               sub_compact->ShouldStopBefore(
-                   key, sub_compact->current_output_file_size) &&
-               sub_compact->builder != nullptr) {
-      CompactionIterationStats range_del_out_stats;
-      status = FinishCompactionOutputFile(input->status(), sub_compact,
-                                          range_del_agg.get(),
-                                          &range_del_out_stats, &key);
-      RecordDroppedKeys(range_del_out_stats,
-                        &sub_compact->compaction_job_stats);
-      if (!status.ok()) {
-        break;
-      }
     }
-
     if (c_iter_stats.num_input_records % kRecordStatsEvery ==
         kRecordStatsEvery - 1) {
       RecordDroppedKeys(c_iter_stats, &sub_compact->compaction_job_stats);
@@ -852,17 +845,37 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
       }
     }
 
-    // Close output file if it is big enough
+    // Close output file if it is big enough. Two possibilities determine it's
+    // time to close it: (1) the current key should be this file's last key, (2)
+    // the next key should not be in this file.
+    //
     // TODO(aekmekji): determine if file should be closed earlier than this
     // during subcompactions (i.e. if output size, estimated by input size, is
     // going to be 1.2MB and max_output_file_size = 1MB, prefer to have 0.6MB
     // and 0.6MB instead of 1MB and 0.2MB)
+    bool output_file_ended = false;
+    Status input_status;
     if (sub_compact->compaction->output_level() != 0 &&
         sub_compact->current_output_file_size >=
             sub_compact->compaction->max_output_file_size()) {
-      Status input_status = input->status();
-      c_iter->Next();
-
+      // (1) this key terminates the file. For historical reasons, the iterator
+      // status before advancing will be given to FinishCompactionOutputFile().
+      input_status = input->status();
+      output_file_ended = true;
+    }
+    c_iter->Next();
+    if (!output_file_ended && c_iter->Valid() &&
+        sub_compact->compaction->output_level() != 0 &&
+        sub_compact->ShouldStopBefore(
+          c_iter->key(), sub_compact->current_output_file_size) &&
+        sub_compact->builder != nullptr) {
+      // (2) this key belongs to the next file. For historical reasons, the
+      // iterator status after advancing will be given to
+      // FinishCompactionOutputFile().
+      input_status = input->status();
+      output_file_ended = true;
+    }
+    if (output_file_ended) {
       const Slice* next_key = nullptr;
       if (c_iter->Valid()) {
         next_key = &c_iter->key();
@@ -878,8 +891,6 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
         // files.
         sub_compact->compression_dict = std::move(compression_dict);
       }
-    } else {
-      c_iter->Next();
     }
   }
 
@@ -1047,7 +1058,7 @@ Status CompactionJob::FinishCompactionOutputFile(
   sub_compact->total_bytes += current_bytes;
 
   // Finish and check for file errors
-  if (s.ok() && !db_options_.disable_data_sync) {
+  if (s.ok()) {
     StopWatch sw(env_, stats_, COMPACTION_OUTFILE_SYNC_MICROS);
     s = sub_compact->outfile->Sync(db_options_.use_fsync);
   }
@@ -1211,8 +1222,8 @@ Status CompactionJob::OpenCompactionOutputFile(
   writable_file->SetIOPriority(Env::IO_LOW);
   writable_file->SetPreallocationBlockSize(static_cast<size_t>(
       sub_compact->compaction->OutputFilePreallocationSize()));
-  sub_compact->outfile.reset(
-      new WritableFileWriter(std::move(writable_file), env_options_));
+  sub_compact->outfile.reset(new WritableFileWriter(
+      std::move(writable_file), env_options_, db_options_.statistics.get()));
 
   // If the Column family flag is to only optimize filters for hits,
   // we can skip creating filters if this is the bottommost_level where

db/compaction_job_stats_test.cc

@@ -61,7 +61,6 @@
 #include "util/testharness.h"
 #include "util/testutil.h"
 #include "util/thread_status_util.h"
-#include "util/xfunc.h"
 #include "utilities/merge_operators.h"
 
 #if !defined(IOS_CROSS_COMPILE)

db/compaction_picker.cc

@@ -18,7 +18,6 @@
 #include <queue>
 #include <string>
 #include <utility>
-
 #include "db/column_family.h"
 #include "db/filename.h"
 #include "util/log_buffer.h"
@@ -466,49 +465,66 @@ bool CompactionPicker::SetupOtherInputs(
   // user key, while excluding other entries for the same user key. This
   // can happen when one user key spans multiple files.
   if (!output_level_inputs->empty()) {
-    CompactionInputFiles expanded0;
-    expanded0.level = input_level;
-    // Get entire range covered by compaction
+    const uint64_t limit = mutable_cf_options.max_compaction_bytes;
+    const uint64_t output_level_inputs_size =
+        TotalCompensatedFileSize(output_level_inputs->files);
+    const uint64_t inputs_size = TotalCompensatedFileSize(inputs->files);
+    bool expand_inputs = false;
+
+    CompactionInputFiles expanded_inputs;
+    expanded_inputs.level = input_level;
+    // Get closed interval of output level
     InternalKey all_start, all_limit;
     GetRange(*inputs, *output_level_inputs, &all_start, &all_limit);
-
+    bool try_overlapping_inputs = true;
     vstorage->GetOverlappingInputs(input_level, &all_start, &all_limit,
-                                   &expanded0.files, base_index, nullptr);
-    const uint64_t inputs0_size = TotalCompensatedFileSize(inputs->files);
-    const uint64_t inputs1_size =
-        TotalCompensatedFileSize(output_level_inputs->files);
-    const uint64_t expanded0_size = TotalCompensatedFileSize(expanded0.files);
-    uint64_t limit = mutable_cf_options.max_compaction_bytes;
-    if (expanded0.size() > inputs->size() &&
-        inputs1_size + expanded0_size < limit &&
-        !FilesInCompaction(expanded0.files) &&
-        !vstorage->HasOverlappingUserKey(&expanded0.files, input_level)) {
+                                   &expanded_inputs.files, base_index, nullptr);
+    uint64_t expanded_inputs_size =
+        TotalCompensatedFileSize(expanded_inputs.files);
+    if (!ExpandWhileOverlapping(cf_name, vstorage, &expanded_inputs)) {
+      try_overlapping_inputs = false;
+    }
+    if (try_overlapping_inputs && expanded_inputs.size() > inputs->size() &&
+        output_level_inputs_size + expanded_inputs_size < limit &&
+        !FilesInCompaction(expanded_inputs.files)) {
       InternalKey new_start, new_limit;
-      GetRange(expanded0, &new_start, &new_limit);
-      CompactionInputFiles expanded1;
-      expanded1.level = output_level;
+      GetRange(expanded_inputs, &new_start, &new_limit);
+      CompactionInputFiles expanded_output_level_inputs;
+      expanded_output_level_inputs.level = output_level;
       vstorage->GetOverlappingInputs(output_level, &new_start, &new_limit,
-                                     &expanded1.files, *parent_index,
-                                     parent_index);
-      assert(!expanded1.empty());
-      if (!FilesInCompaction(expanded1.files) &&
-          ExpandWhileOverlapping(cf_name, vstorage, &expanded1) &&
-          expanded1.size() == output_level_inputs->size()) {
-        Log(InfoLogLevel::INFO_LEVEL, ioptions_.info_log,
-            "[%s] Expanding@%d %" ROCKSDB_PRIszt "+%" ROCKSDB_PRIszt "(%" PRIu64
-            "+%" PRIu64 " bytes) to %" ROCKSDB_PRIszt "+%" ROCKSDB_PRIszt
-            " (%" PRIu64 "+%" PRIu64 "bytes)\n",
-            cf_name.c_str(), input_level, inputs->size(),
-            output_level_inputs->size(), inputs0_size, inputs1_size,
-            expanded0.size(), expanded1.size(), expanded0_size, inputs1_size);
-        smallest = new_start;
-        largest = new_limit;
-        inputs->files = expanded0.files;
-        output_level_inputs->files = expanded1.files;
+                                     &expanded_output_level_inputs.files,
+                                     *parent_index, parent_index);
+      assert(!expanded_output_level_inputs.empty());
+      if (!FilesInCompaction(expanded_output_level_inputs.files) &&
+          ExpandWhileOverlapping(cf_name, vstorage,
+                                 &expanded_output_level_inputs) &&
+          expanded_output_level_inputs.size() == output_level_inputs->size()) {
+        expand_inputs = true;
       }
     }
+    if (!expand_inputs) {
+      vstorage->GetCleanInputsWithinInterval(input_level, &all_start,
+                                             &all_limit, &expanded_inputs.files,
+                                             base_index, nullptr);
+      expanded_inputs_size = TotalCompensatedFileSize(expanded_inputs.files);
+      if (expanded_inputs.size() > inputs->size() &&
+          output_level_inputs_size + expanded_inputs_size < limit &&
+          !FilesInCompaction(expanded_inputs.files)) {
+        expand_inputs = true;
+      }
+    }
+    if (expand_inputs) {
+      Log(InfoLogLevel::INFO_LEVEL, ioptions_.info_log,
+          "[%s] Expanding@%d %" ROCKSDB_PRIszt "+%" ROCKSDB_PRIszt "(%" PRIu64
+          "+%" PRIu64 " bytes) to %" ROCKSDB_PRIszt "+%" ROCKSDB_PRIszt
+          " (%" PRIu64 "+%" PRIu64 "bytes)\n",
+          cf_name.c_str(), input_level, inputs->size(),
+          output_level_inputs->size(), inputs_size, output_level_inputs_size,
+          expanded_inputs.size(), output_level_inputs->size(),
+          expanded_inputs_size, output_level_inputs_size);
+      inputs->files = expanded_inputs.files;
+    }
   }
-
   return true;
 }
 

db/compaction_picker_test.cc

@@ -758,6 +758,70 @@ TEST_F(CompactionPickerTest, OverlappingUserKeys7) {
   ASSERT_EQ(5U, compaction->inputs(1)->back()->fd.GetNumber());
 }
 
+TEST_F(CompactionPickerTest, OverlappingUserKeys8) {
+  NewVersionStorage(6, kCompactionStyleLevel);
+  mutable_cf_options_.max_compaction_bytes = 100000000000u;
+  // grow the number of inputs in "level" without
+  // changing the number of "level+1" files we pick up
+  // Expand input level as much as possible
+  // no overlapping case
+  Add(1, 1U, "101", "150", 1U);
+  Add(1, 2U, "151", "200", 1U);
+  Add(1, 3U, "201", "300", 1000000000U);
+  Add(1, 4U, "301", "400", 1U);
+  Add(1, 5U, "401", "500", 1U);
+  Add(2, 6U, "150", "200", 1U);
+  Add(2, 7U, "200", "450", 1U, 0, 0);
+  Add(2, 8U, "500", "600", 1U);
+
+  UpdateVersionStorageInfo();
+
+  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
+      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
+  ASSERT_TRUE(compaction.get() != nullptr);
+  ASSERT_EQ(2U, compaction->num_input_levels());
+  ASSERT_EQ(3U, compaction->num_input_files(0));
+  ASSERT_EQ(2U, compaction->num_input_files(1));
+  ASSERT_EQ(2U, compaction->input(0, 0)->fd.GetNumber());
+  ASSERT_EQ(3U, compaction->input(0, 1)->fd.GetNumber());
+  ASSERT_EQ(4U, compaction->input(0, 2)->fd.GetNumber());
+  ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber());
+  ASSERT_EQ(7U, compaction->input(1, 1)->fd.GetNumber());
+}
+
+TEST_F(CompactionPickerTest, OverlappingUserKeys9) {
+  NewVersionStorage(6, kCompactionStyleLevel);
+  mutable_cf_options_.max_compaction_bytes = 100000000000u;
+  // grow the number of inputs in "level" without
+  // changing the number of "level+1" files we pick up
+  // Expand input level as much as possible
+  // overlapping case
+  Add(1, 1U, "121", "150", 1U);
+  Add(1, 2U, "151", "200", 1U);
+  Add(1, 3U, "201", "300", 1000000000U);
+  Add(1, 4U, "301", "400", 1U);
+  Add(1, 5U, "401", "500", 1U);
+  Add(2, 6U, "100", "120", 1U);
+  Add(2, 7U, "150", "200", 1U);
+  Add(2, 8U, "200", "450", 1U, 0, 0);
+  Add(2, 9U, "501", "600", 1U);
+
+  UpdateVersionStorageInfo();
+
+  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
+      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
+  ASSERT_TRUE(compaction.get() != nullptr);
+  ASSERT_EQ(2U, compaction->num_input_levels());
+  ASSERT_EQ(5U, compaction->num_input_files(0));
+  ASSERT_EQ(2U, compaction->num_input_files(1));
+  ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
+  ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber());
+  ASSERT_EQ(3U, compaction->input(0, 2)->fd.GetNumber());
+  ASSERT_EQ(4U, compaction->input(0, 3)->fd.GetNumber());
+  ASSERT_EQ(7U, compaction->input(1, 0)->fd.GetNumber());
+  ASSERT_EQ(8U, compaction->input(1, 1)->fd.GetNumber());
+}
+
 TEST_F(CompactionPickerTest, NotScheduleL1IfL0WithHigherPri1) {
   NewVersionStorage(6, kCompactionStyleLevel);
   mutable_cf_options_.level0_file_num_compaction_trigger = 2;
@@ -1132,13 +1196,13 @@ TEST_F(CompactionPickerTest, MaxCompactionBytesHit) {
   ioptions_.level_compaction_dynamic_level_bytes = false;
   NewVersionStorage(6, kCompactionStyleLevel);
   // A compaction should be triggered and pick file 2 and 5.
-  // It cannot expand because adding file 1 and 3, the compaction size will
+  // It can expand because adding file 1 and 3, the compaction size will
   // exceed mutable_cf_options_.max_bytes_for_level_base.
   Add(1, 1U, "100", "150", 300000U);
   Add(1, 2U, "151", "200", 300001U, 0, 0);
   Add(1, 3U, "201", "250", 300000U, 0, 0);
   Add(1, 4U, "251", "300", 300000U, 0, 0);
-  Add(2, 5U, "160", "256", 1U);
+  Add(2, 5U, "100", "256", 1U);
   UpdateVersionStorageInfo();
 
   std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
@@ -1152,7 +1216,7 @@ TEST_F(CompactionPickerTest, MaxCompactionBytesHit) {
 }
 
 TEST_F(CompactionPickerTest, MaxCompactionBytesNotHit) {
-  mutable_cf_options_.max_bytes_for_level_base = 1000000u;
+  mutable_cf_options_.max_bytes_for_level_base = 800000u;
   mutable_cf_options_.max_compaction_bytes = 1000000u;
   ioptions_.level_compaction_dynamic_level_bytes = false;
   NewVersionStorage(6, kCompactionStyleLevel);
@@ -1162,7 +1226,7 @@ TEST_F(CompactionPickerTest, MaxCompactionBytesNotHit) {
   Add(1, 2U, "151", "200", 300001U, 0, 0);
   Add(1, 3U, "201", "250", 300000U, 0, 0);
   Add(1, 4U, "251", "300", 300000U, 0, 0);
-  Add(2, 5U, "000", "233", 1U);
+  Add(2, 5U, "000", "251", 1U);
   UpdateVersionStorageInfo();
 
   std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(

db/db_basic_test.cc

@@ -0,0 +1,852 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+#include "db/db_test_util.h"
+#include "port/stack_trace.h"
+#include "rocksdb/perf_context.h"
+#if !defined(ROCKSDB_LITE)
+#include "util/sync_point.h"
+#endif
+
+namespace rocksdb {
+
+class DBBasicTest : public DBTestBase {
+ public:
+  DBBasicTest() : DBTestBase("/db_basic_test") {}
+};
+
+TEST_F(DBBasicTest, OpenWhenOpen) {
+  Options options = CurrentOptions();
+  options.env = env_;
+  rocksdb::DB* db2 = nullptr;
+  rocksdb::Status s = DB::Open(options, dbname_, &db2);
+
+  ASSERT_EQ(Status::Code::kIOError, s.code());
+  ASSERT_EQ(Status::SubCode::kNone, s.subcode());
+  ASSERT_TRUE(strstr(s.getState(), "lock ") != nullptr);
+
+  delete db2;
+}
+
+#ifndef ROCKSDB_LITE
+TEST_F(DBBasicTest, ReadOnlyDB) {
+  ASSERT_OK(Put("foo", "v1"));
+  ASSERT_OK(Put("bar", "v2"));
+  ASSERT_OK(Put("foo", "v3"));
+  Close();
+
+  auto options = CurrentOptions();
+  assert(options.env = env_);
+  ASSERT_OK(ReadOnlyReopen(options));
+  ASSERT_EQ("v3", Get("foo"));
+  ASSERT_EQ("v2", Get("bar"));
+  Iterator* iter = db_->NewIterator(ReadOptions());
+  int count = 0;
+  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+    ASSERT_OK(iter->status());
+    ++count;
+  }
+  ASSERT_EQ(count, 2);
+  delete iter;
+  Close();
+
+  // Reopen and flush memtable.
+  Reopen(options);
+  Flush();
+  Close();
+  // Now check keys in read only mode.
+  ASSERT_OK(ReadOnlyReopen(options));
+  ASSERT_EQ("v3", Get("foo"));
+  ASSERT_EQ("v2", Get("bar"));
+  ASSERT_TRUE(db_->SyncWAL().IsNotSupported());
+}
+
+TEST_F(DBBasicTest, CompactedDB) {
+  const uint64_t kFileSize = 1 << 20;
+  Options options = CurrentOptions();
+  options.disable_auto_compactions = true;
+  options.write_buffer_size = kFileSize;
+  options.target_file_size_base = kFileSize;
+  options.max_bytes_for_level_base = 1 << 30;
+  options.compression = kNoCompression;
+  Reopen(options);
+  // 1 L0 file, use CompactedDB if max_open_files = -1
+  ASSERT_OK(Put("aaa", DummyString(kFileSize / 2, '1')));
+  Flush();
+  Close();
+  ASSERT_OK(ReadOnlyReopen(options));
+  Status s = Put("new", "value");
+  ASSERT_EQ(s.ToString(),
+            "Not implemented: Not supported operation in read only mode.");
+  ASSERT_EQ(DummyString(kFileSize / 2, '1'), Get("aaa"));
+  Close();
+  options.max_open_files = -1;
+  ASSERT_OK(ReadOnlyReopen(options));
+  s = Put("new", "value");
+  ASSERT_EQ(s.ToString(),
+            "Not implemented: Not supported in compacted db mode.");
+  ASSERT_EQ(DummyString(kFileSize / 2, '1'), Get("aaa"));
+  Close();
+  Reopen(options);
+  // Add more L0 files
+  ASSERT_OK(Put("bbb", DummyString(kFileSize / 2, '2')));
+  Flush();
+  ASSERT_OK(Put("aaa", DummyString(kFileSize / 2, 'a')));
+  Flush();
+  ASSERT_OK(Put("bbb", DummyString(kFileSize / 2, 'b')));
+  ASSERT_OK(Put("eee", DummyString(kFileSize / 2, 'e')));
+  Flush();
+  Close();
+
+  ASSERT_OK(ReadOnlyReopen(options));
+  // Fallback to read-only DB
+  s = Put("new", "value");
+  ASSERT_EQ(s.ToString(),
+            "Not implemented: Not supported operation in read only mode.");
+  Close();
+
+  // Full compaction
+  Reopen(options);
+  // Add more keys
+  ASSERT_OK(Put("fff", DummyString(kFileSize / 2, 'f')));
+  ASSERT_OK(Put("hhh", DummyString(kFileSize / 2, 'h')));
+  ASSERT_OK(Put("iii", DummyString(kFileSize / 2, 'i')));
+  ASSERT_OK(Put("jjj", DummyString(kFileSize / 2, 'j')));
+  db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
+  ASSERT_EQ(3, NumTableFilesAtLevel(1));
+  Close();
+
+  // CompactedDB
+  ASSERT_OK(ReadOnlyReopen(options));
+  s = Put("new", "value");
+  ASSERT_EQ(s.ToString(),
+            "Not implemented: Not supported in compacted db mode.");
+  ASSERT_EQ("NOT_FOUND", Get("abc"));
+  ASSERT_EQ(DummyString(kFileSize / 2, 'a'), Get("aaa"));
+  ASSERT_EQ(DummyString(kFileSize / 2, 'b'), Get("bbb"));
+  ASSERT_EQ("NOT_FOUND", Get("ccc"));
+  ASSERT_EQ(DummyString(kFileSize / 2, 'e'), Get("eee"));
+  ASSERT_EQ(DummyString(kFileSize / 2, 'f'), Get("fff"));
+  ASSERT_EQ("NOT_FOUND", Get("ggg"));
+  ASSERT_EQ(DummyString(kFileSize / 2, 'h'), Get("hhh"));
+  ASSERT_EQ(DummyString(kFileSize / 2, 'i'), Get("iii"));
+  ASSERT_EQ(DummyString(kFileSize / 2, 'j'), Get("jjj"));
+  ASSERT_EQ("NOT_FOUND", Get("kkk"));
+
+  // MultiGet
+  std::vector<std::string> values;
+  std::vector<Status> status_list = dbfull()->MultiGet(
+      ReadOptions(),
+      std::vector<Slice>({Slice("aaa"), Slice("ccc"), Slice("eee"),
+                          Slice("ggg"), Slice("iii"), Slice("kkk")}),
+      &values);
+  ASSERT_EQ(status_list.size(), static_cast<uint64_t>(6));
+  ASSERT_EQ(values.size(), static_cast<uint64_t>(6));
+  ASSERT_OK(status_list[0]);
+  ASSERT_EQ(DummyString(kFileSize / 2, 'a'), values[0]);
+  ASSERT_TRUE(status_list[1].IsNotFound());
+  ASSERT_OK(status_list[2]);
+  ASSERT_EQ(DummyString(kFileSize / 2, 'e'), values[2]);
+  ASSERT_TRUE(status_list[3].IsNotFound());
+  ASSERT_OK(status_list[4]);
+  ASSERT_EQ(DummyString(kFileSize / 2, 'i'), values[4]);
+  ASSERT_TRUE(status_list[5].IsNotFound());
+
+  Reopen(options);
+  // Add a key
+  ASSERT_OK(Put("fff", DummyString(kFileSize / 2, 'f')));
+  Close();
+  ASSERT_OK(ReadOnlyReopen(options));
+  s = Put("new", "value");
+  ASSERT_EQ(s.ToString(),
+            "Not implemented: Not supported operation in read only mode.");
+}
+
+TEST_F(DBBasicTest, LevelLimitReopen) {
+  Options options = CurrentOptions();
+  CreateAndReopenWithCF({"pikachu"}, options);
+
+  const std::string value(1024 * 1024, ' ');
+  int i = 0;
+  while (NumTableFilesAtLevel(2, 1) == 0) {
+    ASSERT_OK(Put(1, Key(i++), value));
+  }
+
+  options.num_levels = 1;
+  options.max_bytes_for_level_multiplier_additional.resize(1, 1);
+  Status s = TryReopenWithColumnFamilies({"default", "pikachu"}, options);
+  ASSERT_EQ(s.IsInvalidArgument(), true);
+  ASSERT_EQ(s.ToString(),
+            "Invalid argument: db has more levels than options.num_levels");
+
+  options.num_levels = 10;
+  options.max_bytes_for_level_multiplier_additional.resize(10, 1);
+  ASSERT_OK(TryReopenWithColumnFamilies({"default", "pikachu"}, options));
+}
+#endif  // ROCKSDB_LITE
+
+TEST_F(DBBasicTest, PutDeleteGet) {
+  do {
+    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
+    ASSERT_OK(Put(1, "foo", "v1"));
+    ASSERT_EQ("v1", Get(1, "foo"));
+    ASSERT_OK(Put(1, "foo", "v2"));
+    ASSERT_EQ("v2", Get(1, "foo"));
+    ASSERT_OK(Delete(1, "foo"));
+    ASSERT_EQ("NOT_FOUND", Get(1, "foo"));
+  } while (ChangeOptions());
+}
+
+TEST_F(DBBasicTest, PutSingleDeleteGet) {
+  do {
+    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
+    ASSERT_OK(Put(1, "foo", "v1"));
+    ASSERT_EQ("v1", Get(1, "foo"));
+    ASSERT_OK(Put(1, "foo2", "v2"));
+    ASSERT_EQ("v2", Get(1, "foo2"));
+    ASSERT_OK(SingleDelete(1, "foo"));
+    ASSERT_EQ("NOT_FOUND", Get(1, "foo"));
+    // Skip HashCuckooRep as it does not support single delete. FIFO and
+    // universal compaction do not apply to the test case. Skip MergePut
+    // because single delete does not get removed when it encounters a merge.
+  } while (ChangeOptions(kSkipHashCuckoo | kSkipFIFOCompaction |
+                         kSkipUniversalCompaction | kSkipMergePut));
+}
+
+TEST_F(DBBasicTest, EmptyFlush) {
+  // It is possible to produce empty flushes when using single deletes. Tests
+  // whether empty flushes cause issues.
+  do {
+    Random rnd(301);
+
+    Options options = CurrentOptions();
+    options.disable_auto_compactions = true;
+    CreateAndReopenWithCF({"pikachu"}, options);
+
+    Put(1, "a", Slice());
+    SingleDelete(1, "a");
+    ASSERT_OK(Flush(1));
+
+    ASSERT_EQ("[ ]", AllEntriesFor("a", 1));
+    // Skip HashCuckooRep as it does not support single delete. FIFO and
+    // universal compaction do not apply to the test case. Skip MergePut
+    // because merges cannot be combined with single deletions.
+  } while (ChangeOptions(kSkipHashCuckoo | kSkipFIFOCompaction |
+                         kSkipUniversalCompaction | kSkipMergePut));
+}
+
+TEST_F(DBBasicTest, GetFromVersions) {
+  do {
+    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
+    ASSERT_OK(Put(1, "foo", "v1"));
+    ASSERT_OK(Flush(1));
+    ASSERT_EQ("v1", Get(1, "foo"));
+    ASSERT_EQ("NOT_FOUND", Get(0, "foo"));
+  } while (ChangeOptions());
+}
+
+#ifndef ROCKSDB_LITE
+TEST_F(DBBasicTest, GetSnapshot) {
+  anon::OptionsOverride options_override;
+  options_override.skip_policy = kSkipNoSnapshot;
+  do {
+    CreateAndReopenWithCF({"pikachu"}, CurrentOptions(options_override));
+    // Try with both a short key and a long key
+    for (int i = 0; i < 2; i++) {
+      std::string key = (i == 0) ? std::string("foo") : std::string(200, 'x');
+      ASSERT_OK(Put(1, key, "v1"));
+      const Snapshot* s1 = db_->GetSnapshot();
+      if (option_config_ == kHashCuckoo) {
+        // Unsupported case.
+        ASSERT_TRUE(s1 == nullptr);
+        break;
+      }
+      ASSERT_OK(Put(1, key, "v2"));
+      ASSERT_EQ("v2", Get(1, key));
+      ASSERT_EQ("v1", Get(1, key, s1));
+      ASSERT_OK(Flush(1));
+      ASSERT_EQ("v2", Get(1, key));
+      ASSERT_EQ("v1", Get(1, key, s1));
+      db_->ReleaseSnapshot(s1);
+    }
+  } while (ChangeOptions());
+}
+#endif  // ROCKSDB_LITE
+
+TEST_F(DBBasicTest, CheckLock) {
+  do {
+    DB* localdb;
+    Options options = CurrentOptions();
+    ASSERT_OK(TryReopen(options));
+
+    // second open should fail
+    ASSERT_TRUE(!(DB::Open(options, dbname_, &localdb)).ok());
+  } while (ChangeCompactOptions());
+}
+
+TEST_F(DBBasicTest, FlushMultipleMemtable) {
+  do {
+    Options options = CurrentOptions();
+    WriteOptions writeOpt = WriteOptions();
+    writeOpt.disableWAL = true;
+    options.max_write_buffer_number = 4;
+    options.min_write_buffer_number_to_merge = 3;
+    options.max_write_buffer_number_to_maintain = -1;
+    CreateAndReopenWithCF({"pikachu"}, options);
+    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v1"));
+    ASSERT_OK(Flush(1));
+    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1"));
+
+    ASSERT_EQ("v1", Get(1, "foo"));
+    ASSERT_EQ("v1", Get(1, "bar"));
+    ASSERT_OK(Flush(1));
+  } while (ChangeCompactOptions());
+}
+
+TEST_F(DBBasicTest, FlushEmptyColumnFamily) {
+  // Block flush thread and disable compaction thread
+  env_->SetBackgroundThreads(1, Env::HIGH);
+  env_->SetBackgroundThreads(1, Env::LOW);
+  test::SleepingBackgroundTask sleeping_task_low;
+  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
+                 Env::Priority::LOW);
+  test::SleepingBackgroundTask sleeping_task_high;
+  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
+                 &sleeping_task_high, Env::Priority::HIGH);
+
+  Options options = CurrentOptions();
+  // disable compaction
+  options.disable_auto_compactions = true;
+  WriteOptions writeOpt = WriteOptions();
+  writeOpt.disableWAL = true;
+  options.max_write_buffer_number = 2;
+  options.min_write_buffer_number_to_merge = 1;
+  options.max_write_buffer_number_to_maintain = 1;
+  CreateAndReopenWithCF({"pikachu"}, options);
+
+  // Compaction can still go through even if no thread can flush the
+  // mem table.
+  ASSERT_OK(Flush(0));
+  ASSERT_OK(Flush(1));
+
+  // Insert can go through
+  ASSERT_OK(dbfull()->Put(writeOpt, handles_[0], "foo", "v1"));
+  ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1"));
+
+  ASSERT_EQ("v1", Get(0, "foo"));
+  ASSERT_EQ("v1", Get(1, "bar"));
+
+  sleeping_task_high.WakeUp();
+  sleeping_task_high.WaitUntilDone();
+
+  // Flush can still go through.
+  ASSERT_OK(Flush(0));
+  ASSERT_OK(Flush(1));
+
+  sleeping_task_low.WakeUp();
+  sleeping_task_low.WaitUntilDone();
+}
+
+TEST_F(DBBasicTest, FLUSH) {
+  do {
+    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
+    WriteOptions writeOpt = WriteOptions();
+    writeOpt.disableWAL = true;
+    SetPerfLevel(kEnableTime);
+    ;
+    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v1"));
+    // this will now also flush the last 2 writes
+    ASSERT_OK(Flush(1));
+    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1"));
+
+    perf_context.Reset();
+    Get(1, "foo");
+    ASSERT_TRUE((int)perf_context.get_from_output_files_time > 0);
+
+    ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
+    ASSERT_EQ("v1", Get(1, "foo"));
+    ASSERT_EQ("v1", Get(1, "bar"));
+
+    writeOpt.disableWAL = true;
+    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v2"));
+    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v2"));
+    ASSERT_OK(Flush(1));
+
+    ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
+    ASSERT_EQ("v2", Get(1, "bar"));
+    perf_context.Reset();
+    ASSERT_EQ("v2", Get(1, "foo"));
+    ASSERT_TRUE((int)perf_context.get_from_output_files_time > 0);
+
+    writeOpt.disableWAL = false;
+    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v3"));
+    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v3"));
+    ASSERT_OK(Flush(1));
+
+    ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
+    // 'foo' should be there because its put
+    // has WAL enabled.
+    ASSERT_EQ("v3", Get(1, "foo"));
+    ASSERT_EQ("v3", Get(1, "bar"));
+
+    SetPerfLevel(kDisable);
+  } while (ChangeCompactOptions());
+}
+
+TEST_F(DBBasicTest, ManifestRollOver) {
+  do {
+    Options options;
+    options.max_manifest_file_size = 10;  // 10 bytes
+    options = CurrentOptions(options);
+    CreateAndReopenWithCF({"pikachu"}, options);
+    {
+      ASSERT_OK(Put(1, "manifest_key1", std::string(1000, '1')));
+      ASSERT_OK(Put(1, "manifest_key2", std::string(1000, '2')));
+      ASSERT_OK(Put(1, "manifest_key3", std::string(1000, '3')));
+      uint64_t manifest_before_flush = dbfull()->TEST_Current_Manifest_FileNo();
+      ASSERT_OK(Flush(1));  // This should trigger LogAndApply.
+      uint64_t manifest_after_flush = dbfull()->TEST_Current_Manifest_FileNo();
+      ASSERT_GT(manifest_after_flush, manifest_before_flush);
+      ReopenWithColumnFamilies({"default", "pikachu"}, options);
+      ASSERT_GT(dbfull()->TEST_Current_Manifest_FileNo(), manifest_after_flush);
+      // check if a new manifest file got inserted or not.
+      ASSERT_EQ(std::string(1000, '1'), Get(1, "manifest_key1"));
+      ASSERT_EQ(std::string(1000, '2'), Get(1, "manifest_key2"));
+      ASSERT_EQ(std::string(1000, '3'), Get(1, "manifest_key3"));
+    }
+  } while (ChangeCompactOptions());
+}
+
+TEST_F(DBBasicTest, IdentityAcrossRestarts) {
+  do {
+    std::string id1;
+    ASSERT_OK(db_->GetDbIdentity(id1));
+
+    Options options = CurrentOptions();
+    Reopen(options);
+    std::string id2;
+    ASSERT_OK(db_->GetDbIdentity(id2));
+    // id1 should match id2 because identity was not regenerated
+    ASSERT_EQ(id1.compare(id2), 0);
+
+    std::string idfilename = IdentityFileName(dbname_);
+    ASSERT_OK(env_->DeleteFile(idfilename));
+    Reopen(options);
+    std::string id3;
+    ASSERT_OK(db_->GetDbIdentity(id3));
+    // id1 should NOT match id3 because identity was regenerated
+    ASSERT_NE(id1.compare(id3), 0);
+  } while (ChangeCompactOptions());
+}
+
+#ifndef ROCKSDB_LITE
+TEST_F(DBBasicTest, Snapshot) {
+  anon::OptionsOverride options_override;
+  options_override.skip_policy = kSkipNoSnapshot;
+  do {
+    CreateAndReopenWithCF({"pikachu"}, CurrentOptions(options_override));
+    Put(0, "foo", "0v1");
+    Put(1, "foo", "1v1");
+
+    const Snapshot* s1 = db_->GetSnapshot();
+    ASSERT_EQ(1U, GetNumSnapshots());
+    uint64_t time_snap1 = GetTimeOldestSnapshots();
+    ASSERT_GT(time_snap1, 0U);
+    Put(0, "foo", "0v2");
+    Put(1, "foo", "1v2");
+
+    env_->addon_time_.fetch_add(1);
+
+    const Snapshot* s2 = db_->GetSnapshot();
+    ASSERT_EQ(2U, GetNumSnapshots());
+    ASSERT_EQ(time_snap1, GetTimeOldestSnapshots());
+    Put(0, "foo", "0v3");
+    Put(1, "foo", "1v3");
+
+    {
+      ManagedSnapshot s3(db_);
+      ASSERT_EQ(3U, GetNumSnapshots());
+      ASSERT_EQ(time_snap1, GetTimeOldestSnapshots());
+
+      Put(0, "foo", "0v4");
+      Put(1, "foo", "1v4");
+      ASSERT_EQ("0v1", Get(0, "foo", s1));
+      ASSERT_EQ("1v1", Get(1, "foo", s1));
+      ASSERT_EQ("0v2", Get(0, "foo", s2));
+      ASSERT_EQ("1v2", Get(1, "foo", s2));
+      ASSERT_EQ("0v3", Get(0, "foo", s3.snapshot()));
+      ASSERT_EQ("1v3", Get(1, "foo", s3.snapshot()));
+      ASSERT_EQ("0v4", Get(0, "foo"));
+      ASSERT_EQ("1v4", Get(1, "foo"));
+    }
+
+    ASSERT_EQ(2U, GetNumSnapshots());
+    ASSERT_EQ(time_snap1, GetTimeOldestSnapshots());
+    ASSERT_EQ("0v1", Get(0, "foo", s1));
+    ASSERT_EQ("1v1", Get(1, "foo", s1));
+    ASSERT_EQ("0v2", Get(0, "foo", s2));
+    ASSERT_EQ("1v2", Get(1, "foo", s2));
+    ASSERT_EQ("0v4", Get(0, "foo"));
+    ASSERT_EQ("1v4", Get(1, "foo"));
+
+    db_->ReleaseSnapshot(s1);
+    ASSERT_EQ("0v2", Get(0, "foo", s2));
+    ASSERT_EQ("1v2", Get(1, "foo", s2));
+    ASSERT_EQ("0v4", Get(0, "foo"));
+    ASSERT_EQ("1v4", Get(1, "foo"));
+    ASSERT_EQ(1U, GetNumSnapshots());
+    ASSERT_LT(time_snap1, GetTimeOldestSnapshots());
+
+    db_->ReleaseSnapshot(s2);
+    ASSERT_EQ(0U, GetNumSnapshots());
+    ASSERT_EQ("0v4", Get(0, "foo"));
+    ASSERT_EQ("1v4", Get(1, "foo"));
+  } while (ChangeOptions(kSkipHashCuckoo));
+}
+
+#endif  // ROCKSDB_LITE
+
+TEST_F(DBBasicTest, CompactBetweenSnapshots) {
+  anon::OptionsOverride options_override;
+  options_override.skip_policy = kSkipNoSnapshot;
+  do {
+    Options options = CurrentOptions(options_override);
+    options.disable_auto_compactions = true;
+    CreateAndReopenWithCF({"pikachu"}, options);
+    Random rnd(301);
+    FillLevels("a", "z", 1);
+
+    Put(1, "foo", "first");
+    const Snapshot* snapshot1 = db_->GetSnapshot();
+    Put(1, "foo", "second");
+    Put(1, "foo", "third");
+    Put(1, "foo", "fourth");
+    const Snapshot* snapshot2 = db_->GetSnapshot();
+    Put(1, "foo", "fifth");
+    Put(1, "foo", "sixth");
+
+    // All entries (including duplicates) exist
+    // before any compaction or flush is triggered.
+    ASSERT_EQ(AllEntriesFor("foo", 1),
+              "[ sixth, fifth, fourth, third, second, first ]");
+    ASSERT_EQ("sixth", Get(1, "foo"));
+    ASSERT_EQ("fourth", Get(1, "foo", snapshot2));
+    ASSERT_EQ("first", Get(1, "foo", snapshot1));
+
+    // After a flush, "second", "third" and "fifth" should
+    // be removed
+    ASSERT_OK(Flush(1));
+    ASSERT_EQ(AllEntriesFor("foo", 1), "[ sixth, fourth, first ]");
+
+    // after we release the snapshot1, only two values left
+    db_->ReleaseSnapshot(snapshot1);
+    FillLevels("a", "z", 1);
+    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
+                           nullptr);
+
+    // We have only one valid snapshot snapshot2. Since snapshot1 is
+    // not valid anymore, "first" should be removed by a compaction.
+    ASSERT_EQ("sixth", Get(1, "foo"));
+    ASSERT_EQ("fourth", Get(1, "foo", snapshot2));
+    ASSERT_EQ(AllEntriesFor("foo", 1), "[ sixth, fourth ]");
+
+    // after we release the snapshot2, only one value should be left
+    db_->ReleaseSnapshot(snapshot2);
+    FillLevels("a", "z", 1);
+    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
+                           nullptr);
+    ASSERT_EQ("sixth", Get(1, "foo"));
+    ASSERT_EQ(AllEntriesFor("foo", 1), "[ sixth ]");
+    // skip HashCuckooRep as it does not support snapshot
+  } while (ChangeOptions(kSkipHashCuckoo | kSkipFIFOCompaction));
+}
+
+TEST_F(DBBasicTest, DBOpen_Options) {
+  Options options = CurrentOptions();
+  std::string dbname = test::TmpDir(env_) + "/db_options_test";
+  ASSERT_OK(DestroyDB(dbname, options));
+
+  // Does not exist, and create_if_missing == false: error
+  DB* db = nullptr;
+  options.create_if_missing = false;
+  Status s = DB::Open(options, dbname, &db);
+  ASSERT_TRUE(strstr(s.ToString().c_str(), "does not exist") != nullptr);
+  ASSERT_TRUE(db == nullptr);
+
+  // Does not exist, and create_if_missing == true: OK
+  options.create_if_missing = true;
+  s = DB::Open(options, dbname, &db);
+  ASSERT_OK(s);
+  ASSERT_TRUE(db != nullptr);
+
+  delete db;
+  db = nullptr;
+
+  // Does exist, and error_if_exists == true: error
+  options.create_if_missing = false;
+  options.error_if_exists = true;
+  s = DB::Open(options, dbname, &db);
+  ASSERT_TRUE(strstr(s.ToString().c_str(), "exists") != nullptr);
+  ASSERT_TRUE(db == nullptr);
+
+  // Does exist, and error_if_exists == false: OK
+  options.create_if_missing = true;
+  options.error_if_exists = false;
+  s = DB::Open(options, dbname, &db);
+  ASSERT_OK(s);
+  ASSERT_TRUE(db != nullptr);
+
+  delete db;
+  db = nullptr;
+}
+
+TEST_F(DBBasicTest, CompactOnFlush) {
+  anon::OptionsOverride options_override;
+  options_override.skip_policy = kSkipNoSnapshot;
+  do {
+    Options options = CurrentOptions(options_override);
+    options.disable_auto_compactions = true;
+    CreateAndReopenWithCF({"pikachu"}, options);
+
+    Put(1, "foo", "v1");
+    ASSERT_OK(Flush(1));
+    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v1 ]");
+
+    // Write two new keys
+    Put(1, "a", "begin");
+    Put(1, "z", "end");
+    Flush(1);
+
+    // Case1: Delete followed by a put
+    Delete(1, "foo");
+    Put(1, "foo", "v2");
+    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2, DEL, v1 ]");
+
+    // After the current memtable is flushed, the DEL should
+    // have been removed
+    ASSERT_OK(Flush(1));
+    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2, v1 ]");
+
+    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
+                           nullptr);
+    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2 ]");
+
+    // Case 2: Delete followed by another delete
+    Delete(1, "foo");
+    Delete(1, "foo");
+    ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, DEL, v2 ]");
+    ASSERT_OK(Flush(1));
+    ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, v2 ]");
+    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
+                           nullptr);
+    ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]");
+
+    // Case 3: Put followed by a delete
+    Put(1, "foo", "v3");
+    Delete(1, "foo");
+    ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, v3 ]");
+    ASSERT_OK(Flush(1));
+    ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL ]");
+    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
+                           nullptr);
+    ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]");
+
+    // Case 4: Put followed by another Put
+    Put(1, "foo", "v4");
+    Put(1, "foo", "v5");
+    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v5, v4 ]");
+    ASSERT_OK(Flush(1));
+    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v5 ]");
+    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
+                           nullptr);
+    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v5 ]");
+
+    // clear database
+    Delete(1, "foo");
+    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
+                           nullptr);
+    ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]");
+
+    // Case 5: Put followed by snapshot followed by another Put
+    // Both puts should remain.
+    Put(1, "foo", "v6");
+    const Snapshot* snapshot = db_->GetSnapshot();
+    Put(1, "foo", "v7");
+    ASSERT_OK(Flush(1));
+    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v7, v6 ]");
+    db_->ReleaseSnapshot(snapshot);
+
+    // clear database
+    Delete(1, "foo");
+    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
+                           nullptr);
+    ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]");
+
+    // Case 5: snapshot followed by a put followed by another Put
+    // Only the last put should remain.
+    const Snapshot* snapshot1 = db_->GetSnapshot();
+    Put(1, "foo", "v8");
+    Put(1, "foo", "v9");
+    ASSERT_OK(Flush(1));
+    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v9 ]");
+    db_->ReleaseSnapshot(snapshot1);
+  } while (ChangeCompactOptions());
+}
+
+TEST_F(DBBasicTest, FlushOneColumnFamily) {
+  Options options = CurrentOptions();
+  CreateAndReopenWithCF({"pikachu", "ilya", "muromec", "dobrynia", "nikitich",
+                         "alyosha", "popovich"},
+                        options);
+
+  ASSERT_OK(Put(0, "Default", "Default"));
+  ASSERT_OK(Put(1, "pikachu", "pikachu"));
+  ASSERT_OK(Put(2, "ilya", "ilya"));
+  ASSERT_OK(Put(3, "muromec", "muromec"));
+  ASSERT_OK(Put(4, "dobrynia", "dobrynia"));
+  ASSERT_OK(Put(5, "nikitich", "nikitich"));
+  ASSERT_OK(Put(6, "alyosha", "alyosha"));
+  ASSERT_OK(Put(7, "popovich", "popovich"));
+
+  for (int i = 0; i < 8; ++i) {
+    Flush(i);
+    auto tables = ListTableFiles(env_, dbname_);
+    ASSERT_EQ(tables.size(), i + 1U);
+  }
+}
+
+TEST_F(DBBasicTest, MultiGetSimple) {
+  do {
+    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
+    ASSERT_OK(Put(1, "k1", "v1"));
+    ASSERT_OK(Put(1, "k2", "v2"));
+    ASSERT_OK(Put(1, "k3", "v3"));
+    ASSERT_OK(Put(1, "k4", "v4"));
+    ASSERT_OK(Delete(1, "k4"));
+    ASSERT_OK(Put(1, "k5", "v5"));
+    ASSERT_OK(Delete(1, "no_key"));
+
+    std::vector<Slice> keys({"k1", "k2", "k3", "k4", "k5", "no_key"});
+
+    std::vector<std::string> values(20, "Temporary data to be overwritten");
+    std::vector<ColumnFamilyHandle*> cfs(keys.size(), handles_[1]);
+
+    std::vector<Status> s = db_->MultiGet(ReadOptions(), cfs, keys, &values);
+    ASSERT_EQ(values.size(), keys.size());
+    ASSERT_EQ(values[0], "v1");
+    ASSERT_EQ(values[1], "v2");
+    ASSERT_EQ(values[2], "v3");
+    ASSERT_EQ(values[4], "v5");
+
+    ASSERT_OK(s[0]);
+    ASSERT_OK(s[1]);
+    ASSERT_OK(s[2]);
+    ASSERT_TRUE(s[3].IsNotFound());
+    ASSERT_OK(s[4]);
+    ASSERT_TRUE(s[5].IsNotFound());
+  } while (ChangeCompactOptions());
+}
+
+TEST_F(DBBasicTest, MultiGetEmpty) {
+  do {
+    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
+    // Empty Key Set
+    std::vector<Slice> keys;
+    std::vector<std::string> values;
+    std::vector<ColumnFamilyHandle*> cfs;
+    std::vector<Status> s = db_->MultiGet(ReadOptions(), cfs, keys, &values);
+    ASSERT_EQ(s.size(), 0U);
+
+    // Empty Database, Empty Key Set
+    Options options = CurrentOptions();
+    options.create_if_missing = true;
+    DestroyAndReopen(options);
+    CreateAndReopenWithCF({"pikachu"}, options);
+    s = db_->MultiGet(ReadOptions(), cfs, keys, &values);
+    ASSERT_EQ(s.size(), 0U);
+
+    // Empty Database, Search for Keys
+    keys.resize(2);
+    keys[0] = "a";
+    keys[1] = "b";
+    cfs.push_back(handles_[0]);
+    cfs.push_back(handles_[1]);
+    s = db_->MultiGet(ReadOptions(), cfs, keys, &values);
+    ASSERT_EQ(static_cast<int>(s.size()), 2);
+    ASSERT_TRUE(s[0].IsNotFound() && s[1].IsNotFound());
+  } while (ChangeCompactOptions());
+}
+
+TEST_F(DBBasicTest, ChecksumTest) {
+  BlockBasedTableOptions table_options;
+  Options options = CurrentOptions();
+
+  table_options.checksum = kCRC32c;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  Reopen(options);
+  ASSERT_OK(Put("a", "b"));
+  ASSERT_OK(Put("c", "d"));
+  ASSERT_OK(Flush());  // table with crc checksum
+
+  table_options.checksum = kxxHash;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  Reopen(options);
+  ASSERT_OK(Put("e", "f"));
+  ASSERT_OK(Put("g", "h"));
+  ASSERT_OK(Flush());  // table with xxhash checksum
+
+  table_options.checksum = kCRC32c;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  Reopen(options);
+  ASSERT_EQ("b", Get("a"));
+  ASSERT_EQ("d", Get("c"));
+  ASSERT_EQ("f", Get("e"));
+  ASSERT_EQ("h", Get("g"));
+
+  table_options.checksum = kCRC32c;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  Reopen(options);
+  ASSERT_EQ("b", Get("a"));
+  ASSERT_EQ("d", Get("c"));
+  ASSERT_EQ("f", Get("e"));
+  ASSERT_EQ("h", Get("g"));
+}
+
+// On Windows you can have either memory mapped file or a file
+// with unbuffered access. So this asserts and does not make
+// sense to run
+#ifndef OS_WIN
+TEST_F(DBBasicTest, MmapAndBufferOptions) {
+  Options options = CurrentOptions();
+
+  options.use_direct_reads = true;
+  options.allow_mmap_reads = true;
+  ASSERT_NOK(TryReopen(options));
+
+  // All other combinations are acceptable
+  options.use_direct_reads = false;
+  ASSERT_OK(TryReopen(options));
+
+  if (IsDirectIOSupported()) {
+    options.use_direct_reads = true;
+    options.allow_mmap_reads = false;
+    ASSERT_OK(TryReopen(options));
+  }
+
+  options.use_direct_reads = false;
+  ASSERT_OK(TryReopen(options));
+}
+#endif
+
+}  // namespace rocksdb
+
+int main(int argc, char** argv) {
+  rocksdb::port::InstallStackTraceHandler();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

db/db_bloom_filter_test.cc

@@ -788,8 +788,6 @@ void PrefixScanInit(DBBloomFilterTest* dbtest) {
 }  // namespace
 
 TEST_F(DBBloomFilterTest, PrefixScan) {
-  XFUNC_TEST("", "dbtest_prefix", prefix_skip1, XFuncPoint::SetSkip,
-             kSkipNoPrefix);
   while (ChangeFilterOptions()) {
     int count;
     Slice prefix;
@@ -836,7 +834,6 @@ TEST_F(DBBloomFilterTest, PrefixScan) {
     ASSERT_EQ(env_->random_read_counter_.Read(), 2);
     Close();
   }  // end of while
-  XFUNC_TEST("", "dbtest_prefix", prefix_skip1, XFuncPoint::SetSkip, 0);
 }
 
 TEST_F(DBBloomFilterTest, OptimizeFiltersForHits) {

db/db_flush_test.cc

@@ -74,7 +74,9 @@ TEST_F(DBFlushTest, SyncFail) {
   TEST_SYNC_POINT("DBFlushTest::SyncFail:2");
   fault_injection_env->SetFilesystemActive(true);
   dbfull()->TEST_WaitForFlushMemTable();
+#ifndef ROCKSDB_LITE
   ASSERT_EQ("", FilesPerLevel());  // flush failed.
+#endif                             // ROCKSDB_LITE
   // Flush job should release ref count to current version.
   ASSERT_EQ(refs_before, cfd->current()->TEST_refs());
   Destroy(options);

db/db_impl.cc

@@ -58,7 +58,6 @@
 #include "db/version_set.h"
 #include "db/write_batch_internal.h"
 #include "db/write_callback.h"
-#include "db/xfunc_test_points.h"
 #include "memtable/hash_linklist_rep.h"
 #include "memtable/hash_skiplist_rep.h"
 #include "port/likely.h"
@@ -100,7 +99,6 @@
 #include "util/sync_point.h"
 #include "util/thread_status_updater.h"
 #include "util/thread_status_util.h"
-#include "util/xfunc.h"
 
 namespace rocksdb {
 
@@ -150,6 +148,7 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src) {
       result.info_log = nullptr;
     }
   }
+
   if (!result.write_buffer_manager) {
     result.write_buffer_manager.reset(
         new WriteBufferManager(result.db_write_buffer_size));
@@ -198,6 +197,10 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src) {
     result.db_paths.emplace_back(dbname, std::numeric_limits<uint64_t>::max());
   }
 
+  if (result.use_direct_reads && result.compaction_readahead_size == 0) {
+    result.compaction_readahead_size = 1024 * 1024 * 2;
+  }
+
   if (result.compaction_readahead_size > 0) {
     result.new_table_reader_for_compaction_inputs = true;
   }
@@ -388,7 +391,7 @@ void DBImpl::CancelAllBackgroundWork(bool wait) {
       "Shutdown: canceling all background work");
 
   if (!shutting_down_.load(std::memory_order_acquire) &&
-      has_unpersisted_data_ &&
+      has_unpersisted_data_.load(std::memory_order_relaxed) &&
       !mutable_db_options_.avoid_flush_during_shutdown) {
     for (auto cfd : *versions_->GetColumnFamilySet()) {
       if (!cfd->IsDropped() && !cfd->mem()->IsEmpty()) {
@@ -606,7 +609,7 @@ static void DumpMallocStats(std::string* stats) {
 #ifdef ROCKSDB_JEMALLOC
   MallocStatus mstat;
   const uint kMallocStatusLen = 1000000;
-  std::unique_ptr<char> buf{new char[kMallocStatusLen + 1]};
+  std::unique_ptr<char[]> buf{new char[kMallocStatusLen + 1]};
   mstat.cur = buf.get();
   mstat.end = buf.get() + kMallocStatusLen;
   malloc_stats_print(GetJemallocStatus, &mstat, "");
@@ -1915,8 +1918,7 @@ Status DBImpl::FlushMemTableToOutputFile(
 
   Status s;
   if (logfile_number_ > 0 &&
-      versions_->GetColumnFamilySet()->NumberOfColumnFamilies() > 0 &&
-      !immutable_db_options_.disable_data_sync) {
+      versions_->GetColumnFamilySet()->NumberOfColumnFamilies() > 0) {
     // If there are more than one column families, we need to make sure that
     // all the log files except the most recent one are synced. Otherwise if
     // the host crashes after flushing and before WAL is persistent, the
@@ -1969,8 +1971,9 @@ Status DBImpl::FlushMemTableToOutputFile(
       sfm->OnAddFile(file_path);
       if (sfm->IsMaxAllowedSpaceReached() && bg_error_.ok()) {
         bg_error_ = Status::IOError("Max allowed space was reached");
-        TEST_SYNC_POINT(
-            "DBImpl::FlushMemTableToOutputFile:MaxAllowedSpaceReached");
+        TEST_SYNC_POINT_CALLBACK(
+            "DBImpl::FlushMemTableToOutputFile:MaxAllowedSpaceReached",
+            &bg_error_);
       }
     }
 #endif  // ROCKSDB_LITE
@@ -3916,8 +3919,10 @@ InternalIterator* DBImpl::NewInternalIterator(
   }
   if (s.ok()) {
     // Collect iterators for files in L0 - Ln
-    super_version->current->AddIterators(read_options, env_options_,
-                                         &merge_iter_builder, range_del_agg);
+    if (read_options.read_tier != kMemtableTier) {
+      super_version->current->AddIterators(read_options, env_options_,
+                                           &merge_iter_builder, range_del_agg);
+    }
     internal_iter = merge_iter_builder.Finish();
     IterState* cleanup =
         new IterState(this, &mutex_, super_version,
@@ -4037,8 +4042,8 @@ Status DBImpl::GetImpl(const ReadOptions& read_options,
   LookupKey lkey(key, snapshot);
   PERF_TIMER_STOP(get_snapshot_time);
 
-  bool skip_memtable =
-      (read_options.read_tier == kPersistedTier && has_unpersisted_data_);
+  bool skip_memtable = (read_options.read_tier == kPersistedTier &&
+                        has_unpersisted_data_.load(std::memory_order_relaxed));
   bool done = false;
   if (!skip_memtable) {
     if (sv->mem->Get(lkey, value, &s, &merge_context, &range_del_agg,
@@ -4143,7 +4148,8 @@ std::vector<Status> DBImpl::MultiGet(
     auto mgd = mgd_iter->second;
     auto super_version = mgd->super_version;
     bool skip_memtable =
-        (read_options.read_tier == kPersistedTier && has_unpersisted_data_);
+        (read_options.read_tier == kPersistedTier &&
+         has_unpersisted_data_.load(std::memory_order_relaxed));
     bool done = false;
     if (!skip_memtable) {
       if (super_version->mem->Get(lkey, value, &s, &merge_context,
@@ -4393,10 +4399,6 @@ Iterator* DBImpl::NewIterator(const ReadOptions& read_options,
   }
   auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
   auto cfd = cfh->cfd();
-
-  XFUNC_TEST("", "managed_new", managed_new1, xf_manage_new,
-             reinterpret_cast<DBImpl*>(this),
-             const_cast<ReadOptions*>(&read_options), is_snapshot_supported_);
   if (read_options.managed) {
 #ifdef ROCKSDB_LITE
     // not supported in lite version
@@ -4505,9 +4507,6 @@ Status DBImpl::NewIterators(
   }
   iterators->clear();
   iterators->reserve(column_families.size());
-  XFUNC_TEST("", "managed_new", managed_new1, xf_manage_new,
-             reinterpret_cast<DBImpl*>(this),
-             const_cast<ReadOptions*>(&read_options), is_snapshot_supported_);
   if (read_options.managed) {
 #ifdef ROCKSDB_LITE
     return Status::InvalidArgument(
@@ -4647,22 +4646,9 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
   if (my_batch == nullptr) {
     return Status::Corruption("Batch is nullptr!");
   }
-  if (write_options.timeout_hint_us != 0) {
-    return Status::InvalidArgument("timeout_hint_us is deprecated");
-  }
 
   Status status;
 
-  bool xfunc_attempted_write = false;
-  XFUNC_TEST("transaction", "transaction_xftest_write_impl",
-             xf_transaction_write1, xf_transaction_write, write_options,
-             immutable_db_options_, my_batch, callback, this, &status,
-             &xfunc_attempted_write);
-  if (xfunc_attempted_write) {
-    // Test already did the write
-    return status;
-  }
-
   PERF_TIMER_GUARD(write_pre_and_post_process_time);
   WriteThread::Writer w;
   w.batch = my_batch;
@@ -4740,7 +4726,8 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
   if (UNLIKELY(!single_column_family_mode_ &&
                total_log_size_ > GetMaxTotalWalSize())) {
     MaybeFlushColumnFamilies();
-  } else if (UNLIKELY(write_buffer_manager_->ShouldFlush())) {
+  }
+  if (UNLIKELY(write_buffer_manager_->ShouldFlush())) {
     // Before a new memtable is added in SwitchMemtable(),
     // write_buffer_manager_->ShouldFlush() will keep returning true. If another
     // thread is writing to another DB with the same write buffer, they may also
@@ -4876,7 +4863,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
     PERF_TIMER_STOP(write_pre_and_post_process_time);
 
     if (write_options.disableWAL) {
-      has_unpersisted_data_ = true;
+      has_unpersisted_data_.store(true, std::memory_order_relaxed);
     }
 
     uint64_t log_size = 0;
@@ -5069,14 +5056,25 @@ void DBImpl::MaybeFlushColumnFamilies() {
   auto oldest_log_with_uncommited_prep = FindMinLogContainingOutstandingPrep();
 
   if (allow_2pc() &&
-      unable_to_flush_oldest_log_ &&
       oldest_log_with_uncommited_prep > 0 &&
       oldest_log_with_uncommited_prep <= oldest_alive_log) {
-    // we already attempted to flush all column families dependent on
-    // the oldest alive log but the log still contained uncommited transactions.
-    // the oldest alive log STILL contains uncommited transaction so there
-    // is still nothing that we can do.
-    return;
+    if (unable_to_flush_oldest_log_) {
+        // we already attempted to flush all column families dependent on
+        // the oldest alive log but the log still contained uncommited transactions.
+        // the oldest alive log STILL contains uncommited transaction so there
+        // is still nothing that we can do.
+        return;
+    } else {
+      Log(InfoLogLevel::WARN_LEVEL, immutable_db_options_.info_log,
+          "Unable to release oldest log due to uncommited transaction");
+      unable_to_flush_oldest_log_ = true;
+    }
+  } else {
+    // we only mark this log as getting flushed if we have successfully
+    // flushed all data in this log. If this log contains outstanding prepared
+    // transactions then we cannot flush this log until those transactions are commited.
+    unable_to_flush_oldest_log_ = false;
+    alive_log_files_.begin()->getting_flushed = true;
   }
 
   WriteContext context;
@@ -5102,25 +5100,6 @@ void DBImpl::MaybeFlushColumnFamilies() {
   }
   MaybeScheduleFlushOrCompaction();
 
-  // we only mark this log as getting flushed if we have successfully
-  // flushed all data in this log. If this log contains outstanding prepred
-  // transactions then we cannot flush this log until those transactions are commited.
-
-  unable_to_flush_oldest_log_ = false;
-
-  if (allow_2pc()) {
-    if (oldest_log_with_uncommited_prep == 0 ||
-        oldest_log_with_uncommited_prep > oldest_alive_log) {
-      // this log contains no outstanding prepared transactions
-      alive_log_files_.begin()->getting_flushed = true;
-    } else {
-      Log(InfoLogLevel::WARN_LEVEL, immutable_db_options_.info_log,
-          "Unable to release oldest log due to uncommited transaction");
-      unable_to_flush_oldest_log_ = true;
-    }
-  } else {
-    alive_log_files_.begin()->getting_flushed = true;
-  }
 }
 
 uint64_t DBImpl::GetMaxTotalWalSize() const {
@@ -6542,10 +6521,15 @@ Status DBImpl::IngestExternalFile(
                                             immutable_db_options_, env_options_,
                                             &snapshots_, ingestion_options);
 
-  // Make sure that bg cleanup wont delete the files that we are ingesting
   std::list<uint64_t>::iterator pending_output_elem;
   {
     InstrumentedMutexLock l(&mutex_);
+    if (!bg_error_.ok()) {
+      // Don't ingest files when there is a bg_error
+      return bg_error_;
+    }
+
+    // Make sure that bg cleanup wont delete the files that we are ingesting
     pending_output_elem = CaptureCurrentFileNumberInPendingOutputs();
   }
 

db/db_impl.h

@@ -465,6 +465,8 @@ class DBImpl : public DB {
     return num_running_compactions_;
   }
 
+  const WriteController& write_controller() { return write_controller_; }
+
   // hollow transactions shell used for recovery.
   // these will then be passed to TransactionDB so that
   // locks can be reacquired before writing can resume.
@@ -565,8 +567,10 @@ class DBImpl : public DB {
   void NotifyOnMemTableSealed(ColumnFamilyData* cfd,
                               const MemTableInfo& mem_table_info);
 
+#ifndef ROCKSDB_LITE
   void NotifyOnExternalFileIngested(
       ColumnFamilyData* cfd, const ExternalSstFileIngestionJob& ingestion_job);
+#endif  // !ROCKSDB_LITE
 
   void NewThreadStatusCfInfo(ColumnFamilyData* cfd) const;
 
@@ -998,8 +1002,7 @@ class DBImpl : public DB {
   // A flag indicating whether the current rocksdb database has any
   // data that is not yet persisted into either WAL or SST file.
   // Used when disableWAL is true.
-  bool has_unpersisted_data_;
-
+  std::atomic<bool> has_unpersisted_data_;
 
   // if an attempt was made to flush all column families that
   // the oldest log depends on but uncommited data in the oldest

db/db_sst_test.cc

@@ -328,9 +328,10 @@ TEST_F(DBSSTTest, RateLimitedDelete) {
   std::string trash_dir = test::TmpDir(env_) + "/trash";
   int64_t rate_bytes_per_sec = 1024 * 10;  // 10 Kbs / Sec
   Status s;
-  options.sst_file_manager.reset(NewSstFileManager(
-      env_, nullptr, trash_dir, rate_bytes_per_sec, false, &s));
+  options.sst_file_manager.reset(
+      NewSstFileManager(env_, nullptr, trash_dir, 0, false, &s));
   ASSERT_OK(s);
+  options.sst_file_manager->SetDeleteRateBytesPerSecond(rate_bytes_per_sec);
   auto sfm = static_cast<SstFileManagerImpl*>(options.sst_file_manager.get());
 
   ASSERT_OK(TryReopen(options));
@@ -516,18 +517,25 @@ TEST_F(DBSSTTest, DBWithMaxSpaceAllowedRandomized) {
   // than the limit.
 
   std::vector<int> max_space_limits_mbs = {1, 2, 4, 8, 10};
-
+  decltype(max_space_limits_mbs)::value_type limit_mb_cb;
   bool bg_error_set = false;
   uint64_t total_sst_files_size = 0;
 
+  std::atomic<int> estimate_multiplier(1);
   int reached_max_space_on_flush = 0;
   int reached_max_space_on_compaction = 0;
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
       "DBImpl::FlushMemTableToOutputFile:MaxAllowedSpaceReached",
       [&](void* arg) {
+        Status* bg_error = static_cast<Status*>(arg);
         bg_error_set = true;
         GetAllSSTFiles(&total_sst_files_size);
         reached_max_space_on_flush++;
+        // low limit for size calculated using sst files
+        ASSERT_GE(total_sst_files_size, limit_mb_cb * 1024 * 1024);
+        // clear error to ensure compaction callback is called
+        *bg_error = Status::OK();
+        estimate_multiplier++;  // used in the main loop assert
       });
 
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
@@ -541,6 +549,8 @@ TEST_F(DBSSTTest, DBWithMaxSpaceAllowedRandomized) {
   for (auto limit_mb : max_space_limits_mbs) {
     bg_error_set = false;
     total_sst_files_size = 0;
+    estimate_multiplier = 1;
+    limit_mb_cb = limit_mb;
     rocksdb::SyncPoint::GetInstance()->ClearTrace();
     rocksdb::SyncPoint::GetInstance()->EnableProcessing();
     std::shared_ptr<SstFileManager> sst_file_manager(NewSstFileManager(env_));
@@ -565,7 +575,8 @@ TEST_F(DBSSTTest, DBWithMaxSpaceAllowedRandomized) {
       // Check the estimated db size vs the db limit just to make sure we
       // dont run into an infinite loop
       estimated_db_size = keys_written * 60;  // ~60 bytes per key
-      ASSERT_LT(estimated_db_size, limit_mb * 1024 * 1024 * 2);
+      ASSERT_LT(estimated_db_size,
+                estimate_multiplier * limit_mb * 1024 * 1024 * 2);
     }
     ASSERT_TRUE(bg_error_set);
     ASSERT_GE(total_sst_files_size, limit_mb * 1024 * 1024);

db/db_tailing_iter_test.cc

@@ -294,8 +294,6 @@ TEST_F(DBTestTailingIterator, TailingIteratorDeletes) {
 }
 
 TEST_F(DBTestTailingIterator, TailingIteratorPrefixSeek) {
-  XFUNC_TEST("", "dbtest_prefix", prefix_skip1, XFuncPoint::SetSkip,
-             kSkipNoPrefix);
   ReadOptions read_options;
   read_options.tailing = true;
 
@@ -326,7 +324,6 @@ TEST_F(DBTestTailingIterator, TailingIteratorPrefixSeek) {
 
   iter->Next();
   ASSERT_TRUE(!iter->Valid());
-  XFUNC_TEST("", "dbtest_prefix", prefix_skip1, XFuncPoint::SetSkip, 0);
 }
 
 TEST_F(DBTestTailingIterator, TailingIteratorIncomplete) {
@@ -614,8 +611,6 @@ TEST_F(DBTestTailingIterator, ManagedTailingIteratorDeletes) {
 }
 
 TEST_F(DBTestTailingIterator, ManagedTailingIteratorPrefixSeek) {
-  XFUNC_TEST("", "dbtest_prefix", prefix_skip1, XFuncPoint::SetSkip,
-             kSkipNoPrefix);
   ReadOptions read_options;
   read_options.tailing = true;
   read_options.managed = true;
@@ -647,7 +642,6 @@ TEST_F(DBTestTailingIterator, ManagedTailingIteratorPrefixSeek) {
 
   iter->Next();
   ASSERT_TRUE(!iter->Valid());
-  XFUNC_TEST("", "dbtest_prefix", prefix_skip1, XFuncPoint::SetSkip, 0);
 }
 
 TEST_F(DBTestTailingIterator, ManagedTailingIteratorIncomplete) {

db/db_test.cc

@@ -67,7 +67,6 @@
 #include "util/testharness.h"
 #include "util/testutil.h"
 #include "util/thread_status_util.h"
-#include "util/xfunc.h"
 #include "utilities/merge_operators.h"
 
 namespace rocksdb {
@@ -198,19 +197,6 @@ TEST_F(DBTest, MemEnvTest) {
 }
 #endif  // ROCKSDB_LITE
 
-TEST_F(DBTest, OpenWhenOpen) {
-  Options options = CurrentOptions();
-  options.env = env_;
-  rocksdb::DB* db2 = nullptr;
-  rocksdb::Status s = DB::Open(options, dbname_, &db2);
-
-  ASSERT_EQ(Status::Code::kIOError, s.code());
-  ASSERT_EQ(Status::SubCode::kNone, s.subcode());
-  ASSERT_TRUE(strstr(s.getState(), "lock ") != nullptr);
-
-  delete db2;
-}
-
 TEST_F(DBTest, WriteEmptyBatch) {
   Options options = CurrentOptions();
   options.env = env_;
@@ -274,138 +260,6 @@ TEST_F(DBTest, SkipDelay) {
 }
 
 #ifndef ROCKSDB_LITE
-TEST_F(DBTest, ReadOnlyDB) {
-  ASSERT_OK(Put("foo", "v1"));
-  ASSERT_OK(Put("bar", "v2"));
-  ASSERT_OK(Put("foo", "v3"));
-  Close();
-
-  auto options = CurrentOptions();
-  assert(options.env = env_);
-  ASSERT_OK(ReadOnlyReopen(options));
-  ASSERT_EQ("v3", Get("foo"));
-  ASSERT_EQ("v2", Get("bar"));
-  Iterator* iter = db_->NewIterator(ReadOptions());
-  int count = 0;
-  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
-    ASSERT_OK(iter->status());
-    ++count;
-  }
-  ASSERT_EQ(count, 2);
-  delete iter;
-  Close();
-
-  // Reopen and flush memtable.
-  Reopen(options);
-  Flush();
-  Close();
-  // Now check keys in read only mode.
-  ASSERT_OK(ReadOnlyReopen(options));
-  ASSERT_EQ("v3", Get("foo"));
-  ASSERT_EQ("v2", Get("bar"));
-  ASSERT_TRUE(db_->SyncWAL().IsNotSupported());
-}
-
-TEST_F(DBTest, CompactedDB) {
-  const uint64_t kFileSize = 1 << 20;
-  Options options = CurrentOptions();
-  options.disable_auto_compactions = true;
-  options.write_buffer_size = kFileSize;
-  options.target_file_size_base = kFileSize;
-  options.max_bytes_for_level_base = 1 << 30;
-  options.compression = kNoCompression;
-  Reopen(options);
-  // 1 L0 file, use CompactedDB if max_open_files = -1
-  ASSERT_OK(Put("aaa", DummyString(kFileSize / 2, '1')));
-  Flush();
-  Close();
-  ASSERT_OK(ReadOnlyReopen(options));
-  Status s = Put("new", "value");
-  ASSERT_EQ(s.ToString(),
-            "Not implemented: Not supported operation in read only mode.");
-  ASSERT_EQ(DummyString(kFileSize / 2, '1'), Get("aaa"));
-  Close();
-  options.max_open_files = -1;
-  ASSERT_OK(ReadOnlyReopen(options));
-  s = Put("new", "value");
-  ASSERT_EQ(s.ToString(),
-            "Not implemented: Not supported in compacted db mode.");
-  ASSERT_EQ(DummyString(kFileSize / 2, '1'), Get("aaa"));
-  Close();
-  Reopen(options);
-  // Add more L0 files
-  ASSERT_OK(Put("bbb", DummyString(kFileSize / 2, '2')));
-  Flush();
-  ASSERT_OK(Put("aaa", DummyString(kFileSize / 2, 'a')));
-  Flush();
-  ASSERT_OK(Put("bbb", DummyString(kFileSize / 2, 'b')));
-  ASSERT_OK(Put("eee", DummyString(kFileSize / 2, 'e')));
-  Flush();
-  Close();
-
-  ASSERT_OK(ReadOnlyReopen(options));
-  // Fallback to read-only DB
-  s = Put("new", "value");
-  ASSERT_EQ(s.ToString(),
-            "Not implemented: Not supported operation in read only mode.");
-  Close();
-
-  // Full compaction
-  Reopen(options);
-  // Add more keys
-  ASSERT_OK(Put("fff", DummyString(kFileSize / 2, 'f')));
-  ASSERT_OK(Put("hhh", DummyString(kFileSize / 2, 'h')));
-  ASSERT_OK(Put("iii", DummyString(kFileSize / 2, 'i')));
-  ASSERT_OK(Put("jjj", DummyString(kFileSize / 2, 'j')));
-  db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
-  ASSERT_EQ(3, NumTableFilesAtLevel(1));
-  Close();
-
-  // CompactedDB
-  ASSERT_OK(ReadOnlyReopen(options));
-  s = Put("new", "value");
-  ASSERT_EQ(s.ToString(),
-            "Not implemented: Not supported in compacted db mode.");
-  ASSERT_EQ("NOT_FOUND", Get("abc"));
-  ASSERT_EQ(DummyString(kFileSize / 2, 'a'), Get("aaa"));
-  ASSERT_EQ(DummyString(kFileSize / 2, 'b'), Get("bbb"));
-  ASSERT_EQ("NOT_FOUND", Get("ccc"));
-  ASSERT_EQ(DummyString(kFileSize / 2, 'e'), Get("eee"));
-  ASSERT_EQ(DummyString(kFileSize / 2, 'f'), Get("fff"));
-  ASSERT_EQ("NOT_FOUND", Get("ggg"));
-  ASSERT_EQ(DummyString(kFileSize / 2, 'h'), Get("hhh"));
-  ASSERT_EQ(DummyString(kFileSize / 2, 'i'), Get("iii"));
-  ASSERT_EQ(DummyString(kFileSize / 2, 'j'), Get("jjj"));
-  ASSERT_EQ("NOT_FOUND", Get("kkk"));
-
-  // MultiGet
-  std::vector<std::string> values;
-  std::vector<Status> status_list = dbfull()->MultiGet(
-      ReadOptions(),
-      std::vector<Slice>({Slice("aaa"), Slice("ccc"), Slice("eee"),
-                          Slice("ggg"), Slice("iii"), Slice("kkk")}),
-      &values);
-  ASSERT_EQ(status_list.size(), static_cast<uint64_t>(6));
-  ASSERT_EQ(values.size(), static_cast<uint64_t>(6));
-  ASSERT_OK(status_list[0]);
-  ASSERT_EQ(DummyString(kFileSize / 2, 'a'), values[0]);
-  ASSERT_TRUE(status_list[1].IsNotFound());
-  ASSERT_OK(status_list[2]);
-  ASSERT_EQ(DummyString(kFileSize / 2, 'e'), values[2]);
-  ASSERT_TRUE(status_list[3].IsNotFound());
-  ASSERT_OK(status_list[4]);
-  ASSERT_EQ(DummyString(kFileSize / 2, 'i'), values[4]);
-  ASSERT_TRUE(status_list[5].IsNotFound());
-
-  Reopen(options);
-  // Add a key
-  ASSERT_OK(Put("fff", DummyString(kFileSize / 2, 'f')));
-  Close();
-  ASSERT_OK(ReadOnlyReopen(options));
-  s = Put("new", "value");
-  ASSERT_EQ(s.ToString(),
-            "Not implemented: Not supported operation in read only mode.");
-}
 
 TEST_F(DBTest, LevelLimitReopen) {
   Options options = CurrentOptions();
@@ -430,17 +284,6 @@ TEST_F(DBTest, LevelLimitReopen) {
 }
 #endif  // ROCKSDB_LITE
 
-TEST_F(DBTest, PutDeleteGet) {
-  do {
-    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
-    ASSERT_OK(Put(1, "foo", "v1"));
-    ASSERT_EQ("v1", Get(1, "foo"));
-    ASSERT_OK(Put(1, "foo", "v2"));
-    ASSERT_EQ("v2", Get(1, "foo"));
-    ASSERT_OK(Delete(1, "foo"));
-    ASSERT_EQ("NOT_FOUND", Get(1, "foo"));
-  } while (ChangeOptions());
-}
 
 TEST_F(DBTest, PutSingleDeleteGet) {
   do {
@@ -634,28 +477,6 @@ TEST_F(DBTest, SingleDeletePutFlush) {
                          kSkipUniversalCompaction | kSkipMergePut));
 }
 
-TEST_F(DBTest, EmptyFlush) {
-  // It is possible to produce empty flushes when using single deletes. Tests
-  // whether empty flushes cause issues.
-  do {
-    Random rnd(301);
-
-    Options options = CurrentOptions();
-    options.disable_auto_compactions = true;
-    CreateAndReopenWithCF({"pikachu"}, options);
-
-    Put(1, "a", Slice());
-    SingleDelete(1, "a");
-    ASSERT_OK(Flush(1));
-
-    ASSERT_EQ("[ ]", AllEntriesFor("a", 1));
-    // Skip HashCuckooRep as it does not support single delete. FIFO and
-    // universal compaction do not apply to the test case. Skip MergePut
-    // because merges cannot be combined with single deletions.
-  } while (ChangeOptions(kSkipHashCuckoo | kSkipFIFOCompaction |
-                         kSkipUniversalCompaction | kSkipMergePut));
-}
-
 // Disable because not all platform can run it.
 // It requires more than 9GB memory to run it, With single allocation
 // of more than 3GB.
@@ -730,43 +551,6 @@ TEST_F(DBTest, GetFromImmutableLayer) {
   } while (ChangeOptions());
 }
 
-TEST_F(DBTest, GetFromVersions) {
-  do {
-    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
-    ASSERT_OK(Put(1, "foo", "v1"));
-    ASSERT_OK(Flush(1));
-    ASSERT_EQ("v1", Get(1, "foo"));
-    ASSERT_EQ("NOT_FOUND", Get(0, "foo"));
-  } while (ChangeOptions());
-}
-
-#ifndef ROCKSDB_LITE
-TEST_F(DBTest, GetSnapshot) {
-  anon::OptionsOverride options_override;
-  options_override.skip_policy = kSkipNoSnapshot;
-  do {
-    CreateAndReopenWithCF({"pikachu"}, CurrentOptions(options_override));
-    // Try with both a short key and a long key
-    for (int i = 0; i < 2; i++) {
-      std::string key = (i == 0) ? std::string("foo") : std::string(200, 'x');
-      ASSERT_OK(Put(1, key, "v1"));
-      const Snapshot* s1 = db_->GetSnapshot();
-      if (option_config_ == kHashCuckoo) {
-        // Unsupported case.
-        ASSERT_TRUE(s1 == nullptr);
-        break;
-      }
-      ASSERT_OK(Put(1, key, "v2"));
-      ASSERT_EQ("v2", Get(1, key));
-      ASSERT_EQ("v1", Get(1, key, s1));
-      ASSERT_OK(Flush(1));
-      ASSERT_EQ("v2", Get(1, key));
-      ASSERT_EQ("v1", Get(1, key, s1));
-      db_->ReleaseSnapshot(s1);
-    }
-  } while (ChangeOptions());
-}
-#endif  // ROCKSDB_LITE
 
 TEST_F(DBTest, GetLevel0Ordering) {
   do {
@@ -827,7 +611,6 @@ TEST_F(DBTest, GetPicksCorrectFile) {
 TEST_F(DBTest, GetEncountersEmptyLevel) {
   do {
     Options options = CurrentOptions();
-    options.disableDataSync = true;
     CreateAndReopenWithCF({"pikachu"}, options);
     // Arrange for the following to happen:
     //   * sstable A in level 0
@@ -868,17 +651,6 @@ TEST_F(DBTest, GetEncountersEmptyLevel) {
 }
 #endif  // ROCKSDB_LITE
 
-TEST_F(DBTest, CheckLock) {
-  do {
-    DB* localdb;
-    Options options = CurrentOptions();
-    ASSERT_OK(TryReopen(options));
-
-    // second open should fail
-    ASSERT_TRUE(!(DB::Open(options, dbname_, &localdb)).ok());
-  } while (ChangeCompactOptions());
-}
-
 TEST_F(DBTest, FlushMultipleMemtable) {
   do {
     Options options = CurrentOptions();
@@ -897,97 +669,6 @@ TEST_F(DBTest, FlushMultipleMemtable) {
     ASSERT_OK(Flush(1));
   } while (ChangeCompactOptions());
 }
-
-TEST_F(DBTest, FlushEmptyColumnFamily) {
-  // Block flush thread and disable compaction thread
-  env_->SetBackgroundThreads(1, Env::HIGH);
-  env_->SetBackgroundThreads(1, Env::LOW);
-  test::SleepingBackgroundTask sleeping_task_low;
-  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
-                 Env::Priority::LOW);
-  test::SleepingBackgroundTask sleeping_task_high;
-  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
-                 &sleeping_task_high, Env::Priority::HIGH);
-
-  Options options = CurrentOptions();
-  // disable compaction
-  options.disable_auto_compactions = true;
-  WriteOptions writeOpt = WriteOptions();
-  writeOpt.disableWAL = true;
-  options.max_write_buffer_number = 2;
-  options.min_write_buffer_number_to_merge = 1;
-  options.max_write_buffer_number_to_maintain = 1;
-  CreateAndReopenWithCF({"pikachu"}, options);
-
-  // Compaction can still go through even if no thread can flush the
-  // mem table.
-  ASSERT_OK(Flush(0));
-  ASSERT_OK(Flush(1));
-
-  // Insert can go through
-  ASSERT_OK(dbfull()->Put(writeOpt, handles_[0], "foo", "v1"));
-  ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1"));
-
-  ASSERT_EQ("v1", Get(0, "foo"));
-  ASSERT_EQ("v1", Get(1, "bar"));
-
-  sleeping_task_high.WakeUp();
-  sleeping_task_high.WaitUntilDone();
-
-  // Flush can still go through.
-  ASSERT_OK(Flush(0));
-  ASSERT_OK(Flush(1));
-
-  sleeping_task_low.WakeUp();
-  sleeping_task_low.WaitUntilDone();
-}
-
-TEST_F(DBTest, FLUSH) {
-  do {
-    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
-    WriteOptions writeOpt = WriteOptions();
-    writeOpt.disableWAL = true;
-    SetPerfLevel(kEnableTime);
-    ;
-    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v1"));
-    // this will now also flush the last 2 writes
-    ASSERT_OK(Flush(1));
-    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1"));
-
-    perf_context.Reset();
-    Get(1, "foo");
-    ASSERT_TRUE((int)perf_context.get_from_output_files_time > 0);
-
-    ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
-    ASSERT_EQ("v1", Get(1, "foo"));
-    ASSERT_EQ("v1", Get(1, "bar"));
-
-    writeOpt.disableWAL = true;
-    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v2"));
-    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v2"));
-    ASSERT_OK(Flush(1));
-
-    ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
-    ASSERT_EQ("v2", Get(1, "bar"));
-    perf_context.Reset();
-    ASSERT_EQ("v2", Get(1, "foo"));
-    ASSERT_TRUE((int)perf_context.get_from_output_files_time > 0);
-
-    writeOpt.disableWAL = false;
-    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v3"));
-    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v3"));
-    ASSERT_OK(Flush(1));
-
-    ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
-    // 'foo' should be there because its put
-    // has WAL enabled.
-    ASSERT_EQ("v3", Get(1, "foo"));
-    ASSERT_EQ("v3", Get(1, "bar"));
-
-    SetPerfLevel(kDisable);
-  } while (ChangeCompactOptions());
-}
-
 #ifndef ROCKSDB_LITE
 TEST_F(DBTest, FlushSchedule) {
   Options options = CurrentOptions();
@@ -1031,52 +712,6 @@ TEST_F(DBTest, FlushSchedule) {
 }
 #endif  // ROCKSDB_LITE
 
-TEST_F(DBTest, ManifestRollOver) {
-  do {
-    Options options;
-    options.max_manifest_file_size = 10;  // 10 bytes
-    options = CurrentOptions(options);
-    CreateAndReopenWithCF({"pikachu"}, options);
-    {
-      ASSERT_OK(Put(1, "manifest_key1", std::string(1000, '1')));
-      ASSERT_OK(Put(1, "manifest_key2", std::string(1000, '2')));
-      ASSERT_OK(Put(1, "manifest_key3", std::string(1000, '3')));
-      uint64_t manifest_before_flush = dbfull()->TEST_Current_Manifest_FileNo();
-      ASSERT_OK(Flush(1));  // This should trigger LogAndApply.
-      uint64_t manifest_after_flush = dbfull()->TEST_Current_Manifest_FileNo();
-      ASSERT_GT(manifest_after_flush, manifest_before_flush);
-      ReopenWithColumnFamilies({"default", "pikachu"}, options);
-      ASSERT_GT(dbfull()->TEST_Current_Manifest_FileNo(), manifest_after_flush);
-      // check if a new manifest file got inserted or not.
-      ASSERT_EQ(std::string(1000, '1'), Get(1, "manifest_key1"));
-      ASSERT_EQ(std::string(1000, '2'), Get(1, "manifest_key2"));
-      ASSERT_EQ(std::string(1000, '3'), Get(1, "manifest_key3"));
-    }
-  } while (ChangeCompactOptions());
-}
-
-TEST_F(DBTest, IdentityAcrossRestarts) {
-  do {
-    std::string id1;
-    ASSERT_OK(db_->GetDbIdentity(id1));
-
-    Options options = CurrentOptions();
-    Reopen(options);
-    std::string id2;
-    ASSERT_OK(db_->GetDbIdentity(id2));
-    // id1 should match id2 because identity was not regenerated
-    ASSERT_EQ(id1.compare(id2), 0);
-
-    std::string idfilename = IdentityFileName(dbname_);
-    ASSERT_OK(env_->DeleteFile(idfilename));
-    Reopen(options);
-    std::string id3;
-    ASSERT_OK(db_->GetDbIdentity(id3));
-    // id1 should NOT match id3 because identity was regenerated
-    ASSERT_NE(id1.compare(id3), 0);
-  } while (ChangeCompactOptions());
-}
-
 namespace {
 class KeepFilter : public CompactionFilter {
  public:
@@ -1752,61 +1387,6 @@ TEST_F(DBTest, HiddenValuesAreRemoved) {
 }
 #endif  // ROCKSDB_LITE
 
-TEST_F(DBTest, CompactBetweenSnapshots) {
-  anon::OptionsOverride options_override;
-  options_override.skip_policy = kSkipNoSnapshot;
-  do {
-    Options options = CurrentOptions(options_override);
-    options.disable_auto_compactions = true;
-    CreateAndReopenWithCF({"pikachu"}, options);
-    Random rnd(301);
-    FillLevels("a", "z", 1);
-
-    Put(1, "foo", "first");
-    const Snapshot* snapshot1 = db_->GetSnapshot();
-    Put(1, "foo", "second");
-    Put(1, "foo", "third");
-    Put(1, "foo", "fourth");
-    const Snapshot* snapshot2 = db_->GetSnapshot();
-    Put(1, "foo", "fifth");
-    Put(1, "foo", "sixth");
-
-    // All entries (including duplicates) exist
-    // before any compaction or flush is triggered.
-    ASSERT_EQ(AllEntriesFor("foo", 1),
-              "[ sixth, fifth, fourth, third, second, first ]");
-    ASSERT_EQ("sixth", Get(1, "foo"));
-    ASSERT_EQ("fourth", Get(1, "foo", snapshot2));
-    ASSERT_EQ("first", Get(1, "foo", snapshot1));
-
-    // After a flush, "second", "third" and "fifth" should
-    // be removed
-    ASSERT_OK(Flush(1));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ sixth, fourth, first ]");
-
-    // after we release the snapshot1, only two values left
-    db_->ReleaseSnapshot(snapshot1);
-    FillLevels("a", "z", 1);
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-
-    // We have only one valid snapshot snapshot2. Since snapshot1 is
-    // not valid anymore, "first" should be removed by a compaction.
-    ASSERT_EQ("sixth", Get(1, "foo"));
-    ASSERT_EQ("fourth", Get(1, "foo", snapshot2));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ sixth, fourth ]");
-
-    // after we release the snapshot2, only one value should be left
-    db_->ReleaseSnapshot(snapshot2);
-    FillLevels("a", "z", 1);
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-    ASSERT_EQ("sixth", Get(1, "foo"));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ sixth ]");
-    // skip HashCuckooRep as it does not support snapshot
-  } while (ChangeOptions(kSkipHashCuckoo | kSkipFIFOCompaction));
-}
-
 TEST_F(DBTest, UnremovableSingleDelete) {
   // If we compact:
   //
@@ -2291,121 +1871,6 @@ TEST_F(DBTest, SnapshotFiles) {
 }
 #endif
 
-TEST_F(DBTest, CompactOnFlush) {
-  anon::OptionsOverride options_override;
-  options_override.skip_policy = kSkipNoSnapshot;
-  do {
-    Options options = CurrentOptions(options_override);
-    options.disable_auto_compactions = true;
-    CreateAndReopenWithCF({"pikachu"}, options);
-
-    Put(1, "foo", "v1");
-    ASSERT_OK(Flush(1));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v1 ]");
-
-    // Write two new keys
-    Put(1, "a", "begin");
-    Put(1, "z", "end");
-    Flush(1);
-
-    // Case1: Delete followed by a put
-    Delete(1, "foo");
-    Put(1, "foo", "v2");
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2, DEL, v1 ]");
-
-    // After the current memtable is flushed, the DEL should
-    // have been removed
-    ASSERT_OK(Flush(1));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2, v1 ]");
-
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2 ]");
-
-    // Case 2: Delete followed by another delete
-    Delete(1, "foo");
-    Delete(1, "foo");
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, DEL, v2 ]");
-    ASSERT_OK(Flush(1));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, v2 ]");
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]");
-
-    // Case 3: Put followed by a delete
-    Put(1, "foo", "v3");
-    Delete(1, "foo");
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, v3 ]");
-    ASSERT_OK(Flush(1));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL ]");
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]");
-
-    // Case 4: Put followed by another Put
-    Put(1, "foo", "v4");
-    Put(1, "foo", "v5");
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v5, v4 ]");
-    ASSERT_OK(Flush(1));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v5 ]");
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v5 ]");
-
-    // clear database
-    Delete(1, "foo");
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]");
-
-    // Case 5: Put followed by snapshot followed by another Put
-    // Both puts should remain.
-    Put(1, "foo", "v6");
-    const Snapshot* snapshot = db_->GetSnapshot();
-    Put(1, "foo", "v7");
-    ASSERT_OK(Flush(1));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v7, v6 ]");
-    db_->ReleaseSnapshot(snapshot);
-
-    // clear database
-    Delete(1, "foo");
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]");
-
-    // Case 5: snapshot followed by a put followed by another Put
-    // Only the last put should remain.
-    const Snapshot* snapshot1 = db_->GetSnapshot();
-    Put(1, "foo", "v8");
-    Put(1, "foo", "v9");
-    ASSERT_OK(Flush(1));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v9 ]");
-    db_->ReleaseSnapshot(snapshot1);
-  } while (ChangeCompactOptions());
-}
-
-TEST_F(DBTest, FlushOneColumnFamily) {
-  Options options = CurrentOptions();
-  CreateAndReopenWithCF({"pikachu", "ilya", "muromec", "dobrynia", "nikitich",
-                         "alyosha", "popovich"},
-                        options);
-
-  ASSERT_OK(Put(0, "Default", "Default"));
-  ASSERT_OK(Put(1, "pikachu", "pikachu"));
-  ASSERT_OK(Put(2, "ilya", "ilya"));
-  ASSERT_OK(Put(3, "muromec", "muromec"));
-  ASSERT_OK(Put(4, "dobrynia", "dobrynia"));
-  ASSERT_OK(Put(5, "nikitich", "nikitich"));
-  ASSERT_OK(Put(6, "alyosha", "alyosha"));
-  ASSERT_OK(Put(7, "popovich", "popovich"));
-
-  for (int i = 0; i < 8; ++i) {
-    Flush(i);
-    auto tables = ListTableFiles(env_, dbname_);
-    ASSERT_EQ(tables.size(), i + 1U);
-  }
-}
-
 TEST_F(DBTest, PurgeInfoLogs) {
   Options options = CurrentOptions();
   options.keep_log_file_num = 5;
@@ -2693,7 +2158,7 @@ TEST_F(DBTest, GroupCommitTest) {
     ASSERT_TRUE(!itr->Valid());
     delete itr;
 
-    HistogramData hist_data = {0, 0, 0, 0, 0};
+    HistogramData hist_data;
     options.statistics->histogramData(DB_WRITE, &hist_data);
     ASSERT_GT(hist_data.average, 0.0);
   } while (ChangeOptions(kSkipNoSeekToLast));
@@ -3199,68 +2664,6 @@ TEST_P(DBTestRandomized, Randomized) {
   if (db_snap != nullptr) db_->ReleaseSnapshot(db_snap);
 }
 
-TEST_F(DBTest, MultiGetSimple) {
-  do {
-    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
-    ASSERT_OK(Put(1, "k1", "v1"));
-    ASSERT_OK(Put(1, "k2", "v2"));
-    ASSERT_OK(Put(1, "k3", "v3"));
-    ASSERT_OK(Put(1, "k4", "v4"));
-    ASSERT_OK(Delete(1, "k4"));
-    ASSERT_OK(Put(1, "k5", "v5"));
-    ASSERT_OK(Delete(1, "no_key"));
-
-    std::vector<Slice> keys({"k1", "k2", "k3", "k4", "k5", "no_key"});
-
-    std::vector<std::string> values(20, "Temporary data to be overwritten");
-    std::vector<ColumnFamilyHandle*> cfs(keys.size(), handles_[1]);
-
-    std::vector<Status> s = db_->MultiGet(ReadOptions(), cfs, keys, &values);
-    ASSERT_EQ(values.size(), keys.size());
-    ASSERT_EQ(values[0], "v1");
-    ASSERT_EQ(values[1], "v2");
-    ASSERT_EQ(values[2], "v3");
-    ASSERT_EQ(values[4], "v5");
-
-    ASSERT_OK(s[0]);
-    ASSERT_OK(s[1]);
-    ASSERT_OK(s[2]);
-    ASSERT_TRUE(s[3].IsNotFound());
-    ASSERT_OK(s[4]);
-    ASSERT_TRUE(s[5].IsNotFound());
-  } while (ChangeCompactOptions());
-}
-
-TEST_F(DBTest, MultiGetEmpty) {
-  do {
-    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
-    // Empty Key Set
-    std::vector<Slice> keys;
-    std::vector<std::string> values;
-    std::vector<ColumnFamilyHandle*> cfs;
-    std::vector<Status> s = db_->MultiGet(ReadOptions(), cfs, keys, &values);
-    ASSERT_EQ(s.size(), 0U);
-
-    // Empty Database, Empty Key Set
-    Options options = CurrentOptions();
-    options.create_if_missing = true;
-    DestroyAndReopen(options);
-    CreateAndReopenWithCF({"pikachu"}, options);
-    s = db_->MultiGet(ReadOptions(), cfs, keys, &values);
-    ASSERT_EQ(s.size(), 0U);
-
-    // Empty Database, Search for Keys
-    keys.resize(2);
-    keys[0] = "a";
-    keys[1] = "b";
-    cfs.push_back(handles_[0]);
-    cfs.push_back(handles_[1]);
-    s = db_->MultiGet(ReadOptions(), cfs, keys, &values);
-    ASSERT_EQ(static_cast<int>(s.size()), 2);
-    ASSERT_TRUE(s[0].IsNotFound() && s[1].IsNotFound());
-  } while (ChangeCompactOptions());
-}
-
 TEST_F(DBTest, BlockBasedTablePrefixIndexTest) {
   // create a DB with block prefix index
   BlockBasedTableOptions table_options;
@@ -3364,15 +2767,6 @@ TEST_P(DBTestWithParam, FIFOCompactionTest) {
 }
 #endif  // ROCKSDB_LITE
 
-// verify that we correctly deprecated timeout_hint_us
-TEST_F(DBTest, SimpleWriteTimeoutTest) {
-  WriteOptions write_opt;
-  write_opt.timeout_hint_us = 0;
-  ASSERT_OK(Put(Key(1), Key(1) + std::string(100, 'v'), write_opt));
-  write_opt.timeout_hint_us = 10;
-  ASSERT_NOK(Put(Key(1), Key(1) + std::string(100, 'v'), write_opt));
-}
-
 #ifndef ROCKSDB_LITE
 /*
  * This test is not reliable enough as it heavily depends on disk behavior.
@@ -3387,6 +2781,7 @@ TEST_F(DBTest, RateLimitingTest) {
   options.compression = kNoCompression;
   options.create_if_missing = true;
   options.env = env_;
+  options.statistics = rocksdb::CreateDBStatistics();
   options.IncreaseParallelism(4);
   DestroyAndReopen(options);
 
@@ -3403,6 +2798,9 @@ TEST_F(DBTest, RateLimitingTest) {
   }
   uint64_t elapsed = env_->NowMicros() - start;
   double raw_rate = env_->bytes_written_ * 1000000.0 / elapsed;
+  uint64_t rate_limiter_drains =
+      TestGetTickerCount(options, NUMBER_RATE_LIMITER_DRAINS);
+  ASSERT_EQ(0, rate_limiter_drains);
   Close();
 
   // # rate limiting with 0.7 x threshold
@@ -3417,9 +2815,16 @@ TEST_F(DBTest, RateLimitingTest) {
     ASSERT_OK(
         Put(RandomString(&rnd, 32), RandomString(&rnd, (1 << 10) + 1), wo));
   }
+  rate_limiter_drains =
+      TestGetTickerCount(options, NUMBER_RATE_LIMITER_DRAINS) -
+      rate_limiter_drains;
   elapsed = env_->NowMicros() - start;
   Close();
   ASSERT_EQ(options.rate_limiter->GetTotalBytesThrough(), env_->bytes_written_);
+  // Most intervals should've been drained (interval time is 100ms, elapsed is
+  // micros)
+  ASSERT_GT(rate_limiter_drains, 0);
+  ASSERT_LE(rate_limiter_drains, elapsed / 100000 + 1);
   double ratio = env_->bytes_written_ * 1000000 / elapsed / raw_rate;
   fprintf(stderr, "write rate ratio = %.2lf, expected 0.7\n", ratio);
   ASSERT_TRUE(ratio < 0.8);
@@ -3437,8 +2842,15 @@ TEST_F(DBTest, RateLimitingTest) {
         Put(RandomString(&rnd, 32), RandomString(&rnd, (1 << 10) + 1), wo));
   }
   elapsed = env_->NowMicros() - start;
+  rate_limiter_drains =
+      TestGetTickerCount(options, NUMBER_RATE_LIMITER_DRAINS) -
+      rate_limiter_drains;
   Close();
   ASSERT_EQ(options.rate_limiter->GetTotalBytesThrough(), env_->bytes_written_);
+  // Most intervals should've been drained (interval time is 100ms, elapsed is
+  // micros)
+  ASSERT_GT(rate_limiter_drains, elapsed / 100000 / 2);
+  ASSERT_LE(rate_limiter_drains, elapsed / 100000 + 1);
   ratio = env_->bytes_written_ * 1000000 / elapsed / raw_rate;
   fprintf(stderr, "write rate ratio = %.2lf, expected 0.5\n", ratio);
   ASSERT_LT(ratio, 0.6);
@@ -3467,32 +2879,6 @@ TEST_F(DBTest, TableOptionsSanitizeTest) {
   ASSERT_OK(TryReopen(options));
 }
 
-// On Windows you can have either memory mapped file or a file
-// with unbuffered access. So this asserts and does not make
-// sense to run
-#ifndef OS_WIN
-TEST_F(DBTest, MmapAndBufferOptions) {
-  Options options = CurrentOptions();
-
-  options.use_direct_reads = true;
-  options.allow_mmap_reads = true;
-  ASSERT_NOK(TryReopen(options));
-
-  // All other combinations are acceptable
-  options.use_direct_reads = false;
-  ASSERT_OK(TryReopen(options));
-
-  if (IsDirectIOSupported()) {
-    options.use_direct_reads = true;
-    options.allow_mmap_reads = false;
-    ASSERT_OK(TryReopen(options));
-  }
-
-  options.use_direct_reads = false;
-  ASSERT_OK(TryReopen(options));
-}
-#endif
-
 TEST_F(DBTest, ConcurrentMemtableNotSupported) {
   Options options = CurrentOptions();
   options.allow_concurrent_memtable_write = true;
@@ -3574,31 +2960,6 @@ TEST_F(DBTest, WriteSingleThreadEntry) {
   }
 }
 
-TEST_F(DBTest, DisableDataSyncTest) {
-  env_->sync_counter_.store(0);
-  // iter 0 -- no sync
-  // iter 1 -- sync
-  for (int iter = 0; iter < 2; ++iter) {
-    Options options = CurrentOptions();
-    options.disableDataSync = iter == 0;
-    options.create_if_missing = true;
-    options.num_levels = 10;
-    options.env = env_;
-    Reopen(options);
-    CreateAndReopenWithCF({"pikachu"}, options);
-
-    MakeTables(10, "a", "z");
-    Compact("a", "z");
-
-    if (iter == 0) {
-      ASSERT_EQ(env_->sync_counter_.load(), 0);
-    } else {
-      ASSERT_GT(env_->sync_counter_.load(), 0);
-    }
-    Destroy(options);
-  }
-}
-
 #ifndef ROCKSDB_LITE
 TEST_F(DBTest, DynamicMemtableOptions) {
   const uint64_t k64KB = 1 << 16;
@@ -4727,12 +4088,6 @@ TEST_F(DBTest, DynamicMiscOptions) {
   ASSERT_OK(dbfull()->TEST_GetLatestMutableCFOptions(handles_[1],
                                                      &mutable_cf_options));
   ASSERT_EQ(true, mutable_cf_options.report_bg_io_stats);
-  // Test min_partial_merge_operands
-  ASSERT_OK(
-      dbfull()->SetOptions(handles_[1], {{"min_partial_merge_operands", "4"}}));
-  ASSERT_OK(dbfull()->TEST_GetLatestMutableCFOptions(handles_[1],
-                                                     &mutable_cf_options));
-  ASSERT_EQ(4, mutable_cf_options.min_partial_merge_operands);
   // Test compression
   // sanity check
   ASSERT_OK(dbfull()->SetOptions({{"compression", "kNoCompression"}}));
@@ -5853,7 +5208,7 @@ TEST_F(DBTest, PauseBackgroundWorkTest) {
   });
   env_->SleepForMicroseconds(200000);
   // make sure the thread is not done
-  ASSERT_EQ(false, done.load());
+  ASSERT_FALSE(done.load());
   db_->ContinueBackgroundWork();
   for (auto& t : threads) {
     t.join();

db/db_test2.cc

@@ -2298,6 +2298,70 @@ TEST_F(DBTest2, DirectIO) {
   ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
   Reopen(options);
 }
+
+TEST_F(DBTest2, MemtableOnlyIterator) {
+  Options options = CurrentOptions();
+  CreateAndReopenWithCF({"pikachu"}, options);
+
+  ASSERT_OK(Put(1, "foo", "first"));
+  ASSERT_OK(Put(1, "bar", "second"));
+
+  ReadOptions ropt;
+  ropt.read_tier = kMemtableTier;
+  std::string value;
+  Iterator* it = nullptr;
+
+  // Before flushing
+  // point lookups
+  ASSERT_OK(db_->Get(ropt, handles_[1], "foo", &value));
+  ASSERT_EQ("first", value);
+  ASSERT_OK(db_->Get(ropt, handles_[1], "bar", &value));
+  ASSERT_EQ("second", value);
+
+  // Memtable-only iterator (read_tier=kMemtableTier); data not flushed yet.
+  it = db_->NewIterator(ropt, handles_[1]);
+  int count = 0;
+  for (it->SeekToFirst(); it->Valid(); it->Next()) {
+    ASSERT_TRUE(it->Valid());
+    count++;
+  }
+  ASSERT_TRUE(!it->Valid());
+  ASSERT_EQ(2, count);
+  delete it;
+
+  Flush(1);
+
+  // After flushing
+  // point lookups
+  ASSERT_OK(db_->Get(ropt, handles_[1], "foo", &value));
+  ASSERT_EQ("first", value);
+  ASSERT_OK(db_->Get(ropt, handles_[1], "bar", &value));
+  ASSERT_EQ("second", value);
+  // nothing should be returned using memtable-only iterator after flushing.
+  it = db_->NewIterator(ropt, handles_[1]);
+  count = 0;
+  for (it->SeekToFirst(); it->Valid(); it->Next()) {
+    ASSERT_TRUE(it->Valid());
+    count++;
+  }
+  ASSERT_TRUE(!it->Valid());
+  ASSERT_EQ(0, count);
+  delete it;
+
+  // Add a key to memtable
+  ASSERT_OK(Put(1, "foobar", "third"));
+  it = db_->NewIterator(ropt, handles_[1]);
+  count = 0;
+  for (it->SeekToFirst(); it->Valid(); it->Next()) {
+    ASSERT_TRUE(it->Valid());
+    ASSERT_EQ("foobar", it->key().ToString());
+    ASSERT_EQ("third", it->value().ToString());
+    count++;
+  }
+  ASSERT_TRUE(!it->Valid());
+  ASSERT_EQ(1, count);
+  delete it;
+}
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/db_test_util.cc

@@ -191,6 +191,36 @@ bool DBTestBase::ChangeCompactOptions() {
   }
 }
 
+// Switch between different WAL settings
+bool DBTestBase::ChangeWalOptions() {
+  if (option_config_ == kDefault) {
+    option_config_ = kDBLogDir;
+    Destroy(last_options_);
+    auto options = CurrentOptions();
+    Destroy(options);
+    options.create_if_missing = true;
+    TryReopen(options);
+    return true;
+  } else if (option_config_ == kDBLogDir) {
+    option_config_ = kWalDirAndMmapReads;
+    Destroy(last_options_);
+    auto options = CurrentOptions();
+    Destroy(options);
+    options.create_if_missing = true;
+    TryReopen(options);
+    return true;
+  } else if (option_config_ == kWalDirAndMmapReads) {
+    option_config_ = kRecycleLogFiles;
+    Destroy(last_options_);
+    auto options = CurrentOptions();
+    Destroy(options);
+    TryReopen(options);
+    return true;
+  } else {
+    return false;
+  }
+}
+
 // Switch between different filter policy
 // Jump from kDefault to kFilter to kFullFilter
 bool DBTestBase::ChangeFilterOptions() {
@@ -229,9 +259,6 @@ Options DBTestBase::CurrentOptions(
     const anon::OptionsOverride& options_override) {
   // this redundant copy is to minimize code change w/o having lint error.
   Options options = defaultOptions;
-  XFUNC_TEST("", "dbtest_options", inplace_options1, GetXFTestOptions,
-             reinterpret_cast<Options*>(&options),
-             options_override.skip_policy);
   BlockBasedTableOptions table_options;
   bool set_block_based_table_factory = true;
   switch (option_config_) {
@@ -484,6 +511,13 @@ Status DBTestBase::ReadOnlyReopen(const Options& options) {
 
 Status DBTestBase::TryReopen(const Options& options) {
   Close();
+  last_options_.table_factory.reset();
+  // Note: operator= is an unsafe approach here since it destructs shared_ptr in
+  // the same order of their creation, in contrast to destructors which
+  // destructs them in the opposite order of creation. One particular problme is
+  // that the cache destructor might invoke callback functions that use Option
+  // members such as statistics. To work around this problem, we manually call
+  // destructor of table_facotry which eventually clears the block cache.
   last_options_ = options;
   return DB::Open(options, dbname_, &db_);
 }

db/db_test_util.h

@@ -58,7 +58,6 @@
 #endif  // !(defined NDEBUG) || !defined(OS_WIN)
 #include "util/testharness.h"
 #include "util/testutil.h"
-#include "util/xfunc.h"
 #include "utilities/merge_operators.h"
 
 namespace rocksdb {
@@ -120,6 +119,8 @@ struct OptionsOverride {
 
 }  // namespace anon
 
+enum SkipPolicy { kSkipNone = 0, kSkipNoSnapshot = 1, kSkipNoPrefix = 2 };
+
 // A hacky skip list mem table that triggers flush after number of entries.
 class SpecialMemTableRep : public MemTableRep {
  public:
@@ -670,9 +671,12 @@ class DBTestBase : public testing::Test {
   // test.  Return false if there are no more configurations to test.
   bool ChangeOptions(int skip_mask = kNoSkip);
 
-  // Switch between different compaction styles (we have only 2 now).
+  // Switch between different compaction styles.
   bool ChangeCompactOptions();
 
+  // Switch between different WAL-realted options.
+  bool ChangeWalOptions();
+
   // Switch between different filter policy
   // Jump from kDefault to kFilter to kFullFilter
   bool ChangeFilterOptions();

db/db_universal_compaction_test.cc

@@ -1028,13 +1028,15 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest1) {
 // Test that checks trivial move in universal compaction
 TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest2) {
   int32_t trivial_move = 0;
-  int32_t non_trivial_move = 0;
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
       "DBImpl::BackgroundCompaction:TrivialMove",
       [&](void* arg) { trivial_move++; });
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::BackgroundCompaction:NonTrivial",
-      [&](void* arg) { non_trivial_move++; });
+      "DBImpl::BackgroundCompaction:NonTrivial", [&](void* arg) {
+        ASSERT_TRUE(arg != nullptr);
+        int output_level = *(static_cast<int*>(arg));
+        ASSERT_EQ(output_level, 0);
+      });
 
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 
@@ -1044,7 +1046,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest2) {
   options.num_levels = 15;
   options.write_buffer_size = 100 << 10;  // 100KB
   options.level0_file_num_compaction_trigger = 8;
-  options.max_background_compactions = 1;
+  options.max_background_compactions = 2;
   options.target_file_size_base = 64 * 1024;
   DestroyAndReopen(options);
   CreateAndReopenWithCF({"pikachu"}, options);
@@ -1065,7 +1067,6 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest2) {
   dbfull()->TEST_WaitForCompact();
 
   ASSERT_GT(trivial_move, 0);
-  ASSERT_EQ(non_trivial_move, 0);
 
   rocksdb::SyncPoint::GetInstance()->DisableProcessing();
 }

db/db_wal_test.cc

@@ -51,7 +51,7 @@ TEST_F(DBWALTest, WAL) {
     // again both values should be present.
     ASSERT_EQ("v3", Get(1, "foo"));
     ASSERT_EQ("v3", Get(1, "bar"));
-  } while (ChangeCompactOptions());
+  } while (ChangeWalOptions());
 }
 
 TEST_F(DBWALTest, RollLog) {
@@ -68,7 +68,7 @@ TEST_F(DBWALTest, RollLog) {
     for (int i = 0; i < 10; i++) {
       ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
     }
-  } while (ChangeOptions());
+  } while (ChangeWalOptions());
 }
 
 TEST_F(DBWALTest, SyncWALNotBlockWrite) {
@@ -151,7 +151,7 @@ TEST_F(DBWALTest, Recover) {
     ASSERT_EQ("v4", Get(1, "foo"));
     ASSERT_EQ("v2", Get(1, "bar"));
     ASSERT_EQ("v5", Get(1, "baz"));
-  } while (ChangeOptions());
+  } while (ChangeWalOptions());
 }
 
 TEST_F(DBWALTest, RecoverWithTableHandle) {
@@ -188,23 +188,22 @@ TEST_F(DBWALTest, RecoverWithTableHandle) {
         }
       }
     }
-  } while (ChangeOptions());
+  } while (ChangeWalOptions());
 }
 
 TEST_F(DBWALTest, IgnoreRecoveredLog) {
   std::string backup_logs = dbname_ + "/backup_logs";
 
-  // delete old files in backup_logs directory
-  env_->CreateDirIfMissing(backup_logs);
-  std::vector<std::string> old_files;
-  env_->GetChildren(backup_logs, &old_files);
-  for (auto& file : old_files) {
-    if (file != "." && file != "..") {
-      env_->DeleteFile(backup_logs + "/" + file);
-    }
-  }
-
   do {
+    // delete old files in backup_logs directory
+    env_->CreateDirIfMissing(backup_logs);
+    std::vector<std::string> old_files;
+    env_->GetChildren(backup_logs, &old_files);
+    for (auto& file : old_files) {
+      if (file != "." && file != "..") {
+        env_->DeleteFile(backup_logs + "/" + file);
+      }
+    }
     Options options = CurrentOptions();
     options.create_if_missing = true;
     options.merge_operator = MergeOperators::CreateUInt64AddOperator();
@@ -277,7 +276,8 @@ TEST_F(DBWALTest, IgnoreRecoveredLog) {
     }
     Status s = TryReopen(options);
     ASSERT_TRUE(!s.ok());
-  } while (ChangeOptions(kSkipHashCuckoo));
+    Destroy(options);
+  } while (ChangeWalOptions());
 }
 
 TEST_F(DBWALTest, RecoveryWithEmptyLog) {
@@ -290,7 +290,7 @@ TEST_F(DBWALTest, RecoveryWithEmptyLog) {
     ASSERT_OK(Put(1, "foo", "v3"));
     ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
     ASSERT_EQ("v3", Get(1, "foo"));
-  } while (ChangeOptions());
+  } while (ChangeWalOptions());
 }
 
 #if !(defined NDEBUG) || !defined(OS_WIN)
@@ -429,7 +429,7 @@ TEST_F(DBWALTest, GetSortedWalFiles) {
     ASSERT_OK(Put(1, "foo", "v1"));
     ASSERT_OK(dbfull()->GetSortedWalFiles(log_files));
     ASSERT_EQ(1, log_files.size());
-  } while (ChangeOptions());
+  } while (ChangeWalOptions());
 }
 
 TEST_F(DBWALTest, RecoveryWithLogDataForSomeCFs) {
@@ -454,7 +454,7 @@ TEST_F(DBWALTest, RecoveryWithLogDataForSomeCFs) {
     }
     // Check at least the first WAL was cleaned up during the recovery.
     ASSERT_LT(earliest_log_nums[0], earliest_log_nums[1]);
-  } while (ChangeOptions());
+  } while (ChangeWalOptions());
 }
 
 TEST_F(DBWALTest, RecoverWithLargeLog) {
@@ -481,7 +481,7 @@ TEST_F(DBWALTest, RecoverWithLargeLog) {
     ASSERT_EQ(std::string(10, '3'), Get(1, "small3"));
     ASSERT_EQ(std::string(10, '4'), Get(1, "small4"));
     ASSERT_GT(NumTableFilesAtLevel(0, 1), 1);
-  } while (ChangeCompactOptions());
+  } while (ChangeWalOptions());
 }
 
 // In https://reviews.facebook.net/D20661 we change
@@ -679,9 +679,9 @@ class RecoveryTestHelper {
   // Starting number for the WAL file name like 00010.log
   static const int kWALFileOffset = 10;
   // Keys to be written per WAL file
-  static const int kKeysPerWALFile = 1024;
+  static const int kKeysPerWALFile = 133;
   // Size of the value
-  static const int kValueSize = 10;
+  static const int kValueSize = 96;
 
   // Create WAL files with values filled in
   static void FillData(DBWALTest* test, const Options& options,
@@ -690,7 +690,7 @@ class RecoveryTestHelper {
 
     *count = 0;
 
-    shared_ptr<Cache> table_cache = NewLRUCache(50000, 16);
+    shared_ptr<Cache> table_cache = NewLRUCache(50, 0);
     EnvOptions env_options;
     WriteBufferManager write_buffer_manager(db_options.db_write_buffer_size);
 
@@ -717,12 +717,13 @@ class RecoveryTestHelper {
           new log::Writer(std::move(file_writer), current_log_number,
                           db_options.recycle_log_file_num > 0));
 
+      WriteBatch batch;
       for (int i = 0; i < kKeysPerWALFile; i++) {
         std::string key = "key" + ToString((*count)++);
         std::string value = test->DummyString(kValueSize);
         assert(current_log_writer.get() != nullptr);
         uint64_t seq = versions->LastSequence() + 1;
-        WriteBatch batch;
+        batch.Clear();
         batch.Put(key, value);
         WriteBatchInternal::SetSequence(&batch, seq);
         current_log_writer->AddRecord(WriteBatchInternal::Contents(&batch));
@@ -773,7 +774,7 @@ class RecoveryTestHelper {
     if (trunc) {
       ASSERT_EQ(0, truncate(fname.c_str(), static_cast<int64_t>(size * off)));
     } else {
-      InduceCorruption(fname, static_cast<size_t>(size * off),
+      InduceCorruption(fname, static_cast<size_t>(size * off + 8),
                        static_cast<size_t>(size * len));
     }
   }
@@ -792,7 +793,7 @@ class RecoveryTestHelper {
     ASSERT_EQ(offset, lseek(fd, static_cast<long>(offset), SEEK_SET));
 
     void* buf = alloca(len);
-    memset(buf, 'a', len);
+    memset(buf, 'b', len);
     ASSERT_EQ(len, write(fd, buf, static_cast<unsigned int>(len)));
 
     close(fd);
@@ -808,7 +809,7 @@ TEST_F(DBWALTest, kTolerateCorruptedTailRecords) {
   const int jend = jstart + RecoveryTestHelper::kWALFilesCount;
 
   for (auto trunc : {true, false}) {        /* Corruption style */
-    for (int i = 0; i < 4; i++) {           /* Corruption offset position */
+    for (int i = 0; i < 3; i++) {           /* Corruption offset position */
       for (int j = jstart; j < jend; j++) { /* WAL file */
         // Fill data for testing
         Options options = CurrentOptions();

db/external_sst_file_basic_test.cc

@@ -0,0 +1,323 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+
+#include <functional>
+
+#include "db/db_test_util.h"
+#include "port/port.h"
+#include "port/stack_trace.h"
+#include "rocksdb/sst_file_writer.h"
+#include "util/testutil.h"
+
+namespace rocksdb {
+
+#ifndef ROCKSDB_LITE
+class ExternalSSTFileBasicTest : public DBTestBase {
+ public:
+  ExternalSSTFileBasicTest() : DBTestBase("/external_sst_file_test") {
+    sst_files_dir_ = dbname_ + "/sst_files/";
+    DestroyAndRecreateExternalSSTFilesDir();
+  }
+
+  void DestroyAndRecreateExternalSSTFilesDir() {
+    test::DestroyDir(env_, sst_files_dir_);
+    env_->CreateDir(sst_files_dir_);
+  }
+
+  Status DeprecatedAddFile(const std::vector<std::string>& files,
+                           bool move_files = false,
+                           bool skip_snapshot_check = false) {
+    IngestExternalFileOptions opts;
+    opts.move_files = move_files;
+    opts.snapshot_consistency = !skip_snapshot_check;
+    opts.allow_global_seqno = false;
+    opts.allow_blocking_flush = false;
+    return db_->IngestExternalFile(files, opts);
+  }
+
+  Status GenerateAndAddExternalFile(
+      const Options options, std::vector<int> keys, int file_id,
+      std::map<std::string, std::string>* true_data) {
+    std::string file_path = sst_files_dir_ + ToString(file_id);
+    SstFileWriter sst_file_writer(EnvOptions(), options, options.comparator,
+                                  nullptr);
+
+    Status s = sst_file_writer.Open(file_path);
+    if (!s.ok()) {
+      return s;
+    }
+    for (int k : keys) {
+      std::string key = Key(k);
+      std::string value = Key(k) + ToString(file_id);
+      s = sst_file_writer.Add(key, value);
+      (*true_data)[key] = value;
+      if (!s.ok()) {
+        sst_file_writer.Finish();
+        return s;
+      }
+    }
+    s = sst_file_writer.Finish();
+
+    if (s.ok()) {
+      IngestExternalFileOptions ifo;
+      ifo.allow_global_seqno = true;
+      s = db_->IngestExternalFile({file_path}, ifo);
+    }
+
+    return s;
+  }
+
+  ~ExternalSSTFileBasicTest() { test::DestroyDir(env_, sst_files_dir_); }
+
+ protected:
+  std::string sst_files_dir_;
+};
+
+TEST_F(ExternalSSTFileBasicTest, Basic) {
+  Options options = CurrentOptions();
+
+  SstFileWriter sst_file_writer(EnvOptions(), options, options.comparator);
+
+  // Current file size should be 0 after sst_file_writer init and before open a
+  // file.
+  ASSERT_EQ(sst_file_writer.FileSize(), 0);
+
+  // file1.sst (0 => 99)
+  std::string file1 = sst_files_dir_ + "file1.sst";
+  ASSERT_OK(sst_file_writer.Open(file1));
+  for (int k = 0; k < 100; k++) {
+    ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val"));
+  }
+  ExternalSstFileInfo file1_info;
+  Status s = sst_file_writer.Finish(&file1_info);
+  ASSERT_TRUE(s.ok()) << s.ToString();
+
+  // Current file size should be non-zero after success write.
+  ASSERT_GT(sst_file_writer.FileSize(), 0);
+
+  ASSERT_EQ(file1_info.file_path, file1);
+  ASSERT_EQ(file1_info.num_entries, 100);
+  ASSERT_EQ(file1_info.smallest_key, Key(0));
+  ASSERT_EQ(file1_info.largest_key, Key(99));
+  // sst_file_writer already finished, cannot add this value
+  s = sst_file_writer.Add(Key(100), "bad_val");
+  ASSERT_FALSE(s.ok()) << s.ToString();
+
+  DestroyAndReopen(options);
+  // Add file using file path
+  s = DeprecatedAddFile({file1});
+  ASSERT_TRUE(s.ok()) << s.ToString();
+  ASSERT_EQ(db_->GetLatestSequenceNumber(), 0U);
+  for (int k = 0; k < 100; k++) {
+    ASSERT_EQ(Get(Key(k)), Key(k) + "_val");
+  }
+
+  DestroyAndRecreateExternalSSTFilesDir();
+}
+
+TEST_F(ExternalSSTFileBasicTest, NoCopy) {
+  Options options = CurrentOptions();
+  const ImmutableCFOptions ioptions(options);
+
+  SstFileWriter sst_file_writer(EnvOptions(), options, options.comparator);
+
+  // file1.sst (0 => 99)
+  std::string file1 = sst_files_dir_ + "file1.sst";
+  ASSERT_OK(sst_file_writer.Open(file1));
+  for (int k = 0; k < 100; k++) {
+    ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val"));
+  }
+  ExternalSstFileInfo file1_info;
+  Status s = sst_file_writer.Finish(&file1_info);
+  ASSERT_TRUE(s.ok()) << s.ToString();
+  ASSERT_EQ(file1_info.file_path, file1);
+  ASSERT_EQ(file1_info.num_entries, 100);
+  ASSERT_EQ(file1_info.smallest_key, Key(0));
+  ASSERT_EQ(file1_info.largest_key, Key(99));
+
+  // file2.sst (100 => 299)
+  std::string file2 = sst_files_dir_ + "file2.sst";
+  ASSERT_OK(sst_file_writer.Open(file2));
+  for (int k = 100; k < 300; k++) {
+    ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val"));
+  }
+  ExternalSstFileInfo file2_info;
+  s = sst_file_writer.Finish(&file2_info);
+  ASSERT_TRUE(s.ok()) << s.ToString();
+  ASSERT_EQ(file2_info.file_path, file2);
+  ASSERT_EQ(file2_info.num_entries, 200);
+  ASSERT_EQ(file2_info.smallest_key, Key(100));
+  ASSERT_EQ(file2_info.largest_key, Key(299));
+
+  // file3.sst (110 => 124) .. overlap with file2.sst
+  std::string file3 = sst_files_dir_ + "file3.sst";
+  ASSERT_OK(sst_file_writer.Open(file3));
+  for (int k = 110; k < 125; k++) {
+    ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val_overlap"));
+  }
+  ExternalSstFileInfo file3_info;
+  s = sst_file_writer.Finish(&file3_info);
+  ASSERT_TRUE(s.ok()) << s.ToString();
+  ASSERT_EQ(file3_info.file_path, file3);
+  ASSERT_EQ(file3_info.num_entries, 15);
+  ASSERT_EQ(file3_info.smallest_key, Key(110));
+  ASSERT_EQ(file3_info.largest_key, Key(124));
+  s = DeprecatedAddFile({file1}, true /* move file */);
+  ASSERT_TRUE(s.ok()) << s.ToString();
+  ASSERT_EQ(Status::NotFound(), env_->FileExists(file1));
+
+  s = DeprecatedAddFile({file2}, false /* copy file */);
+  ASSERT_TRUE(s.ok()) << s.ToString();
+  ASSERT_OK(env_->FileExists(file2));
+
+  // This file have overlapping values with the existing data
+  s = DeprecatedAddFile({file2}, true /* move file */);
+  ASSERT_FALSE(s.ok()) << s.ToString();
+  ASSERT_OK(env_->FileExists(file3));
+
+  for (int k = 0; k < 300; k++) {
+    ASSERT_EQ(Get(Key(k)), Key(k) + "_val");
+  }
+}
+
+TEST_F(ExternalSSTFileBasicTest, IngestFileWithGlobalSeqnoPickedSeqno) {
+  Options options = CurrentOptions();
+  DestroyAndReopen(options);
+  std::map<std::string, std::string> true_data;
+
+  int file_id = 1;
+
+  ASSERT_OK(GenerateAndAddExternalFile(options, {1, 2, 3, 4, 5, 6}, file_id++,
+                                       &true_data));
+  // File dont overwrite any keys, No seqno needed
+  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 0);
+
+  ASSERT_OK(GenerateAndAddExternalFile(options, {10, 11, 12, 13}, file_id++,
+                                       &true_data));
+  // File dont overwrite any keys, No seqno needed
+  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 0);
+
+  ASSERT_OK(
+      GenerateAndAddExternalFile(options, {1, 4, 6}, file_id++, &true_data));
+  // File overwrite some keys, a seqno will be assigned
+  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 1);
+
+  ASSERT_OK(
+      GenerateAndAddExternalFile(options, {11, 15, 19}, file_id++, &true_data));
+  // File overwrite some keys, a seqno will be assigned
+  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 2);
+
+  ASSERT_OK(
+      GenerateAndAddExternalFile(options, {120, 130}, file_id++, &true_data));
+  // File dont overwrite any keys, No seqno needed
+  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 2);
+
+  ASSERT_OK(
+      GenerateAndAddExternalFile(options, {1, 130}, file_id++, &true_data));
+  // File overwrite some keys, a seqno will be assigned
+  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 3);
+
+  // Write some keys through normal write path
+  for (int i = 0; i < 50; i++) {
+    ASSERT_OK(Put(Key(i), "memtable"));
+    true_data[Key(i)] = "memtable";
+  }
+  SequenceNumber last_seqno = dbfull()->GetLatestSequenceNumber();
+
+  ASSERT_OK(
+      GenerateAndAddExternalFile(options, {60, 61, 62}, file_id++, &true_data));
+  // File dont overwrite any keys, No seqno needed
+  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno);
+
+  ASSERT_OK(
+      GenerateAndAddExternalFile(options, {40, 41, 42}, file_id++, &true_data));
+  // File overwrite some keys, a seqno will be assigned
+  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 1);
+
+  ASSERT_OK(
+      GenerateAndAddExternalFile(options, {20, 30, 40}, file_id++, &true_data));
+  // File overwrite some keys, a seqno will be assigned
+  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 2);
+
+  const Snapshot* snapshot = db_->GetSnapshot();
+
+  // We will need a seqno for the file regardless if the file overwrite
+  // keys in the DB or not because we have a snapshot
+  ASSERT_OK(
+      GenerateAndAddExternalFile(options, {1000, 1002}, file_id++, &true_data));
+  // A global seqno will be assigned anyway because of the snapshot
+  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 3);
+
+  ASSERT_OK(
+      GenerateAndAddExternalFile(options, {2000, 3002}, file_id++, &true_data));
+  // A global seqno will be assigned anyway because of the snapshot
+  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 4);
+
+  ASSERT_OK(GenerateAndAddExternalFile(options, {1, 20, 40, 100, 150},
+                                       file_id++, &true_data));
+  // A global seqno will be assigned anyway because of the snapshot
+  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 5);
+
+  db_->ReleaseSnapshot(snapshot);
+
+  ASSERT_OK(
+      GenerateAndAddExternalFile(options, {5000, 5001}, file_id++, &true_data));
+  // No snapshot anymore, no need to assign a seqno
+  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 5);
+
+  size_t kcnt = 0;
+  VerifyDBFromMap(true_data, &kcnt, false);
+}
+
+TEST_F(ExternalSSTFileBasicTest, FadviseTrigger) {
+  Options options = CurrentOptions();
+  const int kNumKeys = 10000;
+
+  size_t total_fadvised_bytes = 0;
+  rocksdb::SyncPoint::GetInstance()->SetCallBack(
+      "SstFileWriter::InvalidatePageCache", [&](void* arg) {
+        size_t fadvise_size = *(reinterpret_cast<size_t*>(arg));
+        total_fadvised_bytes += fadvise_size;
+      });
+  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+
+  std::unique_ptr<SstFileWriter> sst_file_writer;
+
+  std::string sst_file_path = sst_files_dir_ + "file_fadvise_disable.sst";
+  sst_file_writer.reset(new SstFileWriter(EnvOptions(), options,
+                                          options.comparator, nullptr, false));
+  ASSERT_OK(sst_file_writer->Open(sst_file_path));
+  for (int i = 0; i < kNumKeys; i++) {
+    ASSERT_OK(sst_file_writer->Add(Key(i), Key(i)));
+  }
+  ASSERT_OK(sst_file_writer->Finish());
+  // fadvise disabled
+  ASSERT_EQ(total_fadvised_bytes, 0);
+
+
+  sst_file_path = sst_files_dir_ + "file_fadvise_enable.sst";
+  sst_file_writer.reset(new SstFileWriter(EnvOptions(), options,
+                                          options.comparator, nullptr, true));
+  ASSERT_OK(sst_file_writer->Open(sst_file_path));
+  for (int i = 0; i < kNumKeys; i++) {
+    ASSERT_OK(sst_file_writer->Add(Key(i), Key(i)));
+  }
+  ASSERT_OK(sst_file_writer->Finish());
+  // fadvise enabled
+  ASSERT_EQ(total_fadvised_bytes, sst_file_writer->FileSize());
+  ASSERT_GT(total_fadvised_bytes, 0);
+
+  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
+}
+
+#endif  // ROCKSDB_LITE
+
+}  // namespace rocksdb
+
+int main(int argc, char** argv) {
+  rocksdb::port::InstallStackTraceHandler();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

db/external_sst_file_ingestion_job.cc

@@ -3,6 +3,8 @@
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
 
+#ifndef ROCKSDB_LITE
+
 #include "db/external_sst_file_ingestion_job.h"
 
 #define __STDC_FORMAT_MACROS
@@ -309,6 +311,7 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo(
   } else if (file_to_ingest->version == 1) {
     // SST file V1 should not have global seqno field
     assert(seqno_iter == uprops.end());
+    file_to_ingest->original_seqno = 0;
     if (ingestion_options_.allow_blocking_flush ||
             ingestion_options_.allow_global_seqno) {
       return Status::InvalidArgument(
@@ -524,3 +527,5 @@ bool ExternalSstFileIngestionJob::IngestedFileFitInLevel(
 }
 
 }  // namespace rocksdb
+
+#endif  // !ROCKSDB_LITE

db/external_sst_file_test.cc

@@ -3,8 +3,9 @@
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
 
-#include <functional>
+#ifndef ROCKSDB_LITE
 
+#include <functional>
 #include "db/db_test_util.h"
 #include "port/port.h"
 #include "port/stack_trace.h"
@@ -13,7 +14,6 @@
 
 namespace rocksdb {
 
-#ifndef ROCKSDB_LITE
 class ExternalSSTFileTest : public DBTestBase {
  public:
   ExternalSSTFileTest() : DBTestBase("/external_sst_file_test") {
@@ -650,71 +650,6 @@ TEST_F(ExternalSSTFileTest, PurgeObsoleteFilesBug) {
   rocksdb::SyncPoint::GetInstance()->DisableProcessing();
 }
 
-TEST_F(ExternalSSTFileTest, NoCopy) {
-  Options options = CurrentOptions();
-  const ImmutableCFOptions ioptions(options);
-
-  SstFileWriter sst_file_writer(EnvOptions(), options, options.comparator);
-
-  // file1.sst (0 => 99)
-  std::string file1 = sst_files_dir_ + "file1.sst";
-  ASSERT_OK(sst_file_writer.Open(file1));
-  for (int k = 0; k < 100; k++) {
-    ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val"));
-  }
-  ExternalSstFileInfo file1_info;
-  Status s = sst_file_writer.Finish(&file1_info);
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  ASSERT_EQ(file1_info.file_path, file1);
-  ASSERT_EQ(file1_info.num_entries, 100);
-  ASSERT_EQ(file1_info.smallest_key, Key(0));
-  ASSERT_EQ(file1_info.largest_key, Key(99));
-
-  // file2.sst (100 => 299)
-  std::string file2 = sst_files_dir_ + "file2.sst";
-  ASSERT_OK(sst_file_writer.Open(file2));
-  for (int k = 100; k < 300; k++) {
-    ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val"));
-  }
-  ExternalSstFileInfo file2_info;
-  s = sst_file_writer.Finish(&file2_info);
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  ASSERT_EQ(file2_info.file_path, file2);
-  ASSERT_EQ(file2_info.num_entries, 200);
-  ASSERT_EQ(file2_info.smallest_key, Key(100));
-  ASSERT_EQ(file2_info.largest_key, Key(299));
-
-  // file3.sst (110 => 124) .. overlap with file2.sst
-  std::string file3 = sst_files_dir_ + "file3.sst";
-  ASSERT_OK(sst_file_writer.Open(file3));
-  for (int k = 110; k < 125; k++) {
-    ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val_overlap"));
-  }
-  ExternalSstFileInfo file3_info;
-  s = sst_file_writer.Finish(&file3_info);
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  ASSERT_EQ(file3_info.file_path, file3);
-  ASSERT_EQ(file3_info.num_entries, 15);
-  ASSERT_EQ(file3_info.smallest_key, Key(110));
-  ASSERT_EQ(file3_info.largest_key, Key(124));
-  s = DeprecatedAddFile({file1}, true /* move file */);
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  ASSERT_EQ(Status::NotFound(), env_->FileExists(file1));
-
-  s = DeprecatedAddFile({file2}, false /* copy file */);
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  ASSERT_OK(env_->FileExists(file2));
-
-  // This file have overlapping values with the existing data
-  s = DeprecatedAddFile({file2}, true /* move file */);
-  ASSERT_FALSE(s.ok()) << s.ToString();
-  ASSERT_OK(env_->FileExists(file3));
-
-  for (int k = 0; k < 300; k++) {
-    ASSERT_EQ(Get(Key(k)), Key(k) + "_val");
-  }
-}
-
 TEST_F(ExternalSSTFileTest, SkipSnapshot) {
   Options options = CurrentOptions();
 
@@ -1537,93 +1472,6 @@ TEST_F(ExternalSSTFileTest, IngestFileWithGlobalSeqnoAssignedLevel) {
   VerifyDBFromMap(true_data, &kcnt, false);
 }
 
-TEST_F(ExternalSSTFileTest, IngestFileWithGlobalSeqnoPickedSeqno) {
-  Options options = CurrentOptions();
-  DestroyAndReopen(options);
-  std::map<std::string, std::string> true_data;
-
-  ASSERT_OK(GenerateAndAddExternalFile(options, {1, 2, 3, 4, 5, 6}, -1, true,
-                                       false, &true_data));
-  // File dont overwrite any keys, No seqno needed
-  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 0);
-
-  ASSERT_OK(GenerateAndAddExternalFile(options, {10, 11, 12, 13}, -1, true,
-                                       false, &true_data));
-  // File dont overwrite any keys, No seqno needed
-  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 0);
-
-  ASSERT_OK(GenerateAndAddExternalFile(options, {1, 4, 6}, -1, true, false,
-                                       &true_data));
-  // File overwrite some keys, a seqno will be assigned
-  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 1);
-
-  ASSERT_OK(GenerateAndAddExternalFile(options, {11, 15, 19}, -1, true, false,
-                                       &true_data));
-  // File overwrite some keys, a seqno will be assigned
-  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 2);
-
-  ASSERT_OK(GenerateAndAddExternalFile(options, {120, 130}, -1, true, false,
-                                       &true_data));
-  // File dont overwrite any keys, No seqno needed
-  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 2);
-
-  ASSERT_OK(GenerateAndAddExternalFile(options, {1, 130}, -1, true, false,
-                                       &true_data));
-  // File overwrite some keys, a seqno will be assigned
-  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 3);
-
-  // Write some keys through normal write path
-  for (int i = 0; i < 50; i++) {
-    ASSERT_OK(Put(Key(i), "memtable"));
-    true_data[Key(i)] = "memtable";
-  }
-  SequenceNumber last_seqno = dbfull()->GetLatestSequenceNumber();
-
-  ASSERT_OK(GenerateAndAddExternalFile(options, {60, 61, 62}, -1, true, false,
-                                       &true_data));
-  // File dont overwrite any keys, No seqno needed
-  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno);
-
-  ASSERT_OK(GenerateAndAddExternalFile(options, {40, 41, 42}, -1, true, false,
-                                       &true_data));
-  // File overwrite some keys, a seqno will be assigned
-  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 1);
-
-  ASSERT_OK(GenerateAndAddExternalFile(options, {20, 30, 40}, -1, true, false,
-                                       &true_data));
-  // File overwrite some keys, a seqno will be assigned
-  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 2);
-
-  const Snapshot* snapshot = db_->GetSnapshot();
-
-  // We will need a seqno for the file regardless if the file overwrite
-  // keys in the DB or not because we have a snapshot
-  ASSERT_OK(GenerateAndAddExternalFile(options, {1000, 1002}, -1, true, false,
-                                       &true_data));
-  // A global seqno will be assigned anyway because of the snapshot
-  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 3);
-
-  ASSERT_OK(GenerateAndAddExternalFile(options, {2000, 3002}, -1, true, false,
-                                       &true_data));
-  // A global seqno will be assigned anyway because of the snapshot
-  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 4);
-
-  ASSERT_OK(GenerateAndAddExternalFile(options, {1, 20, 40, 100, 150}, -1, true,
-                                       false, &true_data));
-  // A global seqno will be assigned anyway because of the snapshot
-  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 5);
-
-  db_->ReleaseSnapshot(snapshot);
-
-  ASSERT_OK(GenerateAndAddExternalFile(options, {5000, 5001}, -1, true, false,
-                                       &true_data));
-  // No snapshot anymore, no need to assign a seqno
-  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 5);
-
-  size_t kcnt = 0;
-  VerifyDBFromMap(true_data, &kcnt, false);
-}
-
 TEST_F(ExternalSSTFileTest, IngestFileWithGlobalSeqnoMemtableFlush) {
   Options options = CurrentOptions();
   DestroyAndReopen(options);
@@ -1944,50 +1792,6 @@ TEST_F(ExternalSSTFileTest, SnapshotInconsistencyBug) {
 
   db_->ReleaseSnapshot(snap);
 }
-
-TEST_F(ExternalSSTFileTest, FadviseTrigger) {
-  Options options = CurrentOptions();
-  const int kNumKeys = 10000;
-
-  size_t total_fadvised_bytes = 0;
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "SstFileWriter::InvalidatePageCache", [&](void* arg) {
-        size_t fadvise_size = *(reinterpret_cast<size_t*>(arg));
-        total_fadvised_bytes += fadvise_size;
-      });
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-
-  std::unique_ptr<SstFileWriter> sst_file_writer;
-
-  std::string sst_file_path = sst_files_dir_ + "file_fadvise_disable.sst";
-  sst_file_writer.reset(new SstFileWriter(EnvOptions(), options,
-                                          options.comparator, nullptr, false));
-  ASSERT_OK(sst_file_writer->Open(sst_file_path));
-  for (int i = 0; i < kNumKeys; i++) {
-    ASSERT_OK(sst_file_writer->Add(Key(i), Key(i)));
-  }
-  ASSERT_OK(sst_file_writer->Finish());
-  // fadvise disabled
-  ASSERT_EQ(total_fadvised_bytes, 0);
-
-
-  sst_file_path = sst_files_dir_ + "file_fadvise_enable.sst";
-  sst_file_writer.reset(new SstFileWriter(EnvOptions(), options,
-                                          options.comparator, nullptr, true));
-  ASSERT_OK(sst_file_writer->Open(sst_file_path));
-  for (int i = 0; i < kNumKeys; i++) {
-    ASSERT_OK(sst_file_writer->Add(Key(i), Key(i)));
-  }
-  ASSERT_OK(sst_file_writer->Finish());
-  // fadvise enabled
-  ASSERT_EQ(total_fadvised_bytes, sst_file_writer->FileSize());
-  ASSERT_GT(total_fadvised_bytes, 0);
-
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-}
-
-#endif  // ROCKSDB_LITE
-
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {
@@ -1995,3 +1799,15 @@ int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }
+
+#else
+#include <stdio.h>
+
+int main(int argc, char** argv) {
+  fprintf(stderr,
+          "SKIPPED as External SST File Writer and Ingestion are not supported "
+          "in ROCKSDB_LITE\n");
+  return 0;
+}
+
+#endif  // !ROCKSDB_LITE

db/filename.cc

@@ -389,12 +389,8 @@ Status SetIdentityFile(Env* env, const std::string& dbname) {
 Status SyncManifest(Env* env, const ImmutableDBOptions* db_options,
                     WritableFileWriter* file) {
   TEST_KILL_RANDOM("SyncManifest:0", rocksdb_kill_odds * REDUCE_ODDS2);
-  if (db_options->disable_data_sync) {
-    return Status::OK();
-  } else {
-    StopWatch sw(env, db_options->statistics.get(), MANIFEST_FILE_SYNC_MICROS);
-    return file->Sync(db_options->use_fsync);
-  }
+  StopWatch sw(env, db_options->statistics.get(), MANIFEST_FILE_SYNC_MICROS);
+  return file->Sync(db_options->use_fsync);
 }
 
 }  // namespace rocksdb

db/flush_job.cc

@@ -313,7 +313,7 @@ Status FlushJob::WriteLevel0Table() {
         meta_.fd.GetFileSize(), s.ToString().c_str(),
         meta_.marked_for_compaction ? " (needs compaction)" : "");
 
-    if (!db_options_.disable_data_sync && output_file_directory_ != nullptr) {
+    if (output_file_directory_ != nullptr) {
       output_file_directory_->Fsync();
     }
     TEST_SYNC_POINT("FlushJob::WriteLevel0Table");

db/internal_stats.cc

@@ -232,6 +232,9 @@ static const std::string aggregated_table_properties_at_level =
     aggregated_table_properties + "-at-level";
 static const std::string num_running_compactions = "num-running-compactions";
 static const std::string num_running_flushes = "num-running-flushes";
+static const std::string actual_delayed_write_rate =
+    "actual-delayed-write-rate";
+static const std::string is_write_stopped = "is-write-stopped";
 
 const std::string DB::Properties::kNumFilesAtLevelPrefix =
                       rocksdb_prefix + num_files_at_level_prefix;
@@ -297,6 +300,10 @@ const std::string DB::Properties::kAggregatedTableProperties =
     rocksdb_prefix + aggregated_table_properties;
 const std::string DB::Properties::kAggregatedTablePropertiesAtLevel =
     rocksdb_prefix + aggregated_table_properties_at_level;
+const std::string DB::Properties::kActualDelayedWriteRate =
+    rocksdb_prefix + actual_delayed_write_rate;
+const std::string DB::Properties::kIsWriteStopped =
+    rocksdb_prefix + is_write_stopped;
 
 const std::unordered_map<std::string, DBPropertyInfo>
     InternalStats::ppt_name_to_info = {
@@ -385,6 +392,11 @@ const std::unordered_map<std::string, DBPropertyInfo>
         {DB::Properties::kNumRunningCompactions,
          {false, nullptr, &InternalStats::HandleNumRunningCompactions,
           nullptr}},
+        {DB::Properties::kActualDelayedWriteRate,
+         {false, nullptr, &InternalStats::HandleActualDelayedWriteRate,
+          nullptr}},
+        {DB::Properties::kIsWriteStopped,
+         {false, nullptr, &InternalStats::HandleIsWriteStopped, nullptr}},
 };
 
 const DBPropertyInfo* GetPropertyInfo(const Slice& property) {
@@ -716,6 +728,23 @@ bool InternalStats::HandleMinLogNumberToKeep(uint64_t* value, DBImpl* db,
   return true;
 }
 
+bool InternalStats::HandleActualDelayedWriteRate(uint64_t* value, DBImpl* db,
+                                                 Version* version) {
+  const WriteController& wc = db->write_controller();
+  if (!wc.NeedsDelay()) {
+    *value = 0;
+  } else {
+    *value = wc.delayed_write_rate();
+  }
+  return true;
+}
+
+bool InternalStats::HandleIsWriteStopped(uint64_t* value, DBImpl* db,
+                                         Version* version) {
+  *value = db->write_controller().IsStopped() ? 1 : 0;
+  return true;
+}
+
 void InternalStats::DumpDBStats(std::string* value) {
   char buf[1000];
   // DB-level stats, only available from default column family

db/internal_stats.h

@@ -402,6 +402,9 @@ class InternalStats {
   bool HandleEstimateLiveDataSize(uint64_t* value, DBImpl* db,
                                   Version* version);
   bool HandleMinLogNumberToKeep(uint64_t* value, DBImpl* db, Version* version);
+  bool HandleActualDelayedWriteRate(uint64_t* value, DBImpl* db,
+                                    Version* version);
+  bool HandleIsWriteStopped(uint64_t* value, DBImpl* db, Version* version);
 
   // Total number of background errors encountered. Every time a flush task
   // or compaction task fails, this counter is incremented. The failure can

db/managed_iterator.cc

@@ -15,12 +15,10 @@
 #include "db/db_impl.h"
 #include "db/db_iter.h"
 #include "db/dbformat.h"
-#include "db/xfunc_test_points.h"
 #include "rocksdb/env.h"
 #include "rocksdb/slice.h"
 #include "rocksdb/slice_transform.h"
 #include "table/merging_iterator.h"
-#include "util/xfunc.h"
 
 namespace rocksdb {
 
@@ -42,8 +40,6 @@ class MILock {
   }
   ~MILock() {
     this->mu_->unlock();
-    XFUNC_TEST("managed_xftest_release", "managed_unlock", managed_unlock1,
-               xf_manage_release, mi_);
   }
   ManagedIterator* GetManagedIterator() { return mi_; }
 
@@ -84,8 +80,6 @@ ManagedIterator::ManagedIterator(DBImpl* db, const ReadOptions& read_options,
   }
   cfh_.SetCFD(cfd);
   mutable_iter_ = unique_ptr<Iterator>(db->NewIterator(read_options_, &cfh_));
-  XFUNC_TEST("managed_xftest_dropold", "managed_create", xf_managed_create1,
-             xf_manage_create, this);
 }
 
 ManagedIterator::~ManagedIterator() {
@@ -261,8 +255,6 @@ bool ManagedIterator::TryLock() { return in_use_.try_lock(); }
 
 void ManagedIterator::UnLock() {
   in_use_.unlock();
-  XFUNC_TEST("managed_xftest_release", "managed_unlock", managed_unlock1,
-             xf_manage_release, this);
 }
 
 }  // namespace rocksdb

db/merge_helper.cc

@@ -283,8 +283,7 @@ Status MergeHelper::MergeUntil(InternalIterator* iter,
     // Attempt to use the user's associative merge function to
     // merge the stacked merge operands into a single operand.
     s = Status::MergeInProgress();
-    if (merge_context_.GetNumOperands() >= 2 &&
-        merge_context_.GetNumOperands() >= min_partial_merge_operands_) {
+    if (merge_context_.GetNumOperands() >= 2) {
       bool merge_success = false;
       std::string merge_result;
       {

db/merge_helper.h

@@ -32,7 +32,6 @@ class MergeHelper {
   MergeHelper(Env* env, const Comparator* user_comparator,
               const MergeOperator* user_merge_operator,
               const CompactionFilter* compaction_filter, Logger* logger,
-              unsigned min_partial_merge_operands,
               bool assert_valid_internal_key, SequenceNumber latest_snapshot,
               int level = 0, Statistics* stats = nullptr,
               const std::atomic<bool>* shutting_down = nullptr)
@@ -42,7 +41,6 @@ class MergeHelper {
         compaction_filter_(compaction_filter),
         shutting_down_(shutting_down),
         logger_(logger),
-        min_partial_merge_operands_(min_partial_merge_operands),
         assert_valid_internal_key_(assert_valid_internal_key),
         latest_snapshot_(latest_snapshot),
         level_(level),
@@ -156,7 +154,6 @@ class MergeHelper {
   const CompactionFilter* compaction_filter_;
   const std::atomic<bool>* shutting_down_;
   Logger* logger_;
-  unsigned min_partial_merge_operands_;
   bool assert_valid_internal_key_; // enforce no internal key corruption?
   SequenceNumber latest_snapshot_;
   int level_;

db/merge_helper_test.cc

@@ -28,7 +28,7 @@ class MergeHelperTest : public testing::Test {
     iter_->SeekToFirst();
     merge_helper_.reset(new MergeHelper(env_, BytewiseComparator(),
                                         merge_op_.get(), filter_.get(), nullptr,
-                                        2U, false, latest_snapshot));
+                                        false, latest_snapshot));
     return merge_helper_->MergeUntil(iter_.get(), nullptr /* range_del_agg */,
                                      stop_before, at_bottom);
   }

db/merge_test.cc

@@ -76,14 +76,12 @@ class CountMergeOperator : public AssociativeMergeOperator {
 
 namespace {
 std::shared_ptr<DB> OpenDb(const std::string& dbname, const bool ttl = false,
-                           const size_t max_successive_merges = 0,
-                           const uint32_t min_partial_merge_operands = 2) {
+                           const size_t max_successive_merges = 0) {
   DB* db;
   Options options;
   options.create_if_missing = true;
   options.merge_operator = std::make_shared<CountMergeOperator>();
   options.max_successive_merges = max_successive_merges;
-  options.min_partial_merge_operands = min_partial_merge_operands;
   Status s;
   DestroyDB(dbname, Options());
 // DBWithTTL is not supported in ROCKSDB_LITE
@@ -448,20 +446,20 @@ void runTest(int argc, const std::string& dbname, const bool use_ttl = false) {
   {
     std::cout << "Test Partial-Merge\n";
     size_t max_merge = 100;
-    for (uint32_t min_merge = 5; min_merge < 25; min_merge += 5) {
-      for (uint32_t count = min_merge - 1; count <= min_merge + 1; count++) {
-        auto db = OpenDb(dbname, use_ttl, max_merge, min_merge);
-        MergeBasedCounters counters(db, 0);
-        testPartialMerge(&counters, db.get(), max_merge, min_merge, count);
-        DestroyDB(dbname, Options());
-      }
-      {
-        auto db = OpenDb(dbname, use_ttl, max_merge, min_merge);
-        MergeBasedCounters counters(db, 0);
-        testPartialMerge(&counters, db.get(), max_merge, min_merge,
-                         min_merge * 10);
-        DestroyDB(dbname, Options());
-      }
+    // Min merge is hard-coded to 2.
+    uint32_t min_merge = 2;
+    for (uint32_t count = min_merge - 1; count <= min_merge + 1; count++) {
+      auto db = OpenDb(dbname, use_ttl, max_merge);
+      MergeBasedCounters counters(db, 0);
+      testPartialMerge(&counters, db.get(), max_merge, min_merge, count);
+      DestroyDB(dbname, Options());
+    }
+    {
+      auto db = OpenDb(dbname, use_ttl, max_merge);
+      MergeBasedCounters counters(db, 0);
+      testPartialMerge(&counters, db.get(), max_merge, min_merge,
+                       min_merge * 10);
+      DestroyDB(dbname, Options());
     }
   }
 

db/repair.cc

@@ -486,6 +486,7 @@ class Repairer {
         if (empty) {
           empty = false;
           t->meta.smallest.DecodeFrom(key);
+          t->min_sequence = parsed.sequence;
         }
         t->meta.largest.DecodeFrom(key);
         if (parsed.sequence < t->min_sequence) {

db/repair_test.cc

@@ -3,6 +3,8 @@
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
 
+#ifndef ROCKSDB_LITE
+
 #include <algorithm>
 #include <string>
 #include <vector>
@@ -17,6 +19,7 @@
 
 namespace rocksdb {
 
+#ifndef ROCKSDB_LITE
 class RepairTest : public DBTestBase {
  public:
   RepairTest() : DBTestBase("/repair_test") {}
@@ -253,7 +256,10 @@ TEST_F(RepairTest, RepairColumnFamilyOptions) {
   db_->GetPropertiesOfAllTables(handles_[1], &fname_to_props);
   ASSERT_EQ(fname_to_props.size(), 2U);
   for (const auto& fname_and_props : fname_to_props) {
-    ASSERT_EQ(InternalKeyComparator(rev_opts.comparator).Name(),
+    std::string comparator_name (
+      InternalKeyComparator(rev_opts.comparator).Name());
+    comparator_name = comparator_name.substr(comparator_name.find(':') + 1);
+    ASSERT_EQ(comparator_name,
               fname_and_props.second->comparator_name);
   }
 
@@ -268,9 +274,21 @@ TEST_F(RepairTest, RepairColumnFamilyOptions) {
     }
   }
 }
+
+#endif  // ROCKSDB_LITE
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }
+
+#else
+#include <stdio.h>
+
+int main(int argc, char** argv) {
+  fprintf(stderr, "SKIPPED as RepairDB is not supported in ROCKSDB_LITE\n");
+  return 0;
+}
+
+#endif  // ROCKSDB_LITE

db/version_set.cc

@@ -22,7 +22,6 @@
 #include <unordered_map>
 #include <vector>
 #include <string>
-
 #include "db/compaction.h"
 #include "db/filename.h"
 #include "db/internal_stats.h"
@@ -1640,10 +1639,11 @@ void VersionStorageInfo::GetOverlappingInputs(
   }
   const Comparator* user_cmp = user_comparator_;
   if (begin != nullptr && end != nullptr && level > 0) {
-    GetOverlappingInputsBinarySearch(level, user_begin, user_end, inputs,
-      hint_index, file_index);
+    GetOverlappingInputsRangeBinarySearch(level, user_begin, user_end, inputs,
+                                          hint_index, file_index);
     return;
   }
+
   for (size_t i = 0; i < level_files_brief_[level].num_files; ) {
     FdWithKeyRange* f = &(level_files_brief_[level].files[i++]);
     const Slice file_start = ExtractUserKey(f->smallest_key);
@@ -1674,13 +1674,49 @@ void VersionStorageInfo::GetOverlappingInputs(
   }
 }
 
+// Store in "*inputs" files in "level" that within range [begin,end]
+// Guarantee a "clean cut" boundary between the files in inputs
+// and the surrounding files and the maxinum number of files.
+// This will ensure that no parts of a key are lost during compaction.
+// If hint_index is specified, then it points to a file in the range.
+// The file_index returns a pointer to any file in an overlapping range.
+void VersionStorageInfo::GetCleanInputsWithinInterval(
+    int level, const InternalKey* begin, const InternalKey* end,
+    std::vector<FileMetaData*>* inputs, int hint_index, int* file_index) const {
+  if (level >= num_non_empty_levels_) {
+    // this level is empty, no inputs within range
+    return;
+  }
+
+  inputs->clear();
+  Slice user_begin, user_end;
+  if (begin != nullptr) {
+    user_begin = begin->user_key();
+  }
+  if (end != nullptr) {
+    user_end = end->user_key();
+  }
+  if (file_index) {
+    *file_index = -1;
+  }
+  if (begin != nullptr && end != nullptr && level > 0) {
+    GetOverlappingInputsRangeBinarySearch(level, user_begin, user_end, inputs,
+                                          hint_index, file_index,
+                                          true /* within_interval */);
+  }
+}
+
 // Store in "*inputs" all files in "level" that overlap [begin,end]
 // Employ binary search to find at least one file that overlaps the
 // specified range. From that file, iterate backwards and
 // forwards to find all overlapping files.
-void VersionStorageInfo::GetOverlappingInputsBinarySearch(
+// if within_range is set, then only store the maximum clean inputs
+// within range [begin, end]. "clean" means there is a boudnary
+// between the files in "*inputs" and the surrounding files
+void VersionStorageInfo::GetOverlappingInputsRangeBinarySearch(
     int level, const Slice& user_begin, const Slice& user_end,
-    std::vector<FileMetaData*>* inputs, int hint_index, int* file_index) const {
+    std::vector<FileMetaData*>* inputs, int hint_index, int* file_index,
+    bool within_interval) const {
   assert(level > 0);
   int min = 0;
   int mid = 0;
@@ -1700,9 +1736,13 @@ void VersionStorageInfo::GetOverlappingInputsBinarySearch(
     FdWithKeyRange* f = &(level_files_brief_[level].files[mid]);
     const Slice file_start = ExtractUserKey(f->smallest_key);
     const Slice file_limit = ExtractUserKey(f->largest_key);
-    if (user_cmp->Compare(file_limit, user_begin) < 0) {
+    if ((!within_interval && user_cmp->Compare(file_limit, user_begin) < 0) ||
+        (within_interval && user_cmp->Compare(file_start, user_begin) < 0)) {
       min = mid + 1;
-    } else if (user_cmp->Compare(user_end, file_start) < 0) {
+    } else if ((!within_interval &&
+                user_cmp->Compare(user_end, file_start) < 0) ||
+               (within_interval &&
+                user_cmp->Compare(user_end, file_limit) < 0)) {
       max = mid - 1;
     } else {
       foundOverlap = true;
@@ -1718,24 +1758,38 @@ void VersionStorageInfo::GetOverlappingInputsBinarySearch(
   if (file_index) {
     *file_index = mid;
   }
-  ExtendOverlappingInputs(level, user_begin, user_end, inputs, mid);
+
+  int start_index, end_index;
+  if (within_interval) {
+    ExtendFileRangeWithinInterval(level, user_begin, user_end, mid, &start_index,
+                                  &end_index);
+  } else {
+    ExtendFileRangeOverlappingInterval(level, user_begin, user_end, mid,
+                                       &start_index, &end_index);
+  }
+  assert(end_index >= start_index);
+  // insert overlapping files into vector
+  for (int i = start_index; i <= end_index; i++) {
+    inputs->push_back(files_[level][i]);
+  }
 }
 
-// Store in "*inputs" all files in "level" that overlap [begin,end]
-// The midIndex specifies the index of at least one file that
+// Store in *start_index and *end_index the range of all files in
+// "level" that overlap [begin,end]
+// The mid_index specifies the index of at least one file that
 // overlaps the specified range. From that file, iterate backward
 // and forward to find all overlapping files.
 // Use FileLevel in searching, make it faster
-void VersionStorageInfo::ExtendOverlappingInputs(
+void VersionStorageInfo::ExtendFileRangeOverlappingInterval(
     int level, const Slice& user_begin, const Slice& user_end,
-    std::vector<FileMetaData*>* inputs, unsigned int midIndex) const {
+    unsigned int mid_index, int* start_index, int* end_index) const {
   const Comparator* user_cmp = user_comparator_;
   const FdWithKeyRange* files = level_files_brief_[level].files;
 #ifndef NDEBUG
   {
-    // assert that the file at midIndex overlaps with the range
-    assert(midIndex < level_files_brief_[level].num_files);
-    const FdWithKeyRange* f = &files[midIndex];
+    // assert that the file at mid_index overlaps with the range
+    assert(mid_index < level_files_brief_[level].num_files);
+    const FdWithKeyRange* f = &files[mid_index];
     const Slice fstart = ExtractUserKey(f->smallest_key);
     const Slice flimit = ExtractUserKey(f->largest_key);
     if (user_cmp->Compare(fstart, user_begin) >= 0) {
@@ -1745,91 +1799,105 @@ void VersionStorageInfo::ExtendOverlappingInputs(
     }
   }
 #endif
-  int startIndex = midIndex + 1;
-  int endIndex = midIndex;
+  *start_index = mid_index + 1;
+  *end_index = mid_index;
   int count __attribute__((unused)) = 0;
 
   // check backwards from 'mid' to lower indices
-  for (int i = midIndex; i >= 0 ; i--) {
+  for (int i = mid_index; i >= 0 ; i--) {
     const FdWithKeyRange* f = &files[i];
     const Slice file_limit = ExtractUserKey(f->largest_key);
     if (user_cmp->Compare(file_limit, user_begin) >= 0) {
-      startIndex = i;
+      *start_index = i;
       assert((count++, true));
     } else {
       break;
     }
   }
   // check forward from 'mid+1' to higher indices
-  for (unsigned int i = midIndex+1;
+  for (unsigned int i = mid_index+1;
        i < level_files_brief_[level].num_files; i++) {
     const FdWithKeyRange* f = &files[i];
     const Slice file_start = ExtractUserKey(f->smallest_key);
     if (user_cmp->Compare(file_start, user_end) <= 0) {
       assert((count++, true));
-      endIndex = i;
+      *end_index = i;
     } else {
       break;
     }
   }
-  assert(count == endIndex - startIndex + 1);
-
-  // insert overlapping files into vector
-  for (int i = startIndex; i <= endIndex; i++) {
-    FileMetaData* f = files_[level][i];
-    inputs->push_back(f);
-  }
+  assert(count == *end_index - *start_index + 1);
 }
 
-// Returns true iff the first or last file in inputs contains
-// an overlapping user key to the file "just outside" of it (i.e.
-// just after the last file, or just before the first file)
-// REQUIRES: "*inputs" is a sorted list of non-overlapping files
-bool VersionStorageInfo::HasOverlappingUserKey(
-    const std::vector<FileMetaData*>* inputs, int level) {
-
-  // If inputs empty, there is no overlap.
-  // If level == 0, it is assumed that all needed files were already included.
-  if (inputs->empty() || level == 0){
-    return false;
-  }
-
+// Store in *start_index and *end_index the clean range of all files in
+// "level" within [begin,end]
+// The mid_index specifies the index of at least one file within
+// the specified range. From that file, iterate backward
+// and forward to find all overlapping files and then "shrink" to
+// the clean range required.
+// Use FileLevel in searching, make it faster
+void VersionStorageInfo::ExtendFileRangeWithinInterval(
+    int level, const Slice& user_begin, const Slice& user_end,
+    unsigned int mid_index, int* start_index, int* end_index) const {
+  assert(level != 0);
   const Comparator* user_cmp = user_comparator_;
-  const rocksdb::LevelFilesBrief& file_level = level_files_brief_[level];
   const FdWithKeyRange* files = level_files_brief_[level].files;
-  const size_t kNumFiles = file_level.num_files;
-
-  // Check the last file in inputs against the file after it
-  size_t last_file = FindFile(*internal_comparator_, file_level,
-                              inputs->back()->largest.Encode());
-  assert(last_file < kNumFiles);  // File should exist!
-  if (last_file < kNumFiles-1) {                    // If not the last file
-    const Slice last_key_in_input = ExtractUserKey(
-        files[last_file].largest_key);
-    const Slice first_key_after = ExtractUserKey(
-        files[last_file+1].smallest_key);
-    if (user_cmp->Equal(last_key_in_input, first_key_after)) {
-      // The last user key in input overlaps with the next file's first key
-      return true;
+#ifndef NDEBUG
+  {
+    // assert that the file at mid_index is within the range
+    assert(mid_index < level_files_brief_[level].num_files);
+    const FdWithKeyRange* f = &files[mid_index];
+    const Slice fstart = ExtractUserKey(f->smallest_key);
+    const Slice flimit = ExtractUserKey(f->largest_key);
+    assert(user_cmp->Compare(fstart, user_begin) >= 0 &&
+           user_cmp->Compare(flimit, user_end) <= 0);
+  }
+#endif
+  ExtendFileRangeOverlappingInterval(level, user_begin, user_end, mid_index,
+                                     start_index, end_index);
+  int left = *start_index;
+  int right = *end_index;
+  // shrink from left to right
+  while (left <= right) {
+    const Slice& first_key_in_range = ExtractUserKey(files[left].smallest_key);
+    if (user_cmp->Compare(first_key_in_range, user_begin) < 0) {
+      left++;
+      continue;
     }
+    if (left > 0) {  // If not first file
+      const Slice& last_key_before =
+          ExtractUserKey(files[left - 1].largest_key);
+      if (user_cmp->Equal(first_key_in_range, last_key_before)) {
+        // The first user key in range overlaps with the previous file's last
+        // key
+        left++;
+        continue;
+      }
+    }
+    break;
+  }
+  // shrink from right to left
+  while (left <= right) {
+    const Slice last_key_in_range = ExtractUserKey(files[right].largest_key);
+    if (user_cmp->Compare(last_key_in_range, user_end) > 0) {
+      right--;
+      continue;
+    }
+    if (right < static_cast<int>(level_files_brief_[level].num_files) -
+                    1) {  // If not the last file
+      const Slice first_key_after =
+          ExtractUserKey(files[right + 1].smallest_key);
+      if (user_cmp->Equal(last_key_in_range, first_key_after)) {
+        // The last user key in range overlaps with the next file's first key
+        right--;
+        continue;
+      }
+    }
+    break;
   }
 
-  // Check the first file in inputs against the file just before it
-  size_t first_file = FindFile(*internal_comparator_, file_level,
-                               inputs->front()->smallest.Encode());
-  assert(first_file <= last_file);   // File should exist!
-  if (first_file > 0) {                                 // If not first file
-    const Slice& first_key_in_input = ExtractUserKey(
-        files[first_file].smallest_key);
-    const Slice& last_key_before = ExtractUserKey(
-        files[first_file-1].largest_key);
-    if (user_cmp->Equal(first_key_in_input, last_key_before)) {
-      // The first user key in input overlaps with the previous file's last key
-      return true;
-    }
-  }
-
-  return false;
+  *start_index = left;
+  *end_index = right;
 }
 
 uint64_t VersionStorageInfo::NumLevelBytes(int level) const {
@@ -2361,9 +2429,8 @@ Status VersionSet::LogAndApply(ColumnFamilyData* column_family_data,
     // If we just created a new descriptor file, install it by writing a
     // new CURRENT file that points to it.
     if (s.ok() && new_descriptor_log) {
-      s = SetCurrentFile(
-          env_, dbname_, pending_manifest_file_number_,
-          db_options_->disable_data_sync ? nullptr : db_directory);
+      s = SetCurrentFile(env_, dbname_, pending_manifest_file_number_,
+                         db_directory);
     }
 
     if (s.ok()) {
@@ -3364,8 +3431,7 @@ InternalIterator* VersionSet::MakeInputIterator(
     const Compaction* c, RangeDelAggregator* range_del_agg) {
   auto cfd = c->column_family_data();
   ReadOptions read_options;
-  read_options.verify_checksums =
-    c->mutable_cf_options()->verify_checksums_in_compaction;
+  read_options.verify_checksums = true;
   read_options.fill_cache = false;
   if (c->ShouldFormSubcompactions()) {
     read_options.total_order_seek = true;

db/version_set.h

@@ -163,21 +163,41 @@ class VersionStorageInfo {
       bool expand_range = true)   // if set, returns files which overlap the
       const;                      // range and overlap each other. If false,
                                   // then just files intersecting the range
+  void GetCleanInputsWithinInterval(
+      int level, const InternalKey* begin,  // nullptr means before all keys
+      const InternalKey* end,               // nullptr means after all keys
+      std::vector<FileMetaData*>* inputs,
+      int hint_index = -1,        // index of overlap file
+      int* file_index = nullptr)  // return index of overlap file
+      const;
 
-  void GetOverlappingInputsBinarySearch(
-      int level,
+  void GetOverlappingInputsRangeBinarySearch(
+      int level,           // level > 0
       const Slice& begin,  // nullptr means before all keys
       const Slice& end,    // nullptr means after all keys
       std::vector<FileMetaData*>* inputs,
-      int hint_index,          // index of overlap file
-      int* file_index) const;  // return index of overlap file
+      int hint_index,                // index of overlap file
+      int* file_index,               // return index of overlap file
+      bool within_interval = false)  // if set, force the inputs within interval
+      const;
 
-  void ExtendOverlappingInputs(
+  void ExtendFileRangeOverlappingInterval(
       int level,
       const Slice& begin,  // nullptr means before all keys
       const Slice& end,    // nullptr means after all keys
-      std::vector<FileMetaData*>* inputs,
-      unsigned int index) const;  // start extending from this index
+      unsigned int index,  // start extending from this index
+      int* startIndex,     // return the startIndex of input range
+      int* endIndex)       // return the endIndex of input range
+      const;
+
+  void ExtendFileRangeWithinInterval(
+      int level,
+      const Slice& begin,  // nullptr means before all keys
+      const Slice& end,    // nullptr means after all keys
+      unsigned int index,  // start extending from this index
+      int* startIndex,     // return the startIndex of input range
+      int* endIndex)       // return the endIndex of input range
+      const;
 
   // Returns true iff some file in the specified level overlaps
   // some part of [*smallest_user_key,*largest_user_key].

db/write_controller.cc

@@ -7,6 +7,7 @@
 
 #include <atomic>
 #include <cassert>
+#include <ratio>
 #include "rocksdb/env.h"
 
 namespace rocksdb {
@@ -56,7 +57,7 @@ uint64_t WriteController::GetDelay(Env* env, uint64_t num_bytes) {
   }
   // The frequency to get time inside DB mutex is less than one per refill
   // interval.
-  auto time_now = env->NowMicros();
+  auto time_now = NowMicrosMonotonic(env);
 
   uint64_t sleep_debt = 0;
   uint64_t time_since_last_refill = 0;
@@ -103,6 +104,10 @@ uint64_t WriteController::GetDelay(Env* env, uint64_t num_bytes) {
   return sleep_amount;
 }
 
+uint64_t WriteController::NowMicrosMonotonic(Env* env) {
+  return env->NowNanos() / std::milli::den;
+}
+
 StopWriteToken::~StopWriteToken() {
   assert(controller_->total_stopped_ >= 1);
   --controller_->total_stopped_;

db/write_controller.h

@@ -78,6 +78,8 @@ class WriteController {
   uint64_t max_delayed_write_rate() const { return max_delayed_write_rate_; }
 
  private:
+  uint64_t NowMicrosMonotonic(Env* env);
+
   friend class WriteControllerToken;
   friend class StopWriteToken;
   friend class DelayWriteToken;

db/write_controller_test.cc

@@ -3,6 +3,8 @@
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
 //
+#include <ratio>
+
 #include "db/write_controller.h"
 
 #include "rocksdb/env.h"
@@ -16,7 +18,7 @@ class TimeSetEnv : public EnvWrapper {
  public:
   explicit TimeSetEnv() : EnvWrapper(nullptr) {}
   uint64_t now_micros_ = 6666;
-  virtual uint64_t NowMicros() override { return now_micros_; }
+  virtual uint64_t NowNanos() override { return now_micros_ * std::milli::den; }
 };
 
 TEST_F(WriteControllerTest, ChangeDelayRateTest) {

db/xfunc_test_points.cc

@@ -1,145 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-
-#include "db/xfunc_test_points.h"
-#include "util/xfunc.h"
-
-namespace rocksdb {
-
-#ifdef XFUNC
-
-void xf_manage_release(ManagedIterator* iter) {
-  if (!(XFuncPoint::GetSkip() & kSkipNoPrefix)) {
-    iter->ReleaseIter(false);
-  }
-}
-
-void xf_manage_create(ManagedIterator* iter) { iter->SetDropOld(false); }
-
-void xf_manage_new(DBImpl* db, ReadOptions* read_options,
-                   bool is_snapshot_supported) {
-  if ((!XFuncPoint::Check("managed_xftest_dropold") &&
-       (!XFuncPoint::Check("managed_xftest_release"))) ||
-      (!read_options->managed)) {
-    return;
-  }
-  if ((!read_options->tailing) && (read_options->snapshot == nullptr) &&
-      (!is_snapshot_supported)) {
-    read_options->managed = false;
-    return;
-  }
-  if (db->GetOptions().prefix_extractor != nullptr) {
-    if (strcmp(db->GetOptions().table_factory.get()->Name(), "PlainTable")) {
-      if (!(XFuncPoint::GetSkip() & kSkipNoPrefix)) {
-        read_options->total_order_seek = true;
-      }
-    } else {
-      read_options->managed = false;
-    }
-  }
-}
-
-class XFTransactionWriteHandler : public WriteBatch::Handler {
- public:
-  Transaction* txn_;
-  DBImpl* db_impl_;
-
-  XFTransactionWriteHandler(Transaction* txn, DBImpl* db_impl)
-      : txn_(txn), db_impl_(db_impl) {}
-
-  virtual Status PutCF(uint32_t column_family_id, const Slice& key,
-                       const Slice& value) override {
-    InstrumentedMutexLock l(&db_impl_->mutex_);
-
-    ColumnFamilyHandle* cfh = db_impl_->GetColumnFamilyHandle(column_family_id);
-    if (cfh == nullptr) {
-      return Status::InvalidArgument(
-          "XFUNC test could not find column family "
-          "handle for id ",
-          ToString(column_family_id));
-    }
-
-    txn_->Put(cfh, key, value);
-
-    return Status::OK();
-  }
-
-  virtual Status MergeCF(uint32_t column_family_id, const Slice& key,
-                         const Slice& value) override {
-    InstrumentedMutexLock l(&db_impl_->mutex_);
-
-    ColumnFamilyHandle* cfh = db_impl_->GetColumnFamilyHandle(column_family_id);
-    if (cfh == nullptr) {
-      return Status::InvalidArgument(
-          "XFUNC test could not find column family "
-          "handle for id ",
-          ToString(column_family_id));
-    }
-
-    txn_->Merge(cfh, key, value);
-
-    return Status::OK();
-  }
-
-  virtual Status DeleteCF(uint32_t column_family_id,
-                          const Slice& key) override {
-    InstrumentedMutexLock l(&db_impl_->mutex_);
-
-    ColumnFamilyHandle* cfh = db_impl_->GetColumnFamilyHandle(column_family_id);
-    if (cfh == nullptr) {
-      return Status::InvalidArgument(
-          "XFUNC test could not find column family "
-          "handle for id ",
-          ToString(column_family_id));
-    }
-
-    txn_->Delete(cfh, key);
-
-    return Status::OK();
-  }
-
-  virtual void LogData(const Slice& blob) override { txn_->PutLogData(blob); }
-};
-
-// Whenever DBImpl::Write is called, create a transaction and do the write via
-// the transaction.
-void xf_transaction_write(const WriteOptions& write_options,
-                          const DBOptions& db_options, WriteBatch* my_batch,
-                          WriteCallback* callback, DBImpl* db_impl, Status* s,
-                          bool* write_attempted) {
-  if (callback != nullptr) {
-    // We may already be in a transaction, don't force a transaction
-    *write_attempted = false;
-    return;
-  }
-
-  OptimisticTransactionDB* txn_db = new OptimisticTransactionDB(db_impl);
-  Transaction* txn = Transaction::BeginTransaction(txn_db, write_options);
-
-  XFTransactionWriteHandler handler(txn, db_impl);
-  *s = my_batch->Iterate(&handler);
-
-  if (!s->ok()) {
-    Log(InfoLogLevel::ERROR_LEVEL, db_options.info_log,
-        "XFUNC test could not iterate batch.  status: $s\n",
-        s->ToString().c_str());
-  }
-
-  *s = txn->Commit();
-
-  if (!s->ok()) {
-    Log(InfoLogLevel::ERROR_LEVEL, db_options.info_log,
-        "XFUNC test could not commit transaction.  status: $s\n",
-        s->ToString().c_str());
-  }
-
-  *write_attempted = true;
-  delete txn;
-  delete txn_db;
-}
-
-#endif  // XFUNC
-
-}  // namespace rocksdb

db/xfunc_test_points.h

@@ -1,33 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-
-#pragma once
-
-#include "db/db_impl.h"
-#include "db/managed_iterator.h"
-#include "db/write_callback.h"
-#include "rocksdb/options.h"
-#include "rocksdb/write_batch.h"
-#include "util/xfunc.h"
-
-namespace rocksdb {
-
-#ifdef XFUNC
-
-// DB-specific test points for the cross-functional test framework (see
-// util/xfunc.h).
-void xf_manage_release(ManagedIterator* iter);
-void xf_manage_create(ManagedIterator* iter);
-void xf_manage_new(DBImpl* db, ReadOptions* readoptions,
-                   bool is_snapshot_supported);
-void xf_transaction_write(const WriteOptions& write_options,
-                          const DBOptions& db_options,
-                          class WriteBatch* my_batch,
-                          class WriteCallback* callback, DBImpl* db_impl,
-                          Status* success, bool* write_attempted);
-
-#endif  // XFUNC
-
-}  // namespace rocksdb

docs/_posts/2017-02-17-bulkoad-ingest-sst-file.markdown

@@ -0,0 +1,50 @@
+---
+title: Bulkoading by ingesting external SST files
+layout: post
+author: IslamAbdelRahman
+category: blog
+---
+
+## Introduction
+
+One of the basic operations of RocksDB is writing to RocksDB, Writes happen when user call (DB::Put, DB::Write, DB::Delete ... ), but what happens when you write to RocksDB ? .. this is a brief description of what happens.
+- User insert a new key/value by calling DB::Put() (or DB::Write())
+- We create a new entry for the new key/value in our in-memory structure (memtable / SkipList by default) and we assign it a new sequence number.
+- When the memtable exceeds a specific size (64 MB for example), we convert this memtable to a SST file, and put this file in level 0 of our LSM-Tree
+- Later, compaction will kick in and move data from level 0 to level 1, and then from level 1 to level 2 .. and so on 
+
+But what if we can skip these steps and add data to the lowest possible level directly ? This is what bulk-loading does
+
+## Bulkloading
+
+- Write all of our keys and values into SST file outside of the DB
+- Add the SST file into the LSM directly
+
+This is bulk-loading, and in specific use-cases it allow users to achieve faster data loading and better write-amplification.
+
+and doing it is as simple as 
+```cpp
+Options options;
+SstFileWriter sst_file_writer(EnvOptions(), options, options.comparator);
+Status s = sst_file_writer.Open(file_path);
+assert(s.ok());
+
+// Insert rows into the SST file, note that inserted keys must be 
+// strictly increasing (based on options.comparator)
+for (...) {
+  s = sst_file_writer.Add(key, value);
+  assert(s.ok());
+}
+
+// Ingest the external SST file into the DB
+s = db_->IngestExternalFile({"/home/usr/file1.sst"}, IngestExternalFileOptions());
+assert(s.ok());
+```
+
+You can find more details about how to generate SST files and ingesting them into RocksDB in this [wiki page](https://github.com/facebook/rocksdb/wiki/Creating-and-Ingesting-SST-files)
+
+## Use cases
+There are multiple use cases where bulkloading could be useful, for example
+- Generating SST files in offline jobs in Hadoop, then downloading and ingesting the SST files into RocksDB
+- Migrating shards between machines by dumping key-range in SST File and loading the file in a different machine
+- Migrating from a different storage (InnoDB to RocksDB migration in MyRocks)

examples/Makefile

@@ -7,7 +7,7 @@ ifndef DISABLE_JEMALLOC
 	EXEC_LDFLAGS := $(JEMALLOC_LIB) $(EXEC_LDFLAGS) -lpthread
 	PLATFORM_CXXFLAGS += $(JEMALLOC_INCLUDE)
 endif
-	
+
 .PHONY: clean librocksdb
 
 all: simple_example column_families_example compact_files_example c_simple_example optimistic_transaction_example transaction_example compaction_filter_example options_file_example

examples/rocksdb_option_file_example.ini

@@ -62,13 +62,11 @@
   create_missing_column_families=false
   paranoid_checks=true
   delete_obsolete_files_period_micros=21600000000
-  disable_data_sync=false
   log_file_time_to_roll=0
   compaction_readahead_size=0
   create_if_missing=false
   use_adaptive_mutex=false
   enable_thread_tracking=false
-  disableDataSync=false
   allow_fallocate=true
   error_if_exists=false
   recycle_log_file_num=0
@@ -132,7 +130,6 @@
 [TableOptions/BlockBasedTable "default"]
   format_version=2
   whole_key_filtering=true
-  skip_table_builder_flush=false
   no_block_cache=false
   checksum=kCRC32c
   filter_policy=rocksdb.BuiltinBloomFilter

include/rocksdb/advanced_options.h

@@ -0,0 +1,558 @@
+// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#pragma once
+
+#include <memory>
+
+#include "rocksdb/memtablerep.h"
+#include "rocksdb/universal_compaction.h"
+
+namespace rocksdb {
+
+class Slice;
+class SliceTransform;
+enum CompressionType : unsigned char;
+class TablePropertiesCollectorFactory;
+class TableFactory;
+struct Options;
+
+enum CompactionStyle : char {
+  // level based compaction style
+  kCompactionStyleLevel = 0x0,
+  // Universal compaction style
+  // Not supported in ROCKSDB_LITE.
+  kCompactionStyleUniversal = 0x1,
+  // FIFO compaction style
+  // Not supported in ROCKSDB_LITE
+  kCompactionStyleFIFO = 0x2,
+  // Disable background compaction. Compaction jobs are submitted
+  // via CompactFiles().
+  // Not supported in ROCKSDB_LITE
+  kCompactionStyleNone = 0x3,
+};
+
+// In Level-based comapction, it Determines which file from a level to be
+// picked to merge to the next level. We suggest people try
+// kMinOverlappingRatio first when you tune your database.
+enum CompactionPri : char {
+  // Slightly Priotize larger files by size compensated by #deletes
+  kByCompensatedSize = 0x0,
+  // First compact files whose data's latest update time is oldest.
+  // Try this if you only update some hot keys in small ranges.
+  kOldestLargestSeqFirst = 0x1,
+  // First compact files whose range hasn't been compacted to the next level
+  // for the longest. If your updates are random across the key space,
+  // write amplification is slightly better with this option.
+  kOldestSmallestSeqFirst = 0x2,
+  // First compact files whose ratio between overlapping size in next level
+  // and its size is the smallest. It in many cases can optimize write
+  // amplification.
+  kMinOverlappingRatio = 0x3,
+};
+
+struct CompactionOptionsFIFO {
+  // once the total sum of table files reaches this, we will delete the oldest
+  // table file
+  // Default: 1GB
+  uint64_t max_table_files_size;
+
+  CompactionOptionsFIFO() : max_table_files_size(1 * 1024 * 1024 * 1024) {}
+};
+
+// Compression options for different compression algorithms like Zlib
+struct CompressionOptions {
+  int window_bits;
+  int level;
+  int strategy;
+  // Maximum size of dictionary used to prime the compression library. Currently
+  // this dictionary will be constructed by sampling the first output file in a
+  // subcompaction when the target level is bottommost. This dictionary will be
+  // loaded into the compression library before compressing/uncompressing each
+  // data block of subsequent files in the subcompaction. Effectively, this
+  // improves compression ratios when there are repetitions across data blocks.
+  // A value of 0 indicates the feature is disabled.
+  // Default: 0.
+  uint32_t max_dict_bytes;
+
+  CompressionOptions()
+      : window_bits(-14), level(-1), strategy(0), max_dict_bytes(0) {}
+  CompressionOptions(int wbits, int _lev, int _strategy, int _max_dict_bytes)
+      : window_bits(wbits),
+        level(_lev),
+        strategy(_strategy),
+        max_dict_bytes(_max_dict_bytes) {}
+};
+
+enum UpdateStatus {    // Return status For inplace update callback
+  UPDATE_FAILED   = 0, // Nothing to update
+  UPDATED_INPLACE = 1, // Value updated inplace
+  UPDATED         = 2, // No inplace update. Merged value set
+};
+
+
+struct AdvancedColumnFamilyOptions {
+  // The maximum number of write buffers that are built up in memory.
+  // The default and the minimum number is 2, so that when 1 write buffer
+  // is being flushed to storage, new writes can continue to the other
+  // write buffer.
+  // If max_write_buffer_number > 3, writing will be slowed down to
+  // options.delayed_write_rate if we are writing to the last write buffer
+  // allowed.
+  //
+  // Default: 2
+  //
+  // Dynamically changeable through SetOptions() API
+  int max_write_buffer_number = 2;
+
+  // The minimum number of write buffers that will be merged together
+  // before writing to storage.  If set to 1, then
+  // all write buffers are flushed to L0 as individual files and this increases
+  // read amplification because a get request has to check in all of these
+  // files. Also, an in-memory merge may result in writing lesser
+  // data to storage if there are duplicate records in each of these
+  // individual write buffers.  Default: 1
+  int min_write_buffer_number_to_merge = 1;
+
+  // The total maximum number of write buffers to maintain in memory including
+  // copies of buffers that have already been flushed.  Unlike
+  // max_write_buffer_number, this parameter does not affect flushing.
+  // This controls the minimum amount of write history that will be available
+  // in memory for conflict checking when Transactions are used.
+  //
+  // When using an OptimisticTransactionDB:
+  // If this value is too low, some transactions may fail at commit time due
+  // to not being able to determine whether there were any write conflicts.
+  //
+  // When using a TransactionDB:
+  // If Transaction::SetSnapshot is used, TransactionDB will read either
+  // in-memory write buffers or SST files to do write-conflict checking.
+  // Increasing this value can reduce the number of reads to SST files
+  // done for conflict detection.
+  //
+  // Setting this value to 0 will cause write buffers to be freed immediately
+  // after they are flushed.
+  // If this value is set to -1, 'max_write_buffer_number' will be used.
+  //
+  // Default:
+  // If using a TransactionDB/OptimisticTransactionDB, the default value will
+  // be set to the value of 'max_write_buffer_number' if it is not explicitly
+  // set by the user.  Otherwise, the default is 0.
+  int max_write_buffer_number_to_maintain = 0;
+
+  // Allows thread-safe inplace updates. If this is true, there is no way to
+  // achieve point-in-time consistency using snapshot or iterator (assuming
+  // concurrent updates). Hence iterator and multi-get will return results
+  // which are not consistent as of any point-in-time.
+  // If inplace_callback function is not set,
+  //   Put(key, new_value) will update inplace the existing_value iff
+  //   * key exists in current memtable
+  //   * new sizeof(new_value) <= sizeof(existing_value)
+  //   * existing_value for that key is a put i.e. kTypeValue
+  // If inplace_callback function is set, check doc for inplace_callback.
+  // Default: false.
+  bool inplace_update_support = false;
+
+  // Number of locks used for inplace update
+  // Default: 10000, if inplace_update_support = true, else 0.
+  //
+  // Dynamically changeable through SetOptions() API
+  size_t inplace_update_num_locks = 10000;
+
+  // existing_value - pointer to previous value (from both memtable and sst).
+  //                  nullptr if key doesn't exist
+  // existing_value_size - pointer to size of existing_value).
+  //                       nullptr if key doesn't exist
+  // delta_value - Delta value to be merged with the existing_value.
+  //               Stored in transaction logs.
+  // merged_value - Set when delta is applied on the previous value.
+
+  // Applicable only when inplace_update_support is true,
+  // this callback function is called at the time of updating the memtable
+  // as part of a Put operation, lets say Put(key, delta_value). It allows the
+  // 'delta_value' specified as part of the Put operation to be merged with
+  // an 'existing_value' of the key in the database.
+
+  // If the merged value is smaller in size that the 'existing_value',
+  // then this function can update the 'existing_value' buffer inplace and
+  // the corresponding 'existing_value'_size pointer, if it wishes to.
+  // The callback should return UpdateStatus::UPDATED_INPLACE.
+  // In this case. (In this case, the snapshot-semantics of the rocksdb
+  // Iterator is not atomic anymore).
+
+  // If the merged value is larger in size than the 'existing_value' or the
+  // application does not wish to modify the 'existing_value' buffer inplace,
+  // then the merged value should be returned via *merge_value. It is set by
+  // merging the 'existing_value' and the Put 'delta_value'. The callback should
+  // return UpdateStatus::UPDATED in this case. This merged value will be added
+  // to the memtable.
+
+  // If merging fails or the application does not wish to take any action,
+  // then the callback should return UpdateStatus::UPDATE_FAILED.
+
+  // Please remember that the original call from the application is Put(key,
+  // delta_value). So the transaction log (if enabled) will still contain (key,
+  // delta_value). The 'merged_value' is not stored in the transaction log.
+  // Hence the inplace_callback function should be consistent across db reopens.
+
+  // Default: nullptr
+  UpdateStatus (*inplace_callback)(char* existing_value,
+                                   uint32_t* existing_value_size,
+                                   Slice delta_value,
+                                   std::string* merged_value) = nullptr;
+
+  // if prefix_extractor is set and memtable_prefix_bloom_size_ratio is not 0,
+  // create prefix bloom for memtable with the size of
+  // write_buffer_size * memtable_prefix_bloom_size_ratio.
+  // If it is larger than 0.25, it is santinized to 0.25.
+  //
+  // Default: 0 (disable)
+  //
+  // Dynamically changeable through SetOptions() API
+  double memtable_prefix_bloom_size_ratio = 0.0;
+
+  // Page size for huge page for the arena used by the memtable. If <=0, it
+  // won't allocate from huge page but from malloc.
+  // Users are responsible to reserve huge pages for it to be allocated. For
+  // example:
+  //      sysctl -w vm.nr_hugepages=20
+  // See linux doc Documentation/vm/hugetlbpage.txt
+  // If there isn't enough free huge page available, it will fall back to
+  // malloc.
+  //
+  // Dynamically changeable through SetOptions() API
+  size_t memtable_huge_page_size = 0;
+
+  // If non-nullptr, memtable will use the specified function to extract
+  // prefixes for keys, and for each prefix maintain a hint of insert location
+  // to reduce CPU usage for inserting keys with the prefix. Keys out of
+  // domain of the prefix extractor will be insert without using hints.
+  //
+  // Currently only the default skiplist based memtable implements the feature.
+  // All other memtable implementation will ignore the option. It incurs ~250
+  // additional bytes of memory overhead to store a hint for each prefix.
+  // Also concurrent writes (when allow_concurrent_memtable_write is true) will
+  // ignore the option.
+  //
+  // The option is best suited for workloads where keys will likely to insert
+  // to a location close the the last inserted key with the same prefix.
+  // One example could be inserting keys of the form (prefix + timestamp),
+  // and keys of the same prefix always comes in with time order. Another
+  // example would be updating the same key over and over again, in which case
+  // the prefix can be the key itself.
+  //
+  // Default: nullptr (disable)
+  std::shared_ptr<const SliceTransform>
+      memtable_insert_with_hint_prefix_extractor = nullptr;
+
+  // Control locality of bloom filter probes to improve cache miss rate.
+  // This option only applies to memtable prefix bloom and plaintable
+  // prefix bloom. It essentially limits every bloom checking to one cache line.
+  // This optimization is turned off when set to 0, and positive number to turn
+  // it on.
+  // Default: 0
+  uint32_t bloom_locality = 0;
+
+  // size of one block in arena memory allocation.
+  // If <= 0, a proper value is automatically calculated (usually 1/8 of
+  // writer_buffer_size, rounded up to a multiple of 4KB).
+  //
+  // There are two additional restriction of the The specified size:
+  // (1) size should be in the range of [4096, 2 << 30] and
+  // (2) be the multiple of the CPU word (which helps with the memory
+  // alignment).
+  //
+  // We'll automatically check and adjust the size number to make sure it
+  // conforms to the restrictions.
+  //
+  // Default: 0
+  //
+  // Dynamically changeable through SetOptions() API
+  size_t arena_block_size = 0;
+
+  // Different levels can have different compression policies. There
+  // are cases where most lower levels would like to use quick compression
+  // algorithms while the higher levels (which have more data) use
+  // compression algorithms that have better compression but could
+  // be slower. This array, if non-empty, should have an entry for
+  // each level of the database; these override the value specified in
+  // the previous field 'compression'.
+  //
+  // NOTICE if level_compaction_dynamic_level_bytes=true,
+  // compression_per_level[0] still determines L0, but other elements
+  // of the array are based on base level (the level L0 files are merged
+  // to), and may not match the level users see from info log for metadata.
+  // If L0 files are merged to level-n, then, for i>0, compression_per_level[i]
+  // determines compaction type for level n+i-1.
+  // For example, if we have three 5 levels, and we determine to merge L0
+  // data to L4 (which means L1..L3 will be empty), then the new files go to
+  // L4 uses compression type compression_per_level[1].
+  // If now L0 is merged to L2. Data goes to L2 will be compressed
+  // according to compression_per_level[1], L3 using compression_per_level[2]
+  // and L4 using compression_per_level[3]. Compaction for each level can
+  // change when data grows.
+  std::vector<CompressionType> compression_per_level;
+
+  // Number of levels for this database
+  int num_levels = 7;
+
+  // Soft limit on number of level-0 files. We start slowing down writes at this
+  // point. A value <0 means that no writing slow down will be triggered by
+  // number of files in level-0.
+  //
+  // Default: 20
+  //
+  // Dynamically changeable through SetOptions() API
+  int level0_slowdown_writes_trigger = 20;
+
+  // Maximum number of level-0 files.  We stop writes at this point.
+  //
+  // Default: 36
+  //
+  // Dynamically changeable through SetOptions() API
+  int level0_stop_writes_trigger = 36;
+
+  // Target file size for compaction.
+  // target_file_size_base is per-file size for level-1.
+  // Target file size for level L can be calculated by
+  // target_file_size_base * (target_file_size_multiplier ^ (L-1))
+  // For example, if target_file_size_base is 2MB and
+  // target_file_size_multiplier is 10, then each file on level-1 will
+  // be 2MB, and each file on level 2 will be 20MB,
+  // and each file on level-3 will be 200MB.
+  //
+  // Default: 64MB.
+  //
+  // Dynamically changeable through SetOptions() API
+  uint64_t target_file_size_base = 64 * 1048576;
+
+  // By default target_file_size_multiplier is 1, which means
+  // by default files in different levels will have similar size.
+  //
+  // Dynamically changeable through SetOptions() API
+  int target_file_size_multiplier = 1;
+
+  // If true, RocksDB will pick target size of each level dynamically.
+  // We will pick a base level b >= 1. L0 will be directly merged into level b,
+  // instead of always into level 1. Level 1 to b-1 need to be empty.
+  // We try to pick b and its target size so that
+  // 1. target size is in the range of
+  //   (max_bytes_for_level_base / max_bytes_for_level_multiplier,
+  //    max_bytes_for_level_base]
+  // 2. target size of the last level (level num_levels-1) equals to extra size
+  //    of the level.
+  // At the same time max_bytes_for_level_multiplier and
+  // max_bytes_for_level_multiplier_additional are still satisfied.
+  //
+  // With this option on, from an empty DB, we make last level the base level,
+  // which means merging L0 data into the last level, until it exceeds
+  // max_bytes_for_level_base. And then we make the second last level to be
+  // base level, to start to merge L0 data to second last level, with its
+  // target size to be 1/max_bytes_for_level_multiplier of the last level's
+  // extra size. After the data accumulates more so that we need to move the
+  // base level to the third last one, and so on.
+  //
+  // For example, assume max_bytes_for_level_multiplier=10, num_levels=6,
+  // and max_bytes_for_level_base=10MB.
+  // Target sizes of level 1 to 5 starts with:
+  // [- - - - 10MB]
+  // with base level is level. Target sizes of level 1 to 4 are not applicable
+  // because they will not be used.
+  // Until the size of Level 5 grows to more than 10MB, say 11MB, we make
+  // base target to level 4 and now the targets looks like:
+  // [- - - 1.1MB 11MB]
+  // While data are accumulated, size targets are tuned based on actual data
+  // of level 5. When level 5 has 50MB of data, the target is like:
+  // [- - - 5MB 50MB]
+  // Until level 5's actual size is more than 100MB, say 101MB. Now if we keep
+  // level 4 to be the base level, its target size needs to be 10.1MB, which
+  // doesn't satisfy the target size range. So now we make level 3 the target
+  // size and the target sizes of the levels look like:
+  // [- - 1.01MB 10.1MB 101MB]
+  // In the same way, while level 5 further grows, all levels' targets grow,
+  // like
+  // [- - 5MB 50MB 500MB]
+  // Until level 5 exceeds 1000MB and becomes 1001MB, we make level 2 the
+  // base level and make levels' target sizes like this:
+  // [- 1.001MB 10.01MB 100.1MB 1001MB]
+  // and go on...
+  //
+  // By doing it, we give max_bytes_for_level_multiplier a priority against
+  // max_bytes_for_level_base, for a more predictable LSM tree shape. It is
+  // useful to limit worse case space amplification.
+  //
+  // max_bytes_for_level_multiplier_additional is ignored with this flag on.
+  //
+  // Turning this feature on or off for an existing DB can cause unexpected
+  // LSM tree structure so it's not recommended.
+  //
+  // NOTE: this option is experimental
+  //
+  // Default: false
+  bool level_compaction_dynamic_level_bytes = false;
+
+  // Default: 10.
+  //
+  // Dynamically changeable through SetOptions() API
+  double max_bytes_for_level_multiplier = 10;
+
+  // Different max-size multipliers for different levels.
+  // These are multiplied by max_bytes_for_level_multiplier to arrive
+  // at the max-size of each level.
+  //
+  // Default: 1
+  //
+  // Dynamically changeable through SetOptions() API
+  std::vector<int> max_bytes_for_level_multiplier_additional =
+      std::vector<int>(num_levels, 1);
+
+  // We try to limit number of bytes in one compaction to be lower than this
+  // threshold. But it's not guaranteed.
+  // Value 0 will be sanitized.
+  //
+  // Default: result.target_file_size_base * 25
+  uint64_t max_compaction_bytes = 0;
+
+  // All writes will be slowed down to at least delayed_write_rate if estimated
+  // bytes needed to be compaction exceed this threshold.
+  //
+  // Default: 64GB
+  uint64_t soft_pending_compaction_bytes_limit = 64 * 1073741824ull;
+
+  // All writes are stopped if estimated bytes needed to be compaction exceed
+  // this threshold.
+  //
+  // Default: 256GB
+  uint64_t hard_pending_compaction_bytes_limit = 256 * 1073741824ull;
+
+  // The compaction style. Default: kCompactionStyleLevel
+  CompactionStyle compaction_style = kCompactionStyleLevel;
+
+  // If level compaction_style = kCompactionStyleLevel, for each level,
+  // which files are prioritized to be picked to compact.
+  // Default: kByCompensatedSize
+  CompactionPri compaction_pri = kByCompensatedSize;
+
+  // The options needed to support Universal Style compactions
+  CompactionOptionsUniversal compaction_options_universal;
+
+  // The options for FIFO compaction style
+  CompactionOptionsFIFO compaction_options_fifo;
+
+  // An iteration->Next() sequentially skips over keys with the same
+  // user-key unless this option is set. This number specifies the number
+  // of keys (with the same userkey) that will be sequentially
+  // skipped before a reseek is issued.
+  //
+  // Default: 8
+  //
+  // Dynamically changeable through SetOptions() API
+  uint64_t max_sequential_skip_in_iterations = 8;
+
+  // This is a factory that provides MemTableRep objects.
+  // Default: a factory that provides a skip-list-based implementation of
+  // MemTableRep.
+  std::shared_ptr<MemTableRepFactory> memtable_factory =
+      std::shared_ptr<SkipListFactory>(new SkipListFactory);
+
+  // Block-based table related options are moved to BlockBasedTableOptions.
+  // Related options that were originally here but now moved include:
+  //   no_block_cache
+  //   block_cache
+  //   block_cache_compressed
+  //   block_size
+  //   block_size_deviation
+  //   block_restart_interval
+  //   filter_policy
+  //   whole_key_filtering
+  // If you'd like to customize some of these options, you will need to
+  // use NewBlockBasedTableFactory() to construct a new table factory.
+
+  // This option allows user to collect their own interested statistics of
+  // the tables.
+  // Default: empty vector -- no user-defined statistics collection will be
+  // performed.
+  typedef std::vector<std::shared_ptr<TablePropertiesCollectorFactory>>
+      TablePropertiesCollectorFactories;
+  TablePropertiesCollectorFactories table_properties_collector_factories;
+
+  // Maximum number of successive merge operations on a key in the memtable.
+  //
+  // When a merge operation is added to the memtable and the maximum number of
+  // successive merges is reached, the value of the key will be calculated and
+  // inserted into the memtable instead of the merge operation. This will
+  // ensure that there are never more than max_successive_merges merge
+  // operations in the memtable.
+  //
+  // Default: 0 (disabled)
+  //
+  // Dynamically changeable through SetOptions() API
+  size_t max_successive_merges = 0;
+
+  // This flag specifies that the implementation should optimize the filters
+  // mainly for cases where keys are found rather than also optimize for keys
+  // missed. This would be used in cases where the application knows that
+  // there are very few misses or the performance in the case of misses is not
+  // important.
+  //
+  // For now, this flag allows us to not store filters for the last level i.e
+  // the largest level which contains data of the LSM store. For keys which
+  // are hits, the filters in this level are not useful because we will search
+  // for the data anyway. NOTE: the filters in other levels are still useful
+  // even for key hit because they tell us whether to look in that level or go
+  // to the higher level.
+  //
+  // Default: false
+  bool optimize_filters_for_hits = false;
+
+  // After writing every SST file, reopen it and read all the keys.
+  // Default: false
+  bool paranoid_file_checks = false;
+
+  // In debug mode, RocksDB run consistency checks on the LSM everytime the LSM
+  // change (Flush, Compaction, AddFile). These checks are disabled in release
+  // mode, use this option to enable them in release mode as well.
+  // Default: false
+  bool force_consistency_checks = false;
+
+  // Measure IO stats in compactions and flushes, if true.
+  // Default: false
+  bool report_bg_io_stats = false;
+
+  // Create ColumnFamilyOptions with default values for all fields
+  AdvancedColumnFamilyOptions();
+  // Create ColumnFamilyOptions from Options
+  explicit AdvancedColumnFamilyOptions(const Options& options);
+
+  // ---------------- DEPRECATED OPTIONS ----------------
+
+  // DEPRECATED
+  // This does not do anything anymore.
+  int max_mem_compaction_level;
+
+  // DEPRECATED -- this options is no longer used
+  // Puts are delayed to options.delayed_write_rate when any level has a
+  // compaction score that exceeds soft_rate_limit. This is ignored when == 0.0.
+  //
+  // Default: 0 (disabled)
+  //
+  // Dynamically changeable through SetOptions() API
+  double soft_rate_limit = 0.0;
+
+  // DEPRECATED -- this options is no longer used
+  double hard_rate_limit = 0.0;
+
+  // DEPRECATED -- this options is no longer used
+  unsigned int rate_limit_delay_max_milliseconds = 100;
+
+  // DEPREACTED
+  // Does not have any effect.
+  bool purge_redundant_kvs_while_flush = true;
+};
+
+}  // namespace rocksdb

include/rocksdb/c.h

@@ -502,9 +502,6 @@ rocksdb_block_based_options_set_cache_index_and_filter_blocks(
 extern ROCKSDB_LIBRARY_API void
 rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache(
     rocksdb_block_based_table_options_t*, unsigned char);
-extern ROCKSDB_LIBRARY_API void
-rocksdb_block_based_options_set_skip_table_builder_flush(
-    rocksdb_block_based_table_options_t* options, unsigned char);
 extern ROCKSDB_LIBRARY_API void rocksdb_options_set_block_based_table_factory(
     rocksdb_options_t* opt, rocksdb_block_based_table_options_t* table_options);
 
@@ -700,13 +697,8 @@ extern ROCKSDB_LIBRARY_API void
 rocksdb_options_set_enable_write_thread_adaptive_yield(rocksdb_options_t*,
                                                        unsigned char);
 extern ROCKSDB_LIBRARY_API void
-rocksdb_options_set_verify_checksums_in_compaction(rocksdb_options_t*,
-                                                   unsigned char);
-extern ROCKSDB_LIBRARY_API void
 rocksdb_options_set_max_sequential_skip_in_iterations(rocksdb_options_t*,
                                                       uint64_t);
-extern ROCKSDB_LIBRARY_API void rocksdb_options_set_disable_data_sync(
-    rocksdb_options_t*, int);
 extern ROCKSDB_LIBRARY_API void rocksdb_options_set_disable_auto_compactions(
     rocksdb_options_t*, int);
 extern ROCKSDB_LIBRARY_API void rocksdb_options_set_optimize_filters_for_hits(
@@ -737,8 +729,6 @@ extern ROCKSDB_LIBRARY_API void rocksdb_options_set_memtable_huge_page_size(
 
 extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_successive_merges(
     rocksdb_options_t*, size_t);
-extern ROCKSDB_LIBRARY_API void rocksdb_options_set_min_partial_merge_operands(
-    rocksdb_options_t*, uint32_t);
 extern ROCKSDB_LIBRARY_API void rocksdb_options_set_bloom_locality(
     rocksdb_options_t*, uint32_t);
 extern ROCKSDB_LIBRARY_API void rocksdb_options_set_inplace_update_support(

include/rocksdb/db.h

@@ -527,6 +527,13 @@ class DB {
     //      one but only returns the aggregated table properties of the
     //      specified level "N" at the target column family.
     static const std::string kAggregatedTablePropertiesAtLevel;
+
+    //  "rocksdb.actual-delayed-write-rate" - returns the current actual delayed
+    //      write rate. 0 means no delay.
+    static const std::string kActualDelayedWriteRate;
+
+    //  "rocksdb.is-write-stopped" - Return 1 if write has been stopped.
+    static const std::string kIsWriteStopped;
   };
 #endif /* ROCKSDB_LITE */
 
@@ -575,6 +582,8 @@ class DB {
   //  "rocksdb.estimate-pending-compaction-bytes"
   //  "rocksdb.num-running-compactions"
   //  "rocksdb.num-running-flushes"
+  //  "rocksdb.actual-delayed-write-rate"
+  //  "rocksdb.is-write-stopped"
   virtual bool GetIntProperty(ColumnFamilyHandle* column_family,
                               const Slice& property, uint64_t* value) = 0;
   virtual bool GetIntProperty(const Slice& property, uint64_t* value) {

include/rocksdb/env.h

@@ -318,15 +318,16 @@ class Env {
   virtual Status NewLogger(const std::string& fname,
                            shared_ptr<Logger>* result) = 0;
 
-  // Returns the number of micro-seconds since some fixed point in time. Only
-  // useful for computing deltas of time.
-  // However, it is often used as system time such as in GenericRateLimiter
+  // Returns the number of micro-seconds since some fixed point in time.
+  // It is often used as system time such as in GenericRateLimiter
   // and other places so a port needs to return system time in order to work.
   virtual uint64_t NowMicros() = 0;
 
   // Returns the number of nano-seconds since some fixed point in time. Only
   // useful for computing deltas of time in one run.
-  // Default implementation simply relies on NowMicros
+  // Default implementation simply relies on NowMicros.
+  // In platform-specific implementations, NowNanos() should return time points
+  // that are MONOTONIC.
   virtual uint64_t NowNanos() {
     return NowMicros() * 1000;
   }
@@ -982,6 +983,7 @@ class EnvWrapper : public Env {
     return target_->NewLogger(fname, result);
   }
   uint64_t NowMicros() override { return target_->NowMicros(); }
+
   void SleepForMicroseconds(int micros) override {
     target_->SleepForMicroseconds(micros);
   }

include/rocksdb/merge_operator.h

@@ -165,10 +165,8 @@ class MergeOperator {
   //
   // The string that new_value is pointing to will be empty.
   //
-  // The PartialMergeMulti function will be called only when the list of
-  // operands are long enough. The minimum amount of operands that will be
-  // passed to the function are specified by the "min_partial_merge_operands"
-  // option.
+  // The PartialMergeMulti function will be called when there are at least two
+  // operands.
   //
   // In the default implementation, PartialMergeMulti will invoke PartialMerge
   // multiple times, where each time it only merges two operands.  Developers

include/rocksdb/options.h

@@ -17,10 +17,10 @@
 #include <limits>
 #include <unordered_map>
 
+#include "rocksdb/advanced_options.h"
 #include "rocksdb/comparator.h"
 #include "rocksdb/env.h"
 #include "rocksdb/listener.h"
-#include "rocksdb/memtablerep.h"
 #include "rocksdb/universal_compaction.h"
 #include "rocksdb/version.h"
 #include "rocksdb/write_buffer_manager.h"
@@ -42,12 +42,9 @@ class FilterPolicy;
 class Logger;
 class MergeOperator;
 class Snapshot;
-class TableFactory;
 class MemTableRepFactory;
-class TablePropertiesCollectorFactory;
 class RateLimiter;
 class Slice;
-class SliceTransform;
 class Statistics;
 class InternalKeyComparator;
 class WalFilter;
@@ -79,113 +76,9 @@ enum CompressionType : unsigned char {
   kDisableCompressionOption = 0xff,
 };
 
-enum CompactionStyle : char {
-  // level based compaction style
-  kCompactionStyleLevel = 0x0,
-  // Universal compaction style
-  // Not supported in ROCKSDB_LITE.
-  kCompactionStyleUniversal = 0x1,
-  // FIFO compaction style
-  // Not supported in ROCKSDB_LITE
-  kCompactionStyleFIFO = 0x2,
-  // Disable background compaction. Compaction jobs are submitted
-  // via CompactFiles().
-  // Not supported in ROCKSDB_LITE
-  kCompactionStyleNone = 0x3,
-};
-
-// In Level-based comapction, it Determines which file from a level to be
-// picked to merge to the next level. We suggest people try
-// kMinOverlappingRatio first when you tune your database.
-enum CompactionPri : char {
-  // Slightly Priotize larger files by size compensated by #deletes
-  kByCompensatedSize = 0x0,
-  // First compact files whose data's latest update time is oldest.
-  // Try this if you only update some hot keys in small ranges.
-  kOldestLargestSeqFirst = 0x1,
-  // First compact files whose range hasn't been compacted to the next level
-  // for the longest. If your updates are random across the key space,
-  // write amplification is slightly better with this option.
-  kOldestSmallestSeqFirst = 0x2,
-  // First compact files whose ratio between overlapping size in next level
-  // and its size is the smallest. It in many cases can optimize write
-  // amplification.
-  kMinOverlappingRatio = 0x3,
-};
-
-enum class WALRecoveryMode : char {
-  // Original levelDB recovery
-  // We tolerate incomplete record in trailing data on all logs
-  // Use case : This is legacy behavior (default)
-  kTolerateCorruptedTailRecords = 0x00,
-  // Recover from clean shutdown
-  // We don't expect to find any corruption in the WAL
-  // Use case : This is ideal for unit tests and rare applications that
-  // can require high consistency guarantee
-  kAbsoluteConsistency = 0x01,
-  // Recover to point-in-time consistency
-  // We stop the WAL playback on discovering WAL inconsistency
-  // Use case : Ideal for systems that have disk controller cache like
-  // hard disk, SSD without super capacitor that store related data
-  kPointInTimeRecovery = 0x02,
-  // Recovery after a disaster
-  // We ignore any corruption in the WAL and try to salvage as much data as
-  // possible
-  // Use case : Ideal for last ditch effort to recover data or systems that
-  // operate with low grade unrelated data
-  kSkipAnyCorruptedRecords = 0x03,
-};
-
-struct CompactionOptionsFIFO {
-  // once the total sum of table files reaches this, we will delete the oldest
-  // table file
-  // Default: 1GB
-  uint64_t max_table_files_size;
-
-  CompactionOptionsFIFO() : max_table_files_size(1 * 1024 * 1024 * 1024) {}
-};
-
-// Compression options for different compression algorithms like Zlib
-struct CompressionOptions {
-  int window_bits;
-  int level;
-  int strategy;
-  // Maximum size of dictionary used to prime the compression library. Currently
-  // this dictionary will be constructed by sampling the first output file in a
-  // subcompaction when the target level is bottommost. This dictionary will be
-  // loaded into the compression library before compressing/uncompressing each
-  // data block of subsequent files in the subcompaction. Effectively, this
-  // improves compression ratios when there are repetitions across data blocks.
-  // A value of 0 indicates the feature is disabled.
-  // Default: 0.
-  uint32_t max_dict_bytes;
-
-  CompressionOptions()
-      : window_bits(-14), level(-1), strategy(0), max_dict_bytes(0) {}
-  CompressionOptions(int wbits, int _lev, int _strategy, int _max_dict_bytes)
-      : window_bits(wbits),
-        level(_lev),
-        strategy(_strategy),
-        max_dict_bytes(_max_dict_bytes) {}
-};
-
-enum UpdateStatus {    // Return status For inplace update callback
-  UPDATE_FAILED   = 0, // Nothing to update
-  UPDATED_INPLACE = 1, // Value updated inplace
-  UPDATED         = 2, // No inplace update. Merged value set
-};
-
-struct DbPath {
-  std::string path;
-  uint64_t target_size;  // Target size of total files under the path, in byte.
-
-  DbPath() : target_size(0) {}
-  DbPath(const std::string& p, uint64_t t) : path(p), target_size(t) {}
-};
-
 struct Options;
 
-struct ColumnFamilyOptions {
+struct ColumnFamilyOptions : public AdvancedColumnFamilyOptions {
   // The function recovers options to a previous version. Only 4.6 or later
   // versions are supported.
   ColumnFamilyOptions* OldDefaults(int rocksdb_major_version = 4,
@@ -295,54 +188,6 @@ struct ColumnFamilyOptions {
   // Dynamically changeable through SetOptions() API
   size_t write_buffer_size = 64 << 20;
 
-  // The maximum number of write buffers that are built up in memory.
-  // The default and the minimum number is 2, so that when 1 write buffer
-  // is being flushed to storage, new writes can continue to the other
-  // write buffer.
-  // If max_write_buffer_number > 3, writing will be slowed down to
-  // options.delayed_write_rate if we are writing to the last write buffer
-  // allowed.
-  //
-  // Default: 2
-  //
-  // Dynamically changeable through SetOptions() API
-  int max_write_buffer_number = 2;
-
-  // The minimum number of write buffers that will be merged together
-  // before writing to storage.  If set to 1, then
-  // all write buffers are flushed to L0 as individual files and this increases
-  // read amplification because a get request has to check in all of these
-  // files. Also, an in-memory merge may result in writing lesser
-  // data to storage if there are duplicate records in each of these
-  // individual write buffers.  Default: 1
-  int min_write_buffer_number_to_merge = 1;
-
-  // The total maximum number of write buffers to maintain in memory including
-  // copies of buffers that have already been flushed.  Unlike
-  // max_write_buffer_number, this parameter does not affect flushing.
-  // This controls the minimum amount of write history that will be available
-  // in memory for conflict checking when Transactions are used.
-  //
-  // When using an OptimisticTransactionDB:
-  // If this value is too low, some transactions may fail at commit time due
-  // to not being able to determine whether there were any write conflicts.
-  //
-  // When using a TransactionDB:
-  // If Transaction::SetSnapshot is used, TransactionDB will read either
-  // in-memory write buffers or SST files to do write-conflict checking.
-  // Increasing this value can reduce the number of reads to SST files
-  // done for conflict detection.
-  //
-  // Setting this value to 0 will cause write buffers to be freed immediately
-  // after they are flushed.
-  // If this value is set to -1, 'max_write_buffer_number' will be used.
-  //
-  // Default:
-  // If using a TransactionDB/OptimisticTransactionDB, the default value will
-  // be set to the value of 'max_write_buffer_number' if it is not explicitly
-  // set by the user.  Otherwise, the default is 0.
-  int max_write_buffer_number_to_maintain = 0;
-
   // Compress blocks using the specified compression algorithm.  This
   // parameter can be changed dynamically.
   //
@@ -359,29 +204,6 @@ struct ColumnFamilyOptions {
   // efficiently detect that and will switch to uncompressed mode.
   CompressionType compression;
 
-  // Different levels can have different compression policies. There
-  // are cases where most lower levels would like to use quick compression
-  // algorithms while the higher levels (which have more data) use
-  // compression algorithms that have better compression but could
-  // be slower. This array, if non-empty, should have an entry for
-  // each level of the database; these override the value specified in
-  // the previous field 'compression'.
-  //
-  // NOTICE if level_compaction_dynamic_level_bytes=true,
-  // compression_per_level[0] still determines L0, but other elements
-  // of the array are based on base level (the level L0 files are merged
-  // to), and may not match the level users see from info log for metadata.
-  // If L0 files are merged to level-n, then, for i>0, compression_per_level[i]
-  // determines compaction type for level n+i-1.
-  // For example, if we have three 5 levels, and we determine to merge L0
-  // data to L4 (which means L1..L3 will be empty), then the new files go to
-  // L4 uses compression type compression_per_level[1].
-  // If now L0 is merged to L2. Data goes to L2 will be compressed
-  // according to compression_per_level[1], L3 using compression_per_level[2]
-  // and L4 using compression_per_level[3]. Compaction for each level can
-  // change when data grows.
-  std::vector<CompressionType> compression_per_level;
-
   // Compression algorithm that will be used for the bottommost level that
   // contain files. If level-compaction is used, this option will only affect
   // levels after base level.
@@ -392,6 +214,14 @@ struct ColumnFamilyOptions {
   // different options for compression algorithms
   CompressionOptions compression_opts;
 
+  // Number of files to trigger level-0 compaction. A value <0 means that
+  // level-0 compaction will not be triggered by number of files at all.
+  //
+  // Default: 4
+  //
+  // Dynamically changeable through SetOptions() API
+  int level0_file_num_compaction_trigger = 4;
+
   // If non-nullptr, use the specified function to determine the
   // prefixes for keys.  These prefixes will be placed in the filter.
   // Depending on the workload, this can reduce the number of read-IOP
@@ -408,56 +238,6 @@ struct ColumnFamilyOptions {
   // Default: nullptr
   std::shared_ptr<const SliceTransform> prefix_extractor = nullptr;
 
-  // Number of levels for this database
-  int num_levels = 7;
-
-  // Number of files to trigger level-0 compaction. A value <0 means that
-  // level-0 compaction will not be triggered by number of files at all.
-  //
-  // Default: 4
-  //
-  // Dynamically changeable through SetOptions() API
-  int level0_file_num_compaction_trigger = 4;
-
-  // Soft limit on number of level-0 files. We start slowing down writes at this
-  // point. A value <0 means that no writing slow down will be triggered by
-  // number of files in level-0.
-  //
-  // Default: 20
-  //
-  // Dynamically changeable through SetOptions() API
-  int level0_slowdown_writes_trigger = 20;
-
-  // Maximum number of level-0 files.  We stop writes at this point.
-  //
-  // Default: 36
-  //
-  // Dynamically changeable through SetOptions() API
-  int level0_stop_writes_trigger = 36;
-
-  // This does not do anything anymore. Deprecated.
-  int max_mem_compaction_level;
-
-  // Target file size for compaction.
-  // target_file_size_base is per-file size for level-1.
-  // Target file size for level L can be calculated by
-  // target_file_size_base * (target_file_size_multiplier ^ (L-1))
-  // For example, if target_file_size_base is 2MB and
-  // target_file_size_multiplier is 10, then each file on level-1 will
-  // be 2MB, and each file on level 2 will be 20MB,
-  // and each file on level-3 will be 200MB.
-  //
-  // Default: 64MB.
-  //
-  // Dynamically changeable through SetOptions() API
-  uint64_t target_file_size_base = 64 * 1048576;
-
-  // By default target_file_size_multiplier is 1, which means
-  // by default files in different levels will have similar size.
-  //
-  // Dynamically changeable through SetOptions() API
-  int target_file_size_multiplier = 1;
-
   // Control maximum total data size for a level.
   // max_bytes_for_level_base is the max total for level-1.
   // Maximum number of bytes for level L can be calculated as
@@ -472,371 +252,18 @@ struct ColumnFamilyOptions {
   // Dynamically changeable through SetOptions() API
   uint64_t max_bytes_for_level_base = 256 * 1048576;
 
-  // If true, RocksDB will pick target size of each level dynamically.
-  // We will pick a base level b >= 1. L0 will be directly merged into level b,
-  // instead of always into level 1. Level 1 to b-1 need to be empty.
-  // We try to pick b and its target size so that
-  // 1. target size is in the range of
-  //   (max_bytes_for_level_base / max_bytes_for_level_multiplier,
-  //    max_bytes_for_level_base]
-  // 2. target size of the last level (level num_levels-1) equals to extra size
-  //    of the level.
-  // At the same time max_bytes_for_level_multiplier and
-  // max_bytes_for_level_multiplier_additional are still satisfied.
-  //
-  // With this option on, from an empty DB, we make last level the base level,
-  // which means merging L0 data into the last level, until it exceeds
-  // max_bytes_for_level_base. And then we make the second last level to be
-  // base level, to start to merge L0 data to second last level, with its
-  // target size to be 1/max_bytes_for_level_multiplier of the last level's
-  // extra size. After the data accumulates more so that we need to move the
-  // base level to the third last one, and so on.
-  //
-  // For example, assume max_bytes_for_level_multiplier=10, num_levels=6,
-  // and max_bytes_for_level_base=10MB.
-  // Target sizes of level 1 to 5 starts with:
-  // [- - - - 10MB]
-  // with base level is level. Target sizes of level 1 to 4 are not applicable
-  // because they will not be used.
-  // Until the size of Level 5 grows to more than 10MB, say 11MB, we make
-  // base target to level 4 and now the targets looks like:
-  // [- - - 1.1MB 11MB]
-  // While data are accumulated, size targets are tuned based on actual data
-  // of level 5. When level 5 has 50MB of data, the target is like:
-  // [- - - 5MB 50MB]
-  // Until level 5's actual size is more than 100MB, say 101MB. Now if we keep
-  // level 4 to be the base level, its target size needs to be 10.1MB, which
-  // doesn't satisfy the target size range. So now we make level 3 the target
-  // size and the target sizes of the levels look like:
-  // [- - 1.01MB 10.1MB 101MB]
-  // In the same way, while level 5 further grows, all levels' targets grow,
-  // like
-  // [- - 5MB 50MB 500MB]
-  // Until level 5 exceeds 1000MB and becomes 1001MB, we make level 2 the
-  // base level and make levels' target sizes like this:
-  // [- 1.001MB 10.01MB 100.1MB 1001MB]
-  // and go on...
-  //
-  // By doing it, we give max_bytes_for_level_multiplier a priority against
-  // max_bytes_for_level_base, for a more predictable LSM tree shape. It is
-  // useful to limit worse case space amplification.
-  //
-  // max_bytes_for_level_multiplier_additional is ignored with this flag on.
-  //
-  // Turning this feature on or off for an existing DB can cause unexpected
-  // LSM tree structure so it's not recommended.
-  //
-  // NOTE: this option is experimental
-  //
-  // Default: false
-  bool level_compaction_dynamic_level_bytes = false;
-
-  // Default: 10.
-  //
-  // Dynamically changeable through SetOptions() API
-  double max_bytes_for_level_multiplier = 10;
-
-  // Different max-size multipliers for different levels.
-  // These are multiplied by max_bytes_for_level_multiplier to arrive
-  // at the max-size of each level.
-  //
-  // Default: 1
-  //
-  // Dynamically changeable through SetOptions() API
-  std::vector<int> max_bytes_for_level_multiplier_additional =
-      std::vector<int>(num_levels, 1);
-
-  // We try to limit number of bytes in one compaction to be lower than this
-  // threshold. But it's not guaranteed.
-  // Value 0 will be sanitized.
-  //
-  // Default: result.target_file_size_base * 25
-  uint64_t max_compaction_bytes = 0;
-
-  // DEPRECATED -- this options is no longer used
-  // Puts are delayed to options.delayed_write_rate when any level has a
-  // compaction score that exceeds soft_rate_limit. This is ignored when == 0.0.
-  //
-  // Default: 0 (disabled)
-  //
-  // Dynamically changeable through SetOptions() API
-  double soft_rate_limit = 0.0;
-
-  // DEPRECATED -- this options is no longer used
-  double hard_rate_limit = 0.0;
-
-  // All writes will be slowed down to at least delayed_write_rate if estimated
-  // bytes needed to be compaction exceed this threshold.
-  //
-  // Default: 64GB
-  uint64_t soft_pending_compaction_bytes_limit = 64 * 1073741824ull;
-
-  // All writes are stopped if estimated bytes needed to be compaction exceed
-  // this threshold.
-  //
-  // Default: 256GB
-  uint64_t hard_pending_compaction_bytes_limit = 256 * 1073741824ull;
-
-  // DEPRECATED -- this options is no longer used
-  unsigned int rate_limit_delay_max_milliseconds = 100;
-
-  // size of one block in arena memory allocation.
-  // If <= 0, a proper value is automatically calculated (usually 1/8 of
-  // writer_buffer_size, rounded up to a multiple of 4KB).
-  //
-  // There are two additional restriction of the The specified size:
-  // (1) size should be in the range of [4096, 2 << 30] and
-  // (2) be the multiple of the CPU word (which helps with the memory
-  // alignment).
-  //
-  // We'll automatically check and adjust the size number to make sure it
-  // conforms to the restrictions.
-  //
-  // Default: 0
-  //
-  // Dynamically changeable through SetOptions() API
-  size_t arena_block_size = 0;
-
   // Disable automatic compactions. Manual compactions can still
   // be issued on this column family
   //
   // Dynamically changeable through SetOptions() API
   bool disable_auto_compactions = false;
 
-  // DEPREACTED
-  // Does not have any effect.
-  bool purge_redundant_kvs_while_flush = true;
-
-  // The compaction style. Default: kCompactionStyleLevel
-  CompactionStyle compaction_style = kCompactionStyleLevel;
-
-  // If level compaction_style = kCompactionStyleLevel, for each level,
-  // which files are prioritized to be picked to compact.
-  // Default: kByCompensatedSize
-  CompactionPri compaction_pri = kByCompensatedSize;
-
-  // If true, compaction will verify checksum on every read that happens
-  // as part of compaction
-  //
-  // Default: true
-  //
-  // Dynamically changeable through SetOptions() API
-  bool verify_checksums_in_compaction = true;
-
-  // The options needed to support Universal Style compactions
-  CompactionOptionsUniversal compaction_options_universal;
-
-  // The options for FIFO compaction style
-  CompactionOptionsFIFO compaction_options_fifo;
-
-  // An iteration->Next() sequentially skips over keys with the same
-  // user-key unless this option is set. This number specifies the number
-  // of keys (with the same userkey) that will be sequentially
-  // skipped before a reseek is issued.
-  //
-  // Default: 8
-  //
-  // Dynamically changeable through SetOptions() API
-  uint64_t max_sequential_skip_in_iterations = 8;
-
-  // This is a factory that provides MemTableRep objects.
-  // Default: a factory that provides a skip-list-based implementation of
-  // MemTableRep.
-  std::shared_ptr<MemTableRepFactory> memtable_factory =
-      std::shared_ptr<SkipListFactory>(new SkipListFactory);
-
   // This is a factory that provides TableFactory objects.
   // Default: a block-based table factory that provides a default
   // implementation of TableBuilder and TableReader with default
   // BlockBasedTableOptions.
   std::shared_ptr<TableFactory> table_factory;
 
-  // Block-based table related options are moved to BlockBasedTableOptions.
-  // Related options that were originally here but now moved include:
-  //   no_block_cache
-  //   block_cache
-  //   block_cache_compressed
-  //   block_size
-  //   block_size_deviation
-  //   block_restart_interval
-  //   filter_policy
-  //   whole_key_filtering
-  // If you'd like to customize some of these options, you will need to
-  // use NewBlockBasedTableFactory() to construct a new table factory.
-
-  // This option allows user to collect their own interested statistics of
-  // the tables.
-  // Default: empty vector -- no user-defined statistics collection will be
-  // performed.
-  typedef std::vector<std::shared_ptr<TablePropertiesCollectorFactory>>
-      TablePropertiesCollectorFactories;
-  TablePropertiesCollectorFactories table_properties_collector_factories;
-
-  // Allows thread-safe inplace updates. If this is true, there is no way to
-  // achieve point-in-time consistency using snapshot or iterator (assuming
-  // concurrent updates). Hence iterator and multi-get will return results
-  // which are not consistent as of any point-in-time.
-  // If inplace_callback function is not set,
-  //   Put(key, new_value) will update inplace the existing_value iff
-  //   * key exists in current memtable
-  //   * new sizeof(new_value) <= sizeof(existing_value)
-  //   * existing_value for that key is a put i.e. kTypeValue
-  // If inplace_callback function is set, check doc for inplace_callback.
-  // Default: false.
-  bool inplace_update_support = false;
-
-  // Number of locks used for inplace update
-  // Default: 10000, if inplace_update_support = true, else 0.
-  //
-  // Dynamically changeable through SetOptions() API
-  size_t inplace_update_num_locks = 10000;
-
-  // existing_value - pointer to previous value (from both memtable and sst).
-  //                  nullptr if key doesn't exist
-  // existing_value_size - pointer to size of existing_value).
-  //                       nullptr if key doesn't exist
-  // delta_value - Delta value to be merged with the existing_value.
-  //               Stored in transaction logs.
-  // merged_value - Set when delta is applied on the previous value.
-
-  // Applicable only when inplace_update_support is true,
-  // this callback function is called at the time of updating the memtable
-  // as part of a Put operation, lets say Put(key, delta_value). It allows the
-  // 'delta_value' specified as part of the Put operation to be merged with
-  // an 'existing_value' of the key in the database.
-
-  // If the merged value is smaller in size that the 'existing_value',
-  // then this function can update the 'existing_value' buffer inplace and
-  // the corresponding 'existing_value'_size pointer, if it wishes to.
-  // The callback should return UpdateStatus::UPDATED_INPLACE.
-  // In this case. (In this case, the snapshot-semantics of the rocksdb
-  // Iterator is not atomic anymore).
-
-  // If the merged value is larger in size than the 'existing_value' or the
-  // application does not wish to modify the 'existing_value' buffer inplace,
-  // then the merged value should be returned via *merge_value. It is set by
-  // merging the 'existing_value' and the Put 'delta_value'. The callback should
-  // return UpdateStatus::UPDATED in this case. This merged value will be added
-  // to the memtable.
-
-  // If merging fails or the application does not wish to take any action,
-  // then the callback should return UpdateStatus::UPDATE_FAILED.
-
-  // Please remember that the original call from the application is Put(key,
-  // delta_value). So the transaction log (if enabled) will still contain (key,
-  // delta_value). The 'merged_value' is not stored in the transaction log.
-  // Hence the inplace_callback function should be consistent across db reopens.
-
-  // Default: nullptr
-  UpdateStatus (*inplace_callback)(char* existing_value,
-                                   uint32_t* existing_value_size,
-                                   Slice delta_value,
-                                   std::string* merged_value) = nullptr;
-
-  // if prefix_extractor is set and memtable_prefix_bloom_size_ratio is not 0,
-  // create prefix bloom for memtable with the size of
-  // write_buffer_size * memtable_prefix_bloom_size_ratio.
-  // If it is larger than 0.25, it is santinized to 0.25.
-  //
-  // Default: 0 (disable)
-  //
-  // Dynamically changeable through SetOptions() API
-  double memtable_prefix_bloom_size_ratio = 0.0;
-
-  // Page size for huge page for the arena used by the memtable. If <=0, it
-  // won't allocate from huge page but from malloc.
-  // Users are responsible to reserve huge pages for it to be allocated. For
-  // example:
-  //      sysctl -w vm.nr_hugepages=20
-  // See linux doc Documentation/vm/hugetlbpage.txt
-  // If there isn't enough free huge page available, it will fall back to
-  // malloc.
-  //
-  // Dynamically changeable through SetOptions() API
-  size_t memtable_huge_page_size = 0;
-
-  // If non-nullptr, memtable will use the specified function to extract
-  // prefixes for keys, and for each prefix maintain a hint of insert location
-  // to reduce CPU usage for inserting keys with the prefix. Keys out of
-  // domain of the prefix extractor will be insert without using hints.
-  //
-  // Currently only the default skiplist based memtable implements the feature.
-  // All other memtable implementation will ignore the option. It incurs ~250
-  // additional bytes of memory overhead to store a hint for each prefix.
-  // Also concurrent writes (when allow_concurrent_memtable_write is true) will
-  // ignore the option.
-  //
-  // The option is best suited for workloads where keys will likely to insert
-  // to a location close the the last inserted key with the same prefix.
-  // One example could be inserting keys of the form (prefix + timestamp),
-  // and keys of the same prefix always comes in with time order. Another
-  // example would be updating the same key over and over again, in which case
-  // the prefix can be the key itself.
-  //
-  // Default: nullptr (disable)
-  std::shared_ptr<const SliceTransform>
-      memtable_insert_with_hint_prefix_extractor = nullptr;
-
-  // Control locality of bloom filter probes to improve cache miss rate.
-  // This option only applies to memtable prefix bloom and plaintable
-  // prefix bloom. It essentially limits every bloom checking to one cache line.
-  // This optimization is turned off when set to 0, and positive number to turn
-  // it on.
-  // Default: 0
-  uint32_t bloom_locality = 0;
-
-  // Maximum number of successive merge operations on a key in the memtable.
-  //
-  // When a merge operation is added to the memtable and the maximum number of
-  // successive merges is reached, the value of the key will be calculated and
-  // inserted into the memtable instead of the merge operation. This will
-  // ensure that there are never more than max_successive_merges merge
-  // operations in the memtable.
-  //
-  // Default: 0 (disabled)
-  //
-  // Dynamically changeable through SetOptions() API
-  size_t max_successive_merges = 0;
-
-  // The number of partial merge operands to accumulate before partial
-  // merge will be performed. Partial merge will not be called
-  // if the list of values to merge is less than min_partial_merge_operands.
-  //
-  // If min_partial_merge_operands < 2, then it will be treated as 2.
-  //
-  // Default: 2
-  uint32_t min_partial_merge_operands = 2;
-
-  // This flag specifies that the implementation should optimize the filters
-  // mainly for cases where keys are found rather than also optimize for keys
-  // missed. This would be used in cases where the application knows that
-  // there are very few misses or the performance in the case of misses is not
-  // important.
-  //
-  // For now, this flag allows us to not store filters for the last level i.e
-  // the largest level which contains data of the LSM store. For keys which
-  // are hits, the filters in this level are not useful because we will search
-  // for the data anyway. NOTE: the filters in other levels are still useful
-  // even for key hit because they tell us whether to look in that level or go
-  // to the higher level.
-  //
-  // Default: false
-  bool optimize_filters_for_hits = false;
-
-  // After writing every SST file, reopen it and read all the keys.
-  // Default: false
-  bool paranoid_file_checks = false;
-
-  // In debug mode, RocksDB run consistency checks on the LSM everytime the LSM
-  // change (Flush, Compaction, AddFile). These checks are disabled in release
-  // mode, use this option to enable them in release mode as well.
-  // Default: false
-  bool force_consistency_checks = false;
-
-  // Measure IO stats in compactions and flushes, if true.
-  // Default: false
-  bool report_bg_io_stats = false;
-
   // Create ColumnFamilyOptions with default values for all fields
   ColumnFamilyOptions();
   // Create ColumnFamilyOptions from Options
@@ -845,6 +272,38 @@ struct ColumnFamilyOptions {
   void Dump(Logger* log) const;
 };
 
+enum class WALRecoveryMode : char {
+  // Original levelDB recovery
+  // We tolerate incomplete record in trailing data on all logs
+  // Use case : This is legacy behavior (default)
+  kTolerateCorruptedTailRecords = 0x00,
+  // Recover from clean shutdown
+  // We don't expect to find any corruption in the WAL
+  // Use case : This is ideal for unit tests and rare applications that
+  // can require high consistency guarantee
+  kAbsoluteConsistency = 0x01,
+  // Recover to point-in-time consistency
+  // We stop the WAL playback on discovering WAL inconsistency
+  // Use case : Ideal for systems that have disk controller cache like
+  // hard disk, SSD without super capacitor that store related data
+  kPointInTimeRecovery = 0x02,
+  // Recovery after a disaster
+  // We ignore any corruption in the WAL and try to salvage as much data as
+  // possible
+  // Use case : Ideal for last ditch effort to recover data or systems that
+  // operate with low grade unrelated data
+  kSkipAnyCorruptedRecords = 0x03,
+};
+
+struct DbPath {
+  std::string path;
+  uint64_t target_size;  // Target size of total files under the path, in byte.
+
+  DbPath() : target_size(0) {}
+  DbPath(const std::string& p, uint64_t t) : path(p), target_size(t) {}
+};
+
+
 struct DBOptions {
   // The function recovers options to the option as in version 4.6.
   DBOptions* OldDefaults(int rocksdb_major_version = 4,
@@ -948,14 +407,6 @@ struct DBOptions {
   // If non-null, then we should collect metrics about database operations
   std::shared_ptr<Statistics> statistics = nullptr;
 
-  // If true, then the contents of manifest and data files are not synced
-  // to stable storage. Their contents remain in the OS buffers till the
-  // OS decides to flush them. This option is good for bulk-loading
-  // of data. Once the bulk-loading is complete, please issue a
-  // sync to the OS to flush all dirty buffers to stable storage.
-  // Default: false
-  bool disableDataSync = false;
-
   // If true, then every store to stable storage will issue a fsync.
   // If false, then every store to stable storage will issue a fdatasync.
   // This parameter should be set to true while storing data to
@@ -1063,6 +514,7 @@ struct DBOptions {
   // If specified with non-zero value, log file will be rolled
   // if it has been active longer than `log_file_time_to_roll`.
   // Default: 0 (disabled)
+  // Not supported in ROCKSDB_LITE mode!
   size_t log_file_time_to_roll = 0;
 
   // Maximal info log files to be kept.
@@ -1129,10 +581,12 @@ struct DBOptions {
 
   // Use O_DIRECT for reading file
   // Default: false
+  // Not supported in ROCKSDB_LITE mode!
   bool use_direct_reads = false;
 
   // Use O_DIRECT for writing file
   // Default: false
+  // Not supported in ROCKSDB_LITE mode!
   bool use_direct_writes = false;
 
   // If false, fallocate() calls are bypassed
@@ -1434,10 +888,11 @@ struct Options : public DBOptions, public ColumnFamilyOptions {
 enum ReadTier {
   kReadAllTier = 0x0,     // data in memtable, block cache, OS cache or storage
   kBlockCacheTier = 0x1,  // data in memtable or block cache
-  kPersistedTier = 0x2    // persisted data.  When WAL is disabled, this option
+  kPersistedTier = 0x2,   // persisted data.  When WAL is disabled, this option
                           // will skip data in memtable.
                           // Note that this ReadTier currently only supports
                           // Get and MultiGet and does not support iterators.
+  kMemtableTier = 0x3     // data in memtable. used for memtable-only iterators.
 };
 
 // Options that control read operations
@@ -1585,9 +1040,6 @@ struct WriteOptions {
   // and the write may got lost after a crash.
   bool disableWAL;
 
-  // The option is deprecated. It's not used anymore.
-  uint64_t timeout_hint_us;
-
   // If true and if user is trying to write to column families that don't exist
   // (they were dropped),  ignore the write (don't return an error). If there
   // are multiple writes in a WriteBatch, other writes will succeed.
@@ -1601,7 +1053,6 @@ struct WriteOptions {
   WriteOptions()
       : sync(false),
         disableWAL(false),
-        timeout_hint_us(0),
         ignore_missing_column_families(false),
         no_slowdown(false) {}
 };

include/rocksdb/rate_limiter.h

@@ -10,6 +10,7 @@
 #pragma once
 
 #include "rocksdb/env.h"
+#include "rocksdb/statistics.h"
 
 namespace rocksdb {
 
@@ -24,7 +25,21 @@ class RateLimiter {
   // Request for token to write bytes. If this request can not be satisfied,
   // the call is blocked. Caller is responsible to make sure
   // bytes <= GetSingleBurstBytes()
-  virtual void Request(const int64_t bytes, const Env::IOPriority pri) = 0;
+  virtual void Request(const int64_t bytes, const Env::IOPriority pri) {
+    // Deprecated. New RateLimiter derived classes should override
+    // Request(const int64_t, const Env::IOPriority, Statistics*) instead.
+    assert(false);
+  }
+
+  // Request for token to write bytes and potentially update statistics. If this
+  // request can not be satisfied, the call is blocked. Caller is responsible to
+  // make sure bytes <= GetSingleBurstBytes().
+  virtual void Request(const int64_t bytes, const Env::IOPriority pri,
+                       Statistics* /* stats */) {
+    // For API compatibility, default implementation calls the older API in
+    // which statistics are unsupported.
+    Request(bytes, pri);
+  }
 
   // Max bytes can be granted in a single burst
   virtual int64_t GetSingleBurstBytes() const = 0;

include/rocksdb/sst_file_manager.h

@@ -50,6 +50,11 @@ class SstFileManager {
   // Return delete rate limit in bytes per second.
   // thread-safe
   virtual int64_t GetDeleteRateBytesPerSecond() = 0;
+
+  // Update the delete rate limit in bytes per second.
+  // zero means disable delete rate limiting and delete files immediately
+  // thread-safe
+  virtual void SetDeleteRateBytesPerSecond(int64_t delete_rate) = 0;
 };
 
 // Create a new SstFileManager that can be shared among multiple RocksDB

include/rocksdb/sst_file_writer.h

@@ -3,6 +3,8 @@
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
 
+#ifndef ROCKSDB_LITE
+
 #pragma once
 #include <string>
 #include "rocksdb/env.h"
@@ -79,3 +81,5 @@ class SstFileWriter {
   Rep* rep_;
 };
 }  // namespace rocksdb
+
+#endif  // !ROCKSDB_LITE

include/rocksdb/statistics.h

@@ -216,6 +216,9 @@ enum Tickers : uint32_t {
   READ_AMP_ESTIMATE_USEFUL_BYTES,  // Estimate of total bytes actually used.
   READ_AMP_TOTAL_READ_BYTES,       // Total size of loaded data blocks.
 
+  // Number of refill intervals where rate limiter's bytes are fully consumed.
+  NUMBER_RATE_LIMITER_DRAINS,
+
   TICKER_ENUM_MAX
 };
 
@@ -318,6 +321,7 @@ const std::vector<std::pair<Tickers, std::string>> TickersNameMap = {
     {ROW_CACHE_MISS, "rocksdb.row.cache.miss"},
     {READ_AMP_ESTIMATE_USEFUL_BYTES, "rocksdb.read.amp.estimate.useful.bytes"},
     {READ_AMP_TOTAL_READ_BYTES, "rocksdb.read.amp.total.read.bytes"},
+    {NUMBER_RATE_LIMITER_DRAINS, "rocksdb.number.rate_limiter.drains"},
 };
 
 /**
@@ -407,6 +411,9 @@ struct HistogramData {
   double percentile99;
   double average;
   double standard_deviation;
+  // zero-initialize new members since old Statistics::histogramData()
+  // implementations won't write them.
+  double max = 0.0;
 };
 
 enum StatsLevel {

include/rocksdb/status.h

@@ -70,6 +70,7 @@ class Status {
     kLockLimit = 3,
     kNoSpace = 4,
     kDeadlock = 5,
+    kStaleFile = 6,
     kMaxSubCode
   };
 

include/rocksdb/table.h

@@ -148,6 +148,15 @@ struct BlockBasedTableOptions {
   // i.e., the number of data blocks covered by each index partition
   uint64_t index_per_partition = 1024;
 
+  // Note: currently this option requires kTwoLevelIndexSearch to be set as
+  // well.
+  // TODO(myabandeh): remove the note above once the limitation is lifted
+  // TODO(myabandeh): this feature is in experimental phase and shall not be
+  // used in production; either remove the feature or remove this comment if
+  // it is ready to be used in production.
+  // Use partitioned full filters for each SST file
+  bool partition_filters = false;
+
   // Use delta encoding to compress keys in blocks.
   // ReadOptions::pin_data requires this option to be disabled.
   //
@@ -163,20 +172,6 @@ struct BlockBasedTableOptions {
   // This must generally be true for gets to be efficient.
   bool whole_key_filtering = true;
 
-  // If true, block will not be explicitly flushed to disk during building
-  // a SstTable. Instead, buffer in WritableFileWriter will take
-  // care of the flushing when it is full.
-  //
-  // This option helps a lot when direct I/O writes
-  // (use_direct_writes = true) is used, since it avoids small
-  // direct disk write.
-  //
-  // User may also adjust writable_file_max_buffer_size to optimize disk I/O
-  // size.
-  //
-  // Default: false
-  bool skip_table_builder_flush = false;
-
   // Verify that decompressing the compressed block gives back the input. This
   // is a verification mode that we use to detect bugs in compression
   // algorithms.

include/rocksdb/utilities/backupable_db.h

@@ -227,6 +227,12 @@ class BackupEngineReadOnly {
 
   // checks that each file exists and that the size of the file matches our
   // expectations. it does not check file checksum.
+  //
+  // If this BackupEngine created the backup, it compares the files' current
+  // sizes against the number of bytes written to them during creation.
+  // Otherwise, it compares the files' current sizes against their sizes when
+  // the BackupEngine was opened.
+  //
   // Returns Status::OK() if all checks are good
   virtual Status VerifyBackup(BackupID backup_id) = 0;
 };

include/rocksdb/utilities/checkpoint.h

@@ -27,7 +27,14 @@ class Checkpoint {
   // (2) a copied manifest files and other files
   // The directory should not already exist and will be created by this API.
   // The directory will be an absolute path
-  virtual Status CreateCheckpoint(const std::string& checkpoint_dir);
+  // log_size_for_flush: if the total log file size is equal or larger than
+  // this value, then a flush is triggered for all the column families. The
+  // default value is 0, which means flush is always triggered. If you move
+  // away from the default, the checkpoint may not contain up-to-date data
+  // if WAL writing is not always enabled.
+  // Flush will always trigger if it is 2PC.
+  virtual Status CreateCheckpoint(const std::string& checkpoint_dir,
+                                  uint64_t log_size_for_flush = 0);
 
   virtual ~Checkpoint() {}
 };

include/rocksdb/utilities/write_batch_with_index.h

@@ -12,6 +12,7 @@
 
 #ifndef ROCKSDB_LITE
 
+#include <memory>
 #include <string>
 
 #include "rocksdb/comparator.h"
@@ -205,7 +206,7 @@ class WriteBatchWithIndex : public WriteBatchBase {
 
  private:
   struct Rep;
-  Rep* rep;
+  std::unique_ptr<Rep> rep;
 };
 
 }  // namespace rocksdb

include/rocksdb/version.h

@@ -5,8 +5,8 @@
 #pragma once
 
 #define ROCKSDB_MAJOR 5
-#define ROCKSDB_MINOR 2
-#define ROCKSDB_PATCH 1
+#define ROCKSDB_MINOR 3
+#define ROCKSDB_PATCH 6
 
 // Do not use these. We made the mistake of declaring macros starting with
 // double underscore. Now we have to live with our choice. We'll deprecate these

java/Makefile

@@ -18,7 +18,6 @@ NATIVE_JAVA_CLASSES = org.rocksdb.AbstractCompactionFilter\
 	org.rocksdb.ExternalSstFileInfo\
 	org.rocksdb.FlushOptions\
 	org.rocksdb.Filter\
-	org.rocksdb.GenericRateLimiterConfig\
 	org.rocksdb.HashLinkedListMemTableConfig\
 	org.rocksdb.HashSkipListMemTableConfig\
 	org.rocksdb.Logger\
@@ -91,6 +90,7 @@ JAVA_TESTS = org.rocksdb.BackupableDBOptionsTest\
         org.rocksdb.NativeLibraryLoaderTest\
 	org.rocksdb.OptionsTest\
 	org.rocksdb.PlainTableConfigTest\
+	org.rocksdb.RateLimiterTest\
 	org.rocksdb.ReadOnlyTest\
 	org.rocksdb.ReadOptionsTest\
 	org.rocksdb.RocksDBTest\
@@ -136,6 +136,14 @@ JAVA_TESTCLASSPATH = $(JAVA_JUNIT_JAR):$(JAVA_HAMCR_JAR):$(JAVA_MOCKITO_JAR):$(J
 
 MVN_LOCAL = ~/.m2/repository
 
+# Set the default JAVA_ARGS to "" for DEBUG_LEVEL=0
+JAVA_ARGS? =
+
+# When debugging add -Xcheck:jni to the java args
+ifneq ($(DEBUG_LEVEL),0)
+	JAVA_ARGS = -ea -Xcheck:jni
+endif
+
 clean:
 	$(AM_V_at)rm -rf include/*
 	$(AM_V_at)rm -rf test-libs/
@@ -164,7 +172,7 @@ sample: java
 	$(AM_V_at)javac -cp $(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/RocksDBSample.java
 	$(AM_V_at)@rm -rf /tmp/rocksdbjni
 	$(AM_V_at)@rm -rf /tmp/rocksdbjni_not_found
-	java -ea -Xcheck:jni -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) RocksDBSample /tmp/rocksdbjni
+	java $(JAVA_ARGS) -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) RocksDBSample /tmp/rocksdbjni
 	$(AM_V_at)@rm -rf /tmp/rocksdbjni
 	$(AM_V_at)@rm -rf /tmp/rocksdbjni_not_found
 
@@ -172,7 +180,7 @@ column_family_sample: java
 	$(AM_V_GEN)mkdir -p $(SAMPLES_MAIN_CLASSES)
 	$(AM_V_at)javac -cp $(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/RocksDBColumnFamilySample.java
 	$(AM_V_at)@rm -rf /tmp/rocksdbjni
-	java -ea -Xcheck:jni -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) RocksDBColumnFamilySample /tmp/rocksdbjni
+	java $(JAVA_ARGS) -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) RocksDBColumnFamilySample /tmp/rocksdbjni
 	$(AM_V_at)@rm -rf /tmp/rocksdbjni
 
 resolve_test_deps:
@@ -194,7 +202,7 @@ java_test: java resolve_test_deps
 test: java java_test run_test
 
 run_test:
-	java -ea -Djava.library.path=target -cp "$(MAIN_CLASSES):$(TEST_CLASSES):$(JAVA_TESTCLASSPATH):target/*" org.rocksdb.test.RocksJunitRunner $(JAVA_TESTS)
+	java $(JAVA_ARGS) -Djava.library.path=target -cp "$(MAIN_CLASSES):$(TEST_CLASSES):$(JAVA_TESTCLASSPATH):target/*" org.rocksdb.test.RocksJunitRunner $(JAVA_TESTS)
 
 db_bench: java
 	$(AM_V_GEN)mkdir -p $(BENCHMARK_MAIN_CLASSES)

java/benchmark/src/main/java/org/rocksdb/benchmark/DbBenchmark.java

@@ -545,8 +545,6 @@ public class DbBenchmark {
         (Integer)flags_.get(Flag.max_background_flushes));
     options.setMaxOpenFiles(
         (Integer)flags_.get(Flag.open_files));
-    options.setDisableDataSync(
-        (Boolean)flags_.get(Flag.disable_data_sync));
     options.setUseFsync(
         (Boolean)flags_.get(Flag.use_fsync));
     options.setWalDir(
@@ -574,7 +572,7 @@ public class DbBenchmark {
         (Integer)flags_.get(Flag.num_levels));
     options.setTargetFileSizeBase(
         (Integer)flags_.get(Flag.target_file_size_base));
-    options.setTargetFileSizeMultiplier((Double) flags_.get(Flag.target_file_size_multiplier));
+    options.setTargetFileSizeMultiplier((Integer)flags_.get(Flag.target_file_size_multiplier));
     options.setMaxBytesForLevelBase(
         (Integer)flags_.get(Flag.max_bytes_for_level_base));
     options.setMaxBytesForLevelMultiplier((Double) flags_.get(Flag.max_bytes_for_level_multiplier));
@@ -590,12 +588,10 @@ public class DbBenchmark {
         (Double)flags_.get(Flag.hard_rate_limit));
     options.setRateLimitDelayMaxMilliseconds(
         (Integer)flags_.get(Flag.rate_limit_delay_max_milliseconds));
-    options.setMaxGrandparentOverlapFactor(
-        (Integer)flags_.get(Flag.max_grandparent_overlap_factor));
+    options.setMaxCompactionBytes(
+        (Long) flags_.get(Flag.max_compaction_bytes));
     options.setDisableAutoCompactions(
         (Boolean)flags_.get(Flag.disable_auto_compactions));
-    options.setSourceCompactionFactor(
-        (Integer)flags_.get(Flag.source_compaction_factor));
     options.setMaxSuccessiveMerges(
         (Integer)flags_.get(Flag.max_successive_merges));
     options.setWalTtlSeconds((Long)flags_.get(Flag.wal_ttl_seconds));
@@ -980,7 +976,7 @@ public class DbBenchmark {
         return Integer.parseInt(value);
       }
     },
-    write_buffer_size(4 * SizeUnit.MB,
+    write_buffer_size(4L * SizeUnit.MB,
         "Number of bytes to buffer in memtable before compacting\n" +
         "\t(initialized to default value by 'main'.)") {
       @Override public Object parseValue(String value) {
@@ -1058,7 +1054,7 @@ public class DbBenchmark {
         return Integer.parseInt(value);
       }
     },
-    numdistinct(1000,
+    numdistinct(1000L,
         "Number of distinct keys to use. Used in RandomWithVerify to\n" +
         "\tread/write on fewer keys so that gets are more likely to find the\n" +
         "\tkey and puts are more likely to update the same key.") {
@@ -1066,7 +1062,7 @@ public class DbBenchmark {
         return Long.parseLong(value);
       }
     },
-    merge_keys(-1,
+    merge_keys(-1L,
         "Number of distinct keys to use for MergeRandom and\n" +
         "\tReadRandomMergeRandom.\n" +
         "\tIf negative, there will be FLAGS_num keys.") {
@@ -1171,7 +1167,7 @@ public class DbBenchmark {
         return Long.parseLong(value);
       }
     },
-    compressed_cache_size(-1,
+    compressed_cache_size(-1L,
         "Number of bytes to use as a cache of compressed data.") {
       @Override public Object parseValue(String value) {
         return Long.parseLong(value);
@@ -1190,7 +1186,7 @@ public class DbBenchmark {
         return Integer.parseInt(value);
       }
     },
-    memtable_bloom_size_ratio(0, "Ratio of memtable used by the bloom filter.\n"
+    memtable_bloom_size_ratio(0.0d, "Ratio of memtable used by the bloom filter.\n"
             + "\t0 means no bloom filter.") {
       @Override public Object parseValue(String value) {
         return Double.parseDouble(value);
@@ -1214,7 +1210,7 @@ public class DbBenchmark {
         return parseBoolean(value);
       }
     },
-    writes(-1,"Number of write operations to do. If negative, do\n" +
+    writes(-1L, "Number of write operations to do. If negative, do\n" +
         "\t--num reads.") {
       @Override public Object parseValue(String value) {
         return Long.parseLong(value);
@@ -1225,12 +1221,6 @@ public class DbBenchmark {
         return parseBoolean(value);
       }
     },
-    disable_data_sync(false,"If true, do not wait until data is\n" +
-        "\tsynced to disk.") {
-      @Override public Object parseValue(String value) {
-        return parseBoolean(value);
-      }
-    },
     use_fsync(false,"If true, issue fsync instead of fdatasync.") {
       @Override public Object parseValue(String value) {
         return parseBoolean(value);
@@ -1263,7 +1253,7 @@ public class DbBenchmark {
         return Integer.parseInt(value);
       }
     },
-    max_bytes_for_level_multiplier(10,
+    max_bytes_for_level_multiplier(10.0d,
         "A multiplier to compute max bytes for level-N (N >= 2)") {
       @Override public Object parseValue(String value) {
         return Double.parseDouble(value);
@@ -1345,7 +1335,7 @@ public class DbBenchmark {
         return Integer.parseInt(value);
       }
     },
-    stats_interval(0,"Stats are reported every N operations when\n" +
+    stats_interval(0L, "Stats are reported every N operations when\n" +
         "\tthis is greater than zero. When 0 the interval grows over time.") {
       @Override public Object parseValue(String value) {
         return Long.parseLong(value);
@@ -1362,12 +1352,12 @@ public class DbBenchmark {
         return Integer.parseInt(value);
       }
     },
-    soft_rate_limit(0.0,"") {
+    soft_rate_limit(0.0d,"") {
       @Override public Object parseValue(String value) {
         return Double.parseDouble(value);
       }
     },
-    hard_rate_limit(0.0,"When not equal to 0 this make threads\n" +
+    hard_rate_limit(0.0d,"When not equal to 0 this make threads\n" +
         "\tsleep at each stats reporting interval until the compaction\n" +
         "\tscore for all levels is less than or equal to this value.") {
       @Override public Object parseValue(String value) {
@@ -1381,11 +1371,10 @@ public class DbBenchmark {
         return Integer.parseInt(value);
       }
     },
-    max_grandparent_overlap_factor(10,"Control maximum bytes of\n" +
-        "\toverlaps in grandparent (i.e., level+2) before we stop building a\n" +
-        "\tsingle file in a level->level+1 compaction.") {
+    max_compaction_bytes(0L, "Limit number of bytes in one compaction to be lower than this\n" +
+            "\threshold. But it's not guaranteed.") {
       @Override public Object parseValue(String value) {
-        return Integer.parseInt(value);
+        return Long.parseLong(value);
       }
     },
     readonly(false,"Run read only benchmarks.") {
@@ -1398,13 +1387,6 @@ public class DbBenchmark {
         return parseBoolean(value);
       }
     },
-    source_compaction_factor(1,"Cap the size of data in level-K for\n" +
-        "\ta compaction run that compacts Level-K with Level-(K+1) (for\n" +
-        "\tK >= 1)") {
-      @Override public Object parseValue(String value) {
-        return Integer.parseInt(value);
-      }
-    },
     wal_ttl_seconds(0L,"Set the TTL for the WAL Files in seconds.") {
       @Override public Object parseValue(String value) {
         return Long.parseLong(value);

java/crossbuild/Vagrantfile

@@ -7,11 +7,11 @@ VAGRANTFILE_API_VERSION = "2"
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
   config.vm.define "linux32" do |linux32|
-    linux32.vm.box = "hansode/centos-5.6-i386"
+    linux32.vm.box = "hansode/centos-5.11-i386"
   end
 
   config.vm.define "linux64" do |linux64|
-    linux64.vm.box = "hansode/centos-5.6-x86_64"
+    linux64.vm.box = "hansode/centos-5.11-x86_64"
   end
 
   config.vm.provider "virtualbox" do |v|

java/crossbuild/build-linux-centos.sh

@@ -1,17 +1,35 @@
 #!/usr/bin/env bash
+
+set -e
+
+# CentOS 5 is now end of life so we need to switch its repos over to the vault
+sed -i -e '/^mirrorlist=/d' -e 's%^#baseurl=http://mirror.centos.org/centos/$releasever/%baseurl=http://vault.centos.org/5.11/%g' /etc/yum.repos.d/CentOS-Base.repo
+sed -i -e '/^mirrorlist=/d' -e 's%^#baseurl=http://mirror.centos.org/centos/$releasever/%baseurl=http://vault.centos.org/5.11/%g' /etc/yum.repos.d/CentOS-fasttrack.repo
+if [ -f /etc/yum.repos.d/libselinux.repo ]; then
+	sed -i -e '/^mirrorlist=/d' -e 's%^#baseurl=http://mirror.centos.org/centos/$releasever/%baseurl=http://vault.centos.org/5.11/%g' /etc/yum.repos.d/libselinux.repo
+fi
+
+# enable EPEL
+sudo yum -y install epel-release
+
 # install all required packages for rocksdb that are available through yum
 ARCH=$(uname -i)
-sudo yum -y install openssl java-1.7.0-openjdk-devel.$ARCH 
+sudo yum -y install openssl java-1.7.0-openjdk-devel.$ARCH zlib-devel bzip2-devel lz4-devel snappy-devel
 
 # install gcc/g++ 4.8.2 via CERN (http://linux.web.cern.ch/linux/devtoolset/)
 sudo wget -O /etc/yum.repos.d/slc5-devtoolset.repo http://linuxsoft.cern.ch/cern/devtoolset/slc5-devtoolset.repo
-sudo wget -O /etc/pki/rpm-gpg/RPM-GPG-KEY-cern http://ftp.mirrorservice.org/sites/ftp.scientificlinux.org/linux/scientific/51/i386/RPM-GPG-KEYs/RPM-GPG-KEY-cern
+sudo wget -O /etc/pki/rpm-gpg/RPM-GPG-KEY-cern http://ftp.mirrorservice.org/sites/ftp.scientificlinux.org/linux/scientific/obsolete/51/i386/RPM-GPG-KEYs/RPM-GPG-KEY-cern
 sudo yum -y install devtoolset-2
 
-wget http://gflags.googlecode.com/files/gflags-2.0-no-svn-files.tar.gz
-tar xvfz gflags-2.0-no-svn-files.tar.gz; cd gflags-2.0; scl enable devtoolset-2 ./configure; scl enable devtoolset-2 make; sudo make install
+# install gflags
+wget https://github.com/gflags/gflags/archive/v2.0.tar.gz -O gflags-2.0.tar.gz
+tar xvfz gflags-2.0.tar.gz; cd gflags-2.0; scl enable devtoolset-2 ./configure; scl enable devtoolset-2 make; sudo make install
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
 
+# install zstandard
+wget https://github.com/facebook/zstd/archive/v1.1.3.tar.gz -O zstd-1.1.3.tar.gz
+tar zxvf zstd-1.1.3.tar.gz; cd zstd-1.1.3; scl enable devtoolset-2 make; sudo make install
+
 # set java home so we can build rocksdb jars
 export JAVA_HOME=/usr/lib/jvm/java-1.7.0
 

java/crossbuild/docker-build-linux-centos.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -e
+
+rm -rf /rocksdb-local
+cp -r /rocksdb-host /rocksdb-local
+cd /rocksdb-local
+scl enable devtoolset-2 'make jclean clean'
+scl enable devtoolset-2 'PORTABLE=1 make rocksdbjavastatic'
+cp java/target/librocksdbjni-linux*.so java/target/rocksdbjni-*-linux*.jar /rocksdb-host/java/target
+

java/rocksjni/backupablejni.cc

@@ -27,8 +27,12 @@
  */
 jlong Java_org_rocksdb_BackupableDBOptions_newBackupableDBOptions(
     JNIEnv* env, jclass jcls, jstring jpath) {
-  const char* cpath = env->GetStringUTFChars(jpath, NULL);
-  auto bopt = new rocksdb::BackupableDBOptions(cpath);
+  const char* cpath = env->GetStringUTFChars(jpath, nullptr);
+  if(cpath == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return 0;
+  }
+  auto* bopt = new rocksdb::BackupableDBOptions(cpath);
   env->ReleaseStringUTFChars(jpath, cpath);
   return reinterpret_cast<jlong>(bopt);
 }
@@ -40,7 +44,7 @@ jlong Java_org_rocksdb_BackupableDBOptions_newBackupableDBOptions(
  */
 jstring Java_org_rocksdb_BackupableDBOptions_backupDir(
     JNIEnv* env, jobject jopt, jlong jhandle) {
-  auto bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
+  auto* bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
   return env->NewStringUTF(bopt->backup_dir.c_str());
 }
 
@@ -51,7 +55,7 @@ jstring Java_org_rocksdb_BackupableDBOptions_backupDir(
  */
 void Java_org_rocksdb_BackupableDBOptions_setShareTableFiles(
     JNIEnv* env, jobject jobj, jlong jhandle, jboolean flag) {
-  auto bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
+  auto* bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
   bopt->share_table_files = flag;
 }
 
@@ -62,7 +66,7 @@ void Java_org_rocksdb_BackupableDBOptions_setShareTableFiles(
  */
 jboolean Java_org_rocksdb_BackupableDBOptions_shareTableFiles(
     JNIEnv* env, jobject jobj, jlong jhandle) {
-  auto bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
+  auto* bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
   return bopt->share_table_files;
 }
 
@@ -73,7 +77,7 @@ jboolean Java_org_rocksdb_BackupableDBOptions_shareTableFiles(
  */
 void Java_org_rocksdb_BackupableDBOptions_setSync(
     JNIEnv* env, jobject jobj, jlong jhandle, jboolean flag) {
-  auto bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
+  auto* bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
   bopt->sync = flag;
 }
 
@@ -84,7 +88,7 @@ void Java_org_rocksdb_BackupableDBOptions_setSync(
  */
 jboolean Java_org_rocksdb_BackupableDBOptions_sync(
     JNIEnv* env, jobject jobj, jlong jhandle) {
-  auto bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
+  auto* bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
   return bopt->sync;
 }
 
@@ -95,7 +99,7 @@ jboolean Java_org_rocksdb_BackupableDBOptions_sync(
  */
 void Java_org_rocksdb_BackupableDBOptions_setDestroyOldData(
     JNIEnv* env, jobject jobj, jlong jhandle, jboolean flag) {
-  auto bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
+  auto* bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
   bopt->destroy_old_data = flag;
 }
 
@@ -106,7 +110,7 @@ void Java_org_rocksdb_BackupableDBOptions_setDestroyOldData(
  */
 jboolean Java_org_rocksdb_BackupableDBOptions_destroyOldData(
     JNIEnv* env, jobject jobj, jlong jhandle) {
-  auto bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
+  auto* bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
   return bopt->destroy_old_data;
 }
 
@@ -117,7 +121,7 @@ jboolean Java_org_rocksdb_BackupableDBOptions_destroyOldData(
  */
 void Java_org_rocksdb_BackupableDBOptions_setBackupLogFiles(
     JNIEnv* env, jobject jobj, jlong jhandle, jboolean flag) {
-  auto bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
+  auto* bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
   bopt->backup_log_files = flag;
 }
 
@@ -128,7 +132,7 @@ void Java_org_rocksdb_BackupableDBOptions_setBackupLogFiles(
  */
 jboolean Java_org_rocksdb_BackupableDBOptions_backupLogFiles(
     JNIEnv* env, jobject jobj, jlong jhandle) {
-  auto bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
+  auto* bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
   return bopt->backup_log_files;
 }
 
@@ -139,7 +143,7 @@ jboolean Java_org_rocksdb_BackupableDBOptions_backupLogFiles(
  */
 void Java_org_rocksdb_BackupableDBOptions_setBackupRateLimit(
     JNIEnv* env, jobject jobj, jlong jhandle, jlong jbackup_rate_limit) {
-  auto bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
+  auto* bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
   bopt->backup_rate_limit = jbackup_rate_limit;
 }
 
@@ -150,7 +154,7 @@ void Java_org_rocksdb_BackupableDBOptions_setBackupRateLimit(
  */
 jlong Java_org_rocksdb_BackupableDBOptions_backupRateLimit(
     JNIEnv* env, jobject jobj, jlong jhandle) {
-  auto bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
+  auto* bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
   return bopt->backup_rate_limit;
 }
 
@@ -161,7 +165,7 @@ jlong Java_org_rocksdb_BackupableDBOptions_backupRateLimit(
  */
 void Java_org_rocksdb_BackupableDBOptions_setRestoreRateLimit(
     JNIEnv* env, jobject jobj, jlong jhandle, jlong jrestore_rate_limit) {
-  auto bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
+  auto* bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
   bopt->restore_rate_limit = jrestore_rate_limit;
 }
 
@@ -172,7 +176,7 @@ void Java_org_rocksdb_BackupableDBOptions_setRestoreRateLimit(
  */
 jlong Java_org_rocksdb_BackupableDBOptions_restoreRateLimit(
     JNIEnv* env, jobject jobj, jlong jhandle) {
-  auto bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
+  auto* bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
   return bopt->restore_rate_limit;
 }
 
@@ -183,7 +187,7 @@ jlong Java_org_rocksdb_BackupableDBOptions_restoreRateLimit(
  */
 void Java_org_rocksdb_BackupableDBOptions_setShareFilesWithChecksum(
     JNIEnv* env, jobject jobj, jlong jhandle, jboolean flag) {
-  auto bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
+  auto* bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
   bopt->share_files_with_checksum = flag;
 }
 
@@ -194,7 +198,7 @@ void Java_org_rocksdb_BackupableDBOptions_setShareFilesWithChecksum(
  */
 jboolean Java_org_rocksdb_BackupableDBOptions_shareFilesWithChecksum(
     JNIEnv* env, jobject jobj, jlong jhandle) {
-  auto bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
+  auto* bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
   return bopt->share_files_with_checksum;
 }
 
@@ -205,7 +209,7 @@ jboolean Java_org_rocksdb_BackupableDBOptions_shareFilesWithChecksum(
  */
 void Java_org_rocksdb_BackupableDBOptions_disposeInternal(
     JNIEnv* env, jobject jopt, jlong jhandle) {
-  auto bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
-  assert(bopt);
+  auto* bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
+  assert(bopt != nullptr);
   delete bopt;
 }

java/rocksjni/backupenginejni.cc

@@ -82,11 +82,15 @@ jintArray Java_org_rocksdb_BackupEngine_getCorruptedBackups(
   backup_engine->GetCorruptedBackups(&backup_ids);
   // store backupids in int array
   std::vector<jint> int_backup_ids(backup_ids.begin(), backup_ids.end());
+  
   // Store ints in java array
-  jintArray ret_backup_ids;
   // Its ok to loose precision here (64->32)
   jsize ret_backup_ids_size = static_cast<jsize>(backup_ids.size());
-  ret_backup_ids = env->NewIntArray(ret_backup_ids_size);
+  jintArray ret_backup_ids = env->NewIntArray(ret_backup_ids_size);
+  if(ret_backup_ids == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return nullptr;
+  }
   env->SetIntArrayRegion(ret_backup_ids, 0, ret_backup_ids_size,
       int_backup_ids.data());
   return ret_backup_ids;
@@ -155,14 +159,24 @@ void Java_org_rocksdb_BackupEngine_restoreDbFromBackup(
     JNIEnv* env, jobject jbe, jlong jbe_handle, jint jbackup_id,
     jstring jdb_dir, jstring jwal_dir, jlong jrestore_options_handle) {
   auto* backup_engine = reinterpret_cast<rocksdb::BackupEngine*>(jbe_handle);
-  const char* db_dir = env->GetStringUTFChars(jdb_dir, 0);
-  const char* wal_dir = env->GetStringUTFChars(jwal_dir, 0);
+  const char* db_dir = env->GetStringUTFChars(jdb_dir, nullptr);
+  if(db_dir == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return;
+  }
+  const char* wal_dir = env->GetStringUTFChars(jwal_dir, nullptr);
+  if(wal_dir == nullptr) {
+    // exception thrown: OutOfMemoryError
+    env->ReleaseStringUTFChars(jdb_dir, db_dir);
+    return;
+  }
   auto* restore_options =
       reinterpret_cast<rocksdb::RestoreOptions*>(jrestore_options_handle);
   auto status =
       backup_engine->RestoreDBFromBackup(
           static_cast<rocksdb::BackupID>(jbackup_id), db_dir, wal_dir,
           *restore_options);
+
   env->ReleaseStringUTFChars(jwal_dir, wal_dir);
   env->ReleaseStringUTFChars(jdb_dir, db_dir);
 
@@ -182,13 +196,23 @@ void Java_org_rocksdb_BackupEngine_restoreDbFromLatestBackup(
     JNIEnv* env, jobject jbe, jlong jbe_handle, jstring jdb_dir,
     jstring jwal_dir, jlong jrestore_options_handle) {
   auto* backup_engine = reinterpret_cast<rocksdb::BackupEngine*>(jbe_handle);
-  const char* db_dir = env->GetStringUTFChars(jdb_dir, 0);
-  const char* wal_dir = env->GetStringUTFChars(jwal_dir, 0);
+  const char* db_dir = env->GetStringUTFChars(jdb_dir, nullptr);
+  if(db_dir == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return;
+  }
+  const char* wal_dir = env->GetStringUTFChars(jwal_dir, nullptr);
+  if(wal_dir == nullptr) {
+    // exception thrown: OutOfMemoryError
+    env->ReleaseStringUTFChars(jdb_dir, db_dir);
+    return;
+  }
   auto* restore_options =
       reinterpret_cast<rocksdb::RestoreOptions*>(jrestore_options_handle);
   auto status =
       backup_engine->RestoreDBFromLatestBackup(db_dir, wal_dir,
           *restore_options);
+
   env->ReleaseStringUTFChars(jwal_dir, wal_dir);
   env->ReleaseStringUTFChars(jdb_dir, db_dir);
 
@@ -206,5 +230,7 @@ void Java_org_rocksdb_BackupEngine_restoreDbFromLatestBackup(
  */
 void Java_org_rocksdb_BackupEngine_disposeInternal(
     JNIEnv* env, jobject jbe, jlong jbe_handle) {
-  delete reinterpret_cast<rocksdb::BackupEngine*>(jbe_handle);
+  auto* be = reinterpret_cast<rocksdb::BackupEngine*>(jbe_handle);
+  assert(be != nullptr);
+  delete be;
 }

java/rocksjni/checkpoint.cc

@@ -22,7 +22,7 @@
  */
 jlong Java_org_rocksdb_Checkpoint_newCheckpoint(JNIEnv* env,
     jclass jclazz, jlong jdb_handle) {
-  auto db = reinterpret_cast<rocksdb::DB*>(jdb_handle);
+  auto* db = reinterpret_cast<rocksdb::DB*>(jdb_handle);
   rocksdb::Checkpoint* checkpoint;
   rocksdb::Checkpoint::Create(db, &checkpoint);
   return reinterpret_cast<jlong>(checkpoint);
@@ -35,8 +35,8 @@ jlong Java_org_rocksdb_Checkpoint_newCheckpoint(JNIEnv* env,
  */
 void Java_org_rocksdb_Checkpoint_disposeInternal(JNIEnv* env, jobject jobj,
     jlong jhandle) {
-  auto checkpoint = reinterpret_cast<rocksdb::Checkpoint*>(jhandle);
-  assert(checkpoint);
+  auto* checkpoint = reinterpret_cast<rocksdb::Checkpoint*>(jhandle);
+  assert(checkpoint != nullptr);
   delete checkpoint;
 }
 
@@ -48,13 +48,20 @@ void Java_org_rocksdb_Checkpoint_disposeInternal(JNIEnv* env, jobject jobj,
 void Java_org_rocksdb_Checkpoint_createCheckpoint(
     JNIEnv* env, jobject jobj, jlong jcheckpoint_handle,
     jstring jcheckpoint_path) {
-  auto checkpoint = reinterpret_cast<rocksdb::Checkpoint*>(
-      jcheckpoint_handle);
   const char* checkpoint_path = env->GetStringUTFChars(
       jcheckpoint_path, 0);
+  if(checkpoint_path == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return;
+  }
+
+  auto* checkpoint = reinterpret_cast<rocksdb::Checkpoint*>(
+      jcheckpoint_handle);
   rocksdb::Status s = checkpoint->CreateCheckpoint(
       checkpoint_path);
+  
   env->ReleaseStringUTFChars(jcheckpoint_path, checkpoint_path);
+  
   if (!s.ok()) {
       rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
   }

java/rocksjni/columnfamilyhandle.cc

@@ -20,6 +20,7 @@
  */
 void Java_org_rocksdb_ColumnFamilyHandle_disposeInternal(
     JNIEnv* env, jobject jobj, jlong handle) {
-  auto it = reinterpret_cast<rocksdb::ColumnFamilyHandle*>(handle);
-  delete it;
+  auto* cfh = reinterpret_cast<rocksdb::ColumnFamilyHandle*>(handle);
+  assert(cfh != nullptr);
+  delete cfh;
 }

java/rocksjni/compaction_filter.cc

@@ -20,6 +20,8 @@
  */
 void Java_org_rocksdb_AbstractCompactionFilter_disposeInternal(
     JNIEnv* env, jobject jobj, jlong handle) {
-  delete reinterpret_cast<rocksdb::CompactionFilter*>(handle);
+  auto* cf = reinterpret_cast<rocksdb::CompactionFilter*>(handle);
+  assert(cf != nullptr);
+  delete cf;
 }
 // </editor-fold>

java/rocksjni/comparator.cc

@@ -27,7 +27,9 @@
  */
 void Java_org_rocksdb_AbstractComparator_disposeInternal(
     JNIEnv* env, jobject jobj, jlong handle) {
-  delete reinterpret_cast<rocksdb::BaseComparatorJniCallback*>(handle);
+  auto* bcjc = reinterpret_cast<rocksdb::BaseComparatorJniCallback*>(handle);
+  assert(bcjc != nullptr);
+  delete bcjc;
 }
 // </editor-fold>
 

java/rocksjni/comparatorjnicallback.cc

@@ -17,35 +17,60 @@ BaseComparatorJniCallback::BaseComparatorJniCallback(
     mtx_findShortestSeparator(new port::Mutex(copt->use_adaptive_mutex)) {
   // Note: Comparator methods may be accessed by multiple threads,
   // so we ref the jvm not the env
-  const jint rs __attribute__((unused)) = env->GetJavaVM(&m_jvm);
-  assert(rs == JNI_OK);
+  const jint rs = env->GetJavaVM(&m_jvm);
+  if(rs != JNI_OK) {
+    // exception thrown
+    return;
+  }
 
   // Note: we want to access the Java Comparator instance
   // across multiple method calls, so we create a global ref
+  assert(jComparator != nullptr);
   m_jComparator = env->NewGlobalRef(jComparator);
+  if(m_jComparator == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return;
+  }
 
   // Note: The name of a Comparator will not change during it's lifetime,
   // so we cache it in a global var
   jmethodID jNameMethodId = AbstractComparatorJni::getNameMethodId(env);
+  if(jNameMethodId == nullptr) {
+    // exception thrown: NoSuchMethodException or OutOfMemoryError
+    return;
+  }
   jstring jsName = (jstring)env->CallObjectMethod(m_jComparator, jNameMethodId);
-  m_name = JniUtil::copyString(env, jsName);  // also releases jsName
+  if(env->ExceptionCheck()) {
+    // exception thrown
+    return;
+  }
+  jboolean has_exception = JNI_FALSE;
+  m_name = JniUtil::copyString(env, jsName,
+      &has_exception);  // also releases jsName
+  if (has_exception == JNI_TRUE) {
+    // exception thrown
+    return;
+  }
 
   m_jCompareMethodId = AbstractComparatorJni::getCompareMethodId(env);
+  if(m_jCompareMethodId == nullptr) {
+    // exception thrown: NoSuchMethodException or OutOfMemoryError
+    return;
+  }
+
   m_jFindShortestSeparatorMethodId =
     AbstractComparatorJni::getFindShortestSeparatorMethodId(env);
+  if(m_jFindShortestSeparatorMethodId == nullptr) {
+    // exception thrown: NoSuchMethodException or OutOfMemoryError
+    return;
+  }
+
   m_jFindShortSuccessorMethodId =
     AbstractComparatorJni::getFindShortSuccessorMethodId(env);
-}
-
-/**
- * Attach/Get a JNIEnv for the current native thread
- */
-JNIEnv* BaseComparatorJniCallback::getJniEnv() const {
-  JNIEnv *env;
-  jint rs __attribute__((unused)) =
-      m_jvm->AttachCurrentThread(reinterpret_cast<void**>(&env), NULL);
-  assert(rs == JNI_OK);
-  return env;
+  if(m_jFindShortSuccessorMethodId == nullptr) {
+    // exception thrown: NoSuchMethodException or OutOfMemoryError
+    return;
+  }
 }
 
 const char* BaseComparatorJniCallback::Name() const {
@@ -53,22 +78,50 @@ const char* BaseComparatorJniCallback::Name() const {
 }
 
 int BaseComparatorJniCallback::Compare(const Slice& a, const Slice& b) const {
-  JNIEnv* m_env = getJniEnv();
+  jboolean attached_thread = JNI_FALSE;
+  JNIEnv* env = JniUtil::getJniEnv(m_jvm, &attached_thread);
+  assert(env != nullptr);
 
   // TODO(adamretter): slice objects can potentially be cached using thread
   // local variables to avoid locking. Could make this configurable depending on
   // performance.
   mtx_compare->Lock();
 
-  AbstractSliceJni::setHandle(m_env, m_jSliceA, &a, JNI_FALSE);
-  AbstractSliceJni::setHandle(m_env, m_jSliceB, &b, JNI_FALSE);
+  bool pending_exception =
+      AbstractSliceJni::setHandle(env, m_jSliceA, &a, JNI_FALSE);
+  if(pending_exception) {
+    if(env->ExceptionCheck()) {
+      // exception thrown from setHandle or descendant
+      env->ExceptionDescribe(); // print out exception to stderr
+    }
+    JniUtil::releaseJniEnv(m_jvm, attached_thread);
+    return 0;
+  }
+
+  pending_exception =
+      AbstractSliceJni::setHandle(env, m_jSliceB, &b, JNI_FALSE);
+  if(pending_exception) {
+    if(env->ExceptionCheck()) {
+      // exception thrown from setHandle or descendant
+      env->ExceptionDescribe(); // print out exception to stderr
+    }
+    JniUtil::releaseJniEnv(m_jvm, attached_thread);
+    return 0;
+  }
+  
   jint result =
-    m_env->CallIntMethod(m_jComparator, m_jCompareMethodId, m_jSliceA,
+    env->CallIntMethod(m_jComparator, m_jCompareMethodId, m_jSliceA,
       m_jSliceB);
 
   mtx_compare->Unlock();
 
-  m_jvm->DetachCurrentThread();
+  if(env->ExceptionCheck()) {
+    // exception thrown from CallIntMethod
+    env->ExceptionDescribe(); // print out exception to stderr
+    result = 0; // we could not get a result from java callback so use 0
+  }
+
+  JniUtil::releaseJniEnv(m_jvm, attached_thread);
 
   return result;
 }
@@ -79,32 +132,80 @@ void BaseComparatorJniCallback::FindShortestSeparator(
     return;
   }
 
-  JNIEnv* m_env = getJniEnv();
+  jboolean attached_thread = JNI_FALSE;
+  JNIEnv* env = JniUtil::getJniEnv(m_jvm, &attached_thread);
+  assert(env != nullptr);
 
   const char* startUtf = start->c_str();
-  jstring jsStart = m_env->NewStringUTF(startUtf);
+  jstring jsStart = env->NewStringUTF(startUtf);
+  if(jsStart == nullptr) {
+    // unable to construct string
+    if(env->ExceptionCheck()) {
+      env->ExceptionDescribe(); // print out exception to stderr
+    }
+    JniUtil::releaseJniEnv(m_jvm, attached_thread);
+    return;
+  }
+  if(env->ExceptionCheck()) {
+    // exception thrown: OutOfMemoryError
+    env->ExceptionDescribe(); // print out exception to stderr
+    env->DeleteLocalRef(jsStart);
+    JniUtil::releaseJniEnv(m_jvm, attached_thread);
+    return;
+  }
 
   // TODO(adamretter): slice object can potentially be cached using thread local
   // variable to avoid locking. Could make this configurable depending on
   // performance.
   mtx_findShortestSeparator->Lock();
 
-  AbstractSliceJni::setHandle(m_env, m_jSliceLimit, &limit, JNI_FALSE);
+  bool pending_exception =
+      AbstractSliceJni::setHandle(env, m_jSliceLimit, &limit, JNI_FALSE);
+  if(pending_exception) {
+    if(env->ExceptionCheck()) {
+      // exception thrown from setHandle or descendant
+      env->ExceptionDescribe(); // print out exception to stderr
+    }
+    if(jsStart != nullptr) {
+      env->DeleteLocalRef(jsStart);
+    }
+    JniUtil::releaseJniEnv(m_jvm, attached_thread);
+    return;
+  }
+
   jstring jsResultStart =
-    (jstring)m_env->CallObjectMethod(m_jComparator,
+    (jstring)env->CallObjectMethod(m_jComparator,
       m_jFindShortestSeparatorMethodId, jsStart, m_jSliceLimit);
 
   mtx_findShortestSeparator->Unlock();
 
-  m_env->DeleteLocalRef(jsStart);
+  if(env->ExceptionCheck()) {
+    // exception thrown from CallObjectMethod
+    env->ExceptionDescribe();  // print out exception to stderr
+    env->DeleteLocalRef(jsStart);
+    JniUtil::releaseJniEnv(m_jvm, attached_thread);
+    return;
+  }
+
+  env->DeleteLocalRef(jsStart);
 
   if (jsResultStart != nullptr) {
     // update start with result
-    *start =
-      JniUtil::copyString(m_env, jsResultStart);  // also releases jsResultStart
+    jboolean has_exception = JNI_FALSE;
+    std::string result = JniUtil::copyString(env, jsResultStart,
+        &has_exception);  // also releases jsResultStart
+    if (has_exception == JNI_TRUE) {
+      if (env->ExceptionCheck()) {
+        env->ExceptionDescribe();  // print out exception to stderr
+      }
+      JniUtil::releaseJniEnv(m_jvm, attached_thread);
+      return;
+    }
+
+    *start = result;
   }
 
-  m_jvm->DetachCurrentThread();
+  JniUtil::releaseJniEnv(m_jvm, attached_thread);
 }
 
 void BaseComparatorJniCallback::FindShortSuccessor(std::string* key) const {
@@ -112,34 +213,69 @@ void BaseComparatorJniCallback::FindShortSuccessor(std::string* key) const {
     return;
   }
 
-  JNIEnv* m_env = getJniEnv();
+  jboolean attached_thread = JNI_FALSE;
+  JNIEnv* env = JniUtil::getJniEnv(m_jvm, &attached_thread);
+  assert(env != nullptr);
 
   const char* keyUtf = key->c_str();
-  jstring jsKey = m_env->NewStringUTF(keyUtf);
+  jstring jsKey = env->NewStringUTF(keyUtf);
+  if(jsKey == nullptr) {
+    // unable to construct string
+    if(env->ExceptionCheck()) {
+      env->ExceptionDescribe(); // print out exception to stderr
+    }
+    JniUtil::releaseJniEnv(m_jvm, attached_thread);
+    return;
+  } else if(env->ExceptionCheck()) {
+    // exception thrown: OutOfMemoryError
+    env->ExceptionDescribe(); // print out exception to stderr
+    env->DeleteLocalRef(jsKey);
+    JniUtil::releaseJniEnv(m_jvm, attached_thread);
+    return;
+  }
 
   jstring jsResultKey =
-    (jstring)m_env->CallObjectMethod(m_jComparator,
+    (jstring)env->CallObjectMethod(m_jComparator,
       m_jFindShortSuccessorMethodId, jsKey);
 
-  m_env->DeleteLocalRef(jsKey);
+  if(env->ExceptionCheck()) {
+    // exception thrown from CallObjectMethod
+    env->ExceptionDescribe(); // print out exception to stderr
+    env->DeleteLocalRef(jsKey);
+    JniUtil::releaseJniEnv(m_jvm, attached_thread);
+    return;
+  }
+
+  env->DeleteLocalRef(jsKey);
 
   if (jsResultKey != nullptr) {
     // updates key with result, also releases jsResultKey.
-    *key = JniUtil::copyString(m_env, jsResultKey);
+    jboolean has_exception = JNI_FALSE;
+    std::string result = JniUtil::copyString(env, jsResultKey, &has_exception);
+    if (has_exception == JNI_TRUE) {
+      if (env->ExceptionCheck()) {
+        env->ExceptionDescribe();  // print out exception to stderr
+      }
+      JniUtil::releaseJniEnv(m_jvm, attached_thread);
+      return;
+    }
+
+    *key = result;
   }
 
-  m_jvm->DetachCurrentThread();
+  JniUtil::releaseJniEnv(m_jvm, attached_thread);
 }
 
 BaseComparatorJniCallback::~BaseComparatorJniCallback() {
-  JNIEnv* m_env = getJniEnv();
+  jboolean attached_thread = JNI_FALSE;
+  JNIEnv* env = JniUtil::getJniEnv(m_jvm, &attached_thread);
+  assert(env != nullptr);
 
-  m_env->DeleteGlobalRef(m_jComparator);
+  if(m_jComparator != nullptr) {
+    env->DeleteGlobalRef(m_jComparator);
+  }
 
-  // Note: do not need to explicitly detach, as this function is effectively
-  // called from the Java class's disposeInternal method, and so already
-  // has an attached thread, getJniEnv above is just a no-op Attach to get
-  // the env jvm->DetachCurrentThread();
+  JniUtil::releaseJniEnv(m_jvm, attached_thread);
 }
 
 ComparatorJniCallback::ComparatorJniCallback(
@@ -147,15 +283,42 @@ ComparatorJniCallback::ComparatorJniCallback(
     const ComparatorJniCallbackOptions* copt) :
     BaseComparatorJniCallback(env, jComparator, copt) {
   m_jSliceA = env->NewGlobalRef(SliceJni::construct0(env));
+  if(m_jSliceA == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return;
+  }
+
   m_jSliceB = env->NewGlobalRef(SliceJni::construct0(env));
+  if(m_jSliceB == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return;
+  }
+
   m_jSliceLimit = env->NewGlobalRef(SliceJni::construct0(env));
+  if(m_jSliceLimit == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return;
+  }
 }
 
 ComparatorJniCallback::~ComparatorJniCallback() {
-  JNIEnv* m_env = getJniEnv();
-  m_env->DeleteGlobalRef(m_jSliceA);
-  m_env->DeleteGlobalRef(m_jSliceB);
-  m_env->DeleteGlobalRef(m_jSliceLimit);
+  jboolean attached_thread = JNI_FALSE;
+  JNIEnv* env = JniUtil::getJniEnv(m_jvm, &attached_thread);
+  assert(env != nullptr);
+
+  if(m_jSliceA != nullptr) {
+    env->DeleteGlobalRef(m_jSliceA);
+  }
+
+  if(m_jSliceB != nullptr) {
+    env->DeleteGlobalRef(m_jSliceB);
+  }
+
+  if(m_jSliceLimit != nullptr) {
+    env->DeleteGlobalRef(m_jSliceLimit);
+  }
+
+  JniUtil::releaseJniEnv(m_jvm, attached_thread);
 }
 
 DirectComparatorJniCallback::DirectComparatorJniCallback(
@@ -163,14 +326,41 @@ DirectComparatorJniCallback::DirectComparatorJniCallback(
     const ComparatorJniCallbackOptions* copt) :
     BaseComparatorJniCallback(env, jComparator, copt) {
   m_jSliceA = env->NewGlobalRef(DirectSliceJni::construct0(env));
+  if(m_jSliceA == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return;
+  }
+
   m_jSliceB = env->NewGlobalRef(DirectSliceJni::construct0(env));
+  if(m_jSliceB == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return;
+  }
+
   m_jSliceLimit = env->NewGlobalRef(DirectSliceJni::construct0(env));
+  if(m_jSliceLimit == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return;
+  }
 }
 
 DirectComparatorJniCallback::~DirectComparatorJniCallback() {
-  JNIEnv* m_env = getJniEnv();
-  m_env->DeleteGlobalRef(m_jSliceA);
-  m_env->DeleteGlobalRef(m_jSliceB);
-  m_env->DeleteGlobalRef(m_jSliceLimit);
+  jboolean attached_thread = JNI_FALSE;
+  JNIEnv* env = JniUtil::getJniEnv(m_jvm, &attached_thread);
+  assert(env != nullptr);
+
+  if(m_jSliceA != nullptr) {
+    env->DeleteGlobalRef(m_jSliceA);
+  }
+
+  if(m_jSliceB != nullptr) {
+    env->DeleteGlobalRef(m_jSliceB);
+  }
+
+  if(m_jSliceLimit != nullptr) {
+    env->DeleteGlobalRef(m_jSliceLimit);
+  }
+
+  JniUtil::releaseJniEnv(m_jvm, attached_thread);
 }
 }  // namespace rocksdb

java/rocksjni/comparatorjnicallback.h

@@ -61,7 +61,6 @@ class BaseComparatorJniCallback : public Comparator {
     port::Mutex* mtx_compare;
     // used for synchronisation in findShortestSeparator method
     port::Mutex* mtx_findShortestSeparator;
-    JavaVM* m_jvm;
     jobject m_jComparator;
     std::string m_name;
     jmethodID m_jCompareMethodId;
@@ -69,7 +68,7 @@ class BaseComparatorJniCallback : public Comparator {
     jmethodID m_jFindShortSuccessorMethodId;
 
  protected:
-    JNIEnv* getJniEnv() const;
+    JavaVM* m_jvm;
     jobject m_jSliceA;
     jobject m_jSliceB;
     jobject m_jSliceLimit;

java/rocksjni/env.cc

@@ -75,5 +75,7 @@ jlong Java_org_rocksdb_RocksMemEnv_createMemEnv(
  */
 void Java_org_rocksdb_RocksMemEnv_disposeInternal(
     JNIEnv* env, jobject jobj, jlong jhandle) {
-  delete reinterpret_cast<rocksdb::Env*>(jhandle);
+  auto* e = reinterpret_cast<rocksdb::Env*>(jhandle);
+  assert(e != nullptr);
+  delete e;
 }

java/rocksjni/env_options.cc

@@ -44,7 +44,9 @@ jlong Java_org_rocksdb_EnvOptions_newEnvOptions(JNIEnv *env, jclass jcls) {
  */
 void Java_org_rocksdb_EnvOptions_disposeInternal(JNIEnv *env, jobject jobj,
                                                  jlong jhandle) {
-  delete reinterpret_cast<rocksdb::EnvOptions *>(jhandle);
+  auto* eo = reinterpret_cast<rocksdb::EnvOptions *>(jhandle);
+  assert(eo != nullptr);
+  delete eo;
 }
 
 /*
@@ -288,7 +290,8 @@ jlong Java_org_rocksdb_EnvOptions_writableFileMaxBufferSize(JNIEnv *env,
 void Java_org_rocksdb_EnvOptions_setRateLimiter(JNIEnv *env, jobject jobj,
                                                 jlong jhandle,
                                                 jlong rl_handle) {
-  auto *rate_limiter = reinterpret_cast<rocksdb::RateLimiter *>(rl_handle);
-  auto *env_opt = reinterpret_cast<rocksdb::EnvOptions *>(jhandle);
-  env_opt->rate_limiter = rate_limiter;
+  auto* sptr_rate_limiter =
+      reinterpret_cast<std::shared_ptr<rocksdb::RateLimiter> *>(rl_handle);
+  auto* env_opt = reinterpret_cast<rocksdb::EnvOptions *>(jhandle);
+  env_opt->rate_limiter = sptr_rate_limiter->get();
 }

java/rocksjni/external_sst_file_info.cc

@@ -31,17 +31,35 @@ jlong Java_org_rocksdb_ExternalSstFileInfo_newExternalSstFileInfo__Ljava_lang_St
     JNIEnv *env, jclass jcls, jstring jfile_path, jstring jsmallest_key,
     jstring jlargest_key, jlong jsequence_number, jlong jfile_size,
     jint jnum_entries, jint jversion) {
-  const char *file_path = env->GetStringUTFChars(jfile_path, NULL);
-  const char *smallest_key = env->GetStringUTFChars(jsmallest_key, NULL);
-  const char *largest_key = env->GetStringUTFChars(jlargest_key, NULL);
+  const char *file_path = env->GetStringUTFChars(jfile_path, nullptr);
+  if(file_path == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return 0;
+  }
+  const char *smallest_key = env->GetStringUTFChars(jsmallest_key, nullptr);
+  if(smallest_key == nullptr) {
+    // exception thrown: OutOfMemoryError
+    env->ReleaseStringUTFChars(jfile_path, file_path);
+    return 0;
+  }
+  const char *largest_key = env->GetStringUTFChars(jlargest_key, nullptr);
+  if(largest_key == nullptr) {
+    // exception thrown: OutOfMemoryError
+    env->ReleaseStringUTFChars(jsmallest_key, smallest_key);
+    env->ReleaseStringUTFChars(jfile_path, file_path);
+    return 0;
+  }
+
   auto *external_sst_file_info = new rocksdb::ExternalSstFileInfo(
       file_path, smallest_key, largest_key,
       static_cast<rocksdb::SequenceNumber>(jsequence_number),
       static_cast<uint64_t>(jfile_size), static_cast<int32_t>(jnum_entries),
       static_cast<int32_t>(jversion));
-  env->ReleaseStringUTFChars(jfile_path, file_path);
-  env->ReleaseStringUTFChars(jsmallest_key, smallest_key);
+
   env->ReleaseStringUTFChars(jlargest_key, largest_key);
+  env->ReleaseStringUTFChars(jsmallest_key, smallest_key);
+  env->ReleaseStringUTFChars(jfile_path, file_path);
+
   return reinterpret_cast<jlong>(external_sst_file_info);
 }
 
@@ -55,7 +73,11 @@ void Java_org_rocksdb_ExternalSstFileInfo_setFilePath(JNIEnv *env, jobject jobj,
                                                       jstring jfile_path) {
   auto *external_sst_file_info =
       reinterpret_cast<rocksdb::ExternalSstFileInfo *>(jhandle);
-  const char *file_path = env->GetStringUTFChars(jfile_path, NULL);
+  const char *file_path = env->GetStringUTFChars(jfile_path, nullptr);
+  if(file_path == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return;
+  }
   external_sst_file_info->file_path = file_path;
   env->ReleaseStringUTFChars(jfile_path, file_path);
 }
@@ -81,7 +103,11 @@ void Java_org_rocksdb_ExternalSstFileInfo_setSmallestKey(
     JNIEnv *env, jobject jobj, jlong jhandle, jstring jsmallest_key) {
   auto *external_sst_file_info =
       reinterpret_cast<rocksdb::ExternalSstFileInfo *>(jhandle);
-  const char *smallest_key = env->GetStringUTFChars(jsmallest_key, NULL);
+  const char *smallest_key = env->GetStringUTFChars(jsmallest_key, nullptr);
+  if(smallest_key == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return;
+  }
   external_sst_file_info->smallest_key = smallest_key;
   env->ReleaseStringUTFChars(jsmallest_key, smallest_key);
 }
@@ -111,6 +137,10 @@ void Java_org_rocksdb_ExternalSstFileInfo_setLargestKey(JNIEnv *env,
   auto *external_sst_file_info =
       reinterpret_cast<rocksdb::ExternalSstFileInfo *>(jhandle);
   const char *largest_key = env->GetStringUTFChars(jlargest_key, NULL);
+  if(largest_key == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return;
+  }
   external_sst_file_info->largest_key = largest_key;
   env->ReleaseStringUTFChars(jlargest_key, largest_key);
 }
@@ -238,5 +268,7 @@ jint Java_org_rocksdb_ExternalSstFileInfo_version(JNIEnv *env, jobject jobj,
 void Java_org_rocksdb_ExternalSstFileInfo_disposeInternal(JNIEnv *env,
                                                           jobject jobj,
                                                           jlong jhandle) {
-  delete reinterpret_cast<rocksdb::ExternalSstFileInfo *>(jhandle);
+  auto* esfi = reinterpret_cast<rocksdb::ExternalSstFileInfo *>(jhandle);
+  assert(esfi != nullptr);
+  delete esfi;
 }

java/rocksjni/filter.cc

@@ -24,12 +24,10 @@
 jlong Java_org_rocksdb_BloomFilter_createNewBloomFilter(
     JNIEnv* env, jclass jcls, jint bits_per_key,
     jboolean use_block_base_builder) {
-  auto* fp = const_cast<rocksdb::FilterPolicy *>(
-      rocksdb::NewBloomFilterPolicy(bits_per_key, use_block_base_builder));
-  auto* pFilterPolicy =
-      new std::shared_ptr<rocksdb::FilterPolicy>;
-  *pFilterPolicy = std::shared_ptr<rocksdb::FilterPolicy>(fp);
-  return reinterpret_cast<jlong>(pFilterPolicy);
+  auto* sptr_filter =
+      new std::shared_ptr<const rocksdb::FilterPolicy>(
+          rocksdb::NewBloomFilterPolicy(bits_per_key, use_block_base_builder));
+  return reinterpret_cast<jlong>(sptr_filter);
 }
 
 /*
@@ -39,8 +37,7 @@ jlong Java_org_rocksdb_BloomFilter_createNewBloomFilter(
  */
 void Java_org_rocksdb_Filter_disposeInternal(
     JNIEnv* env, jobject jobj, jlong jhandle) {
-
-  std::shared_ptr<rocksdb::FilterPolicy> *handle =
-      reinterpret_cast<std::shared_ptr<rocksdb::FilterPolicy> *>(jhandle);
-  handle->reset();
+  auto* handle =
+      reinterpret_cast<std::shared_ptr<const rocksdb::FilterPolicy> *>(jhandle);
+  delete handle;  // delete std::shared_ptr
 }

java/rocksjni/iterator.cc

@@ -21,7 +21,8 @@
  */
 void Java_org_rocksdb_RocksIterator_disposeInternal(
     JNIEnv* env, jobject jobj, jlong handle) {
-  auto it = reinterpret_cast<rocksdb::Iterator*>(handle);
+  auto* it = reinterpret_cast<rocksdb::Iterator*>(handle);
+  assert(it != nullptr);
   delete it;
 }
 
@@ -83,11 +84,16 @@ void Java_org_rocksdb_RocksIterator_prev0(
 void Java_org_rocksdb_RocksIterator_seek0(
     JNIEnv* env, jobject jobj, jlong handle,
     jbyteArray jtarget, jint jtarget_len) {
-  auto it = reinterpret_cast<rocksdb::Iterator*>(handle);
-  jbyte* target = env->GetByteArrayElements(jtarget, 0);
+  jbyte* target = env->GetByteArrayElements(jtarget, nullptr);
+  if(target == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return;
+  }
+
   rocksdb::Slice target_slice(
       reinterpret_cast<char*>(target), jtarget_len);
 
+  auto* it = reinterpret_cast<rocksdb::Iterator*>(handle);
   it->Seek(target_slice);
 
   env->ReleaseByteArrayElements(jtarget, target, JNI_ABORT);
@@ -100,7 +106,7 @@ void Java_org_rocksdb_RocksIterator_seek0(
  */
 void Java_org_rocksdb_RocksIterator_status0(
     JNIEnv* env, jobject jobj, jlong handle) {
-  auto it = reinterpret_cast<rocksdb::Iterator*>(handle);
+  auto* it = reinterpret_cast<rocksdb::Iterator*>(handle);
   rocksdb::Status s = it->status();
 
   if (s.ok()) {
@@ -117,12 +123,16 @@ void Java_org_rocksdb_RocksIterator_status0(
  */
 jbyteArray Java_org_rocksdb_RocksIterator_key0(
     JNIEnv* env, jobject jobj, jlong handle) {
-  auto it = reinterpret_cast<rocksdb::Iterator*>(handle);
+  auto* it = reinterpret_cast<rocksdb::Iterator*>(handle);
   rocksdb::Slice key_slice = it->key();
 
   jbyteArray jkey = env->NewByteArray(static_cast<jsize>(key_slice.size()));
+  if(jkey == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return nullptr;
+  }
   env->SetByteArrayRegion(jkey, 0, static_cast<jsize>(key_slice.size()),
-                          reinterpret_cast<const jbyte*>(key_slice.data()));
+                          const_cast<jbyte*>(reinterpret_cast<const jbyte*>(key_slice.data())));
   return jkey;
 }
 
@@ -133,12 +143,16 @@ jbyteArray Java_org_rocksdb_RocksIterator_key0(
  */
 jbyteArray Java_org_rocksdb_RocksIterator_value0(
     JNIEnv* env, jobject jobj, jlong handle) {
-  auto it = reinterpret_cast<rocksdb::Iterator*>(handle);
+  auto* it = reinterpret_cast<rocksdb::Iterator*>(handle);
   rocksdb::Slice value_slice = it->value();
 
   jbyteArray jkeyValue =
       env->NewByteArray(static_cast<jsize>(value_slice.size()));
+  if(jkeyValue == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return nullptr;
+  }
   env->SetByteArrayRegion(jkeyValue, 0, static_cast<jsize>(value_slice.size()),
-                          reinterpret_cast<const jbyte*>(value_slice.data()));
+                          const_cast<jbyte*>(reinterpret_cast<const jbyte*>(value_slice.data())));
   return jkeyValue;
 }

java/rocksjni/loggerjnicallback.cc

@@ -10,53 +10,106 @@
 
 #include "rocksjni/loggerjnicallback.h"
 #include "rocksjni/portal.h"
+#include <cstdarg>
+#include <cstdio>
 
 namespace rocksdb {
 
 LoggerJniCallback::LoggerJniCallback(
     JNIEnv* env, jobject jlogger) {
-  const jint rs __attribute__((unused)) = env->GetJavaVM(&m_jvm);
-  assert(rs == JNI_OK);
+  // Note: Logger methods may be accessed by multiple threads,
+  // so we ref the jvm not the env
+  const jint rs = env->GetJavaVM(&m_jvm);
+  if(rs != JNI_OK) {
+    // exception thrown
+    return;
+  }
 
   // Note: we want to access the Java Logger instance
   // across multiple method calls, so we create a global ref
+  assert(jlogger != nullptr);
   m_jLogger = env->NewGlobalRef(jlogger);
+  if(m_jLogger == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return;
+  }
   m_jLogMethodId = LoggerJni::getLogMethodId(env);
+  if(m_jLogMethodId == nullptr) {
+    // exception thrown: NoSuchMethodException or OutOfMemoryError
+    return;
+  }
 
   jobject jdebug_level = InfoLogLevelJni::DEBUG_LEVEL(env);
-  assert(jdebug_level != nullptr);
+  if(jdebug_level == nullptr) {
+    // exception thrown: NoSuchFieldError, ExceptionInInitializerError
+    // or OutOfMemoryError
+    return;
+  }
   m_jdebug_level = env->NewGlobalRef(jdebug_level);
+  if(m_jdebug_level == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return;
+  }
 
   jobject jinfo_level = InfoLogLevelJni::INFO_LEVEL(env);
-  assert(jinfo_level != nullptr);
+  if(jinfo_level == nullptr) {
+    // exception thrown: NoSuchFieldError, ExceptionInInitializerError
+    // or OutOfMemoryError
+    return;
+  }
   m_jinfo_level = env->NewGlobalRef(jinfo_level);
+  if(m_jinfo_level == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return;
+  }
 
   jobject jwarn_level = InfoLogLevelJni::WARN_LEVEL(env);
-  assert(jwarn_level != nullptr);
+  if(jwarn_level == nullptr) {
+    // exception thrown: NoSuchFieldError, ExceptionInInitializerError
+    // or OutOfMemoryError
+    return;
+  }
   m_jwarn_level = env->NewGlobalRef(jwarn_level);
+  if(m_jwarn_level == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return;
+  }
 
   jobject jerror_level = InfoLogLevelJni::ERROR_LEVEL(env);
-  assert(jerror_level != nullptr);
+  if(jerror_level == nullptr) {
+    // exception thrown: NoSuchFieldError, ExceptionInInitializerError
+    // or OutOfMemoryError
+    return;
+  }
   m_jerror_level = env->NewGlobalRef(jerror_level);
+  if(m_jerror_level == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return;
+  }
 
   jobject jfatal_level = InfoLogLevelJni::FATAL_LEVEL(env);
-  assert(jfatal_level != nullptr);
+  if(jfatal_level == nullptr) {
+    // exception thrown: NoSuchFieldError, ExceptionInInitializerError
+    // or OutOfMemoryError
+    return;
+  }
   m_jfatal_level = env->NewGlobalRef(jfatal_level);
+  if(m_jfatal_level == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return;
+  }
 
   jobject jheader_level = InfoLogLevelJni::HEADER_LEVEL(env);
-  assert(jheader_level != nullptr);
+  if(jheader_level == nullptr) {
+    // exception thrown: NoSuchFieldError, ExceptionInInitializerError
+    // or OutOfMemoryError
+    return;
+  }
   m_jheader_level = env->NewGlobalRef(jheader_level);
-}
-
-/**
- * Get JNIEnv for current native thread
- */
-JNIEnv* LoggerJniCallback::getJniEnv() const {
-  JNIEnv *env;
-  jint rs __attribute__((unused)) =
-      m_jvm->AttachCurrentThread(reinterpret_cast<void**>(&env), NULL);
-  assert(rs == JNI_OK);
-  return env;
+  if(m_jheader_level == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return;
+  }
 }
 
 void LoggerJniCallback::Logv(const char* format, va_list ap) {
@@ -94,69 +147,96 @@ void LoggerJniCallback::Logv(const InfoLogLevel log_level,
         break;
     }
 
-    // We try twice: the first time with a fixed-size stack allocated buffer,
-    // and the second time with a much larger dynamically allocated buffer.
-    char buffer[500];
-    for (int iter = 0; iter < 2; iter++) {
-      char* base;
-      int bufsize;
-      if (iter == 0) {
-        bufsize = sizeof(buffer);
-        base = buffer;
-      } else {
-        bufsize = 30000;
-        base = new char[bufsize];
-      }
-      char* p = base;
-      char* limit = base + bufsize;
-      // Print the message
-      if (p < limit) {
-        va_list backup_ap;
-        va_copy(backup_ap, ap);
-        p += vsnprintf(p, limit - p, format, backup_ap);
-        va_end(backup_ap);
-      }
-      // Truncate to available space if necessary
-      if (p >= limit) {
-        if (iter == 0) {
-          continue;       // Try again with larger buffer
-        } else {
-          p = limit - 1;
-        }
-      }
-      assert(p < limit);
-      *p++ = '\0';
+    assert(format != nullptr);
+    assert(ap != nullptr);
+    const std::unique_ptr<char[]> msg = format_str(format, ap);
 
-      JNIEnv* env = getJniEnv();
+    // pass msg to java callback handler
+    jboolean attached_thread = JNI_FALSE;
+    JNIEnv* env = JniUtil::getJniEnv(m_jvm, &attached_thread);
+    assert(env != nullptr);
 
-      // pass java string to callback handler
-      env->CallVoidMethod(
-          m_jLogger,
-          m_jLogMethodId,
-          jlog_level,
-          env->NewStringUTF(base));
-
-      if (base != buffer) {
-        delete[] base;
+    jstring jmsg = env->NewStringUTF(msg.get());
+    if(jmsg == nullptr) {
+      // unable to construct string
+      if(env->ExceptionCheck()) {
+        env->ExceptionDescribe(); // print out exception to stderr
       }
-      break;
+      JniUtil::releaseJniEnv(m_jvm, attached_thread);
+      return;
     }
-    m_jvm->DetachCurrentThread();
+    if(env->ExceptionCheck()) {
+      // exception thrown: OutOfMemoryError
+      env->ExceptionDescribe(); // print out exception to stderr
+      env->DeleteLocalRef(jmsg);
+      JniUtil::releaseJniEnv(m_jvm, attached_thread);
+      return;
+    }
+
+    env->CallVoidMethod(m_jLogger, m_jLogMethodId, jlog_level, jmsg);
+    if(env->ExceptionCheck()) {
+      // exception thrown
+      env->ExceptionDescribe(); // print out exception to stderr
+      env->DeleteLocalRef(jmsg);
+      JniUtil::releaseJniEnv(m_jvm, attached_thread);
+      return;
+    }
+
+    env->DeleteLocalRef(jmsg);
+    JniUtil::releaseJniEnv(m_jvm, attached_thread);
   }
 }
 
+std::unique_ptr<char[]> LoggerJniCallback::format_str(const char* format, va_list ap) const {
+  va_list ap_copy;
+
+  va_copy(ap_copy, ap);
+  const size_t required = vsnprintf(nullptr, 0, format, ap_copy) + 1; // Extra space for '\0'
+  va_end(ap_copy);
+
+  std::unique_ptr<char[]> buf(new char[required]);
+
+  va_copy(ap_copy, ap);
+  vsnprintf(buf.get(), required, format, ap_copy);
+  va_end(ap_copy);
+
+  return buf;
+}
+
 LoggerJniCallback::~LoggerJniCallback() {
-  JNIEnv* env = getJniEnv();
-  env->DeleteGlobalRef(m_jLogger);
+  jboolean attached_thread = JNI_FALSE;
+  JNIEnv* env = JniUtil::getJniEnv(m_jvm, &attached_thread);
+  assert(env != nullptr);
 
-  env->DeleteGlobalRef(m_jdebug_level);
-  env->DeleteGlobalRef(m_jinfo_level);
-  env->DeleteGlobalRef(m_jwarn_level);
-  env->DeleteGlobalRef(m_jerror_level);
-  env->DeleteGlobalRef(m_jfatal_level);
-  env->DeleteGlobalRef(m_jheader_level);
+  if(m_jLogger != nullptr) {
+    env->DeleteGlobalRef(m_jLogger);
+  }
 
-  m_jvm->DetachCurrentThread();
+  if(m_jdebug_level != nullptr) {
+    env->DeleteGlobalRef(m_jdebug_level);
+  }
+
+  if(m_jinfo_level != nullptr) {
+    env->DeleteGlobalRef(m_jinfo_level);
+  }
+
+  if(m_jwarn_level != nullptr) {
+    env->DeleteGlobalRef(m_jwarn_level);
+  }
+
+  if(m_jerror_level != nullptr) {
+    env->DeleteGlobalRef(m_jerror_level);
+  }
+
+  if(m_jfatal_level != nullptr) {
+    env->DeleteGlobalRef(m_jfatal_level);
+  }
+
+  if(m_jheader_level != nullptr) {
+    env->DeleteGlobalRef(m_jheader_level);
+  }
+
+  JniUtil::releaseJniEnv(m_jvm, attached_thread);
 }
 
 }  // namespace rocksdb
@@ -168,15 +248,14 @@ LoggerJniCallback::~LoggerJniCallback() {
  */
 jlong Java_org_rocksdb_Logger_createNewLoggerOptions(
     JNIEnv* env, jobject jobj, jlong joptions) {
-  rocksdb::LoggerJniCallback* c =
-      new rocksdb::LoggerJniCallback(env, jobj);
+  auto* sptr_logger = new std::shared_ptr<rocksdb::LoggerJniCallback>(
+      new rocksdb::LoggerJniCallback(env, jobj));
+
   // set log level
-  c->SetInfoLogLevel(reinterpret_cast<rocksdb::Options*>
-      (joptions)->info_log_level);
-  std::shared_ptr<rocksdb::LoggerJniCallback> *pLoggerJniCallback =
-      new std::shared_ptr<rocksdb::LoggerJniCallback>;
-  *pLoggerJniCallback = std::shared_ptr<rocksdb::LoggerJniCallback>(c);
-  return reinterpret_cast<jlong>(pLoggerJniCallback);
+  auto* options = reinterpret_cast<rocksdb::Options*>(joptions);
+  sptr_logger->get()->SetInfoLogLevel(options->info_log_level);
+
+  return reinterpret_cast<jlong>(sptr_logger);
 }
 
 /*
@@ -186,15 +265,14 @@ jlong Java_org_rocksdb_Logger_createNewLoggerOptions(
  */
 jlong Java_org_rocksdb_Logger_createNewLoggerDbOptions(
     JNIEnv* env, jobject jobj, jlong jdb_options) {
-  rocksdb::LoggerJniCallback* c =
-      new rocksdb::LoggerJniCallback(env, jobj);
+  auto* sptr_logger = new std::shared_ptr<rocksdb::LoggerJniCallback>(
+    new rocksdb::LoggerJniCallback(env, jobj));
+
   // set log level
-  c->SetInfoLogLevel(reinterpret_cast<rocksdb::DBOptions*>
-      (jdb_options)->info_log_level);
-  std::shared_ptr<rocksdb::LoggerJniCallback> *pLoggerJniCallback =
-      new std::shared_ptr<rocksdb::LoggerJniCallback>;
-  *pLoggerJniCallback = std::shared_ptr<rocksdb::LoggerJniCallback>(c);
-  return reinterpret_cast<jlong>(pLoggerJniCallback);
+  auto* db_options = reinterpret_cast<rocksdb::DBOptions*>(jdb_options);
+  sptr_logger->get()->SetInfoLogLevel(db_options->info_log_level);
+
+  return reinterpret_cast<jlong>(sptr_logger);
 }
 
 /*
@@ -204,9 +282,10 @@ jlong Java_org_rocksdb_Logger_createNewLoggerDbOptions(
  */
 void Java_org_rocksdb_Logger_setInfoLogLevel(
     JNIEnv* env, jobject jobj, jlong jhandle, jbyte jlog_level) {
-  std::shared_ptr<rocksdb::LoggerJniCallback> *handle =
+  auto* handle =
       reinterpret_cast<std::shared_ptr<rocksdb::LoggerJniCallback> *>(jhandle);
-  (*handle)->SetInfoLogLevel(static_cast<rocksdb::InfoLogLevel>(jlog_level));
+  handle->get()->
+      SetInfoLogLevel(static_cast<rocksdb::InfoLogLevel>(jlog_level));
 }
 
 /*
@@ -216,9 +295,9 @@ void Java_org_rocksdb_Logger_setInfoLogLevel(
  */
 jbyte Java_org_rocksdb_Logger_infoLogLevel(
     JNIEnv* env, jobject jobj, jlong jhandle) {
-  std::shared_ptr<rocksdb::LoggerJniCallback> *handle =
+  auto* handle =
       reinterpret_cast<std::shared_ptr<rocksdb::LoggerJniCallback> *>(jhandle);
-  return static_cast<jbyte>((*handle)->GetInfoLogLevel());
+  return static_cast<jbyte>(handle->get()->GetInfoLogLevel());
 }
 
 /*
@@ -228,7 +307,7 @@ jbyte Java_org_rocksdb_Logger_infoLogLevel(
  */
 void Java_org_rocksdb_Logger_disposeInternal(
     JNIEnv* env, jobject jobj, jlong jhandle) {
-  std::shared_ptr<rocksdb::LoggerJniCallback> *handle =
+  auto* handle =
       reinterpret_cast<std::shared_ptr<rocksdb::LoggerJniCallback> *>(jhandle);
-  handle->reset();
+  delete handle;  // delete std::shared_ptr
 }

java/rocksjni/loggerjnicallback.h

@@ -10,6 +10,7 @@
 #define JAVA_ROCKSJNI_LOGGERJNICALLBACK_H_
 
 #include <jni.h>
+#include <memory>
 #include <string>
 #include "port/port.h"
 #include "rocksdb/env.h"
@@ -32,8 +33,6 @@ namespace rocksdb {
      virtual void Logv(const InfoLogLevel log_level,
          const char* format, va_list ap);
 
-   protected:
-     JNIEnv* getJniEnv() const;
    private:
      JavaVM* m_jvm;
      jobject m_jLogger;
@@ -44,6 +43,7 @@ namespace rocksdb {
      jobject m_jerror_level;
      jobject m_jfatal_level;
      jobject m_jheader_level;
+     std::unique_ptr<char[]> format_str(const char* format, va_list ap) const;
   };
 }  // namespace rocksdb
 

java/rocksjni/merge_operator.cc

@@ -25,13 +25,24 @@
 
 /*
  * Class:     org_rocksdb_StringAppendOperator
- * Method:    newMergeOperatorHandle
+ * Method:    newSharedStringAppendOperator
  * Signature: ()J
  */
-jlong Java_org_rocksdb_StringAppendOperator_newMergeOperatorHandleImpl
-(JNIEnv* env, jobject jobj) {
-  std::shared_ptr<rocksdb::MergeOperator> *op =
-    new std::shared_ptr<rocksdb::MergeOperator>();
-  *op = rocksdb::MergeOperators::CreateFromStringId("stringappend");
-  return reinterpret_cast<jlong>(op);
+jlong Java_org_rocksdb_StringAppendOperator_newSharedStringAppendOperator
+(JNIEnv* env, jclass jclazz) {
+  auto* sptr_string_append_op = new std::shared_ptr<rocksdb::MergeOperator>(
+    rocksdb::MergeOperators::CreateFromStringId("stringappend"));
+  return reinterpret_cast<jlong>(sptr_string_append_op);
+}
+
+/*
+ * Class:     org_rocksdb_StringAppendOperator
+ * Method:    disposeInternal
+ * Signature: (J)V
+ */
+void Java_org_rocksdb_StringAppendOperator_disposeInternal(
+    JNIEnv* env, jobject jobj, jlong jhandle) {
+  auto* sptr_string_append_op =
+      reinterpret_cast<std::shared_ptr<rocksdb::MergeOperator>* >(jhandle);
+  delete sptr_string_append_op;  // delete std::shared_ptr
 }

java/rocksjni/options.cc

@@ -39,7 +39,7 @@
  * Signature: ()J
  */
 jlong Java_org_rocksdb_Options_newOptions__(JNIEnv* env, jclass jcls) {
-  rocksdb::Options* op = new rocksdb::Options();
+  auto* op = new rocksdb::Options();
   return reinterpret_cast<jlong>(op);
 }
 
@@ -53,7 +53,7 @@ jlong Java_org_rocksdb_Options_newOptions__JJ(JNIEnv* env, jclass jcls,
   auto* dbOpt = reinterpret_cast<const rocksdb::DBOptions*>(jdboptions);
   auto* cfOpt = reinterpret_cast<const rocksdb::ColumnFamilyOptions*>(
       jcfoptions);
-  rocksdb::Options* op = new rocksdb::Options(*dbOpt, *cfOpt);
+  auto* op = new rocksdb::Options(*dbOpt, *cfOpt);
   return reinterpret_cast<jlong>(op);
 }
 
@@ -64,7 +64,9 @@ jlong Java_org_rocksdb_Options_newOptions__JJ(JNIEnv* env, jclass jcls,
  */
 void Java_org_rocksdb_Options_disposeInternal(
     JNIEnv* env, jobject jobj, jlong handle) {
-  delete reinterpret_cast<rocksdb::Options*>(handle);
+  auto* op = reinterpret_cast<rocksdb::Options*>(handle);
+  assert(op != nullptr);
+  delete op;
 }
 
 /*
@@ -157,10 +159,16 @@ void Java_org_rocksdb_Options_setComparatorHandle__JJ(
  */
 void Java_org_rocksdb_Options_setMergeOperatorName(
     JNIEnv* env, jobject jobj, jlong jhandle, jstring jop_name) {
-  auto options = reinterpret_cast<rocksdb::Options*>(jhandle);
-  const char* op_name = env->GetStringUTFChars(jop_name, 0);
+  const char* op_name = env->GetStringUTFChars(jop_name, nullptr);
+  if(op_name == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return;
+  }
+
+  auto* options = reinterpret_cast<rocksdb::Options*>(jhandle);
   options->merge_operator = rocksdb::MergeOperators::CreateFromStringId(
         op_name);
+
   env->ReleaseStringUTFChars(jop_name, op_name);
 }
 
@@ -231,7 +239,7 @@ void Java_org_rocksdb_Options_createStatistics(
  */
 jlong Java_org_rocksdb_Options_statisticsPtr(
     JNIEnv* env, jobject jobj, jlong jOptHandle) {
-  auto st = reinterpret_cast<rocksdb::Options*>(jOptHandle)->statistics.get();
+  auto* st = reinterpret_cast<rocksdb::Options*>(jOptHandle)->statistics.get();
   return reinterpret_cast<jlong>(st);
 }
 
@@ -342,27 +350,6 @@ void Java_org_rocksdb_Options_setMaxOpenFiles(
       static_cast<int>(max_open_files);
 }
 
-/*
- * Class:     org_rocksdb_Options
- * Method:    disableDataSync
- * Signature: (J)Z
- */
-jboolean Java_org_rocksdb_Options_disableDataSync(
-    JNIEnv* env, jobject jobj, jlong jhandle) {
-  return reinterpret_cast<rocksdb::Options*>(jhandle)->disableDataSync;
-}
-
-/*
- * Class:     org_rocksdb_Options
- * Method:    setDisableDataSync
- * Signature: (JZ)V
- */
-void Java_org_rocksdb_Options_setDisableDataSync(
-    JNIEnv* env, jobject jobj, jlong jhandle, jboolean disableDataSync) {
-  reinterpret_cast<rocksdb::Options*>(jhandle)->disableDataSync =
-      static_cast<bool>(disableDataSync);
-}
-
 /*
  * Class:     org_rocksdb_Options
  * Method:    useFsync
@@ -402,7 +389,11 @@ jstring Java_org_rocksdb_Options_dbLogDir(
  */
 void Java_org_rocksdb_Options_setDbLogDir(
     JNIEnv* env, jobject jobj, jlong jhandle, jstring jdb_log_dir) {
-  const char* log_dir = env->GetStringUTFChars(jdb_log_dir, 0);
+  const char* log_dir = env->GetStringUTFChars(jdb_log_dir, nullptr);
+  if(log_dir == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return;
+  }
   reinterpret_cast<rocksdb::Options*>(jhandle)->db_log_dir.assign(log_dir);
   env->ReleaseStringUTFChars(jdb_log_dir, log_dir);
 }
@@ -425,7 +416,11 @@ jstring Java_org_rocksdb_Options_walDir(
  */
 void Java_org_rocksdb_Options_setWalDir(
     JNIEnv* env, jobject jobj, jlong jhandle, jstring jwal_dir) {
-  const char* wal_dir = env->GetStringUTFChars(jwal_dir, 0);
+  const char* wal_dir = env->GetStringUTFChars(jwal_dir, nullptr);
+  if(wal_dir == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return;
+  }
   reinterpret_cast<rocksdb::Options*>(jhandle)->wal_dir.assign(wal_dir);
   env->ReleaseStringUTFChars(jwal_dir, wal_dir);
 }
@@ -515,8 +510,7 @@ void Java_org_rocksdb_Options_setMaxSubcompactions(
  */
 jint Java_org_rocksdb_Options_maxSubcompactions(
     JNIEnv* env, jobject jobj, jlong jhandle) {
-  return reinterpret_cast<rocksdb::Options*>(jhandle)
-      ->max_subcompactions;
+  return reinterpret_cast<rocksdb::Options*>(jhandle)->max_subcompactions;
 }
 
 /*
@@ -662,7 +656,7 @@ jlong Java_org_rocksdb_Options_maxManifestFileSize(
  */
 jstring Java_org_rocksdb_Options_memTableFactoryName(
     JNIEnv* env, jobject jobj, jlong jhandle) {
-  auto opt = reinterpret_cast<rocksdb::Options*>(jhandle);
+  auto* opt = reinterpret_cast<rocksdb::Options*>(jhandle);
   rocksdb::MemTableRepFactory* tf = opt->memtable_factory.get();
 
   // Should never be nullptr.
@@ -698,17 +692,6 @@ void Java_org_rocksdb_Options_setMemTableFactory(
       reinterpret_cast<rocksdb::MemTableRepFactory*>(jfactory_handle));
 }
 
-/*
- * Class:     org_rocksdb_Options
- * Method:    setOldRateLimiter
- * Signature: (JJ)V
- */
-void Java_org_rocksdb_Options_setOldRateLimiter(
-    JNIEnv* env, jobject jobj, jlong jhandle, jlong jrate_limiter_handle) {
-  reinterpret_cast<rocksdb::Options*>(jhandle)->rate_limiter.reset(
-      reinterpret_cast<rocksdb::RateLimiter*>(jrate_limiter_handle));
-}
-
 /*
  * Class:     org_rocksdb_Options
  * Method:    setRateLimiter
@@ -1165,7 +1148,7 @@ jlong Java_org_rocksdb_Options_writeThreadSlowYieldUsec(
  */
 jstring Java_org_rocksdb_Options_tableFactoryName(
     JNIEnv* env, jobject jobj, jlong jhandle) {
-  auto opt = reinterpret_cast<rocksdb::Options*>(jhandle);
+  auto* opt = reinterpret_cast<rocksdb::Options*>(jhandle);
   rocksdb::TableFactory* tf = opt->table_factory.get();
 
   // Should never be nullptr.
@@ -1245,46 +1228,78 @@ jbyte Java_org_rocksdb_Options_compressionType(
   return reinterpret_cast<rocksdb::Options*>(jhandle)->compression;
 }
 
-/*
- * Helper method to convert a Java list to a CompressionType
- * vector.
+/**
+ * Helper method to convert a Java byte array of compression levels
+ * to a C++ vector of rocksdb::CompressionType
+ *
+ * @param env A pointer to the Java environment
+ * @param jcompression_levels A reference to a java byte array
+ *     where each byte indicates a compression level
+ *
+ * @return A unique_ptr to the vector, or unique_ptr(nullptr) if a JNI exception occurs
  */
-std::vector<rocksdb::CompressionType> rocksdb_compression_vector_helper(
-    JNIEnv* env, jbyteArray jcompressionLevels) {
-  std::vector<rocksdb::CompressionType> compressionLevels;
-
-  jsize len = env->GetArrayLength(jcompressionLevels);
-  jbyte* jcompressionLevel = env->GetByteArrayElements(jcompressionLevels,
-    NULL);
-  for(int i = 0; i < len; i++) {
-    jbyte jcl;
-    jcl = jcompressionLevel[i];
-    compressionLevels.push_back(static_cast<rocksdb::CompressionType>(jcl));
+std::unique_ptr<std::vector<rocksdb::CompressionType>> rocksdb_compression_vector_helper(
+    JNIEnv* env, jbyteArray jcompression_levels) {
+  jsize len = env->GetArrayLength(jcompression_levels);
+  jbyte* jcompression_level =
+      env->GetByteArrayElements(jcompression_levels, nullptr);
+  if(jcompression_level == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return std::unique_ptr<std::vector<rocksdb::CompressionType>>();
   }
-  env->ReleaseByteArrayElements(jcompressionLevels, jcompressionLevel,
+
+  auto* compression_levels = new std::vector<rocksdb::CompressionType>();
+  std::unique_ptr<std::vector<rocksdb::CompressionType>> uptr_compression_levels(compression_levels);
+
+  for(jsize i = 0; i < len; i++) {
+    jbyte jcl = jcompression_level[i];
+    compression_levels->push_back(static_cast<rocksdb::CompressionType>(jcl));
+  }
+
+  env->ReleaseByteArrayElements(jcompression_levels, jcompression_level,
       JNI_ABORT);
 
-  return compressionLevels;
+  return uptr_compression_levels;
 }
 
-/*
- * Helper method to convert a CompressionType vector to a Java
- * List.
+/**
+ * Helper method to convert a C++ vector of rocksdb::CompressionType
+ * to a Java byte array of compression levels
+ *
+ * @param env A pointer to the Java environment
+ * @param jcompression_levels A reference to a java byte array
+ *     where each byte indicates a compression level
+ *
+ * @return A jbytearray or nullptr if an exception occurs
  */
 jbyteArray rocksdb_compression_list_helper(JNIEnv* env,
-    std::vector<rocksdb::CompressionType> compressionLevels) {
-  std::unique_ptr<jbyte[]> jbuf =
-      std::unique_ptr<jbyte[]>(new jbyte[compressionLevels.size()]);
-  for (std::vector<rocksdb::CompressionType>::size_type i = 0;
-        i != compressionLevels.size(); i++) {
-      jbuf[i] = compressionLevels[i];
+    std::vector<rocksdb::CompressionType> compression_levels) {
+  const size_t len = compression_levels.size();
+  jbyte* jbuf = new jbyte[len];
+
+  for (size_t i = 0; i < len; i++) {
+      jbuf[i] = compression_levels[i];
   }
+
   // insert in java array
-  jbyteArray jcompressionLevels = env->NewByteArray(
-      static_cast<jsize>(compressionLevels.size()));
-  env->SetByteArrayRegion(jcompressionLevels, 0,
-      static_cast<jsize>(compressionLevels.size()), jbuf.get());
-  return jcompressionLevels;
+  jbyteArray jcompression_levels = env->NewByteArray(static_cast<jsize>(len));
+  if(jcompression_levels == nullptr) {
+      // exception thrown: OutOfMemoryError
+      delete [] jbuf;
+      return nullptr;
+  }
+  env->SetByteArrayRegion(jcompression_levels, 0, static_cast<jsize>(len),
+      jbuf);
+  if(env->ExceptionCheck()) {
+      // exception thrown: ArrayIndexOutOfBoundsException
+      env->DeleteLocalRef(jcompression_levels);
+      delete [] jbuf;
+      return nullptr;
+  }
+
+  delete [] jbuf;
+
+  return jcompression_levels;
 }
 
 /*
@@ -1295,10 +1310,14 @@ jbyteArray rocksdb_compression_list_helper(JNIEnv* env,
 void Java_org_rocksdb_Options_setCompressionPerLevel(
     JNIEnv* env, jobject jobj, jlong jhandle,
     jbyteArray jcompressionLevels) {
-  auto* options = reinterpret_cast<rocksdb::Options*>(jhandle);
-  std::vector<rocksdb::CompressionType> compressionLevels =
+  auto uptr_compression_levels =
       rocksdb_compression_vector_helper(env, jcompressionLevels);
-  options->compression_per_level = compressionLevels;
+  if(!uptr_compression_levels) {
+    // exception occurred
+    return;
+  }
+  auto* options = reinterpret_cast<rocksdb::Options*>(jhandle);
+  options->compression_per_level = *(uptr_compression_levels.get());
 }
 
 /*
@@ -1728,30 +1747,6 @@ void Java_org_rocksdb_Options_setPurgeRedundantKvsWhileFlush(
           static_cast<bool>(jpurge_redundant_kvs_while_flush);
 }
 
-/*
- * Class:     org_rocksdb_Options
- * Method:    verifyChecksumsInCompaction
- * Signature: (J)Z
- */
-jboolean Java_org_rocksdb_Options_verifyChecksumsInCompaction(
-    JNIEnv* env, jobject jobj, jlong jhandle) {
-  return reinterpret_cast<rocksdb::Options*>(
-      jhandle)->verify_checksums_in_compaction;
-}
-
-/*
- * Class:     org_rocksdb_Options
- * Method:    setVerifyChecksumsInCompaction
- * Signature: (JZ)V
- */
-void Java_org_rocksdb_Options_setVerifyChecksumsInCompaction(
-    JNIEnv* env, jobject jobj, jlong jhandle,
-    jboolean jverify_checksums_in_compaction) {
-  reinterpret_cast<rocksdb::Options*>(
-      jhandle)->verify_checksums_in_compaction =
-          static_cast<bool>(jverify_checksums_in_compaction);
-}
-
 /*
  * Class:     org_rocksdb_Options
  * Method:    maxSequentialSkipInIterations
@@ -1903,30 +1898,6 @@ void Java_org_rocksdb_Options_setMaxSuccessiveMerges(
   }
 }
 
-/*
- * Class:     org_rocksdb_Options
- * Method:    minPartialMergeOperands
- * Signature: (J)I
- */
-jint Java_org_rocksdb_Options_minPartialMergeOperands(
-    JNIEnv* env, jobject jobj, jlong jhandle) {
-  return reinterpret_cast<rocksdb::Options*>(
-      jhandle)->min_partial_merge_operands;
-}
-
-/*
- * Class:     org_rocksdb_Options
- * Method:    setMinPartialMergeOperands
- * Signature: (JI)V
- */
-void Java_org_rocksdb_Options_setMinPartialMergeOperands(
-    JNIEnv* env, jobject jobj, jlong jhandle,
-    jint jmin_partial_merge_operands) {
-  reinterpret_cast<rocksdb::Options*>(
-      jhandle)->min_partial_merge_operands =
-          static_cast<int32_t>(jmin_partial_merge_operands);
-}
-
 /*
  * Class:     org_rocksdb_Options
  * Method:    optimizeFiltersForHits
@@ -2015,7 +1986,6 @@ jlong Java_org_rocksdb_Options_memtableHugePageSize(
 void Java_org_rocksdb_Options_setMemtableHugePageSize(
     JNIEnv* env, jobject jobj, jlong jhandle,
     jlong jmemtable_huge_page_size) {
-
   rocksdb::Status s = rocksdb::check_if_jlong_fits_size_t(
       jmemtable_huge_page_size);
   if (s.ok()) {
@@ -2152,8 +2122,9 @@ void Java_org_rocksdb_Options_setLevel0StopWritesTrigger(
  */
 jintArray Java_org_rocksdb_Options_maxBytesForLevelMultiplierAdditional(
     JNIEnv* env, jobject jobj, jlong jhandle) {
-  auto mbflma = reinterpret_cast<rocksdb::Options*>(
-      jhandle)->max_bytes_for_level_multiplier_additional;
+  auto mbflma =
+      reinterpret_cast<rocksdb::Options*>(jhandle)->
+          max_bytes_for_level_multiplier_additional;
 
   const size_t size = mbflma.size();
 
@@ -2164,7 +2135,19 @@ jintArray Java_org_rocksdb_Options_maxBytesForLevelMultiplierAdditional(
 
   jsize jlen = static_cast<jsize>(size);
   jintArray result = env->NewIntArray(jlen);
+  if(result == nullptr) {
+      // exception thrown: OutOfMemoryError
+      delete [] additionals;
+      return nullptr;
+  }
+
   env->SetIntArrayRegion(result, 0, jlen, additionals);
+  if(env->ExceptionCheck()) {
+      // exception thrown: ArrayIndexOutOfBoundsException
+      env->DeleteLocalRef(result);
+      delete [] additionals;
+      return nullptr;
+  }
 
   delete [] additionals;
 
@@ -2181,12 +2164,20 @@ void Java_org_rocksdb_Options_setMaxBytesForLevelMultiplierAdditional(
     jintArray jmax_bytes_for_level_multiplier_additional) {
   jsize len = env->GetArrayLength(jmax_bytes_for_level_multiplier_additional);
   jint *additionals =
-      env->GetIntArrayElements(jmax_bytes_for_level_multiplier_additional, 0);
+      env->GetIntArrayElements(jmax_bytes_for_level_multiplier_additional, nullptr);
+  if(additionals == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return;
+  }
+
   auto* opt = reinterpret_cast<rocksdb::Options*>(jhandle);
   opt->max_bytes_for_level_multiplier_additional.clear();
   for (jsize i = 0; i < len; i++) {
     opt->max_bytes_for_level_multiplier_additional.push_back(static_cast<int32_t>(additionals[i]));
   }
+
+  env->ReleaseIntArrayElements(jmax_bytes_for_level_multiplier_additional,
+      additionals, JNI_ABORT);
 }
 
 /*
@@ -2222,7 +2213,7 @@ void Java_org_rocksdb_Options_setParanoidFileChecks(
  */
 jlong Java_org_rocksdb_ColumnFamilyOptions_newColumnFamilyOptions(
     JNIEnv* env, jclass jcls) {
-  rocksdb::ColumnFamilyOptions* op = new rocksdb::ColumnFamilyOptions();
+  auto* op = new rocksdb::ColumnFamilyOptions();
   return reinterpret_cast<jlong>(op);
 }
 
@@ -2233,14 +2224,20 @@ jlong Java_org_rocksdb_ColumnFamilyOptions_newColumnFamilyOptions(
  */
 jlong Java_org_rocksdb_ColumnFamilyOptions_getColumnFamilyOptionsFromProps(
     JNIEnv* env, jclass jclazz, jstring jopt_string) {
-  jlong ret_value = 0;
-  rocksdb::ColumnFamilyOptions* cf_options =
-      new rocksdb::ColumnFamilyOptions();
-  const char* opt_string = env->GetStringUTFChars(jopt_string, 0);
+  const char* opt_string = env->GetStringUTFChars(jopt_string, nullptr);
+  if(opt_string == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return 0;
+  }
+
+  auto* cf_options = new rocksdb::ColumnFamilyOptions();
   rocksdb::Status status = rocksdb::GetColumnFamilyOptionsFromString(
       rocksdb::ColumnFamilyOptions(), opt_string, cf_options);
+
   env->ReleaseStringUTFChars(jopt_string, opt_string);
+
   // Check if ColumnFamilyOptions creation was possible.
+  jlong ret_value = 0;
   if (status.ok()) {
     ret_value = reinterpret_cast<jlong>(cf_options);
   } else {
@@ -2258,7 +2255,9 @@ jlong Java_org_rocksdb_ColumnFamilyOptions_getColumnFamilyOptionsFromProps(
  */
 void Java_org_rocksdb_ColumnFamilyOptions_disposeInternal(
     JNIEnv* env, jobject jobj, jlong handle) {
-  delete reinterpret_cast<rocksdb::ColumnFamilyOptions*>(handle);
+  auto* cfo = reinterpret_cast<rocksdb::ColumnFamilyOptions*>(handle);
+  assert(cfo != nullptr);
+  delete cfo;
 }
 
 /*
@@ -2334,10 +2333,15 @@ void Java_org_rocksdb_ColumnFamilyOptions_setComparatorHandle__JJ(
  */
 void Java_org_rocksdb_ColumnFamilyOptions_setMergeOperatorName(
     JNIEnv* env, jobject jobj, jlong jhandle, jstring jop_name) {
-  auto options = reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jhandle);
-  const char* op_name = env->GetStringUTFChars(jop_name, 0);
-  options->merge_operator = rocksdb::MergeOperators::CreateFromStringId(
-        op_name);
+  auto* options = reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jhandle);
+  const char* op_name = env->GetStringUTFChars(jop_name, nullptr);
+  if(op_name == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return;
+  }
+
+  options->merge_operator =
+      rocksdb::MergeOperators::CreateFromStringId(op_name);
   env->ReleaseStringUTFChars(jop_name, op_name);
 }
 
@@ -2433,7 +2437,7 @@ void Java_org_rocksdb_ColumnFamilyOptions_setMemTableFactory(
  */
 jstring Java_org_rocksdb_ColumnFamilyOptions_memTableFactoryName(
     JNIEnv* env, jobject jobj, jlong jhandle) {
-  auto opt = reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jhandle);
+  auto* opt = reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jhandle);
   rocksdb::MemTableRepFactory* tf = opt->memtable_factory.get();
 
   // Should never be nullptr.
@@ -2487,7 +2491,7 @@ void Java_org_rocksdb_ColumnFamilyOptions_setTableFactory(
  */
 jstring Java_org_rocksdb_ColumnFamilyOptions_tableFactoryName(
     JNIEnv* env, jobject jobj, jlong jhandle) {
-  auto opt = reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jhandle);
+  auto* opt = reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jhandle);
   rocksdb::TableFactory* tf = opt->table_factory.get();
 
   // Should never be nullptr.
@@ -2577,9 +2581,13 @@ void Java_org_rocksdb_ColumnFamilyOptions_setCompressionPerLevel(
     JNIEnv* env, jobject jobj, jlong jhandle,
     jbyteArray jcompressionLevels) {
   auto* options = reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jhandle);
-  std::vector<rocksdb::CompressionType> compressionLevels =
+  auto uptr_compression_levels =
       rocksdb_compression_vector_helper(env, jcompressionLevels);
-  options->compression_per_level = compressionLevels;
+  if(!uptr_compression_levels) {
+      // exception occurred
+      return;
+  }
+  options->compression_per_level = *(uptr_compression_levels.get());
 }
 
 /*
@@ -2589,9 +2597,9 @@ void Java_org_rocksdb_ColumnFamilyOptions_setCompressionPerLevel(
  */
 jbyteArray Java_org_rocksdb_ColumnFamilyOptions_compressionPerLevel(
     JNIEnv* env, jobject jobj, jlong jhandle) {
-  auto* options = reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jhandle);
+  auto* cf_options = reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jhandle);
   return rocksdb_compression_list_helper(env,
-      options->compression_per_level);
+      cf_options->compression_per_level);
 }
 
 /*
@@ -2737,7 +2745,7 @@ void Java_org_rocksdb_ColumnFamilyOptions_setLevelZeroStopWritesTrigger(
  */
 jint Java_org_rocksdb_ColumnFamilyOptions_maxMemCompactionLevel(
     JNIEnv* env, jobject jobj, jlong jhandle) {
-  return 0;
+  return 0; // deprecated and intentionally not implemented, see the Java code
 }
 
 /*
@@ -2746,7 +2754,9 @@ jint Java_org_rocksdb_ColumnFamilyOptions_maxMemCompactionLevel(
  * Signature: (JI)V
  */
 void Java_org_rocksdb_ColumnFamilyOptions_setMaxMemCompactionLevel(
-    JNIEnv* env, jobject jobj, jlong jhandle, jint jmax_mem_compaction_level) {}
+    JNIEnv* env, jobject jobj, jlong jhandle, jint jmax_mem_compaction_level) {
+  // deprecated and intentionally not implemented, see the Java code
+}
 
 /*
  * Class:     org_rocksdb_ColumnFamilyOptions
@@ -3034,30 +3044,6 @@ void Java_org_rocksdb_ColumnFamilyOptions_setPurgeRedundantKvsWhileFlush(
           static_cast<bool>(jpurge_redundant_kvs_while_flush);
 }
 
-/*
- * Class:     org_rocksdb_ColumnFamilyOptions
- * Method:    verifyChecksumsInCompaction
- * Signature: (J)Z
- */
-jboolean Java_org_rocksdb_ColumnFamilyOptions_verifyChecksumsInCompaction(
-    JNIEnv* env, jobject jobj, jlong jhandle) {
-  return reinterpret_cast<rocksdb::ColumnFamilyOptions*>(
-      jhandle)->verify_checksums_in_compaction;
-}
-
-/*
- * Class:     org_rocksdb_ColumnFamilyOptions
- * Method:    setVerifyChecksumsInCompaction
- * Signature: (JZ)V
- */
-void Java_org_rocksdb_ColumnFamilyOptions_setVerifyChecksumsInCompaction(
-    JNIEnv* env, jobject jobj, jlong jhandle,
-    jboolean jverify_checksums_in_compaction) {
-  reinterpret_cast<rocksdb::ColumnFamilyOptions*>(
-      jhandle)->verify_checksums_in_compaction =
-          static_cast<bool>(jverify_checksums_in_compaction);
-}
-
 /*
  * Class:     org_rocksdb_ColumnFamilyOptions
  * Method:    maxSequentialSkipInIterations
@@ -3210,30 +3196,6 @@ void Java_org_rocksdb_ColumnFamilyOptions_setMaxSuccessiveMerges(
   }
 }
 
-/*
- * Class:     org_rocksdb_ColumnFamilyOptions
- * Method:    minPartialMergeOperands
- * Signature: (J)I
- */
-jint Java_org_rocksdb_ColumnFamilyOptions_minPartialMergeOperands(
-    JNIEnv* env, jobject jobj, jlong jhandle) {
-  return reinterpret_cast<rocksdb::ColumnFamilyOptions*>(
-      jhandle)->min_partial_merge_operands;
-}
-
-/*
- * Class:     org_rocksdb_ColumnFamilyOptions
- * Method:    setMinPartialMergeOperands
- * Signature: (JI)V
- */
-void Java_org_rocksdb_ColumnFamilyOptions_setMinPartialMergeOperands(
-    JNIEnv* env, jobject jobj, jlong jhandle,
-    jint jmin_partial_merge_operands) {
-  reinterpret_cast<rocksdb::ColumnFamilyOptions*>(
-      jhandle)->min_partial_merge_operands =
-          static_cast<int32_t>(jmin_partial_merge_operands);
-}
-
 /*
  * Class:     org_rocksdb_ColumnFamilyOptions
  * Method:    optimizeFiltersForHits
@@ -3425,9 +3387,19 @@ jintArray Java_org_rocksdb_ColumnFamilyOptions_maxBytesForLevelMultiplierAdditio
   }
 
   jsize jlen = static_cast<jsize>(size);
-  jintArray result;
-  result = env->NewIntArray(jlen);
+  jintArray result = env->NewIntArray(jlen);
+  if(result == nullptr) {
+    // exception thrown: OutOfMemoryError
+    delete [] additionals;
+    return nullptr;
+  }
   env->SetIntArrayRegion(result, 0, jlen, additionals);
+  if(env->ExceptionCheck()) {
+      // exception thrown: ArrayIndexOutOfBoundsException
+      env->DeleteLocalRef(result);
+      delete [] additionals;
+      return nullptr;
+  }
 
   delete [] additionals;
 
@@ -3445,11 +3417,19 @@ void Java_org_rocksdb_ColumnFamilyOptions_setMaxBytesForLevelMultiplierAdditiona
   jsize len = env->GetArrayLength(jmax_bytes_for_level_multiplier_additional);
   jint *additionals =
       env->GetIntArrayElements(jmax_bytes_for_level_multiplier_additional, 0);
+  if(additionals == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return;
+  }
+
   auto* cf_opt = reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jhandle);
   cf_opt->max_bytes_for_level_multiplier_additional.clear();
   for (jsize i = 0; i < len; i++) {
     cf_opt->max_bytes_for_level_multiplier_additional.push_back(static_cast<int32_t>(additionals[i]));
   }
+
+  env->ReleaseIntArrayElements(jmax_bytes_for_level_multiplier_additional,
+      additionals, JNI_ABORT);
 }
 
 /*
@@ -3486,7 +3466,7 @@ void Java_org_rocksdb_ColumnFamilyOptions_setParanoidFileChecks(
  */
 jlong Java_org_rocksdb_DBOptions_newDBOptions(JNIEnv* env,
     jclass jcls) {
-  rocksdb::DBOptions* dbop = new rocksdb::DBOptions();
+  auto* dbop = new rocksdb::DBOptions();
   return reinterpret_cast<jlong>(dbop);
 }
 
@@ -3497,14 +3477,20 @@ jlong Java_org_rocksdb_DBOptions_newDBOptions(JNIEnv* env,
  */
 jlong Java_org_rocksdb_DBOptions_getDBOptionsFromProps(
     JNIEnv* env, jclass jclazz, jstring jopt_string) {
-  jlong ret_value = 0;
-  rocksdb::DBOptions* db_options =
-      new rocksdb::DBOptions();
-  const char* opt_string = env->GetStringUTFChars(jopt_string, 0);
+  const char* opt_string = env->GetStringUTFChars(jopt_string, nullptr);
+  if(opt_string == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return 0;
+  }
+
+  auto* db_options = new rocksdb::DBOptions();
   rocksdb::Status status = rocksdb::GetDBOptionsFromString(
       rocksdb::DBOptions(), opt_string, db_options);
+
   env->ReleaseStringUTFChars(jopt_string, opt_string);
+
   // Check if DBOptions creation was possible.
+  jlong ret_value = 0;
   if (status.ok()) {
     ret_value = reinterpret_cast<jlong>(db_options);
   } else {
@@ -3522,7 +3508,9 @@ jlong Java_org_rocksdb_DBOptions_getDBOptionsFromProps(
  */
 void Java_org_rocksdb_DBOptions_disposeInternal(
     JNIEnv* env, jobject jobj, jlong handle) {
-  delete reinterpret_cast<rocksdb::DBOptions*>(handle);
+  auto* dbo = reinterpret_cast<rocksdb::DBOptions*>(handle);
+  assert(dbo != nullptr);
+  delete dbo;
 }
 
 /*
@@ -3622,17 +3610,6 @@ jboolean Java_org_rocksdb_DBOptions_paranoidChecks(
   return reinterpret_cast<rocksdb::DBOptions*>(jhandle)->paranoid_checks;
 }
 
-/*
- * Class:     org_rocksdb_DBOptions
- * Method:    setOldRateLimiter
- * Signature: (JJ)V
- */
-void Java_org_rocksdb_DBOptions_setOldRateLimiter(
-    JNIEnv* env, jobject jobj, jlong jhandle, jlong jrate_limiter_handle) {
-  reinterpret_cast<rocksdb::DBOptions*>(jhandle)->rate_limiter.reset(
-      reinterpret_cast<rocksdb::RateLimiter*>(jrate_limiter_handle));
-}
-
 /*
  * Class:     org_rocksdb_DBOptions
  * Method:    setRateLimiter
@@ -3743,32 +3720,11 @@ void Java_org_rocksdb_DBOptions_createStatistics(
  */
 jlong Java_org_rocksdb_DBOptions_statisticsPtr(
     JNIEnv* env, jobject jobj, jlong jOptHandle) {
-  auto st = reinterpret_cast<rocksdb::DBOptions*>(jOptHandle)->
+  auto* st = reinterpret_cast<rocksdb::DBOptions*>(jOptHandle)->
       statistics.get();
   return reinterpret_cast<jlong>(st);
 }
 
-/*
- * Class:     org_rocksdb_DBOptions
- * Method:    setDisableDataSync
- * Signature: (JZ)V
- */
-void Java_org_rocksdb_DBOptions_setDisableDataSync(
-    JNIEnv* env, jobject jobj, jlong jhandle, jboolean disableDataSync) {
-  reinterpret_cast<rocksdb::DBOptions*>(jhandle)->disableDataSync =
-      static_cast<bool>(disableDataSync);
-}
-
-/*
- * Class:     org_rocksdb_DBOptions
- * Method:    disableDataSync
- * Signature: (J)Z
- */
-jboolean Java_org_rocksdb_DBOptions_disableDataSync(
-    JNIEnv* env, jobject jobj, jlong jhandle) {
-  return reinterpret_cast<rocksdb::DBOptions*>(jhandle)->disableDataSync;
-}
-
 /*
  * Class:     org_rocksdb_DBOptions
  * Method:    setUseFsync
@@ -3797,7 +3753,12 @@ jboolean Java_org_rocksdb_DBOptions_useFsync(
  */
 void Java_org_rocksdb_DBOptions_setDbLogDir(
     JNIEnv* env, jobject jobj, jlong jhandle, jstring jdb_log_dir) {
-  const char* log_dir = env->GetStringUTFChars(jdb_log_dir, 0);
+  const char* log_dir = env->GetStringUTFChars(jdb_log_dir, nullptr);
+  if(log_dir == nullptr) {
+    // exception thrown: OutOfMemoryError
+    return;
+  }
+
   reinterpret_cast<rocksdb::DBOptions*>(jhandle)->db_log_dir.assign(log_dir);
   env->ReleaseStringUTFChars(jdb_log_dir, log_dir);
 }
@@ -4445,19 +4406,17 @@ jlong Java_org_rocksdb_DBOptions_writeThreadSlowYieldUsec(
 }
 
 void Java_org_rocksdb_DBOptions_setDelayedWriteRate(
-  JNIEnv* env, jobject jobj, jlong jhandle, jlong delay_write_rate){
+    JNIEnv* env, jobject jobj, jlong jhandle, jlong delay_write_rate) {
+  reinterpret_cast<rocksdb::DBOptions*>(jhandle)->delayed_write_rate =
+      static_cast<int64_t>(delay_write_rate);
+}
 
-    reinterpret_cast<rocksdb::DBOptions*>(jhandle)->
-        delayed_write_rate = static_cast<int64_t>(delay_write_rate);
+jlong Java_org_rocksdb_DBOptions_delayedWriteRate(
+    JNIEnv* env, jobject jobj, jlong jhandle) {
+  return reinterpret_cast<rocksdb::DBOptions*>(jhandle)->
+      delayed_write_rate;
+}
 
-  }
-
-  jlong Java_org_rocksdb_DBOptions_delayedWriteRate(
-    JNIEnv* env, jobject jobj, jlong jhandle){
-
-      return reinterpret_cast<rocksdb::DBOptions*>(jhandle)->
-          delayed_write_rate;
-    }
 //////////////////////////////////////////////////////////////////////////////
 // rocksdb::WriteOptions
 
@@ -4468,7 +4427,7 @@ void Java_org_rocksdb_DBOptions_setDelayedWriteRate(
  */
 jlong Java_org_rocksdb_WriteOptions_newWriteOptions(
     JNIEnv* env, jclass jcls) {
-  rocksdb::WriteOptions* op = new rocksdb::WriteOptions();
+  auto* op = new rocksdb::WriteOptions();
   return reinterpret_cast<jlong>(op);
 }
 
@@ -4479,7 +4438,8 @@ jlong Java_org_rocksdb_WriteOptions_newWriteOptions(
  */
 void Java_org_rocksdb_WriteOptions_disposeInternal(
     JNIEnv* env, jobject jwrite_options, jlong jhandle) {
-  auto write_options = reinterpret_cast<rocksdb::WriteOptions*>(jhandle);
+  auto* write_options = reinterpret_cast<rocksdb::WriteOptions*>(jhandle);
+  assert(write_options != nullptr);
   delete write_options;
 }
 
@@ -4533,8 +4493,8 @@ jboolean Java_org_rocksdb_WriteOptions_disableWAL(
  */
 jlong Java_org_rocksdb_ReadOptions_newReadOptions(
     JNIEnv* env, jclass jcls) {
-  auto read_opt = new rocksdb::ReadOptions();
-  return reinterpret_cast<jlong>(read_opt);
+  auto* read_options = new rocksdb::ReadOptions();
+  return reinterpret_cast<jlong>(read_options);
 }
 
 /*
@@ -4544,7 +4504,9 @@ jlong Java_org_rocksdb_ReadOptions_newReadOptions(
  */
 void Java_org_rocksdb_ReadOptions_disposeInternal(
     JNIEnv* env, jobject jobj, jlong jhandle) {
-  delete reinterpret_cast<rocksdb::ReadOptions*>(jhandle);
+  auto* read_options = reinterpret_cast<rocksdb::ReadOptions*>(jhandle);
+  assert(read_options != nullptr);
+  delete read_options;
 }
 
 /*
@@ -4751,7 +4713,7 @@ void Java_org_rocksdb_ReadOptions_setReadTier(
  */
 jlong Java_org_rocksdb_ComparatorOptions_newComparatorOptions(
     JNIEnv* env, jclass jcls) {
-  auto comparator_opt = new rocksdb::ComparatorJniCallbackOptions();
+  auto* comparator_opt = new rocksdb::ComparatorJniCallbackOptions();
   return reinterpret_cast<jlong>(comparator_opt);
 }
 
@@ -4784,7 +4746,10 @@ void Java_org_rocksdb_ComparatorOptions_setUseAdaptiveMutex(
  */
 void Java_org_rocksdb_ComparatorOptions_disposeInternal(
     JNIEnv * env, jobject jobj, jlong jhandle) {
-  delete reinterpret_cast<rocksdb::ComparatorJniCallbackOptions*>(jhandle);
+  auto* comparator_opt =
+      reinterpret_cast<rocksdb::ComparatorJniCallbackOptions*>(jhandle);
+  assert(comparator_opt != nullptr);
+  delete comparator_opt;
 }
 
 /////////////////////////////////////////////////////////////////////
@@ -4797,7 +4762,7 @@ void Java_org_rocksdb_ComparatorOptions_disposeInternal(
  */
 jlong Java_org_rocksdb_FlushOptions_newFlushOptions(
     JNIEnv* env, jclass jcls) {
-  auto flush_opt = new rocksdb::FlushOptions();
+  auto* flush_opt = new rocksdb::FlushOptions();
   return reinterpret_cast<jlong>(flush_opt);
 }
 
@@ -4830,5 +4795,7 @@ jboolean Java_org_rocksdb_FlushOptions_waitForFlush(
  */
 void Java_org_rocksdb_FlushOptions_disposeInternal(
     JNIEnv * env, jobject jobj, jlong jhandle) {
-  delete reinterpret_cast<rocksdb::FlushOptions*>(jhandle);
+  auto* flush_opt = reinterpret_cast<rocksdb::FlushOptions*>(jhandle);
+  assert(flush_opt != nullptr);
+  delete flush_opt;
 }