release 5.6.2

Summary:
`KeyNotExistsBeyondOutputLevel` didn't consider L0 files' key-ranges. So if a key only was covered by older L0 files' key-ranges, we would incorrectly drop deletions of that key. This PR just skips the deletion-dropping optimization when output level is L0.
Closes https://github.com/facebook/rocksdb/pull/2726

Differential Revision: D5617286

Pulled By: ajkr

fbshipit-source-id: 4bff1396b06d49a828ba4542f249191052915bce

.gitignore

@@ -31,6 +31,7 @@ build/
 ldb
 manifest_dump
 sst_dump
+blob_dump
 column_aware_encoding_exp
 util/build_version.cc
 build_tools/VALGRIND_LOGS/

.travis.yml

@@ -7,6 +7,7 @@ os:
 compiler:
   - clang
   - gcc
+osx_image: xcode8.3
 jdk:
   - oraclejdk7
 cache:
@@ -15,56 +16,55 @@ cache:
 
 addons:
    apt:
-      sources: ['ubuntu-toolchain-r-test', 'llvm-toolchain-precise-3.6']
-      packages: ['clang-3.6' , 'g++-6', 'zlib1g-dev', 'libbz2-dev', 'libsnappy-dev', 'curl', 'libgflags-dev', 'mingw-w64']
+      packages: ['zlib1g-dev', 'libbz2-dev', 'libsnappy-dev', 'curl', 'libgflags-dev', 'mingw-w64']
 env:
-  - TEST_GROUP=platform_dependent1
-  - TEST_GROUP=platform_dependent2
-  - TEST_GROUP=platform_dependent3
-  - TEST_GROUP=1
-  - TEST_GROUP=2
+  - TEST_GROUP=platform_dependent # 16-18 minutes
+  - TEST_GROUP=1 # 33-35 minutes
+  - TEST_GROUP=2 # 30-32 minutes
   # Run java tests
-  - JOB_NAME=java_test
+  - JOB_NAME=java_test # 4-11 minutes
   # Build ROCKSDB_LITE
-  - JOB_NAME=lite_build
+  - JOB_NAME=lite_build # 3-4 minutes
   # Build examples
-  - JOB_NAME=examples
-  - JOB_NAME=cmake
-  - JOB_NAME=cmake-mingw
+  - JOB_NAME=examples # 5-7 minutes
+  - JOB_NAME=cmake # 3-5 minutes
+  - JOB_NAME=cmake-mingw # 3 minutes
 
 matrix:
   exclude:
-  - os: osx
-    compiler: gcc
   - os: osx
     env: TEST_GROUP=1
   - os: osx
     env: TEST_GROUP=2
   - os : osx
     env: JOB_NAME=cmake-mingw
-  - os: linux
-    env: JOB_NAME=cmake-mingw
+  - os : linux
     compiler: clang
+  - os : osx
+    compiler: gcc
+
+# https://docs.travis-ci.com/user/caching/#ccache-cache
+install:
+  - if [ "${TRAVIS_OS_NAME}" == osx ]; then
+      brew install ccache;
+      PATH=$PATH:/usr/local/opt/ccache/libexec;
+    fi
 
 before_script:
-  - if [[ "${TRAVIS_OS_NAME}" == 'linux' && "${CXX}" == 'clang++' ]]; then CXX=clang++-3.6; fi
-  # test one linux g++ build with g++-6
-  - if [[ "${TRAVIS_OS_NAME}" == 'linux' && "${CXX}" == 'g++' && "${JOB_NAME}" == 'unittests' ]]; then CXX=g++-6; fi
-  # Limit the maximum number of open file descriptors to 8192
-  - ulimit -n 8192 || true
+  # Increase the maximum number of open file descriptors, since some tests use
+  # more FDs than the default limit.
+  - ulimit -n 8192
 
 script:
   - ${CXX} --version
-  - if [[ "${TEST_GROUP}" == 'platform_dependent1' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_END=external_sst_file_basic_test make -j4 check_some; fi
-  - if [[ "${TEST_GROUP}" == 'platform_dependent2' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=external_sst_file_basic_test ROCKSDBTESTS_END=checkpoint_test make -j4 check_some; fi
-  - if [[ "${TEST_GROUP}" == 'platform_dependent3' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=checkpoint_test ROCKSDBTESTS_END=db_block_cache_test make -j4 check_some; fi
-  - if [[ "${TEST_GROUP}" == '1' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=db_block_cache_test ROCKSDBTESTS_END=comparator_db_test make -j4 check_some; fi
-  - if [[ "${TEST_GROUP}" == '2' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=comparator_db_test make -j4 check_some; fi
-  - if [[ "${JOB_NAME}" == 'java_test' ]]; then OPT=-DTRAVIS V=1 make clean jclean && make rocksdbjava jtest; fi
-  - if [[ "${JOB_NAME}" == 'lite_build' ]]; then OPT="-DTRAVIS -DROCKSDB_LITE" V=1 make -j4 static_lib; fi
-  - if [[ "${JOB_NAME}" == 'examples' ]]; then OPT=-DTRAVIS V=1 make -j4 static_lib; cd examples; make -j4; fi
-  - if [[ "${JOB_NAME}" == 'cmake' ]]; then mkdir build && cd build && cmake .. && make -j4 rocksdb; fi
-  - if [[ "${JOB_NAME}" == 'cmake-mingw' ]]; then mkdir build && cd build && cmake .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb; fi
+  - if [ "${TEST_GROUP}" == 'platform_dependent' ]; then OPT=-DTRAVIS V=1 make -j4 all; OPT=-DTRAVIS V=1 ROCKSDBTESTS_END=db_block_cache_test make -j4 check_some; fi
+  - if [ "${TEST_GROUP}" == '1' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=db_block_cache_test ROCKSDBTESTS_END=comparator_db_test make -j4 check_some; fi
+  - if [ "${TEST_GROUP}" == '2' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=comparator_db_test make -j4 check_some; fi
+  - if [ "${JOB_NAME}" == 'java_test' ]; then OPT=-DTRAVIS V=1 make clean jclean && make rocksdbjava jtest; fi
+  - if [ "${JOB_NAME}" == 'lite_build' ]; then OPT="-DTRAVIS -DROCKSDB_LITE" V=1 make -j4 static_lib; fi
+  - if [ "${JOB_NAME}" == 'examples' ]; then OPT=-DTRAVIS V=1 make -j4 static_lib; cd examples; make -j4; fi
+  - if [ "${JOB_NAME}" == 'cmake' ]; then mkdir build && cd build && cmake .. && make -j4 rocksdb; fi
+  - if [ "${JOB_NAME}" == 'cmake-mingw' ]; then mkdir build && cd build && cmake .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb; fi
 notifications:
     email:
       - leveldb@fb.com

CMakeLists.txt

@@ -106,29 +106,56 @@ if(NOT WIN32)
   string(STRIP "${ROCKSDB_VERSION_MAJOR}" ROCKSDB_VERSION_MAJOR)
 endif()
 
-if(WIN32)
-  option(WITH_AVX2 "build with AVX2" ON)
-  if(WITH_AVX2)
-    if(MSVC)
-      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2")
-    else()
-      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2")
-    endif()
+option(WITH_MD_LIBRARY "build with MD" ON)
+if(WIN32 AND MSVC)
+  if(WITH_MD_LIBRARY)
+    set(RUNTIME_LIBRARY "MD")
+  else()
+    set(RUNTIME_LIBRARY "MT")
   endif()
+endif()
 
-  option(WITH_MD_LIBRARY "build with MD" ON)
-  if(MSVC)
-    if(WITH_MD_LIBRARY)
-      set(RUNTIME_LIBRARY "MD")
-    else()
-      set(RUNTIME_LIBRARY "MT")
-    endif()
-  endif()
-else()
-  option(WITH_SSE42 "build with SSE4.2" ON)
-  if(WITH_SSE42)
+option(PORTABLE "build a portable binary" OFF)
+option(FORCE_SSE42 "force building with SSE4.2, even when PORTABLE=ON" OFF)
+if(PORTABLE)
+  # MSVC does not need a separate compiler flag to enable SSE4.2; if nmmintrin.h
+  # is available, it is available by default.
+  if(FORCE_SSE42 AND NOT MSVC)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2")
   endif()
+else()
+  if(MSVC)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2")
+  else()
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
+  endif()
+endif()
+
+set(CMAKE_REQUIRED_FLAGS ${CMAKE_CXX_FLAGS})
+include(CheckCXXSourceCompiles)
+CHECK_CXX_SOURCE_COMPILES("
+#include <cstdint>
+#include <nmmintrin.h>
+int main() {
+  volatile uint32_t x = _mm_crc32_u32(0, 0);
+}
+" HAVE_SSE42)
+if(HAVE_SSE42)
+  add_definitions(-DHAVE_SSE42)
+elseif(FORCE_SSE42)
+  message(FATAL_ERROR "FORCE_SSE42=ON but unable to compile with SSE4.2 enabled")
+endif()
+
+CHECK_CXX_SOURCE_COMPILES("
+#if defined(_MSC_VER) && !defined(__thread)
+#define __thread __declspec(thread)
+#endif
+int main() {
+  static __thread int tls;
+}
+" HAVE_THREAD_LOCAL)
+if(HAVE_THREAD_LOCAL)
+  add_definitions(-DROCKSDB_SUPPORT_THREAD_LOCAL)
 endif()
 
 set(BUILD_VERSION_CC ${CMAKE_BINARY_DIR}/build_version.cc)
@@ -168,7 +195,6 @@ endif()
 
 option(WITH_ASAN "build with ASAN" OFF)
 if(WITH_ASAN)
-  add_definitions(-DROCKSDB_TSAN_RUN)
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address")
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address")
@@ -180,7 +206,6 @@ endif()
 option(WITH_TSAN "build with TSAN" OFF)
 if(WITH_TSAN)
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread -pie")
-  add_definitions(-DROCKSDB_TSAN_RUN)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread -fPIC")
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=thread -fPIC")
   if(WITH_JEMALLOC)
@@ -190,6 +215,7 @@ endif()
 
 option(WITH_UBSAN "build with UBSAN" OFF)
 if(WITH_UBSAN)
+  add_definitions(-DROCKSDB_UBSAN_RUN)
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined")
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=undefined")
@@ -332,6 +358,7 @@ set(SOURCES
         db/internal_stats.cc
         db/log_reader.cc
         db/log_writer.cc
+        db/malloc_stats.cc
         db/managed_iterator.cc
         db/memtable.cc
         db/memtable_list.cc
@@ -354,13 +381,14 @@ set(SOURCES
         env/env.cc
         env/env_chroot.cc
         env/env_hdfs.cc
-        env/memenv.cc
+        env/mock_env.cc
+        memtable/alloc_tracker.cc
         memtable/hash_cuckoo_rep.cc
         memtable/hash_linklist_rep.cc
         memtable/hash_skiplist_rep.cc
-        memtable/memtable_allocator.cc
         memtable/skiplistrep.cc
         memtable/vectorrep.cc
+        memtable/write_buffer_manager.cc
         monitoring/histogram.cc
         monitoring/histogram_windowing.cc
         monitoring/instrumented_mutex.cc
@@ -449,6 +477,7 @@ set(SOURCES
         utilities/blob_db/blob_db.cc
         utilities/blob_db/blob_db_impl.cc
         utilities/blob_db/blob_db_options_impl.cc
+        utilities/blob_db/blob_dump_tool.cc
         utilities/blob_db/blob_file.cc
         utilities/blob_db/blob_log_reader.cc
         utilities/blob_db/blob_log_writer.cc
@@ -471,6 +500,8 @@ set(SOURCES
         utilities/memory/memory_util.cc
         utilities/merge_operators/max.cc
         utilities/merge_operators/put.cc
+        utilities/merge_operators/cassandra/format.cc
+        utilities/merge_operators/cassandra/merge_operator.cc
         utilities/merge_operators/string_append/stringappend.cc
         utilities/merge_operators/string_append/stringappend2.cc
         utilities/merge_operators/uint64add.cc
@@ -521,7 +552,7 @@ if(WIN32)
   set(SYSTEM_LIBS ${SYSTEM_LIBS} Shlwapi.lib Rpcrt4.lib)
   set(LIBS ${ROCKSDB_STATIC_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
 else()
-  set(SYSTEM_LIBS ${CMAKE_THREAD_LIBS_INIT} rt)
+  set(SYSTEM_LIBS ${CMAKE_THREAD_LIBS_INIT})
   set(LIBS ${ROCKSDB_SHARED_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
 
   add_library(${ROCKSDB_SHARED_LIB} SHARED ${SOURCES})
@@ -605,6 +636,7 @@ set(TESTS
         db/db_test2.cc
         db/db_universal_compaction_test.cc
         db/db_wal_test.cc
+        db/db_write_test.cc
         db/dbformat_test.cc
         db/deletefile_test.cc
         db/external_sst_file_basic_test.cc
@@ -637,6 +669,7 @@ set(TESTS
         env/mock_env_test.cc
         memtable/inlineskiplist_test.cc
         memtable/skiplist_test.cc
+        memtable/write_buffer_manager_test.cc
         monitoring/histogram_test.cc
         monitoring/iostats_context_test.cc
         monitoring/statistics_test.cc
@@ -663,6 +696,7 @@ set(TESTS
         util/event_logger_test.cc
         util/file_reader_writer_test.cc
         util/filelock_test.cc
+        util/hash_test.cc
         util/heap_test.cc
         util/rate_limiter_test.cc
         util/slice_transform_test.cc
@@ -670,6 +704,7 @@ set(TESTS
         util/thread_list_test.cc
         util/thread_local_test.cc
         utilities/backupable/backupable_db_test.cc
+        utilities/blob_db/blob_db_test.cc
         utilities/checkpoint/checkpoint_test.cc
         utilities/column_aware_encoding_test.cc
         utilities/date_tiered/date_tiered_test.cc
@@ -678,6 +713,10 @@ set(TESTS
         utilities/geodb/geodb_test.cc
         utilities/lua/rocks_lua_test.cc
         utilities/memory/memory_test.cc
+        utilities/merge_operators/cassandra/cassandra_merge_test.cc
+        utilities/merge_operators/cassandra/cassandra_format_test.cc
+        utilities/merge_operators/cassandra/cassandra_row_merge_test.cc
+        utilities/merge_operators/cassandra/cassandra_serialize_test.cc
         utilities/merge_operators/string_append/stringappend_test.cc
         utilities/object_registry_test.cc
         utilities/option_change_migration/option_change_migration_test.cc
@@ -715,10 +754,10 @@ endforeach(sourcefile ${BENCHMARKS})
 # and linked to tests. Add test only code that is not #ifdefed for Release here.
 set(TESTUTIL_SOURCE
     db/db_test_util.cc
-    env/mock_env.cc
     monitoring/thread_status_updater_debug.cc
     table/mock_table.cc
     util/fault_injection_test_env.cc
+    utilities/merge_operators/cassandra/test_utils.cc
 )
 # test utilities are only build in debug
 enable_testing()

DEFAULT_OPTIONS_HISTORY.md

@@ -1,5 +1,8 @@
 # RocksDB default options change log
 ## Unreleased
+* delayed_write_rate takes the rate given by rate_limiter if not specified.
+
+## 5.2
 * Change the default of delayed slowdown value to 16MB/s and further increase the L0 stop condition to 36 files.
 
 ## 5.0 (11/17/2016)

HISTORY.md

@@ -1,23 +1,26 @@
 # Rocksdb Change Log
-## 5.5.6 (08/12/2017)
+## 5.6.2 (08/12/2017)
 ### Bug Fixes
 * Fix incorrect dropping of deletions during intra-L0 compaction.
 
-## 5.5.5 (07/18/2017)
-* Change license section of java/rocksjni.pom
-* Clear up some files still mentioning PATENTS file 
-
-## 5.5.4 (07/18/2017)
-* Change License to Apache + GPLv2
-
-## 5.5.3 (07/17/2017)
+## 5.6.1 (07/25/2017)
 ### Bug Fixes
-* Fix LITE build error.
+* Fix lite build.
 
-## 5.5.2 (07/13/2017)
-No change from 5.5.1.
+## 5.6.0 (06/06/2017)
+### Public API Change
+* Scheduling flushes and compactions in the same thread pool is no longer supported by setting `max_background_flushes=0`. Instead, users can achieve this by configuring their high-pri thread pool to have zero threads.
+* Replace `Options::max_background_flushes`, `Options::max_background_compactions`, and `Options::base_background_compactions` all with `Options::max_background_jobs`, which automatically decides how many threads to allocate towards flush/compaction.
+* options.delayed_write_rate by default take the value of options.rate_limiter rate.
+* Replace global variable `IOStatsContext iostats_context` with `IOStatsContext* get_iostats_context()`; replace global variable `PerfContext perf_context` with `PerfContext* get_perf_context()`.
+
+### New Features
+* Change ticker/histogram statistics implementations to use core-local storage. This improves aggregation speed compared to our previous thread-local approach, particularly for applications with many threads.
+* Users can pass a cache object to write buffer manager, so that they can cap memory usage for memtable and block cache using one single limit.
+* Flush will be triggered when 7/8 of the limit introduced by write_buffer_manager or db_write_buffer_size is triggered, so that the hard threshold is hard to hit.
+* Introduce WriteOptions.low_pri. If it is true, low priority writes will be throttled if the compaction is behind.
+* `DB::IngestExternalFile()` now supports ingesting files into a database containing range deletions.
 
-## 5.5.1 (06/27/2017)
 ### Bug Fixes
 * Shouldn't ignore return value of fsync() in flush.
 

INSTALL.md

@@ -21,9 +21,11 @@ depend on gflags. You will need to have gflags installed to run `make all`. This
 use binaries compiled by `make all` in production.
 
 * By default the binary we produce is optimized for the platform you're compiling on
-(-march=native or the equivalent). If you want to build a portable binary, add 'PORTABLE=1' before
-your make commands, like this: `PORTABLE=1 make static_lib`. If you want to build a binary that
-makes use of SSE4, add 'USE_SSE=1' before your make commands, like this: `USE_SSE=1 make static_lib`.
+(`-march=native` or the equivalent). SSE4.2 will thus be enabled automatically if your
+CPU supports it. To print a warning if your CPU does not support SSE4.2, build with
+`USE_SSE=1 make static_lib` or, if using CMake, `cmake -DFORCE_SSE42=ON`. If you want
+to build a portable binary, add `PORTABLE=1` before your make commands, like this:
+`PORTABLE=1 make static_lib`.
 
 ## Dependencies
 

Makefile

@@ -193,8 +193,8 @@ endif
 ifdef COMPILE_WITH_TSAN
 	DISABLE_JEMALLOC=1
 	EXEC_LDFLAGS += -fsanitize=thread
-	PLATFORM_CCFLAGS += -fsanitize=thread -fPIC -DROCKSDB_TSAN_RUN
-	PLATFORM_CXXFLAGS += -fsanitize=thread -fPIC -DROCKSDB_TSAN_RUN
+	PLATFORM_CCFLAGS += -fsanitize=thread -fPIC
+	PLATFORM_CXXFLAGS += -fsanitize=thread -fPIC
         # Turn off -pg when enabling TSAN testing, because that induces
         # a link failure.  TODO: find the root cause
 	PROFILING_FLAGS =
@@ -211,8 +211,8 @@ endif
 ifdef COMPILE_WITH_UBSAN
 	DISABLE_JEMALLOC=1
 	EXEC_LDFLAGS += -fsanitize=undefined
-	PLATFORM_CCFLAGS += -fsanitize=undefined
-	PLATFORM_CXXFLAGS += -fsanitize=undefined
+	PLATFORM_CCFLAGS += -fsanitize=undefined -DROCKSDB_UBSAN_RUN
+	PLATFORM_CXXFLAGS += -fsanitize=undefined -DROCKSDB_UBSAN_RUN
 endif
 
 ifndef DISABLE_JEMALLOC
@@ -335,6 +335,7 @@ TESTS = \
 	inlineskiplist_test \
 	env_basic_test \
 	env_test \
+	hash_test \
 	thread_local_test \
 	rate_limiter_test \
 	perf_context_test \
@@ -362,7 +363,9 @@ TESTS = \
 	db_properties_test \
 	db_table_properties_test \
 	db_statistics_test \
+	db_write_test \
 	autovector_test \
+	blob_db_test \
 	cleanable_test \
 	column_family_test \
 	table_properties_collector_test \
@@ -398,7 +401,12 @@ TESTS = \
 	external_sst_file_test \
 	prefix_test \
 	skiplist_test \
+	write_buffer_manager_test \
 	stringappend_test \
+	cassandra_format_test \
+	cassandra_merge_test \
+	cassandra_row_merge_test \
+	cassandra_serialize_test \
 	ttl_test \
 	date_tiered_test \
 	backupable_db_test \
@@ -485,7 +493,8 @@ TOOLS = \
 	ldb \
 	db_repl_stress \
 	rocksdb_dump \
-	rocksdb_undump
+	rocksdb_undump \
+	blob_dump \
 
 TEST_LIBS = \
 	librocksdb_env_basic_test.a
@@ -901,7 +910,7 @@ clean:
 	rm -rf $(CLEAN_FILES) ios-x86 ios-arm scan_build_report
 	find . -name "*.[oda]" -exec rm -f {} \;
 	find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm {} \;
-	rm -rf bzip2* snappy* zlib* lz4*
+	rm -rf bzip2* snappy* zlib* lz4* zstd*
 	cd java; $(MAKE) clean
 
 tags:
@@ -981,12 +990,27 @@ cache_test: cache/cache_test.o $(LIBOBJECTS) $(TESTHARNESS)
 coding_test: util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
+hash_test: util/hash_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
 option_change_migration_test: utilities/option_change_migration/option_change_migration_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
 stringappend_test: utilities/merge_operators/string_append/stringappend_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
+cassandra_format_test: utilities/merge_operators/cassandra/cassandra_format_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
+cassandra_merge_test: utilities/merge_operators/cassandra/cassandra_merge_test.o utilities/merge_operators/cassandra/test_utils.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
+cassandra_row_merge_test: utilities/merge_operators/cassandra/cassandra_row_merge_test.o utilities/merge_operators/cassandra/test_utils.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
+cassandra_serialize_test: utilities/merge_operators/cassandra/cassandra_serialize_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
 redis_test: utilities/redis/redis_lists_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -1062,6 +1086,9 @@ db_sst_test: db/db_sst_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 db_statistics_test: db/db_statistics_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
+db_write_test: db/db_write_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
 external_sst_file_basic_test: db/external_sst_file_basic_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -1217,6 +1244,9 @@ inlineskiplist_test: memtable/inlineskiplist_test.o $(LIBOBJECTS) $(TESTHARNESS)
 skiplist_test: memtable/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
+write_buffer_manager_test: memtable/write_buffer_manager_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
 version_edit_test: db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -1343,6 +1373,9 @@ transaction_test: utilities/transactions/transaction_test.o $(LIBOBJECTS) $(TEST
 sst_dump: tools/sst_dump.o $(LIBOBJECTS)
 	$(AM_LINK)
 
+blob_dump: tools/blob_dump.o $(LIBOBJECTS)
+	$(AM_LINK)
+
 column_aware_encoding_exp: utilities/column_aware_encoding_exp.o $(EXPOBJECTS)
 	$(AM_LINK)
 
@@ -1373,6 +1406,9 @@ lua_test: utilities/lua/rocks_lua_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTH
 range_del_aggregator_test: db/range_del_aggregator_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
+blob_db_test: utilities/blob_db/blob_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
 #-------------------------------------------------
 # make install related stuff
 INSTALL_PATH ?= /usr/local

TARGETS

@@ -1,4 +1,8 @@
-REPO_PATH = "internal_repo_rocksdb/repo/"
+
+import os
+
+TARGETS_PATH = os.path.dirname(__file__)
+REPO_PATH = TARGETS_PATH[(TARGETS_PATH.find('fbcode/') + len('fbcode/')):] + "/"
 BUCK_BINS = "buck-out/gen/" + REPO_PATH
 TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh"
 rocksdb_compiler_flags = [
@@ -11,6 +15,8 @@ rocksdb_compiler_flags = [
   "-DROCKSDB_MALLOC_USABLE_SIZE",
   "-DROCKSDB_RANGESYNC_PRESENT",
   "-DROCKSDB_SCHED_GETCPU_PRESENT",
+  "-DROCKSDB_SUPPORT_THREAD_LOCAL",
+  "-DHAVE_SSE42",
   "-DOS_LINUX",
   # Flags to enable libs we include
   "-DSNAPPY",
@@ -33,7 +39,7 @@ rocksdb_external_deps = [
   ('lz4', None, 'lz4'),
   ('zstd', None),
   ('tbb', None),
-  ("numa", "2.0.8", "numa"),
+  ("numa", None, "numa"),
   ("googletest", None, "gtest"),
 ]
 
@@ -82,6 +88,7 @@ cpp_library(
       "db/internal_stats.cc",
       "db/log_reader.cc",
       "db/log_writer.cc",
+      "db/malloc_stats.cc",
       "db/managed_iterator.cc",
       "db/memtable.cc",
       "db/memtable_list.cc",
@@ -106,13 +113,14 @@ cpp_library(
       "env/env_hdfs.cc",
       "env/env_posix.cc",
       "env/io_posix.cc",
-      "env/memenv.cc",
+      "env/mock_env.cc",
+      "memtable/alloc_tracker.cc",
       "memtable/hash_cuckoo_rep.cc",
       "memtable/hash_linklist_rep.cc",
       "memtable/hash_skiplist_rep.cc",
-      "memtable/memtable_allocator.cc",
       "memtable/skiplistrep.cc",
       "memtable/vectorrep.cc",
+      "memtable/write_buffer_manager.cc",
       "monitoring/histogram.cc",
       "monitoring/histogram_windowing.cc",
       "monitoring/instrumented_mutex.cc",
@@ -217,6 +225,8 @@ cpp_library(
       "utilities/leveldb_options/leveldb_options.cc",
       "utilities/lua/rocks_lua_compaction_filter.cc",
       "utilities/memory/memory_util.cc",
+      "utilities/merge_operators/cassandra/format.cc",
+      "utilities/merge_operators/cassandra/merge_operator.cc",
       "utilities/merge_operators/max.cc",
       "utilities/merge_operators/put.cc",
       "utilities/merge_operators/string_append/stringappend.cc",
@@ -259,12 +269,12 @@ cpp_library(
     name = "rocksdb_test_lib",
     headers = AutoHeaders.RECURSIVE_GLOB,
     srcs = [
-      "env/mock_env.cc",
       "table/mock_table.cc",
       "util/fault_injection_test_env.cc",
       "util/testharness.cc",
       "util/testutil.cc",
       "db/db_test_util.cc",
+      "utilities/merge_operators/cassandra/test_utils.cc",
       "utilities/col_buf_encoder.cc",
       "utilities/col_buf_decoder.cc",
       "utilities/column_aware_encoding_util.cc",
@@ -299,163 +309,182 @@ cpp_library(
 )
 
 # [test_name, test_src, test_type]
-ROCKS_TESTS = [['merger_test', 'table/merger_test.cc', 'serial'],
- ['cache_test', 'cache/cache_test.cc', 'serial'],
- ['options_file_test', 'db/options_file_test.cc', 'serial'],
- ['compaction_picker_test', 'db/compaction_picker_test.cc', 'serial'],
- ['db_log_iter_test', 'db/db_log_iter_test.cc', 'serial'],
- ['thread_list_test', 'util/thread_list_test.cc', 'serial'],
- ['table_properties_collector_test',
-  'db/table_properties_collector_test.cc',
-  'serial'],
- ['document_db_test', 'utilities/document/document_db_test.cc', 'serial'],
- ['event_logger_test', 'util/event_logger_test.cc', 'serial'],
- ['coding_test', 'util/coding_test.cc', 'serial'],
- ['statistics_test', 'monitoring/statistics_test.cc', 'serial'],
- ['options_settable_test', 'options/options_settable_test.cc', 'serial'],
- ['sst_dump_test', 'tools/sst_dump_test.cc', 'serial'],
- ['column_aware_encoding_test',
-  'utilities/column_aware_encoding_test.cc',
-  'serial'],
- ['db_iterator_test', 'db/db_iterator_test.cc', 'serial'],
- ['db_sst_test', 'db/db_sst_test.cc', 'parallel'],
- ['geodb_test', 'utilities/geodb/geodb_test.cc', 'serial'],
- ['listener_test', 'db/listener_test.cc', 'serial'],
- ['write_callback_test', 'db/write_callback_test.cc', 'serial'],
- ['version_set_test', 'db/version_set_test.cc', 'serial'],
- ['full_filter_block_test', 'table/full_filter_block_test.cc', 'serial'],
- ['cleanable_test', 'table/cleanable_test.cc', 'serial'],
- ['checkpoint_test', 'utilities/checkpoint/checkpoint_test.cc', 'serial'],
- ['compact_files_test', 'db/compact_files_test.cc', 'serial'],
- ['db_options_test', 'db/db_options_test.cc', 'serial'],
- ['object_registry_test', 'utilities/object_registry_test.cc', 'serial'],
+ROCKS_TESTS = [['arena_test', 'util/arena_test.cc', 'serial'],
  ['auto_roll_logger_test', 'util/auto_roll_logger_test.cc', 'serial'],
- ['dbformat_test', 'db/dbformat_test.cc', 'serial'],
- ['write_batch_with_index_test',
-  'utilities/write_batch_with_index/write_batch_with_index_test.cc',
-  'serial'],
- ['json_document_test', 'utilities/document/json_document_test.cc', 'serial'],
- ['file_reader_writer_test', 'util/file_reader_writer_test.cc', 'serial'],
- ['repair_test', 'db/repair_test.cc', 'serial'],
- ['persistent_cache_test',
-  'utilities/persistent_cache/persistent_cache_test.cc',
-  'parallel'],
- ['db_bloom_filter_test', 'db/db_bloom_filter_test.cc', 'serial'],
- ['external_sst_file_basic_test',
-  'db/external_sst_file_basic_test.cc',
-  'serial'],
- ['options_test', 'options/options_test.cc', 'serial'],
- ['perf_context_test', 'db/perf_context_test.cc', 'serial'],
- ['db_block_cache_test', 'db/db_block_cache_test.cc', 'serial'],
- ['heap_test', 'util/heap_test.cc', 'serial'],
- ['db_test2', 'db/db_test2.cc', 'serial'],
- ['filelock_test', 'util/filelock_test.cc', 'serial'],
- ['write_controller_test', 'db/write_controller_test.cc', 'serial'],
- ['compaction_iterator_test', 'db/compaction_iterator_test.cc', 'serial'],
- ['spatial_db_test', 'utilities/spatialdb/spatial_db_test.cc', 'serial'],
- ['c_test', 'db/c_test.c', 'serial'],
- ['range_del_aggregator_test', 'db/range_del_aggregator_test.cc', 'serial'],
- ['date_tiered_test', 'utilities/date_tiered/date_tiered_test.cc', 'serial'],
- ['ldb_cmd_test', 'tools/ldb_cmd_test.cc', 'serial'],
- ['db_test', 'db/db_test.cc', 'parallel'],
- ['block_based_filter_block_test',
-  'table/block_based_filter_block_test.cc',
-  'serial'],
- ['merge_test', 'db/merge_test.cc', 'serial'],
- ['bloom_test', 'util/bloom_test.cc', 'serial'],
- ['block_test', 'table/block_test.cc', 'serial'],
- ['cuckoo_table_builder_test', 'table/cuckoo_table_builder_test.cc', 'serial'],
+ ['autovector_test', 'util/autovector_test.cc', 'serial'],
  ['backupable_db_test',
   'utilities/backupable/backupable_db_test.cc',
   'parallel'],
- ['db_flush_test', 'db/db_flush_test.cc', 'serial'],
- ['filename_test', 'db/filename_test.cc', 'serial'],
- ['cuckoo_table_reader_test', 'table/cuckoo_table_reader_test.cc', 'serial'],
- ['slice_transform_test', 'util/slice_transform_test.cc', 'serial'],
- ['cuckoo_table_db_test', 'db/cuckoo_table_db_test.cc', 'serial'],
- ['inlineskiplist_test', 'memtable/inlineskiplist_test.cc', 'parallel'],
- ['optimistic_transaction_test',
-  'utilities/transactions/optimistic_transaction_test.cc',
+ ['blob_db_test', 'utilities/blob_db/blob_db_test.cc', 'serial'],
+ ['block_based_filter_block_test',
+  'table/block_based_filter_block_test.cc',
   'serial'],
- ['hash_table_test',
-  'utilities/persistent_cache/hash_table_test.cc',
+ ['block_test', 'table/block_test.cc', 'serial'],
+ ['bloom_test', 'util/bloom_test.cc', 'serial'],
+ ['c_test', 'db/c_test.c', 'serial'],
+ ['cache_test', 'cache/cache_test.cc', 'serial'],
+ ['cassandra_format_test',
+  'utilities/merge_operators/cassandra/cassandra_format_test.cc',
   'serial'],
- ['db_dynamic_level_test', 'db/db_dynamic_level_test.cc', 'serial'],
- ['option_change_migration_test',
-  'utilities/option_change_migration/option_change_migration_test.cc',
+ ['cassandra_merge_test',
+  'utilities/merge_operators/cassandra/cassandra_merge_test.cc',
   'serial'],
- ['db_inplace_update_test', 'db/db_inplace_update_test.cc', 'serial'],
- ['autovector_test', 'util/autovector_test.cc', 'serial'],
- ['db_iter_test', 'db/db_iter_test.cc', 'serial'],
- ['flush_job_test', 'db/flush_job_test.cc', 'serial'],
- ['wal_manager_test', 'db/wal_manager_test.cc', 'serial'],
- ['write_batch_test', 'db/write_batch_test.cc', 'serial'],
- ['crc32c_test', 'util/crc32c_test.cc', 'serial'],
- ['rate_limiter_test', 'util/rate_limiter_test.cc', 'serial'],
- ['external_sst_file_test', 'db/external_sst_file_test.cc', 'parallel'],
- ['compaction_job_test', 'db/compaction_job_test.cc', 'serial'],
- ['mock_env_test', 'env/mock_env_test.cc', 'serial'],
- ['db_table_properties_test', 'db/db_table_properties_test.cc', 'serial'],
- ['db_compaction_test', 'db/db_compaction_test.cc', 'parallel'],
- ['arena_test', 'util/arena_test.cc', 'serial'],
- ['stringappend_test',
-  'utilities/merge_operators/string_append/stringappend_test.cc',
+ ['cassandra_row_merge_test',
+  'utilities/merge_operators/cassandra/cassandra_row_merge_test.cc',
   'serial'],
- ['reduce_levels_test', 'tools/reduce_levels_test.cc', 'serial'],
- ['prefix_test', 'db/prefix_test.cc', 'serial'],
- ['ttl_test', 'utilities/ttl/ttl_test.cc', 'serial'],
- ['merge_helper_test', 'db/merge_helper_test.cc', 'serial'],
- ['file_indexer_test', 'db/file_indexer_test.cc', 'serial'],
- ['db_statistics_test', 'db/db_statistics_test.cc', 'serial'],
- ['memory_test', 'utilities/memory/memory_test.cc', 'serial'],
- ['log_test', 'db/log_test.cc', 'serial'],
- ['env_timed_test', 'utilities/env_timed_test.cc', 'serial'],
- ['deletefile_test', 'db/deletefile_test.cc', 'serial'],
- ['partitioned_filter_block_test',
-  'table/partitioned_filter_block_test.cc',
+ ['cassandra_serialize_test',
+  'utilities/merge_operators/cassandra/cassandra_serialize_test.cc',
   'serial'],
- ['comparator_db_test', 'db/comparator_db_test.cc', 'serial'],
- ['compaction_job_stats_test', 'db/compaction_job_stats_test.cc', 'serial'],
- ['thread_local_test', 'util/thread_local_test.cc', 'serial'],
- ['version_builder_test', 'db/version_builder_test.cc', 'serial'],
- ['db_range_del_test', 'db/db_range_del_test.cc', 'serial'],
- ['table_test', 'table/table_test.cc', 'parallel'],
- ['db_tailing_iter_test', 'db/db_tailing_iter_test.cc', 'serial'],
- ['db_compaction_filter_test', 'db/db_compaction_filter_test.cc', 'parallel'],
- ['options_util_test', 'utilities/options/options_util_test.cc', 'serial'],
- ['dynamic_bloom_test', 'util/dynamic_bloom_test.cc', 'serial'],
- ['db_basic_test', 'db/db_basic_test.cc', 'serial'],
- ['db_merge_operator_test', 'db/db_merge_operator_test.cc', 'serial'],
- ['manual_compaction_test', 'db/manual_compaction_test.cc', 'parallel'],
- ['delete_scheduler_test', 'util/delete_scheduler_test.cc', 'serial'],
- ['transaction_test', 'utilities/transactions/transaction_test.cc', 'serial'],
- ['db_io_failure_test', 'db/db_io_failure_test.cc', 'serial'],
- ['corruption_test', 'db/corruption_test.cc', 'serial'],
+ ['checkpoint_test', 'utilities/checkpoint/checkpoint_test.cc', 'serial'],
+ ['cleanable_test', 'table/cleanable_test.cc', 'serial'],
+ ['coding_test', 'util/coding_test.cc', 'serial'],
+ ['column_aware_encoding_test',
+  'utilities/column_aware_encoding_test.cc',
+  'serial'],
+ ['column_family_test', 'db/column_family_test.cc', 'serial'],
+ ['compact_files_test', 'db/compact_files_test.cc', 'serial'],
  ['compact_on_deletion_collector_test',
   'utilities/table_properties_collectors/compact_on_deletion_collector_test.cc',
   'serial'],
- ['env_test', 'env/env_test.cc', 'serial'],
- ['db_wal_test', 'db/db_wal_test.cc', 'parallel'],
- ['timer_queue_test', 'util/timer_queue_test.cc', 'serial'],
- ['sim_cache_test', 'utilities/simulator_cache/sim_cache_test.cc', 'serial'],
+ ['compaction_iterator_test', 'db/compaction_iterator_test.cc', 'serial'],
+ ['compaction_job_stats_test', 'db/compaction_job_stats_test.cc', 'serial'],
+ ['compaction_job_test', 'db/compaction_job_test.cc', 'serial'],
+ ['compaction_picker_test', 'db/compaction_picker_test.cc', 'serial'],
+ ['comparator_db_test', 'db/comparator_db_test.cc', 'serial'],
+ ['corruption_test', 'db/corruption_test.cc', 'serial'],
+ ['crc32c_test', 'util/crc32c_test.cc', 'serial'],
+ ['cuckoo_table_builder_test', 'table/cuckoo_table_builder_test.cc', 'serial'],
+ ['cuckoo_table_db_test', 'db/cuckoo_table_db_test.cc', 'serial'],
+ ['cuckoo_table_reader_test', 'table/cuckoo_table_reader_test.cc', 'serial'],
+ ['date_tiered_test', 'utilities/date_tiered/date_tiered_test.cc', 'serial'],
+ ['db_basic_test', 'db/db_basic_test.cc', 'serial'],
+ ['db_block_cache_test', 'db/db_block_cache_test.cc', 'serial'],
+ ['db_bloom_filter_test', 'db/db_bloom_filter_test.cc', 'serial'],
+ ['db_compaction_filter_test', 'db/db_compaction_filter_test.cc', 'parallel'],
+ ['db_compaction_test', 'db/db_compaction_test.cc', 'parallel'],
+ ['db_dynamic_level_test', 'db/db_dynamic_level_test.cc', 'serial'],
+ ['db_encryption_test', 'db/db_encryption_test.cc', 'serial'],
+ ['db_flush_test', 'db/db_flush_test.cc', 'serial'],
+ ['db_inplace_update_test', 'db/db_inplace_update_test.cc', 'serial'],
+ ['db_io_failure_test', 'db/db_io_failure_test.cc', 'serial'],
+ ['db_iter_test', 'db/db_iter_test.cc', 'serial'],
+ ['db_iterator_test', 'db/db_iterator_test.cc', 'serial'],
+ ['db_log_iter_test', 'db/db_log_iter_test.cc', 'serial'],
  ['db_memtable_test', 'db/db_memtable_test.cc', 'serial'],
+ ['db_merge_operator_test', 'db/db_merge_operator_test.cc', 'serial'],
+ ['db_options_test', 'db/db_options_test.cc', 'serial'],
+ ['db_properties_test', 'db/db_properties_test.cc', 'serial'],
+ ['db_range_del_test', 'db/db_range_del_test.cc', 'serial'],
+ ['db_sst_test', 'db/db_sst_test.cc', 'parallel'],
+ ['db_statistics_test', 'db/db_statistics_test.cc', 'serial'],
+ ['db_table_properties_test', 'db/db_table_properties_test.cc', 'serial'],
+ ['db_tailing_iter_test', 'db/db_tailing_iter_test.cc', 'serial'],
+ ['db_test', 'db/db_test.cc', 'parallel'],
+ ['db_test2', 'db/db_test2.cc', 'serial'],
  ['db_universal_compaction_test',
   'db/db_universal_compaction_test.cc',
   'parallel'],
+ ['db_wal_test', 'db/db_wal_test.cc', 'parallel'],
+ ['db_write_test', 'db/db_write_test.cc', 'serial'],
+ ['dbformat_test', 'db/dbformat_test.cc', 'serial'],
+ ['delete_scheduler_test', 'util/delete_scheduler_test.cc', 'serial'],
+ ['deletefile_test', 'db/deletefile_test.cc', 'serial'],
+ ['document_db_test', 'utilities/document/document_db_test.cc', 'serial'],
+ ['dynamic_bloom_test', 'util/dynamic_bloom_test.cc', 'serial'],
+ ['env_basic_test', 'env/env_basic_test.cc', 'serial'],
+ ['env_test', 'env/env_test.cc', 'serial'],
+ ['env_timed_test', 'utilities/env_timed_test.cc', 'serial'],
+ ['event_logger_test', 'util/event_logger_test.cc', 'serial'],
+ ['external_sst_file_basic_test',
+  'db/external_sst_file_basic_test.cc',
+  'serial'],
+ ['external_sst_file_test', 'db/external_sst_file_test.cc', 'parallel'],
+ ['fault_injection_test', 'db/fault_injection_test.cc', 'parallel'],
+ ['file_indexer_test', 'db/file_indexer_test.cc', 'serial'],
+ ['file_reader_writer_test', 'util/file_reader_writer_test.cc', 'serial'],
+ ['filelock_test', 'util/filelock_test.cc', 'serial'],
+ ['filename_test', 'db/filename_test.cc', 'serial'],
+ ['flush_job_test', 'db/flush_job_test.cc', 'serial'],
+ ['full_filter_block_test', 'table/full_filter_block_test.cc', 'serial'],
+ ['geodb_test', 'utilities/geodb/geodb_test.cc', 'serial'],
+ ['hash_table_test',
+  'utilities/persistent_cache/hash_table_test.cc',
+  'serial'],
+ ['hash_test', 'util/hash_test.cc', 'serial'],
+ ['heap_test', 'util/heap_test.cc', 'serial'],
  ['histogram_test', 'monitoring/histogram_test.cc', 'serial'],
+ ['inlineskiplist_test', 'memtable/inlineskiplist_test.cc', 'parallel'],
+ ['iostats_context_test', 'monitoring/iostats_context_test.cc', 'serial'],
+ ['json_document_test', 'utilities/document/json_document_test.cc', 'serial'],
+ ['ldb_cmd_test', 'tools/ldb_cmd_test.cc', 'serial'],
+ ['listener_test', 'db/listener_test.cc', 'serial'],
+ ['log_test', 'db/log_test.cc', 'serial'],
+ ['lru_cache_test', 'cache/lru_cache_test.cc', 'serial'],
+ ['manual_compaction_test', 'db/manual_compaction_test.cc', 'parallel'],
+ ['memory_test', 'utilities/memory/memory_test.cc', 'serial'],
+ ['memtable_list_test', 'db/memtable_list_test.cc', 'serial'],
+ ['merge_helper_test', 'db/merge_helper_test.cc', 'serial'],
+ ['merge_test', 'db/merge_test.cc', 'serial'],
+ ['merger_test', 'table/merger_test.cc', 'serial'],
+ ['mock_env_test', 'env/mock_env_test.cc', 'serial'],
+ ['object_registry_test', 'utilities/object_registry_test.cc', 'serial'],
+ ['optimistic_transaction_test',
+  'utilities/transactions/optimistic_transaction_test.cc',
+  'serial'],
+ ['option_change_migration_test',
+  'utilities/option_change_migration/option_change_migration_test.cc',
+  'serial'],
+ ['options_file_test', 'db/options_file_test.cc', 'serial'],
+ ['options_settable_test', 'options/options_settable_test.cc', 'serial'],
+ ['options_test', 'options/options_test.cc', 'serial'],
+ ['options_util_test', 'utilities/options/options_util_test.cc', 'serial'],
+ ['partitioned_filter_block_test',
+  'table/partitioned_filter_block_test.cc',
+  'serial'],
+ ['perf_context_test', 'db/perf_context_test.cc', 'serial'],
+ ['persistent_cache_test',
+  'utilities/persistent_cache/persistent_cache_test.cc',
+  'parallel'],
+ ['plain_table_db_test', 'db/plain_table_db_test.cc', 'serial'],
+ ['prefix_test', 'db/prefix_test.cc', 'serial'],
+ ['range_del_aggregator_test', 'db/range_del_aggregator_test.cc', 'serial'],
+ ['rate_limiter_test', 'util/rate_limiter_test.cc', 'serial'],
+ ['reduce_levels_test', 'tools/reduce_levels_test.cc', 'serial'],
+ ['repair_test', 'db/repair_test.cc', 'serial'],
+ ['sim_cache_test', 'utilities/simulator_cache/sim_cache_test.cc', 'serial'],
+ ['skiplist_test', 'memtable/skiplist_test.cc', 'serial'],
+ ['slice_transform_test', 'util/slice_transform_test.cc', 'serial'],
+ ['spatial_db_test', 'utilities/spatialdb/spatial_db_test.cc', 'serial'],
+ ['sst_dump_test', 'tools/sst_dump_test.cc', 'serial'],
+ ['statistics_test', 'monitoring/statistics_test.cc', 'serial'],
+ ['stringappend_test',
+  'utilities/merge_operators/string_append/stringappend_test.cc',
+  'serial'],
+ ['table_properties_collector_test',
+  'db/table_properties_collector_test.cc',
+  'serial'],
+ ['table_test', 'table/table_test.cc', 'parallel'],
+ ['thread_list_test', 'util/thread_list_test.cc', 'serial'],
+ ['thread_local_test', 'util/thread_local_test.cc', 'serial'],
+ ['timer_queue_test', 'util/timer_queue_test.cc', 'serial'],
+ ['transaction_test', 'utilities/transactions/transaction_test.cc', 'serial'],
+ ['ttl_test', 'utilities/ttl/ttl_test.cc', 'serial'],
  ['util_merge_operators_test',
   'utilities/util_merge_operators_test.cc',
   'serial'],
- ['fault_injection_test', 'db/fault_injection_test.cc', 'parallel'],
- ['env_basic_test', 'env/env_basic_test.cc', 'serial'],
- ['iostats_context_test', 'monitoring/iostats_context_test.cc', 'serial'],
- ['memtable_list_test', 'db/memtable_list_test.cc', 'serial'],
- ['column_family_test', 'db/column_family_test.cc', 'serial'],
- ['db_properties_test', 'db/db_properties_test.cc', 'serial'],
+ ['version_builder_test', 'db/version_builder_test.cc', 'serial'],
  ['version_edit_test', 'db/version_edit_test.cc', 'serial'],
- ['skiplist_test', 'memtable/skiplist_test.cc', 'serial'],
- ['lru_cache_test', 'cache/lru_cache_test.cc', 'serial'],
- ['plain_table_db_test', 'db/plain_table_db_test.cc', 'serial']]
+ ['version_set_test', 'db/version_set_test.cc', 'serial'],
+ ['wal_manager_test', 'db/wal_manager_test.cc', 'serial'],
+ ['write_batch_test', 'db/write_batch_test.cc', 'serial'],
+ ['write_batch_with_index_test',
+  'utilities/write_batch_with_index/write_batch_with_index_test.cc',
+  'serial'],
+ ['write_buffer_manager_test',
+  'memtable/write_buffer_manager_test.cc',
+  'serial'],
+ ['write_callback_test', 'db/write_callback_test.cc', 'serial'],
+ ['write_controller_test', 'db/write_controller_test.cc', 'serial']]
 
 
 # Generate a test rule for each entry in ROCKS_TESTS
@@ -480,3 +509,15 @@ for test_cfg in ROCKS_TESTS:
       deps = [":" + test_bin],
       command = [TEST_RUNNER, BUCK_BINS + test_bin]
     )
+
+custom_unittest(
+    name = "make_rocksdbjavastatic",
+    type = "simple",
+    command = ["internal_repo_rocksdb/make_rocksdbjavastatic.sh"],
+)
+
+custom_unittest(
+    name = "make_rocksdb_lite_release",
+    type = "simple",
+    command = ["internal_repo_rocksdb/make_rocksdb_lite_release.sh"],
+)

arcanist_util/INTERNAL_ONLY_DIR

@@ -0,0 +1,2 @@
+arcanist_util are only used internaly, If you want to change it please check
+<internal_rocksdb_repo>/arcanist_util

buckifier/buckify_rocksdb.py

@@ -84,7 +84,7 @@ def get_tests(repo_path):
             else:
                 # we consumed all the parallel tests
                 break
-
+    
     return tests
 
 
@@ -122,7 +122,7 @@ def generate_targets(repo_path):
         [":rocksdb_lib"])
 
     # test for every test we found in the Makefile
-    for test in tests:
+    for test in sorted(tests):
         match_src = [src for src in cc_files if ("/%s.c" % test) in src]
         if len(match_src) == 0:
             print(ColorString.warning("Cannot find .cc file for %s" % test))

buckifier/targets_cfg.py

@@ -2,7 +2,11 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
-rocksdb_target_header = """REPO_PATH = "internal_repo_rocksdb/repo/"
+rocksdb_target_header = """
+import os
+
+TARGETS_PATH = os.path.dirname(__file__)
+REPO_PATH = TARGETS_PATH[(TARGETS_PATH.find('fbcode/') + len('fbcode/')):] + "/"
 BUCK_BINS = "buck-out/gen/" + REPO_PATH
 TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh"
 rocksdb_compiler_flags = [
@@ -16,6 +20,7 @@ rocksdb_compiler_flags = [
   "-DROCKSDB_RANGESYNC_PRESENT",
   "-DROCKSDB_SCHED_GETCPU_PRESENT",
   "-DROCKSDB_SUPPORT_THREAD_LOCAL",
+  "-DHAVE_SSE42",
   "-DOS_LINUX",
   # Flags to enable libs we include
   "-DSNAPPY",
@@ -38,7 +43,7 @@ rocksdb_external_deps = [
   ('lz4', None, 'lz4'),
   ('zstd', None),
   ('tbb', None),
-  ("numa", "2.0.8", "numa"),
+  ("numa", None, "numa"),
   ("googletest", None, "gtest"),
 ]
 

build_tools/build_detect_platform

@@ -51,13 +51,11 @@ if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
     FBCODE_BUILD="true"
     # If we're compiling with TSAN we need pic build
     PIC_BUILD=$COMPILE_WITH_TSAN
-    if [ -n "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
-      # we need this to build with MySQL. Don't use for other purposes.
-      source "$PWD/build_tools/fbcode_config4.8.1.sh"
-    elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_5xx" ]; then
+    if [ -z "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
       source "$PWD/build_tools/fbcode_config.sh"
     else
-      source "$PWD/build_tools/fbcode_config_platform007.sh"
+      # we need this to build with MySQL. Don't use for other purposes.
+      source "$PWD/build_tools/fbcode_config4.8.1.sh"
     fi
 fi
 
@@ -444,14 +442,8 @@ if test "$USE_HDFS"; then
   JAVA_LDFLAGS="$JAVA_LDFLAGS $HDFS_LDFLAGS"
 fi
 
-if [ "$TARGET_OS" = FreeBSD -a "$TARGET_ARCHITECTURE" = i386 ]; then
-  # Intel SSE instructions breaks compilation on FreeBSD i386
-  unset USE_SSE
-fi
-
 if test "$USE_SSE"; then
-  # if Intel SSE instruction set is supported, set USE_SSE=1
-  COMMON_FLAGS="$COMMON_FLAGS -msse -msse4.2 "
+  COMMON_FLAGS="$COMMON_FLAGS -msse4.2"
 elif test -z "$PORTABLE"; then
   if test -n "`echo $TARGET_ARCHITECTURE | grep ^ppc64`"; then
     # Tune for this POWER processor, treating '+' models as base models
@@ -464,6 +456,36 @@ elif test -z "$PORTABLE"; then
   fi
 fi
 
+$CXX $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
+  #include <cstdint>
+  #include <nmmintrin.h>
+  int main() {
+    volatile uint32_t x = _mm_crc32_u32(0, 0);
+  }
+EOF
+if [ "$?" = 0 ]; then
+  COMMON_FLAGS="$COMMON_FLAGS -DHAVE_SSE42"
+elif test "$USE_SSE"; then
+  echo "warning: USE_SSE specified but compiler could not use SSE intrinsics, disabling"
+fi
+
+# iOS doesn't support thread-local storage, but this check would erroneously
+# succeed because the cross-compiler flags are added by the Makefile, not this
+# script.
+if [ "$PLATFORM" != IOS ]; then
+  $CXX $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
+  #if defined(_MSC_VER) && !defined(__thread)
+  #define __thread __declspec(thread)
+  #endif
+  int main() {
+    static __thread int tls;
+  }
+EOF
+  if [ "$?" = 0 ]; then
+    COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_SUPPORT_THREAD_LOCAL"
+  fi
+fi
+
 PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS"
 PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS"
 

build_tools/dependencies_platform007.sh

@@ -1,18 +0,0 @@
-GCC_BASE=/mnt/gvfs/third-party2/gcc/6e8e715624fd15256a7970073387793dfcf79b46/7.x/centos7-native/b2ef2b6
-CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/ef37e1faa1c29782abfac1ae65a291b9b7966f6d/stable/centos7-native/c9f9104
-LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/c67031f0f739ac61575a061518d6ef5038f99f90/7.x/platform007/5620abc
-GLIBC_BASE=/mnt/gvfs/third-party2/glibc/60d6f124a78798b73944f5ba87c2306ae3460153/2.26/platform007/f259413
-SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/platform007/ca4da3d
-ZLIB_BASE=/mnt/gvfs/third-party2/zlib/22c2d65676fb7c23cfa797c4f6937f38b026f3cf/1.2.8/platform007/ca4da3d
-BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/platform007/ca4da3d
-LZ4_BASE=/mnt/gvfs/third-party2/lz4/907b498203d297947f3bb70b9466f47e100f1873/r131/platform007/ca4da3d
-ZSTD_BASE=/mnt/gvfs/third-party2/zstd/3ee276cbacfad3074e3f07bf826ac47f06970f4e/1.3.5/platform007/15a3614
-GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/platform007/ca4da3d
-JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/9c910d36d6235cc40e8ff559358f1833452300ca/master/platform007/5b0f53e
-NUMA_BASE=/mnt/gvfs/third-party2/numa/9cbf2460284c669ed19c3ccb200a71f7dd7e53c7/2.0.11/platform007/ca4da3d
-LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/bf3d7497fe4e6d007354f0adffa16ce3003f8338/1.3/platform007/6f3e0a9
-TBB_BASE=/mnt/gvfs/third-party2/tbb/ff4e0b093534704d8abab678a4fd7f5ea7b094c7/2018_U5/platform007/ca4da3d
-KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/b5c4a61a5c483ba24722005ae07895971a2ac707/fb/platform007/da39a3e
-BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/92ff90349e2f43ea0a8246d8b1cf17b6869013e3/2.29.1/centos7-native/da39a3e
-VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/f3f697a28122e6bcd513273dd9c1ff23852fc59f/3.13.0/platform007/ca4da3d
-LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.3.4/platform007/5007832

build_tools/fbcode_config.sh

@@ -128,7 +128,7 @@ else
 fi
 
 CFLAGS+=" $DEPS_INCLUDE"
-CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL"
+CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
 CXXFLAGS+=" $CFLAGS"
 
 EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"

build_tools/fbcode_config4.8.1.sh

@@ -93,7 +93,7 @@ else
 fi
 
 CFLAGS+=" $DEPS_INCLUDE"
-CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL"
+CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
 CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2 -DLZ4 -DZSTD -DNUMA -DTBB"
 CXXFLAGS+=" $CFLAGS"
 

build_tools/fbcode_config_platform007.sh

@@ -1,157 +0,0 @@
-#!/bin/sh
-#
-# Set environment variables so that we can compile rocksdb using
-# fbcode settings.  It uses the latest g++ and clang compilers and also
-# uses jemalloc
-# Environment variables that change the behavior of this script:
-# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included
-
-
-BASEDIR=`dirname $BASH_SOURCE`
-source "$BASEDIR/dependencies_platform007.sh"
-
-CFLAGS=""
-
-# libgcc
-LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/7.3.0"
-LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
-
-# glibc
-GLIBC_INCLUDE="$GLIBC_BASE/include"
-GLIBC_LIBS=" -L $GLIBC_BASE/lib"
-
-# snappy
-SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
-if test -z $PIC_BUILD; then
-  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
-else
-  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a"
-fi
-CFLAGS+=" -DSNAPPY"
-
-if test -z $PIC_BUILD; then
-  # location of zlib headers and libraries
-  ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
-  ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
-  CFLAGS+=" -DZLIB"
-
-  # location of bzip headers and libraries
-  BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
-  BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a"
-  CFLAGS+=" -DBZIP2"
-
-  LZ4_INCLUDE=" -I $LZ4_BASE/include/"
-  LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
-  CFLAGS+=" -DLZ4"
-fi
-
-ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
-if test -z $PIC_BUILD; then
-  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
-else
-  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a"
-fi
-CFLAGS+=" -DZSTD"
-
-# location of gflags headers and libraries
-GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
-if test -z $PIC_BUILD; then
-  GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
-else
-  GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
-fi
-CFLAGS+=" -DGFLAGS=gflags"
-
-# location of jemalloc
-JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
-JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a"
-
-if test -z $PIC_BUILD; then
-  # location of numa
-  NUMA_INCLUDE=" -I $NUMA_BASE/include/"
-  NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
-  CFLAGS+=" -DNUMA"
-
-  # location of libunwind
-  LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
-fi
-
-# location of TBB
-TBB_INCLUDE=" -isystem $TBB_BASE/include/"
-if test -z $PIC_BUILD; then
-  TBB_LIBS="$TBB_BASE/lib/libtbb.a"
-else
-  TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a"
-fi
-CFLAGS+=" -DTBB"
-
-# use Intel SSE support for checksum calculations
-export USE_SSE=1
-export PORTABLE=1
-
-BINUTILS="$BINUTILS_BASE/bin"
-AR="$BINUTILS/ar"
-
-DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE"
-
-STDLIBS="-L $GCC_BASE/lib64"
-
-CLANG_BIN="$CLANG_BASE/bin"
-CLANG_LIB="$CLANG_BASE/lib"
-CLANG_SRC="$CLANG_BASE/../../src"
-
-CLANG_ANALYZER="$CLANG_BIN/clang++"
-CLANG_SCAN_BUILD="$CLANG_SRC/llvm/tools/clang/tools/scan-build/bin/scan-build"
-
-if [ -z "$USE_CLANG" ]; then
-  # gcc
-  CC="$GCC_BASE/bin/gcc"
-  CXX="$GCC_BASE/bin/g++"
-
-  CFLAGS+=" -B$BINUTILS/gold"
-  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
-  CFLAGS+=" -isystem $GLIBC_INCLUDE"
-  JEMALLOC=1
-else
-  # clang
-  CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
-  CC="$CLANG_BIN/clang"
-  CXX="$CLANG_BIN/clang++"
-
-  KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
-
-  CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x "
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x/x86_64-facebook-linux "
-  CFLAGS+=" -isystem $GLIBC_INCLUDE"
-  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
-  CFLAGS+=" -isystem $CLANG_INCLUDE"
-  CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
-  CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
-  CFLAGS+=" -Wno-expansion-to-defined "
-  CXXFLAGS="-nostdinc++"
-fi
-
-CFLAGS+=" $DEPS_INCLUDE"
-CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
-CXXFLAGS+=" $CFLAGS"
-
-EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
-EXEC_LDFLAGS+=" -B$BINUTILS/gold"
-EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform007/lib/ld.so"
-EXEC_LDFLAGS+=" $LIBUNWIND"
-EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform007/lib"
-# required by libtbb
-EXEC_LDFLAGS+=" -ldl"
-
-PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
-
-EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS"
-
-VALGRIND_VER="$VALGRIND_BASE/bin/"
-
-# lua not supported because it's on track for deprecation, I think
-LUA_PATH=
-LUA_LIB=
-
-export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB

build_tools/make_new_version.sh

@@ -1,53 +0,0 @@
-#!/bin/bash
-#  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-#  This source code is licensed under both the GPLv2 (found in the
-#  COPYING file in the root directory) and Apache 2.0 License
-#  (found in the LICENSE.Apache file in the root directory).
-
-set -e
-if [ -z "$GIT" ]
-then
-  GIT="git"
-fi
-
-# Print out the colored progress info so that it can be brainlessly
-# distinguished by users.
-function title() {
-  echo -e "\033[1;32m$*\033[0m"
-}
-
-usage="Create new RocksDB version and prepare it for the release process\n"
-usage+="USAGE: ./make_new_version.sh <version> [<remote>]\n"
-usage+="  version: specify a version without '.fb' suffix (e.g. 5.4).\n"
-usage+="  remote: name of the remote to push the branch to (default: origin)."
-
-# -- Pre-check
-if [[ $# < 1 ]]; then
-  echo -e $usage
-  exit 1
-fi
-
-ROCKSDB_VERSION=$1
-
-REMOTE="origin"
-if [[ $# > 1 ]]; then
-  REMOTE=$2
-fi
-
-GIT_BRANCH=`git rev-parse --abbrev-ref HEAD`
-echo $GIT_BRANCH
-
-if [ $GIT_BRANCH != "master" ]; then
-  echo "Error: Current branch is '$GIT_BRANCH', Please switch to master branch."
-  exit 1
-fi
-
-title "Adding new tag for this release ..."
-BRANCH="$ROCKSDB_VERSION.fb"
-$GIT checkout -b $BRANCH
-
-# Setting up the proxy for remote repo access
-title "Pushing new branch to remote repo ..."
-git push $REMOTE --set-upstream $BRANCH
-
-title "Branch $BRANCH is pushed to github;"

build_tools/rocksdb-lego-determinator

@@ -85,9 +85,8 @@ NON_SHM="TMPD=/tmp/rocksdb_test_tmp"
 GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1"
 ASAN="COMPILE_WITH_ASAN=1"
 CLANG="USE_CLANG=1"
-# in gcc-5 there are known problems with TSAN like https://gcc.gnu.org/bugzilla/show_bug.cgi?id=71090.
-# using platform007 gives us gcc-8 or higher which has that bug fixed.
-TSAN="ROCKSDB_FBCODE_BUILD_WITH_PLATFORM007=1 COMPILE_WITH_TSAN=1"
+LITE="OPT=\"-DROCKSDB_LITE -g\""
+TSAN="COMPILE_WITH_TSAN=1"
 UBSAN="COMPILE_WITH_UBSAN=1"
 DISABLE_JEMALLOC="DISABLE_JEMALLOC=1"
 HTTP_PROXY="https_proxy=http://fwdproxy.29.prn1:8080 http_proxy=http://fwdproxy.29.prn1:8080 ftp_proxy=http://fwdproxy.29.prn1:8080"

build_tools/run_ci_db_test.ps1

@@ -333,7 +333,7 @@ function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
             $Cases = @{}
 
             if($Suites.Count -gt 0) {
-              # Will teh first one
+              # Will the first one
               ForEach($e in $Suites.Keys) {
                 $exe_name = $e
                 $Cases = $Suites[$e]

build_tools/update_dependencies.sh

@@ -52,45 +52,6 @@ function get_lib_base()
   log_variable $__res_var
 }
 
-###########################################################
-#                platform007 dependencies                 #
-###########################################################
-
-OUTPUT="$BASEDIR/dependencies_platform007.sh"
-
-rm -f "$OUTPUT"
-touch "$OUTPUT"
-
-echo "Writing dependencies to $OUTPUT"
-
-# Compilers locations
-GCC_BASE=`readlink -f $TP2_LATEST/gcc/7.x/centos7-native/*/`
-CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos7-native/*/`
-
-log_variable GCC_BASE
-log_variable CLANG_BASE
-
-# Libraries locations
-get_lib_base libgcc     7.x     platform007
-get_lib_base glibc      2.26    platform007
-get_lib_base snappy     LATEST  platform007
-get_lib_base zlib       LATEST  platform007
-get_lib_base bzip2      LATEST  platform007
-get_lib_base lz4        LATEST  platform007
-get_lib_base zstd       LATEST  platform007
-get_lib_base gflags     LATEST  platform007
-get_lib_base jemalloc   LATEST  platform007
-get_lib_base numa       LATEST  platform007
-get_lib_base libunwind  LATEST  platform007
-get_lib_base tbb        LATEST  platform007
-
-get_lib_base kernel-headers fb platform007
-get_lib_base binutils   LATEST centos7-native
-get_lib_base valgrind   LATEST platform007
-get_lib_base lua        5.3.4  platform007
-
-git diff $OUTPUT
-
 ###########################################################
 #                   5.x dependencies                      #
 ###########################################################

db/c.cc

@@ -1372,24 +1372,23 @@ void rocksdb_writebatch_put_log_data(
   b->rep.PutLogData(Slice(blob, len));
 }
 
-class H : public WriteBatch::Handler {
- public:
-  void* state_;
-  void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
-  void (*deleted_)(void*, const char* k, size_t klen);
-  virtual void Put(const Slice& key, const Slice& value) override {
-    (*put_)(state_, key.data(), key.size(), value.data(), value.size());
-  }
-  virtual void Delete(const Slice& key) override {
-    (*deleted_)(state_, key.data(), key.size());
-  }
-};
-
 void rocksdb_writebatch_iterate(
     rocksdb_writebatch_t* b,
     void* state,
     void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen),
     void (*deleted)(void*, const char* k, size_t klen)) {
+  class H : public WriteBatch::Handler {
+   public:
+    void* state_;
+    void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
+    void (*deleted_)(void*, const char* k, size_t klen);
+    virtual void Put(const Slice& key, const Slice& value) override {
+      (*put_)(state_, key.data(), key.size(), value.data(), value.size());
+    }
+    virtual void Delete(const Slice& key) override {
+      (*deleted_)(state_, key.data(), key.size());
+    }
+  };
   H handler;
   handler.state_ = state;
   handler.put_ = put;
@@ -1634,6 +1633,18 @@ void rocksdb_writebatch_wi_iterate(
     void* state,
     void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen),
     void (*deleted)(void*, const char* k, size_t klen)) {
+  class H : public WriteBatch::Handler {
+   public:
+    void* state_;
+    void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
+    void (*deleted_)(void*, const char* k, size_t klen);
+    virtual void Put(const Slice& key, const Slice& value) override {
+      (*put_)(state_, key.data(), key.size(), value.data(), value.size());
+    }
+    virtual void Delete(const Slice& key) override {
+      (*deleted_)(state_, key.data(), key.size());
+    }
+  };
   H handler;
   handler.state_ = state;
   handler.put_ = put;
@@ -2839,7 +2850,26 @@ void rocksdb_sstfilewriter_open(rocksdb_sstfilewriter_t* writer,
 void rocksdb_sstfilewriter_add(rocksdb_sstfilewriter_t* writer, const char* key,
                                size_t keylen, const char* val, size_t vallen,
                                char** errptr) {
-  SaveError(errptr, writer->rep->Add(Slice(key, keylen), Slice(val, vallen)));
+  SaveError(errptr, writer->rep->Put(Slice(key, keylen), Slice(val, vallen)));
+}
+
+void rocksdb_sstfilewriter_put(rocksdb_sstfilewriter_t* writer, const char* key,
+                               size_t keylen, const char* val, size_t vallen,
+                               char** errptr) {
+  SaveError(errptr, writer->rep->Put(Slice(key, keylen), Slice(val, vallen)));
+}
+
+void rocksdb_sstfilewriter_merge(rocksdb_sstfilewriter_t* writer,
+                                 const char* key, size_t keylen,
+                                 const char* val, size_t vallen,
+                                 char** errptr) {
+  SaveError(errptr, writer->rep->Merge(Slice(key, keylen), Slice(val, vallen)));
+}
+
+void rocksdb_sstfilewriter_delete(rocksdb_sstfilewriter_t* writer,
+                                  const char* key, size_t keylen,
+                                  char** errptr) {
+  SaveError(errptr, writer->rep->Delete(Slice(key, keylen)));
 }
 
 void rocksdb_sstfilewriter_finish(rocksdb_sstfilewriter_t* writer,
@@ -2936,21 +2966,20 @@ void rocksdb_slicetransform_destroy(rocksdb_slicetransform_t* st) {
   delete st;
 }
 
-struct Wrapper : public rocksdb_slicetransform_t {
-  const SliceTransform* rep_;
-  ~Wrapper() { delete rep_; }
-  const char* Name() const override { return rep_->Name(); }
-  Slice Transform(const Slice& src) const override {
-    return rep_->Transform(src);
-  }
-  bool InDomain(const Slice& src) const override {
-    return rep_->InDomain(src);
-  }
-  bool InRange(const Slice& src) const override { return rep_->InRange(src); }
-  static void DoNothing(void*) { }
-};
-
 rocksdb_slicetransform_t* rocksdb_slicetransform_create_fixed_prefix(size_t prefixLen) {
+  struct Wrapper : public rocksdb_slicetransform_t {
+    const SliceTransform* rep_;
+    ~Wrapper() { delete rep_; }
+    const char* Name() const override { return rep_->Name(); }
+    Slice Transform(const Slice& src) const override {
+      return rep_->Transform(src);
+    }
+    bool InDomain(const Slice& src) const override {
+      return rep_->InDomain(src);
+    }
+    bool InRange(const Slice& src) const override { return rep_->InRange(src); }
+    static void DoNothing(void*) { }
+  };
   Wrapper* wrapper = new Wrapper;
   wrapper->rep_ = rocksdb::NewFixedPrefixTransform(prefixLen);
   wrapper->state_ = nullptr;
@@ -2959,6 +2988,19 @@ rocksdb_slicetransform_t* rocksdb_slicetransform_create_fixed_prefix(size_t pref
 }
 
 rocksdb_slicetransform_t* rocksdb_slicetransform_create_noop() {
+  struct Wrapper : public rocksdb_slicetransform_t {
+    const SliceTransform* rep_;
+    ~Wrapper() { delete rep_; }
+    const char* Name() const override { return rep_->Name(); }
+    Slice Transform(const Slice& src) const override {
+      return rep_->Transform(src);
+    }
+    bool InDomain(const Slice& src) const override {
+      return rep_->InDomain(src);
+    }
+    bool InRange(const Slice& src) const override { return rep_->InRange(src); }
+    static void DoNothing(void*) { }
+  };
   Wrapper* wrapper = new Wrapper;
   wrapper->rep_ = rocksdb::NewNoopTransform();
   wrapper->state_ = nullptr;

db/c_test.c

@@ -569,11 +569,11 @@ int main(int argc, char** argv) {
     unlink(sstfilename);
     rocksdb_sstfilewriter_open(writer, sstfilename, &err);
     CheckNoError(err);
-    rocksdb_sstfilewriter_add(writer, "sstk1", 5, "v1", 2, &err);
+    rocksdb_sstfilewriter_put(writer, "sstk1", 5, "v1", 2, &err);
     CheckNoError(err);
-    rocksdb_sstfilewriter_add(writer, "sstk2", 5, "v2", 2, &err);
+    rocksdb_sstfilewriter_put(writer, "sstk2", 5, "v2", 2, &err);
     CheckNoError(err);
-    rocksdb_sstfilewriter_add(writer, "sstk3", 5, "v3", 2, &err);
+    rocksdb_sstfilewriter_put(writer, "sstk3", 5, "v3", 2, &err);
     CheckNoError(err);
     rocksdb_sstfilewriter_finish(writer, &err);
     CheckNoError(err);
@@ -590,11 +590,11 @@ int main(int argc, char** argv) {
     unlink(sstfilename);
     rocksdb_sstfilewriter_open(writer, sstfilename, &err);
     CheckNoError(err);
-    rocksdb_sstfilewriter_add(writer, "sstk2", 5, "v4", 2, &err);
+    rocksdb_sstfilewriter_put(writer, "sstk2", 5, "v4", 2, &err);
     CheckNoError(err);
-    rocksdb_sstfilewriter_add(writer, "sstk22", 6, "v5", 2, &err);
+    rocksdb_sstfilewriter_put(writer, "sstk22", 6, "v5", 2, &err);
     CheckNoError(err);
-    rocksdb_sstfilewriter_add(writer, "sstk3", 5, "v6", 2, &err);
+    rocksdb_sstfilewriter_put(writer, "sstk3", 5, "v6", 2, &err);
     CheckNoError(err);
     rocksdb_sstfilewriter_finish(writer, &err);
     CheckNoError(err);

db/column_family.cc

@@ -347,6 +347,7 @@ ColumnFamilyData::ColumnFamilyData(
       dummy_versions_(_dummy_versions),
       current_(nullptr),
       refs_(0),
+      initialized_(false),
       dropped_(false),
       internal_comparator_(cf_options.comparator),
       initial_cf_options_(SanitizeOptions(db_options, cf_options)),
@@ -549,7 +550,7 @@ std::unique_ptr<WriteControllerToken> SetupDelay(
     // If DB just falled into the stop condition, we need to further reduce
     // the write rate to avoid the stop condition.
     if (penalize_stop) {
-      // Penalize the near stop or stop condition by more agressive slowdown.
+      // Penalize the near stop or stop condition by more aggressive slowdown.
       // This is to provide the long term slowdown increase signal.
       // The penalty is more than the reward of recovering to the normal
       // condition.
@@ -727,7 +728,7 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
               mutable_cf_options.level0_slowdown_writes_trigger)) {
         write_controller_token_ =
             write_controller->GetCompactionPressureToken();
-        ROCKS_LOG_WARN(
+        ROCKS_LOG_INFO(
             ioptions_.info_log,
             "[%s] Increasing compaction threads because we have %d level-0 "
             "files ",
@@ -741,7 +742,7 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
         write_controller_token_ =
             write_controller->GetCompactionPressureToken();
         if (mutable_cf_options.soft_pending_compaction_bytes_limit > 0) {
-          ROCKS_LOG_WARN(
+          ROCKS_LOG_INFO(
               ioptions_.info_log,
               "[%s] Increasing compaction threads because of estimated pending "
               "compaction "
@@ -758,6 +759,12 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
         uint64_t write_rate = write_controller->delayed_write_rate();
         write_controller->set_delayed_write_rate(static_cast<uint64_t>(
             static_cast<double>(write_rate) * kDelayRecoverSlowdownRatio));
+        // Set the low pri limit to be 1/4 the delayed write rate.
+        // Note we don't reset this value even after delay condition is relased.
+        // Low-pri rate will continue to apply if there is a compaction
+        // pressure.
+        write_controller->low_pri_rate_limiter()->SetBytesPerSecond(write_rate /
+                                                                    4);
       }
     }
     prev_compaction_needed_bytes_ = compaction_needed_bytes;
@@ -783,7 +790,7 @@ uint64_t ColumnFamilyData::GetTotalSstFilesSize() const {
 MemTable* ColumnFamilyData::ConstructNewMemtable(
     const MutableCFOptions& mutable_cf_options, SequenceNumber earliest_seq) {
   return new MemTable(internal_comparator_, ioptions_, mutable_cf_options,
-                      write_buffer_manager_, earliest_seq);
+                      write_buffer_manager_, earliest_seq, id_);
 }
 
 void ColumnFamilyData::CreateNewMemtable(

db/column_family.h

@@ -333,6 +333,10 @@ class ColumnFamilyData {
   void RecalculateWriteStallConditions(
       const MutableCFOptions& mutable_cf_options);
 
+  void set_initialized() { initialized_.store(true); }
+
+  bool initialized() const { return initialized_.load(); }
+
  private:
   friend class ColumnFamilySet;
   ColumnFamilyData(uint32_t id, const std::string& name,
@@ -349,6 +353,7 @@ class ColumnFamilyData {
   Version* current_;         // == dummy_versions->prev_
 
   std::atomic<int> refs_;      // outstanding references to ColumnFamilyData
+  std::atomic<bool> initialized_;
   bool dropped_;               // true if client dropped it
 
   const InternalKeyComparator internal_comparator_;

db/column_family_test.cc

@@ -520,6 +520,30 @@ TEST_F(ColumnFamilyTest, DontReuseColumnFamilyID) {
   }
 }
 
+#ifndef ROCKSDB_LITE
+TEST_F(ColumnFamilyTest, CreateCFRaceWithGetAggProperty) {
+  Open();
+
+  rocksdb::SyncPoint::GetInstance()->LoadDependency(
+      {{"DBImpl::WriteOptionsFile:1",
+        "ColumnFamilyTest.CreateCFRaceWithGetAggProperty:1"},
+       {"ColumnFamilyTest.CreateCFRaceWithGetAggProperty:2",
+        "DBImpl::WriteOptionsFile:2"}});
+  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+
+  rocksdb::port::Thread thread([&] { CreateColumnFamilies({"one"}); });
+
+  TEST_SYNC_POINT("ColumnFamilyTest.CreateCFRaceWithGetAggProperty:1");
+  uint64_t pv;
+  db_->GetAggregatedIntProperty(DB::Properties::kEstimateTableReadersMem, &pv);
+  TEST_SYNC_POINT("ColumnFamilyTest.CreateCFRaceWithGetAggProperty:2");
+
+  thread.join();
+
+  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
+}
+#endif  // !ROCKSDB_LITE
+
 class FlushEmptyCFTestWithParam : public ColumnFamilyTest,
                                   public testing::WithParamInterface<bool> {
  public:
@@ -1379,7 +1403,6 @@ TEST_F(ColumnFamilyTest, AutomaticAndManualCompactions) {
   ColumnFamilyOptions default_cf, one, two;
   db_options_.max_open_files = 20;  // only 10 files in file cache
   db_options_.max_background_compactions = 3;
-  db_options_.base_background_compactions = 3;
 
   default_cf.compaction_style = kCompactionStyleLevel;
   default_cf.num_levels = 3;
@@ -1403,6 +1426,9 @@ TEST_F(ColumnFamilyTest, AutomaticAndManualCompactions) {
   two.write_buffer_size = 100000;
 
   Reopen({default_cf, one, two});
+  // make sure all background compaction jobs can be scheduled
+  auto stop_token =
+      dbfull()->TEST_write_controler().GetCompactionPressureToken();
 
   bool cf_1_1 = true;
   rocksdb::SyncPoint::GetInstance()->LoadDependency(
@@ -1462,7 +1488,6 @@ TEST_F(ColumnFamilyTest, AutomaticAndManualCompactions) {
     ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
     key_iter++;
   }
-  Close();
 }
 
 TEST_F(ColumnFamilyTest, ManualAndAutomaticCompactions) {
@@ -1471,7 +1496,6 @@ TEST_F(ColumnFamilyTest, ManualAndAutomaticCompactions) {
   ColumnFamilyOptions default_cf, one, two;
   db_options_.max_open_files = 20;  // only 10 files in file cache
   db_options_.max_background_compactions = 3;
-  db_options_.base_background_compactions = 3;
 
   default_cf.compaction_style = kCompactionStyleLevel;
   default_cf.num_levels = 3;
@@ -1495,6 +1519,9 @@ TEST_F(ColumnFamilyTest, ManualAndAutomaticCompactions) {
   two.write_buffer_size = 100000;
 
   Reopen({default_cf, one, two});
+  // make sure all background compaction jobs can be scheduled
+  auto stop_token =
+      dbfull()->TEST_write_controler().GetCompactionPressureToken();
 
   // SETUP column family "one" -- universal style
   for (int i = 0; i < one.level0_file_num_compaction_trigger - 2; ++i) {
@@ -1557,7 +1584,6 @@ TEST_F(ColumnFamilyTest, ManualAndAutomaticCompactions) {
     ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
     key_iter++;
   }
-  Close();
 }
 
 TEST_F(ColumnFamilyTest, SameCFManualManualCompactions) {
@@ -1566,7 +1592,6 @@ TEST_F(ColumnFamilyTest, SameCFManualManualCompactions) {
   ColumnFamilyOptions default_cf, one;
   db_options_.max_open_files = 20;  // only 10 files in file cache
   db_options_.max_background_compactions = 3;
-  db_options_.base_background_compactions = 3;
 
   default_cf.compaction_style = kCompactionStyleLevel;
   default_cf.num_levels = 3;
@@ -1585,6 +1610,9 @@ TEST_F(ColumnFamilyTest, SameCFManualManualCompactions) {
   one.write_buffer_size = 120000;
 
   Reopen({default_cf, one});
+  // make sure all background compaction jobs can be scheduled
+  auto stop_token =
+      dbfull()->TEST_write_controler().GetCompactionPressureToken();
 
   // SETUP column family "one" -- universal style
   for (int i = 0; i < one.level0_file_num_compaction_trigger - 2; ++i) {
@@ -1655,7 +1683,6 @@ TEST_F(ColumnFamilyTest, SameCFManualManualCompactions) {
     ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
     key_iter++;
   }
-  Close();
 }
 
 TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactions) {
@@ -1664,7 +1691,6 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactions) {
   ColumnFamilyOptions default_cf, one;
   db_options_.max_open_files = 20;  // only 10 files in file cache
   db_options_.max_background_compactions = 3;
-  db_options_.base_background_compactions = 3;
 
   default_cf.compaction_style = kCompactionStyleLevel;
   default_cf.num_levels = 3;
@@ -1683,6 +1709,9 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactions) {
   one.write_buffer_size = 120000;
 
   Reopen({default_cf, one});
+  // make sure all background compaction jobs can be scheduled
+  auto stop_token =
+      dbfull()->TEST_write_controler().GetCompactionPressureToken();
 
   // SETUP column family "one" -- universal style
   for (int i = 0; i < one.level0_file_num_compaction_trigger - 2; ++i) {
@@ -1744,7 +1773,6 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactions) {
     ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
     key_iter++;
   }
-  Close();
 }
 
 TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactionsLevel) {
@@ -1753,7 +1781,6 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactionsLevel) {
   ColumnFamilyOptions default_cf, one;
   db_options_.max_open_files = 20;  // only 10 files in file cache
   db_options_.max_background_compactions = 3;
-  db_options_.base_background_compactions = 3;
 
   default_cf.compaction_style = kCompactionStyleLevel;
   default_cf.num_levels = 3;
@@ -1772,6 +1799,9 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactionsLevel) {
   one.write_buffer_size = 120000;
 
   Reopen({default_cf, one});
+  // make sure all background compaction jobs can be scheduled
+  auto stop_token =
+      dbfull()->TEST_write_controler().GetCompactionPressureToken();
 
   // SETUP column family "one" -- level style
   for (int i = 0; i < one.level0_file_num_compaction_trigger - 2; ++i) {
@@ -1833,7 +1863,6 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactionsLevel) {
     ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
     key_iter++;
   }
-  Close();
 }
 
 // This test checks for automatic getting a conflict if there is a
@@ -1849,7 +1878,6 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticConflict) {
   ColumnFamilyOptions default_cf, one;
   db_options_.max_open_files = 20;  // only 10 files in file cache
   db_options_.max_background_compactions = 3;
-  db_options_.base_background_compactions = 3;
 
   default_cf.compaction_style = kCompactionStyleLevel;
   default_cf.num_levels = 3;
@@ -1868,6 +1896,9 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticConflict) {
   one.write_buffer_size = 120000;
 
   Reopen({default_cf, one});
+  // make sure all background compaction jobs can be scheduled
+  auto stop_token =
+      dbfull()->TEST_write_controler().GetCompactionPressureToken();
 
   // SETUP column family "one" -- universal style
   for (int i = 0; i < one.level0_file_num_compaction_trigger - 2; ++i) {
@@ -1952,8 +1983,6 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticConflict) {
     ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
     key_iter++;
   }
-
-  Close();
 }
 
 // In this test, we generate enough files to trigger automatic compactions.
@@ -1968,7 +1997,6 @@ TEST_F(ColumnFamilyTest, SameCFAutomaticManualCompactions) {
   ColumnFamilyOptions default_cf, one;
   db_options_.max_open_files = 20;  // only 10 files in file cache
   db_options_.max_background_compactions = 3;
-  db_options_.base_background_compactions = 3;
 
   default_cf.compaction_style = kCompactionStyleLevel;
   default_cf.num_levels = 3;
@@ -1987,6 +2015,9 @@ TEST_F(ColumnFamilyTest, SameCFAutomaticManualCompactions) {
   one.write_buffer_size = 120000;
 
   Reopen({default_cf, one});
+  // make sure all background compaction jobs can be scheduled
+  auto stop_token =
+      dbfull()->TEST_write_controler().GetCompactionPressureToken();
 
   bool cf_1_1 = true;
   bool cf_1_2 = true;
@@ -2041,8 +2072,6 @@ TEST_F(ColumnFamilyTest, SameCFAutomaticManualCompactions) {
     ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
     key_iter++;
   }
-
-  Close();
 }
 #endif  // !ROCKSDB_LITE
 
@@ -2495,7 +2524,6 @@ TEST_F(ColumnFamilyTest, CreateAndDropRace) {
 TEST_F(ColumnFamilyTest, WriteStallSingleColumnFamily) {
   const uint64_t kBaseRate = 800000u;
   db_options_.delayed_write_rate = kBaseRate;
-  db_options_.base_background_compactions = 2;
   db_options_.max_background_compactions = 6;
 
   Open({"default"});
@@ -2676,7 +2704,6 @@ TEST_F(ColumnFamilyTest, WriteStallSingleColumnFamily) {
 }
 
 TEST_F(ColumnFamilyTest, CompactionSpeedupSingleColumnFamily) {
-  db_options_.base_background_compactions = 2;
   db_options_.max_background_compactions = 6;
   Open({"default"});
   ColumnFamilyData* cfd =
@@ -2696,7 +2723,7 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupSingleColumnFamily) {
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(40);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed());
+  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(50);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
@@ -2708,11 +2735,11 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupSingleColumnFamily) {
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(45);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed());
+  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
 
   vstorage->set_l0_delay_trigger_count(7);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed());
+  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
 
   vstorage->set_l0_delay_trigger_count(9);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
@@ -2720,7 +2747,7 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupSingleColumnFamily) {
 
   vstorage->set_l0_delay_trigger_count(6);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed());
+  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
 
   // Speed up threshold = min(4 * 2, 4 + (12 - 4)/4) = 6
   mutable_cf_options.level0_file_num_compaction_trigger = 4;
@@ -2729,7 +2756,7 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupSingleColumnFamily) {
 
   vstorage->set_l0_delay_trigger_count(5);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed());
+  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
 
   vstorage->set_l0_delay_trigger_count(7);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
@@ -2737,7 +2764,7 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupSingleColumnFamily) {
 
   vstorage->set_l0_delay_trigger_count(3);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed());
+  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
 }
 
 TEST_F(ColumnFamilyTest, WriteStallTwoColumnFamilies) {
@@ -2816,7 +2843,6 @@ TEST_F(ColumnFamilyTest, WriteStallTwoColumnFamilies) {
 }
 
 TEST_F(ColumnFamilyTest, CompactionSpeedupTwoColumnFamilies) {
-  db_options_.base_background_compactions = 2;
   db_options_.max_background_compactions = 6;
   column_family_options_.soft_pending_compaction_bytes_limit = 200;
   column_family_options_.hard_pending_compaction_bytes_limit = 2000;
@@ -2844,11 +2870,11 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupTwoColumnFamilies) {
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(40);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed());
+  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(60);
   cfd1->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed());
+  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
   ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed());
 
@@ -2866,7 +2892,7 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupTwoColumnFamilies) {
 
   vstorage1->TEST_set_estimated_compaction_needed_bytes(3);
   cfd1->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed());
+  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
 
   vstorage->set_l0_delay_trigger_count(9);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
@@ -2878,7 +2904,7 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupTwoColumnFamilies) {
 
   vstorage->set_l0_delay_trigger_count(0);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed());
+  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
 }
 
 #ifndef ROCKSDB_LITE

db/compaction_job_stats_test.cc

@@ -654,7 +654,6 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
   Options options;
   options.listeners.emplace_back(stats_checker);
   options.create_if_missing = true;
-  options.max_background_flushes = 0;
   // just enough setting to hold off auto-compaction.
   options.level0_file_num_compaction_trigger = kTestScale + 1;
   options.num_levels = 3;
@@ -876,7 +875,6 @@ TEST_P(CompactionJobStatsTest, DeletionStatsTest) {
   Options options;
   options.listeners.emplace_back(stats_checker);
   options.create_if_missing = true;
-  options.max_background_flushes = 0;
   options.level0_file_num_compaction_trigger = kTestScale+1;
   options.num_levels = 3;
   options.compression = kNoCompression;

db/compaction_picker_test.cc

@@ -950,7 +950,7 @@ TEST_F(CompactionPickerTest, NotScheduleL1IfL0WithHigherPri3) {
 TEST_F(CompactionPickerTest, EstimateCompactionBytesNeeded1) {
   int num_levels = ioptions_.num_levels;
   ioptions_.level_compaction_dynamic_level_bytes = false;
-  mutable_cf_options_.level0_file_num_compaction_trigger = 3;
+  mutable_cf_options_.level0_file_num_compaction_trigger = 4;
   mutable_cf_options_.max_bytes_for_level_base = 1000;
   mutable_cf_options_.max_bytes_for_level_multiplier = 10;
   NewVersionStorage(num_levels, kCompactionStyleLevel);

db/compaction_picker_universal.cc

@@ -619,7 +619,7 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSizeAmp(
     const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
     VersionStorageInfo* vstorage, double score,
     const std::vector<SortedRun>& sorted_runs, LogBuffer* log_buffer) {
-  // percentage flexibilty while reducing size amplification
+  // percentage flexibility while reducing size amplification
   uint64_t ratio =
       ioptions_.compaction_options_universal.max_size_amplification_percent;
 

db/db_basic_test.cc

@@ -364,9 +364,9 @@ TEST_F(DBBasicTest, FLUSH) {
     ASSERT_OK(Flush(1));
     ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1"));
 
-    perf_context.Reset();
+    get_perf_context()->Reset();
     Get(1, "foo");
-    ASSERT_TRUE((int)perf_context.get_from_output_files_time > 0);
+    ASSERT_TRUE((int)get_perf_context()->get_from_output_files_time > 0);
 
     ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
     ASSERT_EQ("v1", Get(1, "foo"));
@@ -379,9 +379,9 @@ TEST_F(DBBasicTest, FLUSH) {
 
     ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
     ASSERT_EQ("v2", Get(1, "bar"));
-    perf_context.Reset();
+    get_perf_context()->Reset();
     ASSERT_EQ("v2", Get(1, "foo"));
-    ASSERT_TRUE((int)perf_context.get_from_output_files_time > 0);
+    ASSERT_TRUE((int)get_perf_context()->get_from_output_files_time > 0);
 
     writeOpt.disableWAL = false;
     ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v3"));

db/db_bloom_filter_test.cc

@@ -686,12 +686,12 @@ class BloomStatsTestWithParam
     }
     options_.env = env_;
 
-    perf_context.Reset();
+    get_perf_context()->Reset();
     DestroyAndReopen(options_);
   }
 
   ~BloomStatsTestWithParam() {
-    perf_context.Reset();
+    get_perf_context()->Reset();
     Destroy(options_);
   }
 
@@ -724,33 +724,33 @@ TEST_P(BloomStatsTestWithParam, BloomStatsTest) {
 
   // check memtable bloom stats
   ASSERT_EQ(value1, Get(key1));
-  ASSERT_EQ(1, perf_context.bloom_memtable_hit_count);
+  ASSERT_EQ(1, get_perf_context()->bloom_memtable_hit_count);
   ASSERT_EQ(value3, Get(key3));
-  ASSERT_EQ(2, perf_context.bloom_memtable_hit_count);
-  ASSERT_EQ(0, perf_context.bloom_memtable_miss_count);
+  ASSERT_EQ(2, get_perf_context()->bloom_memtable_hit_count);
+  ASSERT_EQ(0, get_perf_context()->bloom_memtable_miss_count);
 
   ASSERT_EQ("NOT_FOUND", Get(key2));
-  ASSERT_EQ(1, perf_context.bloom_memtable_miss_count);
-  ASSERT_EQ(2, perf_context.bloom_memtable_hit_count);
+  ASSERT_EQ(1, get_perf_context()->bloom_memtable_miss_count);
+  ASSERT_EQ(2, get_perf_context()->bloom_memtable_hit_count);
 
   // sanity checks
-  ASSERT_EQ(0, perf_context.bloom_sst_hit_count);
-  ASSERT_EQ(0, perf_context.bloom_sst_miss_count);
+  ASSERT_EQ(0, get_perf_context()->bloom_sst_hit_count);
+  ASSERT_EQ(0, get_perf_context()->bloom_sst_miss_count);
 
   Flush();
 
   // sanity checks
-  ASSERT_EQ(0, perf_context.bloom_sst_hit_count);
-  ASSERT_EQ(0, perf_context.bloom_sst_miss_count);
+  ASSERT_EQ(0, get_perf_context()->bloom_sst_hit_count);
+  ASSERT_EQ(0, get_perf_context()->bloom_sst_miss_count);
 
   // check SST bloom stats
   ASSERT_EQ(value1, Get(key1));
-  ASSERT_EQ(1, perf_context.bloom_sst_hit_count);
+  ASSERT_EQ(1, get_perf_context()->bloom_sst_hit_count);
   ASSERT_EQ(value3, Get(key3));
-  ASSERT_EQ(2, perf_context.bloom_sst_hit_count);
+  ASSERT_EQ(2, get_perf_context()->bloom_sst_hit_count);
 
   ASSERT_EQ("NOT_FOUND", Get(key2));
-  ASSERT_EQ(1, perf_context.bloom_sst_miss_count);
+  ASSERT_EQ(1, get_perf_context()->bloom_sst_miss_count);
 }
 
 // Same scenario as in BloomStatsTest but using an iterator
@@ -771,21 +771,21 @@ TEST_P(BloomStatsTestWithParam, BloomStatsTestWithIter) {
   ASSERT_OK(iter->status());
   ASSERT_TRUE(iter->Valid());
   ASSERT_EQ(value1, iter->value().ToString());
-  ASSERT_EQ(1, perf_context.bloom_memtable_hit_count);
-  ASSERT_EQ(0, perf_context.bloom_memtable_miss_count);
+  ASSERT_EQ(1, get_perf_context()->bloom_memtable_hit_count);
+  ASSERT_EQ(0, get_perf_context()->bloom_memtable_miss_count);
 
   iter->Seek(key3);
   ASSERT_OK(iter->status());
   ASSERT_TRUE(iter->Valid());
   ASSERT_EQ(value3, iter->value().ToString());
-  ASSERT_EQ(2, perf_context.bloom_memtable_hit_count);
-  ASSERT_EQ(0, perf_context.bloom_memtable_miss_count);
+  ASSERT_EQ(2, get_perf_context()->bloom_memtable_hit_count);
+  ASSERT_EQ(0, get_perf_context()->bloom_memtable_miss_count);
 
   iter->Seek(key2);
   ASSERT_OK(iter->status());
   ASSERT_TRUE(!iter->Valid());
-  ASSERT_EQ(1, perf_context.bloom_memtable_miss_count);
-  ASSERT_EQ(2, perf_context.bloom_memtable_hit_count);
+  ASSERT_EQ(1, get_perf_context()->bloom_memtable_miss_count);
+  ASSERT_EQ(2, get_perf_context()->bloom_memtable_hit_count);
 
   Flush();
 
@@ -796,19 +796,19 @@ TEST_P(BloomStatsTestWithParam, BloomStatsTestWithIter) {
   ASSERT_OK(iter->status());
   ASSERT_TRUE(iter->Valid());
   ASSERT_EQ(value1, iter->value().ToString());
-  ASSERT_EQ(1, perf_context.bloom_sst_hit_count);
+  ASSERT_EQ(1, get_perf_context()->bloom_sst_hit_count);
 
   iter->Seek(key3);
   ASSERT_OK(iter->status());
   ASSERT_TRUE(iter->Valid());
   ASSERT_EQ(value3, iter->value().ToString());
-  ASSERT_EQ(2, perf_context.bloom_sst_hit_count);
+  ASSERT_EQ(2, get_perf_context()->bloom_sst_hit_count);
 
   iter->Seek(key2);
   ASSERT_OK(iter->status());
   ASSERT_TRUE(!iter->Valid());
-  ASSERT_EQ(1, perf_context.bloom_sst_miss_count);
-  ASSERT_EQ(2, perf_context.bloom_sst_hit_count);
+  ASSERT_EQ(1, get_perf_context()->bloom_sst_miss_count);
+  ASSERT_EQ(2, get_perf_context()->bloom_sst_hit_count);
 }
 
 INSTANTIATE_TEST_CASE_P(BloomStatsTestWithParam, BloomStatsTestWithParam,

db/db_compaction_filter_test.cc

@@ -778,7 +778,7 @@ TEST_F(DBTestCompactionFilter, SkipUntil) {
 
   cfilter_skips = 0;
   ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
-  // Numberof skips in tables: 2, 3, 3, 3.
+  // Number of skips in tables: 2, 3, 3, 3.
   ASSERT_EQ(11, cfilter_skips);
 
   for (int table = 0; table < 4; ++table) {
@@ -799,6 +799,43 @@ TEST_F(DBTestCompactionFilter, SkipUntil) {
   }
 }
 
+TEST_F(DBTestCompactionFilter, SkipUntilWithBloomFilter) {
+  BlockBasedTableOptions table_options;
+  table_options.whole_key_filtering = false;
+  table_options.filter_policy.reset(NewBloomFilterPolicy(100, false));
+
+  Options options = CurrentOptions();
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  options.prefix_extractor.reset(NewCappedPrefixTransform(9));
+  options.compaction_filter_factory = std::make_shared<SkipEvenFilterFactory>();
+  options.disable_auto_compactions = true;
+  options.create_if_missing = true;
+  DestroyAndReopen(options);
+
+  Put("0000000010", "v10");
+  Put("0000000020", "v20"); // skipped
+  Put("0000000050", "v50");
+  Flush();
+
+  cfilter_skips = 0;
+  EXPECT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+  EXPECT_EQ(1, cfilter_skips);
+
+  Status s;
+  std::string val;
+
+  s = db_->Get(ReadOptions(), "0000000010", &val);
+  ASSERT_OK(s);
+  EXPECT_EQ("v10", val);
+
+  s = db_->Get(ReadOptions(), "0000000020", &val);
+  EXPECT_TRUE(s.IsNotFound());
+
+  s = db_->Get(ReadOptions(), "0000000050", &val);
+  ASSERT_OK(s);
+  EXPECT_EQ("v50", val);
+}
+
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/db_compaction_test.cc

@@ -519,7 +519,6 @@ TEST_F(DBCompactionTest, BGCompactionsAllowed) {
   options.level0_file_num_compaction_trigger = 2;
   options.level0_slowdown_writes_trigger = 20;
   options.soft_pending_compaction_bytes_limit = 1 << 30;  // Infinitely large
-  options.base_background_compactions = 1;
   options.max_background_compactions = 3;
   options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
 
@@ -1228,6 +1227,9 @@ TEST_F(DBCompactionTest, DISABLED_ManualPartialFill) {
   options.max_background_compactions = 3;
 
   DestroyAndReopen(options);
+  // make sure all background compaction jobs can be scheduled
+  auto stop_token =
+      dbfull()->TEST_write_controler().GetCompactionPressureToken();
   int32_t value_size = 10 * 1024;  // 10 KB
 
   // Add 2 non-overlapping files
@@ -1975,7 +1977,6 @@ TEST_P(DBCompactionTestWithParam, ManualCompaction) {
 
     if (iter == 0) {
       options = CurrentOptions();
-      options.max_background_flushes = 0;
       options.num_levels = 3;
       options.create_if_missing = true;
       options.statistics = rocksdb::CreateDBStatistics();
@@ -2421,15 +2422,12 @@ TEST_F(DBCompactionTest, SanitizeCompactionOptionsTest) {
   options.hard_pending_compaction_bytes_limit = 100;
   options.create_if_missing = true;
   DestroyAndReopen(options);
-  ASSERT_EQ(5, db_->GetOptions().base_background_compactions);
   ASSERT_EQ(100, db_->GetOptions().soft_pending_compaction_bytes_limit);
 
-  options.base_background_compactions = 4;
   options.max_background_compactions = 3;
   options.soft_pending_compaction_bytes_limit = 200;
   options.hard_pending_compaction_bytes_limit = 150;
   DestroyAndReopen(options);
-  ASSERT_EQ(3, db_->GetOptions().base_background_compactions);
   ASSERT_EQ(150, db_->GetOptions().soft_pending_compaction_bytes_limit);
 }
 

db/db_dynamic_level_test.cc

@@ -57,7 +57,6 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase) {
       Options options;
       options.env = env.get();
       options.create_if_missing = true;
-      options.db_write_buffer_size = 2048;
       options.write_buffer_size = 2048;
       options.max_write_buffer_number = 2;
       options.level0_file_num_compaction_trigger = 2;
@@ -127,7 +126,6 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase2) {
 
   Options options = CurrentOptions();
   options.create_if_missing = true;
-  options.db_write_buffer_size = 204800;
   options.write_buffer_size = 20480;
   options.max_write_buffer_number = 2;
   options.level0_file_num_compaction_trigger = 2;
@@ -284,7 +282,6 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesCompactRange) {
 
   Options options = CurrentOptions();
   options.create_if_missing = true;
-  options.db_write_buffer_size = 2048;
   options.write_buffer_size = 2048;
   options.max_write_buffer_number = 2;
   options.level0_file_num_compaction_trigger = 2;
@@ -362,7 +359,6 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesCompactRange) {
 TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBaseInc) {
   Options options = CurrentOptions();
   options.create_if_missing = true;
-  options.db_write_buffer_size = 2048;
   options.write_buffer_size = 2048;
   options.max_write_buffer_number = 2;
   options.level0_file_num_compaction_trigger = 2;
@@ -375,6 +371,7 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBaseInc) {
   options.soft_rate_limit = 1.1;
   options.max_background_compactions = 2;
   options.num_levels = 5;
+  options.max_compaction_bytes = 100000000;
 
   DestroyAndReopen(options);
 
@@ -414,7 +411,6 @@ TEST_F(DBTestDynamicLevel, DISABLED_MigrateToDynamicLevelMaxBytesBase) {
 
   Options options;
   options.create_if_missing = true;
-  options.db_write_buffer_size = 2048;
   options.write_buffer_size = 2048;
   options.max_write_buffer_number = 8;
   options.level0_file_num_compaction_trigger = 4;

db/db_flush_test.cc

@@ -89,6 +89,43 @@ TEST_F(DBFlushTest, SyncFail) {
   Destroy(options);
 }
 
+TEST_F(DBFlushTest, FlushInLowPriThreadPool) {
+  // Verify setting an empty high-pri (flush) thread pool causes flushes to be
+  // scheduled in the low-pri (compaction) thread pool.
+  Options options = CurrentOptions();
+  options.level0_file_num_compaction_trigger = 4;
+  options.memtable_factory.reset(new SpecialSkipListFactory(1));
+  Reopen(options);
+  env_->SetBackgroundThreads(0, Env::HIGH);
+
+  std::thread::id tid;
+  int num_flushes = 0, num_compactions = 0;
+  SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::BGWorkFlush", [&](void* arg) {
+        if (tid == std::thread::id()) {
+          tid = std::this_thread::get_id();
+        } else {
+          ASSERT_EQ(tid, std::this_thread::get_id());
+        }
+        ++num_flushes;
+      });
+  SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::BGWorkCompaction", [&](void* arg) {
+        ASSERT_EQ(tid, std::this_thread::get_id());
+        ++num_compactions;
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  ASSERT_OK(Put("key", "val"));
+  for (int i = 0; i < 4; ++i) {
+    ASSERT_OK(Put("key", "val"));
+    dbfull()->TEST_WaitForFlushMemTable();
+  }
+  dbfull()->TEST_WaitForCompact();
+  ASSERT_EQ(4, num_flushes);
+  ASSERT_EQ(1, num_compactions);
+}
+
 TEST_P(DBFlushDirectIOTest, DirectIO) {
   Options options;
   options.create_if_missing = true;

db/db_impl.cc

@@ -16,9 +16,6 @@
 #ifdef OS_SOLARIS
 #include <alloca.h>
 #endif
-#ifdef ROCKSDB_JEMALLOC
-#include "jemalloc/jemalloc.h"
-#endif
 
 #include <algorithm>
 #include <climits>
@@ -44,6 +41,7 @@
 #include "db/job_context.h"
 #include "db/log_reader.h"
 #include "db/log_writer.h"
+#include "db/malloc_stats.h"
 #include "db/managed_iterator.h"
 #include "db/memtable.h"
 #include "db/memtable_list.h"
@@ -134,6 +132,8 @@ void DumpSupportInfo(Logger* logger) {
   ROCKS_LOG_HEADER(logger, "Fast CRC32 supported: %d",
                    crc32c::IsFastCrc32Supported());
 }
+
+int64_t kDefaultLowPriThrottledRate = 2 * 1024 * 1024;
 } // namespace
 
 DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
@@ -159,6 +159,11 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
       write_buffer_manager_(immutable_db_options_.write_buffer_manager.get()),
       write_thread_(immutable_db_options_),
       write_controller_(mutable_db_options_.delayed_write_rate),
+      // Use delayed_write_rate as a base line to determine the initial
+      // low pri write rate limit. It may be adjusted later.
+      low_pri_write_rate_limiter_(NewGenericRateLimiter(std::min(
+          static_cast<int64_t>(mutable_db_options_.delayed_write_rate / 8),
+          kDefaultLowPriThrottledRate))),
       last_batch_group_size_(0),
       unscheduled_flushes_(0),
       unscheduled_compactions_(0),
@@ -217,7 +222,7 @@ void DBImpl::CancelAllBackgroundWork(bool wait) {
       has_unpersisted_data_.load(std::memory_order_relaxed) &&
       !mutable_db_options_.avoid_flush_during_shutdown) {
     for (auto cfd : *versions_->GetColumnFamilySet()) {
-      if (!cfd->IsDropped() && !cfd->mem()->IsEmpty()) {
+      if (!cfd->IsDropped() && cfd->initialized() && !cfd->mem()->IsEmpty()) {
         cfd->Ref();
         mutex_.Unlock();
         FlushMemTable(cfd, FlushOptions());
@@ -368,39 +373,6 @@ void DBImpl::PrintStatistics() {
   }
 }
 
-#ifndef ROCKSDB_LITE
-#ifdef ROCKSDB_JEMALLOC
-typedef struct {
-  char* cur;
-  char* end;
-} MallocStatus;
-
-static void GetJemallocStatus(void* mstat_arg, const char* status) {
-  MallocStatus* mstat = reinterpret_cast<MallocStatus*>(mstat_arg);
-  size_t status_len = status ? strlen(status) : 0;
-  size_t buf_size = (size_t)(mstat->end - mstat->cur);
-  if (!status_len || status_len > buf_size) {
-    return;
-  }
-
-  snprintf(mstat->cur, buf_size, "%s", status);
-  mstat->cur += status_len;
-}
-#endif  // ROCKSDB_JEMALLOC
-
-static void DumpMallocStats(std::string* stats) {
-#ifdef ROCKSDB_JEMALLOC
-  MallocStatus mstat;
-  const unsigned int kMallocStatusLen = 1000000;
-  std::unique_ptr<char[]> buf{new char[kMallocStatusLen + 1]};
-  mstat.cur = buf.get();
-  mstat.end = buf.get() + kMallocStatusLen;
-  je_malloc_stats_print(GetJemallocStatus, &mstat, "");
-  stats->append(buf.get());
-#endif  // ROCKSDB_JEMALLOC
-}
-#endif  // !ROCKSDB_LITE
-
 void DBImpl::MaybeDumpStats() {
   mutex_.Lock();
   unsigned int stats_dump_period_sec =
@@ -432,12 +404,17 @@ void DBImpl::MaybeDumpStats() {
       default_cf_internal_stats_->GetStringProperty(
           *db_property_info, DB::Properties::kDBStats, &stats);
       for (auto cfd : *versions_->GetColumnFamilySet()) {
-        cfd->internal_stats()->GetStringProperty(
-            *cf_property_info, DB::Properties::kCFStatsNoFileHistogram, &stats);
+        if (cfd->initialized()) {
+          cfd->internal_stats()->GetStringProperty(
+              *cf_property_info, DB::Properties::kCFStatsNoFileHistogram,
+              &stats);
+        }
       }
       for (auto cfd : *versions_->GetColumnFamilySet()) {
-        cfd->internal_stats()->GetStringProperty(
-            *cf_property_info, DB::Properties::kCFFileHistogram, &stats);
+        if (cfd->initialized()) {
+          cfd->internal_stats()->GetStringProperty(
+              *cf_property_info, DB::Properties::kCFFileHistogram, &stats);
+        }
       }
     }
     ROCKS_LOG_WARN(immutable_db_options_.info_log,
@@ -1234,6 +1211,8 @@ Status DBImpl::CreateColumnFamilyImpl(const ColumnFamilyOptions& cf_options,
         is_snapshot_supported_ = false;
       }
 
+      cfd->set_initialized();
+
       *handle = new ColumnFamilyHandleImpl(cfd, this, &mutex_);
       ROCKS_LOG_INFO(immutable_db_options_.info_log,
                      "Created column family [%s] (ID %u)",
@@ -1735,7 +1714,9 @@ bool DBImpl::GetIntPropertyInternal(ColumnFamilyData* cfd,
 Status DBImpl::ResetStats() {
   InstrumentedMutexLock l(&mutex_);
   for (auto* cfd : *versions_->GetColumnFamilySet()) {
-    cfd->internal_stats()->Clear();
+    if (cfd->initialized()) {
+      cfd->internal_stats()->Clear();
+    }
   }
   return Status::OK();
 }
@@ -1754,6 +1735,9 @@ bool DBImpl::GetAggregatedIntProperty(const Slice& property,
     InstrumentedMutexLock l(&mutex_);
     uint64_t value;
     for (auto* cfd : *versions_->GetColumnFamilySet()) {
+      if (!cfd->initialized()) {
+        continue;
+      }
       if (GetIntPropertyInternal(cfd, *property_info, true, &value)) {
         sum += value;
       } else {
@@ -2346,6 +2330,9 @@ Status DBImpl::WriteOptionsFile(bool need_mutex_lock,
       BuildDBOptions(immutable_db_options_, mutable_db_options_);
   mutex_.Unlock();
 
+  TEST_SYNC_POINT("DBImpl::WriteOptionsFile:1");
+  TEST_SYNC_POINT("DBImpl::WriteOptionsFile:2");
+
   std::string file_name =
       TempOptionsFileName(GetName(), versions_->NewFileNumber());
   Status s =
@@ -2481,7 +2468,7 @@ void DBImpl::EraseThreadStatusDbInfo() const {
 // A global method that can dump out the build version
 void DumpRocksDBBuildVersion(Logger * log) {
 #if !defined(IOS_CROSS_COMPILE)
-  // if we compile with Xcode, we don't run build_detect_vesion, so we don't
+  // if we compile with Xcode, we don't run build_detect_version, so we don't
   // generate util/build_version.cc
   ROCKS_LOG_HEADER(log, "RocksDB version: %d.%d.%d\n", ROCKSDB_MAJOR,
                    ROCKSDB_MINOR, ROCKSDB_PATCH);

db/db_impl.h

@@ -399,12 +399,21 @@ class DBImpl : public DB {
   uint64_t TEST_FindMinPrepLogReferencedByMemTable();
 
   int TEST_BGCompactionsAllowed() const;
+  int TEST_BGFlushesAllowed() const;
 
 #endif  // NDEBUG
 
-  // Return maximum background compaction allowed to be scheduled based on
-  // compaction status.
-  int BGCompactionsAllowed() const;
+  struct BGJobLimits {
+    int max_flushes;
+    int max_compactions;
+  };
+  // Returns maximum background flushes and compactions allowed to be scheduled
+  BGJobLimits GetBGJobLimits() const;
+  // Need a static version that can be called during SanitizeOptions().
+  static BGJobLimits GetBGJobLimits(int max_background_flushes,
+                                    int max_background_compactions,
+                                    int max_background_jobs,
+                                    bool parallelize_compactions);
 
   // move logs pending closing from job_context to the DB queue and
   // schedule a purge
@@ -711,6 +720,9 @@ class DBImpl : public DB {
   //            `num_bytes` going through.
   Status DelayWrite(uint64_t num_bytes, const WriteOptions& write_options);
 
+  Status ThrottleLowPriWritesIfNeeded(const WriteOptions& write_options,
+                                      WriteBatch* my_batch);
+
   Status ScheduleFlushes(WriteContext* context);
 
   Status SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context);
@@ -930,6 +942,8 @@ class DBImpl : public DB {
 
   WriteController write_controller_;
 
+  unique_ptr<RateLimiter> low_pri_write_rate_limiter_;
+
   // Size of the last batch group. In slowdown mode, next write needs to
   // sleep if it uses up the quota.
   uint64_t last_batch_group_size_;
@@ -1184,7 +1198,7 @@ extern DBOptions SanitizeOptions(const std::string& db, const DBOptions& src);
 extern CompressionType GetCompressionFlush(
     const ImmutableCFOptions& ioptions,
     const MutableCFOptions& mutable_cf_options);
- 
+
 // Fix user-supplied options to be reasonable
 template <class T, class V>
 static void ClipToRange(T* ptr, V minvalue, V maxvalue) {

db/db_impl_compaction_flush.cc

@@ -981,22 +981,22 @@ void DBImpl::MaybeScheduleFlushOrCompaction() {
     // DB is being deleted; no more background compactions
     return;
   }
-
-  while (unscheduled_flushes_ > 0 &&
-         bg_flush_scheduled_ < immutable_db_options_.max_background_flushes) {
+  auto bg_job_limits = GetBGJobLimits();
+  bool is_flush_pool_empty =
+    env_->GetBackgroundThreads(Env::Priority::HIGH) == 0;
+  while (!is_flush_pool_empty && unscheduled_flushes_ > 0 &&
+         bg_flush_scheduled_ < bg_job_limits.max_flushes) {
     unscheduled_flushes_--;
     bg_flush_scheduled_++;
     env_->Schedule(&DBImpl::BGWorkFlush, this, Env::Priority::HIGH, this);
   }
 
-  auto bg_compactions_allowed = BGCompactionsAllowed();
-
-  // special case -- if max_background_flushes == 0, then schedule flush on a
-  // compaction thread
-  if (immutable_db_options_.max_background_flushes == 0) {
+  // special case -- if high-pri (flush) thread pool is empty, then schedule
+  // flushes in low-pri (compaction) thread pool.
+  if (is_flush_pool_empty) {
     while (unscheduled_flushes_ > 0 &&
            bg_flush_scheduled_ + bg_compaction_scheduled_ <
-               bg_compactions_allowed) {
+               bg_job_limits.max_flushes) {
       unscheduled_flushes_--;
       bg_flush_scheduled_++;
       env_->Schedule(&DBImpl::BGWorkFlush, this, Env::Priority::LOW, this);
@@ -1014,7 +1014,7 @@ void DBImpl::MaybeScheduleFlushOrCompaction() {
     return;
   }
 
-  while (bg_compaction_scheduled_ < bg_compactions_allowed &&
+  while (bg_compaction_scheduled_ < bg_job_limits.max_compactions &&
          unscheduled_compactions_ > 0) {
     CompactionArg* ca = new CompactionArg;
     ca->db = this;
@@ -1026,13 +1026,35 @@ void DBImpl::MaybeScheduleFlushOrCompaction() {
   }
 }
 
-int DBImpl::BGCompactionsAllowed() const {
+DBImpl::BGJobLimits DBImpl::GetBGJobLimits() const {
   mutex_.AssertHeld();
-  if (write_controller_.NeedSpeedupCompaction()) {
-    return mutable_db_options_.max_background_compactions;
+  return GetBGJobLimits(immutable_db_options_.max_background_flushes,
+                        mutable_db_options_.max_background_compactions,
+                        mutable_db_options_.max_background_jobs,
+                        write_controller_.NeedSpeedupCompaction());
+}
+
+DBImpl::BGJobLimits DBImpl::GetBGJobLimits(int max_background_flushes,
+                                           int max_background_compactions,
+                                           int max_background_jobs,
+                                           bool parallelize_compactions) {
+  BGJobLimits res;
+  if (max_background_flushes == -1 && max_background_compactions == -1) {
+    // for our first stab implementing max_background_jobs, simply allocate a
+    // quarter of the threads to flushes.
+    res.max_flushes = std::max(1, max_background_jobs / 4);
+    res.max_compactions = std::max(1, max_background_jobs - res.max_flushes);
   } else {
-    return mutable_db_options_.base_background_compactions;
+    // compatibility code in case users haven't migrated to max_background_jobs,
+    // which automatically computes flush/compaction limits
+    res.max_flushes = std::max(1, max_background_flushes);
+    res.max_compactions = std::max(1, max_background_compactions);
   }
+  if (!parallelize_compactions) {
+    // throttle background compactions until we deem necessary
+    res.max_compactions = 1;
+  }
+  return res;
 }
 
 void DBImpl::AddToCompactionQueue(ColumnFamilyData* cfd) {
@@ -1154,13 +1176,15 @@ Status DBImpl::BackgroundFlush(bool* made_progress, JobContext* job_context,
   if (cfd != nullptr) {
     const MutableCFOptions mutable_cf_options =
         *cfd->GetLatestMutableCFOptions();
+    auto bg_job_limits = GetBGJobLimits();
     ROCKS_LOG_BUFFER(
         log_buffer,
         "Calling FlushMemTableToOutputFile with column "
-        "family [%s], flush slots available %d, compaction slots allowed %d, "
-        "compaction slots scheduled %d",
-        cfd->GetName().c_str(), immutable_db_options_.max_background_flushes -
-        bg_flush_scheduled_, BGCompactionsAllowed(), bg_compaction_scheduled_);
+        "family [%s], flush slots available %d, compaction slots available %d, "
+        "flush slots scheduled %d, compaction slots scheduled %d",
+        cfd->GetName().c_str(), bg_job_limits.max_flushes,
+        bg_job_limits.max_compactions, bg_flush_scheduled_,
+        bg_compaction_scheduled_);
     status = FlushMemTableToOutputFile(cfd, mutable_cf_options, made_progress,
                                        job_context, log_buffer);
     if (cfd->Unref()) {

db/db_impl_debug.cc

@@ -186,7 +186,12 @@ Status DBImpl::TEST_GetLatestMutableCFOptions(
 
 int DBImpl::TEST_BGCompactionsAllowed() const {
   InstrumentedMutexLock l(&mutex_);
-  return BGCompactionsAllowed();
+  return GetBGJobLimits().max_compactions;
+}
+
+int DBImpl::TEST_BGFlushesAllowed() const {
+  InstrumentedMutexLock l(&mutex_);
+  return GetBGJobLimits().max_flushes;
 }
 
 }  // namespace rocksdb

db/db_impl_open.cc

@@ -16,6 +16,7 @@
 #include "db/builder.h"
 #include "options/options_helper.h"
 #include "rocksdb/wal_filter.h"
+#include "util/rate_limiter.h"
 #include "util/sst_file_manager_impl.h"
 #include "util/sync_point.h"
 
@@ -53,15 +54,13 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src) {
     result.write_buffer_manager.reset(
         new WriteBufferManager(result.db_write_buffer_size));
   }
-  if (result.base_background_compactions == -1) {
-    result.base_background_compactions = result.max_background_compactions;
-  }
-  if (result.base_background_compactions > result.max_background_compactions) {
-    result.base_background_compactions = result.max_background_compactions;
-  }
-  result.env->IncBackgroundThreadsIfNeeded(src.max_background_compactions,
+  auto bg_job_limits = DBImpl::GetBGJobLimits(result.max_background_flushes,
+                                              result.max_background_compactions,
+                                              result.max_background_jobs,
+                                              true /* parallelize_compactions */);
+  result.env->IncBackgroundThreadsIfNeeded(bg_job_limits.max_compactions,
                                            Env::Priority::LOW);
-  result.env->IncBackgroundThreadsIfNeeded(src.max_background_flushes,
+  result.env->IncBackgroundThreadsIfNeeded(bg_job_limits.max_flushes,
                                            Env::Priority::HIGH);
 
   if (result.rate_limiter.get() != nullptr) {
@@ -70,6 +69,15 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src) {
     }
   }
 
+  if (result.delayed_write_rate == 0) {
+    if (result.rate_limiter.get() != nullptr) {
+      result.delayed_write_rate = result.rate_limiter->GetBytesPerSecond();
+    }
+    if (result.delayed_write_rate == 0) {
+      result.delayed_write_rate = 16 * 1024 * 1024;
+    }
+  }
+
   if (result.WAL_ttl_seconds > 0 || result.WAL_size_limit_MB > 0) {
     result.recycle_log_file_num = false;
   }
@@ -890,7 +898,7 @@ Status DB::Open(const Options& options, const std::string& dbname, DB** dbptr) {
   }
   return s;
 }
-  
+
 Status DB::Open(const DBOptions& db_options, const std::string& dbname,
                 const std::vector<ColumnFamilyDescriptor>& column_families,
                 std::vector<ColumnFamilyHandle*>* handles, DB** dbptr) {

db/db_impl_write.cc

@@ -64,13 +64,19 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
     return Status::Corruption("Batch is nullptr!");
   }
 
+  Status status;
+  if (write_options.low_pri) {
+    status = ThrottleLowPriWritesIfNeeded(write_options, my_batch);
+    if (!status.ok()) {
+      return status;
+    }
+  }
+
   if (immutable_db_options_.enable_pipelined_write) {
     return PipelinedWriteImpl(write_options, my_batch, callback, log_used,
                               log_ref, disable_memtable);
   }
 
-  Status status;
-
   PERF_TIMER_GUARD(write_pre_and_post_process_time);
   WriteThread::Writer w(write_options, my_batch, callback, log_ref,
                         disable_memtable);
@@ -89,9 +95,8 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
     if (w.ShouldWriteToMemtable()) {
       ColumnFamilyMemTablesImpl column_family_memtables(
           versions_->GetColumnFamilySet());
-      WriteBatchInternal::SetSequence(w.batch, w.sequence);
       w.status = WriteBatchInternal::InsertInto(
-          &w, &column_family_memtables, &flush_scheduler_,
+          &w, w.sequence, &column_family_memtables, &flush_scheduler_,
           write_options.ignore_missing_column_families, 0 /*log_number*/, this,
           true /*concurrent_memtable_writes*/);
     }
@@ -237,9 +242,8 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
           ColumnFamilyMemTablesImpl column_family_memtables(
               versions_->GetColumnFamilySet());
           assert(w.sequence == current_sequence);
-          WriteBatchInternal::SetSequence(w.batch, w.sequence);
           w.status = WriteBatchInternal::InsertInto(
-              &w, &column_family_memtables, &flush_scheduler_,
+              &w, w.sequence, &column_family_memtables, &flush_scheduler_,
               write_options.ignore_missing_column_families, 0 /*log_number*/,
               this, true /*concurrent_memtable_writes*/);
         }
@@ -382,11 +386,10 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
 
   if (w.state == WriteThread::STATE_PARALLEL_MEMTABLE_WRITER) {
     assert(w.ShouldWriteToMemtable());
-    WriteBatchInternal::SetSequence(w.batch, w.sequence);
     ColumnFamilyMemTablesImpl column_family_memtables(
         versions_->GetColumnFamilySet());
     w.status = WriteBatchInternal::InsertInto(
-        &w, &column_family_memtables, &flush_scheduler_,
+        &w, w.sequence, &column_family_memtables, &flush_scheduler_,
         write_options.ignore_missing_column_families, 0 /*log_number*/, this,
         true /*concurrent_memtable_writes*/);
     if (write_thread_.CompleteParallelMemTableWriter(&w)) {
@@ -743,6 +746,34 @@ Status DBImpl::DelayWrite(uint64_t num_bytes,
   return bg_error_;
 }
 
+Status DBImpl::ThrottleLowPriWritesIfNeeded(const WriteOptions& write_options,
+                                            WriteBatch* my_batch) {
+  assert(write_options.low_pri);
+  // This is called outside the DB mutex. Although it is safe to make the call,
+  // the consistency condition is not guaranteed to hold. It's OK to live with
+  // it in this case.
+  // If we need to speed compaction, it means the compaction is left behind
+  // and we start to limit low pri writes to a limit.
+  if (write_controller_.NeedSpeedupCompaction()) {
+    if (allow_2pc() && (my_batch->HasCommit() || my_batch->HasRollback())) {
+      // For 2PC, we only rate limit prepare, not commit.
+      return Status::OK();
+    }
+    if (write_options.no_slowdown) {
+      return Status::Incomplete();
+    } else {
+      assert(my_batch != nullptr);
+      // Rate limit those writes. The reason that we don't completely wait
+      // is that in case the write is heavy, low pri writes may never have
+      // a chance to run. Now we guarantee we are still slowly making
+      // progress.
+      write_controller_.low_pri_rate_limiter()->Request(my_batch->GetDataSize(),
+                                                        Env::IO_HIGH, nullptr);
+    }
+  }
+  return Status::OK();
+}
+
 Status DBImpl::ScheduleFlushes(WriteContext* context) {
   ColumnFamilyData* cfd;
   while ((cfd = flush_scheduler_.TakeNextColumnFamily()) != nullptr) {

db/db_iter.cc

@@ -560,7 +560,7 @@ void DBIter::MergeValuesNewToOld() {
       const Slice val = iter_->value();
       s = MergeHelper::TimedFullMerge(
           merge_operator_, ikey.user_key, &val, merge_context_.GetOperands(),
-          &saved_value_, logger_, statistics_, env_, &pinned_value_);
+          &saved_value_, logger_, statistics_, env_, &pinned_value_, true);
       if (!s.ok()) {
         status_ = s;
       }
@@ -585,7 +585,7 @@ void DBIter::MergeValuesNewToOld() {
   s = MergeHelper::TimedFullMerge(merge_operator_, saved_key_.GetUserKey(),
                                   nullptr, merge_context_.GetOperands(),
                                   &saved_value_, logger_, statistics_, env_,
-                                  &pinned_value_);
+                                  &pinned_value_, true);
   if (!s.ok()) {
     status_ = s;
   }
@@ -804,13 +804,13 @@ bool DBIter::FindValueForCurrentKey() {
         s = MergeHelper::TimedFullMerge(
             merge_operator_, saved_key_.GetUserKey(), nullptr,
             merge_context_.GetOperands(), &saved_value_, logger_, statistics_,
-            env_, &pinned_value_);
+            env_, &pinned_value_, true);
       } else {
         assert(last_not_merge_type == kTypeValue);
         s = MergeHelper::TimedFullMerge(
             merge_operator_, saved_key_.GetUserKey(), &pinned_value_,
             merge_context_.GetOperands(), &saved_value_, logger_, statistics_,
-            env_, &pinned_value_);
+            env_, &pinned_value_, true);
       }
       break;
     case kTypeValue:
@@ -882,7 +882,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
     s = MergeHelper::TimedFullMerge(merge_operator_, saved_key_.GetUserKey(),
                                     nullptr, merge_context_.GetOperands(),
                                     &saved_value_, logger_, statistics_, env_,
-                                    &pinned_value_);
+                                    &pinned_value_, true);
     // Make iter_ valid and point to saved_key_
     if (!iter_->Valid() ||
         !user_comparator_->Equal(ikey.user_key, saved_key_.GetUserKey())) {
@@ -900,7 +900,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
   s = MergeHelper::TimedFullMerge(merge_operator_, saved_key_.GetUserKey(),
                                   &val, merge_context_.GetOperands(),
                                   &saved_value_, logger_, statistics_, env_,
-                                  &pinned_value_);
+                                  &pinned_value_, true);
   valid_ = true;
   if (!s.ok()) {
     status_ = s;

db/db_iter_test.cc

@@ -352,11 +352,11 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
     SetPerfLevel(kEnableCount);
     ASSERT_TRUE(GetPerfLevel() == kEnableCount);
 
-    perf_context.Reset();
+    get_perf_context()->Reset();
     db_iter->SeekToLast();
 
     ASSERT_TRUE(db_iter->Valid());
-    ASSERT_EQ(static_cast<int>(perf_context.internal_key_skipped_count), 7);
+    ASSERT_EQ(static_cast<int>(get_perf_context()->internal_key_skipped_count), 7);
     ASSERT_EQ(db_iter->key().ToString(), "b");
 
     SetPerfLevel(kDisable);
@@ -471,11 +471,11 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
     SetPerfLevel(kEnableCount);
     ASSERT_TRUE(GetPerfLevel() == kEnableCount);
 
-    perf_context.Reset();
+    get_perf_context()->Reset();
     db_iter->SeekToLast();
 
     ASSERT_TRUE(db_iter->Valid());
-    ASSERT_EQ(static_cast<int>(perf_context.internal_delete_skipped_count), 1);
+    ASSERT_EQ(static_cast<int>(get_perf_context()->internal_delete_skipped_count), 1);
     ASSERT_EQ(db_iter->key().ToString(), "b");
 
     SetPerfLevel(kDisable);

db/db_iterator_test.cc

@@ -972,11 +972,11 @@ TEST_F(DBIteratorTest, DBIteratorBoundTest) {
     ASSERT_TRUE(iter->Valid());
     ASSERT_EQ(iter->key().compare(("b1")), 0);
 
-    perf_context.Reset();
+    get_perf_context()->Reset();
     iter->Next();
 
     ASSERT_TRUE(iter->Valid());
-    ASSERT_EQ(static_cast<int>(perf_context.internal_delete_skipped_count), 2);
+    ASSERT_EQ(static_cast<int>(get_perf_context()->internal_delete_skipped_count), 2);
 
     // now testing with iterate_bound
     Slice prefix("c");
@@ -984,7 +984,7 @@ TEST_F(DBIteratorTest, DBIteratorBoundTest) {
 
     iter.reset(db_->NewIterator(ro));
 
-    perf_context.Reset();
+    get_perf_context()->Reset();
 
     iter->Seek("b");
     ASSERT_TRUE(iter->Valid());
@@ -999,7 +999,7 @@ TEST_F(DBIteratorTest, DBIteratorBoundTest) {
     // even though the key is deleted
     // hence internal_delete_skipped_count should be 0
     ASSERT_TRUE(!iter->Valid());
-    ASSERT_EQ(static_cast<int>(perf_context.internal_delete_skipped_count), 0);
+    ASSERT_EQ(static_cast<int>(get_perf_context()->internal_delete_skipped_count), 0);
   }
 }
 
@@ -1886,7 +1886,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipRecentDuplicatesTest) {
 #endif
 
   // Seek iterator to a smaller key.
-  perf_context.Reset();
+  get_perf_context()->Reset();
   iter->Seek("a");
   ASSERT_TRUE(iter->Valid());
   EXPECT_EQ("b", iter->key().ToString());
@@ -1894,17 +1894,17 @@ TEST_F(DBIteratorTest, DBIteratorSkipRecentDuplicatesTest) {
 
   // Check that the seek didn't do too much work.
   // Checks are not tight, just make sure that everything is well below 100.
-  EXPECT_LT(perf_context.internal_key_skipped_count, 4);
-  EXPECT_LT(perf_context.internal_recent_skipped_count, 8);
-  EXPECT_LT(perf_context.seek_on_memtable_count, 10);
-  EXPECT_LT(perf_context.next_on_memtable_count, 10);
-  EXPECT_LT(perf_context.prev_on_memtable_count, 10);
+  EXPECT_LT(get_perf_context()->internal_key_skipped_count, 4);
+  EXPECT_LT(get_perf_context()->internal_recent_skipped_count, 8);
+  EXPECT_LT(get_perf_context()->seek_on_memtable_count, 10);
+  EXPECT_LT(get_perf_context()->next_on_memtable_count, 10);
+  EXPECT_LT(get_perf_context()->prev_on_memtable_count, 10);
 
   // Check that iterator did something like what we expect.
-  EXPECT_EQ(perf_context.internal_delete_skipped_count, 0);
-  EXPECT_EQ(perf_context.internal_merge_count, 0);
-  EXPECT_GE(perf_context.internal_recent_skipped_count, 2);
-  EXPECT_GE(perf_context.seek_on_memtable_count, 2);
+  EXPECT_EQ(get_perf_context()->internal_delete_skipped_count, 0);
+  EXPECT_EQ(get_perf_context()->internal_merge_count, 0);
+  EXPECT_GE(get_perf_context()->internal_recent_skipped_count, 2);
+  EXPECT_GE(get_perf_context()->seek_on_memtable_count, 2);
   EXPECT_EQ(1, options.statistics->getTickerCount(
                  NUMBER_OF_RESEEKS_IN_ITERATION));
 }

db/db_memtable_test.cc

@@ -21,7 +21,7 @@ class DBMemTableTest : public DBTestBase {
 
 class MockMemTableRep : public MemTableRep {
  public:
-  explicit MockMemTableRep(MemTableAllocator* allocator, MemTableRep* rep)
+  explicit MockMemTableRep(Allocator* allocator, MemTableRep* rep)
       : MemTableRep(allocator), rep_(rep), num_insert_with_hint_(0) {}
 
   virtual KeyHandle Allocate(const size_t len, char** buf) override {
@@ -72,7 +72,7 @@ class MockMemTableRep : public MemTableRep {
 class MockMemTableRepFactory : public MemTableRepFactory {
  public:
   virtual MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator& cmp,
-                                         MemTableAllocator* allocator,
+                                         Allocator* allocator,
                                          const SliceTransform* transform,
                                          Logger* logger) override {
     SkipListFactory factory;
@@ -82,14 +82,27 @@ class MockMemTableRepFactory : public MemTableRepFactory {
     return mock_rep_;
   }
 
+  virtual MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator& cmp,
+                                         Allocator* allocator,
+                                         const SliceTransform* transform,
+                                         Logger* logger,
+                                         uint32_t column_family_id) override {
+    last_column_family_id_ = column_family_id;
+    return CreateMemTableRep(cmp, allocator, transform, logger);
+  }
+
   virtual const char* Name() const override { return "MockMemTableRepFactory"; }
 
   MockMemTableRep* rep() { return mock_rep_; }
 
   bool IsInsertConcurrentlySupported() const override { return false; }
 
+  uint32_t GetLastColumnFamilyId() { return last_column_family_id_; }
+
  private:
   MockMemTableRep* mock_rep_;
+  // workaround since there's no port::kMaxUint32 yet.
+  uint32_t last_column_family_id_ = static_cast<uint32_t>(-1);
 };
 
 class TestPrefixExtractor : public SliceTransform {
@@ -155,6 +168,24 @@ TEST_F(DBMemTableTest, InsertWithHint) {
   ASSERT_EQ("vvv", Get("whitelisted"));
 }
 
+TEST_F(DBMemTableTest, ColumnFamilyId) {
+  // Verifies MemTableRepFactory is told the right column family id.
+  Options options;
+  options.allow_concurrent_memtable_write = false;
+  options.create_if_missing = true;
+  options.memtable_factory.reset(new MockMemTableRepFactory());
+  DestroyAndReopen(options);
+  CreateAndReopenWithCF({"pikachu"}, options);
+
+  for (int cf = 0; cf < 2; ++cf) {
+    ASSERT_OK(Put(cf, "key", "val"));
+    ASSERT_OK(Flush(cf));
+    ASSERT_EQ(
+        cf, static_cast<MockMemTableRepFactory*>(options.memtable_factory.get())
+                ->GetLastColumnFamilyId());
+  }
+}
+
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/db_merge_operator_test.cc

@@ -18,28 +18,6 @@ class DBMergeOperatorTest : public DBTestBase {
   DBMergeOperatorTest() : DBTestBase("/db_merge_operator_test") {}
 };
 
-// A test merge operator mimics put but also fails if one of merge operands is
-// "corrupted".
-class TestPutOperator : public MergeOperator {
- public:
-  virtual bool FullMergeV2(const MergeOperationInput& merge_in,
-                           MergeOperationOutput* merge_out) const override {
-    if (merge_in.existing_value != nullptr &&
-        *(merge_in.existing_value) == "corrupted") {
-      return false;
-    }
-    for (auto value : merge_in.operand_list) {
-      if (value == "corrupted") {
-        return false;
-      }
-    }
-    merge_out->existing_operand = merge_in.operand_list.back();
-    return true;
-  }
-
-  virtual const char* Name() const override { return "TestPutOperator"; }
-};
-
 TEST_F(DBMergeOperatorTest, MergeErrorOnRead) {
   Options options;
   options.create_if_missing = true;

db/db_options_test.cc

@@ -17,6 +17,7 @@
 #include "port/stack_trace.h"
 #include "rocksdb/cache.h"
 #include "rocksdb/convenience.h"
+#include "rocksdb/rate_limiter.h"
 #include "util/random.h"
 #include "util/sync_point.h"
 #include "util/testutil.h"
@@ -252,18 +253,44 @@ TEST_F(DBOptionsTest, SetOptionsMayTriggerCompaction) {
 TEST_F(DBOptionsTest, SetBackgroundCompactionThreads) {
   Options options;
   options.create_if_missing = true;
-  options.base_background_compactions = 1;  // default value
   options.max_background_compactions = 1;   // default value
   options.env = env_;
   Reopen(options);
   ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
-  ASSERT_OK(dbfull()->SetDBOptions({{"base_background_compactions", "2"},
-                                    {"max_background_compactions", "3"}}));
-  ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed());
+  ASSERT_OK(dbfull()->SetDBOptions({{"max_background_compactions", "3"}}));
+  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
   auto stop_token = dbfull()->TEST_write_controler().GetStopToken();
   ASSERT_EQ(3, dbfull()->TEST_BGCompactionsAllowed());
 }
 
+TEST_F(DBOptionsTest, SetBackgroundJobs) {
+  Options options;
+  options.create_if_missing = true;
+  options.max_background_jobs = 8;
+  options.env = env_;
+  Reopen(options);
+
+  for (int i = 0; i < 2; ++i) {
+    if (i > 0) {
+      options.max_background_jobs = 12;
+      ASSERT_OK(dbfull()->SetDBOptions(
+          {{"max_background_jobs",
+            std::to_string(options.max_background_jobs)}}));
+    }
+
+    ASSERT_EQ(options.max_background_jobs / 4,
+              dbfull()->TEST_BGFlushesAllowed());
+    ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
+
+    auto stop_token = dbfull()->TEST_write_controler().GetStopToken();
+
+    ASSERT_EQ(options.max_background_jobs / 4,
+              dbfull()->TEST_BGFlushesAllowed());
+    ASSERT_EQ(3 * options.max_background_jobs / 4,
+              dbfull()->TEST_BGCompactionsAllowed());
+  }
+}
+
 TEST_F(DBOptionsTest, AvoidFlushDuringShutdown) {
   Options options;
   options.create_if_missing = true;
@@ -402,6 +429,17 @@ TEST_F(DBOptionsTest, MaxOpenFilesChange) {
   Close();
 }
 
+TEST_F(DBOptionsTest, SanitizeDelayedWriteRate) {
+  Options options;
+  options.delayed_write_rate = 0;
+  Reopen(options);
+  ASSERT_EQ(16 * 1024 * 1024, dbfull()->GetDBOptions().delayed_write_rate);
+
+  options.rate_limiter.reset(NewGenericRateLimiter(31 * 1024 * 1024));
+  Reopen(options);
+  ASSERT_EQ(31 * 1024 * 1024, dbfull()->GetDBOptions().delayed_write_rate);
+}
+
 #endif  // ROCKSDB_LITE
 
 }  // namespace rocksdb

db/db_properties_test.cc

@@ -533,9 +533,9 @@ TEST_F(DBPropertiesTest, NumImmutableMemTable) {
     ASSERT_TRUE(dbfull()->GetProperty(
         handles_[1], "rocksdb.num-entries-active-mem-table", &num));
     ASSERT_EQ(num, "1");
-    perf_context.Reset();
+    get_perf_context()->Reset();
     Get(1, "k1");
-    ASSERT_EQ(1, static_cast<int>(perf_context.get_from_memtable_count));
+    ASSERT_EQ(1, static_cast<int>(get_perf_context()->get_from_memtable_count));
 
     ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k2", big_value));
     ASSERT_TRUE(dbfull()->GetProperty(handles_[1],
@@ -548,12 +548,12 @@ TEST_F(DBPropertiesTest, NumImmutableMemTable) {
         handles_[1], "rocksdb.num-entries-imm-mem-tables", &num));
     ASSERT_EQ(num, "1");
 
-    perf_context.Reset();
+    get_perf_context()->Reset();
     Get(1, "k1");
-    ASSERT_EQ(2, static_cast<int>(perf_context.get_from_memtable_count));
-    perf_context.Reset();
+    ASSERT_EQ(2, static_cast<int>(get_perf_context()->get_from_memtable_count));
+    get_perf_context()->Reset();
     Get(1, "k2");
-    ASSERT_EQ(1, static_cast<int>(perf_context.get_from_memtable_count));
+    ASSERT_EQ(1, static_cast<int>(get_perf_context()->get_from_memtable_count));
 
     ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k3", big_value));
     ASSERT_TRUE(dbfull()->GetProperty(
@@ -567,15 +567,15 @@ TEST_F(DBPropertiesTest, NumImmutableMemTable) {
     ASSERT_TRUE(dbfull()->GetProperty(
         handles_[1], "rocksdb.num-entries-imm-mem-tables", &num));
     ASSERT_EQ(num, "2");
-    perf_context.Reset();
+    get_perf_context()->Reset();
     Get(1, "k2");
-    ASSERT_EQ(2, static_cast<int>(perf_context.get_from_memtable_count));
-    perf_context.Reset();
+    ASSERT_EQ(2, static_cast<int>(get_perf_context()->get_from_memtable_count));
+    get_perf_context()->Reset();
     Get(1, "k3");
-    ASSERT_EQ(1, static_cast<int>(perf_context.get_from_memtable_count));
-    perf_context.Reset();
+    ASSERT_EQ(1, static_cast<int>(get_perf_context()->get_from_memtable_count));
+    get_perf_context()->Reset();
     Get(1, "k1");
-    ASSERT_EQ(3, static_cast<int>(perf_context.get_from_memtable_count));
+    ASSERT_EQ(3, static_cast<int>(get_perf_context()->get_from_memtable_count));
 
     ASSERT_OK(Flush(1));
     ASSERT_TRUE(dbfull()->GetProperty(handles_[1],
@@ -670,7 +670,7 @@ TEST_F(DBPropertiesTest, DISABLED_GetProperty) {
   ASSERT_EQ(num, "0");
   ASSERT_TRUE(dbfull()->GetProperty("rocksdb.estimate-num-keys", &num));
   ASSERT_EQ(num, "1");
-  perf_context.Reset();
+  get_perf_context()->Reset();
 
   ASSERT_OK(dbfull()->Put(writeOpt, "k2", big_value));
   ASSERT_TRUE(dbfull()->GetProperty("rocksdb.num-immutable-mem-table", &num));
@@ -913,7 +913,7 @@ TEST_F(DBPropertiesTest, EstimatePendingCompBytes) {
   Flush();
   ASSERT_TRUE(dbfull()->GetIntProperty(
       "rocksdb.estimate-pending-compaction-bytes", &int_num));
-  ASSERT_EQ(int_num, 0U);
+  ASSERT_GT(int_num, 0U);
 
   ASSERT_OK(dbfull()->Put(writeOpt, "k3", big_value));
   Flush();
@@ -940,7 +940,6 @@ TEST_F(DBPropertiesTest, EstimateCompressionRatio) {
   Options options = CurrentOptions();
   options.compression_per_level = {kNoCompression, kSnappyCompression};
   options.disable_auto_compactions = true;
-  options.max_background_flushes = 0;
   options.num_levels = 2;
   Reopen(options);
 
@@ -1068,7 +1067,6 @@ class CountingDeleteTabPropCollectorFactory
 TEST_F(DBPropertiesTest, GetUserDefinedTableProperties) {
   Options options = CurrentOptions();
   options.level0_file_num_compaction_trigger = (1 << 30);
-  options.max_background_flushes = 0;
   options.table_properties_collector_factories.resize(1);
   std::shared_ptr<CountingUserTblPropCollectorFactory> collector_factory =
       std::make_shared<CountingUserTblPropCollectorFactory>(0);
@@ -1109,7 +1107,6 @@ TEST_F(DBPropertiesTest, GetUserDefinedTableProperties) {
 TEST_F(DBPropertiesTest, UserDefinedTablePropertiesContext) {
   Options options = CurrentOptions();
   options.level0_file_num_compaction_trigger = 3;
-  options.max_background_flushes = 0;
   options.table_properties_collector_factories.resize(1);
   std::shared_ptr<CountingUserTblPropCollectorFactory> collector_factory =
       std::make_shared<CountingUserTblPropCollectorFactory>(1);
@@ -1231,7 +1228,7 @@ TEST_F(DBPropertiesTest, TablePropertiesNeedCompactTest) {
 
   {
     SetPerfLevel(kEnableCount);
-    perf_context.Reset();
+    get_perf_context()->Reset();
     int c = 0;
     std::unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions()));
     iter->Seek(Key(kMaxKey - 100));
@@ -1239,8 +1236,8 @@ TEST_F(DBPropertiesTest, TablePropertiesNeedCompactTest) {
       iter->Next();
     }
     ASSERT_EQ(c, 0);
-    ASSERT_LT(perf_context.internal_delete_skipped_count, 30u);
-    ASSERT_LT(perf_context.internal_key_skipped_count, 30u);
+    ASSERT_LT(get_perf_context()->internal_delete_skipped_count, 30u);
+    ASSERT_LT(get_perf_context()->internal_key_skipped_count, 30u);
     SetPerfLevel(kDisable);
   }
 }
@@ -1287,16 +1284,16 @@ TEST_F(DBPropertiesTest, NeedCompactHintPersistentTest) {
   ASSERT_EQ(NumTableFilesAtLevel(0), 0);
   {
     SetPerfLevel(kEnableCount);
-    perf_context.Reset();
+    get_perf_context()->Reset();
     int c = 0;
     std::unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions()));
     for (iter->Seek(Key(0)); iter->Valid(); iter->Next()) {
       c++;
     }
     ASSERT_EQ(c, 2);
-    ASSERT_EQ(perf_context.internal_delete_skipped_count, 0);
+    ASSERT_EQ(get_perf_context()->internal_delete_skipped_count, 0);
     // We iterate every key twice. Is it a bug?
-    ASSERT_LE(perf_context.internal_key_skipped_count, 2);
+    ASSERT_LE(get_perf_context()->internal_key_skipped_count, 2);
     SetPerfLevel(kDisable);
   }
 }

db/db_range_del_test.cc

@@ -789,6 +789,7 @@ TEST_F(DBRangeDelTest, IteratorIgnoresRangeDeletions) {
   db_->ReleaseSnapshot(snapshot);
 }
 
+#ifndef ROCKSDB_UBSAN_RUN
 TEST_F(DBRangeDelTest, TailingIteratorRangeTombstoneUnsupported) {
   db_->Put(WriteOptions(), "key", "val");
   // snapshot prevents key from being deleted during flush
@@ -815,6 +816,8 @@ TEST_F(DBRangeDelTest, TailingIteratorRangeTombstoneUnsupported) {
   }
   db_->ReleaseSnapshot(snapshot);
 }
+#endif  // !ROCKSDB_UBSAN_RUN
+
 #endif  // ROCKSDB_LITE
 
 }  // namespace rocksdb

db/db_test.cc

@@ -60,7 +60,6 @@
 #include "util/file_reader_writer.h"
 #include "util/filename.h"
 #include "util/hash.h"
-#include "util/logging.h"
 #include "util/mutexlock.h"
 #include "util/rate_limiter.h"
 #include "util/string_util.h"
@@ -1439,7 +1438,6 @@ TEST_F(DBTest, UnremovableSingleDelete) {
 #ifndef ROCKSDB_LITE
 TEST_F(DBTest, DeletionMarkers1) {
   Options options = CurrentOptions();
-  options.max_background_flushes = 0;
   CreateAndReopenWithCF({"pikachu"}, options);
   Put(1, "foo", "v1");
   ASSERT_OK(Flush(1));
@@ -3073,9 +3071,9 @@ TEST_F(DBTest, DynamicMemtableOptions) {
                  Env::Priority::LOW);
   // Start from scratch and disable compaction/flush. Flush can only happen
   // during compaction but trigger is pretty high
-  options.max_background_flushes = 0;
   options.disable_auto_compactions = true;
   DestroyAndReopen(options);
+  env_->SetBackgroundThreads(0, Env::HIGH);
 
   // Put until writes are stopped, bounded by 256 puts. We should see stop at
   // ~128KB
@@ -3352,7 +3350,6 @@ TEST_P(DBTestWithParam, ThreadStatusSingleCompaction) {
 
 TEST_P(DBTestWithParam, PreShutdownManualCompaction) {
   Options options = CurrentOptions();
-  options.max_background_flushes = 0;
   options.max_subcompactions = max_subcompactions_;
   CreateAndReopenWithCF({"pikachu"}, options);
 
@@ -3391,7 +3388,6 @@ TEST_P(DBTestWithParam, PreShutdownManualCompaction) {
 
     if (iter == 0) {
       options = CurrentOptions();
-      options.max_background_flushes = 0;
       options.num_levels = 3;
       options.create_if_missing = true;
       DestroyAndReopen(options);
@@ -3402,7 +3398,6 @@ TEST_P(DBTestWithParam, PreShutdownManualCompaction) {
 
 TEST_F(DBTest, PreShutdownFlush) {
   Options options = CurrentOptions();
-  options.max_background_flushes = 0;
   CreateAndReopenWithCF({"pikachu"}, options);
   ASSERT_OK(Put(1, "key", "value"));
   CancelAllBackgroundWork(db_);
@@ -3694,19 +3689,14 @@ TEST_F(DBTest, DynamicLevelCompressionPerLevel2) {
   Random rnd(301);
   Options options;
   options.create_if_missing = true;
-  options.db_write_buffer_size = 6000;
-  options.write_buffer_size = 6000;
+  options.db_write_buffer_size = 6000000;
+  options.write_buffer_size = 600000;
   options.max_write_buffer_number = 2;
   options.level0_file_num_compaction_trigger = 2;
   options.level0_slowdown_writes_trigger = 2;
   options.level0_stop_writes_trigger = 2;
   options.soft_pending_compaction_bytes_limit = 1024 * 1024;
-
-  // Use file size to distinguish levels
-  // L1: 10, L2: 20, L3 40, L4 80
-  // L0 is less than 30
-  options.target_file_size_base = 10;
-  options.target_file_size_multiplier = 2;
+  options.target_file_size_base = 20;
 
   options.level_compaction_dynamic_level_bytes = true;
   options.max_bytes_for_level_base = 200;
@@ -3743,10 +3733,11 @@ TEST_F(DBTest, DynamicLevelCompressionPerLevel2) {
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 
   for (int i = 0; i < 100; i++) {
-    ASSERT_OK(Put(Key(keys[i]), RandomString(&rnd, 200)));
-
-    if (i % 25 == 0) {
-      dbfull()->TEST_WaitForFlushMemTable();
+    std::string value = RandomString(&rnd, 200);
+    ASSERT_OK(Put(Key(keys[i]), value));
+    if (i % 25 == 24) {
+      Flush();
+      dbfull()->TEST_WaitForCompact();
     }
   }
 
@@ -3787,7 +3778,8 @@ TEST_F(DBTest, DynamicLevelCompressionPerLevel2) {
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 
   for (int i = 101; i < 500; i++) {
-    ASSERT_OK(Put(Key(keys[i]), RandomString(&rnd, 200)));
+    std::string value = RandomString(&rnd, 200);
+    ASSERT_OK(Put(Key(keys[i]), value));
     if (i % 100 == 99) {
       Flush();
       dbfull()->TEST_WaitForCompact();
@@ -5161,7 +5153,6 @@ TEST_F(DBTest, PauseBackgroundWorkTest) {
   // now it's done
   ASSERT_TRUE(done.load());
 }
-
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/db_test2.cc

@@ -165,20 +165,48 @@ TEST_F(DBTest2, MaxSuccessiveMergesChangeWithDBRecovery) {
 #ifndef ROCKSDB_LITE
 class DBTestSharedWriteBufferAcrossCFs
     : public DBTestBase,
-      public testing::WithParamInterface<bool> {
+      public testing::WithParamInterface<std::tuple<bool, bool>> {
  public:
   DBTestSharedWriteBufferAcrossCFs()
       : DBTestBase("/db_test_shared_write_buffer") {}
-  void SetUp() override { use_old_interface_ = GetParam(); }
+  void SetUp() override {
+    use_old_interface_ = std::get<0>(GetParam());
+    cost_cache_ = std::get<1>(GetParam());
+  }
   bool use_old_interface_;
+  bool cost_cache_;
 };
 
 TEST_P(DBTestSharedWriteBufferAcrossCFs, SharedWriteBufferAcrossCFs) {
   Options options = CurrentOptions();
+  options.arena_block_size = 4096;
+
+  // Avoid undeterministic value by malloc_usable_size();
+  // Force arena block size to 1
+  rocksdb::SyncPoint::GetInstance()->SetCallBack(
+      "Arena::Arena:0", [&](void* arg) {
+        size_t* block_size = static_cast<size_t*>(arg);
+        *block_size = 1;
+      });
+
+  rocksdb::SyncPoint::GetInstance()->SetCallBack(
+      "Arena::AllocateNewBlock:0", [&](void* arg) {
+        std::pair<size_t*, size_t*>* pair =
+            static_cast<std::pair<size_t*, size_t*>*>(arg);
+        *std::get<0>(*pair) = *std::get<1>(*pair);
+      });
+  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+
+  // The total soft write buffer size is about 105000
+  std::shared_ptr<Cache> cache = NewLRUCache(4 * 1024 * 1024, 2);
+  ASSERT_LT(cache->GetUsage(), 1024 * 1024);
+
   if (use_old_interface_) {
-    options.db_write_buffer_size = 100000;  // this is the real limit
+    options.db_write_buffer_size = 120000;  // this is the real limit
+  } else if (!cost_cache_) {
+    options.write_buffer_manager.reset(new WriteBufferManager(114285));
   } else {
-    options.write_buffer_manager.reset(new WriteBufferManager(100000));
+    options.write_buffer_manager.reset(new WriteBufferManager(114285, cache));
   }
   options.write_buffer_size = 500000;  // this is never hit
   CreateAndReopenWithCF({"pikachu", "dobrynia", "nikitich"}, options);
@@ -186,6 +214,13 @@ TEST_P(DBTestSharedWriteBufferAcrossCFs, SharedWriteBufferAcrossCFs) {
   WriteOptions wo;
   wo.disableWAL = true;
 
+  std::function<void()> wait_flush = [&]() {
+    dbfull()->TEST_WaitForFlushMemTable(handles_[0]);
+    dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
+    dbfull()->TEST_WaitForFlushMemTable(handles_[2]);
+    dbfull()->TEST_WaitForFlushMemTable(handles_[3]);
+  };
+
   // Create some data and flush "default" and "nikitich" so that they
   // are newer CFs created.
   ASSERT_OK(Put(3, Key(1), DummyString(1), wo));
@@ -199,13 +234,20 @@ TEST_P(DBTestSharedWriteBufferAcrossCFs, SharedWriteBufferAcrossCFs) {
             static_cast<uint64_t>(1));
 
   ASSERT_OK(Put(3, Key(1), DummyString(30000), wo));
+  if (cost_cache_) {
+    ASSERT_GE(cache->GetUsage(), 1024 * 1024);
+    ASSERT_LE(cache->GetUsage(), 2 * 1024 * 1024);
+  }
+  wait_flush();
   ASSERT_OK(Put(0, Key(1), DummyString(60000), wo));
+  if (cost_cache_) {
+    ASSERT_GE(cache->GetUsage(), 1024 * 1024);
+    ASSERT_LE(cache->GetUsage(), 2 * 1024 * 1024);
+  }
+  wait_flush();
   ASSERT_OK(Put(2, Key(1), DummyString(1), wo));
   // No flush should trigger
-  dbfull()->TEST_WaitForFlushMemTable(handles_[0]);
-  dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
-  dbfull()->TEST_WaitForFlushMemTable(handles_[2]);
-  dbfull()->TEST_WaitForFlushMemTable(handles_[3]);
+  wait_flush();
   {
     ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"),
               static_cast<uint64_t>(1));
@@ -219,11 +261,9 @@ TEST_P(DBTestSharedWriteBufferAcrossCFs, SharedWriteBufferAcrossCFs) {
 
   // Trigger a flush. Flushing "nikitich".
   ASSERT_OK(Put(3, Key(2), DummyString(30000), wo));
+  wait_flush();
   ASSERT_OK(Put(0, Key(1), DummyString(1), wo));
-  dbfull()->TEST_WaitForFlushMemTable(handles_[0]);
-  dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
-  dbfull()->TEST_WaitForFlushMemTable(handles_[2]);
-  dbfull()->TEST_WaitForFlushMemTable(handles_[3]);
+  wait_flush();
   {
     ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"),
               static_cast<uint64_t>(1));
@@ -237,12 +277,11 @@ TEST_P(DBTestSharedWriteBufferAcrossCFs, SharedWriteBufferAcrossCFs) {
 
   // Without hitting the threshold, no flush should trigger.
   ASSERT_OK(Put(2, Key(1), DummyString(30000), wo));
+  wait_flush();
   ASSERT_OK(Put(2, Key(1), DummyString(1), wo));
+  wait_flush();
   ASSERT_OK(Put(2, Key(1), DummyString(1), wo));
-  dbfull()->TEST_WaitForFlushMemTable(handles_[0]);
-  dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
-  dbfull()->TEST_WaitForFlushMemTable(handles_[2]);
-  dbfull()->TEST_WaitForFlushMemTable(handles_[3]);
+  wait_flush();
   {
     ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"),
               static_cast<uint64_t>(1));
@@ -257,14 +296,15 @@ TEST_P(DBTestSharedWriteBufferAcrossCFs, SharedWriteBufferAcrossCFs) {
   // Hit the write buffer limit again. "default"
   // will have been flushed.
   ASSERT_OK(Put(2, Key(2), DummyString(10000), wo));
+  wait_flush();
   ASSERT_OK(Put(3, Key(1), DummyString(1), wo));
+  wait_flush();
   ASSERT_OK(Put(0, Key(1), DummyString(1), wo));
+  wait_flush();
   ASSERT_OK(Put(0, Key(1), DummyString(1), wo));
+  wait_flush();
   ASSERT_OK(Put(0, Key(1), DummyString(1), wo));
-  dbfull()->TEST_WaitForFlushMemTable(handles_[0]);
-  dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
-  dbfull()->TEST_WaitForFlushMemTable(handles_[2]);
-  dbfull()->TEST_WaitForFlushMemTable(handles_[3]);
+  wait_flush();
   {
     ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"),
               static_cast<uint64_t>(2));
@@ -279,13 +319,14 @@ TEST_P(DBTestSharedWriteBufferAcrossCFs, SharedWriteBufferAcrossCFs) {
   // Trigger another flush. This time "dobrynia". "pikachu" should not
   // be flushed, althrough it was never flushed.
   ASSERT_OK(Put(1, Key(1), DummyString(1), wo));
+  wait_flush();
   ASSERT_OK(Put(2, Key(1), DummyString(80000), wo));
+  wait_flush();
   ASSERT_OK(Put(1, Key(1), DummyString(1), wo));
+  wait_flush();
   ASSERT_OK(Put(2, Key(1), DummyString(1), wo));
-  dbfull()->TEST_WaitForFlushMemTable(handles_[0]);
-  dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
-  dbfull()->TEST_WaitForFlushMemTable(handles_[2]);
-  dbfull()->TEST_WaitForFlushMemTable(handles_[3]);
+  wait_flush();
+
   {
     ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"),
               static_cast<uint64_t>(2));
@@ -296,16 +337,45 @@ TEST_P(DBTestSharedWriteBufferAcrossCFs, SharedWriteBufferAcrossCFs) {
     ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "nikitich"),
               static_cast<uint64_t>(2));
   }
+  if (cost_cache_) {
+    ASSERT_GE(cache->GetUsage(), 1024 * 1024);
+    Close();
+    options.write_buffer_manager.reset();
+    ASSERT_LT(cache->GetUsage(), 1024 * 1024);
+  }
+  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
 }
 
 INSTANTIATE_TEST_CASE_P(DBTestSharedWriteBufferAcrossCFs,
-                        DBTestSharedWriteBufferAcrossCFs, ::testing::Bool());
+                        DBTestSharedWriteBufferAcrossCFs,
+                        ::testing::Values(std::make_tuple(true, false),
+                                          std::make_tuple(false, false),
+                                          std::make_tuple(false, true)));
 
 TEST_F(DBTest2, SharedWriteBufferLimitAcrossDB) {
   std::string dbname2 = test::TmpDir(env_) + "/db_shared_wb_db2";
   Options options = CurrentOptions();
+  options.arena_block_size = 4096;
+  // Avoid undeterministic value by malloc_usable_size();
+  // Force arena block size to 1
+  rocksdb::SyncPoint::GetInstance()->SetCallBack(
+      "Arena::Arena:0", [&](void* arg) {
+        size_t* block_size = static_cast<size_t*>(arg);
+        *block_size = 1;
+      });
+
+  rocksdb::SyncPoint::GetInstance()->SetCallBack(
+      "Arena::AllocateNewBlock:0", [&](void* arg) {
+        std::pair<size_t*, size_t*>* pair =
+            static_cast<std::pair<size_t*, size_t*>*>(arg);
+        *std::get<0>(*pair) = *std::get<1>(*pair);
+      });
+  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+
   options.write_buffer_size = 500000;  // this is never hit
-  options.write_buffer_manager.reset(new WriteBufferManager(100000));
+  // Use a write buffer total size so that the soft limit is about
+  // 105000.
+  options.write_buffer_manager.reset(new WriteBufferManager(120000));
   CreateAndReopenWithCF({"cf1", "cf2"}, options);
 
   ASSERT_OK(DestroyDB(dbname2, options));
@@ -315,17 +385,25 @@ TEST_F(DBTest2, SharedWriteBufferLimitAcrossDB) {
   WriteOptions wo;
   wo.disableWAL = true;
 
+  std::function<void()> wait_flush = [&]() {
+    dbfull()->TEST_WaitForFlushMemTable(handles_[0]);
+    dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
+    dbfull()->TEST_WaitForFlushMemTable(handles_[2]);
+    static_cast<DBImpl*>(db2)->TEST_WaitForFlushMemTable();
+  };
+
   // Trigger a flush on cf2
   ASSERT_OK(Put(2, Key(1), DummyString(70000), wo));
+  wait_flush();
   ASSERT_OK(Put(0, Key(1), DummyString(20000), wo));
+  wait_flush();
 
   // Insert to DB2
   ASSERT_OK(db2->Put(wo, Key(2), DummyString(20000)));
+  wait_flush();
 
   ASSERT_OK(Put(2, Key(1), DummyString(1), wo));
-  dbfull()->TEST_WaitForFlushMemTable(handles_[0]);
-  dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
-  dbfull()->TEST_WaitForFlushMemTable(handles_[2]);
+  wait_flush();
   static_cast<DBImpl*>(db2)->TEST_WaitForFlushMemTable();
   {
     ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default") +
@@ -338,10 +416,9 @@ TEST_F(DBTest2, SharedWriteBufferLimitAcrossDB) {
 
   // Triggering to flush another CF in DB1
   ASSERT_OK(db2->Put(wo, Key(2), DummyString(70000)));
+  wait_flush();
   ASSERT_OK(Put(2, Key(1), DummyString(1), wo));
-  dbfull()->TEST_WaitForFlushMemTable(handles_[0]);
-  dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
-  dbfull()->TEST_WaitForFlushMemTable(handles_[2]);
+  wait_flush();
   {
     ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"),
               static_cast<uint64_t>(1));
@@ -355,10 +432,9 @@ TEST_F(DBTest2, SharedWriteBufferLimitAcrossDB) {
 
   // Triggering flush in DB2.
   ASSERT_OK(db2->Put(wo, Key(3), DummyString(40000)));
+  wait_flush();
   ASSERT_OK(db2->Put(wo, Key(1), DummyString(1)));
-  dbfull()->TEST_WaitForFlushMemTable(handles_[0]);
-  dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
-  dbfull()->TEST_WaitForFlushMemTable(handles_[2]);
+  wait_flush();
   static_cast<DBImpl*>(db2)->TEST_WaitForFlushMemTable();
   {
     ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"),
@@ -373,6 +449,8 @@ TEST_F(DBTest2, SharedWriteBufferLimitAcrossDB) {
 
   delete db2;
   ASSERT_OK(DestroyDB(dbname2, options));
+
+  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
 }
 
 namespace {
@@ -1080,7 +1158,6 @@ TEST_F(DBTest2, CompressionOptions) {
   options.max_bytes_for_level_multiplier = 2;
   options.num_levels = 7;
   options.max_background_compactions = 1;
-  options.base_background_compactions = 1;
 
   CompactionCompressionListener* listener =
       new CompactionCompressionListener(&options);
@@ -1157,6 +1234,9 @@ TEST_F(DBTest2, CompactionStall) {
   CompactionStallTestListener* listener = new CompactionStallTestListener();
   options.listeners.emplace_back(listener);
   DestroyAndReopen(options);
+  // make sure all background compaction jobs can be scheduled
+  auto stop_token =
+      dbfull()->TEST_write_controler().GetCompactionPressureToken();
 
   Random rnd(301);
 
@@ -2104,6 +2184,56 @@ TEST_F(DBTest2, MemtableOnlyIterator) {
   ASSERT_EQ(1, count);
   delete it;
 }
+
+TEST_F(DBTest2, LowPriWrite) {
+  Options options = CurrentOptions();
+  // Compaction pressure should trigger since 6 files
+  options.level0_file_num_compaction_trigger = 4;
+  options.level0_slowdown_writes_trigger = 12;
+  options.level0_stop_writes_trigger = 30;
+  options.delayed_write_rate = 8 * 1024 * 1024;
+  Reopen(options);
+
+  std::atomic<int> rate_limit_count(0);
+
+  rocksdb::SyncPoint::GetInstance()->SetCallBack(
+      "GenericRateLimiter::Request:1", [&](void* arg) {
+        rate_limit_count.fetch_add(1);
+        int64_t* rate_bytes_per_sec = static_cast<int64_t*>(arg);
+        ASSERT_EQ(1024 * 1024, *rate_bytes_per_sec);
+      });
+  // Block compaction
+  rocksdb::SyncPoint::GetInstance()->LoadDependency({
+      {"DBTest.LowPriWrite:0", "DBImpl::BGWorkCompaction"},
+  });
+  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+  WriteOptions wo;
+  for (int i = 0; i < 6; i++) {
+    wo.low_pri = false;
+    Put("", "", wo);
+    wo.low_pri = true;
+    Put("", "", wo);
+    Flush();
+  }
+  ASSERT_EQ(0, rate_limit_count.load());
+  wo.low_pri = true;
+  Put("", "", wo);
+  ASSERT_EQ(1, rate_limit_count.load());
+  wo.low_pri = false;
+  Put("", "", wo);
+  ASSERT_EQ(1, rate_limit_count.load());
+
+  TEST_SYNC_POINT("DBTest.LowPriWrite:0");
+  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
+
+  dbfull()->TEST_WaitForCompact();
+  wo.low_pri = true;
+  Put("", "", wo);
+  ASSERT_EQ(1, rate_limit_count.load());
+  wo.low_pri = false;
+  Put("", "", wo);
+  ASSERT_EQ(1, rate_limit_count.load());
+}
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/db_test_util.cc

@@ -43,9 +43,9 @@ SpecialEnv::SpecialEnv(Env* base)
 }
 
 DBTestBase::DBTestBase(const std::string path)
-    : option_config_(kDefault),
-      mem_env_(!getenv("MEM_ENV") ? nullptr : new MockEnv(Env::Default())),
-      env_(new SpecialEnv(mem_env_ ? mem_env_ : Env::Default())) {
+    : mem_env_(!getenv("MEM_ENV") ? nullptr : new MockEnv(Env::Default())),
+      env_(new SpecialEnv(mem_env_ ? mem_env_ : Env::Default())),
+      option_config_(kDefault) {
   env_->SetBackgroundThreads(1, Env::LOW);
   env_->SetBackgroundThreads(1, Env::HIGH);
   dbname_ = test::TmpDir(env_) + path;
@@ -245,24 +245,32 @@ bool DBTestBase::ChangeFilterOptions() {
 
 // Return the current option configuration.
 Options DBTestBase::CurrentOptions(
-    const anon::OptionsOverride& options_override) {
+    const anon::OptionsOverride& options_override) const {
+  return GetOptions(option_config_, GetDefaultOptions(), options_override);
+}
+
+Options DBTestBase::CurrentOptions(
+    const Options& default_options,
+    const anon::OptionsOverride& options_override) const {
+  return GetOptions(option_config_, default_options, options_override);
+}
+
+Options DBTestBase::GetDefaultOptions() {
   Options options;
   options.write_buffer_size = 4090 * 4096;
   options.target_file_size_base = 2 * 1024 * 1024;
   options.max_bytes_for_level_base = 10 * 1024 * 1024;
   options.max_open_files = 5000;
-  options.base_background_compactions = -1;
   options.wal_recovery_mode = WALRecoveryMode::kTolerateCorruptedTailRecords;
   options.compaction_pri = CompactionPri::kByCompensatedSize;
-
-  return CurrentOptions(options, options_override);
+  return options;
 }
 
-Options DBTestBase::CurrentOptions(
-    const Options& defaultOptions,
-    const anon::OptionsOverride& options_override) {
+Options DBTestBase::GetOptions(
+    int option_config, const Options& default_options,
+    const anon::OptionsOverride& options_override) const {
   // this redundant copy is to minimize code change w/o having lint error.
-  Options options = defaultOptions;
+  Options options = default_options;
   BlockBasedTableOptions table_options;
   bool set_block_based_table_factory = true;
 #if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) &&  \
@@ -273,7 +281,7 @@ Options DBTestBase::CurrentOptions(
       "NewWritableFile:O_DIRECT");
 #endif
 
-  switch (option_config_) {
+  switch (option_config) {
 #ifndef ROCKSDB_LITE
     case kHashSkipList:
       options.prefix_extractor.reset(NewFixedPrefixTransform(1));
@@ -457,6 +465,10 @@ Options DBTestBase::CurrentOptions(
 #endif
       break;
     }
+    case kPipelinedWrite: {
+      options.enable_pipelined_write = true;
+      break;
+    }
 
     default:
       break;

db/db_test_util.h

@@ -49,10 +49,7 @@
 #include "util/mutexlock.h"
 
 #include "util/string_util.h"
-// SyncPoint is not supported in Released Windows Mode.
-#if !(defined NDEBUG) || !defined(OS_WIN)
 #include "util/sync_point.h"
-#endif  // !(defined NDEBUG) || !defined(OS_WIN)
 #include "util/testharness.h"
 #include "util/testutil.h"
 #include "utilities/merge_operators.h"
@@ -126,8 +123,8 @@ enum SkipPolicy { kSkipNone = 0, kSkipNoSnapshot = 1, kSkipNoPrefix = 2 };
 // A hacky skip list mem table that triggers flush after number of entries.
 class SpecialMemTableRep : public MemTableRep {
  public:
-  explicit SpecialMemTableRep(MemTableAllocator* allocator,
-                              MemTableRep* memtable, int num_entries_flush)
+  explicit SpecialMemTableRep(Allocator* allocator, MemTableRep* memtable,
+                              int num_entries_flush)
       : MemTableRep(allocator),
         memtable_(memtable),
         num_entries_flush_(num_entries_flush),
@@ -187,8 +184,9 @@ class SpecialSkipListFactory : public MemTableRepFactory {
   explicit SpecialSkipListFactory(int num_entries_flush)
       : num_entries_flush_(num_entries_flush) {}
 
+  using MemTableRepFactory::CreateMemTableRep;
   virtual MemTableRep* CreateMemTableRep(
-      const MemTableRep::KeyComparator& compare, MemTableAllocator* allocator,
+      const MemTableRep::KeyComparator& compare, Allocator* allocator,
       const SliceTransform* transform, Logger* logger) override {
     return new SpecialMemTableRep(
         allocator, factory_.CreateMemTableRep(compare, allocator, transform, 0),
@@ -586,10 +584,32 @@ class OnFileDeletionListener : public EventListener {
 };
 #endif
 
+// A test merge operator mimics put but also fails if one of merge operands is
+// "corrupted".
+class TestPutOperator : public MergeOperator {
+ public:
+  virtual bool FullMergeV2(const MergeOperationInput& merge_in,
+                           MergeOperationOutput* merge_out) const override {
+    if (merge_in.existing_value != nullptr &&
+        *(merge_in.existing_value) == "corrupted") {
+      return false;
+    }
+    for (auto value : merge_in.operand_list) {
+      if (value == "corrupted") {
+        return false;
+      }
+    }
+    merge_out->existing_operand = merge_in.operand_list.back();
+    return true;
+  }
+
+  virtual const char* Name() const override { return "TestPutOperator"; }
+};
+
 class DBTestBase : public testing::Test {
- protected:
+ public:
   // Sequence of option configurations to try
-  enum OptionConfig {
+  enum OptionConfig : int {
     kDefault = 0,
     kBlockBasedTableWithPrefixHashIndex = 1,
     kBlockBasedTableWithWholeKeyHashIndex = 2,
@@ -620,15 +640,15 @@ class DBTestBase : public testing::Test {
     kRowCache = 27,
     kRecycleLogFiles = 28,
     kConcurrentSkipList = 29,
-    kDirectIO = 30,
+    kPipelinedWrite = 30,
     kEnd = 31,
-    kLevelSubcompactions = 32,
-    kUniversalSubcompactions = 33,
-    kBlockBasedTableWithIndexRestartInterval = 34,
-    kBlockBasedTableWithPartitionedIndex = 35,
-    kPartitionedFilterWithNewTableReaderForCompactions = 36,
+    kDirectIO = 32,
+    kLevelSubcompactions = 33,
+    kUniversalSubcompactions = 34,
+    kBlockBasedTableWithIndexRestartInterval = 35,
+    kBlockBasedTableWithPartitionedIndex = 36,
+    kPartitionedFilterWithNewTableReaderForCompactions = 37,
   };
-  int option_config_;
 
  public:
   std::string dbname_;
@@ -639,6 +659,7 @@ class DBTestBase : public testing::Test {
   DB* db_;
   std::vector<ColumnFamilyHandle*> handles_;
 
+  int option_config_;
   Options last_options_;
 
   // Skip some options, as they may not be applicable to a specific test.
@@ -689,12 +710,19 @@ class DBTestBase : public testing::Test {
   bool ChangeFilterOptions();
 
   // Return the current option configuration.
-  Options CurrentOptions(
-      const anon::OptionsOverride& options_override = anon::OptionsOverride());
+  Options CurrentOptions(const anon::OptionsOverride& options_override =
+                             anon::OptionsOverride()) const;
 
-  Options CurrentOptions(
-      const Options& defaultOptions,
-      const anon::OptionsOverride& options_override = anon::OptionsOverride());
+  Options CurrentOptions(const Options& default_options,
+                         const anon::OptionsOverride& options_override =
+                             anon::OptionsOverride()) const;
+
+  static Options GetDefaultOptions();
+
+  Options GetOptions(int option_config,
+                     const Options& default_options = GetDefaultOptions(),
+                     const anon::OptionsOverride& options_override =
+                         anon::OptionsOverride()) const;
 
   DBImpl* dbfull() { return reinterpret_cast<DBImpl*>(db_); }
 

db/db_write_test.cc

@@ -0,0 +1,80 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#include <memory>
+#include <thread>
+#include <vector>
+#include "db/db_test_util.h"
+#include "db/write_batch_internal.h"
+#include "port/stack_trace.h"
+#include "util/sync_point.h"
+
+namespace rocksdb {
+
+// Test variations of WriteImpl.
+class DBWriteTest : public DBTestBase, public testing::WithParamInterface<int> {
+ public:
+  DBWriteTest() : DBTestBase("/db_write_test") {}
+
+  void Open() { DBTestBase::Reopen(GetOptions(GetParam())); }
+};
+
+// Sequence number should be return through input write batch.
+TEST_P(DBWriteTest, ReturnSeuqneceNumber) {
+  Random rnd(4422);
+  Open();
+  for (int i = 0; i < 100; i++) {
+    WriteBatch batch;
+    batch.Put("key" + ToString(i), test::RandomHumanReadableString(&rnd, 10));
+    ASSERT_OK(dbfull()->Write(WriteOptions(), &batch));
+    ASSERT_EQ(dbfull()->GetLatestSequenceNumber(),
+              WriteBatchInternal::Sequence(&batch));
+  }
+}
+
+TEST_P(DBWriteTest, ReturnSeuqneceNumberMultiThreaded) {
+  constexpr size_t kThreads = 16;
+  constexpr size_t kNumKeys = 1000;
+  Open();
+  ASSERT_EQ(0, dbfull()->GetLatestSequenceNumber());
+  // Check each sequence is used once and only once.
+  std::vector<std::atomic_flag> flags(kNumKeys * kThreads + 1);
+  for (size_t i = 0; i < flags.size(); i++) {
+    flags[i].clear();
+  }
+  auto writer = [&](size_t id) {
+    Random rnd(4422 + static_cast<uint32_t>(id));
+    for (size_t k = 0; k < kNumKeys; k++) {
+      WriteBatch batch;
+      batch.Put("key" + ToString(id) + "-" + ToString(k),
+                test::RandomHumanReadableString(&rnd, 10));
+      ASSERT_OK(dbfull()->Write(WriteOptions(), &batch));
+      SequenceNumber sequence = WriteBatchInternal::Sequence(&batch);
+      ASSERT_GT(sequence, 0);
+      ASSERT_LE(sequence, kNumKeys * kThreads);
+      // The sequence isn't consumed by someone else.
+      ASSERT_FALSE(flags[sequence].test_and_set());
+    }
+  };
+  std::vector<port::Thread> threads;
+  for (size_t i = 0; i < kThreads; i++) {
+    threads.emplace_back(writer, i);
+  }
+  for (size_t i = 0; i < kThreads; i++) {
+    threads[i].join();
+  }
+}
+
+INSTANTIATE_TEST_CASE_P(DBWriteTestInstance, DBWriteTest,
+                        testing::Values(DBTestBase::kDefault,
+                                        DBTestBase::kPipelinedWrite));
+
+}  // namespace rocksdb
+
+int main(int argc, char** argv) {
+  rocksdb::port::InstallStackTraceHandler();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

db/external_sst_file_basic_test.cc

@@ -38,8 +38,10 @@ class ExternalSSTFileBasicTest : public DBTestBase {
   }
 
   Status GenerateAndAddExternalFile(
-      const Options options, std::vector<int> keys, int file_id,
+      const Options options, std::vector<int> keys,
+      const std::vector<ValueType>& value_types, int file_id,
       std::map<std::string, std::string>* true_data) {
+    assert(value_types.size() == 1 || keys.size() == value_types.size());
     std::string file_path = sst_files_dir_ + ToString(file_id);
     SstFileWriter sst_file_writer(EnvOptions(), options);
 
@@ -47,11 +49,28 @@ class ExternalSSTFileBasicTest : public DBTestBase {
     if (!s.ok()) {
       return s;
     }
-    for (int k : keys) {
-      std::string key = Key(k);
-      std::string value = Key(k) + ToString(file_id);
-      s = sst_file_writer.Add(key, value);
-      (*true_data)[key] = value;
+    for (size_t i = 0; i < keys.size(); i++) {
+      std::string key = Key(keys[i]);
+      std::string value = Key(keys[i]) + ToString(file_id);
+      ValueType value_type =
+          (value_types.size() == 1 ? value_types[0] : value_types[i]);
+      switch (value_type) {
+        case ValueType::kTypeValue:
+          s = sst_file_writer.Put(key, value);
+          (*true_data)[key] = value;
+          break;
+        case ValueType::kTypeMerge:
+          s = sst_file_writer.Merge(key, value);
+          // we only use TestPutOperator in this test
+          (*true_data)[key] = value;
+          break;
+        case ValueType::kTypeDeletion:
+          s = sst_file_writer.Delete(key);
+          true_data->erase(key);
+          break;
+        default:
+          return Status::InvalidArgument("Value type is not supported");
+      }
       if (!s.ok()) {
         sst_file_writer.Finish();
         return s;
@@ -64,10 +83,17 @@ class ExternalSSTFileBasicTest : public DBTestBase {
       ifo.allow_global_seqno = true;
       s = db_->IngestExternalFile({file_path}, ifo);
     }
-
     return s;
   }
 
+  Status GenerateAndAddExternalFile(
+      const Options options, std::vector<int> keys, const ValueType value_type,
+      int file_id, std::map<std::string, std::string>* true_data) {
+    return GenerateAndAddExternalFile(options, keys,
+                                      std::vector<ValueType>(1, value_type),
+                                      file_id, true_data);
+  }
+
   ~ExternalSSTFileBasicTest() { test::DestroyDir(env_, sst_files_dir_); }
 
  protected:
@@ -87,7 +113,7 @@ TEST_F(ExternalSSTFileBasicTest, Basic) {
   std::string file1 = sst_files_dir_ + "file1.sst";
   ASSERT_OK(sst_file_writer.Open(file1));
   for (int k = 0; k < 100; k++) {
-    ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val"));
+    ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val"));
   }
   ExternalSstFileInfo file1_info;
   Status s = sst_file_writer.Finish(&file1_info);
@@ -101,7 +127,7 @@ TEST_F(ExternalSSTFileBasicTest, Basic) {
   ASSERT_EQ(file1_info.smallest_key, Key(0));
   ASSERT_EQ(file1_info.largest_key, Key(99));
   // sst_file_writer already finished, cannot add this value
-  s = sst_file_writer.Add(Key(100), "bad_val");
+  s = sst_file_writer.Put(Key(100), "bad_val");
   ASSERT_FALSE(s.ok()) << s.ToString();
 
   DestroyAndReopen(options);
@@ -126,7 +152,7 @@ TEST_F(ExternalSSTFileBasicTest, NoCopy) {
   std::string file1 = sst_files_dir_ + "file1.sst";
   ASSERT_OK(sst_file_writer.Open(file1));
   for (int k = 0; k < 100; k++) {
-    ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val"));
+    ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val"));
   }
   ExternalSstFileInfo file1_info;
   Status s = sst_file_writer.Finish(&file1_info);
@@ -140,7 +166,7 @@ TEST_F(ExternalSSTFileBasicTest, NoCopy) {
   std::string file2 = sst_files_dir_ + "file2.sst";
   ASSERT_OK(sst_file_writer.Open(file2));
   for (int k = 100; k < 300; k++) {
-    ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val"));
+    ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val"));
   }
   ExternalSstFileInfo file2_info;
   s = sst_file_writer.Finish(&file2_info);
@@ -154,7 +180,7 @@ TEST_F(ExternalSSTFileBasicTest, NoCopy) {
   std::string file3 = sst_files_dir_ + "file3.sst";
   ASSERT_OK(sst_file_writer.Open(file3));
   for (int k = 110; k < 125; k++) {
-    ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val_overlap"));
+    ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val_overlap"));
   }
   ExternalSstFileInfo file3_info;
   s = sst_file_writer.Finish(&file3_info);
@@ -189,33 +215,36 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithGlobalSeqnoPickedSeqno) {
 
     int file_id = 1;
 
-    ASSERT_OK(GenerateAndAddExternalFile(options, {1, 2, 3, 4, 5, 6}, file_id++,
+    ASSERT_OK(GenerateAndAddExternalFile(options, {1, 2, 3, 4, 5, 6},
+                                         ValueType::kTypeValue, file_id++,
                                          &true_data));
     // File dont overwrite any keys, No seqno needed
     ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 0);
 
-    ASSERT_OK(GenerateAndAddExternalFile(options, {10, 11, 12, 13}, file_id++,
+    ASSERT_OK(GenerateAndAddExternalFile(options, {10, 11, 12, 13},
+                                         ValueType::kTypeValue, file_id++,
+
                                          &true_data));
     // File dont overwrite any keys, No seqno needed
     ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 0);
 
-    ASSERT_OK(
-        GenerateAndAddExternalFile(options, {1, 4, 6}, file_id++, &true_data));
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {1, 4, 6}, ValueType::kTypeValue, file_id++, &true_data));
     // File overwrite some keys, a seqno will be assigned
     ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 1);
 
-    ASSERT_OK(GenerateAndAddExternalFile(options, {11, 15, 19}, file_id++,
-                                         &true_data));
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {11, 15, 19}, ValueType::kTypeValue, file_id++, &true_data));
     // File overwrite some keys, a seqno will be assigned
     ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 2);
 
-    ASSERT_OK(
-        GenerateAndAddExternalFile(options, {120, 130}, file_id++, &true_data));
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {120, 130}, ValueType::kTypeValue, file_id++, &true_data));
     // File dont overwrite any keys, No seqno needed
     ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 2);
 
-    ASSERT_OK(
-        GenerateAndAddExternalFile(options, {1, 130}, file_id++, &true_data));
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {1, 130}, ValueType::kTypeValue, file_id++, &true_data));
     // File overwrite some keys, a seqno will be assigned
     ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 3);
 
@@ -226,17 +255,115 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithGlobalSeqnoPickedSeqno) {
     }
     SequenceNumber last_seqno = dbfull()->GetLatestSequenceNumber();
 
-    ASSERT_OK(GenerateAndAddExternalFile(options, {60, 61, 62}, file_id++,
-                                         &true_data));
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {60, 61, 62}, ValueType::kTypeValue, file_id++, &true_data));
     // File dont overwrite any keys, No seqno needed
     ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno);
 
-    ASSERT_OK(GenerateAndAddExternalFile(options, {40, 41, 42}, file_id++,
-                                         &true_data));
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {40, 41, 42}, ValueType::kTypeValue, file_id++, &true_data));
     // File overwrite some keys, a seqno will be assigned
     ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 1);
 
-    ASSERT_OK(GenerateAndAddExternalFile(options, {20, 30, 40}, file_id++,
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {20, 30, 40}, ValueType::kTypeValue, file_id++, &true_data));
+    // File overwrite some keys, a seqno will be assigned
+    ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 2);
+
+    const Snapshot* snapshot = db_->GetSnapshot();
+
+    // We will need a seqno for the file regardless if the file overwrite
+    // keys in the DB or not because we have a snapshot
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {1000, 1002}, ValueType::kTypeValue, file_id++, &true_data));
+    // A global seqno will be assigned anyway because of the snapshot
+    ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 3);
+
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {2000, 3002}, ValueType::kTypeValue, file_id++, &true_data));
+    // A global seqno will be assigned anyway because of the snapshot
+    ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 4);
+
+    ASSERT_OK(GenerateAndAddExternalFile(options, {1, 20, 40, 100, 150},
+                                         ValueType::kTypeValue, file_id++,
+                                         &true_data));
+    // A global seqno will be assigned anyway because of the snapshot
+    ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 5);
+
+    db_->ReleaseSnapshot(snapshot);
+
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {5000, 5001}, ValueType::kTypeValue, file_id++, &true_data));
+    // No snapshot anymore, no need to assign a seqno
+    ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 5);
+
+    size_t kcnt = 0;
+    VerifyDBFromMap(true_data, &kcnt, false);
+  } while (ChangeCompactOptions());
+}
+
+TEST_F(ExternalSSTFileBasicTest, IngestFileWithMultipleValueType) {
+  do {
+    Options options = CurrentOptions();
+    options.merge_operator.reset(new TestPutOperator());
+    DestroyAndReopen(options);
+    std::map<std::string, std::string> true_data;
+
+    int file_id = 1;
+
+    ASSERT_OK(GenerateAndAddExternalFile(options, {1, 2, 3, 4, 5, 6},
+                                         ValueType::kTypeValue, file_id++,
+                                         &true_data));
+    // File dont overwrite any keys, No seqno needed
+    ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 0);
+
+    ASSERT_OK(GenerateAndAddExternalFile(options, {10, 11, 12, 13},
+                                         ValueType::kTypeValue, file_id++,
+
+                                         &true_data));
+    // File dont overwrite any keys, No seqno needed
+    ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 0);
+
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {1, 4, 6}, ValueType::kTypeMerge, file_id++, &true_data));
+    // File overwrite some keys, a seqno will be assigned
+    ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 1);
+
+    ASSERT_OK(GenerateAndAddExternalFile(options, {11, 15, 19},
+                                         ValueType::kTypeDeletion, file_id++,
+                                         &true_data));
+    // File overwrite some keys, a seqno will be assigned
+    ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 2);
+
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {120, 130}, ValueType::kTypeMerge, file_id++, &true_data));
+    // File dont overwrite any keys, No seqno needed
+    ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 2);
+
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {1, 130}, ValueType::kTypeDeletion, file_id++, &true_data));
+    // File overwrite some keys, a seqno will be assigned
+    ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 3);
+
+    // Write some keys through normal write path
+    for (int i = 0; i < 50; i++) {
+      ASSERT_OK(Put(Key(i), "memtable"));
+      true_data[Key(i)] = "memtable";
+    }
+    SequenceNumber last_seqno = dbfull()->GetLatestSequenceNumber();
+
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {60, 61, 62}, ValueType::kTypeValue, file_id++, &true_data));
+    // File dont overwrite any keys, No seqno needed
+    ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno);
+
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {40, 41, 42}, ValueType::kTypeMerge, file_id++, &true_data));
+    // File overwrite some keys, a seqno will be assigned
+    ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 1);
+
+    ASSERT_OK(GenerateAndAddExternalFile(options, {20, 30, 40},
+                                         ValueType::kTypeDeletion, file_id++,
                                          &true_data));
     // File overwrite some keys, a seqno will be assigned
     ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 2);
@@ -245,25 +372,143 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithGlobalSeqnoPickedSeqno) {
 
     // We will need a seqno for the file regardless if the file overwrite
     // keys in the DB or not because we have a snapshot
-    ASSERT_OK(GenerateAndAddExternalFile(options, {1000, 1002}, file_id++,
-                                         &true_data));
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {1000, 1002}, ValueType::kTypeMerge, file_id++, &true_data));
     // A global seqno will be assigned anyway because of the snapshot
     ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 3);
 
-    ASSERT_OK(GenerateAndAddExternalFile(options, {2000, 3002}, file_id++,
-                                         &true_data));
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {2000, 3002}, ValueType::kTypeMerge, file_id++, &true_data));
     // A global seqno will be assigned anyway because of the snapshot
     ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 4);
 
     ASSERT_OK(GenerateAndAddExternalFile(options, {1, 20, 40, 100, 150},
-                                         file_id++, &true_data));
+                                         ValueType::kTypeMerge, file_id++,
+                                         &true_data));
     // A global seqno will be assigned anyway because of the snapshot
     ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 5);
 
     db_->ReleaseSnapshot(snapshot);
 
-    ASSERT_OK(GenerateAndAddExternalFile(options, {5000, 5001}, file_id++,
-                                         &true_data));
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {5000, 5001}, ValueType::kTypeValue, file_id++, &true_data));
+    // No snapshot anymore, no need to assign a seqno
+    ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 5);
+
+    size_t kcnt = 0;
+    VerifyDBFromMap(true_data, &kcnt, false);
+  } while (ChangeCompactOptions());
+}
+
+TEST_F(ExternalSSTFileBasicTest, IngestFileWithMixedValueType) {
+  do {
+    Options options = CurrentOptions();
+    options.merge_operator.reset(new TestPutOperator());
+    DestroyAndReopen(options);
+    std::map<std::string, std::string> true_data;
+
+    int file_id = 1;
+
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {1, 2, 3, 4, 5, 6},
+        {ValueType::kTypeValue, ValueType::kTypeMerge, ValueType::kTypeValue,
+         ValueType::kTypeMerge, ValueType::kTypeValue, ValueType::kTypeMerge},
+        file_id++, &true_data));
+    // File dont overwrite any keys, No seqno needed
+    ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 0);
+
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {10, 11, 12, 13},
+        {ValueType::kTypeValue, ValueType::kTypeMerge, ValueType::kTypeValue,
+         ValueType::kTypeMerge},
+        file_id++, &true_data));
+    // File dont overwrite any keys, No seqno needed
+    ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 0);
+
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {1, 4, 6}, {ValueType::kTypeDeletion, ValueType::kTypeValue,
+                             ValueType::kTypeMerge},
+        file_id++, &true_data));
+    // File overwrite some keys, a seqno will be assigned
+    ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 1);
+
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {11, 15, 19}, {ValueType::kTypeDeletion, ValueType::kTypeMerge,
+                                ValueType::kTypeValue},
+        file_id++, &true_data));
+    // File overwrite some keys, a seqno will be assigned
+    ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 2);
+
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {120, 130}, {ValueType::kTypeValue, ValueType::kTypeMerge},
+        file_id++, &true_data));
+    // File dont overwrite any keys, No seqno needed
+    ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 2);
+
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {1, 130}, {ValueType::kTypeMerge, ValueType::kTypeDeletion},
+        file_id++, &true_data));
+    // File overwrite some keys, a seqno will be assigned
+    ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 3);
+
+    // Write some keys through normal write path
+    for (int i = 0; i < 50; i++) {
+      ASSERT_OK(Put(Key(i), "memtable"));
+      true_data[Key(i)] = "memtable";
+    }
+    SequenceNumber last_seqno = dbfull()->GetLatestSequenceNumber();
+
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {60, 61, 62},
+        {ValueType::kTypeValue, ValueType::kTypeMerge, ValueType::kTypeValue},
+        file_id++, &true_data));
+    // File dont overwrite any keys, No seqno needed
+    ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno);
+
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {40, 41, 42}, {ValueType::kTypeValue, ValueType::kTypeDeletion,
+                                ValueType::kTypeDeletion},
+        file_id++, &true_data));
+    // File overwrite some keys, a seqno will be assigned
+    ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 1);
+
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {20, 30, 40},
+        {ValueType::kTypeDeletion, ValueType::kTypeDeletion,
+         ValueType::kTypeDeletion},
+        file_id++, &true_data));
+    // File overwrite some keys, a seqno will be assigned
+    ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 2);
+
+    const Snapshot* snapshot = db_->GetSnapshot();
+
+    // We will need a seqno for the file regardless if the file overwrite
+    // keys in the DB or not because we have a snapshot
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {1000, 1002}, {ValueType::kTypeValue, ValueType::kTypeMerge},
+        file_id++, &true_data));
+    // A global seqno will be assigned anyway because of the snapshot
+    ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 3);
+
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {2000, 3002}, {ValueType::kTypeValue, ValueType::kTypeMerge},
+        file_id++, &true_data));
+    // A global seqno will be assigned anyway because of the snapshot
+    ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 4);
+
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {1, 20, 40, 100, 150},
+        {ValueType::kTypeDeletion, ValueType::kTypeDeletion,
+         ValueType::kTypeValue, ValueType::kTypeMerge, ValueType::kTypeMerge},
+        file_id++, &true_data));
+    // A global seqno will be assigned anyway because of the snapshot
+    ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 5);
+
+    db_->ReleaseSnapshot(snapshot);
+
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, {5000, 5001}, {ValueType::kTypeValue, ValueType::kTypeMerge},
+        file_id++, &true_data));
     // No snapshot anymore, no need to assign a seqno
     ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 5);
 
@@ -278,7 +523,7 @@ TEST_F(ExternalSSTFileBasicTest, FadviseTrigger) {
 
   size_t total_fadvised_bytes = 0;
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "SstFileWriter::InvalidatePageCache", [&](void* arg) {
+      "SstFileWriter::Rep::InvalidatePageCache", [&](void* arg) {
         size_t fadvise_size = *(reinterpret_cast<size_t*>(arg));
         total_fadvised_bytes += fadvise_size;
       });
@@ -291,19 +536,18 @@ TEST_F(ExternalSSTFileBasicTest, FadviseTrigger) {
       new SstFileWriter(EnvOptions(), options, nullptr, false));
   ASSERT_OK(sst_file_writer->Open(sst_file_path));
   for (int i = 0; i < kNumKeys; i++) {
-    ASSERT_OK(sst_file_writer->Add(Key(i), Key(i)));
+    ASSERT_OK(sst_file_writer->Put(Key(i), Key(i)));
   }
   ASSERT_OK(sst_file_writer->Finish());
   // fadvise disabled
   ASSERT_EQ(total_fadvised_bytes, 0);
 
-
   sst_file_path = sst_files_dir_ + "file_fadvise_enable.sst";
   sst_file_writer.reset(
       new SstFileWriter(EnvOptions(), options, nullptr, true));
   ASSERT_OK(sst_file_writer->Open(sst_file_path));
   for (int i = 0; i < kNumKeys; i++) {
-    ASSERT_OK(sst_file_writer->Add(Key(i), Key(i)));
+    ASSERT_OK(sst_file_writer->Put(Key(i), Key(i)));
   }
   ASSERT_OK(sst_file_writer->Finish());
   // fadvise enabled
@@ -313,6 +557,55 @@ TEST_F(ExternalSSTFileBasicTest, FadviseTrigger) {
   rocksdb::SyncPoint::GetInstance()->DisableProcessing();
 }
 
+TEST_F(ExternalSSTFileBasicTest, IngestionWithRangeDeletions) {
+  Options options = CurrentOptions();
+  options.disable_auto_compactions = true;
+  Reopen(options);
+
+  std::map<std::string, std::string> true_data;
+  int file_id = 1;
+  // prevent range deletions from being dropped due to becoming obsolete.
+  const Snapshot* snapshot = db_->GetSnapshot();
+
+  // range del [0, 50) in L0 file, [50, 100) in memtable
+  for (int i = 0; i < 2; i++) {
+    if (i == 1) {
+      db_->Flush(FlushOptions());
+    }
+    ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(),
+                               Key(50 * i), Key(50 * (i + 1))));
+  }
+  ASSERT_EQ(1, NumTableFilesAtLevel(0));
+
+  // overlaps with L0 file but not memtable, so flush is skipped
+  SequenceNumber last_seqno = dbfull()->GetLatestSequenceNumber();
+  ASSERT_OK(GenerateAndAddExternalFile(
+      options, {10, 40}, {ValueType::kTypeValue, ValueType::kTypeValue},
+      file_id++, &true_data));
+  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), ++last_seqno);
+  ASSERT_EQ(2, NumTableFilesAtLevel(0));
+
+  // overlaps with memtable, so flush is triggered (thus file count increases by
+  // two at this step).
+  ASSERT_OK(GenerateAndAddExternalFile(
+      options, {50, 90}, {ValueType::kTypeValue, ValueType::kTypeValue},
+      file_id++, &true_data));
+  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), ++last_seqno);
+  ASSERT_EQ(4, NumTableFilesAtLevel(0));
+
+  // snapshot unneeded now that both range deletions are persisted
+  db_->ReleaseSnapshot(snapshot);
+
+  // overlaps with nothing, so places at bottom level and skips incrementing
+  // seqnum.
+  ASSERT_OK(GenerateAndAddExternalFile(
+      options, {101, 125}, {ValueType::kTypeValue, ValueType::kTypeValue},
+      file_id++, &true_data));
+  ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno);
+  ASSERT_EQ(4, NumTableFilesAtLevel(0));
+  ASSERT_EQ(1, NumTableFilesAtLevel(options.num_levels - 1));
+}
+
 #endif  // ROCKSDB_LITE
 
 }  // namespace rocksdb

db/external_sst_file_ingestion_job.cc

@@ -379,6 +379,17 @@ Status ExternalSstFileIngestionJob::IngestedFilesOverlapWithMemtables(
   sv->imm->AddIterators(ro, &merge_iter_builder);
   ScopedArenaIterator memtable_iter(merge_iter_builder.Finish());
 
+  std::vector<InternalIterator*> memtable_range_del_iters;
+  auto* active_range_del_iter = sv->mem->NewRangeTombstoneIterator(ro);
+  if (active_range_del_iter != nullptr) {
+    memtable_range_del_iters.push_back(active_range_del_iter);
+  }
+  sv->imm->AddRangeTombstoneIterators(ro, &memtable_range_del_iters);
+  std::unique_ptr<InternalIterator> memtable_range_del_iter(NewMergingIterator(
+      &cfd_->internal_comparator(),
+      memtable_range_del_iters.empty() ? nullptr : &memtable_range_del_iters[0],
+      static_cast<int>(memtable_range_del_iters.size())));
+
   Status status;
   *overlap = false;
   for (IngestedFileInfo& f : files_to_ingest_) {
@@ -387,6 +398,11 @@ Status ExternalSstFileIngestionJob::IngestedFilesOverlapWithMemtables(
     if (!status.ok() || *overlap == true) {
       break;
     }
+    status = IngestedFileOverlapWithRangeDeletions(
+        &f, memtable_range_del_iter.get(), overlap);
+    if (!status.ok() || *overlap == true) {
+      break;
+    }
   }
 
   return status;
@@ -553,6 +569,34 @@ Status ExternalSstFileIngestionJob::IngestedFileOverlapWithIteratorRange(
   return iter->status();
 }
 
+Status ExternalSstFileIngestionJob::IngestedFileOverlapWithRangeDeletions(
+    const IngestedFileInfo* file_to_ingest, InternalIterator* range_del_iter,
+    bool* overlap) {
+  auto* vstorage = cfd_->current()->storage_info();
+  auto* ucmp = vstorage->InternalComparator()->user_comparator();
+
+  *overlap = false;
+  if (range_del_iter != nullptr) {
+    for (range_del_iter->SeekToFirst(); range_del_iter->Valid();
+         range_del_iter->Next()) {
+      ParsedInternalKey parsed_key;
+      if (!ParseInternalKey(range_del_iter->key(), &parsed_key)) {
+        return Status::Corruption("corrupted range deletion key: " +
+                                  range_del_iter->key().ToString());
+      }
+      RangeTombstone range_del(parsed_key, range_del_iter->value());
+      if (ucmp->Compare(range_del.start_key_,
+                        file_to_ingest->largest_user_key) <= 0 &&
+          ucmp->Compare(file_to_ingest->smallest_user_key,
+                        range_del.end_key_) <= 0) {
+        *overlap = true;
+        break;
+      }
+    }
+  }
+  return Status::OK();
+}
+
 bool ExternalSstFileIngestionJob::IngestedFileFitInLevel(
     const IngestedFileInfo* file_to_ingest, int level) {
   if (level == 0) {
@@ -589,17 +633,25 @@ Status ExternalSstFileIngestionJob::IngestedFileOverlapWithLevel(
   ro.total_order_seek = true;
   MergeIteratorBuilder merge_iter_builder(&cfd_->internal_comparator(),
                                           &arena);
-  RangeDelAggregator range_del_agg(cfd_->internal_comparator(),
-                                   {} /* snapshots */);
-  sv->current->AddIteratorsForLevel(ro, env_options_, &merge_iter_builder,
-                                    lvl, &range_del_agg);
-  if (!range_del_agg.IsEmpty()) {
-    return Status::NotSupported(
-        "file ingestion with range tombstones is currently unsupported");
-  }
+  sv->current->AddIteratorsForLevel(ro, env_options_, &merge_iter_builder, lvl,
+                                    nullptr /* range_del_agg */);
   ScopedArenaIterator level_iter(merge_iter_builder.Finish());
-  return IngestedFileOverlapWithIteratorRange(
+
+  std::vector<InternalIterator*> level_range_del_iters;
+  sv->current->AddRangeDelIteratorsForLevel(ro, env_options_, lvl,
+                                            &level_range_del_iters);
+  std::unique_ptr<InternalIterator> level_range_del_iter(NewMergingIterator(
+      &cfd_->internal_comparator(),
+      level_range_del_iters.empty() ? nullptr : &level_range_del_iters[0],
+      static_cast<int>(level_range_del_iters.size())));
+
+  Status status = IngestedFileOverlapWithIteratorRange(
       file_to_ingest, level_iter.get(), overlap_with_level);
+  if (status.ok() && *overlap_with_level == false) {
+    status = IngestedFileOverlapWithRangeDeletions(
+        file_to_ingest, level_range_del_iter.get(), overlap_with_level);
+  }
+  return status;
 }
 
 }  // namespace rocksdb

db/external_sst_file_ingestion_job.h

@@ -95,7 +95,7 @@ class ExternalSstFileIngestionJob {
   // REQUIRES: Mutex held
   void UpdateStats();
 
-  // Cleanup after successfull/failed job
+  // Cleanup after successful/failed job
   void Cleanup(const Status& status);
 
   VersionEdit* edit() { return &edit_; }
@@ -139,6 +139,13 @@ class ExternalSstFileIngestionJob {
       const IngestedFileInfo* file_to_ingest, InternalIterator* iter,
       bool* overlap);
 
+  // Check if `file_to_ingest` key range overlaps with any range deletions
+  // specified by `iter`.
+  // REQUIRES: Mutex held
+  Status IngestedFileOverlapWithRangeDeletions(
+      const IngestedFileInfo* file_to_ingest, InternalIterator* range_del_iter,
+      bool* overlap);
+
   // Check if `file_to_ingest` key range overlap with level
   // REQUIRES: Mutex held
   Status IngestedFileOverlapWithLevel(SuperVersion* sv,

db/external_sst_file_test.cc

@@ -61,7 +61,7 @@ class ExternalSSTFileTest : public DBTestBase {
       return s;
     }
     for (auto& entry : data) {
-      s = sst_file_writer.Add(entry.first, entry.second);
+      s = sst_file_writer.Put(entry.first, entry.second);
       if (!s.ok()) {
         sst_file_writer.Finish();
         return s;
@@ -123,7 +123,7 @@ class ExternalSSTFileTest : public DBTestBase {
       return s;
     }
     for (auto& entry : data) {
-      s = sst_file_writer.Add(entry.first, entry.second);
+      s = sst_file_writer.Put(entry.first, entry.second);
       if (!s.ok()) {
         sst_file_writer.Finish();
         return s;
@@ -209,7 +209,7 @@ TEST_F(ExternalSSTFileTest, Basic) {
     std::string file1 = sst_files_dir_ + "file1.sst";
     ASSERT_OK(sst_file_writer.Open(file1));
     for (int k = 0; k < 100; k++) {
-      ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val"));
+      ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val"));
     }
     ExternalSstFileInfo file1_info;
     Status s = sst_file_writer.Finish(&file1_info);
@@ -223,17 +223,17 @@ TEST_F(ExternalSSTFileTest, Basic) {
     ASSERT_EQ(file1_info.smallest_key, Key(0));
     ASSERT_EQ(file1_info.largest_key, Key(99));
     // sst_file_writer already finished, cannot add this value
-    s = sst_file_writer.Add(Key(100), "bad_val");
+    s = sst_file_writer.Put(Key(100), "bad_val");
     ASSERT_FALSE(s.ok()) << s.ToString();
 
     // file2.sst (100 => 199)
     std::string file2 = sst_files_dir_ + "file2.sst";
     ASSERT_OK(sst_file_writer.Open(file2));
     for (int k = 100; k < 200; k++) {
-      ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val"));
+      ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val"));
     }
     // Cannot add this key because it's not after last added key
-    s = sst_file_writer.Add(Key(99), "bad_val");
+    s = sst_file_writer.Put(Key(99), "bad_val");
     ASSERT_FALSE(s.ok()) << s.ToString();
     ExternalSstFileInfo file2_info;
     s = sst_file_writer.Finish(&file2_info);
@@ -248,7 +248,7 @@ TEST_F(ExternalSSTFileTest, Basic) {
     std::string file3 = sst_files_dir_ + "file3.sst";
     ASSERT_OK(sst_file_writer.Open(file3));
     for (int k = 195; k < 300; k++) {
-      ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val_overlap"));
+      ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val_overlap"));
     }
     ExternalSstFileInfo file3_info;
     s = sst_file_writer.Finish(&file3_info);
@@ -266,7 +266,7 @@ TEST_F(ExternalSSTFileTest, Basic) {
     std::string file4 = sst_files_dir_ + "file4.sst";
     ASSERT_OK(sst_file_writer.Open(file4));
     for (int k = 30; k < 40; k++) {
-      ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val_overlap"));
+      ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val_overlap"));
     }
     ExternalSstFileInfo file4_info;
     s = sst_file_writer.Finish(&file4_info);
@@ -280,7 +280,7 @@ TEST_F(ExternalSSTFileTest, Basic) {
     std::string file5 = sst_files_dir_ + "file5.sst";
     ASSERT_OK(sst_file_writer.Open(file5));
     for (int k = 400; k < 500; k++) {
-      ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val"));
+      ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val"));
     }
     ExternalSstFileInfo file5_info;
     s = sst_file_writer.Finish(&file5_info);
@@ -442,7 +442,7 @@ TEST_F(ExternalSSTFileTest, AddList) {
     std::string file1 = sst_files_dir_ + "file1.sst";
     ASSERT_OK(sst_file_writer.Open(file1));
     for (int k = 0; k < 100; k++) {
-      ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val"));
+      ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val"));
     }
     ExternalSstFileInfo file1_info;
     Status s = sst_file_writer.Finish(&file1_info);
@@ -452,17 +452,17 @@ TEST_F(ExternalSSTFileTest, AddList) {
     ASSERT_EQ(file1_info.smallest_key, Key(0));
     ASSERT_EQ(file1_info.largest_key, Key(99));
     // sst_file_writer already finished, cannot add this value
-    s = sst_file_writer.Add(Key(100), "bad_val");
+    s = sst_file_writer.Put(Key(100), "bad_val");
     ASSERT_FALSE(s.ok()) << s.ToString();
 
     // file2.sst (100 => 199)
     std::string file2 = sst_files_dir_ + "file2.sst";
     ASSERT_OK(sst_file_writer.Open(file2));
     for (int k = 100; k < 200; k++) {
-      ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val"));
+      ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val"));
     }
     // Cannot add this key because it's not after last added key
-    s = sst_file_writer.Add(Key(99), "bad_val");
+    s = sst_file_writer.Put(Key(99), "bad_val");
     ASSERT_FALSE(s.ok()) << s.ToString();
     ExternalSstFileInfo file2_info;
     s = sst_file_writer.Finish(&file2_info);
@@ -477,7 +477,7 @@ TEST_F(ExternalSSTFileTest, AddList) {
     std::string file3 = sst_files_dir_ + "file3.sst";
     ASSERT_OK(sst_file_writer.Open(file3));
     for (int k = 195; k < 200; k++) {
-      ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val_overlap"));
+      ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val_overlap"));
     }
     ExternalSstFileInfo file3_info;
     s = sst_file_writer.Finish(&file3_info);
@@ -492,7 +492,7 @@ TEST_F(ExternalSSTFileTest, AddList) {
     std::string file4 = sst_files_dir_ + "file4.sst";
     ASSERT_OK(sst_file_writer.Open(file4));
     for (int k = 30; k < 40; k++) {
-      ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val_overlap"));
+      ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val_overlap"));
     }
     ExternalSstFileInfo file4_info;
     s = sst_file_writer.Finish(&file4_info);
@@ -506,7 +506,7 @@ TEST_F(ExternalSSTFileTest, AddList) {
     std::string file5 = sst_files_dir_ + "file5.sst";
     ASSERT_OK(sst_file_writer.Open(file5));
     for (int k = 200; k < 300; k++) {
-      ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val"));
+      ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val"));
     }
     ExternalSstFileInfo file5_info;
     s = sst_file_writer.Finish(&file5_info);
@@ -637,7 +637,7 @@ TEST_F(ExternalSSTFileTest, AddListAtomicity) {
       files[i] = sst_files_dir_ + "file" + std::to_string(i) + ".sst";
       ASSERT_OK(sst_file_writer.Open(files[i]));
       for (int k = i * 100; k < (i + 1) * 100; k++) {
-        ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val"));
+        ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val"));
       }
       Status s = sst_file_writer.Finish(&files_info[i]);
       ASSERT_TRUE(s.ok()) << s.ToString();
@@ -674,7 +674,7 @@ TEST_F(ExternalSSTFileTest, PurgeObsoleteFilesBug) {
   ASSERT_OK(s);
   for (int i = 0; i < 500; i++) {
     std::string k = Key(i);
-    s = sst_file_writer.Add(k, k + "_val");
+    s = sst_file_writer.Put(k, k + "_val");
     ASSERT_OK(s);
   }
 
@@ -717,7 +717,7 @@ TEST_F(ExternalSSTFileTest, SkipSnapshot) {
   std::string file1 = sst_files_dir_ + "file1.sst";
   ASSERT_OK(sst_file_writer.Open(file1));
   for (int k = 0; k < 100; k++) {
-    ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val"));
+    ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val"));
   }
   ExternalSstFileInfo file1_info;
   Status s = sst_file_writer.Finish(&file1_info);
@@ -731,7 +731,7 @@ TEST_F(ExternalSSTFileTest, SkipSnapshot) {
   std::string file2 = sst_files_dir_ + "file2.sst";
   ASSERT_OK(sst_file_writer.Open(file2));
   for (int k = 100; k < 300; k++) {
-    ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val"));
+    ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val"));
   }
   ExternalSstFileInfo file2_info;
   s = sst_file_writer.Finish(&file2_info);
@@ -761,7 +761,7 @@ TEST_F(ExternalSSTFileTest, SkipSnapshot) {
   std::string file3 = sst_files_dir_ + "file3.sst";
   ASSERT_OK(sst_file_writer.Open(file3));
   for (int k = 300; k < 400; k++) {
-    ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val"));
+    ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val"));
   }
   ExternalSstFileInfo file3_info;
   s = sst_file_writer.Finish(&file3_info);
@@ -807,7 +807,7 @@ TEST_F(ExternalSSTFileTest, MultiThreaded) {
       ASSERT_OK(sst_file_writer.Open(file_names[file_idx]));
 
       for (int k = range_start; k < range_end; k++) {
-        ASSERT_OK(sst_file_writer.Add(Key(k), Key(k)));
+        ASSERT_OK(sst_file_writer.Put(Key(k), Key(k)));
       }
 
       Status s = sst_file_writer.Finish();
@@ -958,7 +958,7 @@ TEST_F(ExternalSSTFileTest, OverlappingRanges) {
         std::string file_name = sst_files_dir_ + env_->GenerateUniqueId();
         ASSERT_OK(sst_file_writer.Open(file_name));
         for (int k = range_start; k <= range_end; k++) {
-          s = sst_file_writer.Add(Key(k), range_val);
+          s = sst_file_writer.Put(Key(k), range_val);
           ASSERT_OK(s);
         }
         ExternalSstFileInfo file_info;
@@ -1356,7 +1356,7 @@ TEST_F(ExternalSSTFileTest, AddExternalSstFileWithCustomCompartor) {
     int range_end = i * 10;
     int range_start = range_end + 15;
     for (int k = (range_start - 1); k >= range_end; k--) {
-      ASSERT_OK(sst_file_writer.Add(Key(k), Key(k)));
+      ASSERT_OK(sst_file_writer.Put(Key(k), Key(k)));
     }
     ExternalSstFileInfo file_info;
     ASSERT_OK(sst_file_writer.Finish(&file_info));
@@ -1446,12 +1446,12 @@ TEST_F(ExternalSSTFileTest, SstFileWriterNonSharedKeys) {
 
   std::string suffix(100, 'X');
   ASSERT_OK(sst_file_writer.Open(file_path));
-  ASSERT_OK(sst_file_writer.Add("A" + suffix, "VAL"));
-  ASSERT_OK(sst_file_writer.Add("BB" + suffix, "VAL"));
-  ASSERT_OK(sst_file_writer.Add("CC" + suffix, "VAL"));
-  ASSERT_OK(sst_file_writer.Add("CXD" + suffix, "VAL"));
-  ASSERT_OK(sst_file_writer.Add("CZZZ" + suffix, "VAL"));
-  ASSERT_OK(sst_file_writer.Add("ZAAAX" + suffix, "VAL"));
+  ASSERT_OK(sst_file_writer.Put("A" + suffix, "VAL"));
+  ASSERT_OK(sst_file_writer.Put("BB" + suffix, "VAL"));
+  ASSERT_OK(sst_file_writer.Put("CC" + suffix, "VAL"));
+  ASSERT_OK(sst_file_writer.Put("CXD" + suffix, "VAL"));
+  ASSERT_OK(sst_file_writer.Put("CZZZ" + suffix, "VAL"));
+  ASSERT_OK(sst_file_writer.Put("ZAAAX" + suffix, "VAL"));
 
   ASSERT_OK(sst_file_writer.Finish());
   ASSERT_OK(DeprecatedAddFile({file_path}));
@@ -1746,22 +1746,22 @@ TEST_F(ExternalSSTFileTest, FileWithCFInfo) {
   // default_cf.sst
   const std::string cf_default_sst = sst_files_dir_ + "/default_cf.sst";
   ASSERT_OK(sfw_default.Open(cf_default_sst));
-  ASSERT_OK(sfw_default.Add("K1", "V1"));
-  ASSERT_OK(sfw_default.Add("K2", "V2"));
+  ASSERT_OK(sfw_default.Put("K1", "V1"));
+  ASSERT_OK(sfw_default.Put("K2", "V2"));
   ASSERT_OK(sfw_default.Finish());
 
   // cf1.sst
   const std::string cf1_sst = sst_files_dir_ + "/cf1.sst";
   ASSERT_OK(sfw_cf1.Open(cf1_sst));
-  ASSERT_OK(sfw_cf1.Add("K3", "V1"));
-  ASSERT_OK(sfw_cf1.Add("K4", "V2"));
+  ASSERT_OK(sfw_cf1.Put("K3", "V1"));
+  ASSERT_OK(sfw_cf1.Put("K4", "V2"));
   ASSERT_OK(sfw_cf1.Finish());
 
   // cf_unknown.sst
   const std::string unknown_sst = sst_files_dir_ + "/cf_unknown.sst";
   ASSERT_OK(sfw_unknown.Open(unknown_sst));
-  ASSERT_OK(sfw_unknown.Add("K5", "V1"));
-  ASSERT_OK(sfw_unknown.Add("K6", "V2"));
+  ASSERT_OK(sfw_unknown.Put("K5", "V1"));
+  ASSERT_OK(sfw_unknown.Put("K6", "V2"));
   ASSERT_OK(sfw_unknown.Finish());
 
   IngestExternalFileOptions ifo;
@@ -1862,7 +1862,7 @@ TEST_F(ExternalSSTFileTest, SnapshotInconsistencyBug) {
   SstFileWriter sst_file_writer(EnvOptions(), options);
   ASSERT_OK(sst_file_writer.Open(sst_file_path));
   for (int i = 0; i < kNumKeys; i++) {
-    ASSERT_OK(sst_file_writer.Add(Key(i), Key(i) + "_V2"));
+    ASSERT_OK(sst_file_writer.Put(Key(i), Key(i) + "_V2"));
   }
   ASSERT_OK(sst_file_writer.Finish());
 

db/fault_injection_test.cc

@@ -228,10 +228,12 @@ class FaultInjectionTest : public testing::Test,
     return Status::OK();
   }
 
-#if __clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ >= 9)
-__attribute__((__no_sanitize__("undefined")))
-#elif __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 9)
+#ifdef ROCKSDB_UBSAN_RUN
+#if defined(__clang__)
+__attribute__((__no_sanitize__("shift"), no_sanitize("signed-integer-overflow")))
+#elif defined(__GNUC__)
 __attribute__((__no_sanitize_undefined__))
+#endif
 #endif
   // Return the ith key
   Slice Key(int i, std::string* storage) const {

db/internal_stats.cc

@@ -1137,6 +1137,7 @@ void InternalStats::DumpCFStatsNoFileHistogram(std::string* value) {
            total_stall_count - cf_stats_snapshot_.stall_count);
   value->append(buf);
 
+  cf_stats_snapshot_.seconds_up = seconds_up;
   cf_stats_snapshot_.ingest_bytes_flush = flush_ingest;
   cf_stats_snapshot_.ingest_bytes_addfile = add_file_ingest;
   cf_stats_snapshot_.ingest_files_addfile = ingest_files_addfile;

db/malloc_stats.cc

@@ -0,0 +1,52 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "db/malloc_stats.h"
+
+#ifndef ROCKSDB_LITE
+#include <memory>
+#include <string.h>
+
+namespace rocksdb {
+
+#ifdef ROCKSDB_JEMALLOC
+#include "jemalloc/jemalloc.h"
+
+typedef struct {
+  char* cur;
+  char* end;
+} MallocStatus;
+
+static void GetJemallocStatus(void* mstat_arg, const char* status) {
+  MallocStatus* mstat = reinterpret_cast<MallocStatus*>(mstat_arg);
+  size_t status_len = status ? strlen(status) : 0;
+  size_t buf_size = (size_t)(mstat->end - mstat->cur);
+  if (!status_len || status_len > buf_size) {
+    return;
+  }
+
+  snprintf(mstat->cur, buf_size, "%s", status);
+  mstat->cur += status_len;
+}
+#endif  // ROCKSDB_JEMALLOC
+
+void DumpMallocStats(std::string* stats) {
+#ifdef ROCKSDB_JEMALLOC
+  MallocStatus mstat;
+  const unsigned int kMallocStatusLen = 1000000;
+  std::unique_ptr<char[]> buf{new char[kMallocStatusLen + 1]};
+  mstat.cur = buf.get();
+  mstat.end = buf.get() + kMallocStatusLen;
+  je_malloc_stats_print(GetJemallocStatus, &mstat, "");
+  stats->append(buf.get());
+#endif  // ROCKSDB_JEMALLOC
+}
+
+}
+#endif  // !ROCKSDB_LITE

db/malloc_stats.h

@@ -0,0 +1,22 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#pragma once
+
+#ifndef ROCKSDB_LITE
+
+#include <string>
+
+namespace rocksdb {
+
+void DumpMallocStats(std::string*);
+
+}
+
+#endif  // !ROCKSDB_LITE

db/memtable.cc

@@ -11,7 +11,6 @@
 
 #include <memory>
 #include <algorithm>
-#include <limits>
 
 #include "db/dbformat.h"
 #include "db/merge_context.h"
@@ -35,7 +34,6 @@
 #include "util/memory_usage.h"
 #include "util/murmurhash.h"
 #include "util/mutexlock.h"
-#include "util/stop_watch.h"
 
 namespace rocksdb {
 
@@ -61,20 +59,24 @@ MemTable::MemTable(const InternalKeyComparator& cmp,
                    const ImmutableCFOptions& ioptions,
                    const MutableCFOptions& mutable_cf_options,
                    WriteBufferManager* write_buffer_manager,
-                   SequenceNumber latest_seq)
+                   SequenceNumber latest_seq, uint32_t column_family_id)
     : comparator_(cmp),
       moptions_(ioptions, mutable_cf_options),
       refs_(0),
       kArenaBlockSize(OptimizeBlockSize(moptions_.arena_block_size)),
-      arena_(moptions_.arena_block_size,
-             mutable_cf_options.memtable_huge_page_size),
-      allocator_(&arena_, write_buffer_manager),
+      mem_tracker_(write_buffer_manager),
+      arena_(
+          moptions_.arena_block_size,
+          (write_buffer_manager != nullptr && write_buffer_manager->enabled())
+              ? &mem_tracker_
+              : nullptr,
+          mutable_cf_options.memtable_huge_page_size),
       table_(ioptions.memtable_factory->CreateMemTableRep(
-          comparator_, &allocator_, ioptions.prefix_extractor,
-          ioptions.info_log)),
+          comparator_, &arena_, ioptions.prefix_extractor, ioptions.info_log,
+          column_family_id)),
       range_del_table_(SkipListFactory().CreateMemTableRep(
-          comparator_, &allocator_, nullptr /* transform */,
-          ioptions.info_log)),
+          comparator_, &arena_, nullptr /* transform */, ioptions.info_log,
+          column_family_id)),
       is_range_del_table_empty_(true),
       data_size_(0),
       num_entries_(0),
@@ -101,13 +103,16 @@ MemTable::MemTable(const InternalKeyComparator& cmp,
 
   if (prefix_extractor_ && moptions_.memtable_prefix_bloom_bits > 0) {
     prefix_bloom_.reset(new DynamicBloom(
-        &allocator_, moptions_.memtable_prefix_bloom_bits,
-        ioptions.bloom_locality, 6 /* hard coded 6 probes */, nullptr,
-        moptions_.memtable_huge_page_size, ioptions.info_log));
+        &arena_, moptions_.memtable_prefix_bloom_bits, ioptions.bloom_locality,
+        6 /* hard coded 6 probes */, nullptr, moptions_.memtable_huge_page_size,
+        ioptions.info_log));
   }
 }
 
-MemTable::~MemTable() { assert(refs_ == 0); }
+MemTable::~MemTable() {
+  mem_tracker_.FreeMem();
+  assert(refs_ == 0);
+}
 
 size_t MemTable::ApproximateMemoryUsage() {
   autovector<size_t> usages = {arena_.ApproximateMemoryUsage(),
@@ -576,7 +581,7 @@ static bool SaveValue(void* arg, const char* entry) {
           *(s->status) = MergeHelper::TimedFullMerge(
               merge_operator, s->key->user_key(), &v,
               merge_context->GetOperands(), s->value, s->logger, s->statistics,
-              s->env_);
+              s->env_, nullptr /* result_operand */, true);
         } else if (s->value != nullptr) {
           s->value->assign(v.data(), v.size());
         }
@@ -593,7 +598,7 @@ static bool SaveValue(void* arg, const char* entry) {
           *(s->status) = MergeHelper::TimedFullMerge(
               merge_operator, s->key->user_key(), nullptr,
               merge_context->GetOperands(), s->value, s->logger, s->statistics,
-              s->env_);
+              s->env_, nullptr /* result_operand */, true);
         } else {
           *(s->status) = Status::NotFound();
         }

db/memtable.h

@@ -18,12 +18,12 @@
 #include "db/dbformat.h"
 #include "db/range_del_aggregator.h"
 #include "db/version_edit.h"
-#include "memtable/memtable_allocator.h"
 #include "monitoring/instrumented_mutex.h"
 #include "options/cf_options.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "rocksdb/memtablerep.h"
+#include "util/allocator.h"
 #include "util/concurrent_arena.h"
 #include "util/dynamic_bloom.h"
 #include "util/hash.h"
@@ -101,7 +101,7 @@ class MemTable {
                     const ImmutableCFOptions& ioptions,
                     const MutableCFOptions& mutable_cf_options,
                     WriteBufferManager* write_buffer_manager,
-                    SequenceNumber earliest_seq);
+                    SequenceNumber earliest_seq, uint32_t column_family_id);
 
   // Do not delete this MemTable unless Unref() indicates it not in use.
   ~MemTable();
@@ -317,7 +317,7 @@ class MemTable {
   // write anything to this MemTable().  (Ie. do not call Add() or Update()).
   void MarkImmutable() {
     table_->MarkReadOnly();
-    allocator_.DoneAllocating();
+    mem_tracker_.DoneAllocating();
   }
 
   // return true if the current MemTableRep supports merge operator.
@@ -359,8 +359,8 @@ class MemTable {
   const MemTableOptions moptions_;
   int refs_;
   const size_t kArenaBlockSize;
+  AllocTracker mem_tracker_;
   ConcurrentArena arena_;
-  MemTableAllocator allocator_;
   unique_ptr<MemTableRep> table_;
   unique_ptr<MemTableRep> range_del_table_;
   bool is_range_del_table_empty_;

db/memtable_list.cc

@@ -166,6 +166,18 @@ Status MemTableListVersion::AddRangeTombstoneIterators(
   return Status::OK();
 }
 
+Status MemTableListVersion::AddRangeTombstoneIterators(
+    const ReadOptions& read_opts,
+    std::vector<InternalIterator*>* range_del_iters) {
+  for (auto& m : memlist_) {
+    auto* range_del_iter = m->NewRangeTombstoneIterator(read_opts);
+    if (range_del_iter != nullptr) {
+      range_del_iters->push_back(range_del_iter);
+    }
+  }
+  return Status::OK();
+}
+
 void MemTableListVersion::AddIterators(
     const ReadOptions& options, std::vector<InternalIterator*>* iterator_list,
     Arena* arena) {

db/memtable_list.h

@@ -82,6 +82,9 @@ class MemTableListVersion {
 
   Status AddRangeTombstoneIterators(const ReadOptions& read_opts, Arena* arena,
                                     RangeDelAggregator* range_del_agg);
+  Status AddRangeTombstoneIterators(
+      const ReadOptions& read_opts,
+      std::vector<InternalIterator*>* range_del_iters);
 
   void AddIterators(const ReadOptions& options,
                     std::vector<InternalIterator*>* iterator_list,

db/memtable_list_test.cc

@@ -134,7 +134,7 @@ TEST_F(MemTableListTest, GetTest) {
 
   WriteBufferManager wb(options.db_write_buffer_size);
   MemTable* mem = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb,
-                               kMaxSequenceNumber);
+                               kMaxSequenceNumber, 0 /* column_family_id */);
   mem->Ref();
 
   // Write some keys to this memtable.
@@ -173,7 +173,7 @@ TEST_F(MemTableListTest, GetTest) {
   // Create another memtable and write some keys to it
   WriteBufferManager wb2(options.db_write_buffer_size);
   MemTable* mem2 = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb2,
-                                kMaxSequenceNumber);
+                                kMaxSequenceNumber, 0 /* column_family_id */);
   mem2->Ref();
 
   mem2->Add(++seq, kTypeDeletion, "key1", "");
@@ -241,7 +241,7 @@ TEST_F(MemTableListTest, GetFromHistoryTest) {
 
   WriteBufferManager wb(options.db_write_buffer_size);
   MemTable* mem = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb,
-                               kMaxSequenceNumber);
+                               kMaxSequenceNumber, 0 /* column_family_id */);
   mem->Ref();
 
   // Write some keys to this memtable.
@@ -319,7 +319,7 @@ TEST_F(MemTableListTest, GetFromHistoryTest) {
   // Create another memtable and write some keys to it
   WriteBufferManager wb2(options.db_write_buffer_size);
   MemTable* mem2 = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb2,
-                                kMaxSequenceNumber);
+                                kMaxSequenceNumber, 0 /* column_family_id */);
   mem2->Ref();
 
   mem2->Add(++seq, kTypeDeletion, "key1", "");
@@ -344,7 +344,7 @@ TEST_F(MemTableListTest, GetFromHistoryTest) {
   // Add a third memtable to push the first memtable out of the history
   WriteBufferManager wb3(options.db_write_buffer_size);
   MemTable* mem3 = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb3,
-                                kMaxSequenceNumber);
+                                kMaxSequenceNumber, 0 /* column_family_id */);
   mem3->Ref();
   list.Add(mem3, &to_delete);
   ASSERT_EQ(1, list.NumNotFlushed());
@@ -418,7 +418,7 @@ TEST_F(MemTableListTest, FlushPendingTest) {
   MutableCFOptions mutable_cf_options(options);
   for (int i = 0; i < num_tables; i++) {
     MemTable* mem = new MemTable(cmp, ioptions, mutable_cf_options, &wb,
-                                 kMaxSequenceNumber);
+                                 kMaxSequenceNumber, 0 /* column_family_id */);
     mem->Ref();
 
     std::string value;

db/merge_helper.cc

@@ -23,7 +23,8 @@ Status MergeHelper::TimedFullMerge(const MergeOperator* merge_operator,
                                    const std::vector<Slice>& operands,
                                    std::string* result, Logger* logger,
                                    Statistics* statistics, Env* env,
-                                   Slice* result_operand) {
+                                   Slice* result_operand,
+                                   bool update_num_ops_stats) {
   assert(merge_operator != nullptr);
 
   if (operands.size() == 0) {
@@ -32,6 +33,11 @@ Status MergeHelper::TimedFullMerge(const MergeOperator* merge_operator,
     return Status::OK();
   }
 
+  if (update_num_ops_stats) {
+    MeasureTime(statistics, READ_NUM_MERGE_OPERANDS,
+                static_cast<uint64_t>(operands.size()));
+  }
+
   bool success;
   Slice tmp_result_operand(nullptr, 0);
   const MergeOperator::MergeOperationInput merge_in(key, value, operands,

db/merge_helper.h

@@ -54,6 +54,8 @@ class MergeHelper {
   // Wrapper around MergeOperator::FullMergeV2() that records perf statistics.
   // Result of merge will be written to result if status returned is OK.
   // If operands is empty, the value will simply be copied to result.
+  // Set `update_num_ops_stats` to true if it is from a user read, so that
+  // the latency is sensitive.
   // Returns one of the following statuses:
   // - OK: Entries were successfully merged.
   // - Corruption: Merge operator reported unsuccessful merge.
@@ -62,7 +64,8 @@ class MergeHelper {
                                const std::vector<Slice>& operands,
                                std::string* result, Logger* logger,
                                Statistics* statistics, Env* env,
-                               Slice* result_operand = nullptr);
+                               Slice* result_operand = nullptr,
+                               bool update_num_ops_stats = false);
 
   // Merge entries until we hit
   //     - a corrupted key

db/perf_context_test.cc

@@ -87,13 +87,13 @@ TEST_F(PerfContextTest, SeekIntoDeletion) {
     std::string key = "k" + ToString(i);
     std::string value;
 
-    perf_context.Reset();
+    get_perf_context()->Reset();
     StopWatchNano timer(Env::Default());
     timer.Start();
     auto status = db->Get(read_options, key, &value);
     auto elapsed_nanos = timer.ElapsedNanos();
     ASSERT_TRUE(status.IsNotFound());
-    hist_get.Add(perf_context.user_key_comparison_count);
+    hist_get.Add(get_perf_context()->user_key_comparison_count);
     hist_get_time.Add(elapsed_nanos);
   }
 
@@ -106,19 +106,19 @@ TEST_F(PerfContextTest, SeekIntoDeletion) {
     HistogramImpl hist_seek_to_first;
     std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
 
-    perf_context.Reset();
+    get_perf_context()->Reset();
     StopWatchNano timer(Env::Default(), true);
     iter->SeekToFirst();
-    hist_seek_to_first.Add(perf_context.user_key_comparison_count);
+    hist_seek_to_first.Add(get_perf_context()->user_key_comparison_count);
     auto elapsed_nanos = timer.ElapsedNanos();
 
     if (FLAGS_verbose) {
       std::cout << "SeekToFirst uesr key comparison: \n"
                 << hist_seek_to_first.ToString()
-                << "ikey skipped: " << perf_context.internal_key_skipped_count
+                << "ikey skipped: " << get_perf_context()->internal_key_skipped_count
                 << "\n"
                 << "idelete skipped: "
-                << perf_context.internal_delete_skipped_count << "\n"
+                << get_perf_context()->internal_delete_skipped_count << "\n"
                 << "elapsed: " << elapsed_nanos << "\n";
     }
   }
@@ -128,26 +128,26 @@ TEST_F(PerfContextTest, SeekIntoDeletion) {
     std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
     std::string key = "k" + ToString(i);
 
-    perf_context.Reset();
+    get_perf_context()->Reset();
     StopWatchNano timer(Env::Default(), true);
     iter->Seek(key);
     auto elapsed_nanos = timer.ElapsedNanos();
-    hist_seek.Add(perf_context.user_key_comparison_count);
+    hist_seek.Add(get_perf_context()->user_key_comparison_count);
     if (FLAGS_verbose) {
-      std::cout << "seek cmp: " << perf_context.user_key_comparison_count
-                << " ikey skipped " << perf_context.internal_key_skipped_count
+      std::cout << "seek cmp: " << get_perf_context()->user_key_comparison_count
+                << " ikey skipped " << get_perf_context()->internal_key_skipped_count
                 << " idelete skipped "
-                << perf_context.internal_delete_skipped_count
+                << get_perf_context()->internal_delete_skipped_count
                 << " elapsed: " << elapsed_nanos << "ns\n";
     }
 
-    perf_context.Reset();
+    get_perf_context()->Reset();
     ASSERT_TRUE(iter->Valid());
     StopWatchNano timer2(Env::Default(), true);
     iter->Next();
     auto elapsed_nanos2 = timer2.ElapsedNanos();
     if (FLAGS_verbose) {
-      std::cout << "next cmp: " << perf_context.user_key_comparison_count
+      std::cout << "next cmp: " << get_perf_context()->user_key_comparison_count
                 << "elapsed: " << elapsed_nanos2 << "ns\n";
     }
   }
@@ -263,18 +263,18 @@ void ProfileQueries(bool enabled_time = false) {
 
     std::vector<std::string> values;
 
-    perf_context.Reset();
+    get_perf_context()->Reset();
     db->Put(write_options, key, value);
     if (++num_mutex_waited > 3) {
 #ifndef NDEBUG
       ThreadStatusUtil::TEST_SetStateDelay(ThreadStatus::STATE_MUTEX_WAIT, 0U);
 #endif
     }
-    hist_write_pre_post.Add(perf_context.write_pre_and_post_process_time);
-    hist_write_wal_time.Add(perf_context.write_wal_time);
-    hist_write_memtable_time.Add(perf_context.write_memtable_time);
-    hist_put.Add(perf_context.user_key_comparison_count);
-    total_db_mutex_nanos += perf_context.db_mutex_lock_nanos;
+    hist_write_pre_post.Add(get_perf_context()->write_pre_and_post_process_time);
+    hist_write_wal_time.Add(get_perf_context()->write_wal_time);
+    hist_write_memtable_time.Add(get_perf_context()->write_memtable_time);
+    hist_put.Add(get_perf_context()->user_key_comparison_count);
+    total_db_mutex_nanos += get_perf_context()->db_mutex_lock_nanos;
   }
 #ifndef NDEBUG
   ThreadStatusUtil::TEST_SetStateDelay(ThreadStatus::STATE_MUTEX_WAIT, 0U);
@@ -291,24 +291,24 @@ void ProfileQueries(bool enabled_time = false) {
     std::vector<Slice> multiget_keys = {Slice(key)};
     std::vector<std::string> values;
 
-    perf_context.Reset();
+    get_perf_context()->Reset();
     ASSERT_OK(db->Get(read_options, key, &value));
     ASSERT_EQ(expected_value, value);
-    hist_get_snapshot.Add(perf_context.get_snapshot_time);
-    hist_get_memtable.Add(perf_context.get_from_memtable_time);
-    hist_get_files.Add(perf_context.get_from_output_files_time);
-    hist_num_memtable_checked.Add(perf_context.get_from_memtable_count);
-    hist_get_post_process.Add(perf_context.get_post_process_time);
-    hist_get.Add(perf_context.user_key_comparison_count);
+    hist_get_snapshot.Add(get_perf_context()->get_snapshot_time);
+    hist_get_memtable.Add(get_perf_context()->get_from_memtable_time);
+    hist_get_files.Add(get_perf_context()->get_from_output_files_time);
+    hist_num_memtable_checked.Add(get_perf_context()->get_from_memtable_count);
+    hist_get_post_process.Add(get_perf_context()->get_post_process_time);
+    hist_get.Add(get_perf_context()->user_key_comparison_count);
 
-    perf_context.Reset();
+    get_perf_context()->Reset();
     db->MultiGet(read_options, multiget_keys, &values);
-    hist_mget_snapshot.Add(perf_context.get_snapshot_time);
-    hist_mget_memtable.Add(perf_context.get_from_memtable_time);
-    hist_mget_files.Add(perf_context.get_from_output_files_time);
-    hist_mget_num_memtable_checked.Add(perf_context.get_from_memtable_count);
-    hist_mget_post_process.Add(perf_context.get_post_process_time);
-    hist_mget.Add(perf_context.user_key_comparison_count);
+    hist_mget_snapshot.Add(get_perf_context()->get_snapshot_time);
+    hist_mget_memtable.Add(get_perf_context()->get_from_memtable_time);
+    hist_mget_files.Add(get_perf_context()->get_from_output_files_time);
+    hist_mget_num_memtable_checked.Add(get_perf_context()->get_from_memtable_count);
+    hist_mget_post_process.Add(get_perf_context()->get_post_process_time);
+    hist_mget.Add(get_perf_context()->user_key_comparison_count);
   }
 
   if (FLAGS_verbose) {
@@ -392,24 +392,24 @@ void ProfileQueries(bool enabled_time = false) {
     std::vector<Slice> multiget_keys = {Slice(key)};
     std::vector<std::string> values;
 
-    perf_context.Reset();
+    get_perf_context()->Reset();
     ASSERT_OK(db->Get(read_options, key, &value));
     ASSERT_EQ(expected_value, value);
-    hist_get_snapshot.Add(perf_context.get_snapshot_time);
-    hist_get_memtable.Add(perf_context.get_from_memtable_time);
-    hist_get_files.Add(perf_context.get_from_output_files_time);
-    hist_num_memtable_checked.Add(perf_context.get_from_memtable_count);
-    hist_get_post_process.Add(perf_context.get_post_process_time);
-    hist_get.Add(perf_context.user_key_comparison_count);
+    hist_get_snapshot.Add(get_perf_context()->get_snapshot_time);
+    hist_get_memtable.Add(get_perf_context()->get_from_memtable_time);
+    hist_get_files.Add(get_perf_context()->get_from_output_files_time);
+    hist_num_memtable_checked.Add(get_perf_context()->get_from_memtable_count);
+    hist_get_post_process.Add(get_perf_context()->get_post_process_time);
+    hist_get.Add(get_perf_context()->user_key_comparison_count);
 
-    perf_context.Reset();
+    get_perf_context()->Reset();
     db->MultiGet(read_options, multiget_keys, &values);
-    hist_mget_snapshot.Add(perf_context.get_snapshot_time);
-    hist_mget_memtable.Add(perf_context.get_from_memtable_time);
-    hist_mget_files.Add(perf_context.get_from_output_files_time);
-    hist_mget_num_memtable_checked.Add(perf_context.get_from_memtable_count);
-    hist_mget_post_process.Add(perf_context.get_post_process_time);
-    hist_mget.Add(perf_context.user_key_comparison_count);
+    hist_mget_snapshot.Add(get_perf_context()->get_snapshot_time);
+    hist_mget_memtable.Add(get_perf_context()->get_from_memtable_time);
+    hist_mget_files.Add(get_perf_context()->get_from_output_files_time);
+    hist_mget_num_memtable_checked.Add(get_perf_context()->get_from_memtable_count);
+    hist_mget_post_process.Add(get_perf_context()->get_post_process_time);
+    hist_mget.Add(get_perf_context()->user_key_comparison_count);
   }
 
   if (FLAGS_verbose) {
@@ -512,13 +512,13 @@ TEST_F(PerfContextTest, SeekKeyComparison) {
     std::string key = "k" + ToString(i);
     std::string value = "v" + ToString(i);
 
-    perf_context.Reset();
+    get_perf_context()->Reset();
     timer.Start();
     db->Put(write_options, key, value);
     auto put_time = timer.ElapsedNanos();
     hist_put_time.Add(put_time);
-    hist_wal_time.Add(perf_context.write_wal_time);
-    hist_time_diff.Add(put_time - perf_context.write_wal_time);
+    hist_wal_time.Add(get_perf_context()->write_wal_time);
+    hist_time_diff.Add(put_time - get_perf_context()->write_wal_time);
   }
 
   if (FLAGS_verbose) {
@@ -535,18 +535,18 @@ TEST_F(PerfContextTest, SeekKeyComparison) {
     std::string value = "v" + ToString(i);
 
     std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
-    perf_context.Reset();
+    get_perf_context()->Reset();
     iter->Seek(key);
     ASSERT_TRUE(iter->Valid());
     ASSERT_EQ(iter->value().ToString(), value);
-    hist_seek.Add(perf_context.user_key_comparison_count);
+    hist_seek.Add(get_perf_context()->user_key_comparison_count);
   }
 
   std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
   for (iter->SeekToFirst(); iter->Valid();) {
-    perf_context.Reset();
+    get_perf_context()->Reset();
     iter->Next();
-    hist_next.Add(perf_context.user_key_comparison_count);
+    hist_next.Add(get_perf_context()->user_key_comparison_count);
   }
 
   if (FLAGS_verbose) {
@@ -564,16 +564,16 @@ TEST_F(PerfContextTest, DBMutexLockCounter) {
     mutex.Lock();
     rocksdb::port::Thread child_thread([&] {
       SetPerfLevel(perf_level);
-      perf_context.Reset();
-      ASSERT_EQ(perf_context.db_mutex_lock_nanos, 0);
+      get_perf_context()->Reset();
+      ASSERT_EQ(get_perf_context()->db_mutex_lock_nanos, 0);
       mutex.Lock();
       mutex.Unlock();
       if (perf_level == PerfLevel::kEnableTimeExceptForMutex ||
           stats_code[c] != DB_MUTEX_WAIT_MICROS) {
-        ASSERT_EQ(perf_context.db_mutex_lock_nanos, 0);
+        ASSERT_EQ(get_perf_context()->db_mutex_lock_nanos, 0);
       } else {
         // increment the counter only when it's a DB Mutex
-        ASSERT_GT(perf_context.db_mutex_lock_nanos, 0);
+        ASSERT_GT(get_perf_context()->db_mutex_lock_nanos, 0);
       }
     });
     Env::Default()->SleepForMicroseconds(100);
@@ -589,28 +589,28 @@ TEST_F(PerfContextTest, FalseDBMutexWait) {
   for (int c = 0; c < 2; ++c) {
     InstrumentedMutex mutex(nullptr, Env::Default(), stats_code[c]);
     InstrumentedCondVar lock(&mutex);
-    perf_context.Reset();
+    get_perf_context()->Reset();
     mutex.Lock();
     lock.TimedWait(100);
     mutex.Unlock();
     if (stats_code[c] == static_cast<int>(DB_MUTEX_WAIT_MICROS)) {
       // increment the counter only when it's a DB Mutex
-      ASSERT_GT(perf_context.db_condition_wait_nanos, 0);
+      ASSERT_GT(get_perf_context()->db_condition_wait_nanos, 0);
     } else {
-      ASSERT_EQ(perf_context.db_condition_wait_nanos, 0);
+      ASSERT_EQ(get_perf_context()->db_condition_wait_nanos, 0);
     }
   }
 }
 
 TEST_F(PerfContextTest, ToString) {
-  perf_context.Reset();
-  perf_context.block_read_count = 12345;
+  get_perf_context()->Reset();
+  get_perf_context()->block_read_count = 12345;
 
-  std::string zero_included = perf_context.ToString();
+  std::string zero_included = get_perf_context()->ToString();
   ASSERT_NE(std::string::npos, zero_included.find("= 0"));
   ASSERT_NE(std::string::npos, zero_included.find("= 12345"));
 
-  std::string zero_excluded = perf_context.ToString(true);
+  std::string zero_excluded = get_perf_context()->ToString(true);
   ASSERT_EQ(std::string::npos, zero_excluded.find("= 0"));
   ASSERT_NE(std::string::npos, zero_excluded.find("= 12345"));
 }
@@ -631,36 +631,36 @@ TEST_F(PerfContextTest, MergeOperatorTime) {
   ASSERT_OK(db->Merge(WriteOptions(), "k1", "val4"));
 
   SetPerfLevel(kEnableTime);
-  perf_context.Reset();
+  get_perf_context()->Reset();
   ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
 #ifdef OS_SOLARIS
   for (int i = 0; i < 100; i++) {
     ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
   }
 #endif
-  EXPECT_GT(perf_context.merge_operator_time_nanos, 0);
+  EXPECT_GT(get_perf_context()->merge_operator_time_nanos, 0);
 
   ASSERT_OK(db->Flush(FlushOptions()));
 
-  perf_context.Reset();
+  get_perf_context()->Reset();
   ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
 #ifdef OS_SOLARIS
   for (int i = 0; i < 100; i++) {
     ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
   }
 #endif
-  EXPECT_GT(perf_context.merge_operator_time_nanos, 0);
+  EXPECT_GT(get_perf_context()->merge_operator_time_nanos, 0);
 
   ASSERT_OK(db->CompactRange(CompactRangeOptions(), nullptr, nullptr));
 
-  perf_context.Reset();
+  get_perf_context()->Reset();
   ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
 #ifdef OS_SOLARIS
   for (int i = 0; i < 100; i++) {
     ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
   }
 #endif
-  EXPECT_GT(perf_context.merge_operator_time_nanos, 0);
+  EXPECT_GT(get_perf_context()->merge_operator_time_nanos, 0);
 
   delete db;
 }

db/prefix_test.cc

@@ -603,11 +603,11 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
         Slice key = TestKeyToSlice(s, test_key);
         std::string value(FLAGS_value_size, 0);
 
-        perf_context.Reset();
+        get_perf_context()->Reset();
         StopWatchNano timer(Env::Default(), true);
         ASSERT_OK(db->Put(write_options, key, value));
         hist_put_time.Add(timer.ElapsedNanos());
-        hist_put_comparison.Add(perf_context.user_key_comparison_count);
+        hist_put_comparison.Add(get_perf_context()->user_key_comparison_count);
       }
     }
 
@@ -626,7 +626,7 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
       Slice key = TestKeyToSlice(s, test_key);
       std::string value = "v" + ToString(0);
 
-      perf_context.Reset();
+      get_perf_context()->Reset();
       StopWatchNano timer(Env::Default(), true);
       auto key_prefix = options.prefix_extractor->Transform(key);
       uint64_t total_keys = 0;
@@ -640,7 +640,7 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
         total_keys++;
       }
       hist_seek_time.Add(timer.ElapsedNanos());
-      hist_seek_comparison.Add(perf_context.user_key_comparison_count);
+      hist_seek_comparison.Add(get_perf_context()->user_key_comparison_count);
       ASSERT_EQ(total_keys, FLAGS_items_per_prefix - FLAGS_items_per_prefix/2);
     }
 
@@ -660,11 +660,11 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
       std::string s;
       Slice key = TestKeyToSlice(s, test_key);
 
-      perf_context.Reset();
+      get_perf_context()->Reset();
       StopWatchNano timer(Env::Default(), true);
       iter->Seek(key);
       hist_no_seek_time.Add(timer.ElapsedNanos());
-      hist_no_seek_comparison.Add(perf_context.user_key_comparison_count);
+      hist_no_seek_comparison.Add(get_perf_context()->user_key_comparison_count);
       ASSERT_TRUE(!iter->Valid());
     }
 

db/table_cache.cc

@@ -260,6 +260,44 @@ InternalIterator* TableCache::NewIterator(
   return result;
 }
 
+InternalIterator* TableCache::NewRangeTombstoneIterator(
+    const ReadOptions& options, const EnvOptions& env_options,
+    const InternalKeyComparator& icomparator, const FileDescriptor& fd,
+    HistogramImpl* file_read_hist, bool skip_filters, int level) {
+  Status s;
+  TableReader* table_reader = nullptr;
+  Cache::Handle* handle = nullptr;
+  table_reader = fd.table_reader;
+  if (table_reader == nullptr) {
+    s = FindTable(env_options, icomparator, fd, &handle,
+                  options.read_tier == kBlockCacheTier /* no_io */,
+                  true /* record read_stats */, file_read_hist, skip_filters,
+                  level);
+    if (s.ok()) {
+      table_reader = GetTableReaderFromHandle(handle);
+    }
+  }
+  InternalIterator* result = nullptr;
+  if (s.ok()) {
+    result = table_reader->NewRangeTombstoneIterator(options);
+    if (result != nullptr) {
+      if (handle != nullptr) {
+        result->RegisterCleanup(&UnrefEntry, cache_, handle);
+      }
+    }
+  }
+  if (result == nullptr && handle != nullptr) {
+    // the range deletion block didn't exist, or there was a failure between
+    // getting handle and getting iterator.
+    ReleaseHandle(handle);
+  }
+  if (!s.ok()) {
+    assert(result == nullptr);
+    result = NewErrorInternalIterator(s);
+  }
+  return result;
+}
+
 Status TableCache::Get(const ReadOptions& options,
                        const InternalKeyComparator& internal_comparator,
                        const FileDescriptor& fd, const Slice& k,

db/table_cache.h

@@ -58,6 +58,12 @@ class TableCache {
       HistogramImpl* file_read_hist = nullptr, bool for_compaction = false,
       Arena* arena = nullptr, bool skip_filters = false, int level = -1);
 
+  InternalIterator* NewRangeTombstoneIterator(
+      const ReadOptions& options, const EnvOptions& toptions,
+      const InternalKeyComparator& internal_comparator,
+      const FileDescriptor& file_fd, HistogramImpl* file_read_hist,
+      bool skip_filters, int level);
+
   // If a seek to internal key "k" in specified file finds an entry,
   // call (*handle_result)(arg, found_key, found_value) repeatedly until
   // it returns false.

db/version_set.cc

@@ -134,8 +134,8 @@ class FilePicker {
 
         // Do key range filtering of files or/and fractional cascading if:
         // (1) not all the files are in level 0, or
-        // (2) there are more than 3 Level 0 files
-        // If there are only 3 or less level 0 files in the system, we skip
+        // (2) there are more than 3 current level files
+        // If there are only 3 or less current level files in the system, we skip
         // the key range filtering. In this case, more likely, the system is
         // highly tuned to minimize number of tables queried by each query,
         // so it is unlikely that key range filtering is more efficient than
@@ -863,6 +863,22 @@ void Version::AddIteratorsForLevel(const ReadOptions& read_options,
   }
 }
 
+void Version::AddRangeDelIteratorsForLevel(
+    const ReadOptions& read_options, const EnvOptions& soptions, int level,
+    std::vector<InternalIterator*>* range_del_iters) {
+  range_del_iters->clear();
+  for (size_t i = 0; i < storage_info_.LevelFilesBrief(level).num_files; i++) {
+    const auto& file = storage_info_.LevelFilesBrief(level).files[i];
+    auto* range_del_iter = cfd_->table_cache()->NewRangeTombstoneIterator(
+        read_options, soptions, cfd_->internal_comparator(), file.fd,
+        cfd_->internal_stats()->GetFileReadHist(level),
+        false /* skip_filters */, level);
+    if (range_del_iter != nullptr) {
+      range_del_iters->push_back(range_del_iter);
+    }
+  }
+}
+
 VersionStorageInfo::VersionStorageInfo(
     const InternalKeyComparator* internal_comparator,
     const Comparator* user_comparator, int levels,
@@ -1014,7 +1030,8 @@ void Version::Get(const ReadOptions& read_options, const LookupKey& k,
     std::string* str_value = value != nullptr ? value->GetSelf() : nullptr;
     *status = MergeHelper::TimedFullMerge(
         merge_operator_, user_key, nullptr, merge_context->GetOperands(),
-        str_value, info_log_, db_statistics_, env_);
+        str_value, info_log_, db_statistics_, env_,
+        nullptr /* result_operand */, true);
     if (LIKELY(value != nullptr)) {
       value->PinSelf();
     }
@@ -1214,15 +1231,18 @@ void VersionStorageInfo::EstimateCompactionBytesNeeded(
   // accumulated bytes.
 
   uint64_t bytes_compact_to_next_level = 0;
+  uint64_t level_size = 0;
+  for (auto* f : files_[0]) {
+    level_size += f->fd.GetFileSize();
+  }
   // Level 0
   bool level0_compact_triggered = false;
-  if (static_cast<int>(files_[0].size()) >
-      mutable_cf_options.level0_file_num_compaction_trigger) {
+  if (static_cast<int>(files_[0].size()) >=
+          mutable_cf_options.level0_file_num_compaction_trigger ||
+      level_size >= mutable_cf_options.max_bytes_for_level_base) {
     level0_compact_triggered = true;
-    for (auto* f : files_[0]) {
-      bytes_compact_to_next_level += f->fd.GetFileSize();
-    }
-    estimated_compaction_needed_bytes_ = bytes_compact_to_next_level;
+    estimated_compaction_needed_bytes_ = level_size;
+    bytes_compact_to_next_level = level_size;
   } else {
     estimated_compaction_needed_bytes_ = 0;
   }
@@ -1230,7 +1250,7 @@ void VersionStorageInfo::EstimateCompactionBytesNeeded(
   // Level 1 and up.
   uint64_t bytes_next_level = 0;
   for (int level = base_level(); level <= MaxInputLevel(); level++) {
-    uint64_t level_size = 0;
+    level_size = 0;
     if (bytes_next_level > 0) {
 #ifndef NDEBUG
       uint64_t level_size2 = 0;
@@ -2669,6 +2689,9 @@ Status VersionSet::Recover(
   default_cf_edit.SetColumnFamily(0);
   ColumnFamilyData* default_cfd =
       CreateColumnFamily(default_cf_iter->second, &default_cf_edit);
+  // In recovery, nobody else can access it, so it's fine to set it to be
+  // initialized earlier.
+  default_cfd->set_initialized();
   builders.insert({0, new BaseReferencedVersionBuilder(default_cfd)});
 
   {
@@ -2715,6 +2738,7 @@ Status VersionSet::Recover(
               {edit.column_family_, edit.column_family_name_});
         } else {
           cfd = CreateColumnFamily(cf_options->second, &edit);
+          cfd->set_initialized();
           builders.insert(
               {edit.column_family_, new BaseReferencedVersionBuilder(cfd)});
         }
@@ -2842,6 +2866,7 @@ Status VersionSet::Recover(
       if (cfd->IsDropped()) {
         continue;
       }
+      assert(cfd->initialized());
       auto builders_iter = builders.find(cfd->GetID());
       assert(builders_iter != builders.end());
       auto* builder = builders_iter->second->version_builder();
@@ -3126,6 +3151,7 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname,
           break;
         }
         cfd = CreateColumnFamily(ColumnFamilyOptions(options), &edit);
+        cfd->set_initialized();
         builders.insert(
             {edit.column_family_, new BaseReferencedVersionBuilder(cfd)});
       } else if (edit.is_column_family_drop_) {
@@ -3269,6 +3295,7 @@ Status VersionSet::WriteSnapshot(log::Writer* log) {
     if (cfd->IsDropped()) {
       continue;
     }
+    assert(cfd->initialized());
     {
       // Store column family info
       VersionEdit edit;
@@ -3435,6 +3462,9 @@ void VersionSet::AddLiveFiles(std::vector<FileDescriptor>* live_list) {
   // pre-calculate space requirement
   int64_t total_files = 0;
   for (auto cfd : *column_family_set_) {
+    if (!cfd->initialized()) {
+      continue;
+    }
     Version* dummy_versions = cfd->dummy_versions();
     for (Version* v = dummy_versions->next_; v != dummy_versions;
          v = v->next_) {
@@ -3449,6 +3479,9 @@ void VersionSet::AddLiveFiles(std::vector<FileDescriptor>* live_list) {
   live_list->reserve(live_list->size() + static_cast<size_t>(total_files));
 
   for (auto cfd : *column_family_set_) {
+    if (!cfd->initialized()) {
+      continue;
+    }
     auto* current = cfd->current();
     bool found_current = false;
     Version* dummy_versions = cfd->dummy_versions();
@@ -3473,9 +3506,11 @@ InternalIterator* VersionSet::MakeInputIterator(
   ReadOptions read_options;
   read_options.verify_checksums = true;
   read_options.fill_cache = false;
-  if (c->ShouldFormSubcompactions()) {
-    read_options.total_order_seek = true;
-  }
+  // Compaction iterators shouldn't be confined to a single prefix.
+  // Compactions use Seek() for
+  // (a) concurrent compactions,
+  // (b) CompactionFilter::Decision::kRemoveAndSkipUntil.
+  read_options.total_order_seek = true;
 
   // Level-0 files have to be merged together.  For other levels,
   // we will make a concatenating iterator per level.
@@ -3574,6 +3609,9 @@ Status VersionSet::GetMetadataForFile(uint64_t number, int* filelevel,
                                       FileMetaData** meta,
                                       ColumnFamilyData** cfd) {
   for (auto cfd_iter : *column_family_set_) {
+    if (!cfd_iter->initialized()) {
+      continue;
+    }
     Version* version = cfd_iter->current();
     const auto* vstorage = version->storage_info();
     for (int level = 0; level < vstorage->num_levels(); level++) {
@@ -3592,7 +3630,7 @@ Status VersionSet::GetMetadataForFile(uint64_t number, int* filelevel,
 
 void VersionSet::GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata) {
   for (auto cfd : *column_family_set_) {
-    if (cfd->IsDropped()) {
+    if (cfd->IsDropped() || !cfd->initialized()) {
       continue;
     }
     for (int level = 0; level < cfd->NumberLevels(); level++) {

db/version_set.h

@@ -461,6 +461,10 @@ class Version {
                             MergeIteratorBuilder* merger_iter_builder,
                             int level, RangeDelAggregator* range_del_agg);
 
+  void AddRangeDelIteratorsForLevel(
+      const ReadOptions& read_options, const EnvOptions& soptions, int level,
+      std::vector<InternalIterator*>* range_del_iters);
+
   // Lookup the value for key.  If found, store it in *val and
   // return OK.  Else return a non-OK status.
   // Uses *operands to store merge_operator operations to apply later.

db/write_batch.cc

@@ -850,24 +850,24 @@ class MemTableInserter : public WriteBatch::Handler {
 
 public:
   // cf_mems should not be shared with concurrent inserters
-  MemTableInserter(SequenceNumber sequence, ColumnFamilyMemTables* cf_mems,
-                   FlushScheduler* flush_scheduler,
-                   bool ignore_missing_column_families,
-                   uint64_t recovering_log_number, DB* db,
-                   bool concurrent_memtable_writes,
-                   bool* has_valid_writes = nullptr)
-      : sequence_(sequence),
-        cf_mems_(cf_mems),
-        flush_scheduler_(flush_scheduler),
-        ignore_missing_column_families_(ignore_missing_column_families),
-        recovering_log_number_(recovering_log_number),
-        log_number_ref_(0),
-        db_(reinterpret_cast<DBImpl*>(db)),
-        concurrent_memtable_writes_(concurrent_memtable_writes),
-        post_info_created_(false),
-        has_valid_writes_(has_valid_writes),
-        rebuilding_trx_(nullptr) {
-    assert(cf_mems_);
+ MemTableInserter(SequenceNumber _sequence, ColumnFamilyMemTables* cf_mems,
+                  FlushScheduler* flush_scheduler,
+                  bool ignore_missing_column_families,
+                  uint64_t recovering_log_number, DB* db,
+                  bool concurrent_memtable_writes,
+                  bool* has_valid_writes = nullptr)
+     : sequence_(_sequence),
+       cf_mems_(cf_mems),
+       flush_scheduler_(flush_scheduler),
+       ignore_missing_column_families_(ignore_missing_column_families),
+       recovering_log_number_(recovering_log_number),
+       log_number_ref_(0),
+       db_(reinterpret_cast<DBImpl*>(db)),
+       concurrent_memtable_writes_(concurrent_memtable_writes),
+       post_info_created_(false),
+       has_valid_writes_(has_valid_writes),
+       rebuilding_trx_(nullptr) {
+   assert(cf_mems_);
   }
 
   ~MemTableInserter() {
@@ -882,7 +882,7 @@ public:
 
   void set_log_number_ref(uint64_t log) { log_number_ref_ = log; }
 
-  SequenceNumber get_final_sequence() const { return sequence_; }
+  SequenceNumber sequence() const { return sequence_; }
 
   void PostProcess() {
     assert(concurrent_memtable_writes_);
@@ -1302,6 +1302,7 @@ Status WriteBatchInternal::InsertInto(WriteThread::WriteGroup& write_group,
     if (!w->ShouldWriteToMemtable()) {
       continue;
     }
+    SetSequence(w->batch, inserter.sequence());
     inserter.set_log_number_ref(w->log_ref);
     w->status = w->batch->Iterate(&inserter);
     if (!w->status.ok()) {
@@ -1312,16 +1313,17 @@ Status WriteBatchInternal::InsertInto(WriteThread::WriteGroup& write_group,
 }
 
 Status WriteBatchInternal::InsertInto(WriteThread::Writer* writer,
+                                      SequenceNumber sequence,
                                       ColumnFamilyMemTables* memtables,
                                       FlushScheduler* flush_scheduler,
                                       bool ignore_missing_column_families,
                                       uint64_t log_number, DB* db,
                                       bool concurrent_memtable_writes) {
-  MemTableInserter inserter(WriteBatchInternal::Sequence(writer->batch),
-                            memtables, flush_scheduler,
+  assert(writer->ShouldWriteToMemtable());
+  MemTableInserter inserter(sequence, memtables, flush_scheduler,
                             ignore_missing_column_families, log_number, db,
                             concurrent_memtable_writes);
-  assert(writer->ShouldWriteToMemtable());
+  SetSequence(writer->batch, sequence);
   inserter.set_log_number_ref(writer->log_ref);
   Status s = writer->batch->Iterate(&inserter);
   if (concurrent_memtable_writes) {
@@ -1335,13 +1337,12 @@ Status WriteBatchInternal::InsertInto(
     FlushScheduler* flush_scheduler, bool ignore_missing_column_families,
     uint64_t log_number, DB* db, bool concurrent_memtable_writes,
     SequenceNumber* last_seq_used, bool* has_valid_writes) {
-  MemTableInserter inserter(WriteBatchInternal::Sequence(batch), memtables,
-                            flush_scheduler, ignore_missing_column_families,
-                            log_number, db, concurrent_memtable_writes,
-                            has_valid_writes);
+  MemTableInserter inserter(Sequence(batch), memtables, flush_scheduler,
+                            ignore_missing_column_families, log_number, db,
+                            concurrent_memtable_writes, has_valid_writes);
   Status s = batch->Iterate(&inserter);
   if (last_seq_used != nullptr) {
-    *last_seq_used = inserter.get_final_sequence();
+    *last_seq_used = inserter.sequence();
   }
   if (concurrent_memtable_writes) {
     inserter.PostProcess();

db/write_batch_internal.h

@@ -170,7 +170,7 @@ class WriteBatchInternal {
                            SequenceNumber* last_seq_used = nullptr,
                            bool* has_valid_writes = nullptr);
 
-  static Status InsertInto(WriteThread::Writer* writer,
+  static Status InsertInto(WriteThread::Writer* writer, SequenceNumber sequence,
                            ColumnFamilyMemTables* memtables,
                            FlushScheduler* flush_scheduler,
                            bool ignore_missing_column_families = false,

db/write_batch_test.cc

@@ -32,7 +32,7 @@ static std::string PrintContents(WriteBatch* b) {
   ImmutableCFOptions ioptions(options);
   WriteBufferManager wb(options.db_write_buffer_size);
   MemTable* mem = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb,
-                               kMaxSequenceNumber);
+                               kMaxSequenceNumber, 0 /* column_family_id */);
   mem->Ref();
   std::string state;
   ColumnFamilyMemTablesDefault cf_mems_default(mem);

db/write_controller.cc

@@ -34,17 +34,19 @@ WriteController::GetCompactionPressureToken() {
       new CompactionPressureToken(this));
 }
 
-bool WriteController::IsStopped() const { return total_stopped_ > 0; }
+bool WriteController::IsStopped() const {
+  return total_stopped_.load(std::memory_order_relaxed) > 0;
+}
 // This is inside DB mutex, so we can't sleep and need to minimize
 // frequency to get time.
 // If it turns out to be a performance issue, we can redesign the thread
 // synchronization model here.
 // The function trust caller will sleep micros returned.
 uint64_t WriteController::GetDelay(Env* env, uint64_t num_bytes) {
-  if (total_stopped_ > 0) {
+  if (total_stopped_.load(std::memory_order_relaxed) > 0) {
     return 0;
   }
-  if (total_delayed_.load() == 0) {
+  if (total_delayed_.load(std::memory_order_relaxed) == 0) {
     return 0;
   }
 

db/write_controller.h

@@ -9,6 +9,7 @@
 
 #include <atomic>
 #include <memory>
+#include "rocksdb/rate_limiter.h"
 
 namespace rocksdb {
 
@@ -21,12 +22,15 @@ class WriteControllerToken;
 // to be called while holding DB mutex
 class WriteController {
  public:
-  explicit WriteController(uint64_t _delayed_write_rate = 1024u * 1024u * 32u)
+  explicit WriteController(uint64_t _delayed_write_rate = 1024u * 1024u * 32u,
+                           int64_t low_pri_rate_bytes_per_sec = 1024 * 1024)
       : total_stopped_(0),
         total_delayed_(0),
         total_compaction_pressure_(0),
         bytes_left_(0),
-        last_refill_time_(0) {
+        last_refill_time_(0),
+        low_pri_rate_limiter_(
+            NewGenericRateLimiter(low_pri_rate_bytes_per_sec)) {
     set_max_delayed_write_rate(_delayed_write_rate);
   }
   ~WriteController() = default;
@@ -78,6 +82,8 @@ class WriteController {
 
   uint64_t max_delayed_write_rate() const { return max_delayed_write_rate_; }
 
+  RateLimiter* low_pri_rate_limiter() { return low_pri_rate_limiter_.get(); }
+
  private:
   uint64_t NowMicrosMonotonic(Env* env);
 
@@ -86,15 +92,17 @@ class WriteController {
   friend class DelayWriteToken;
   friend class CompactionPressureToken;
 
-  int total_stopped_;
+  std::atomic<int> total_stopped_;
   std::atomic<int> total_delayed_;
-  int total_compaction_pressure_;
+  std::atomic<int> total_compaction_pressure_;
   uint64_t bytes_left_;
   uint64_t last_refill_time_;
   // write rate set when initialization or by `DBImpl::SetDBOptions`
   uint64_t max_delayed_write_rate_;
   // current write rate
   uint64_t delayed_write_rate_;
+
+  std::unique_ptr<RateLimiter> low_pri_rate_limiter_;
 };
 
 class WriteControllerToken {

docs/_data/authors.yml

@@ -52,3 +52,7 @@ IslamAbdelRahman:
 ajkr:
   full_name: Andrew Kryczka
   fbid: 100010829806660
+
+sagar0:
+  full_name: Sagar Vemuri
+  fbid: 2419111

docs/_posts/2017-05-26-rocksdb-5-4-5-released.markdown

@@ -0,0 +1,39 @@
+---
+title: RocksDB 5.4.5 Released!
+layout: post
+author: sagar0
+category: blog
+---
+
+### Public API Change
+* Support dynamically changing `stats_dump_period_sec` option via SetDBOptions().
+* Added ReadOptions::max_skippable_internal_keys to set a threshold to fail a request as incomplete when too many keys are being skipped while using iterators.
+* DB::Get in place of std::string accepts PinnableSlice, which avoids the extra memcpy of value to std::string in most of cases.
+    * PinnableSlice releases the pinned resources that contain the value when it is destructed or when ::Reset() is called on it.
+    * The old API that accepts std::string, although discouraged, is still supported.
+* Replace Options::use_direct_writes with Options::use_direct_io_for_flush_and_compaction. See Direct IO wiki for details.
+
+### New Features
+* Memtable flush can be avoided during checkpoint creation if total log file size is smaller than a threshold specified by the user.
+* Introduce level-based L0->L0 compactions to reduce file count, so write delays are incurred less often.
+* (Experimental) Partitioning filters which creates an index on the partitions. The feature can be enabled by setting partition_filters when using kFullFilter. Currently the feature also requires two-level indexing to be enabled. Number of partitions is the same as the number of partitions for indexes, which is controlled by metadata_block_size.
+* DB::ResetStats() to reset internal stats.
+* Added CompactionEventListener and EventListener::OnFlushBegin interfaces.
+* Added DB::CreateColumnFamilie() and DB::DropColumnFamilies() to bulk create/drop column families.
+* Facility for cross-building RocksJava using Docker.
+
+### Bug Fixes
+* Fix WriteBatchWithIndex address use after scope error.
+* Fix WritableFile buffer size in direct IO.
+* Add prefetch to PosixRandomAccessFile in buffered io.
+* Fix PinnableSlice access invalid address when row cache is enabled.
+* Fix huge fallocate calls fail and make XFS unhappy.
+* Fix memory alignment with logical sector size.
+* Fix alignment in ReadaheadRandomAccessFile.
+* Fix bias with read amplification stats (READ_AMP_ESTIMATE_USEFUL_BYTES and READ_AMP_TOTAL_READ_BYTES).
+* Fix a manual / auto compaction data race.
+* Fix CentOS 5 cross-building of RocksJava.
+* Build and link with ZStd when creating the static RocksJava build.
+* Fix snprintf's usage to be cross-platform.
+* Fix build errors with blob DB.
+* Fix readamp test type inconsistency.

env/env_posix.cc

@@ -169,7 +169,8 @@ class PosixEnv : public Env {
       fd = open(fname.c_str(), flags, 0644);
     } while (fd < 0 && errno == EINTR);
     if (fd < 0) {
-      return IOError(fname, errno);
+      return IOError("While opening a file for sequentially reading", fname,
+                     errno);
     }
 
     SetFD_CLOEXEC(fd, &options);
@@ -178,7 +179,7 @@ class PosixEnv : public Env {
 #ifdef OS_MACOSX
       if (fcntl(fd, F_NOCACHE, 1) == -1) {
         close(fd);
-        return IOError(fname, errno);
+        return IOError("While fcntl NoCache", fname, errno);
       }
 #endif
     } else {
@@ -188,7 +189,8 @@ class PosixEnv : public Env {
       } while (file == nullptr && errno == EINTR);
       if (file == nullptr) {
         close(fd);
-        return IOError(fname, errno);
+        return IOError("While opening file for sequentially read", fname,
+                       errno);
       }
     }
     result->reset(new PosixSequentialFile(fname, file, fd, options));
@@ -217,7 +219,7 @@ class PosixEnv : public Env {
       fd = open(fname.c_str(), flags, 0644);
     } while (fd < 0 && errno == EINTR);
     if (fd < 0) {
-      return IOError(fname, errno);
+      return IOError("While open a file for random read", fname, errno);
     }
     SetFD_CLOEXEC(fd, &options);
 
@@ -233,7 +235,7 @@ class PosixEnv : public Env {
           result->reset(new PosixMmapReadableFile(fd, fname, base,
                                                   size, options));
         } else {
-          s = IOError(fname, errno);
+          s = IOError("while mmap file for read", fname, errno);
         }
       }
       close(fd);
@@ -242,7 +244,7 @@ class PosixEnv : public Env {
 #ifdef OS_MACOSX
         if (fcntl(fd, F_NOCACHE, 1) == -1) {
           close(fd);
-          return IOError(fname, errno);
+          return IOError("while fcntl NoCache", fname, errno);
         }
 #endif
       }
@@ -289,7 +291,7 @@ class PosixEnv : public Env {
     } while (fd < 0 && errno == EINTR);
 
     if (fd < 0) {
-      s = IOError(fname, errno);
+      s = IOError("While open a file for appending", fname, errno);
       return s;
     }
     SetFD_CLOEXEC(fd, &options);
@@ -310,14 +312,15 @@ class PosixEnv : public Env {
 #ifdef OS_MACOSX
       if (fcntl(fd, F_NOCACHE, 1) == -1) {
         close(fd);
-        s = IOError(fname, errno);
+        s = IOError("While fcntl NoCache an opened file for appending", fname,
+                    errno);
         return s;
       }
 #elif defined(OS_SOLARIS)
       if (directio(fd, DIRECTIO_ON) == -1) {
         if (errno != ENOTTY) { // ZFS filesystems don't support DIRECTIO_ON
           close(fd);
-          s = IOError(fname, errno);
+          s = IOError("While calling directio()", fname, errno);
           return s;
         }
       }
@@ -375,14 +378,14 @@ class PosixEnv : public Env {
       fd = open(old_fname.c_str(), flags, 0644);
     } while (fd < 0 && errno == EINTR);
     if (fd < 0) {
-      s = IOError(fname, errno);
+      s = IOError("while reopen file for write", fname, errno);
       return s;
     }
 
     SetFD_CLOEXEC(fd, &options);
     // rename into place
     if (rename(old_fname.c_str(), fname.c_str()) != 0) {
-      s = IOError(old_fname, errno);
+      s = IOError("while rename file to " + fname, old_fname, errno);
       close(fd);
       return s;
     }
@@ -403,14 +406,15 @@ class PosixEnv : public Env {
 #ifdef OS_MACOSX
       if (fcntl(fd, F_NOCACHE, 1) == -1) {
         close(fd);
-        s = IOError(fname, errno);
+        s = IOError("while fcntl NoCache for reopened file for append", fname,
+                    errno);
         return s;
       }
 #elif defined(OS_SOLARIS)
       if (directio(fd, DIRECTIO_ON) == -1) {
         if (errno != ENOTTY) { // ZFS filesystems don't support DIRECTIO_ON
           close(fd);
-          s = IOError(fname, errno);
+          s = IOError("while calling directio()", fname, errno);
           return s;
         }
       }
@@ -439,7 +443,7 @@ class PosixEnv : public Env {
         if (errno == EINTR) {
           continue;
         }
-        return IOError(fname, errno);
+        return IOError("While open file for random read/write", fname, errno);
       }
     }
 
@@ -457,7 +461,7 @@ class PosixEnv : public Env {
       fd = open(name.c_str(), 0);
     }
     if (fd < 0) {
-      return IOError(name, errno);
+      return IOError("While open directory", name, errno);
     } else {
       result->reset(new PosixDirectory(fd));
     }
@@ -496,7 +500,7 @@ class PosixEnv : public Env {
         case ENOTDIR:
           return Status::NotFound();
         default:
-          return IOError(dir, errno);
+          return IOError("While opendir", dir, errno);
       }
     }
     struct dirent* entry;
@@ -510,7 +514,7 @@ class PosixEnv : public Env {
   virtual Status DeleteFile(const std::string& fname) override {
     Status result;
     if (unlink(fname.c_str()) != 0) {
-      result = IOError(fname, errno);
+      result = IOError("while unlink() file", fname, errno);
     }
     return result;
   };
@@ -518,7 +522,7 @@ class PosixEnv : public Env {
   virtual Status CreateDir(const std::string& name) override {
     Status result;
     if (mkdir(name.c_str(), 0755) != 0) {
-      result = IOError(name, errno);
+      result = IOError("While mkdir", name, errno);
     }
     return result;
   };
@@ -527,7 +531,7 @@ class PosixEnv : public Env {
     Status result;
     if (mkdir(name.c_str(), 0755) != 0) {
       if (errno != EEXIST) {
-        result = IOError(name, errno);
+        result = IOError("While mkdir if missing", name, errno);
       } else if (!DirExists(name)) { // Check that name is actually a
                                      // directory.
         // Message is taken from mkdir
@@ -540,7 +544,7 @@ class PosixEnv : public Env {
   virtual Status DeleteDir(const std::string& name) override {
     Status result;
     if (rmdir(name.c_str()) != 0) {
-      result = IOError(name, errno);
+      result = IOError("file rmdir", name, errno);
     }
     return result;
   };
@@ -551,7 +555,7 @@ class PosixEnv : public Env {
     struct stat sbuf;
     if (stat(fname.c_str(), &sbuf) != 0) {
       *size = 0;
-      s = IOError(fname, errno);
+      s = IOError("while stat a file for size", fname, errno);
     } else {
       *size = sbuf.st_size;
     }
@@ -562,7 +566,7 @@ class PosixEnv : public Env {
                                          uint64_t* file_mtime) override {
     struct stat s;
     if (stat(fname.c_str(), &s) !=0) {
-      return IOError(fname, errno);
+      return IOError("while stat a file for modification time", fname, errno);
     }
     *file_mtime = static_cast<uint64_t>(s.st_mtime);
     return Status::OK();
@@ -571,7 +575,7 @@ class PosixEnv : public Env {
                             const std::string& target) override {
     Status result;
     if (rename(src.c_str(), target.c_str()) != 0) {
-      result = IOError(src, errno);
+      result = IOError("While renaming a file to " + target, src, errno);
     }
     return result;
   }
@@ -583,7 +587,7 @@ class PosixEnv : public Env {
       if (errno == EXDEV) {
         return Status::NotSupported("No cross FS links allowed");
       }
-      result = IOError(src, errno);
+      result = IOError("while link file to " + target, src, errno);
     }
     return result;
   }
@@ -597,9 +601,9 @@ class PosixEnv : public Env {
       fd = open(fname.c_str(), O_RDWR | O_CREAT, 0644);
     }
     if (fd < 0) {
-      result = IOError(fname, errno);
+      result = IOError("while open a file for lock", fname, errno);
     } else if (LockOrUnlock(fname, fd, true) == -1) {
-      result = IOError("lock " + fname, errno);
+      result = IOError("While lock file", fname, errno);
       close(fd);
     } else {
       SetFD_CLOEXEC(fd, nullptr);
@@ -615,7 +619,7 @@ class PosixEnv : public Env {
     PosixFileLock* my_lock = reinterpret_cast<PosixFileLock*>(lock);
     Status result;
     if (LockOrUnlock(my_lock->filename, my_lock->fd_, false) == -1) {
-      result = IOError("unlock", errno);
+      result = IOError("unlock", my_lock->filename, errno);
     }
     close(my_lock->fd_);
     delete my_lock;
@@ -678,7 +682,7 @@ class PosixEnv : public Env {
     }
     if (f == nullptr) {
       result->reset();
-      return IOError(fname, errno);
+      return IOError("when fopen a file for new logger", fname, errno);
     } else {
       int fd = fileno(f);
 #ifdef ROCKSDB_FALLOCATE_PRESENT
@@ -724,7 +728,7 @@ class PosixEnv : public Env {
       if (errno == EFAULT || errno == EINVAL)
         return Status::InvalidArgument(strerror(errno));
       else
-        return IOError("GetHostName", errno);
+        return IOError("GetHostName", name, errno);
     }
     return Status::OK();
   }
@@ -732,7 +736,7 @@ class PosixEnv : public Env {
   virtual Status GetCurrentTime(int64_t* unix_time) override {
     time_t ret = time(nullptr);
     if (ret == (time_t) -1) {
-      return IOError("GetCurrentTime", errno);
+      return IOError("GetCurrentTime", "", errno);
     }
     *unix_time = (int64_t) ret;
     return Status::OK();
@@ -761,6 +765,11 @@ class PosixEnv : public Env {
     thread_pools_[pri].SetBackgroundThreads(num);
   }
 
+  virtual int GetBackgroundThreads(Priority pri) override {
+    assert(pri >= Priority::LOW && pri <= Priority::HIGH);
+    return thread_pools_[pri].GetBackgroundThreads();
+  }
+
   // Allow increasing the number of worker threads.
   virtual void IncBackgroundThreadsIfNeeded(int num, Priority pri) override {
     assert(pri >= Priority::LOW && pri <= Priority::HIGH);

env/io_posix.cc

@@ -167,7 +167,7 @@ Status PosixSequentialFile::Read(size_t n, Slice* result, char* scratch) {
       clearerr(file_);
     } else {
       // A partial read with an error: return a non-ok status
-      s = IOError(filename_, errno);
+      s = IOError("While reading file sequentially", filename_, errno);
     }
   }
   // we need to fadvise away the entire range of pages because
@@ -207,7 +207,9 @@ Status PosixSequentialFile::PositionedRead(uint64_t offset, size_t n,
   }
   if (r < 0) {
     // An error: return a non-ok status
-    s = IOError(filename_, errno);
+    s = IOError(
+        "While pread " + ToString(n) + " bytes from offset " + ToString(offset),
+        filename_, errno);
   }
   *result = Slice(scratch, (r < 0) ? 0 : n - left);
   return s;
@@ -215,7 +217,8 @@ Status PosixSequentialFile::PositionedRead(uint64_t offset, size_t n,
 
 Status PosixSequentialFile::Skip(uint64_t n) {
   if (fseek(file_, static_cast<long int>(n), SEEK_CUR)) {
-    return IOError(filename_, errno);
+    return IOError("While fseek to skip " + ToString(n) + " bytes", filename_,
+                   errno);
   }
   return Status::OK();
 }
@@ -228,7 +231,9 @@ Status PosixSequentialFile::InvalidateCache(size_t offset, size_t length) {
     // free OS pages
     int ret = Fadvise(fd_, offset, length, POSIX_FADV_DONTNEED);
     if (ret != 0) {
-      return IOError(filename_, errno);
+      return IOError("While fadvise NotNeeded offset " + ToString(offset) +
+                         " len " + ToString(length),
+                     filename_, errno);
     }
   }
   return Status::OK();
@@ -336,7 +341,9 @@ Status PosixRandomAccessFile::Read(uint64_t offset, size_t n, Slice* result,
   }
   if (r < 0) {
     // An error: return a non-ok status
-    s = IOError(filename_, errno);
+    s = IOError(
+        "While pread offset " + ToString(offset) + " len " + ToString(n),
+        filename_, errno);
   }
   *result = Slice(scratch, (r < 0) ? 0 : n - left);
   return s;
@@ -356,7 +363,9 @@ Status PosixRandomAccessFile::Prefetch(uint64_t offset, size_t n) {
     r = fcntl(fd_, F_RDADVISE, &advice);
 #endif
     if (r == -1) {
-      s = IOError(filename_, errno);
+      s = IOError("While prefetching offset " + ToString(offset) + " len " +
+                      ToString(n),
+                  filename_, errno);
     }
   }
   return s;
@@ -406,7 +415,9 @@ Status PosixRandomAccessFile::InvalidateCache(size_t offset, size_t length) {
   if (ret == 0) {
     return Status::OK();
   }
-  return IOError(filename_, errno);
+  return IOError("While fadvise NotNeeded offset " + ToString(offset) +
+                     " len " + ToString(length),
+                 filename_, errno);
 #endif
 }
 
@@ -439,7 +450,9 @@ Status PosixMmapReadableFile::Read(uint64_t offset, size_t n, Slice* result,
   Status s;
   if (offset > length_) {
     *result = Slice();
-    return IOError(filename_, EINVAL);
+    return IOError("While mmap read offset " + ToString(offset) +
+                       " larger than file length " + ToString(length_),
+                   filename_, EINVAL);
   } else if (offset + n > length_) {
     n = static_cast<size_t>(length_ - offset);
   }
@@ -456,7 +469,9 @@ Status PosixMmapReadableFile::InvalidateCache(size_t offset, size_t length) {
   if (ret == 0) {
     return Status::OK();
   }
-  return IOError(filename_, errno);
+  return IOError("While fadvise not needed. Offset " + ToString(offset) +
+                     " len" + ToString(length),
+                 filename_, errno);
 #endif
 }
 
@@ -473,7 +488,7 @@ Status PosixMmapFile::UnmapCurrentRegion() {
   if (base_ != nullptr) {
     int munmap_status = munmap(base_, limit_ - base_);
     if (munmap_status != 0) {
-      return IOError(filename_, munmap_status);
+      return IOError("While munmap", filename_, munmap_status);
     }
     file_offset_ += limit_ - base_;
     base_ = nullptr;
@@ -536,7 +551,7 @@ Status PosixMmapFile::Msync() {
   last_sync_ = dst_;
   TEST_KILL_RANDOM("PosixMmapFile::Msync:0", rocksdb_kill_odds);
   if (msync(base_ + p1, p2 - p1 + page_size_, MS_SYNC) < 0) {
-    return IOError(filename_, errno);
+    return IOError("While msync", filename_, errno);
   }
   return Status::OK();
 }
@@ -587,6 +602,7 @@ Status PosixMmapFile::Append(const Slice& data) {
     }
 
     size_t n = (left <= avail) ? left : avail;
+    assert(dst_);
     memcpy(dst_, src, n);
     dst_ += n;
     src += n;
@@ -601,17 +617,17 @@ Status PosixMmapFile::Close() {
 
   s = UnmapCurrentRegion();
   if (!s.ok()) {
-    s = IOError(filename_, errno);
+    s = IOError("While closing mmapped file", filename_, errno);
   } else if (unused > 0) {
     // Trim the extra space at the end of the file
     if (ftruncate(fd_, file_offset_ - unused) < 0) {
-      s = IOError(filename_, errno);
+      s = IOError("While ftruncating mmaped file", filename_, errno);
     }
   }
 
   if (close(fd_) < 0) {
     if (s.ok()) {
-      s = IOError(filename_, errno);
+      s = IOError("While closing mmapped file", filename_, errno);
     }
   }
 
@@ -625,7 +641,7 @@ Status PosixMmapFile::Flush() { return Status::OK(); }
 
 Status PosixMmapFile::Sync() {
   if (fdatasync(fd_) < 0) {
-    return IOError(filename_, errno);
+    return IOError("While fdatasync mmapped file", filename_, errno);
   }
 
   return Msync();
@@ -636,7 +652,7 @@ Status PosixMmapFile::Sync() {
  */
 Status PosixMmapFile::Fsync() {
   if (fsync(fd_) < 0) {
-    return IOError(filename_, errno);
+    return IOError("While fsync mmaped file", filename_, errno);
   }
 
   return Msync();
@@ -661,7 +677,7 @@ Status PosixMmapFile::InvalidateCache(size_t offset, size_t length) {
   if (ret == 0) {
     return Status::OK();
   }
-  return IOError(filename_, errno);
+  return IOError("While fadvise NotNeeded mmapped file", filename_, errno);
 #endif
 }
 
@@ -679,7 +695,9 @@ Status PosixMmapFile::Allocate(uint64_t offset, uint64_t len) {
   if (alloc_status == 0) {
     return Status::OK();
   } else {
-    return IOError(filename_, errno);
+    return IOError(
+        "While fallocate offset " + ToString(offset) + " len " + ToString(len),
+        filename_, errno);
   }
 }
 #endif
@@ -722,7 +740,7 @@ Status PosixWritableFile::Append(const Slice& data) {
       if (errno == EINTR) {
         continue;
       }
-      return IOError(filename_, errno);
+      return IOError("While appending to file", filename_, errno);
     }
     left -= done;
     src += done;
@@ -746,7 +764,8 @@ Status PosixWritableFile::PositionedAppend(const Slice& data, uint64_t offset) {
       if (errno == EINTR) {
         continue;
       }
-      return IOError(filename_, errno);
+      return IOError("While pwrite to file at offset " + ToString(offset),
+                     filename_, errno);
     }
     left -= done;
     offset += done;
@@ -760,7 +779,8 @@ Status PosixWritableFile::Truncate(uint64_t size) {
   Status s;
   int r = ftruncate(fd_, size);
   if (r < 0) {
-    s = IOError(filename_, errno);
+    s = IOError("While ftruncate file to size " + ToString(size), filename_,
+                errno);
   } else {
     filesize_ = size;
   }
@@ -797,10 +817,11 @@ Status PosixWritableFile::Close() {
     // While we work with Travis-CI team to figure out if this is a
     // quirk of Docker/AUFS, we will comment this out.
     struct stat file_stats;
-    fstat(fd_, &file_stats);
+    int result = fstat(fd_, &file_stats);
     // After ftruncate, we check whether ftruncate has the correct behavior.
     // If not, we should hack it with FALLOC_FL_PUNCH_HOLE
-    if ((file_stats.st_size + file_stats.st_blksize - 1) /
+    if (result == 0 &&
+        (file_stats.st_size + file_stats.st_blksize - 1) /
             file_stats.st_blksize !=
         file_stats.st_blocks / (file_stats.st_blksize / 512)) {
       IOSTATS_TIMER_GUARD(allocate_nanos);
@@ -813,7 +834,7 @@ Status PosixWritableFile::Close() {
   }
 
   if (close(fd_) < 0) {
-    s = IOError(filename_, errno);
+    s = IOError("While closing file after writing", filename_, errno);
   }
   fd_ = -1;
   return s;
@@ -824,14 +845,14 @@ Status PosixWritableFile::Flush() { return Status::OK(); }
 
 Status PosixWritableFile::Sync() {
   if (fdatasync(fd_) < 0) {
-    return IOError(filename_, errno);
+    return IOError("While fdatasync", filename_, errno);
   }
   return Status::OK();
 }
 
 Status PosixWritableFile::Fsync() {
   if (fsync(fd_) < 0) {
-    return IOError(filename_, errno);
+    return IOError("While fsync", filename_, errno);
   }
   return Status::OK();
 }
@@ -852,7 +873,7 @@ Status PosixWritableFile::InvalidateCache(size_t offset, size_t length) {
   if (ret == 0) {
     return Status::OK();
   }
-  return IOError(filename_, errno);
+  return IOError("While fadvise NotNeeded", filename_, errno);
 #endif
 }
 
@@ -871,7 +892,9 @@ Status PosixWritableFile::Allocate(uint64_t offset, uint64_t len) {
   if (alloc_status == 0) {
     return Status::OK();
   } else {
-    return IOError(filename_, errno);
+    return IOError(
+        "While fallocate offset " + ToString(offset) + " len " + ToString(len),
+        filename_, errno);
   }
 }
 #endif
@@ -884,7 +907,9 @@ Status PosixWritableFile::RangeSync(uint64_t offset, uint64_t nbytes) {
       static_cast<off_t>(nbytes), SYNC_FILE_RANGE_WRITE) == 0) {
     return Status::OK();
   } else {
-    return IOError(filename_, errno);
+    return IOError("While sync_file_range offset " + ToString(offset) +
+                       " bytes " + ToString(nbytes),
+                   filename_, errno);
   }
 }
 #endif
@@ -920,7 +945,9 @@ Status PosixRandomRWFile::Write(uint64_t offset, const Slice& data) {
         // write was interrupted, try again.
         continue;
       }
-      return IOError(filename_, errno);
+      return IOError(
+          "While write random read/write file at offset " + ToString(offset),
+          filename_, errno);
     }
 
     // Wrote `done` bytes
@@ -944,7 +971,9 @@ Status PosixRandomRWFile::Read(uint64_t offset, size_t n, Slice* result,
         // read was interrupted, try again.
         continue;
       }
-      return IOError(filename_, errno);
+      return IOError("While reading random read/write file offset " +
+                         ToString(offset) + " len " + ToString(n),
+                     filename_, errno);
     } else if (done == 0) {
       // Nothing more to read
       break;
@@ -964,21 +993,21 @@ Status PosixRandomRWFile::Flush() { return Status::OK(); }
 
 Status PosixRandomRWFile::Sync() {
   if (fdatasync(fd_) < 0) {
-    return IOError(filename_, errno);
+    return IOError("While fdatasync random read/write file", filename_, errno);
   }
   return Status::OK();
 }
 
 Status PosixRandomRWFile::Fsync() {
   if (fsync(fd_) < 0) {
-    return IOError(filename_, errno);
+    return IOError("While fsync random read/write file", filename_, errno);
   }
   return Status::OK();
 }
 
 Status PosixRandomRWFile::Close() {
   if (close(fd_) < 0) {
-    return IOError(filename_, errno);
+    return IOError("While close random read/write file", filename_, errno);
   }
   fd_ = -1;
   return Status::OK();
@@ -993,7 +1022,7 @@ PosixDirectory::~PosixDirectory() { close(fd_); }
 Status PosixDirectory::Fsync() {
 #ifndef OS_AIX
   if (fsync(fd_) == -1) {
-    return IOError("directory", errno);
+    return IOError("While fsync", "a directory", errno);
   }
 #endif
   return Status::OK();

env/io_posix.h

@@ -24,15 +24,26 @@
 #endif
 
 namespace rocksdb {
+static std::string IOErrorMsg(const std::string& context,
+                              const std::string& file_name) {
+  if (file_name.empty()) {
+    return context;
+  }
+  return context + ": " + file_name;
+}
 
-static Status IOError(const std::string& context, int err_number) {
+// file_name can be left empty if it is not unkown.
+static Status IOError(const std::string& context, const std::string& file_name,
+                      int err_number) {
   switch (err_number) {
   case ENOSPC:
-    return Status::NoSpace(context, strerror(err_number));
+    return Status::NoSpace(IOErrorMsg(context, file_name),
+                           strerror(err_number));
   case ESTALE:
     return Status::IOError(Status::kStaleFile);
   default:
-    return Status::IOError(context, strerror(err_number));
+    return Status::IOError(IOErrorMsg(context, file_name),
+                           strerror(err_number));
   }
 }
 

env/memenv.cc

@@ -1,451 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include <string.h>
-
-#include <algorithm>
-#include <map>
-#include <string>
-#include <vector>
-
-#include "rocksdb/env.h"
-#include "rocksdb/status.h"
-#include "port/port.h"
-#include "util/mutexlock.h"
-
-namespace rocksdb {
-
-#ifndef ROCKSDB_LITE
-
-namespace {
-
-std::string NormalizeFileName(const std::string fname) {
-  if (fname.find("//") == std::string::npos) {
-    return fname;
-  }
-  std::string out_name = "";
-  bool is_slash = false;
-  for (char c : fname) {
-    if (c == '/' && is_slash) {
-      continue;
-    }
-    out_name.append(1, c);
-    if (c == '/') {
-      is_slash = true;
-    } else {
-      is_slash = false;
-    }
-  }
-  return out_name;
-}
-
-class FileState {
- public:
-  // FileStates are reference counted. The initial reference count is zero
-  // and the caller must call Ref() at least once.
-  FileState() : refs_(0), size_(0) {}
-
-  // Increase the reference count.
-  void Ref() {
-    MutexLock lock(&refs_mutex_);
-    ++refs_;
-  }
-
-  // Decrease the reference count. Delete if this is the last reference.
-  void Unref() {
-    bool do_delete = false;
-
-    {
-      MutexLock lock(&refs_mutex_);
-      --refs_;
-      assert(refs_ >= 0);
-      if (refs_ <= 0) {
-        do_delete = true;
-      }
-    }
-
-    if (do_delete) {
-      delete this;
-    }
-  }
-
-  uint64_t Size() const { return size_; }
-
-  Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const {
-    const uint64_t available = size_ - std::min(size_, offset);
-    if (n > available) {
-      n = available;
-    }
-    if (n == 0) {
-      *result = Slice();
-      return Status::OK();
-    }
-
-    size_t block = offset / kBlockSize;
-    size_t block_offset = offset % kBlockSize;
-
-    if (n <= kBlockSize - block_offset) {
-      // The requested bytes are all in the first block.
-      *result = Slice(blocks_[block] + block_offset, n);
-      return Status::OK();
-    }
-
-    size_t bytes_to_copy = n;
-    char* dst = scratch;
-
-    while (bytes_to_copy > 0) {
-      size_t avail = kBlockSize - block_offset;
-      if (avail > bytes_to_copy) {
-        avail = bytes_to_copy;
-      }
-      memcpy(dst, blocks_[block] + block_offset, avail);
-
-      bytes_to_copy -= avail;
-      dst += avail;
-      block++;
-      block_offset = 0;
-    }
-
-    *result = Slice(scratch, n);
-    return Status::OK();
-  }
-
-  Status Append(const Slice& data) {
-    const char* src = data.data();
-    size_t src_len = data.size();
-
-    while (src_len > 0) {
-      size_t avail;
-      size_t offset = size_ % kBlockSize;
-
-      if (offset != 0) {
-        // There is some room in the last block.
-        avail = kBlockSize - offset;
-      } else {
-        // No room in the last block; push new one.
-        blocks_.push_back(new char[kBlockSize]);
-        avail = kBlockSize;
-      }
-
-      if (avail > src_len) {
-        avail = src_len;
-      }
-      memcpy(blocks_.back() + offset, src, avail);
-      src_len -= avail;
-      src += avail;
-      size_ += avail;
-    }
-
-    return Status::OK();
-  }
-
- private:
-  // Private since only Unref() should be used to delete it.
-  ~FileState() {
-    for (std::vector<char*>::iterator i = blocks_.begin(); i != blocks_.end();
-         ++i) {
-      delete [] *i;
-    }
-  }
-
-  // No copying allowed.
-  FileState(const FileState&);
-  void operator=(const FileState&);
-
-  port::Mutex refs_mutex_;
-  int refs_;  // Protected by refs_mutex_;
-
-  // The following fields are not protected by any mutex. They are only mutable
-  // while the file is being written, and concurrent access is not allowed
-  // to writable files.
-  std::vector<char*> blocks_;
-  uint64_t size_;
-
-  enum { kBlockSize = 8 * 1024 };
-};
-
-class SequentialFileImpl : public SequentialFile {
- public:
-  explicit SequentialFileImpl(FileState* file) : file_(file), pos_(0) {
-    file_->Ref();
-  }
-
-  ~SequentialFileImpl() {
-    file_->Unref();
-  }
-
-  virtual Status Read(size_t n, Slice* result, char* scratch) override {
-    Status s = file_->Read(pos_, n, result, scratch);
-    if (s.ok()) {
-      pos_ += result->size();
-    }
-    return s;
-  }
-
-  virtual Status Skip(uint64_t n) override {
-    if (pos_ > file_->Size()) {
-      return Status::IOError("pos_ > file_->Size()");
-    }
-    const size_t available = file_->Size() - pos_;
-    if (n > available) {
-      n = available;
-    }
-    pos_ += n;
-    return Status::OK();
-  }
-
- private:
-  FileState* file_;
-  size_t pos_;
-};
-
-class RandomAccessFileImpl : public RandomAccessFile {
- public:
-  explicit RandomAccessFileImpl(FileState* file) : file_(file) {
-    file_->Ref();
-  }
-
-  ~RandomAccessFileImpl() {
-    file_->Unref();
-  }
-
-  virtual Status Read(uint64_t offset, size_t n, Slice* result,
-                      char* scratch) const override {
-    return file_->Read(offset, n, result, scratch);
-  }
-
- private:
-  FileState* file_;
-};
-
-class WritableFileImpl : public WritableFile {
- public:
-  explicit WritableFileImpl(FileState* file) : file_(file) { file_->Ref(); }
-
-  ~WritableFileImpl() {
-    file_->Unref();
-  }
-
-  virtual Status Append(const Slice& data) override {
-    return file_->Append(data);
-  }
-  virtual Status Truncate(uint64_t size) override {
-    return Status::OK();
-  }
-  virtual Status Close() override { return Status::OK(); }
-  virtual Status Flush() override { return Status::OK(); }
-  virtual Status Sync() override { return Status::OK(); }
-
- private:
-  FileState* file_;
-};
-
-class InMemoryDirectory : public Directory {
- public:
-  virtual Status Fsync() override { return Status::OK(); }
-};
-
-class InMemoryEnv : public EnvWrapper {
- public:
-  explicit InMemoryEnv(Env* base_env) : EnvWrapper(base_env) { }
-
-  virtual ~InMemoryEnv() {
-    for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i){
-      i->second->Unref();
-    }
-  }
-
-  // Partial implementation of the Env interface.
-  virtual Status NewSequentialFile(const std::string& fname,
-                                   unique_ptr<SequentialFile>* result,
-                                   const EnvOptions& soptions) override {
-    std::string nfname = NormalizeFileName(fname);
-    MutexLock lock(&mutex_);
-    if (file_map_.find(fname) == file_map_.end()) {
-      *result = NULL;
-      return Status::IOError(fname, "File not found");
-    }
-
-    result->reset(new SequentialFileImpl(file_map_[nfname]));
-    return Status::OK();
-  }
-
-  virtual Status NewRandomAccessFile(const std::string& fname,
-                                     unique_ptr<RandomAccessFile>* result,
-                                     const EnvOptions& soptions) override {
-    std::string nfname = NormalizeFileName(fname);
-    MutexLock lock(&mutex_);
-    if (file_map_.find(nfname) == file_map_.end()) {
-      *result = NULL;
-      return Status::IOError(fname, "File not found");
-    }
-
-    result->reset(new RandomAccessFileImpl(file_map_[nfname]));
-    return Status::OK();
-  }
-
-  virtual Status NewWritableFile(const std::string& fname,
-                                 unique_ptr<WritableFile>* result,
-                                 const EnvOptions& soptions) override {
-    std::string nfname = NormalizeFileName(fname);
-    MutexLock lock(&mutex_);
-    if (file_map_.find(nfname) != file_map_.end()) {
-      DeleteFileInternal(nfname);
-    }
-
-    FileState* file = new FileState();
-    file->Ref();
-    file_map_[nfname] = file;
-
-    result->reset(new WritableFileImpl(file));
-    return Status::OK();
-  }
-
-  virtual Status NewDirectory(const std::string& name,
-                              unique_ptr<Directory>* result) override {
-    result->reset(new InMemoryDirectory());
-    return Status::OK();
-  }
-
-  virtual Status FileExists(const std::string& fname) override {
-    std::string nfname = NormalizeFileName(fname);
-    MutexLock lock(&mutex_);
-    if (file_map_.find(nfname) != file_map_.end()) {
-      return Status::OK();
-    } else {
-      return Status::NotFound();
-    }
-  }
-
-  virtual Status GetChildren(const std::string& dir,
-                             std::vector<std::string>* result) override {
-    MutexLock lock(&mutex_);
-    result->clear();
-
-    bool found_dir = false;
-    for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i){
-      const std::string& filename = i->first;
-
-      if (dir == filename) {
-        found_dir = true;
-      } else if (filename.size() >= dir.size() + 1 &&
-                 filename[dir.size()] == '/' &&
-                 Slice(filename).starts_with(Slice(dir))) {
-        found_dir = true;
-        result->push_back(filename.substr(dir.size() + 1));
-      }
-    }
-
-    return found_dir ? Status::OK() : Status::NotFound();
-  }
-
-  void DeleteFileInternal(const std::string& fname) {
-    if (file_map_.find(fname) == file_map_.end()) {
-      return;
-    }
-
-    file_map_[fname]->Unref();
-    file_map_.erase(fname);
-  }
-
-  virtual Status DeleteFile(const std::string& fname) override {
-    std::string nfname = NormalizeFileName(fname);
-    MutexLock lock(&mutex_);
-    if (file_map_.find(nfname) == file_map_.end()) {
-      return Status::IOError(fname, "File not found");
-    }
-
-    DeleteFileInternal(nfname);
-    return Status::OK();
-  }
-
-  virtual Status CreateDir(const std::string& dirname) override {
-    auto ndirname = NormalizeFileName(dirname);
-    if (file_map_.find(ndirname) == file_map_.end()) {
-      FileState* file = new FileState();
-      file->Ref();
-      file_map_[ndirname] = file;
-    } else {
-      return Status::IOError();
-    }
-    return Status::OK();
-  }
-
-  virtual Status CreateDirIfMissing(const std::string& dirname) override {
-    CreateDir(dirname);
-    return Status::OK();
-  }
-
-  virtual Status DeleteDir(const std::string& dirname) override {
-    return DeleteFile(dirname);
-  }
-
-  virtual Status GetFileSize(const std::string& fname,
-                             uint64_t* file_size) override {
-    std::string nfname = NormalizeFileName(fname);
-    MutexLock lock(&mutex_);
-
-    if (file_map_.find(nfname) == file_map_.end()) {
-      return Status::IOError(fname, "File not found");
-    }
-
-    *file_size = file_map_[nfname]->Size();
-    return Status::OK();
-  }
-
-  virtual Status GetFileModificationTime(const std::string& fname,
-                                         uint64_t* time) override {
-    return Status::NotSupported("getFileMTime", "Not supported in MemEnv");
-  }
-
-  virtual Status RenameFile(const std::string& src,
-                            const std::string& dest) override {
-    std::string nsrc = NormalizeFileName(src);
-    std::string ndest = NormalizeFileName(dest);
-    MutexLock lock(&mutex_);
-    if (file_map_.find(nsrc) == file_map_.end()) {
-      return Status::IOError(src, "File not found");
-    }
-
-    DeleteFileInternal(dest);
-    file_map_[ndest] = file_map_[nsrc];
-    file_map_.erase(nsrc);
-    return Status::OK();
-  }
-
-  virtual Status LockFile(const std::string& fname, FileLock** lock) override {
-    *lock = new FileLock;
-    return Status::OK();
-  }
-
-  virtual Status UnlockFile(FileLock* lock) override {
-    delete lock;
-    return Status::OK();
-  }
-
-  virtual Status GetTestDirectory(std::string* path) override {
-    *path = "/test";
-    return Status::OK();
-  }
-
- private:
-  // Map from filenames to FileState objects, representing a simple file system.
-  typedef std::map<std::string, FileState*> FileSystem;
-  port::Mutex mutex_;
-  FileSystem file_map_;  // Protected by mutex_.
-};
-
-}  // namespace
-
-Env* NewMemEnv(Env* base_env) {
-  return new InMemoryEnv(base_env);
-}
-
-#else  // ROCKSDB_LITE
-
-Env* NewMemEnv(Env* base_env) { return nullptr; }
-
-#endif  // !ROCKSDB_LITE
-
-}  // namespace rocksdb

env/mock_env.cc

@@ -376,8 +376,9 @@ class TestMemLogger : public Logger {
       struct timeval now_tv;
       gettimeofday(&now_tv, nullptr);
       const time_t seconds = now_tv.tv_sec;
-      struct tm t;
-      localtime_r(&seconds, &t);
+      struct tm t{0, 0, 0, 0, 0, 0, 0, 0, 0};
+      auto ret __attribute__((__unused__)) = localtime_r(&seconds, &t);
+      assert(ret);
       p += snprintf(p, limit - p,
                     "%04d/%02d/%02d-%02d:%02d:%02d.%06d ",
                     t.tm_year + 1900,
@@ -780,4 +781,14 @@ void MockEnv::FakeSleepForMicroseconds(int64_t micros) {
   fake_sleep_micros_.fetch_add(micros);
 }
 
+#ifndef ROCKSDB_LITE
+// This is to maintain the behavior before swithcing from InMemoryEnv to MockEnv
+Env* NewMemEnv(Env* base_env) { return new MockEnv(base_env); }
+
+#else  // ROCKSDB_LITE
+
+Env* NewMemEnv(Env* base_env) { return nullptr; }
+
+#endif  // !ROCKSDB_LITE
+
 }  // namespace rocksdb

hdfs/env_hdfs.h

@@ -149,6 +149,10 @@ class HdfsEnv : public Env {
     posixEnv->SetBackgroundThreads(number, pri);
   }
 
+  virtual int GetBackgroundThreads(Priority pri = LOW) {
+    return posixEnv->GetBackgroundThreads(pri);
+  }
+
   virtual void IncBackgroundThreadsIfNeeded(int number, Priority pri) override {
     posixEnv->IncBackgroundThreadsIfNeeded(number, pri);
   }
@@ -356,6 +360,7 @@ class HdfsEnv : public Env {
   }
 
   virtual void SetBackgroundThreads(int number, Priority pri = LOW) override {}
+  virtual int GetBackgroundThreads(Priority pri = LOW) override { return 0; }
   virtual void IncBackgroundThreadsIfNeeded(int number, Priority pri) override {
   }
   virtual std::string TimeToString(uint64_t number) override { return ""; }

include/rocksdb/advanced_options.h

@@ -72,7 +72,7 @@ struct CompactionOptionsFIFO {
 
   CompactionOptionsFIFO() : max_table_files_size(1 * 1024 * 1024 * 1024) {}
   CompactionOptionsFIFO(uint64_t _max_table_files_size,
-                        uint64_t _allow_compaction)
+                        bool _allow_compaction)
       : max_table_files_size(_max_table_files_size),
         allow_compaction(_allow_compaction) {}
 };

include/rocksdb/c.h

@@ -639,6 +639,7 @@ extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_format_version(
 enum {
   rocksdb_block_based_table_index_type_binary_search = 0,
   rocksdb_block_based_table_index_type_hash_search = 1,
+  rocksdb_block_based_table_index_type_two_level_index_search = 2,
 };
 extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_index_type(
     rocksdb_block_based_table_options_t*, int);  // uses one of the above enums
@@ -1118,6 +1119,15 @@ extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_open(
 extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_add(
     rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen,
     const char* val, size_t vallen, char** errptr);
+extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_put(
+    rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen,
+    const char* val, size_t vallen, char** errptr);
+extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_merge(
+    rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen,
+    const char* val, size_t vallen, char** errptr);
+extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_delete(
+    rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen,
+    char** errptr);
 extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_finish(
     rocksdb_sstfilewriter_t* writer, char** errptr);
 extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_destroy(

include/rocksdb/compaction_filter.h

@@ -135,19 +135,19 @@ class CompactionFilter {
   //      *skip_until <= key is treated the same as Decision::kKeep
   //      (since the range [key, *skip_until) is empty).
   //
-  //      The keys are skipped even if there are snapshots containing them,
-  //      as if IgnoreSnapshots() was true; i.e. values removed
-  //      by kRemoveAndSkipUntil can disappear from a snapshot - beware
-  //      if you're using TransactionDB or DB::GetSnapshot().
-  //
-  //      Another warning: if value for a key was overwritten or merged into
-  //      (multiple Put()s or Merge()s), and compaction filter skips this key
-  //      with kRemoveAndSkipUntil, it's possible that it will remove only
-  //      the new value, exposing the old value that was supposed to be
-  //      overwritten.
-  //
-  //      If you use kRemoveAndSkipUntil, consider also reducing
-  //      compaction_readahead_size option.
+  //      Caveats:
+  //       - The keys are skipped even if there are snapshots containing them,
+  //         as if IgnoreSnapshots() was true; i.e. values removed
+  //         by kRemoveAndSkipUntil can disappear from a snapshot - beware
+  //         if you're using TransactionDB or DB::GetSnapshot().
+  //       - If value for a key was overwritten or merged into (multiple Put()s
+  //         or Merge()s), and compaction filter skips this key with
+  //         kRemoveAndSkipUntil, it's possible that it will remove only
+  //         the new value, exposing the old value that was supposed to be
+  //         overwritten.
+  //       - Doesn't work with PlainTableFactory in prefix mode.
+  //       - If you use kRemoveAndSkipUntil, consider also reducing
+  //         compaction_readahead_size option.
   //
   // Note: If you are using a TransactionDB, it is not recommended to filter
   // out or modify merge operands (ValueType::kMergeOperand).

include/rocksdb/convenience.h

@@ -161,12 +161,15 @@ namespace rocksdb {
 // @param input_strings_escaped when set to true, each escaped characters
 //     prefixed by '\' in the values of the opts_map will be further converted
 //     back to the raw string before assigning to the associated options.
+// @param ignore_unknown_options when set to true, unknown options are ignored
+//     instead of resulting in an unknown-option error.
 // @return Status::OK() on success.  Otherwise, a non-ok status indicating
 //     error will be returned, and "new_options" will be set to "base_options".
 Status GetColumnFamilyOptionsFromMap(
     const ColumnFamilyOptions& base_options,
     const std::unordered_map<std::string, std::string>& opts_map,
-    ColumnFamilyOptions* new_options, bool input_strings_escaped = false);
+    ColumnFamilyOptions* new_options, bool input_strings_escaped = false,
+    bool ignore_unknown_options = false);
 
 // Take a default DBOptions "base_options" in addition to a
 // map "opts_map" of option name to option value to construct the new
@@ -189,12 +192,15 @@ Status GetColumnFamilyOptionsFromMap(
 // @param input_strings_escaped when set to true, each escaped characters
 //     prefixed by '\' in the values of the opts_map will be further converted
 //     back to the raw string before assigning to the associated options.
+// @param ignore_unknown_options when set to true, unknown options are ignored
+//     instead of resulting in an unknown-option error.
 // @return Status::OK() on success.  Otherwise, a non-ok status indicating
 //     error will be returned, and "new_options" will be set to "base_options".
 Status GetDBOptionsFromMap(
     const DBOptions& base_options,
     const std::unordered_map<std::string, std::string>& opts_map,
-    DBOptions* new_options, bool input_strings_escaped = false);
+    DBOptions* new_options, bool input_strings_escaped = false,
+    bool ignore_unknown_options = false);
 
 // Take a default BlockBasedTableOptions "table_options" in addition to a
 // map "opts_map" of option name to option value to construct the new
@@ -229,6 +235,8 @@ Status GetDBOptionsFromMap(
 // @param input_strings_escaped when set to true, each escaped characters
 //     prefixed by '\' in the values of the opts_map will be further converted
 //     back to the raw string before assigning to the associated options.
+// @param ignore_unknown_options when set to true, unknown options are ignored
+//     instead of resulting in an unknown-option error.
 // @return Status::OK() on success.  Otherwise, a non-ok status indicating
 //     error will be returned, and "new_table_options" will be set to
 //     "table_options".
@@ -236,7 +244,7 @@ Status GetBlockBasedTableOptionsFromMap(
     const BlockBasedTableOptions& table_options,
     const std::unordered_map<std::string, std::string>& opts_map,
     BlockBasedTableOptions* new_table_options,
-    bool input_strings_escaped = false);
+    bool input_strings_escaped = false, bool ignore_unknown_options = false);
 
 // Take a default PlainTableOptions "table_options" in addition to a
 // map "opts_map" of option name to option value to construct the new
@@ -250,14 +258,16 @@ Status GetBlockBasedTableOptionsFromMap(
 // @param input_strings_escaped when set to true, each escaped characters
 //     prefixed by '\' in the values of the opts_map will be further converted
 //     back to the raw string before assigning to the associated options.
+// @param ignore_unknown_options when set to true, unknown options are ignored
+//     instead of resulting in an unknown-option error.
 // @return Status::OK() on success.  Otherwise, a non-ok status indicating
 //     error will be returned, and "new_table_options" will be set to
 //     "table_options".
 Status GetPlainTableOptionsFromMap(
     const PlainTableOptions& table_options,
     const std::unordered_map<std::string, std::string>& opts_map,
-    PlainTableOptions* new_table_options,
-    bool input_strings_escaped = false);
+    PlainTableOptions* new_table_options, bool input_strings_escaped = false,
+    bool ignore_unknown_options = false);
 
 // Take a string representation of option names and  values, apply them into the
 // base_options, and return the new options as a result. The string has the

include/rocksdb/env.h

@@ -57,10 +57,10 @@ const size_t kDefaultPageSize = 4 * 1024;
 // Options while opening a file to read/write
 struct EnvOptions {
 
-  // construct with default Options
+  // Construct with default Options
   EnvOptions();
 
-  // construct from Options
+  // Construct from Options
   explicit EnvOptions(const DBOptions& options);
 
    // If true, then use mmap to read data
@@ -95,10 +95,10 @@ struct EnvOptions {
   // WAL writes
   bool fallocate_with_keep_size = true;
 
-  // See DBOPtions doc
+  // See DBOptions doc
   size_t compaction_readahead_size;
 
-  // See DBOPtions doc
+  // See DBOptions doc
   size_t random_access_max_buffer_size;
 
   // See DBOptions doc
@@ -173,8 +173,7 @@ class Env {
   virtual Status ReopenWritableFile(const std::string& fname,
                                     unique_ptr<WritableFile>* result,
                                     const EnvOptions& options) {
-    Status s;
-    return s;
+    return Status::NotSupported();
   }
 
   // Reuse an existing file by renaming it and opening it as writable.
@@ -364,6 +363,7 @@ class Env {
   // for this environment. 'LOW' is the default pool.
   // default number: 1
   virtual void SetBackgroundThreads(int number, Priority pri = LOW) = 0;
+  virtual int GetBackgroundThreads(Priority pri = LOW) = 0;
 
   // Enlarge number of background worker threads of a specific thread pool
   // for this environment if it is smaller than specified. 'LOW' is the default
@@ -935,8 +935,8 @@ class EnvWrapper : public Env {
                          const EnvOptions& options) override {
     return target_->NewRandomRWFile(fname, result, options);
   }
-  Status NewDirectory(const std::string& name, unique_ptr<Directory>* result)
-      override {
+  Status NewDirectory(const std::string& name,
+                      unique_ptr<Directory>* result) override {
     return target_->NewDirectory(name, result);
   }
   Status FileExists(const std::string& f) override {
@@ -1004,8 +1004,8 @@ class EnvWrapper : public Env {
   Status GetTestDirectory(std::string* path) override {
     return target_->GetTestDirectory(path);
   }
-  Status NewLogger(const std::string& fname, shared_ptr<Logger>* result)
-      override {
+  Status NewLogger(const std::string& fname,
+                   shared_ptr<Logger>* result) override {
     return target_->NewLogger(fname, result);
   }
   uint64_t NowMicros() override { return target_->NowMicros(); }
@@ -1026,6 +1026,9 @@ class EnvWrapper : public Env {
   void SetBackgroundThreads(int num, Priority pri) override {
     return target_->SetBackgroundThreads(num, pri);
   }
+  int GetBackgroundThreads(Priority pri) override {
+    return target_->GetBackgroundThreads(pri);
+  }
 
   void IncBackgroundThreadsIfNeeded(int num, Priority pri) override {
     return target_->IncBackgroundThreadsIfNeeded(num, pri);

include/rocksdb/flush_block_policy.h

@@ -48,9 +48,7 @@ class FlushBlockBySizePolicyFactory : public FlushBlockPolicyFactory {
  public:
   FlushBlockBySizePolicyFactory() {}
 
-  const char* Name() const override {
-    return "FlushBlockBySizePolicyFactory";
-  }
+  const char* Name() const override { return "FlushBlockBySizePolicyFactory"; }
 
   FlushBlockPolicy* NewFlushBlockPolicy(
       const BlockBasedTableOptions& table_options,

include/rocksdb/iostats_context.h

@@ -46,12 +46,7 @@ struct IOStatsContext {
   uint64_t logger_nanos;
 };
 
-#ifndef IOS_CROSS_COMPILE
-# ifdef _MSC_VER
-extern __declspec(thread) IOStatsContext iostats_context;
-# else
-extern __thread IOStatsContext iostats_context;
-# endif
-#endif  // IOS_CROSS_COMPILE
+// Get Thread-local IOStatsContext object pointer
+IOStatsContext* get_iostats_context();
 
 }  // namespace rocksdb

include/rocksdb/memtablerep.h

@@ -43,7 +43,7 @@
 namespace rocksdb {
 
 class Arena;
-class MemTableAllocator;
+class Allocator;
 class LookupKey;
 class Slice;
 class SliceTransform;
@@ -68,7 +68,7 @@ class MemTableRep {
     virtual ~KeyComparator() { }
   };
 
-  explicit MemTableRep(MemTableAllocator* allocator) : allocator_(allocator) {}
+  explicit MemTableRep(Allocator* allocator) : allocator_(allocator) {}
 
   // Allocate a buf of len size for storing key. The idea is that a
   // specific memtable representation knows its underlying data structure
@@ -208,7 +208,7 @@ class MemTableRep {
   // user key.
   virtual Slice UserKey(const char* key) const;
 
-  MemTableAllocator* allocator_;
+  Allocator* allocator_;
 };
 
 // This is the base class for all factories that are used by RocksDB to create
@@ -216,10 +216,17 @@ class MemTableRep {
 class MemTableRepFactory {
  public:
   virtual ~MemTableRepFactory() {}
+
   virtual MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator&,
-                                         MemTableAllocator*,
-                                         const SliceTransform*,
+                                         Allocator*, const SliceTransform*,
                                          Logger* logger) = 0;
+  virtual MemTableRep* CreateMemTableRep(
+      const MemTableRep::KeyComparator& key_cmp, Allocator* allocator,
+      const SliceTransform* slice_transform, Logger* logger,
+      uint32_t /* column_family_id */) {
+    return CreateMemTableRep(key_cmp, allocator, slice_transform, logger);
+  }
+
   virtual const char* Name() const = 0;
 
   // Return true if the current MemTableRep supports concurrent inserts
@@ -238,9 +245,9 @@ class SkipListFactory : public MemTableRepFactory {
  public:
   explicit SkipListFactory(size_t lookahead = 0) : lookahead_(lookahead) {}
 
+  using MemTableRepFactory::CreateMemTableRep;
   virtual MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator&,
-                                         MemTableAllocator*,
-                                         const SliceTransform*,
+                                         Allocator*, const SliceTransform*,
                                          Logger* logger) override;
   virtual const char* Name() const override { return "SkipListFactory"; }
 
@@ -264,10 +271,12 @@ class VectorRepFactory : public MemTableRepFactory {
 
  public:
   explicit VectorRepFactory(size_t count = 0) : count_(count) { }
+
+  using MemTableRepFactory::CreateMemTableRep;
   virtual MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator&,
-                                         MemTableAllocator*,
-                                         const SliceTransform*,
+                                         Allocator*, const SliceTransform*,
                                          Logger* logger) override;
+
   virtual const char* Name() const override {
     return "VectorRepFactory";
   }

include/rocksdb/merge_operator.h

@@ -212,16 +212,12 @@ class AssociativeMergeOperator : public MergeOperator {
 
  private:
   // Default implementations of the MergeOperator functions
-  bool FullMergeV2(
-      const MergeOperationInput& merge_in,
-      MergeOperationOutput* merge_out) const override;
+  bool FullMergeV2(const MergeOperationInput& merge_in,
+                   MergeOperationOutput* merge_out) const override;
 
-  bool PartialMerge(
-      const Slice& key,
-      const Slice& left_operand,
-      const Slice& right_operand,
-      std::string* new_value,
-      Logger* logger) const override;
+  bool PartialMerge(const Slice& key, const Slice& left_operand,
+                    const Slice& right_operand, std::string* new_value,
+                    Logger* logger) const override;
 };
 
 }  // namespace rocksdb

include/rocksdb/options.h

@@ -463,24 +463,27 @@ struct DBOptions {
   // regardless of this setting
   uint64_t delete_obsolete_files_period_micros = 6ULL * 60 * 60 * 1000000;
 
-  // Suggested number of concurrent background compaction jobs, submitted to
-  // the default LOW priority thread pool.
-  //
-  // Default: 1
-  int base_background_compactions = 1;
+  // Maximum number of concurrent background jobs (compactions and flushes).
+  int max_background_jobs = 2;
 
+  // NOT SUPPORTED ANYMORE: RocksDB automatically decides this based on the
+  // value of max_background_jobs. This option is ignored.
+  int base_background_compactions = -1;
+
+  // NOT SUPPORTED ANYMORE: RocksDB automatically decides this based on the
+  // value of max_background_jobs. For backwards compatibility we will set
+  // `max_background_jobs = max_background_compactions + max_background_flushes`
+  // in the case where user sets at least one of `max_background_compactions` or
+  // `max_background_flushes` (we replace -1 by 1 in case one option is unset).
+  //
   // Maximum number of concurrent background compaction jobs, submitted to
   // the default LOW priority thread pool.
-  // We first try to schedule compactions based on
-  // `base_background_compactions`. If the compaction cannot catch up , we
-  // will increase number of compaction threads up to
-  // `max_background_compactions`.
   //
   // If you're increasing this, also consider increasing number of threads in
   // LOW priority thread pool. For more information, see
   // Env::SetBackgroundThreads
-  // Default: 1
-  int max_background_compactions = 1;
+  // Default: -1
+  int max_background_compactions = -1;
 
   // This value represents the maximum number of threads that will
   // concurrently perform a compaction job by breaking it into multiple,
@@ -488,22 +491,27 @@ struct DBOptions {
   // Default: 1 (i.e. no subcompactions)
   uint32_t max_subcompactions = 1;
 
-  // Maximum number of concurrent background memtable flush jobs, submitted to
-  // the HIGH priority thread pool.
+  // NOT SUPPORTED ANYMORE: RocksDB automatically decides this based on the
+  // value of max_background_jobs. For backwards compatibility we will set
+  // `max_background_jobs = max_background_compactions + max_background_flushes`
+  // in the case where user sets at least one of `max_background_compactions` or
+  // `max_background_flushes`.
   //
-  // By default, all background jobs (major compaction and memtable flush) go
-  // to the LOW priority pool. If this option is set to a positive number,
-  // memtable flush jobs will be submitted to the HIGH priority pool.
-  // It is important when the same Env is shared by multiple db instances.
-  // Without a separate pool, long running major compaction jobs could
-  // potentially block memtable flush jobs of other db instances, leading to
-  // unnecessary Put stalls.
+  // Maximum number of concurrent background memtable flush jobs, submitted by
+  // default to the HIGH priority thread pool. If the HIGH priority thread pool
+  // is configured to have zero threads, flush jobs will share the LOW priority
+  // thread pool with compaction jobs.
+  //
+  // It is important to use both thread pools when the same Env is shared by
+  // multiple db instances. Without a separate pool, long running compaction
+  // jobs could potentially block memtable flush jobs of other db instances,
+  // leading to unnecessary Put stalls.
   //
   // If you're increasing this, also consider increasing number of threads in
   // HIGH priority thread pool. For more information, see
   // Env::SetBackgroundThreads
-  // Default: 1
-  int max_background_flushes = 1;
+  // Default: -1
+  int max_background_flushes = -1;
 
   // Specify the maximal size of the info log file. If the log file
   // is larger than `max_log_file_size`, a new info log file will
@@ -746,10 +754,15 @@ struct DBOptions {
   // calculated using size of user write requests before compression.
   // RocksDB may decide to slow down more if the compaction still
   // gets behind further.
+  // If the value is 0, we will infer a value from `rater_limiter` value
+  // if it is not empty, or 16MB if `rater_limiter` is empty. Note that
+  // if users change the rate in `rate_limiter` after DB is opened,
+  // `delayed_write_rate` won't be adjusted.
+  //
   // Unit: byte per second.
   //
-  // Default: 16MB/s
-  uint64_t delayed_write_rate = 16 * 1024U * 1024U;
+  // Default: 0
+  uint64_t delayed_write_rate = 0;
 
   // By default, a single write thread queue is maintained. The thread gets
   // to the head of the queue becomes write batch group leader and responsible
@@ -758,7 +771,7 @@ struct DBOptions {
   // If enable_pipelined_write is true, separate write thread queue is
   // maintained for WAL write and memtable write. A write thread first enter WAL
   // writer queue and then memtable writer queue. Pending thread on the WAL
-  // writer queue thus only have to wait for previous writers to finish thier
+  // writer queue thus only have to wait for previous writers to finish their
   // WAL writing but not the memtable writing. Enabling the feature may improve
   // write throughput and reduce latency of the prepare phase of two-phase
   // commit.
@@ -929,24 +942,6 @@ enum ReadTier {
 
 // Options that control read operations
 struct ReadOptions {
-  // If true, all data read from underlying storage will be
-  // verified against corresponding checksums.
-  // Default: true
-  bool verify_checksums;
-
-  // Should the "data block"/"index block"/"filter block" read for this
-  // iteration be cached in memory?
-  // Callers may wish to set this field to false for bulk scans.
-  // Default: true
-  bool fill_cache;
-
-  // If this option is set and memtable implementation allows, Seek
-  // might only return keys with the same prefix as the seek-key
-  //
-  // ! NOT SUPPORTED ANYMORE: prefix_seek is on by default when prefix_extractor
-  // is configured
-  // bool prefix_seek;
-
   // If "snapshot" is non-nullptr, read as of the supplied snapshot
   // (which must belong to the DB that is being read and which must
   // not have been released).  If "snapshot" is nullptr, use an implicit
@@ -966,12 +961,35 @@ struct ReadOptions {
   // Default: nullptr
   const Slice* iterate_upper_bound;
 
+  // If non-zero, NewIterator will create a new table reader which
+  // performs reads of the given size. Using a large size (> 2MB) can
+  // improve the performance of forward iteration on spinning disks.
+  // Default: 0
+  size_t readahead_size;
+
+  // A threshold for the number of keys that can be skipped before failing an
+  // iterator seek as incomplete. The default value of 0 should be used to
+  // never fail a request as incomplete, even on skipping too many keys.
+  // Default: 0
+  uint64_t max_skippable_internal_keys;
+
   // Specify if this read request should process data that ALREADY
   // resides on a particular cache. If the required data is not
   // found at the specified cache, then Status::Incomplete is returned.
   // Default: kReadAllTier
   ReadTier read_tier;
 
+  // If true, all data read from underlying storage will be
+  // verified against corresponding checksums.
+  // Default: true
+  bool verify_checksums;
+
+  // Should the "data block"/"index block"/"filter block" read for this
+  // iteration be cached in memory?
+  // Callers may wish to set this field to false for bulk scans.
+  // Default: true
+  bool fill_cache;
+
   // Specify to create a tailing iterator -- a special iterator that has a
   // view of the complete database (i.e. it can also be used to read newly
   // added data) and is optimized for sequential reads. It will return records
@@ -1017,24 +1035,12 @@ struct ReadOptions {
   // Default: false
   bool background_purge_on_iterator_cleanup;
 
-  // If non-zero, NewIterator will create a new table reader which
-  // performs reads of the given size. Using a large size (> 2MB) can
-  // improve the performance of forward iteration on spinning disks.
-  // Default: 0
-  size_t readahead_size;
-
   // If true, keys deleted using the DeleteRange() API will be visible to
   // readers until they are naturally deleted during compaction. This improves
   // read performance in DBs with many range deletions.
   // Default: false
   bool ignore_range_deletions;
 
-  // A threshold for the number of keys that can be skipped before failing an
-  // iterator seek as incomplete. The default value of 0 should be used to
-  // never fail a request as incomplete, even on skipping too many keys.
-  // Default: 0
-  uint64_t max_skippable_internal_keys;
-
   ReadOptions();
   ReadOptions(bool cksum, bool cache);
 };
@@ -1073,11 +1079,21 @@ struct WriteOptions {
   // immediately with Status::Incomplete().
   bool no_slowdown;
 
+  // If true, this write request is of lower priority if compaction is
+  // behind. In this case, no_slowdown = true, the request will be cancelled
+  // immediately with Status::Incomplete() returned. Otherwise, it will be
+  // slowed down. The slowdown value is determined by RocksDB to guarantee
+  // it introduces minimum impacts to high priority writes.
+  //
+  // Default: false
+  bool low_pri;
+
   WriteOptions()
       : sync(false),
         disableWAL(false),
         ignore_missing_column_families(false),
-        no_slowdown(false) {}
+        no_slowdown(false),
+        low_pri(false) {}
 };
 
 // Options that control flush operations

include/rocksdb/perf_context.h

@@ -150,18 +150,9 @@ struct PerfContext {
   uint64_t env_new_logger_nanos;
 };
 
-#if defined(NPERF_CONTEXT) || defined(IOS_CROSS_COMPILE)
-extern PerfContext perf_context;
-#elif defined(_MSC_VER)
-extern __declspec(thread) PerfContext perf_context;
-#else
-  #if defined(OS_SOLARIS)
-    PerfContext *getPerfContext();
-    #define perf_context (*getPerfContext())
-  #else
-    extern __thread PerfContext perf_context;
-  #endif
-#endif
+// Get Thread-local PerfContext object pointer
+// if defined(NPERF_CONTEXT), then the pointer is not thread-local
+PerfContext* get_perf_context();
 
 }
 

include/rocksdb/perf_level.h

@@ -12,7 +12,6 @@
 namespace rocksdb {
 
 // How much perf stats to collect. Affects perf_context and iostats_context.
-
 enum PerfLevel : unsigned char {
   kUninitialized = 0,             // unknown setting
   kDisable = 1,                   // disable perf stats

include/rocksdb/rate_limiter.h

@@ -51,6 +51,8 @@ class RateLimiter {
   // Total # of requests that go though rate limiter
   virtual int64_t GetTotalRequests(
       const Env::IOPriority pri = Env::IO_TOTAL) const = 0;
+
+  virtual int64_t GetBytesPerSecond() const = 0;
 };
 
 // Create a RateLimiter object, which can be shared among RocksDB instances to