bump version

Summary:
Fix for #2833.

- In `DeleteFilesInRange`, use `GetCleanInputsWithinInterval` instead of `GetOverlappingInputs` to make sure we get a clean cut set of files to delete.
- In `GetCleanInputsWithinInterval`, support nullptr as `begin_key` or `end_key`.
- In `GetOverlappingInputsRangeBinarySearch`, move the assertion for non-empty range away from `ExtendFileRangeWithinInterval`, which should be allowed to return an empty range (via `end_index < begin_index`).
Closes https://github.com/facebook/rocksdb/pull/2843

Differential Revision: D5772387

Pulled By: ajkr

fbshipit-source-id: e554e8461823c6be82b21a9262a2da02b3957881

.travis.yml

@@ -20,9 +20,7 @@ addons:
 env:
   - TEST_GROUP=platform_dependent # 16-18 minutes
   - TEST_GROUP=1 # 33-35 minutes
-  - TEST_GROUP=2 # 18-20 minutes
-  - TEST_GROUP=3 # 20-22 minutes
-  - TEST_GROUP=4 # 12-14 minutes
+  - TEST_GROUP=2 # 30-32 minutes
   # Run java tests
   - JOB_NAME=java_test # 4-11 minutes
   # Build ROCKSDB_LITE
@@ -38,10 +36,6 @@ matrix:
     env: TEST_GROUP=1
   - os: osx
     env: TEST_GROUP=2
-  - os: osx
-    env: TEST_GROUP=3
-  - os: osx
-    env: TEST_GROUP=4
   - os : osx
     env: JOB_NAME=cmake-mingw
   - os : linux
@@ -55,9 +49,6 @@ install:
       brew install ccache;
       PATH=$PATH:/usr/local/opt/ccache/libexec;
     fi
-  - if [[ "${JOB_NAME}" == cmake* ]] && [ "${TRAVIS_OS_NAME}" == linux ]; then
-      mkdir cmake-dist && curl -sfSL https://cmake.org/files/v3.8/cmake-3.8.1-Linux-x86_64.tar.gz | tar --strip-components=1 -C cmake-dist -xz && export PATH=$PWD/cmake-dist/bin:$PATH;
-    fi
 
 before_script:
   # Increase the maximum number of open file descriptors, since some tests use
@@ -66,17 +57,14 @@ before_script:
 
 script:
   - ${CXX} --version
-  - if [ `command -v ccache` ]; then ccache -C; fi
-  - if [ "${TEST_GROUP}" == 'platform_dependent' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_END=db_block_cache_test make -j4 all_but_some_tests check_some; fi
-  - if [ "${TEST_GROUP}" == '1' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=db_block_cache_test ROCKSDBTESTS_END=full_filter_block_test make -j4 check_some; fi
-  - if [ "${TEST_GROUP}" == '2' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=full_filter_block_test ROCKSDBTESTS_END=write_batch_with_index_test make -j4 check_some; fi
-  - if [ "${TEST_GROUP}" == '3' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=write_batch_with_index_test ROCKSDBTESTS_END=write_prepared_transaction_test make -j4 check_some; fi
-  - if [ "${TEST_GROUP}" == '4' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=write_prepared_transaction_test make -j4 check_some; fi
+  - if [ "${TEST_GROUP}" == 'platform_dependent' ]; then ccache -C && OPT=-DTRAVIS V=1 ROCKSDBTESTS_END=db_block_cache_test make -j4 all_but_some_tests check_some; fi
+  - if [ "${TEST_GROUP}" == '1' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=db_block_cache_test ROCKSDBTESTS_END=comparator_db_test make -j4 check_some; fi
+  - if [ "${TEST_GROUP}" == '2' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=comparator_db_test make -j4 check_some; fi
   - if [ "${JOB_NAME}" == 'java_test' ]; then OPT=-DTRAVIS V=1 make clean jclean && make rocksdbjava jtest; fi
   - if [ "${JOB_NAME}" == 'lite_build' ]; then OPT="-DTRAVIS -DROCKSDB_LITE" V=1 make -j4 static_lib tools; fi
   - if [ "${JOB_NAME}" == 'examples' ]; then OPT=-DTRAVIS V=1 make -j4 static_lib; cd examples; make -j4; fi
-  - if [ "${JOB_NAME}" == 'cmake' ]; then mkdir build && cd build && cmake -DJNI=1 .. && make -j4 rocksdb rocksdbjni; fi
-  - if [ "${JOB_NAME}" == 'cmake-mingw' ]; then mkdir build && cd build && cmake -DJNI=1 .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni; fi
+  - if [ "${JOB_NAME}" == 'cmake' ]; then mkdir build && cd build && cmake .. && make -j4 rocksdb; fi
+  - if [ "${JOB_NAME}" == 'cmake-mingw' ]; then mkdir build && cd build && cmake .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb; fi
 notifications:
     email:
       - leveldb@fb.com

AUTHORS

@@ -9,4 +9,3 @@ Sanjay Ghemawat <sanjay@google.com>
 # Partial list of contributors:
 Kevin Regan <kevin.d.regan@gmail.com>
 Johan Bilien <jobi@litl.com>
-Matthew Von-Maszewski <https://github.com/matthewvon> (Basho Technologies)

CMakeLists.txt

@@ -14,9 +14,9 @@
 #        cd build
 # 3. Run cmake to generate project files for Windows, add more options to enable required third-party libraries.
 #    See thirdparty.inc for more information.
-#        sample command: cmake -G "Visual Studio 15 Win64" -DWITH_GFLAGS=1 -DWITH_SNAPPY=1 -DWITH_JEMALLOC=1 -DWITH_JNI=1 ..
+#        sample command: cmake -G "Visual Studio 14 Win64" -DGFLAGS=1 -DSNAPPY=1 -DJEMALLOC=1 -DJNI=1 ..
 # 4. Then build the project in debug mode (you may want to add /m[:<N>] flag to run msbuild in <N> parallel threads
-#                                          or simply /m to use all avail cores)
+#                                          or simply /m ot use all avail cores)
 #        msbuild rocksdb.sln
 #
 #        rocksdb.sln build features exclusions of test only code in Release. If you build ALL_BUILD then everything
@@ -32,31 +32,21 @@
 # 3. cmake ..
 # 4. make -j
 
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 2.6)
 project(rocksdb)
-enable_language(CXX)
-enable_language(C)
-enable_language(ASM)
 
 if(POLICY CMP0042)
   cmake_policy(SET CMP0042 NEW)
 endif()
 
-list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules/")
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/modules/")
 
 option(WITH_JEMALLOC "build with JeMalloc" OFF)
-option(WITH_SNAPPY "build with SNAPPY" OFF)
-option(WITH_LZ4 "build with lz4" OFF)
-option(WITH_ZLIB "build with zlib" OFF)
 if(MSVC)
-  # Defaults currently different for GFLAGS.
-  #  We will address find_package work a little later
-  option(WITH_GFLAGS "build with GFlags" OFF)
-  option(WITH_XPRESS "build with windows built in compression" OFF)
   include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc)
 else()
   if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
-    # FreeBSD has jemalloc as default malloc
+    # FreeBSD has jemaloc as default malloc
     # but it does not have all the jemalloc files in include/...
     set(WITH_JEMALLOC ON)
   else()
@@ -64,21 +54,10 @@ else()
       find_package(JeMalloc REQUIRED)
       add_definitions(-DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE)
       include_directories(${JEMALLOC_INCLUDE_DIR})
-      list(APPEND THIRDPARTY_LIBS ${JEMALLOC_LIBRARIES})
-    endif()
-  endif()
-
-  # No config file for this
-  option(WITH_GFLAGS "build with GFlags" ON)
-  if(WITH_GFLAGS)
-    find_package(gflags)
-    if(gflags_FOUND)
-      add_definitions(-DGFLAGS=1)
-      include_directories(${gflags_INCLUDE_DIR})
-      list(APPEND THIRDPARTY_LIBS ${gflags_LIBRARIES})
     endif()
   endif()
 
+  option(WITH_SNAPPY "build with SNAPPY" OFF)
   if(WITH_SNAPPY)
     find_package(snappy REQUIRED)
     add_definitions(-DSNAPPY)
@@ -86,6 +65,7 @@ else()
     list(APPEND THIRDPARTY_LIBS ${SNAPPY_LIBRARIES})
   endif()
 
+  option(WITH_ZLIB "build with zlib" OFF)
   if(WITH_ZLIB)
     find_package(zlib REQUIRED)
     add_definitions(-DZLIB)
@@ -101,6 +81,7 @@ else()
     list(APPEND THIRDPARTY_LIBS ${BZIP2_LIBRARIES})
   endif()
 
+  option(WITH_LZ4 "build with lz4" OFF)
   if(WITH_LZ4)
     find_package(lz4 REQUIRED)
     add_definitions(-DLZ4)
@@ -117,7 +98,17 @@ else()
   endif()
 endif()
 
-string(TIMESTAMP GIT_DATE_TIME "%Y/%m/%d %H:%M:%S" UTC)
+if(WIN32)
+  execute_process(COMMAND powershell -noprofile -Command "Get-Date -format MM_dd_yyyy" OUTPUT_VARIABLE DATE)
+  execute_process(COMMAND powershell -noprofile -Command "Get-Date -format HH:mm:ss" OUTPUT_VARIABLE TIME)
+  string(REGEX REPLACE "(..)_(..)_..(..).*" "\\1/\\2/\\3" DATE "${DATE}")
+  string(REGEX REPLACE "(..):(.....).*" " \\1:\\2" TIME "${TIME}")
+  set(GIT_DATE_TIME "${DATE} ${TIME}")
+else()
+  execute_process(COMMAND date "+%Y/%m/%d %H:%M:%S" OUTPUT_VARIABLE DATETIME)
+  string(REGEX REPLACE "\n" "" DATETIME ${DATETIME})
+  set(GIT_DATE_TIME "${DATETIME}")
+endif()
 
 find_package(Git)
 
@@ -133,17 +124,20 @@ endif()
 
 string(REGEX REPLACE "[^0-9a-f]+" "" GIT_SHA "${GIT_SHA}")
 
-
-# Read rocksdb version from version.h header file.
-file(READ include/rocksdb/version.h version_header_file)
-string(REGEX MATCH "#define ROCKSDB_MAJOR ([0-9]+)" _ ${version_header_file})
-set(ROCKSDB_VERSION_MAJOR ${CMAKE_MATCH_1})
-string(REGEX MATCH "#define ROCKSDB_MINOR ([0-9]+)" _ ${version_header_file})
-set(ROCKSDB_VERSION_MINOR ${CMAKE_MATCH_1})
-string(REGEX MATCH "#define ROCKSDB_PATCH ([0-9]+)" _ ${version_header_file})
-set(ROCKSDB_VERSION_PATCH ${CMAKE_MATCH_1})
-set(ROCKSDB_VERSION ${ROCKSDB_VERSION_MAJOR}.${ROCKSDB_VERSION_MINOR}.${ROCKSDB_VERSION_PATCH})
-
+if(NOT WIN32)
+  execute_process(COMMAND
+      "./build_tools/version.sh" "full"
+      WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+      OUTPUT_VARIABLE ROCKSDB_VERSION
+  )
+  string(STRIP "${ROCKSDB_VERSION}" ROCKSDB_VERSION)
+  execute_process(COMMAND
+      "./build_tools/version.sh" "major"
+      WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+      OUTPUT_VARIABLE ROCKSDB_VERSION_MAJOR
+  )
+  string(STRIP "${ROCKSDB_VERSION_MAJOR}" ROCKSDB_VERSION_MAJOR)
+endif()
 
 option(WITH_MD_LIBRARY "build with MD" ON)
 if(WIN32 AND MSVC)
@@ -161,10 +155,10 @@ target_include_directories(build_version PRIVATE
   ${CMAKE_CURRENT_SOURCE_DIR}/util)
 if(MSVC)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W4 /wd4127 /wd4800 /wd4996 /wd4351 /wd4100 /wd4204 /wd4324")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W3 /wd4127 /wd4800 /wd4996 /wd4351")
 else()
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wshadow -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers -Wno-strict-aliasing")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wshadow -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers")
   if(MINGW)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format")
   endif()
@@ -179,56 +173,33 @@ else()
   endif()
 endif()
 
-include(CheckCCompilerFlag)
-if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le")
-  CHECK_C_COMPILER_FLAG("-maltivec" HAS_ALTIVEC)
-  if(HAS_ALTIVEC)
-    message(STATUS " HAS_ALTIVEC yes")
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -maltivec")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -maltivec")
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mcpu=power8")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=power8")
-  endif(HAS_ALTIVEC)
-endif(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le")
-
 option(PORTABLE "build a portable binary" OFF)
 option(FORCE_SSE42 "force building with SSE4.2, even when PORTABLE=ON" OFF)
 if(PORTABLE)
   # MSVC does not need a separate compiler flag to enable SSE4.2; if nmmintrin.h
   # is available, it is available by default.
   if(FORCE_SSE42 AND NOT MSVC)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2 -mpclmul")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2")
   endif()
 else()
   if(MSVC)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2")
   else()
-    if(NOT HAVE_POWER8)
-      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
-    endif()
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
   endif()
 endif()
 
+set(CMAKE_REQUIRED_FLAGS ${CMAKE_CXX_FLAGS})
 include(CheckCXXSourceCompiles)
-if(NOT MSVC)
-  set(CMAKE_REQUIRED_FLAGS "-msse4.2 -mpclmul")
-endif()
 CHECK_CXX_SOURCE_COMPILES("
 #include <cstdint>
 #include <nmmintrin.h>
-#include <wmmintrin.h>
 int main() {
   volatile uint32_t x = _mm_crc32_u32(0, 0);
-  const auto a = _mm_set_epi64x(0, 0);
-  const auto b = _mm_set_epi64x(0, 0);
-  const auto c = _mm_clmulepi64_si128(a, b, 0x00);
-  auto d = _mm_cvtsi128_si64(c);
 }
 " HAVE_SSE42)
-unset(CMAKE_REQUIRED_FLAGS)
 if(HAVE_SSE42)
   add_definitions(-DHAVE_SSE42)
-  add_definitions(-DHAVE_PCLMUL)
 elseif(FORCE_SSE42)
   message(FATAL_ERROR "FORCE_SSE42=ON but unable to compile with SSE4.2 enabled")
 endif()
@@ -285,28 +256,6 @@ if(WITH_UBSAN)
   endif()
 endif()
 
-option(WITH_NUMA "build with NUMA policy support" OFF)
-if(WITH_NUMA)
-  find_package(NUMA REQUIRED)
-  add_definitions(-DNUMA)
-  include_directories(${NUMA_INCLUDE_DIR})
-  list(APPEND THIRDPARTY_LIBS ${NUMA_LIBRARIES})
-endif()
-
-option(WITH_TBB "build with Threading Building Blocks (TBB)" OFF)
-if(WITH_TBB)
-  find_package(TBB REQUIRED)
-  add_definitions(-DTBB)
-  include_directories(${TBB_INCLUDE_DIR})
-  list(APPEND THIRDPARTY_LIBS ${TBB_LIBRARIES})
-endif()
-
-# Stall notifications eat some performance from inserts
-option(DISABLE_STALL_NOTIF "Build with stall notifications" OFF)
-if(DISABLE_STALL_NOTIF)
-  add_definitions(-DROCKSDB_DISABLE_STALL_NOTIFICATION)
-endif()
-
 # Used to run CI build and tests so we can run faster
 set(OPTIMIZE_DEBUG_DEFAULT 0)        # Debug build is unoptimized by default use -DOPTDBG=1 to optimize
 
@@ -316,22 +265,6 @@ else()
    set(OPTIMIZE_DEBUG ${OPTIMIZE_DEBUG_DEFAULT})
 endif()
 
-if(DEFINED USE_RTTI)
-  if(USE_RTTI)
-    message(STATUS "Enabling RTTI")
-    set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DROCKSDB_USE_RTTI")
-    set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -DROCKSDB_USE_RTTI")
-  else()
-    message(STATUS "Disabling RTTI")
-    set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fno-rtti")
-    set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fno-rtti")
-  endif()
-else()
-  message(STATUS "Enabling RTTI in Debug builds only (default)")
-  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DROCKSDB_USE_RTTI")
-  set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fno-rtti")
-endif()
-
 if(MSVC)
   if((${OPTIMIZE_DEBUG} EQUAL 1))
     message(STATUS "Debug optimization is enabled")
@@ -390,8 +323,11 @@ if(NOT WIN32)
 endif()
 
 option(WITH_FALLOCATE "build with fallocate" ON)
+
 if(WITH_FALLOCATE)
-  CHECK_CXX_SOURCE_COMPILES("
+  set(CMAKE_REQUIRED_FLAGS ${CMAKE_C_FLAGS})
+  include(CheckCSourceCompiles)
+  CHECK_C_SOURCE_COMPILES("
 #include <fcntl.h>
 #include <linux/falloc.h>
 int main() {
@@ -404,38 +340,12 @@ int main() {
   endif()
 endif()
 
-CHECK_CXX_SOURCE_COMPILES("
-#include <fcntl.h>
-int main() {
-  int fd = open(\"/dev/null\", 0);
-  sync_file_range(fd, 0, 1024, SYNC_FILE_RANGE_WRITE);
-}
-" HAVE_SYNC_FILE_RANGE_WRITE)
-if(HAVE_SYNC_FILE_RANGE_WRITE)
-  add_definitions(-DROCKSDB_RANGESYNC_PRESENT)
-endif()
-
-CHECK_CXX_SOURCE_COMPILES("
-#include <pthread.h>
-int main() {
-  (void) PTHREAD_MUTEX_ADAPTIVE_NP;
-}
-" HAVE_PTHREAD_MUTEX_ADAPTIVE_NP)
-if(HAVE_PTHREAD_MUTEX_ADAPTIVE_NP)
-  add_definitions(-DROCKSDB_PTHREAD_ADAPTIVE_MUTEX)
-endif()
-
-include(CheckCXXSymbolExists)
-check_cxx_symbol_exists(malloc_usable_size malloc.h HAVE_MALLOC_USABLE_SIZE)
+include(CheckFunctionExists)
+CHECK_FUNCTION_EXISTS(malloc_usable_size HAVE_MALLOC_USABLE_SIZE)
 if(HAVE_MALLOC_USABLE_SIZE)
   add_definitions(-DROCKSDB_MALLOC_USABLE_SIZE)
 endif()
 
-check_cxx_symbol_exists(sched_getcpu sched.h HAVE_SCHED_GETCPU)
-if(HAVE_SCHED_GETCPU)
-  add_definitions(-DROCKSDB_SCHED_GETCPU_PRESENT)
-endif()
-
 include_directories(${PROJECT_SOURCE_DIR})
 include_directories(${PROJECT_SOURCE_DIR}/include)
 include_directories(SYSTEM ${PROJECT_SOURCE_DIR}/third-party/gtest-1.7.0/fused-src)
@@ -479,7 +389,6 @@ set(SOURCES
         db/flush_scheduler.cc
         db/forward_iterator.cc
         db/internal_stats.cc
-        db/logs_with_prep_tracker.cc
         db/log_reader.cc
         db/log_writer.cc
         db/malloc_stats.cc
@@ -539,7 +448,6 @@ set(SOURCES
         table/block_based_table_factory.cc
         table/block_based_table_reader.cc
         table/block_builder.cc
-        table/block_fetcher.cc
         table/block_prefix_index.cc
         table/bloom_block.cc
         table/cuckoo_table_builder.cc
@@ -594,14 +502,12 @@ set(SOURCES
         util/status_message.cc
         util/string_util.cc
         util/sync_point.cc
-        util/sync_point_impl.cc
         util/testutil.cc
         util/thread_local.cc
         util/threadpool_imp.cc
         util/transaction_test_util.cc
         util/xxhash.cc
         utilities/backupable/backupable_db.cc
-        utilities/blob_db/blob_compaction_filter.cc
         utilities/blob_db/blob_db.cc
         utilities/blob_db/blob_db_impl.cc
         utilities/blob_db/blob_dump_tool.cc
@@ -629,7 +535,6 @@ set(SOURCES
         utilities/leveldb_options/leveldb_options.cc
         utilities/lua/rocks_lua_compaction_filter.cc
         utilities/memory/memory_util.cc
-        utilities/merge_operators/bytesxor.cc
         utilities/merge_operators/max.cc
         utilities/merge_operators/put.cc
         utilities/merge_operators/string_append/stringappend.cc
@@ -648,34 +553,18 @@ set(SOURCES
         utilities/table_properties_collectors/compact_on_deletion_collector.cc
         utilities/transactions/optimistic_transaction_db_impl.cc
         utilities/transactions/optimistic_transaction.cc
-        utilities/transactions/pessimistic_transaction.cc
-        utilities/transactions/pessimistic_transaction_db.cc
-        utilities/transactions/snapshot_checker.cc
         utilities/transactions/transaction_base.cc
+        utilities/transactions/pessimistic_transaction_db.cc
         utilities/transactions/transaction_db_mutex_impl.cc
+        utilities/transactions/pessimistic_transaction.cc
         utilities/transactions/transaction_lock_mgr.cc
         utilities/transactions/transaction_util.cc
         utilities/transactions/write_prepared_txn.cc
-        utilities/transactions/write_prepared_txn_db.cc
         utilities/ttl/db_ttl_impl.cc
         utilities/write_batch_with_index/write_batch_with_index.cc
         utilities/write_batch_with_index/write_batch_with_index_internal.cc
         $<TARGET_OBJECTS:build_version>)
 
-if(HAVE_SSE42 AND NOT FORCE_SSE42)
-  if(NOT MSVC)
-set_source_files_properties(
-  util/crc32c.cc
-      PROPERTIES COMPILE_FLAGS "-msse4.2 -mpclmul")
-  endif()
-endif()
-
-if(HAVE_POWER8)
-  list(APPEND SOURCES
-    util/crc32c_ppc.c
-    util/crc32c_ppc_asm.S)
-endif(HAVE_POWER8)
-
 if(WIN32)
   list(APPEND SOURCES
     port/win/io_win.cc
@@ -683,18 +572,14 @@ if(WIN32)
     port/win/env_default.cc
     port/win/port_win.cc
     port/win/win_logger.cc
-    port/win/win_thread.cc)
-
-if(WITH_XPRESS)
-  list(APPEND SOURCES
+    port/win/win_thread.cc
     port/win/xpress_win.cc)
-endif()
-
+	
 if(WITH_JEMALLOC)
   list(APPEND SOURCES
     port/win/win_jemalloc.cc)
 endif()
-
+	
 else()
   list(APPEND SOURCES
     port/port_posix.cc
@@ -705,14 +590,6 @@ endif()
 set(ROCKSDB_STATIC_LIB rocksdb${ARTIFACT_SUFFIX})
 set(ROCKSDB_SHARED_LIB rocksdb-shared${ARTIFACT_SUFFIX})
 set(ROCKSDB_IMPORT_LIB ${ROCKSDB_SHARED_LIB})
-
-option(WITH_LIBRADOS "Build with librados" OFF)
-if(WITH_LIBRADOS)
-  list(APPEND SOURCES
-    utilities/env_librados.cc)
-  list(APPEND THIRDPARTY_LIBS rados)
-endif()
-
 if(WIN32)
   set(SYSTEM_LIBS ${SYSTEM_LIBS} Shlwapi.lib Rpcrt4.lib)
   set(LIBS ${ROCKSDB_STATIC_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
@@ -731,6 +608,13 @@ else()
                         OUTPUT_NAME "rocksdb")
 endif()
 
+option(WITH_LIBRADOS "Build with librados" OFF)
+if(WITH_LIBRADOS)
+  list(APPEND SOURCES
+    utilities/env_librados.cc)
+  list(APPEND THIRDPARTY_LIBS rados)
+endif()
+
 add_library(${ROCKSDB_STATIC_LIB} STATIC ${SOURCES})
 target_link_libraries(${ROCKSDB_STATIC_LIB}
   ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
@@ -775,7 +659,7 @@ if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS)
   set(package_config_destination ${CMAKE_INSTALL_LIBDIR}/cmake/rocksdb)
 
   configure_package_config_file(
-    ${CMAKE_CURRENT_LIST_DIR}/cmake/RocksDBConfig.cmake.in RocksDBConfig.cmake
+    ${CMAKE_SOURCE_DIR}/cmake/RocksDBConfig.cmake.in RocksDBConfig.cmake
     INSTALL_DESTINATION ${package_config_destination}
   )
 
@@ -845,7 +729,6 @@ if(WITH_TESTS)
         db/db_inplace_update_test.cc
         db/db_io_failure_test.cc
         db/db_iter_test.cc
-        db/db_iter_stress_test.cc
         db/db_iterator_test.cc
         db/db_log_iter_test.cc
         db/db_memtable_test.cc
@@ -864,7 +747,6 @@ if(WITH_TESTS)
         db/db_write_test.cc
         db/dbformat_test.cc
         db/deletefile_test.cc
-        db/obsolete_files_test.cc
         db/external_sst_file_basic_test.cc
         db/external_sst_file_test.cc
         db/fault_injection_test.cc
@@ -956,7 +838,6 @@ if(WITH_TESTS)
         utilities/table_properties_collectors/compact_on_deletion_collector_test.cc
         utilities/transactions/optimistic_transaction_test.cc
         utilities/transactions/transaction_test.cc
-        utilities/transactions/write_prepared_transaction_test.cc
         utilities/ttl/ttl_test.cc
         utilities/write_batch_with_index/write_batch_with_index_test.cc
   )
@@ -1007,18 +888,17 @@ if(WITH_TESTS)
 
   foreach(sourcefile ${TEST_EXES})
       get_filename_component(exename ${sourcefile} NAME_WE)
-      add_executable(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} ${sourcefile}
+      add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile}
         $<TARGET_OBJECTS:testharness>)
-      set_target_properties(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX}
+      set_target_properties(${exename}${ARTIFACT_SUFFIX}
         PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
         EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
         EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
-        OUTPUT_NAME ${exename}${ARTIFACT_SUFFIX}
         )
-      target_link_libraries(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} gtest ${LIBS})
+      target_link_libraries(${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} gtest ${LIBS})
       if(NOT "${exename}" MATCHES "db_sanity_test")
         add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
-        add_dependencies(check ${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX})
+        add_dependencies(check ${exename}${ARTIFACT_SUFFIX})
       endif()
   endforeach(sourcefile ${TEST_EXES})
 

CODE_OF_CONDUCT.md

@@ -1,3 +0,0 @@
-# Code of Conduct
-
-Facebook has adopted a Code of Conduct that we expect project participants to adhere to. Please [read the full text](https://code.facebook.com/codeofconduct) so that you can understand what actions will and will not be tolerated.

CONTRIBUTING.md

@@ -1,8 +1,5 @@
 # Contributing to RocksDB
 
-## Code of Conduct
-The code of conduct is described in [`CODE_OF_CONDUCT.md`](CODE_OF_CONDUCT.md)
-
 ## Contributor License Agreement ("CLA")
 
 In order to accept your pull request, we need you to submit a CLA. You

HISTORY.md

@@ -1,136 +1,19 @@
 # Rocksdb Change Log
-## Unreleased
-### Public API Change
-* For users of `Statistics` objects created via `CreateDBStatistics()`, the format of the string returned by its `ToString()` method has changed.
-
-## 5.14.0 (5/16/2018)
-### Public API Change
-* Add a BlockBasedTableOption to align uncompressed data blocks on the smaller of block size or page size boundary, to reduce flash reads by avoiding reads spanning 4K pages.
-* The background thread naming convention changed (on supporting platforms) to "rocksdb:<thread pool priority><thread number>", e.g., "rocksdb:low0".
-* Add a new ticker stat rocksdb.number.multiget.keys.found to count number of keys successfully read in MultiGet calls
-* Touch-up to write-related counters in PerfContext. New counters added: write_scheduling_flushes_compactions_time, write_thread_wait_nanos. Counters whose behavior was fixed or modified: write_memtable_time, write_pre_and_post_process_time, write_delay_time.
-* Posix Env's NewRandomRWFile() will fail if the file doesn't exist.
-* Now, `DBOptions::use_direct_io_for_flush_and_compaction` only applies to background writes, and `DBOptions::use_direct_reads` applies to both user reads and background reads. This conforms with Linux's `open(2)` manpage, which advises against simultaneously reading a file in buffered and direct modes, due to possibly undefined behavior and degraded performance.
-* Iterator::Valid() always returns false if !status().ok(). So, now when doing a Seek() followed by some Next()s, there's no need to check status() after every operation.
-* Iterator::Seek()/SeekForPrev()/SeekToFirst()/SeekToLast() always resets status().
-* Introduced `CompressionOptions::kDefaultCompressionLevel`, which is a generic way to tell RocksDB to use the compression library's default level. It is now the default value for `CompressionOptions::level`. Previously the level defaulted to -1, which gave poor compression ratios in ZSTD.
-
-### New Features
-* Introduce TTL for level compaction so that all files older than ttl go through the compaction process to get rid of old data.
-* TransactionDBOptions::write_policy can be configured to enable WritePrepared 2PC transactions. Read more about them in the wiki.
-* Add DB properties "rocksdb.block-cache-capacity", "rocksdb.block-cache-usage", "rocksdb.block-cache-pinned-usage" to show block cache usage.
-* Add `Env::LowerThreadPoolCPUPriority(Priority)` method, which lowers the CPU priority of background (esp. compaction) threads to minimize interference with foreground tasks.
-* Fsync parent directory after deleting a file in delete scheduler.
-* In level-based compaction, if bottom-pri thread pool was setup via `Env::SetBackgroundThreads()`, compactions to the bottom level will be delegated to that thread pool.
-* `prefix_extractor` has been moved from ImmutableCFOptions to MutableCFOptions, meaning it can be dynamically changed without a DB restart.
-
+## 5.8.8 (12/6/2017)
 ### Bug Fixes
-* Fsync after writing global seq number to the ingestion file in ExternalSstFileIngestionJob.
-* Fix WAL corruption caused by race condition between user write thread and FlushWAL when two_write_queue is not set.
-* Fix `BackupableDBOptions::max_valid_backups_to_open` to not delete backup files when refcount cannot be accurately determined.
-* Fix memory leak when pin_l0_filter_and_index_blocks_in_cache is used with partitioned filters
-* Disable rollback of merge operands in WritePrepared transactions to work around an issue in MyRocks. It can be enabled back by setting TransactionDBOptions::rollback_merge_operands to true.
-* Fix wrong results by ReverseBytewiseComparator::FindShortSuccessor()
-
-### Java API Changes
-* Add `BlockBasedTableConfig.setBlockCache` to allow sharing a block cache across DB instances.
-* Added SstFileManager to the Java API to allow managing SST files across DB instances.
-
-## 5.13.0 (3/20/2018)
-### Public API Change
-* RocksDBOptionsParser::Parse()'s `ignore_unknown_options` argument will only be effective if the option file shows it is generated using a higher version of RocksDB than the current version.
-* Remove CompactionEventListener.
-
-### New Features
-* SstFileManager now can cancel compactions if they will result in max space errors. SstFileManager users can also use SetCompactionBufferSize to specify how much space must be leftover during a compaction for auxiliary file functions such as logging and flushing.
-* Avoid unnecessarily flushing in `CompactRange()` when the range specified by the user does not overlap unflushed memtables.
-* If `ColumnFamilyOptions::max_subcompactions` is set greater than one, we now parallelize large manual level-based compactions.
-* Add "rocksdb.live-sst-files-size" DB property to return total bytes of all SST files belong to the latest LSM tree.
-* NewSstFileManager to add an argument bytes_max_delete_chunk with default 64MB. With this argument, a file larger than 64MB will be ftruncated multiple times based on this size.
-
-### Bug Fixes
-* Fix a leak in prepared_section_completed_ where the zeroed entries would not removed from the map.
-* Fix WAL corruption caused by race condition between user write thread and backup/checkpoint thread.
-
-## 5.12.0 (2/14/2018)
-### Public API Change
-* Iterator::SeekForPrev is now a pure virtual method. This is to prevent user who implement the Iterator interface fail to implement SeekForPrev by mistake.
-* Add `include_end` option to make the range end exclusive when `include_end == false` in `DeleteFilesInRange()`.
-* Add `CompactRangeOptions::allow_write_stall`, which makes `CompactRange` start working immediately, even if it causes user writes to stall. The default value is false, meaning we add delay to `CompactRange` calls until stalling can be avoided when possible. Note this delay is not present in previous RocksDB versions.
-* Creating checkpoint with empty directory now returns `Status::InvalidArgument`; previously, it returned `Status::IOError`.
-* Adds a BlockBasedTableOption to turn off index block compression.
-* Close() method now returns a status when closing a db.
-
-### New Features
-* Improve the performance of iterators doing long range scans by using readahead.
-* Add new function `DeleteFilesInRanges()` to delete files in multiple ranges at once for better performance.
-* FreeBSD build support for RocksDB and RocksJava.
-* Improved performance of long range scans with readahead.
-* Updated to and now continuously tested in Visual Studio 2017.
-
-### Bug Fixes
-* Fix `DisableFileDeletions()` followed by `GetSortedWalFiles()` to not return obsolete WAL files that `PurgeObsoleteFiles()` is going to delete.
-* Fix Handle error return from WriteBuffer() during WAL file close and DB close.
-* Fix advance reservation of arena block addresses.
-* Fix handling of empty string as checkpoint directory.
-
-## 5.11.0 (01/08/2018)
-### Public API Change
-* Add `autoTune` and `getBytesPerSecond()` to RocksJava RateLimiter
-
-### New Features
-* Add a new histogram stat called rocksdb.db.flush.micros for memtable flush.
-* Add "--use_txn" option to use transactional API in db_stress.
-* Disable onboard cache for compaction output in Windows platform.
-* Improve the performance of iterators doing long range scans by using readahead.
-
-### Bug Fixes
-* Fix a stack-use-after-scope bug in ForwardIterator.
-* Fix builds on platforms including Linux, Windows, and PowerPC.
-* Fix buffer overrun in backup engine for DBs with huge number of files.
-* Fix a mislabel bug for bottom-pri compaction threads.
-* Fix DB::Flush() keep waiting after flush finish under certain condition.
-
-## 5.10.0 (12/11/2017)
-### Public API Change
-* When running `make` with environment variable `USE_SSE` set and `PORTABLE` unset, will use all machine features available locally. Previously this combination only compiled SSE-related features.
-
-### New Features
-* Provide lifetime hints when writing files on Linux. This reduces hardware write-amp on storage devices supporting multiple streams.
-* Add a DB stat, `NUMBER_ITER_SKIP`, which returns how many internal keys were skipped during iterations (e.g., due to being tombstones or duplicate versions of a key).
-* Add PerfContext counters, `key_lock_wait_count` and `key_lock_wait_time`, which measure the number of times transactions wait on key locks and total amount of time waiting.
-
-### Bug Fixes
-* Fix IOError on WAL write doesn't propagate to write group follower
-* Make iterator invalid on merge error.
-* Fix performance issue in `IngestExternalFile()` affecting databases with large number of SST files.
 * Fix possible corruption to LSM structure when `DeleteFilesInRange()` deletes a subset of files spanned by a `DeleteRange()` marker.
 
-## 5.9.0 (11/1/2017)
-### Public API Change
-* `BackupableDBOptions::max_valid_backups_to_open == 0` now means no backups will be opened during BackupEngine initialization. Previously this condition disabled limiting backups opened.
-* `DBOptions::preserve_deletes` is a new option that allows one to specify that DB should not drop tombstones for regular deletes if they have sequence number larger than what was set by the new API call `DB::SetPreserveDeletesSequenceNumber(SequenceNumber seqnum)`. Disabled by default.
-* API call `DB::SetPreserveDeletesSequenceNumber(SequenceNumber seqnum)` was added, users who wish to preserve deletes are expected to periodically call this function to advance the cutoff seqnum (all deletes made before this seqnum can be dropped by DB). It's user responsibility to figure out how to advance the seqnum in the way so the tombstones are kept for the desired period of time, yet are eventually processed in time and don't eat up too much space.
-* `ReadOptions::iter_start_seqnum` was added; if set to something > 0 user will see 2 changes in iterators behavior 1) only keys written with sequence larger than this parameter would be returned and 2) the `Slice` returned by iter->key() now points to the memory that keep User-oriented representation of the internal key, rather than user key. New struct `FullKey` was added to represent internal keys, along with a new helper function `ParseFullKey(const Slice& internal_key, FullKey* result);`.
-* Deprecate trash_dir param in NewSstFileManager, right now we will rename deleted files to <name>.trash instead of moving them to trash directory
-* Allow setting a custom trash/DB size ratio limit in the SstFileManager, after which files that are to be scheduled for deletion are deleted immediately, regardless of any delete ratelimit.
-* Return an error on write if write_options.sync = true and write_options.disableWAL = true to warn user of inconsistent options. Previously we will not write to WAL and not respecting the sync options in this case.
-
-### New Features
-* CRC32C is now using the 3-way pipelined SSE algorithm `crc32c_3way` on supported platforms to improve performance. The system will choose to use this algorithm on supported platforms automatically whenever possible. If PCLMULQDQ is not supported it will fall back to the old Fast_CRC32 algorithm.
-* `DBOptions::writable_file_max_buffer_size` can now be changed dynamically.
-* `DBOptions::bytes_per_sync`, `DBOptions::compaction_readahead_size`, and `DBOptions::wal_bytes_per_sync` can now be changed dynamically, `DBOptions::wal_bytes_per_sync` will flush all memtables and switch to a new WAL file.
-* Support dynamic adjustment of rate limit according to demand for background I/O. It can be enabled by passing `true` to the `auto_tuned` parameter in `NewGenericRateLimiter()`. The value passed as `rate_bytes_per_sec` will still be respected as an upper-bound.
-* Support dynamically changing `ColumnFamilyOptions::compaction_options_fifo`.
-* Introduce `EventListener::OnStallConditionsChanged()` callback. Users can implement it to be notified when user writes are stalled, stopped, or resumed.
-* Add a new db property "rocksdb.estimate-oldest-key-time" to return oldest data timestamp. The property is available only for FIFO compaction with compaction_options_fifo.allow_compaction = false.
-* Upon snapshot release, recompact bottommost files containing deleted/overwritten keys that previously could not be dropped due to the snapshot. This alleviates space-amp caused by long-held snapshots.
-* Support lower bound on iterators specified via `ReadOptions::iterate_lower_bound`.
-* Support for differential snapshots (via iterator emitting the sequence of key-values representing the difference between DB state at two different sequence numbers). Supports preserving and emitting puts and regular deletes, doesn't support SingleDeletes, MergeOperator, Blobs and Range Deletes.
-
+## 5.8.7 (11/28/2017)
 ### Bug Fixes
-* Fix a potential data inconsistency issue during point-in-time recovery. `DB:Open()` will abort if column family inconsistency is found during PIT recovery.
-* Fix possible metadata corruption in databases using `DeleteRange()`.
+* Fix IOError on WAL write doesn't propagate to write group follower
+
+## 5.8.6 (11/20/2017)
+### Bug Fixes
+* Fixed aligned_alloc issues with Windows.
+
+## 5.8.1 (10/23/2017)
+### New Features
+* Add a new db property "rocksdb.estimate-oldest-key-time" to return oldest data timestamp. The property is available only for FIFO compaction with compaction_options_fifo.allow_compaction = false.
 
 ## 5.8.0 (08/30/2017)
 ### Public API Change

INSTALL.md

@@ -107,62 +107,6 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
             * run `brew tap homebrew/versions; brew install gcc48 --use-llvm` to install gcc 4.8 (or higher).
     * run `brew install rocksdb`
 
-* **FreeBSD** (11.01):
-
-    * You can either install RocksDB from the Ports system using `cd /usr/ports/databases/rocksdb && make install`, or you can follow the details below to install dependencies and compile from source code:
-
-    * Install the dependencies for RocksDB:
-
-        export BATCH=YES
-        cd /usr/ports/devel/gmake && make install
-        cd /usr/ports/devel/gflags && make install
-
-        cd /usr/ports/archivers/snappy && make install
-        cd /usr/ports/archivers/bzip2 && make install
-        cd /usr/ports/archivers/liblz4 && make install
-        cd /usr/ports/archivesrs/zstd && make install
-
-        cd /usr/ports/devel/git && make install
-
-
-    * Install the dependencies for RocksJava (optional):
-
-        export BATCH=yes
-        cd /usr/ports/java/openjdk7 && make install
-
-    * Build RocksDB from source:
-        cd ~
-        git clone https://github.com/facebook/rocksdb.git
-        cd rocksdb
-        gmake static_lib
-
-    * Build RocksJava from source (optional):
-        cd rocksdb
-        export JAVA_HOME=/usr/local/openjdk7
-        gmake rocksdbjava
-
-* **OpenBSD** (6.3/-current):
-
-    * As RocksDB is not available in the ports yet you have to build it on your own:
-
-    * Install the dependencies for RocksDB:
-
-        pkg_add gmake gflags snappy bzip2 lz4 zstd git jdk bash findutils gnuwatch 
-
-    * Build RocksDB from source:
-
-        cd ~
-        git clone https://github.com/facebook/rocksdb.git
-        cd rocksdb
-        gmake static_lib
-
-    * Build RocksJava from source (optional):
-
-        cd rocksdb
-        export JAVA_HOME=/usr/local/jdk-1.8.0
-        export PATH=$PATH:/usr/local/jdk-1.8.0/bin
-        gmake rocksdbjava
-
 * **iOS**:
   * Run: `TARGET_OS=IOS make static_lib`. When building the project which uses rocksdb iOS library, make sure to define two important pre-processing macros: `ROCKSDB_LITE` and `IOS_CROSS_COMPILE`.
 
@@ -170,7 +114,7 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
   * For building with MS Visual Studio 13 you will need Update 4 installed.
   * Read and follow the instructions at CMakeLists.txt
   * Or install via [vcpkg](https://github.com/microsoft/vcpkg) 
-       * run `vcpkg install rocksdb:x64-windows`
+       * run `vcpkg install rocksdb`
 
 * **AIX 6.1**
     * Install AIX Toolbox rpms with gcc

LANGUAGE-BINDINGS.md

@@ -14,5 +14,3 @@ This is the list of all known third-party language bindings for RocksDB. If some
     * https://github.com/bh1xuw/rust-rocks
 * D programming language - https://github.com/b1naryth1ef/rocksdb
 * Erlang - https://gitlab.com/barrel-db/erlang-rocksdb
-* Elixir - https://github.com/urbint/rox
-* Nim - https://github.com/status-im/nim-rocksdb

Makefile

@@ -76,9 +76,7 @@ ifeq ($(MAKECMDGOALS),install)
 endif
 
 ifeq ($(MAKECMDGOALS),rocksdbjavastatic)
-	ifneq ($(DEBUG_LEVEL),2)
-		DEBUG_LEVEL=0
-	endif
+	DEBUG_LEVEL=0
 endif
 
 ifeq ($(MAKECMDGOALS),rocksdbjavastaticrelease)
@@ -98,21 +96,11 @@ OPT += -momit-leaf-frame-pointer
 endif
 endif
 
-ifeq (,$(shell $(CXX) -fsyntax-only -maltivec -xc /dev/null 2>&1))
-CXXFLAGS += -DHAS_ALTIVEC
-CFLAGS += -DHAS_ALTIVEC
-HAS_ALTIVEC=1
-endif
-
-ifeq (,$(shell $(CXX) -fsyntax-only -mcpu=power8 -xc /dev/null 2>&1))
-CXXFLAGS += -DHAVE_POWER8
-CFLAGS +=  -DHAVE_POWER8
-HAVE_POWER8=1
-endif
-
-# if we're compiling for release, compile without debug code (-DNDEBUG)
+# if we're compiling for release, compile without debug code (-DNDEBUG) and
+# don't treat warnings as errors
 ifeq ($(DEBUG_LEVEL),0)
 OPT += -DNDEBUG
+DISABLE_WARNING_AS_ERROR=1
 
 ifneq ($(USE_RTTI), 1)
 	CXXFLAGS += -fno-rtti
@@ -224,9 +212,6 @@ ifdef COMPILE_WITH_TSAN
 	PROFILING_FLAGS =
 	# LUA is not supported under TSAN
 	LUA_PATH =
-	# Limit keys for crash test under TSAN to avoid error:
-	# "ThreadSanitizer: DenseSlabAllocator overflow. Dying."
-	CRASH_TEST_EXT_ARGS += --max_key=1000000
 endif
 
 # AIX doesn't work with -pg
@@ -237,18 +222,9 @@ endif
 # USAN doesn't work well with jemalloc. If we're compiling with USAN, we should use regular malloc.
 ifdef COMPILE_WITH_UBSAN
 	DISABLE_JEMALLOC=1
-	# Suppress alignment warning because murmurhash relies on casting unaligned
-	# memory to integer. Fixing it may cause performance regression. 3-way crc32
-	# relies on it too, although it can be rewritten to eliminate with minimal
-	# performance regression.
-	EXEC_LDFLAGS += -fsanitize=undefined -fno-sanitize-recover=all
-	PLATFORM_CCFLAGS += -fsanitize=undefined -fno-sanitize-recover=all -DROCKSDB_UBSAN_RUN
-	PLATFORM_CXXFLAGS += -fsanitize=undefined -fno-sanitize-recover=all -DROCKSDB_UBSAN_RUN
-endif
-
-ifdef ROCKSDB_VALGRIND_RUN
-	PLATFORM_CCFLAGS += -DROCKSDB_VALGRIND_RUN
-	PLATFORM_CXXFLAGS += -DROCKSDB_VALGRIND_RUN
+	EXEC_LDFLAGS += -fsanitize=undefined
+	PLATFORM_CCFLAGS += -fsanitize=undefined -DROCKSDB_UBSAN_RUN
+	PLATFORM_CXXFLAGS += -fsanitize=undefined -DROCKSDB_UBSAN_RUN
 endif
 
 ifndef DISABLE_JEMALLOC
@@ -281,11 +257,7 @@ endif
 default: all
 
 WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare -Wshadow \
-  -Wunused-parameter
-
-ifeq ($(PLATFORM), OS_OPENBSD)
-	WARNING_FLAGS += -Wno-unused-lambda-capture
-endif
+  -Wno-unused-parameter
 
 ifndef DISABLE_WARNING_AS_ERROR
 	WARNING_FLAGS += -Werror
@@ -318,9 +290,6 @@ LDFLAGS += $(LUA_LIB)
 
 endif
 
-ifeq ($(NO_THREEWAY_CRC32C), 1)
-	CXXFLAGS += -DNO_THREEWAY_CRC32C
-endif
 
 CFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CCFLAGS) $(OPT)
 CXXFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers
@@ -353,14 +322,6 @@ util/build_version.cc: FORCE
 endif
 
 LIBOBJECTS = $(LIB_SOURCES:.cc=.o)
-ifeq ($(HAVE_POWER8),1)
-LIB_CC_OBJECTS = $(LIB_SOURCES:.cc=.o)
-LIBOBJECTS += $(LIB_SOURCES_C:.c=.o)
-LIBOBJECTS += $(LIB_SOURCES_ASM:.S=.o)
-else
-LIB_CC_OBJECTS = $(LIB_SOURCES:.cc=.o)
-endif
-
 LIBOBJECTS += $(TOOL_LIB_SOURCES:.cc=.o)
 MOCKOBJECTS = $(MOCK_LIB_SOURCES:.cc=.o)
 
@@ -402,7 +363,6 @@ TESTS = \
 	db_blob_index_test \
 	db_bloom_filter_test \
 	db_iter_test \
-	db_iter_stress_test \
 	db_log_iter_test \
 	db_compaction_filter_test \
 	db_compaction_test \
@@ -416,6 +376,7 @@ TESTS = \
 	db_range_del_test \
 	db_sst_test \
 	db_tailing_iter_test \
+	db_universal_compaction_test \
 	db_io_failure_test \
 	db_properties_test \
 	db_table_properties_test \
@@ -480,7 +441,6 @@ TESTS = \
 	write_batch_with_index_test \
 	write_controller_test\
 	deletefile_test \
-	obsolete_files_test \
 	table_test \
 	geodb_test \
 	delete_scheduler_test \
@@ -517,14 +477,11 @@ TESTS = \
 	object_registry_test \
 	repair_test \
 	env_timed_test \
-	write_prepared_transaction_test \
-	db_universal_compaction_test \
 
 PARALLEL_TEST = \
 	backupable_db_test \
 	db_compaction_filter_test \
 	db_compaction_test \
-	db_merge_operator_test \
 	db_sst_test \
 	db_test \
 	db_universal_compaction_test \
@@ -535,13 +492,8 @@ PARALLEL_TEST = \
 	manual_compaction_test \
 	persistent_cache_test \
 	table_test \
-	transaction_test \
-	write_prepared_transaction_test \
+	transaction_test
 
-# options_settable_test doesn't pass with UBSAN as we use hack in the test
-ifdef COMPILE_WITH_UBSAN
-        TESTS := $(shell echo $(TESTS) | sed 's/\boptions_settable_test\b//g')
-endif
 SUBSET := $(TESTS)
 ifdef ROCKSDBTESTS_START
         SUBSET := $(shell echo $(SUBSET) | sed 's/^.*$(ROCKSDBTESTS_START)/$(ROCKSDBTESTS_START)/')
@@ -620,36 +572,15 @@ $(SHARED2): $(SHARED4)
 $(SHARED3): $(SHARED4)
 	ln -fs $(SHARED4) $(SHARED3)
 endif
-ifeq ($(HAVE_POWER8),1)
-SHARED_C_OBJECTS = $(LIB_SOURCES_C:.c=.o)
-SHARED_ASM_OBJECTS = $(LIB_SOURCES_ASM:.S=.o)
-SHARED_C_LIBOBJECTS = $(patsubst %.o,shared-objects/%.o,$(SHARED_C_OBJECTS))
-SHARED_ASM_LIBOBJECTS = $(patsubst %.o,shared-objects/%.o,$(SHARED_ASM_OBJECTS))
-shared_libobjects = $(patsubst %,shared-objects/%,$(LIB_CC_OBJECTS))
-else
+
 shared_libobjects = $(patsubst %,shared-objects/%,$(LIBOBJECTS))
-endif
-
 CLEAN_FILES += shared-objects
-shared_all_libobjects = $(shared_libobjects)
 
-ifeq ($(HAVE_POWER8),1)
-shared-ppc-objects = $(SHARED_C_LIBOBJECTS) $(SHARED_ASM_LIBOBJECTS)
-
-shared-objects/util/crc32c_ppc.o: util/crc32c_ppc.c
-	$(AM_V_CC)$(CC) $(CFLAGS) -c $< -o $@
-
-shared-objects/util/crc32c_ppc_asm.o: util/crc32c_ppc_asm.S
-	$(AM_V_CC)$(CC) $(CFLAGS) -c $< -o $@
-endif
 $(shared_libobjects): shared-objects/%.o: %.cc
 	$(AM_V_CC)mkdir -p $(@D) && $(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) -c $< -o $@
 
-ifeq ($(HAVE_POWER8),1)
-shared_all_libobjects = $(shared_libobjects) $(shared-ppc-objects)
-endif
-$(SHARED4): $(shared_all_libobjects)
-	$(CXX) $(PLATFORM_SHARED_LDFLAGS)$(SHARED3) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(shared_all_libobjects) $(LDFLAGS) -o $@
+$(SHARED4): $(shared_libobjects)
+	$(CXX) $(PLATFORM_SHARED_LDFLAGS)$(SHARED3) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(shared_libobjects) $(LDFLAGS) -o $@
 
 endif  # PLATFORM_SHARED_EXT
 
@@ -685,7 +616,7 @@ coverage:
 	COVERAGEFLAGS="-fprofile-arcs -ftest-coverage" LDFLAGS+="-lgcov" $(MAKE) J=1 all check
 	cd coverage && ./coverage_test.sh
         # Delete intermediate files
-	$(FIND) . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm {} \;
+	find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm {} \;
 
 ifneq (,$(filter check parallel_check,$(MAKECMDGOALS)),)
 # Use /dev/shm if it has the sticky bit set (otherwise, /tmp),
@@ -770,7 +701,7 @@ gen_parallel_tests:
 # 107.816 PASS t/DBTest.EncodeDecompressedBlockSizeTest
 #
 slow_test_regexp = \
-	^.*SnapshotConcurrentAccessTest.*$$|^t/run-table_test-HarnessTest.Randomized$$|^t/run-db_test-.*(?:FileCreationRandomFailure|EncodeDecompressedBlockSizeTest)$$|^.*RecoverFromCorruptedWALWithoutFlush$$
+	^t/run-table_test-HarnessTest.Randomized$$|^t/run-db_test-.*(?:FileCreationRandomFailure|EncodeDecompressedBlockSizeTest)$$|^.*RecoverFromCorruptedWALWithoutFlush$$
 prioritize_long_running_tests =						\
   perl -pe 's,($(slow_test_regexp)),100 $$1,'				\
     | sort -k1,1gr							\
@@ -802,7 +733,7 @@ check_0:
 	  | grep -E '$(tests-regexp)'					\
 	  | build_tools/gnu_parallel -j$(J) --plain --joblog=LOG $$eta --gnu '{} >& t/log-{/}'
 
-valgrind-blacklist-regexp = InlineSkipTest.ConcurrentInsert|TransactionTest.DeadlockStress|DBCompactionTest.SuggestCompactRangeNoTwoLevel0Compactions|BackupableDBTest.RateLimiting|DBTest.CloseSpeedup|DBTest.ThreadStatusFlush|DBTest.RateLimitingTest|DBTest.EncodeDecompressedBlockSizeTest|FaultInjectionTest.UninstalledCompaction|HarnessTest.Randomized|ExternalSSTFileTest.CompactDuringAddFileRandom|ExternalSSTFileTest.IngestFileWithGlobalSeqnoRandomized|MySQLStyleTransactionTest.TransactionStressTest
+valgrind-blacklist-regexp = InlineSkipTest.ConcurrentInsert|TransactionTest.DeadlockStress|DBCompactionTest.SuggestCompactRangeNoTwoLevel0Compactions|BackupableDBTest.RateLimiting|DBTest.CloseSpeedup|DBTest.ThreadStatusFlush|DBTest.RateLimitingTest|DBTest.EncodeDecompressedBlockSizeTest|FaultInjectionTest.UninstalledCompaction|HarnessTest.Randomized|ExternalSSTFileTest.CompactDuringAddFileRandom|ExternalSSTFileTest.IngestFileWithGlobalSeqnoRandomized
 
 .PHONY: valgrind_check_0
 valgrind_check_0:
@@ -832,7 +763,7 @@ CLEAN_FILES += t LOG $(TMPD)
 # regardless of their duration. As with any use of "watch", hit ^C to
 # interrupt.
 watch-log:
-	$(WATCH) --interval=0 'sort -k7,7nr -k4,4gr LOG|$(quoted_perl_command)'
+	watch --interval=0 'sort -k7,7nr -k4,4gr LOG|$(quoted_perl_command)'
 
 # If J != 1 and GNU parallel is installed, run the tests in parallel,
 # via the check_0 rule above.  Otherwise, run them sequentially.
@@ -900,7 +831,7 @@ ubsan_crash_test:
 	$(MAKE) clean
 
 valgrind_test:
-	ROCKSDB_VALGRIND_RUN=1 DISABLE_JEMALLOC=1 $(MAKE) valgrind_check
+	DISABLE_JEMALLOC=1 $(MAKE) valgrind_check
 
 valgrind_check: $(TESTS)
 	$(MAKE) DRIVER="$(VALGRIND_VER) $(VALGRIND_OPTS)" gen_parallel_tests
@@ -997,14 +928,14 @@ rocksdb.h rocksdb.cc: build_tools/amalgamate.py Makefile $(LIB_SOURCES) unity.cc
 clean:
 	rm -f $(BENCHMARKS) $(TOOLS) $(TESTS) $(LIBRARY) $(SHARED)
 	rm -rf $(CLEAN_FILES) ios-x86 ios-arm scan_build_report
-	$(FIND) . -name "*.[oda]" -exec rm -f {} \;
-	$(FIND) . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm {} \;
+	find . -name "*.[oda]" -exec rm -f {} \;
+	find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm {} \;
 	rm -rf bzip2* snappy* zlib* lz4* zstd*
 	cd java; $(MAKE) clean
 
 tags:
-	ctags -R .
-	cscope -b `$(FIND) . -name '*.cc'` `$(FIND) . -name '*.h'` `$(FIND) . -name '*.c'`
+	ctags * -R
+	cscope -b `find . -name '*.cc'` `find . -name '*.h'` `find . -name '*.c'`
 	ctags -e -R -o etags *
 
 format:
@@ -1196,9 +1127,6 @@ db_tailing_iter_test: db/db_tailing_iter_test.o db/db_test_util.o $(LIBOBJECTS)
 db_iter_test: db/db_iter_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-db_iter_stress_test: db/db_iter_stress_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 db_universal_compaction_test: db/db_universal_compaction_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -1303,7 +1231,7 @@ env_test: env/env_test.o $(LIBOBJECTS) $(TESTHARNESS)
 fault_injection_test: db/fault_injection_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-rate_limiter_test: util/rate_limiter_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
+rate_limiter_test: util/rate_limiter_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
 delete_scheduler_test: util/delete_scheduler_test.o $(LIBOBJECTS) $(TESTHARNESS)
@@ -1390,9 +1318,6 @@ options_file_test: db/options_file_test.o $(LIBOBJECTS) $(TESTHARNESS)
 deletefile_test: db/deletefile_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-obsolete_files_test: db/obsolete_files_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 geodb_test: utilities/geodb/geodb_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -1471,9 +1396,6 @@ heap_test: util/heap_test.o $(GTEST)
 transaction_test: utilities/transactions/transaction_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-write_prepared_transaction_test: utilities/transactions/write_prepared_transaction_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 sst_dump: tools/sst_dump.o $(LIBOBJECTS)
 	$(AM_LINK)
 
@@ -1527,10 +1449,10 @@ uninstall:
 
 install-headers:
 	install -d $(INSTALL_PATH)/lib
-	for header_dir in `$(FIND) "include/rocksdb" -type d`; do \
+	for header_dir in `find "include/rocksdb" -type d`; do \
 		install -d $(INSTALL_PATH)/$$header_dir; \
 	done
-	for header in `$(FIND) "include/rocksdb" -type f -name *.h`; do \
+	for header in `find "include/rocksdb" -type f -name *.h`; do \
 		install -C -m 644 $$header $(INSTALL_PATH)/$$header; \
 	done
 
@@ -1557,12 +1479,6 @@ install: install-static
 JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/linux
 ifeq ($(PLATFORM), OS_SOLARIS)
 	ARCH := $(shell isainfo -b)
-else ifeq ($(PLATFORM), OS_OPENBSD)
-	ifneq (,$(filter $(MACHINE), amd64 arm64 sparc64))
-		ARCH := 64
-	else
-		ARCH := 32
-	endif
 else
 	ARCH := $(shell getconf LONG_BIT)
 endif
@@ -1587,13 +1503,12 @@ BZIP2_DOWNLOAD_BASE ?= http://www.bzip.org
 SNAPPY_VER ?= 1.1.4
 SNAPPY_SHA256 ?= 134bfe122fd25599bb807bb8130e7ba6d9bdb851e0b16efcb83ac4f5d0b70057
 SNAPPY_DOWNLOAD_BASE ?= https://github.com/google/snappy/releases/download
-LZ4_VER ?= 1.8.0
-LZ4_SHA256 ?= 2ca482ea7a9bb103603108b5a7510b7592b90158c151ff50a28f1ca8389fccf6
+LZ4_VER ?= 1.7.5
+LZ4_SHA256 ?= 0190cacd63022ccb86f44fa5041dc6c3804407ad61550ca21c382827319e7e7e
 LZ4_DOWNLOAD_BASE ?= https://github.com/lz4/lz4/archive
-ZSTD_VER ?= 1.3.3
-ZSTD_SHA256 ?= a77c47153ee7de02626c5b2a097005786b71688be61e9fb81806a011f90b297b
+ZSTD_VER ?= 1.2.0
+ZSTD_SHA256 ?= 4a7e4593a3638276ca7f2a09dc4f38e674d8317bbea51626393ca73fc047cbfb
 ZSTD_DOWNLOAD_BASE ?= https://github.com/facebook/zstd/archive
-CURL_SSL_OPTS ?= --tlsv1
 
 ifeq ($(PLATFORM), OS_MACOSX)
 	ROCKSDBJNILIB = librocksdbjni-osx.jnilib
@@ -1606,7 +1521,7 @@ else
 endif
 endif
 ifeq ($(PLATFORM), OS_FREEBSD)
-	JAVA_INCLUDE = -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/freebsd
+	JAVA_INCLUDE += -I$(JAVA_HOME)/include/freebsd
 	ROCKSDBJNILIB = librocksdbjni-freebsd$(ARCH).so
 	ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-freebsd$(ARCH).jar
 endif
@@ -1622,11 +1537,6 @@ ifeq ($(PLATFORM), OS_AIX)
 	EXTRACT_SOURCES = gunzip < TAR_GZ | tar xvf -
 	SNAPPY_MAKE_TARGET = libsnappy.la
 endif
-ifeq ($(PLATFORM), OS_OPENBSD)
-        JAVA_INCLUDE = -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/openbsd
-	ROCKSDBJNILIB = librocksdbjni-openbsd$(ARCH).so
-        ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-openbsd$(ARCH).jar
-endif
 
 libz.a:
 	-rm -rf zlib-$(ZLIB_VER)
@@ -1637,7 +1547,7 @@ libz.a:
 		exit 1; \
 	fi
 	tar xvzf zlib-$(ZLIB_VER).tar.gz
-	cd zlib-$(ZLIB_VER) && CFLAGS='-fPIC ${EXTRA_CFLAGS}' LDFLAGS='${EXTRA_LDFLAGS}' ./configure --static && $(MAKE)
+	cd zlib-$(ZLIB_VER) && CFLAGS='-fPIC ${EXTRA_CFLAGS}' LDFLAGS='${EXTRA_LDFLAGS}' ./configure --static && make
 	cp zlib-$(ZLIB_VER)/libz.a .
 
 libbz2.a:
@@ -1649,12 +1559,12 @@ libbz2.a:
 		exit 1; \
 	fi
 	tar xvzf bzip2-$(BZIP2_VER).tar.gz
-	cd bzip2-$(BZIP2_VER) && $(MAKE) CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64 ${EXTRA_CFLAGS}' AR='ar ${EXTRA_ARFLAGS}'
+	cd bzip2-$(BZIP2_VER) && make CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64 ${EXTRA_CFLAGS}' AR='ar ${EXTRA_ARFLAGS}'
 	cp bzip2-$(BZIP2_VER)/libbz2.a .
 
 libsnappy.a:
 	-rm -rf snappy-$(SNAPPY_VER)
-	curl -O -L ${CURL_SSL_OPTS} ${SNAPPY_DOWNLOAD_BASE}/$(SNAPPY_VER)/snappy-$(SNAPPY_VER).tar.gz
+	curl -O -L ${SNAPPY_DOWNLOAD_BASE}/$(SNAPPY_VER)/snappy-$(SNAPPY_VER).tar.gz
 	SNAPPY_SHA256_ACTUAL=`$(SHA256_CMD) snappy-$(SNAPPY_VER).tar.gz | cut -d ' ' -f 1`; \
 	if [ "$(SNAPPY_SHA256)" != "$$SNAPPY_SHA256_ACTUAL" ]; then \
 		echo snappy-$(SNAPPY_VER).tar.gz checksum mismatch, expected=\"$(SNAPPY_SHA256)\" actual=\"$$SNAPPY_SHA256_ACTUAL\"; \
@@ -1662,12 +1572,12 @@ libsnappy.a:
 	fi
 	tar xvzf snappy-$(SNAPPY_VER).tar.gz
 	cd snappy-$(SNAPPY_VER) && CFLAGS='${EXTRA_CFLAGS}' CXXFLAGS='${EXTRA_CXXFLAGS}' LDFLAGS='${EXTRA_LDFLAGS}' ./configure --with-pic --enable-static --disable-shared
-	cd snappy-$(SNAPPY_VER) && $(MAKE) ${SNAPPY_MAKE_TARGET}
+	cd snappy-$(SNAPPY_VER) && make ${SNAPPY_MAKE_TARGET}
 	cp snappy-$(SNAPPY_VER)/.libs/libsnappy.a .
 
 liblz4.a:
 	-rm -rf lz4-$(LZ4_VER)
-	curl -O -L ${CURL_SSL_OPTS} ${LZ4_DOWNLOAD_BASE}/v$(LZ4_VER).tar.gz
+	curl -O -L ${LZ4_DOWNLOAD_BASE}/v$(LZ4_VER).tar.gz
 	mv v$(LZ4_VER).tar.gz lz4-$(LZ4_VER).tar.gz
 	LZ4_SHA256_ACTUAL=`$(SHA256_CMD) lz4-$(LZ4_VER).tar.gz | cut -d ' ' -f 1`; \
 	if [ "$(LZ4_SHA256)" != "$$LZ4_SHA256_ACTUAL" ]; then \
@@ -1675,12 +1585,12 @@ liblz4.a:
 		exit 1; \
 	fi
 	tar xvzf lz4-$(LZ4_VER).tar.gz
-	cd lz4-$(LZ4_VER)/lib && $(MAKE) CFLAGS='-fPIC -O2 ${EXTRA_CFLAGS}' all
+	cd lz4-$(LZ4_VER)/lib && make CFLAGS='-fPIC -O2 ${EXTRA_CFLAGS}' all
 	cp lz4-$(LZ4_VER)/lib/liblz4.a .
 
 libzstd.a:
 	-rm -rf zstd-$(ZSTD_VER)
-	curl -O -L ${CURL_SSL_OPTS} ${ZSTD_DOWNLOAD_BASE}/v$(ZSTD_VER).tar.gz
+	curl -O -L ${ZSTD_DOWNLOAD_BASE}/v$(ZSTD_VER).tar.gz
 	mv v$(ZSTD_VER).tar.gz zstd-$(ZSTD_VER).tar.gz
 	ZSTD_SHA256_ACTUAL=`$(SHA256_CMD) zstd-$(ZSTD_VER).tar.gz | cut -d ' ' -f 1`; \
 	if [ "$(ZSTD_SHA256)" != "$$ZSTD_SHA256_ACTUAL" ]; then \
@@ -1688,49 +1598,31 @@ libzstd.a:
 		exit 1; \
 	fi
 	tar xvzf zstd-$(ZSTD_VER).tar.gz
-	cd zstd-$(ZSTD_VER)/lib && DESTDIR=. PREFIX= $(MAKE) CFLAGS='-fPIC -O2 ${EXTRA_CFLAGS}' install
+	cd zstd-$(ZSTD_VER)/lib && make CFLAGS='-fPIC -O2 ${EXTRA_CFLAGS}' all
 	cp zstd-$(ZSTD_VER)/lib/libzstd.a .
 
 # A version of each $(LIBOBJECTS) compiled with -fPIC and a fixed set of static compression libraries
-java_static_libobjects = $(patsubst %,jls/%,$(LIB_CC_OBJECTS))
+java_static_libobjects = $(patsubst %,jls/%,$(LIBOBJECTS))
 CLEAN_FILES += jls
-java_static_all_libobjects = $(java_static_libobjects)
 
 ifneq ($(ROCKSDB_JAVA_NO_COMPRESSION), 1)
 JAVA_COMPRESSIONS = libz.a libbz2.a libsnappy.a liblz4.a libzstd.a
 endif
 
 JAVA_STATIC_FLAGS = -DZLIB -DBZIP2 -DSNAPPY -DLZ4 -DZSTD
-JAVA_STATIC_INCLUDES = -I./zlib-$(ZLIB_VER) -I./bzip2-$(BZIP2_VER) -I./snappy-$(SNAPPY_VER) -I./lz4-$(LZ4_VER)/lib -I./zstd-$(ZSTD_VER)/lib/include
-
-ifeq ($(HAVE_POWER8),1)
-JAVA_STATIC_C_LIBOBJECTS = $(patsubst %.c.o,jls/%.c.o,$(LIB_SOURCES_C:.c=.o))
-JAVA_STATIC_ASM_LIBOBJECTS = $(patsubst %.S.o,jls/%.S.o,$(LIB_SOURCES_ASM:.S=.o))
-
-java_static_ppc_libobjects = $(JAVA_STATIC_C_LIBOBJECTS) $(JAVA_STATIC_ASM_LIBOBJECTS)
-
-jls/util/crc32c_ppc.o: util/crc32c_ppc.c
-	$(AM_V_CC)$(CC) $(CFLAGS) $(JAVA_STATIC_FLAGS) $(JAVA_STATIC_INCLUDES) -c $< -o $@
-
-jls/util/crc32c_ppc_asm.o: util/crc32c_ppc_asm.S
-	$(AM_V_CC)$(CC) $(CFLAGS) $(JAVA_STATIC_FLAGS) $(JAVA_STATIC_INCLUDES) -c $< -o $@
-
-java_static_all_libobjects += $(java_static_ppc_libobjects)
-endif
+JAVA_STATIC_INCLUDES = -I./zlib-$(ZLIB_VER) -I./bzip2-$(BZIP2_VER) -I./snappy-$(SNAPPY_VER) -I./lz4-$(LZ4_VER)/lib -I./zstd-$(ZSTD_VER)/lib
 
 $(java_static_libobjects): jls/%.o: %.cc $(JAVA_COMPRESSIONS)
 	$(AM_V_CC)mkdir -p $(@D) && $(CXX) $(CXXFLAGS) $(JAVA_STATIC_FLAGS) $(JAVA_STATIC_INCLUDES) -fPIC -c $< -o $@ $(COVERAGEFLAGS)
 
-rocksdbjavastatic: $(java_static_all_libobjects)
+rocksdbjavastatic: $(java_static_libobjects)
 	cd java;$(MAKE) javalib;
 	rm -f ./java/target/$(ROCKSDBJNILIB)
 	$(CXX) $(CXXFLAGS) -I./java/. $(JAVA_INCLUDE) -shared -fPIC \
 	  -o ./java/target/$(ROCKSDBJNILIB) $(JNI_NATIVE_SOURCES) \
-	  $(java_static_all_libobjects) $(COVERAGEFLAGS) \
+	  $(java_static_libobjects) $(COVERAGEFLAGS) \
 	  $(JAVA_COMPRESSIONS) $(JAVA_STATIC_LDFLAGS)
-	cd java/target;if [ "$(DEBUG_LEVEL)" == "0" ]; then \
-		strip $(STRIPFLAGS) $(ROCKSDBJNILIB); \
-	fi
+	cd java/target;strip $(STRIPFLAGS) $(ROCKSDBJNILIB)
 	cd java;jar -cf target/$(ROCKSDB_JAR) HISTORY*.md
 	cd java/target;jar -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB)
 	cd java/target/classes;jar -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class
@@ -1758,14 +1650,6 @@ rocksdbjavastaticreleasedocker: rocksdbjavastatic
 	cd java/target;jar -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib
 	cd java/target/classes;jar -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class
 
-rocksdbjavastaticdockerppc64le:
-	mkdir -p java/target
-	DOCKER_LINUX_PPC64LE_CONTAINER=`docker ps -aqf name=rocksdb_linux_ppc64le-be`; \
-	if [ -z "$$DOCKER_LINUX_PPC64LE_CONTAINER" ]; then \
-		docker container create --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host --name rocksdb_linux_ppc64le-be evolvedbinary/rocksjava:centos7_ppc64le-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh; \
-	fi
-	docker start -a rocksdb_linux_ppc64le-be
-
 rocksdbjavastaticpublish: rocksdbjavastaticrelease rocksdbjavastaticpublishcentral
 
 rocksdbjavastaticpublishdocker: rocksdbjavastaticreleasedocker rocksdbjavastaticpublishcentral
@@ -1780,36 +1664,13 @@ rocksdbjavastaticpublishcentral:
 	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/rocksjni.pom -Dfile=java/target/rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH).jar
 
 # A version of each $(LIBOBJECTS) compiled with -fPIC
-ifeq ($(HAVE_POWER8),1)
-JAVA_CC_OBJECTS = $(SHARED_CC_OBJECTS)
-JAVA_C_OBJECTS = $(SHARED_C_OBJECTS)
-JAVA_ASM_OBJECTS = $(SHARED_ASM_OBJECTS)
-
-JAVA_C_LIBOBJECTS = $(patsubst %.c.o,jl/%.c.o,$(JAVA_C_OBJECTS))
-JAVA_ASM_LIBOBJECTS = $(patsubst %.S.o,jl/%.S.o,$(JAVA_ASM_OBJECTS))
-endif
-
-java_libobjects = $(patsubst %,jl/%,$(LIB_CC_OBJECTS))
+java_libobjects = $(patsubst %,jl/%,$(LIBOBJECTS))
 CLEAN_FILES += jl
-java_all_libobjects = $(java_libobjects)
-
-ifeq ($(HAVE_POWER8),1)
-java_ppc_libobjects = $(JAVA_C_LIBOBJECTS) $(JAVA_ASM_LIBOBJECTS)
-
-jl/crc32c_ppc.o: util/crc32c_ppc.c
-	$(AM_V_CC)$(CC) $(CFLAGS) -c $< -o $@
-
-jl/crc32c_ppc_asm.o: util/crc32c_ppc_asm.S
-	$(AM_V_CC)$(CC) $(CFLAGS) -c $< -o $@
-java_all_libobjects += $(java_ppc_libobjects)
-endif
 
 $(java_libobjects): jl/%.o: %.cc
 	$(AM_V_CC)mkdir -p $(@D) && $(CXX) $(CXXFLAGS) -fPIC -c $< -o $@ $(COVERAGEFLAGS)
 
-
-
-rocksdbjava: $(java_all_libobjects)
+rocksdbjava: $(java_libobjects)
 	$(AM_V_GEN)cd java;$(MAKE) javalib;
 	$(AM_V_at)rm -f ./java/target/$(ROCKSDBJNILIB)
 	$(AM_V_at)$(CXX) $(CXXFLAGS) -I./java/. $(JAVA_INCLUDE) -shared -fPIC -o ./java/target/$(ROCKSDBJNILIB) $(JNI_NATIVE_SOURCES) $(java_libobjects) $(JAVA_LDFLAGS) $(COVERAGEFLAGS)
@@ -1864,54 +1725,30 @@ IOSVERSION=$(shell defaults read $(PLATFORMSROOT)/iPhoneOS.platform/version CFBu
 	lipo ios-x86/$@ ios-arm/$@ -create -output $@
 
 else
-ifeq ($(HAVE_POWER8),1)
-util/crc32c_ppc.o: util/crc32c_ppc.c
-	$(AM_V_CC)$(CC) $(CFLAGS) -c $< -o $@
-
-util/crc32c_ppc_asm.o: util/crc32c_ppc_asm.S
-	$(AM_V_CC)$(CC) $(CFLAGS) -c $< -o $@
-endif
 .cc.o:
 	$(AM_V_CC)$(CXX) $(CXXFLAGS) -c $< -o $@ $(COVERAGEFLAGS)
 
 .c.o:
 	$(AM_V_CC)$(CC) $(CFLAGS) -c $< -o $@
 endif
+
 # ---------------------------------------------------------------------------
 #  	Source files dependencies detection
 # ---------------------------------------------------------------------------
 
 all_sources = $(LIB_SOURCES) $(MAIN_SOURCES) $(MOCK_LIB_SOURCES) $(TOOL_LIB_SOURCES) $(BENCH_LIB_SOURCES) $(TEST_LIB_SOURCES) $(EXP_LIB_SOURCES)
-DEPFILES = $(all_sources:.cc=.cc.d)
+DEPFILES = $(all_sources:.cc=.d)
 
 # Add proper dependency support so changing a .h file forces a .cc file to
 # rebuild.
 
 # The .d file indicates .cc file's dependencies on .h files. We generate such
 # dependency by g++'s -MM option, whose output is a make dependency rule.
-%.cc.d: %.cc
+$(DEPFILES): %.d: %.cc
 	@$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) \
 	  -MM -MT'$@' -MT'$(<:.cc=.o)' "$<" -o '$@'
 
-ifeq ($(HAVE_POWER8),1)
-DEPFILES_C = $(LIB_SOURCES_C:.c=.c.d)
-DEPFILES_ASM = $(LIB_SOURCES_ASM:.S=.S.d)
-
-%.c.d: %.c
-	@$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) \
-	  -MM -MT'$@' -MT'$(<:.c=.o)' "$<" -o '$@'
-
-%.S.d: %.S
-	@$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) \
-	  -MM -MT'$@' -MT'$(<:.S=.o)' "$<" -o '$@'
-
-$(DEPFILES_C): %.c.d
-
-$(DEPFILES_ASM): %.S.d
-depend: $(DEPFILES) $(DEPFILES_C) $(DEPFILES_ASM)
-else
 depend: $(DEPFILES)
-endif
 
 # if the make goal is either "clean" or "format", we shouldn't
 # try to import the *.d files.

README.md

@@ -1,8 +1,7 @@
 ## RocksDB: A Persistent Key-Value Store for Flash and RAM Storage
 
-[![Linux/Mac Build Status](https://travis-ci.org/facebook/rocksdb.svg?branch=master)](https://travis-ci.org/facebook/rocksdb)
-[![Windows Build status](https://ci.appveyor.com/api/projects/status/fbgfu0so3afcno78/branch/master?svg=true)](https://ci.appveyor.com/project/Facebook/rocksdb/branch/master)
-[![PPC64le Build Status](http://140.211.168.68:8080/buildStatus/icon?job=Rocksdb)](http://140.211.168.68:8080/job/Rocksdb)
+[![Build Status](https://travis-ci.org/facebook/rocksdb.svg?branch=master)](https://travis-ci.org/facebook/rocksdb)
+[![Build status](https://ci.appveyor.com/api/projects/status/fbgfu0so3afcno78/branch/master?svg=true)](https://ci.appveyor.com/project/Facebook/rocksdb/branch/master)
 
 
 RocksDB is developed and maintained by Facebook Database Engineering Team.
@@ -26,7 +25,3 @@ rely on the details of any other header files in this package.  Those
 internal APIs may be changed without warning.
 
 Design discussions are conducted in https://www.facebook.com/groups/rocksdb.dev/
-
-## License
-
-RocksDB is dual-licensed under both the GPLv2 (found in the COPYING file in the root directory) and Apache 2.0 License (found in the LICENSE.Apache file in the root directory).  You may select, at your option, one of the above-listed licenses.

ROCKSDB_LITE.md

@@ -5,7 +5,7 @@ RocksDBLite is a project focused on mobile use cases, which don't need a lot of
 Some examples of the features disabled by ROCKSDB_LITE:
 * compiled-in support for LDB tool
 * No backupable DB
-* No support for replication (which we provide in form of TransactionalIterator)
+* No support for replication (which we provide in form of TrasactionalIterator)
 * No advanced monitoring tools
 * No special-purpose memtables that are highly optimized for specific use cases
 * No Transactions

USERS.md

@@ -5,14 +5,12 @@ At Facebook, we use RocksDB as storage engines in multiple data management servi
 
 1. MyRocks -- https://github.com/MySQLOnRocksDB/mysql-5.6
 2. MongoRocks -- https://github.com/mongodb-partners/mongo-rocks
-3. ZippyDB --  Facebook's distributed key-value store with Paxos-style replication, built on top of RocksDB.[1] https://www.youtube.com/watch?v=DfiN7pG0D0khtt
-4. Laser -- Laser is a high query throughput, low (millisecond) latency, key-value storage service built on top of RocksDB.[1]
+3. ZippyDB --  Facebook's distributed key-value store with Paxos-style replication, built on top of RocksDB.[*] https://www.youtube.com/watch?v=DfiN7pG0D0khtt
+4. Laser -- Laser is a high query throughput, low (millisecond) latency, key-value storage service built on top of RocksDB.[*]
 4. Dragon -- a distributed graph query engine. https://code.facebook.com/posts/1737605303120405/dragon-a-distributed-graph-query-engine/
-5. Stylus -- a low-level stream processing framework writtenin C++.[1]
-6. LogDevice -- a distributed data store for logs [2]
+5. Stylus -- a low-level stream processing framework writtenin C++.[*]
 
-[1] https://research.facebook.com/publications/realtime-data-processing-at-facebook/
-[2] https://code.facebook.com/posts/357056558062811/logdevice-a-distributed-data-store-for-logs/
+[*] https://research.facebook.com/publications/realtime-data-processing-at-facebook/
 
 ## LinkedIn
 Two different use cases at Linkedin are using RocksDB as a storage engine:
@@ -85,6 +83,3 @@ quasardb uses a heavily tuned RocksDB as its persistence layer.
 
 ## LzLabs
 LzLabs is using RocksDB as a storage engine in their multi-database distributed framework to store application configuration and user data.
-
-## ProfaneDB
-[ProfaneDB](https://profanedb.gitlab.io/) is a database for Protocol Buffers, and uses RocksDB for storage. It is accessible via gRPC, and the schema is defined using directly `.proto` files.

Vagrantfile

@@ -14,11 +14,6 @@ Vagrant.configure("2") do |config|
     box.vm.box = "chef/centos-6.5"
   end
 
-  config.vm.define "centos7" do |box|
-    box.vm.box = "centos/7"
-    box.vm.provision "shell", path: "build_tools/setup_centos7.sh"
-  end
-
   config.vm.define "FreeBSD10" do |box|
     box.vm.guest = :freebsd
     box.vm.box = "robin/freebsd-10"

appveyor.yml

@@ -1,14 +1,14 @@
 version: 1.0.{build}
-image: Visual Studio 2017
+image: Visual Studio 2015
 before_build:
 - md %APPVEYOR_BUILD_FOLDER%\build
 - cd %APPVEYOR_BUILD_FOLDER%\build
-- cmake -G "Visual Studio 15 Win64" -DOPTDBG=1 -DWITH_XPRESS=1 -DPORTABLE=1 ..
+- cmake -G "Visual Studio 14 2015 Win64" -DOPTDBG=1 -DXPRESS=1 -DPORTABLE=1 ..
 - cd ..
 build:
   project: build\rocksdb.sln
   parallel: true
-  verbosity: normal
+  verbosity: minimal
 test:
 test_script:
 - ps: build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test2,db_test,env_basic_test,env_test -Concurrency 8

buckifier/buckify_rocksdb.py

@@ -36,7 +36,7 @@ def parse_src_mk(repo_path):
 # get all .cc / .c files
 def get_cc_files(repo_path):
     cc_files = []
-    for root, dirnames, filenames in os.walk(repo_path):  # noqa: B007 T25377293 Grandfathered in
+    for root, dirnames, filenames in os.walk(repo_path):
         root = root[(len(repo_path) + 1):]
         if "java" in root:
             # Skip java

buckifier/rocks_test_runner.sh

@@ -1,5 +1,4 @@
 #!/usr/bin/env bash
 # Create a tmp directory for the test to use
 TEST_DIR=$(mktemp -d /dev/shm/fbcode_rocksdb_XXXXXXX)
-# shellcheck disable=SC2068
 TEST_TMPDIR="$TEST_DIR" $@ && rm -rf "$TEST_DIR"

buckifier/targets_builder.py

@@ -3,17 +3,19 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 import targets_cfg
+import pprint
 
-def pretty_list(lst, indent=8):
+# TODO(tec): replace this with PrettyPrinter
+def pretty_list(lst, indent=6):
     if lst is None or len(lst) == 0:
         return ""
 
     if len(lst) == 1:
         return "\"%s\"" % lst[0]
-    
+
     separator = "\",\n%s\"" % (" " * indent)
-    res = separator.join(sorted(lst))
-    res = "\n" + (" " * indent) + "\"" + res + "\",\n" + (" " * (indent - 4))
+    res = separator.join(lst)
+    res = "\n" + (" " * indent) + "\"" + res + "\",\n" + (" " * (indent - 2))
     return res
 
 
@@ -25,7 +27,7 @@ class TARGETSBuilder:
         self.total_lib = 0
         self.total_bin = 0
         self.total_test = 0
-        self.tests_cfg = ""
+        self.tests_cfg = []
 
     def __del__(self):
         self.targets_file.close()
@@ -35,8 +37,8 @@ class TARGETSBuilder:
             headers = "AutoHeaders.RECURSIVE_GLOB"
         self.targets_file.write(targets_cfg.library_template % (
             name,
-            pretty_list(srcs),
             headers,
+            pretty_list(srcs),
             pretty_list(deps)))
         self.total_lib = self.total_lib + 1
 
@@ -51,13 +53,13 @@ class TARGETSBuilder:
         exec_mode = "serial"
         if is_parallel:
             exec_mode = "parallel"
-        self.tests_cfg += targets_cfg.test_cfg_template % (
-            test_name,
-            str(src),
-            str(exec_mode))
+        self.tests_cfg.append([test_name, str(src), str(exec_mode)])
 
         self.total_test = self.total_test + 1
 
     def flush_tests(self):
-        self.targets_file.write(targets_cfg.unittests_template % self.tests_cfg)
-        self.tests_cfg = ""
+        self.targets_file.write(targets_cfg.unittests_template % (
+            pprint.PrettyPrinter().pformat(self.tests_cfg)
+        ))
+
+        self.tests_cfg = []

buckifier/targets_cfg.py

@@ -2,148 +2,123 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
-rocksdb_target_header = """REPO_PATH = package_name() + "/"
+rocksdb_target_header = """
+import os
 
+TARGETS_PATH = os.path.dirname(__file__)
+REPO_PATH = "rocksdb/src/"
 BUCK_BINS = "buck-out/gen/" + REPO_PATH
-
 TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh"
-
 rocksdb_compiler_flags = [
-    "-fno-builtin-memcmp",
-    "-DROCKSDB_PLATFORM_POSIX",
-    "-DROCKSDB_LIB_IO_POSIX",
-    "-DROCKSDB_FALLOCATE_PRESENT",
-    "-DROCKSDB_MALLOC_USABLE_SIZE",
-    "-DROCKSDB_RANGESYNC_PRESENT",
-    "-DROCKSDB_SCHED_GETCPU_PRESENT",
-    "-DROCKSDB_SUPPORT_THREAD_LOCAL",
-    "-DOS_LINUX",
-    # Flags to enable libs we include
-    "-DSNAPPY",
-    "-DZLIB",
-    "-DBZIP2",
-    "-DLZ4",
-    "-DZSTD",
-    "-DGFLAGS=gflags",
-    "-DNUMA",
-    "-DTBB",
-    # Needed to compile in fbcode
-    "-Wno-expansion-to-defined",
-    # Added missing flags from output of build_detect_platform
-    "-DROCKSDB_PTHREAD_ADAPTIVE_MUTEX",
-    "-DROCKSDB_BACKTRACE",
-    "-Wshorten-64-to-32",
+  "-fno-builtin-memcmp",
+  "-DROCKSDB_PLATFORM_POSIX",
+  "-DROCKSDB_LIB_IO_POSIX",
+  "-DROCKSDB_FALLOCATE_PRESENT",
+  "-DROCKSDB_MALLOC_USABLE_SIZE",
+  "-DROCKSDB_RANGESYNC_PRESENT",
+  "-DROCKSDB_SCHED_GETCPU_PRESENT",
+  "-DROCKSDB_SUPPORT_THREAD_LOCAL",
+  "-DOS_LINUX",
+  # Flags to enable libs we include
+  "-DSNAPPY",
+  "-DZLIB",
+  "-DBZIP2",
+  "-DLZ4",
+  "-DZSTD",
+  "-DGFLAGS=gflags",
+  "-DNUMA",
+  "-DTBB",
+  # Needed to compile in fbcode
+  "-Wno-expansion-to-defined",
 ]
 
 rocksdb_external_deps = [
-    ("bzip2", None, "bz2"),
-    ("snappy", None, "snappy"),
-    ("zlib", None, "z"),
-    ("gflags", None, "gflags"),
-    ("lz4", None, "lz4"),
-    ("zstd", None),
-    ("tbb", None),
-    ("numa", None, "numa"),
-    ("googletest", None, "gtest"),
+  ('bzip2', None, 'bz2'),
+  ('snappy', None, "snappy"),
+  ('zlib', None, 'z'),
+  ('gflags', None, 'gflags'),
+  ('lz4', None, 'lz4'),
+  ('zstd', None),
+  ('tbb', None),
+  ("numa", None, "numa"),
+  ("googletest", None, "gtest"),
 ]
 
 rocksdb_preprocessor_flags = [
-    # Directories with files for #include
-    "-I" + REPO_PATH + "include/",
-    "-I" + REPO_PATH,
+  # Directories with files for #include
+  "-I" + REPO_PATH + "include/",
+  "-I" + REPO_PATH,
 ]
 
 rocksdb_arch_preprocessor_flags = {
-    "x86_64": [
-        "-DHAVE_SSE42",
-        "-DHAVE_PCLMUL",
-    ],
+  "x86_64": ["-DHAVE_SSE42"],
 }
-
-build_mode = read_config("fbcode", "build_mode")
-
-is_opt_mode = build_mode.startswith("opt")
-
-# -DNDEBUG is added by default in opt mode in fbcode. But adding it twice
-# doesn't harm and avoid forgetting to add it.
-if is_opt_mode:
-    rocksdb_compiler_flags.append("-DNDEBUG")
 """
 
 
 library_template = """
 cpp_library(
     name = "%s",
-    srcs = [%s],
     headers = %s,
+    srcs = [%s],
+    deps = [%s],
+    preprocessor_flags = rocksdb_preprocessor_flags,
     arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
     compiler_flags = rocksdb_compiler_flags,
-    preprocessor_flags = rocksdb_preprocessor_flags,
-    deps = [%s],
     external_deps = rocksdb_external_deps,
 )
 """
 
 binary_template = """
 cpp_binary(
-    name = "%s",
-    srcs = [%s],
-    arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
-    compiler_flags = rocksdb_compiler_flags,
-    preprocessor_flags = rocksdb_preprocessor_flags,
-    deps = [%s],
-    external_deps = rocksdb_external_deps,
+  name = "%s",
+  srcs = [%s],
+  deps = [%s],
+  preprocessor_flags = rocksdb_preprocessor_flags,
+  arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
+  compiler_flags = rocksdb_compiler_flags,
+  external_deps = rocksdb_external_deps,
 )
 """
 
-test_cfg_template = """    [
-        "%s",
-        "%s",
-        "%s",
-    ],
-"""
-
 unittests_template = """
 # [test_name, test_src, test_type]
-ROCKS_TESTS = [
-%s]
+ROCKS_TESTS = %s
+
 
 # Generate a test rule for each entry in ROCKS_TESTS
-# Do not build the tests in opt mode, since SyncPoint and other test code
-# will not be included.
-if not is_opt_mode:
-    for test_cfg in ROCKS_TESTS:
-        test_name = test_cfg[0]
-        test_cc = test_cfg[1]
-        ttype = "gtest" if test_cfg[2] == "parallel" else "simple"
-        test_bin = test_name + "_bin"
+for test_cfg in ROCKS_TESTS:
+    test_name = test_cfg[0]
+    test_cc = test_cfg[1]
+    ttype = "gtest" if test_cfg[2] == "parallel" else "simple"
+    test_bin = test_name + "_bin"
 
-        cpp_binary (
-          name = test_bin,
-          srcs = [test_cc],
-          deps = [":rocksdb_test_lib"],
-          preprocessor_flags = rocksdb_preprocessor_flags,
-          arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
-          compiler_flags = rocksdb_compiler_flags,
-          external_deps = rocksdb_external_deps,
-        )
+    cpp_binary (
+      name = test_bin,
+      srcs = [test_cc],
+      deps = [":rocksdb_test_lib"],
+      preprocessor_flags = rocksdb_preprocessor_flags,
+      arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
+      compiler_flags = rocksdb_compiler_flags,
+      external_deps = rocksdb_external_deps,
+    )
 
-        custom_unittest(
-          name = test_name,
-          type = ttype,
-          deps = [":" + test_bin],
-          command = [TEST_RUNNER, BUCK_BINS + test_bin]
-        )
+    custom_unittest(
+      name = test_name,
+      type = ttype,
+      deps = [":" + test_bin],
+      command = [TEST_RUNNER, BUCK_BINS + test_bin]
+    )
 
 custom_unittest(
     name = "make_rocksdbjavastatic",
-    command = ["internal_repo_rocksdb/make_rocksdbjavastatic.sh"],
     type = "simple",
+    command = ["internal_repo_rocksdb/make_rocksdbjavastatic.sh"],
 )
 
 custom_unittest(
     name = "make_rocksdb_lite_release",
-    command = ["internal_repo_rocksdb/make_rocksdb_lite_release.sh"],
     type = "simple",
+    command = ["internal_repo_rocksdb/make_rocksdb_lite_release.sh"],
 )
 """

build_tools/build_detect_platform

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/sh
 #
 # Detects OS we're compiling on and outputs a file specified by the first
 # argument, which in turn gets read while processing Makefile.
@@ -16,8 +16,6 @@
 #   PLATFORM_CXXFLAGS           C++ compiler flags.  Will contain:
 #   PLATFORM_SHARED_VERSIONED   Set to 'true' if platform supports versioned
 #                               shared libraries, empty otherwise.
-#   FIND			Command for the find utility
-#   WATCH			Command for the watch utility
 #
 # The PLATFORM_CCFLAGS and PLATFORM_CXXFLAGS might include the following:
 #
@@ -66,23 +64,11 @@ rm -f "$OUTPUT"
 touch "$OUTPUT"
 
 if test -z "$CC"; then
-    if [ -x "$(command -v cc)" ]; then
-        CC=cc
-    elif [ -x "$(command -v clang)" ]; then
-        CC=clang
-    else
-        CC=cc
-    fi
+   CC=cc
 fi
 
 if test -z "$CXX"; then
-    if [ -x "$(command -v g++)" ]; then
-        CXX=g++
-    elif [ -x "$(command -v clang++)" ]; then
-        CXX=clang++
-    else
-        CXX=g++
-    fi
+    CXX=g++
 fi
 
 # Detect OS
@@ -99,15 +85,7 @@ if test -z "$CLANG_SCAN_BUILD"; then
 fi
 
 if test -z "$CLANG_ANALYZER"; then
-    CLANG_ANALYZER=$(command -v clang++ 2> /dev/null)
-fi
-
-if test -z "$FIND"; then
-    FIND=find
-fi
-
-if test -z "$WATCH"; then
-    WATCH=watch
+    CLANG_ANALYZER=$(which clang++ 2> /dev/null)
 fi
 
 COMMON_FLAGS="$COMMON_FLAGS ${CFLAGS}"
@@ -144,8 +122,6 @@ case "$TARGET_OS" in
         COMMON_FLAGS="$COMMON_FLAGS -DOS_LINUX"
         if [ -z "$USE_CLANG" ]; then
             COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp"
-        else
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic"
         fi
         PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
         # PORT_FILES=port/linux/linux_specific.cc
@@ -165,7 +141,6 @@ case "$TARGET_OS" in
         ;;
     FreeBSD)
         PLATFORM=OS_FREEBSD
-        CXX=clang++
         COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_FREEBSD"
         PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread"
         # PORT_FILES=port/freebsd/freebsd_specific.cc
@@ -178,12 +153,9 @@ case "$TARGET_OS" in
         ;;
     OpenBSD)
         PLATFORM=OS_OPENBSD
-	CXX=clang++
         COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_OPENBSD"
         PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -pthread"
         # PORT_FILES=port/openbsd/openbsd_specific.cc
-	FIND=gfind
-	WATCH=gnuwatch
         ;;
     DragonFly)
         PLATFORM=OS_DRAGONFLYBSD
@@ -198,8 +170,6 @@ case "$TARGET_OS" in
         COMMON_FLAGS="$COMMON_FLAGS -DCYGWIN"
         if [ -z "$USE_CLANG" ]; then
             COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp"
-        else
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic"
         fi
         PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
         # PORT_FILES=port/linux/linux_specific.cc
@@ -240,137 +210,119 @@ EOF
         fi
     fi
 
-    if ! test $ROCKSDB_DISABLE_SNAPPY; then
-        # Test whether Snappy library is installed
-        # http://code.google.com/p/snappy/
-        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <snappy.h>
-          int main() {}
+    # Test whether Snappy library is installed
+    # http://code.google.com/p/snappy/
+    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <snappy.h>
+      int main() {}
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DSNAPPY"
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lsnappy"
-            JAVA_LDFLAGS="$JAVA_LDFLAGS -lsnappy"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DSNAPPY"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lsnappy"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -lsnappy"
     fi
 
-    if ! test $ROCKSDB_DISABLE_GFLAGS; then
-        # Test whether gflags library is installed
-        # http://gflags.github.io/gflags/
-        # check if the namespace is gflags
-        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
-          #include <gflags/gflags.h>
-          int main() {}
+    # Test whether gflags library is installed
+    # http://gflags.github.io/gflags/
+    # check if the namespace is gflags
+    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
+      #include <gflags/gflags.h>
+      using namespace gflags;
+      int main() {}
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1"
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
-        else
-          # check if namespace is google
-          $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
-            #include <gflags/gflags.h>
-            using namespace google;
-            int main() {}
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=gflags"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
+    else
+      # check if namespace is google
+      $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
+        #include <gflags/gflags.h>
+        using namespace google;
+        int main() {}
 EOF
-          if [ "$?" = 0 ]; then
-              COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=google"
-              PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
-          fi
-        fi
+      if [ "$?" = 0 ]; then
+          COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=google"
+          PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
+      fi
     fi
 
-    if ! test $ROCKSDB_DISABLE_ZLIB; then
-        # Test whether zlib library is installed
-        $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <zlib.h>
-          int main() {}
+    # Test whether zlib library is installed
+    $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <zlib.h>
+      int main() {}
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DZLIB"
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lz"
-            JAVA_LDFLAGS="$JAVA_LDFLAGS -lz"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DZLIB"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lz"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -lz"
     fi
 
-    if ! test $ROCKSDB_DISABLE_BZIP; then
-        # Test whether bzip library is installed
-        $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <bzlib.h>
-          int main() {}
+    # Test whether bzip library is installed
+    $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <bzlib.h>
+      int main() {}
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DBZIP2"
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lbz2"
-            JAVA_LDFLAGS="$JAVA_LDFLAGS -lbz2"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DBZIP2"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lbz2"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -lbz2"
     fi
 
-    if ! test $ROCKSDB_DISABLE_LZ4; then
-        # Test whether lz4 library is installed
-        $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <lz4.h>
-          #include <lz4hc.h>
-          int main() {}
+    # Test whether lz4 library is installed
+    $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <lz4.h>
+      #include <lz4hc.h>
+      int main() {}
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DLZ4"
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -llz4"
-            JAVA_LDFLAGS="$JAVA_LDFLAGS -llz4"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DLZ4"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -llz4"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -llz4"
     fi
 
-    if ! test $ROCKSDB_DISABLE_ZSTD; then
-        # Test whether zstd library is installed
-        $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <zstd.h>
-          int main() {}
+    # Test whether zstd library is installed
+    $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <zstd.h>
+      int main() {}
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DZSTD"
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lzstd"
-            JAVA_LDFLAGS="$JAVA_LDFLAGS -lzstd"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DZSTD"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lzstd"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -lzstd"
     fi
 
-    if ! test $ROCKSDB_DISABLE_NUMA; then
-        # Test whether numa is available
-        $CXX $CFLAGS -x c++ - -o /dev/null -lnuma 2>/dev/null  <<EOF
-          #include <numa.h>
-          #include <numaif.h>
-          int main() {}
+    # Test whether numa is available
+    $CXX $CFLAGS -x c++ - -o /dev/null -lnuma 2>/dev/null  <<EOF
+      #include <numa.h>
+      #include <numaif.h>
+      int main() {}
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DNUMA"
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lnuma"
-            JAVA_LDFLAGS="$JAVA_LDFLAGS -lnuma"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DNUMA"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lnuma"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -lnuma"
     fi
 
-    if ! test $ROCKSDB_DISABLE_TBB; then
-        # Test whether tbb is available
-        $CXX $CFLAGS $LDFLAGS -x c++ - -o /dev/null -ltbb 2>/dev/null  <<EOF
-          #include <tbb/tbb.h>
-          int main() {}
+    # Test whether tbb is available
+    $CXX $CFLAGS $LDFLAGS -x c++ - -o /dev/null -ltbb 2>/dev/null  <<EOF
+      #include <tbb/tbb.h>
+      int main() {}
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DTBB"
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltbb"
-            JAVA_LDFLAGS="$JAVA_LDFLAGS -ltbb"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DTBB"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltbb"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -ltbb"
     fi
 
-    if ! test $ROCKSDB_DISABLE_JEMALLOC; then
-        # Test whether jemalloc is available
-        if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null -ljemalloc \
-          2>/dev/null; then
-            # This will enable some preprocessor identifiers in the Makefile
-            JEMALLOC=1
-            # JEMALLOC can be enabled either using the flag (like here) or by
-            # providing direct link to the jemalloc library
-            WITH_JEMALLOC_FLAG=1
-        fi
-    fi
-    if ! test $JEMALLOC && ! test $ROCKSDB_DISABLE_TCMALLOC; then
+    # Test whether jemalloc is available
+    if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null -ljemalloc \
+      2>/dev/null; then
+        # This will enable some preprocessor identifiers in the Makefile
+        JEMALLOC=1
+        # JEMALLOC can be enabled either using the flag (like here) or by
+        # providing direct link to the jemalloc library
+        WITH_JEMALLOC_FLAG=1
+    else
         # jemalloc is not available. Let's try tcmalloc
         if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null \
           -ltcmalloc 2>/dev/null; then
@@ -379,100 +331,88 @@ EOF
         fi
     fi
 
-    if ! test $ROCKSDB_DISABLE_MALLOC_USABLE_SIZE; then
-        # Test whether malloc_usable_size is available
-        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <malloc.h>
-          int main() {
-            size_t res = malloc_usable_size(0);
-            return 0;
-          }
+    # Test whether malloc_usable_size is available
+    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <malloc.h>
+      int main() {
+        size_t res = malloc_usable_size(0);
+        return 0;
+      }
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_MALLOC_USABLE_SIZE"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_MALLOC_USABLE_SIZE"
     fi
 
-    if ! test $ROCKSDB_DISABLE_PTHREAD_MUTEX_ADAPTIVE_NP; then
-        # Test whether PTHREAD_MUTEX_ADAPTIVE_NP mutex type is available
-        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <pthread.h>
-          int main() {
-            int x = PTHREAD_MUTEX_ADAPTIVE_NP;
-            return 0;
-          }
+    # Test whether PTHREAD_MUTEX_ADAPTIVE_NP mutex type is available
+    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <pthread.h>
+      int main() {
+        int x = PTHREAD_MUTEX_ADAPTIVE_NP;
+        return 0;
+      }
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_PTHREAD_ADAPTIVE_MUTEX"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_PTHREAD_ADAPTIVE_MUTEX"
     fi
 
-    if ! test $ROCKSDB_DISABLE_BACKTRACE; then
-        # Test whether backtrace is available
-        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <execinfo.h>>
+    # Test whether backtrace is available
+    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <execinfo.h>>
+      int main() {
+        void* frames[1];
+        backtrace_symbols(frames, backtrace(frames, 1));
+        return 0;
+      }
+EOF
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE"
+    else
+        # Test whether execinfo library is installed
+        $CXX $CFLAGS -lexecinfo -x c++ - -o /dev/null 2>/dev/null  <<EOF
+          #include <execinfo.h>
           int main() {
             void* frames[1];
             backtrace_symbols(frames, backtrace(frames, 1));
-            return 0;
           }
 EOF
         if [ "$?" = 0 ]; then
             COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE"
-        else
-            # Test whether execinfo library is installed
-            $CXX $CFLAGS -lexecinfo -x c++ - -o /dev/null 2>/dev/null  <<EOF
-              #include <execinfo.h>
-              int main() {
-                void* frames[1];
-                backtrace_symbols(frames, backtrace(frames, 1));
-              }
-EOF
-            if [ "$?" = 0 ]; then
-                COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE"
-                PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lexecinfo"
-                JAVA_LDFLAGS="$JAVA_LDFLAGS -lexecinfo"
-            fi
+            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lexecinfo"
+            JAVA_LDFLAGS="$JAVA_LDFLAGS -lexecinfo"
         fi
     fi
 
-    if ! test $ROCKSDB_DISABLE_PG; then
-        # Test if -pg is supported
-        $CXX $CFLAGS -pg -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          int main() {
-            return 0;
-          }
+    # Test if -pg is supported
+    $CXX $CFLAGS -pg -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      int main() {
+        return 0;
+      }
 EOF
-        if [ "$?" = 0 ]; then
-            PROFILING_FLAGS=-pg
-        fi
+    if [ "$?" = 0 ]; then
+        PROFILING_FLAGS=-pg
     fi
 
-    if ! test $ROCKSDB_DISABLE_SYNC_FILE_RANGE; then
-        # Test whether sync_file_range is supported for compatibility with an old glibc
-        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <fcntl.h>
-          int main() {
-            int fd = open("/dev/null", 0);
-            sync_file_range(fd, 0, 1024, SYNC_FILE_RANGE_WRITE);
-          }
+    # Test whether sync_file_range is supported for compatibility with an old glibc
+    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <fcntl.h>
+      int main() {
+        int fd = open("/dev/null", 0);
+        sync_file_range(fd, 0, 1024, SYNC_FILE_RANGE_WRITE);
+      }
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_RANGESYNC_PRESENT"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_RANGESYNC_PRESENT"
     fi
 
-    if ! test $ROCKSDB_DISABLE_SCHED_GETCPU; then
-        # Test whether sched_getcpu is supported
-        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <sched.h>
-          int main() {
-            int cpuid = sched_getcpu();
-          }
+    # Test whether sched_getcpu is supported
+    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <sched.h>
+      int main() {
+        int cpuid = sched_getcpu();
+      }
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_SCHED_GETCPU_PRESENT"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_SCHED_GETCPU_PRESENT"
     fi
 fi
 
@@ -504,25 +444,18 @@ if test "$USE_HDFS"; then
   JAVA_LDFLAGS="$JAVA_LDFLAGS $HDFS_LDFLAGS"
 fi
 
-if test -z "$PORTABLE"; then
+if test "$USE_SSE"; then
+  COMMON_FLAGS="$COMMON_FLAGS -msse4.2"
+elif test -z "$PORTABLE"; then
   if test -n "`echo $TARGET_ARCHITECTURE | grep ^ppc64`"; then
     # Tune for this POWER processor, treating '+' models as base models
     POWER=`LD_SHOW_AUXV=1 /bin/true | grep AT_PLATFORM | grep -E -o power[0-9]+`
     COMMON_FLAGS="$COMMON_FLAGS -mcpu=$POWER -mtune=$POWER "
   elif test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then
     COMMON_FLAGS="$COMMON_FLAGS -march=z10 "
-  elif test -n "`echo $TARGET_ARCHITECTURE | grep ^arm`"; then
-    # TODO: Handle this with approprite options.
-    COMMON_FLAGS="$COMMON_FLAGS"
-  elif [ "$TARGET_OS" == "IOS" ]; then
-    COMMON_FLAGS="$COMMON_FLAGS"
-  elif [ "$TARGET_OS" != "AIX" ] && [ "$TARGET_OS" != "SunOS" ]; then
+  elif [ "$TARGET_OS" != AIX ] && [ "$TARGET_OS" != SunOS ]; then
     COMMON_FLAGS="$COMMON_FLAGS -march=native "
-  elif test "$USE_SSE"; then
-    COMMON_FLAGS="$COMMON_FLAGS -msse4.2 -mpclmul"
   fi
-elif test "$USE_SSE"; then
-  COMMON_FLAGS="$COMMON_FLAGS -msse4.2 -mpclmul"
 fi
 
 $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
@@ -536,24 +469,6 @@ if [ "$?" = 0 ]; then
   COMMON_FLAGS="$COMMON_FLAGS -DHAVE_SSE42"
 elif test "$USE_SSE"; then
   echo "warning: USE_SSE specified but compiler could not use SSE intrinsics, disabling"
-  exit 1
-fi
-
-$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
-  #include <cstdint>
-  #include <wmmintrin.h>
-  int main() {
-    const auto a = _mm_set_epi64x(0, 0);
-    const auto b = _mm_set_epi64x(0, 0);
-    const auto c = _mm_clmulepi64_si128(a, b, 0x00);
-    auto d = _mm_cvtsi128_si64(c);
-  }
-EOF
-if [ "$?" = 0 ]; then
-  COMMON_FLAGS="$COMMON_FLAGS -DHAVE_PCLMUL"
-elif test "$USE_SSE"; then
-  echo "warning: USE_SSE specified but compiler could not use PCLMUL intrinsics, disabling"
-  exit 1
 fi
 
 # iOS doesn't support thread-local storage, but this check would erroneously
@@ -604,8 +519,6 @@ echo "ROCKSDB_PATCH=$ROCKSDB_PATCH" >> "$OUTPUT"
 echo "CLANG_SCAN_BUILD=$CLANG_SCAN_BUILD" >> "$OUTPUT"
 echo "CLANG_ANALYZER=$CLANG_ANALYZER" >> "$OUTPUT"
 echo "PROFILING_FLAGS=$PROFILING_FLAGS" >> "$OUTPUT"
-echo "FIND=$FIND" >> "$OUTPUT"
-echo "WATCH=$WATCH" >> "$OUTPUT"
 # This will enable some related identifiers for the preprocessor
 if test -n "$JEMALLOC"; then
   echo "JEMALLOC=1" >> "$OUTPUT"

build_tools/cont_integration.sh

@@ -13,12 +13,10 @@ error=0
 
 function log {
   DATE=`date +%Y-%m-%d:%H:%M:%S`
-  # shellcheck disable=SC2068
   echo $DATE $@
 }
 
 function log_err {
-  # shellcheck disable=SC2145
   log "ERROR: $@ Error code: $error."
 }
 

build_tools/dependencies.sh

@@ -1,19 +1,18 @@
-# shellcheck disable=SC2148
-GCC_BASE=/mnt/gvfs/third-party2/gcc/8219ec1bcedf8ad9da05e121e193364de2cc4f61/5.x/centos6-native/c447969
-CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/64d8d58e3d84f8bde7a029763d4f5baf39d0d5b9/stable/centos6-native/6aaf4de
-LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/ba9be983c81de7299b59fe71950c664a84dcb5f8/5.x/gcc-5-glibc-2.23/339d858
-GLIBC_BASE=/mnt/gvfs/third-party2/glibc/f20197cf3d4bd50339c9777aaa0b2ccadad9e2cb/2.23/gcc-5-glibc-2.23/ca1d1c0
-SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/6427ce8c7496e4ab06c2da81543b94c0de8be3d0/1.1.3/gcc-5-glibc-2.23/9bc6787
-ZLIB_BASE=/mnt/gvfs/third-party2/zlib/8f1e8b867d26efef93eac2fabbdb2e1d512665d7/1.2.8/gcc-5-glibc-2.23/9bc6787
-BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/70471c0571559fe0af7db6d7e8860b93a7eadfe1/1.0.6/gcc-5-glibc-2.23/9bc6787
-LZ4_BASE=/mnt/gvfs/third-party2/lz4/453c89d6f0e68cdf1c151c769197fabedad9cac8/r131/gcc-5-glibc-2.23/9bc6787
-ZSTD_BASE=/mnt/gvfs/third-party2/zstd/00a40fa5f8bd2cd0622f2e868552793aef37ccf4/1.3.0/gcc-5-glibc-2.23/03859b5
-GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/47eef08f9acb77de982fbda6047c26d330739538/2.2.0/gcc-5-glibc-2.23/9bc6787
-JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/4414ddc78df8008b35cc4adac23590ad29148584/master/gcc-5-glibc-2.23/d506c82
-NUMA_BASE=/mnt/gvfs/third-party2/numa/9d7ae2693d05d62f9a579cb21e6b717cf257a75d/2.0.11/gcc-5-glibc-2.23/9bc6787
-LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/2b2dd58e3a52ccf2c1d827def59e5f740de0ad15/1.2/gcc-5-glibc-2.23/b443de1
-TBB_BASE=/mnt/gvfs/third-party2/tbb/379addf7ab2468a2b4293b47456cfcd1c9cb318d/4.3/gcc-5-glibc-2.23/9bc6787
-KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/3f68f5fe65a85b7c2d3e66852268fbd1efdb3151/4.0.9-36_fbk5_2933_gd092e3f/gcc-5-glibc-2.23/da39a3e
-BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/b9fab0aec99d9c36408e810b2677e91c12807afd/2.28/centos6-native/da39a3e
-VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/423431d61786b20bcc3bde8972901130cb29e6b3/3.11.0/gcc-5-glibc-2.23/9bc6787
-LUA_BASE=/mnt/gvfs/third-party2/lua/3b0bb3bd9a0f690a069c479fcc0f7424fc7456d2/5.2.3/gcc-5-glibc-2.23/65372bd
+GCC_BASE=/mnt/gvfs/third-party2/gcc/2928bb3ed95bf64f5b388ee88c30dc74710c3b35/5.x/centos6-native/f4950a1
+CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/a5fea028cb7ba43498976e1f8054b0b2e790c295/stable/centos6-native/6aaf4de
+LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/7a9099f6587ee4378c0b1fa32bb8934019d30ca4/5.x/gcc-5-glibc-2.23/339d858
+GLIBC_BASE=/mnt/gvfs/third-party2/glibc/3b7c6469854dfc7832a1c3cc5b86919a84e5f865/2.23/gcc-5-glibc-2.23/ca1d1c0
+SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/8c38a4c1e52b4c2cc8a9cdc31b9c947ed7dbfcb4/1.1.3/gcc-5-glibc-2.23/9bc6787
+ZLIB_BASE=/mnt/gvfs/third-party2/zlib/d7861abe6f0e27ab98c9303b95a662f0e4cdedb5/1.2.8/gcc-5-glibc-2.23/9bc6787
+BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/740325875f6729f42d28deaa2147b0854f3a347e/1.0.6/gcc-5-glibc-2.23/9bc6787
+LZ4_BASE=/mnt/gvfs/third-party2/lz4/0815d59804160c96caac5f27ca004f51af893dc6/r131/gcc-5-glibc-2.23/9bc6787
+ZSTD_BASE=/mnt/gvfs/third-party2/zstd/c15a4f5f619a2930478d01e2e34dc1e0652b0873/1.1.4/gcc-5-glibc-2.23/03859b5
+GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/f905a5e1032fb30c05db3d3752319857388c0c49/2.2.0/gcc-5-glibc-2.23/9bc6787
+JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/8d60633d822a2a55849c73db24e74a25e52b71db/master/gcc-5-glibc-2.23/1c32b4b
+NUMA_BASE=/mnt/gvfs/third-party2/numa/17c514c4d102a25ca15f4558be564eeed76f4b6a/2.0.8/gcc-5-glibc-2.23/9bc6787
+LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/8db74270cd6d0212ac92d69e7fc7beefe617d772/trunk/gcc-5-glibc-2.23/b1847cb
+TBB_BASE=/mnt/gvfs/third-party2/tbb/9d9a554877d0c5bef330fe818ab7178806dd316a/4.0_update2/gcc-5-glibc-2.23/9bc6787
+KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/90c9734afc5579c9d1db529fa788d09f97763b85/4.0.9-36_fbk5_2933_gd092e3f/gcc-5-glibc-2.23/da39a3e
+BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/9e829389ef61b92c62de8748c80169aaf25ce1f0/2.26.1/centos6-native/da39a3e
+VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d7f4d4d86674a57668e3a96f76f0e17dd0eb8765/3.11.0/gcc-5-glibc-2.23/9bc6787
+LUA_BASE=/mnt/gvfs/third-party2/lua/61e4abf5813bbc39bc4f548757ccfcadde175a48/5.2.3/gcc-5-glibc-2.23/65372bd

build_tools/dependencies_4.8.1.sh

@@ -1,4 +1,3 @@
-# shellcheck disable=SC2148
 GCC_BASE=/mnt/gvfs/third-party2/gcc/cf7d14c625ce30bae1a4661c2319c5a283e4dd22/4.8.1/centos6-native/cc6c9dc
 CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/8598c375b0e94e1448182eb3df034704144a838d/stable/centos6-native/3f16ddd
 LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/d6e0a7da6faba45f5e5b1638f9edd7afc2f34e7d/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc

build_tools/fbcode_config.sh

@@ -83,7 +83,6 @@ CFLAGS+=" -DTBB"
 
 # use Intel SSE support for checksum calculations
 export USE_SSE=1
-export PORTABLE=1
 
 BINUTILS="$BINUTILS_BASE/bin"
 AR="$BINUTILS/ar"

build_tools/fbcode_config4.8.1.sh

@@ -54,7 +54,6 @@ TBB_LIBS="$TBB_BASE/lib/libtbb.a"
 
 # use Intel SSE support for checksum calculations
 export USE_SSE=1
-export PORTABLE=1
 
 BINUTILS="$BINUTILS_BASE/bin"
 AR="$BINUTILS/ar"

build_tools/make_package.sh

@@ -1,4 +1,3 @@
-# shellcheck disable=SC1113
 #/usr/bin/env bash
 
 set -e
@@ -29,14 +28,12 @@ function package() {
     if dpkg --get-selections | grep --quiet $1; then
       log "$1 is already installed. skipping."
     else
-      # shellcheck disable=SC2068
       apt-get install $@ -y
     fi
   elif [[ $OS = "centos" ]]; then
     if rpm -qa | grep --quiet $1; then
       log "$1 is already installed. skipping."
     else
-      # shellcheck disable=SC2068
       yum install $@ -y
     fi
   fi
@@ -55,7 +52,6 @@ function gem_install() {
   if gem list | grep --quiet $1; then
     log "$1 is already installed. skipping."
   else
-    # shellcheck disable=SC2068
     gem install $@
   fi
 }
@@ -129,5 +125,4 @@ function main() {
     include $LIB_DIR
 }
 
-# shellcheck disable=SC2068
 main $@

build_tools/rocksdb-lego-determinator

@@ -352,22 +352,6 @@ LITE_BUILD_COMMANDS="[
     }
 ]"
 
-#
-# Report RocksDB lite binary size to scuba
-REPORT_LITE_BINARY_SIZE_COMMANDS="[
-    {
-        'name':'Rocksdb Lite Binary Size',
-        'oncall':'$ONCALL',
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'Report RocksDB Lite binary size to scuba',
-                'shell':'tools/report_lite_binary_size.sh',
-                'user':'root',
-            },
-        ],
-]"
-
 #
 # RocksDB stress/crash test
 #
@@ -744,9 +728,6 @@ case $1 in
   lite)
     echo $LITE_BUILD_COMMANDS
     ;;
-  report_lite_binary_size)
-    echo $REPORT_LITE_BINARY_SIZE_COMMANDS
-    ;;
   stress_crash)
     echo $STRESS_CRASH_TEST_COMMANDS
     ;;

build_tools/run_ci_db_test.ps1

@@ -1,16 +1,13 @@
 # This script enables you running RocksDB tests by running
 # All the tests concurrently and utilizing all the cores
 Param(
-  [switch]$EnableJE = $false,  # Look for and use test executable, append _je to listed exclusions
+  [switch]$EnableJE = $false,  # Look for and use _je executable, append _je to listed exclusions
   [switch]$RunAll = $false,    # Will attempt discover all *_test[_je].exe binaries and run all
                                # of them as Google suites. I.e. It will run test cases concurrently
                                # except those mentioned as $Run, those will run as individual test cases
                                # And any execlued with $ExcludeExes or $ExcludeCases
                                # It will also not run any individual test cases
                                # excluded but $ExcludeCasese
-  [switch]$RunAllExe = $false, # Look for and use test exdcutables, append _je to exclusions automatically
-                               # It will attempt to run them in parallel w/o breaking them up on individual
-                               # test cases. Those listed with $ExcludeExes will be excluded
   [string]$SuiteRun = "",      # Split test suites in test cases and run in parallel, not compatible with $RunAll
   [string]$Run = "",           # Run specified executables in parallel but do not split to test cases
   [string]$ExcludeCases = "",  # Exclude test cases, expects a comma separated list, no spaces
@@ -42,18 +39,13 @@ $RunOnly.Add("compact_on_deletion_collector_test") | Out-Null
 $RunOnly.Add("merge_test") | Out-Null
 $RunOnly.Add("stringappend_test") | Out-Null # Apparently incorrectly written
 $RunOnly.Add("backupable_db_test") | Out-Null # Disabled
-$RunOnly.Add("timer_queue_test") | Out-Null # Not a gtest
+
 
 if($RunAll -and $SuiteRun -ne "") {
     Write-Error "$RunAll and $SuiteRun are not compatible"
     exit 1
 }
 
-if($RunAllExe -and $Run -ne "") {
-    Write-Error "$RunAllExe and $Run are not compatible"
-    exit 1
-}
-
 # If running under Appveyor assume that root
 [string]$Appveyor = $Env:APPVEYOR_BUILD_FOLDER
 if($Appveyor -ne "") {
@@ -139,8 +131,12 @@ function ExtractTestCases([string]$GTestExe, $HashTable) {
 
       # Leading whitespace is fine
       $l = $l -replace '^\s+',''
+      # but no whitespace any other place
+      if($l -match "\s+") {
+        continue
+      }
       # Trailing dot is a test group but no whitespace
-      if ($l -match "\.$" -and $l -notmatch "\s+") {
+      elseif ( $l -match "\.$" ) {
         $Group = $l
       }  else {
         # Otherwise it is a test name, remove leading space
@@ -227,11 +223,13 @@ $TestExes = [ordered]@{}
 if($Run -ne "") {
 
   $test_list = $Run -split ' '
+
   ForEach($t in $test_list) {
 
     if($EnableJE) {
       $t += "_je"
     }
+
     MakeAndAdd -token $t -HashTable $TestExes
   }
 
@@ -239,38 +237,6 @@ if($Run -ne "") {
      Write-Error "Failed to extract tests from $Run"
      exit 1
   }
-} elseif($RunAllExe) {
-  # Discover all the test binaries
-  if($EnableJE) {
-    $pattern = "*_test_je.exe"
-  } else {
-    $pattern = "*_test.exe"
-  }
-
-  $search_path = -join ($BinariesFolder, $pattern)
-  Write-Host "Binaries Search Path: $search_path"
-
-  $DiscoveredExe = @()
-  dir -Path $search_path | ForEach-Object {
-     $DiscoveredExe += ($_.Name)     
-  }
-
-  # Remove exclusions
-  ForEach($e in $DiscoveredExe) {
-    $e = $e -replace '.exe$', ''
-    $bare_name = $e -replace '_je$', ''
-
-    if($ExcludeExesSet.Contains($bare_name)) {
-      Write-Warning "Test $e is excluded"
-      continue
-    }
-    MakeAndAdd -token $e -HashTable $TestExes
-  }
-
-  if($TestExes.Count -lt 1) {
-     Write-Error "Failed to discover test executables"
-     exit 1
-  }
 }
 
 # Ordered by exe @{ Exe = @{ TestCase = LogName }}
@@ -279,7 +245,9 @@ $CasesToRun = [ordered]@{}
 if($SuiteRun -ne "") {
   $suite_list = $SuiteRun -split ' '
   ProcessSuites -ListOfSuites $suite_list -HashOfHashes $CasesToRun
-} elseif ($RunAll) {
+}
+
+if($RunAll) {
 # Discover all the test binaries
   if($EnableJE) {
     $pattern = "*_test_je.exe"
@@ -287,6 +255,7 @@ if($SuiteRun -ne "") {
     $pattern = "*_test.exe"
   }
 
+
   $search_path = -join ($BinariesFolder, $pattern)
   Write-Host "Binaries Search Path: $search_path"
 
@@ -318,6 +287,8 @@ if($SuiteRun -ne "") {
 }
 
 
+Write-Host "Attempting to start: $NumTestsToStart tests"
+
 # Invoke a test with a filter and redirect all output
 $InvokeTestCase = {
     param($exe, $test, $log);
@@ -336,7 +307,7 @@ $InvokeTestAsync = {
 # Test limiting factor here
 [int]$count = 0
 # Overall status
-[bool]$script:success = $true;
+[bool]$success = $true;
 
 function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
 {
@@ -394,7 +365,6 @@ function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
                  break
                }
 
-              Write-Host "Starting $exe_name"
               [string]$Exe =  -Join ($BinariesFolder, $exe_name)
               $job = Start-Job -Name $exe_name -ScriptBlock $InvokeTestAsync -ArgumentList @($Exe,$log_path)
               $JobToLog.Add($job, $log_path)
@@ -425,7 +395,7 @@ function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
         $log_content = @(Get-Content $log)
 
         if($completed.State -ne "Completed") {
-            $script:success = $false
+            $success = $false
             Write-Warning $message
             $log_content | Write-Warning
         } else {
@@ -449,7 +419,7 @@ function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
             }
 
             if(!$pass_found) {
-                $script:success = $false;
+                $success = $false;
                 Write-Warning $message
                 $log_content | Write-Warning
             } else {
@@ -473,7 +443,7 @@ New-TimeSpan -Start $StartDate -End $EndDate |
   }
 
 
-if(!$script:success) {
+if(!$success) {
 # This does not succeed killing off jobs quick
 # So we simply exit
 #    Remove-Job -Job $jobs -Force

build_tools/setup_centos7.sh

@@ -1,43 +0,0 @@
-#!/bin/bash
-set -e
-
-ROCKSDB_VERSION="5.10.3"
-ZSTD_VERSION="1.1.3"
-
-echo "This script configures CentOS with everything needed to build and run RocksDB"
-
-yum update -y && yum install epel-release -y
-
-yum install -y \
-  wget \
-  gcc-c++ \
-  snappy snappy-devel \
-  zlib zlib-devel \
-  bzip2 bzip2-devel \
-  lz4-devel \
-  libasan \
-  gflags
-
-mkdir -pv /usr/local/rocksdb-${ROCKSDB_VERSION}
-ln -sfT /usr/local/rocksdb-${ROCKSDB_VERSION} /usr/local/rocksdb
-
-wget -qO /tmp/zstd-${ZSTD_VERSION}.tar.gz https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz
-wget -qO /tmp/rocksdb-${ROCKSDB_VERSION}.tar.gz https://github.com/facebook/rocksdb/archive/v${ROCKSDB_VERSION}.tar.gz
-
-cd /tmp
-
-tar xzvf zstd-${ZSTD_VERSION}.tar.gz
-tar xzvf rocksdb-${ROCKSDB_VERSION}.tar.gz -C /usr/local/
-
-echo "Installing ZSTD..."
-pushd zstd-${ZSTD_VERSION}
-make && make install
-popd
-
-echo "Compiling RocksDB..."
-cd /usr/local/rocksdb
-chown -R vagrant:vagrant /usr/local/rocksdb/
-sudo -u vagrant make static_lib
-cd examples/
-sudo -u vagrant make all
-sudo -u vagrant ./c_simple_example

build_tools/update_dependencies.sh

@@ -38,7 +38,6 @@ function get_lib_base()
     # platform is not provided, use latest gcc
     result=`ls -dr1v $result/gcc-*[^fb]/ | head -n1`
   else
-    echo $lib_platform
     result="$result/$lib_platform/"
   fi
   
@@ -72,22 +71,22 @@ log_variable GCC_BASE
 log_variable CLANG_BASE
 
 # Libraries locations
-get_lib_base libgcc     5.x     gcc-5-glibc-2.23
-get_lib_base glibc      2.23    gcc-5-glibc-2.23
-get_lib_base snappy     LATEST  gcc-5-glibc-2.23
-get_lib_base zlib       LATEST  gcc-5-glibc-2.23
-get_lib_base bzip2      LATEST  gcc-5-glibc-2.23
-get_lib_base lz4        LATEST  gcc-5-glibc-2.23
-get_lib_base zstd       LATEST  gcc-5-glibc-2.23
-get_lib_base gflags     LATEST  gcc-5-glibc-2.23
-get_lib_base jemalloc   LATEST  gcc-5-glibc-2.23
-get_lib_base numa       LATEST  gcc-5-glibc-2.23
-get_lib_base libunwind  LATEST  gcc-5-glibc-2.23
-get_lib_base tbb        LATEST  gcc-5-glibc-2.23
+get_lib_base libgcc     5.x
+get_lib_base glibc      2.23
+get_lib_base snappy     LATEST gcc-5-glibc-2.23
+get_lib_base zlib       LATEST
+get_lib_base bzip2      LATEST
+get_lib_base lz4        LATEST
+get_lib_base zstd       LATEST
+get_lib_base gflags     LATEST
+get_lib_base jemalloc   LATEST
+get_lib_base numa       LATEST
+get_lib_base libunwind  LATEST
+get_lib_base tbb        4.0_update2 gcc-5-glibc-2.23
 
-get_lib_base kernel-headers 4.0.9-36_fbk5_2933_gd092e3f gcc-5-glibc-2.23
+get_lib_base kernel-headers LATEST 
 get_lib_base binutils   LATEST centos6-native 
-get_lib_base valgrind   LATEST gcc-5-glibc-2.23
+get_lib_base valgrind   3.10.0 gcc-5-glibc-2.23
 get_lib_base lua        5.2.3 gcc-5-glibc-2.23
 
 git diff $OUTPUT

build_tools/version.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/sh
 if [ "$#" = "0" ]; then
   echo "Usage: $0 major|minor|patch|full"
   exit 1

cache/cache_bench.cc

@@ -17,16 +17,16 @@ int main() {
 #include <inttypes.h>
 #include <sys/types.h>
 #include <stdio.h>
+#include <gflags/gflags.h>
 
-#include "port/port.h"
-#include "rocksdb/cache.h"
 #include "rocksdb/db.h"
+#include "rocksdb/cache.h"
 #include "rocksdb/env.h"
-#include "util/gflags_compat.h"
+#include "port/port.h"
 #include "util/mutexlock.h"
 #include "util/random.h"
 
-using GFLAGS_NAMESPACE::ParseCommandLineFlags;
+using GFLAGS::ParseCommandLineFlags;
 
 static const uint32_t KB = 1024;
 
@@ -52,7 +52,7 @@ namespace rocksdb {
 
 class CacheBench;
 namespace {
-void deleter(const Slice& /*key*/, void* value) {
+void deleter(const Slice& key, void* value) {
     delete reinterpret_cast<char *>(value);
 }
 

cache/cache_test.cc

@@ -40,9 +40,9 @@ static int DecodeValue(void* v) {
 const std::string kLRU = "lru";
 const std::string kClock = "clock";
 
-void dumbDeleter(const Slice& /*key*/, void* /*value*/) {}
+void dumbDeleter(const Slice& key, void* value) {}
 
-void eraseDeleter(const Slice& /*key*/, void* value) {
+void eraseDeleter(const Slice& key, void* value) {
   Cache* cache = reinterpret_cast<Cache*>(value);
   cache->Erase("foo");
 }
@@ -470,7 +470,7 @@ class Value {
 };
 
 namespace {
-void deleter(const Slice& /*key*/, void* value) {
+void deleter(const Slice& key, void* value) {
   delete static_cast<Value *>(value);
 }
 }  // namespace

cache/clock_cache.cc

@@ -13,8 +13,8 @@
 
 namespace rocksdb {
 
-std::shared_ptr<Cache> NewClockCache(size_t /*capacity*/, int /*num_shard_bits*/,
-                                     bool /*strict_capacity_limit*/) {
+std::shared_ptr<Cache> NewClockCache(size_t capacity, int num_shard_bits,
+                                     bool strict_capacity_limit) {
   // Clock cache not supported.
   return nullptr;
 }
@@ -367,9 +367,7 @@ ClockCacheShard::~ClockCacheShard() {
   for (auto& handle : list_) {
     uint32_t flags = handle.flags.load(std::memory_order_relaxed);
     if (InCache(flags) || CountRefs(flags) > 0) {
-      if (handle.deleter != nullptr) {
-        (*handle.deleter)(handle.key, handle.value);
-      }
+      (*handle.deleter)(handle.key, handle.value);
       delete[] handle.key.data();
     }
   }
@@ -588,7 +586,7 @@ Status ClockCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
                                size_t charge,
                                void (*deleter)(const Slice& key, void* value),
                                Cache::Handle** out_handle,
-                               Cache::Priority /*priority*/) {
+                               Cache::Priority priority) {
   CleanupContext context;
   HashTable::accessor accessor;
   char* key_data = new char[key.size()];

cache/lru_cache.cc

@@ -99,20 +99,12 @@ void LRUHandleTable::Resize() {
   length_ = new_length;
 }
 
-LRUCacheShard::LRUCacheShard(size_t capacity, bool strict_capacity_limit,
-                             double high_pri_pool_ratio)
-    : capacity_(0),
-      high_pri_pool_usage_(0),
-      strict_capacity_limit_(strict_capacity_limit),
-      high_pri_pool_ratio_(high_pri_pool_ratio),
-      high_pri_pool_capacity_(0),
-      usage_(0),
-      lru_usage_(0) {
+LRUCacheShard::LRUCacheShard()
+    : high_pri_pool_usage_(0), usage_(0), lru_usage_(0) {
   // Make empty circular linked list
   lru_.next = &lru_;
   lru_.prev = &lru_;
   lru_low_pri_ = &lru_;
-  SetCapacity(capacity);
 }
 
 LRUCacheShard::~LRUCacheShard() {}
@@ -175,11 +167,6 @@ size_t LRUCacheShard::TEST_GetLRUSize() {
   return lru_size;
 }
 
-double LRUCacheShard::GetHighPriPoolRatio() {
-  MutexLock l(&mutex_);
-  return high_pri_pool_ratio_;
-}
-
 void LRUCacheShard::LRU_Remove(LRUHandle* e) {
   assert(e->next != nullptr);
   assert(e->prev != nullptr);
@@ -250,13 +237,17 @@ void* LRUCacheShard::operator new(size_t size) {
   return port::cacheline_aligned_alloc(size);
 }
 
-void* LRUCacheShard::operator new(size_t /*size*/, void* ptr) { return ptr; }
+void* LRUCacheShard::operator new[](size_t size) {
+  return port::cacheline_aligned_alloc(size);
+}
 
 void LRUCacheShard::operator delete(void *memblock) {
   port::cacheline_aligned_free(memblock);
 }
 
-void LRUCacheShard::operator delete(void* /*memblock*/, void* /*ptr*/) {}
+void LRUCacheShard::operator delete[](void* memblock) {
+  port::cacheline_aligned_free(memblock);
+}
 
 void LRUCacheShard::SetCapacity(size_t capacity) {
   autovector<LRUHandle*> last_reference_list;
@@ -362,7 +353,6 @@ Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
   e->deleter = deleter;
   e->charge = charge;
   e->key_length = key.size();
-  e->flags = 0;
   e->hash = hash;
   e->refs = (handle == nullptr
                  ? 1
@@ -475,21 +465,15 @@ LRUCache::LRUCache(size_t capacity, int num_shard_bits,
                    bool strict_capacity_limit, double high_pri_pool_ratio)
     : ShardedCache(capacity, num_shard_bits, strict_capacity_limit) {
   num_shards_ = 1 << num_shard_bits;
-  shards_ = reinterpret_cast<LRUCacheShard*>(
-      port::cacheline_aligned_alloc(sizeof(LRUCacheShard) * num_shards_));
-  size_t per_shard = (capacity + (num_shards_ - 1)) / num_shards_;
+  shards_ = new LRUCacheShard[num_shards_];
+  SetCapacity(capacity);
+  SetStrictCapacityLimit(strict_capacity_limit);
   for (int i = 0; i < num_shards_; i++) {
-    new (&shards_[i])
-        LRUCacheShard(per_shard, strict_capacity_limit, high_pri_pool_ratio);
+    shards_[i].SetHighPriorityPoolRatio(high_pri_pool_ratio);
   }
 }
 
-LRUCache::~LRUCache() {
-  for (int i = 0; i < num_shards_; i++) {
-    shards_[i].~LRUCacheShard();
-  }
-  port::cacheline_aligned_free(shards_);
-}
+LRUCache::~LRUCache() { delete[] shards_; }
 
 CacheShard* LRUCache::GetShard(int shard) {
   return reinterpret_cast<CacheShard*>(&shards_[shard]);
@@ -526,20 +510,6 @@ size_t LRUCache::TEST_GetLRUSize() {
   return lru_size_of_all_shards;
 }
 
-double LRUCache::GetHighPriPoolRatio() {
-  double result = 0.0;
-  if (num_shards_ > 0) {
-    result = shards_[0].GetHighPriPoolRatio();
-  }
-  return result;
-}
-
-std::shared_ptr<Cache> NewLRUCache(const LRUCacheOptions& cache_opts) {
-  return NewLRUCache(cache_opts.capacity, cache_opts.num_shard_bits,
-                     cache_opts.strict_capacity_limit,
-                     cache_opts.high_pri_pool_ratio);
-}
-
 std::shared_ptr<Cache> NewLRUCache(size_t capacity, int num_shard_bits,
                                    bool strict_capacity_limit,
                                    double high_pri_pool_ratio) {

cache/lru_cache.h

@@ -57,7 +57,7 @@ struct LRUHandle {
   // Include the following flags:
   //   in_cache:    whether this entry is referenced by the hash table.
   //   is_high_pri: whether this entry is high priority entry.
-  //   in_high_pri_pool: whether this entry is in high-pri pool.
+  //   in_high_pro_pool: whether this entry is in high-pri pool.
   char flags;
 
   uint32_t hash;     // Hash of key(); used for fast sharding and comparisons
@@ -156,8 +156,7 @@ class LRUHandleTable {
 // A single shard of sharded cache.
 class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard : public CacheShard {
  public:
-  LRUCacheShard(size_t capacity, bool strict_capacity_limit,
-                double high_pri_pool_ratio);
+  LRUCacheShard();
   virtual ~LRUCacheShard();
 
   // Separate from constructor so caller can easily make an array of LRUCache
@@ -203,20 +202,14 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard : public CacheShard {
   //  not threadsafe
   size_t TEST_GetLRUSize();
 
-  //  Retrives high pri pool ratio
-  double GetHighPriPoolRatio();
-
   // Overloading to aligned it to cache line size
-  // They are used by tests.
   void* operator new(size_t);
 
-  // placement new
-  void* operator new(size_t, void*);
+  void* operator new[](size_t);
 
   void operator delete(void *);
 
-  // placement delete, does nothing.
-  void operator delete(void*, void*);
+  void operator delete[](void*);
 
  private:
   void LRU_Remove(LRUHandle* e);
@@ -300,8 +293,6 @@ class LRUCache : public ShardedCache {
 
   //  Retrieves number of elements in LRU, for unit test purpose only
   size_t TEST_GetLRUSize();
-  //  Retrives high pri pool ratio
-  double GetHighPriPoolRatio();
 
  private:
   LRUCacheShard* shards_;

cache/lru_cache_test.cc

@@ -7,7 +7,6 @@
 
 #include <string>
 #include <vector>
-#include "port/port.h"
 #include "util/testharness.h"
 
 namespace rocksdb {
@@ -18,8 +17,19 @@ class LRUCacheTest : public testing::Test {
   ~LRUCacheTest() {}
 
   void NewCache(size_t capacity, double high_pri_pool_ratio = 0.0) {
-    cache_.reset(new LRUCacheShard(capacity, false /*strict_capcity_limit*/,
-                                   high_pri_pool_ratio));
+    cache_.reset(
+#if defined(_MSC_VER)
+#pragma warning(push)
+#pragma warning(disable: 4316) // We've validated the alignment with the new operators
+#endif
+      new LRUCacheShard()
+#if defined(_MSC_VER)
+#pragma warning(pop)
+#endif
+    );
+    cache_->SetCapacity(capacity);
+    cache_->SetStrictCapacityLimit(false);
+    cache_->SetHighPriorityPoolRatio(high_pri_pool_ratio);
   }
 
   void Insert(const std::string& key,
@@ -104,7 +114,7 @@ TEST_F(LRUCacheTest, BasicLRU) {
   ValidateLRUList({"e", "z", "d", "u", "v"});
 }
 
-TEST_F(LRUCacheTest, EntriesWithPriority) {
+TEST_F(LRUCacheTest, MidPointInsertion) {
   // Allocate 2 cache entries to high-pri pool.
   NewCache(5, 0.45);
 

cache/sharded_cache.cc

@@ -53,7 +53,7 @@ Status ShardedCache::Insert(const Slice& key, void* value, size_t charge,
       ->Insert(key, hash, value, charge, deleter, handle, priority);
 }
 
-Cache::Handle* ShardedCache::Lookup(const Slice& key, Statistics* /*stats*/) {
+Cache::Handle* ShardedCache::Lookup(const Slice& key, Statistics* stats) {
   uint32_t hash = HashSlice(key);
   return GetShard(Shard(hash))->Lookup(key, hash);
 }

cmake/modules/FindNUMA.cmake

@@ -1,21 +0,0 @@
-# - Find NUMA
-# Find the NUMA library and includes
-#
-# NUMA_INCLUDE_DIR - where to find numa.h, etc.
-# NUMA_LIBRARIES - List of libraries when using NUMA.
-# NUMA_FOUND - True if NUMA found.
-
-find_path(NUMA_INCLUDE_DIR
-  NAMES numa.h numaif.h
-  HINTS ${NUMA_ROOT_DIR}/include)
-
-find_library(NUMA_LIBRARIES
-  NAMES numa
-  HINTS ${NUMA_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(NUMA DEFAULT_MSG NUMA_LIBRARIES NUMA_INCLUDE_DIR)
-
-mark_as_advanced(
-  NUMA_LIBRARIES
-  NUMA_INCLUDE_DIR)

cmake/modules/FindTBB.cmake

@@ -1,25 +0,0 @@
-# - Find TBB
-# Find the Thread Building Blocks library and includes
-#
-# TBB_INCLUDE_DIR - where to find tbb.h, etc.
-# TBB_LIBRARIES - List of libraries when using TBB.
-# TBB_FOUND - True if TBB found.
-
-if(NOT DEFINED TBB_ROOT_DIR)
-  set(TBB_ROOT_DIR "$ENV{TBBROOT}")
-endif()
-
-find_path(TBB_INCLUDE_DIR
-NAMES tbb/tbb.h
-HINTS ${TBB_ROOT_DIR}/include)
-
-find_library(TBB_LIBRARIES
-NAMES tbb
-HINTS ${TBB_ROOT_DIR}/lib ENV LIBRARY_PATH)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(TBB DEFAULT_MSG TBB_LIBRARIES TBB_INCLUDE_DIR)
-
-mark_as_advanced(
-TBB_LIBRARIES
-TBB_INCLUDE_DIR)

coverage/parse_gcov_output.py

@@ -72,7 +72,7 @@ def display_file_coverage(per_file_coverage, total_coverage):
     header_template = \
         "%" + str(max_file_name_length) + "s\t%s\t%s"
     separator = "-" * (max_file_name_length + 10 + 20)
-    print header_template % ("Filename", "Coverage", "Lines")  # noqa: E999 T25377293 Grandfathered in
+    print header_template % ("Filename", "Coverage", "Lines")
     print separator
 
     # -- Print body

db/builder.cc

@@ -39,7 +39,7 @@ namespace rocksdb {
 class TableFactory;
 
 TableBuilder* NewTableBuilder(
-    const ImmutableCFOptions& ioptions, const MutableCFOptions& moptions,
+    const ImmutableCFOptions& ioptions,
     const InternalKeyComparator& internal_comparator,
     const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
         int_tbl_prop_collector_factories,
@@ -52,11 +52,10 @@ TableBuilder* NewTableBuilder(
           TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
          column_family_name.empty());
   return ioptions.table_factory->NewTableBuilder(
-      TableBuilderOptions(ioptions, moptions, internal_comparator,
-                          int_tbl_prop_collector_factories, compression_type,
-                          compression_opts, compression_dict, skip_filters,
-                          column_family_name, level, creation_time,
-                          oldest_key_time),
+      TableBuilderOptions(
+          ioptions, internal_comparator, int_tbl_prop_collector_factories,
+          compression_type, compression_opts, compression_dict, skip_filters,
+          column_family_name, level, creation_time, oldest_key_time),
       column_family_id, file);
 }
 
@@ -71,12 +70,12 @@ Status BuildTable(
     uint32_t column_family_id, const std::string& column_family_name,
     std::vector<SequenceNumber> snapshots,
     SequenceNumber earliest_write_conflict_snapshot,
-    SnapshotChecker* snapshot_checker, const CompressionType compression,
+    const CompressionType compression,
     const CompressionOptions& compression_opts, bool paranoid_file_checks,
     InternalStats* internal_stats, TableFileCreationReason reason,
     EventLogger* event_logger, int job_id, const Env::IOPriority io_priority,
     TableProperties* table_properties, int level, const uint64_t creation_time,
-    const uint64_t oldest_key_time, Env::WriteLifeTimeHint write_hint) {
+    const uint64_t oldest_key_time) {
   assert((column_family_id ==
           TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
          column_family_name.empty());
@@ -93,7 +92,7 @@ Status BuildTable(
     return s;
   }
 
-  std::string fname = TableFileName(ioptions.cf_paths, meta->fd.GetNumber(),
+  std::string fname = TableFileName(ioptions.db_paths, meta->fd.GetNumber(),
                                     meta->fd.GetPathId());
 #ifndef ROCKSDB_LITE
   EventHelpers::NotifyTableFileCreationStarted(
@@ -118,27 +117,24 @@ Status BuildTable(
         return s;
       }
       file->SetIOPriority(io_priority);
-      file->SetWriteLifeTimeHint(write_hint);
 
       file_writer.reset(new WritableFileWriter(std::move(file), env_options,
                                                ioptions.statistics));
       builder = NewTableBuilder(
-          ioptions, mutable_cf_options, internal_comparator,
-          int_tbl_prop_collector_factories, column_family_id,
-          column_family_name, file_writer.get(), compression, compression_opts,
-          level, nullptr /* compression_dict */, false /* skip_filters */,
-          creation_time, oldest_key_time);
+          ioptions, internal_comparator, int_tbl_prop_collector_factories,
+          column_family_id, column_family_name, file_writer.get(), compression,
+          compression_opts, level, nullptr /* compression_dict */,
+          false /* skip_filters */, creation_time, oldest_key_time);
     }
 
     MergeHelper merge(env, internal_comparator.user_comparator(),
                       ioptions.merge_operator, nullptr, ioptions.info_log,
                       true /* internal key corruption is not ok */,
-                      snapshots.empty() ? 0 : snapshots.back(),
-                      snapshot_checker);
+                      snapshots.empty() ? 0 : snapshots.back());
 
     CompactionIterator c_iter(
         iter, internal_comparator.user_comparator(), &merge, kMaxSequenceNumber,
-        &snapshots, earliest_write_conflict_snapshot, snapshot_checker, env,
+        &snapshots, earliest_write_conflict_snapshot, env,
         true /* internal key corruption is not ok */, range_del_agg.get());
     c_iter.SeekToFirst();
     for (; c_iter.Valid(); c_iter.Next()) {
@@ -197,8 +193,7 @@ Status BuildTable(
       // to cache it here for further user reads
       std::unique_ptr<InternalIterator> it(table_cache->NewIterator(
           ReadOptions(), env_options, internal_comparator, meta->fd,
-          nullptr /* range_del_agg */,
-          mutable_cf_options.prefix_extractor.get(), nullptr,
+          nullptr /* range_del_agg */, nullptr,
           (internal_stats == nullptr) ? nullptr
                                       : internal_stats->GetFileReadHist(0),
           false /* for_compaction */, nullptr /* arena */,

db/builder.h

@@ -29,7 +29,6 @@ struct FileMetaData;
 class Env;
 struct EnvOptions;
 class Iterator;
-class SnapshotChecker;
 class TableCache;
 class VersionEdit;
 class TableBuilder;
@@ -43,7 +42,7 @@ class InternalIterator;
 // @param compression_dict Data for presetting the compression library's
 //    dictionary, or nullptr.
 TableBuilder* NewTableBuilder(
-    const ImmutableCFOptions& options, const MutableCFOptions& moptions,
+    const ImmutableCFOptions& options,
     const InternalKeyComparator& internal_comparator,
     const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
         int_tbl_prop_collector_factories,
@@ -73,13 +72,12 @@ extern Status BuildTable(
     uint32_t column_family_id, const std::string& column_family_name,
     std::vector<SequenceNumber> snapshots,
     SequenceNumber earliest_write_conflict_snapshot,
-    SnapshotChecker* snapshot_checker, const CompressionType compression,
+    const CompressionType compression,
     const CompressionOptions& compression_opts, bool paranoid_file_checks,
     InternalStats* internal_stats, TableFileCreationReason reason,
     EventLogger* event_logger = nullptr, int job_id = 0,
     const Env::IOPriority io_priority = Env::IO_HIGH,
     TableProperties* table_properties = nullptr, int level = -1,
-    const uint64_t creation_time = 0, const uint64_t oldest_key_time = 0,
-    Env::WriteLifeTimeHint write_hint = Env::WLTH_NOT_SET);
+    const uint64_t creation_time = 0, const uint64_t oldest_key_time = 0);
 
 }  // namespace rocksdb

db/c.cc

@@ -21,25 +21,23 @@
 #include "rocksdb/env.h"
 #include "rocksdb/filter_policy.h"
 #include "rocksdb/iterator.h"
-#include "rocksdb/memtablerep.h"
 #include "rocksdb/merge_operator.h"
 #include "rocksdb/options.h"
-#include "rocksdb/rate_limiter.h"
-#include "rocksdb/slice_transform.h"
-#include "rocksdb/statistics.h"
 #include "rocksdb/status.h"
-#include "rocksdb/table.h"
+#include "rocksdb/write_batch.h"
+#include "rocksdb/memtablerep.h"
 #include "rocksdb/universal_compaction.h"
+#include "rocksdb/statistics.h"
+#include "rocksdb/slice_transform.h"
+#include "rocksdb/table.h"
+#include "rocksdb/rate_limiter.h"
 #include "rocksdb/utilities/backupable_db.h"
-#include "rocksdb/utilities/checkpoint.h"
-#include "rocksdb/utilities/db_ttl.h"
-#include "rocksdb/utilities/optimistic_transaction_db.h"
+#include "rocksdb/utilities/write_batch_with_index.h"
+#include "utilities/merge_operators.h"
 #include "rocksdb/utilities/transaction.h"
 #include "rocksdb/utilities/transaction_db.h"
-#include "rocksdb/utilities/write_batch_with_index.h"
-#include "rocksdb/write_batch.h"
-#include "rocksdb/perf_context.h"
-#include "utilities/merge_operators.h"
+#include "rocksdb/utilities/optimistic_transaction_db.h"
+#include "rocksdb/utilities/checkpoint.h"
 
 using rocksdb::BytewiseComparator;
 using rocksdb::Cache;
@@ -102,10 +100,6 @@ using rocksdb::OptimisticTransactionDB;
 using rocksdb::OptimisticTransactionOptions;
 using rocksdb::Transaction;
 using rocksdb::Checkpoint;
-using rocksdb::TransactionLogIterator;
-using rocksdb::BatchResult;
-using rocksdb::PerfLevel;
-using rocksdb::PerfContext;
 
 using std::shared_ptr;
 
@@ -123,9 +117,7 @@ struct rocksdb_flushoptions_t    { FlushOptions      rep; };
 struct rocksdb_fifo_compaction_options_t { CompactionOptionsFIFO rep; };
 struct rocksdb_readoptions_t {
    ReadOptions rep;
-   // stack variables to set pointers to in ReadOptions
-   Slice upper_bound;
-   Slice lower_bound;
+   Slice upper_bound; // stack variable to set pointer to in ReadOptions
 };
 struct rocksdb_writeoptions_t    { WriteOptions      rep; };
 struct rocksdb_options_t         { Options           rep; };
@@ -137,8 +129,6 @@ struct rocksdb_cuckoo_table_options_t  { CuckooTableOptions rep; };
 struct rocksdb_seqfile_t         { SequentialFile*   rep; };
 struct rocksdb_randomfile_t      { RandomAccessFile* rep; };
 struct rocksdb_writablefile_t    { WritableFile*     rep; };
-struct rocksdb_wal_iterator_t { TransactionLogIterator* rep; };
-struct rocksdb_wal_readoptions_t { TransactionLogIterator::ReadOptions rep; };
 struct rocksdb_filelock_t        { FileLock*         rep; };
 struct rocksdb_logger_t          { shared_ptr<Logger>  rep; };
 struct rocksdb_cache_t           { shared_ptr<Cache>   rep; };
@@ -147,8 +137,7 @@ struct rocksdb_column_family_handle_t  { ColumnFamilyHandle* rep; };
 struct rocksdb_envoptions_t      { EnvOptions        rep; };
 struct rocksdb_ingestexternalfileoptions_t  { IngestExternalFileOptions rep; };
 struct rocksdb_sstfilewriter_t   { SstFileWriter*    rep; };
-struct rocksdb_ratelimiter_t     { shared_ptr<RateLimiter>      rep; };
-struct rocksdb_perfcontext_t     { PerfContext*      rep; };
+struct rocksdb_ratelimiter_t     { RateLimiter*      rep; };
 struct rocksdb_pinnableslice_t {
   PinnableSlice rep;
 };
@@ -261,7 +250,7 @@ struct rocksdb_comparator_t : public Comparator {
   // No-ops since the C binding does not support key shortening methods.
   virtual void FindShortestSeparator(std::string*,
                                      const Slice&) const override {}
-  virtual void FindShortSuccessor(std::string* /*key*/) const override {}
+  virtual void FindShortSuccessor(std::string* key) const override {}
 };
 
 struct rocksdb_filterpolicy_t : public FilterPolicy {
@@ -376,7 +365,7 @@ struct rocksdb_mergeoperator_t : public MergeOperator {
   virtual bool PartialMergeMulti(const Slice& key,
                                  const std::deque<Slice>& operand_list,
                                  std::string* new_value,
-                                 Logger* /*logger*/) const override {
+                                 Logger* logger) const override {
     size_t operand_count = operand_list.size();
     std::vector<const char*> operand_pointers(operand_count);
     std::vector<size_t> operand_sizes(operand_count);
@@ -486,20 +475,6 @@ rocksdb_t* rocksdb_open(
   return result;
 }
 
-rocksdb_t* rocksdb_open_with_ttl(
-    const rocksdb_options_t* options,
-    const char* name,
-    int ttl,
-    char** errptr) {
-  rocksdb::DBWithTTL* db;
-  if (SaveError(errptr, rocksdb::DBWithTTL::Open(options->rep, std::string(name), &db, ttl))) {
-    return nullptr;
-  }
-  rocksdb_t* result = new rocksdb_t;
-  result->rep = db;
-  return result;
-}
-
 rocksdb_t* rocksdb_open_for_read_only(
     const rocksdb_options_t* options,
     const char* name,
@@ -934,54 +909,6 @@ rocksdb_iterator_t* rocksdb_create_iterator(
   return result;
 }
 
-rocksdb_wal_iterator_t* rocksdb_get_updates_since(
-        rocksdb_t* db, uint64_t seq_number,
-        const rocksdb_wal_readoptions_t* options,
-        char** errptr) {
-  std::unique_ptr<TransactionLogIterator> iter;
-  TransactionLogIterator::ReadOptions ro;
-  if (options!=nullptr) {
-      ro = options->rep;
-  }
-  if (SaveError(errptr, db->rep->GetUpdatesSince(seq_number, &iter, ro))) {
-    return nullptr;
-  }
-  rocksdb_wal_iterator_t* result = new rocksdb_wal_iterator_t;
-  result->rep = iter.release();
-  return result;
-}
-
-void rocksdb_wal_iter_next(rocksdb_wal_iterator_t* iter) {
-    iter->rep->Next();
-}
-
-unsigned char rocksdb_wal_iter_valid(const rocksdb_wal_iterator_t* iter) {
-    return iter->rep->Valid();
-}
-
-void rocksdb_wal_iter_status (const rocksdb_wal_iterator_t* iter, char** errptr) {
-    SaveError(errptr, iter->rep->status());
-}
-
-void rocksdb_wal_iter_destroy (const rocksdb_wal_iterator_t* iter) {
-  delete iter->rep;
-  delete iter;
-}
-
-rocksdb_writebatch_t* rocksdb_wal_iter_get_batch (const rocksdb_wal_iterator_t* iter, uint64_t* seq) {
-  rocksdb_writebatch_t* result = rocksdb_writebatch_create();
-  BatchResult wal_batch = iter->rep->GetBatch();
-  result->rep = * wal_batch.writeBatchPtr.release();
-  if (seq != nullptr) {
-    *seq = wal_batch.sequence;
-  }
-  return result;
-}
-
-uint64_t rocksdb_get_latest_sequence_number (rocksdb_t *db) {
-    return db->rep->GetLatestSequenceNumber();
-}
-
 rocksdb_iterator_t* rocksdb_create_iterator_cf(
     rocksdb_t* db,
     const rocksdb_readoptions_t* options,
@@ -1459,24 +1386,23 @@ void rocksdb_writebatch_put_log_data(
   b->rep.PutLogData(Slice(blob, len));
 }
 
-class H : public WriteBatch::Handler {
- public:
-  void* state_;
-  void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
-  void (*deleted_)(void*, const char* k, size_t klen);
-  virtual void Put(const Slice& key, const Slice& value) override {
-    (*put_)(state_, key.data(), key.size(), value.data(), value.size());
-  }
-  virtual void Delete(const Slice& key) override {
-    (*deleted_)(state_, key.data(), key.size());
-  }
-};
-
 void rocksdb_writebatch_iterate(
     rocksdb_writebatch_t* b,
     void* state,
     void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen),
     void (*deleted)(void*, const char* k, size_t klen)) {
+  class H : public WriteBatch::Handler {
+   public:
+    void* state_;
+    void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
+    void (*deleted_)(void*, const char* k, size_t klen);
+    virtual void Put(const Slice& key, const Slice& value) override {
+      (*put_)(state_, key.data(), key.size(), value.data(), value.size());
+    }
+    virtual void Delete(const Slice& key) override {
+      (*deleted_)(state_, key.data(), key.size());
+    }
+  };
   H handler;
   handler.state_ = state;
   handler.put_ = put;
@@ -1721,6 +1647,18 @@ void rocksdb_writebatch_wi_iterate(
     void* state,
     void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen),
     void (*deleted)(void*, const char* k, size_t klen)) {
+  class H : public WriteBatch::Handler {
+   public:
+    void* state_;
+    void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
+    void (*deleted_)(void*, const char* k, size_t klen);
+    virtual void Put(const Slice& key, const Slice& value) override {
+      (*put_)(state_, key.data(), key.size(), value.data(), value.size());
+    }
+    virtual void Delete(const Slice& key) override {
+      (*deleted_)(state_, key.data(), key.size());
+    }
+  };
   H handler;
   handler.state_ = state;
   handler.put_ = put;
@@ -1886,26 +1824,6 @@ void rocksdb_block_based_options_set_block_restart_interval(
   options->rep.block_restart_interval = block_restart_interval;
 }
 
-void rocksdb_block_based_options_set_index_block_restart_interval(
-    rocksdb_block_based_table_options_t* options, int index_block_restart_interval) {
-  options->rep.index_block_restart_interval = index_block_restart_interval;
-}
-
-void rocksdb_block_based_options_set_metadata_block_size(
-    rocksdb_block_based_table_options_t* options, uint64_t metadata_block_size) {
-  options->rep.metadata_block_size = metadata_block_size;
-}
-
-void rocksdb_block_based_options_set_partition_filters(
-    rocksdb_block_based_table_options_t* options, unsigned char partition_filters) {
-  options->rep.partition_filters = partition_filters;
-}
-
-void rocksdb_block_based_options_set_use_delta_encoding(
-    rocksdb_block_based_table_options_t* options, unsigned char use_delta_encoding) {
-  options->rep.use_delta_encoding = use_delta_encoding;
-}
-
 void rocksdb_block_based_options_set_filter_policy(
     rocksdb_block_based_table_options_t* options,
     rocksdb_filterpolicy_t* filter_policy) {
@@ -1959,11 +1877,6 @@ void rocksdb_block_based_options_set_cache_index_and_filter_blocks(
   options->rep.cache_index_and_filter_blocks = v;
 }
 
-void rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority(
-    rocksdb_block_based_table_options_t* options, unsigned char v) {
-  options->rep.cache_index_and_filter_blocks_with_high_priority = v;
-}
-
 void rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache(
     rocksdb_block_based_table_options_t* options, unsigned char v) {
   options->rep.pin_l0_filter_and_index_blocks_in_cache = v;
@@ -1978,6 +1891,7 @@ void rocksdb_options_set_block_based_table_factory(
   }
 }
 
+
 rocksdb_cuckoo_table_options_t*
 rocksdb_cuckoo_options_create() {
   return new rocksdb_cuckoo_table_options_t;
@@ -2059,11 +1973,6 @@ void rocksdb_options_optimize_universal_style_compaction(
   opt->rep.OptimizeUniversalStyleCompaction(memtable_memory_budget);
 }
 
-void rocksdb_options_set_allow_ingest_behind(
-    rocksdb_options_t* opt, unsigned char v) {
-  opt->rep.allow_ingest_behind = v;
-}
-
 void rocksdb_options_set_compaction_filter(
     rocksdb_options_t* opt,
     rocksdb_compactionfilter_t* filter) {
@@ -2114,8 +2023,8 @@ void rocksdb_options_set_paranoid_checks(
   opt->rep.paranoid_checks = v;
 }
 
-void rocksdb_options_set_db_paths(rocksdb_options_t* opt,
-                                  const rocksdb_dbpath_t** dbpath_values,
+void rocksdb_options_set_db_paths(rocksdb_options_t* opt, 
+                                  const rocksdb_dbpath_t** dbpath_values, 
                                   size_t num_paths) {
   std::vector<DbPath> db_paths(num_paths);
   for (size_t i = 0; i < num_paths; ++i) {
@@ -2202,8 +2111,7 @@ void rocksdb_options_enable_statistics(rocksdb_options_t* opt) {
   opt->rep.statistics = rocksdb::CreateDBStatistics();
 }
 
-void rocksdb_options_set_skip_stats_update_on_db_open(rocksdb_options_t* opt,
-                                                      unsigned char val) {
+void rocksdb_options_set_skip_stats_update_on_db_open(rocksdb_options_t* opt, unsigned char val) {
   opt->rep.skip_stats_update_on_db_open = val;
 }
 
@@ -2226,8 +2134,8 @@ void rocksdb_options_set_level0_stop_writes_trigger(
   opt->rep.level0_stop_writes_trigger = n;
 }
 
-void rocksdb_options_set_max_mem_compaction_level(rocksdb_options_t* /*opt*/,
-                                                  int /*n*/) {}
+void rocksdb_options_set_max_mem_compaction_level(rocksdb_options_t* opt,
+                                                  int n) {}
 
 void rocksdb_options_set_wal_recovery_mode(rocksdb_options_t* opt,int mode) {
   opt->rep.wal_recovery_mode = static_cast<WALRecoveryMode>(mode);
@@ -2291,8 +2199,8 @@ void rocksdb_options_set_manifest_preallocation_size(
 }
 
 // noop
-void rocksdb_options_set_purge_redundant_kvs_while_flush(
-    rocksdb_options_t* /*opt*/, unsigned char /*v*/) {}
+void rocksdb_options_set_purge_redundant_kvs_while_flush(rocksdb_options_t* opt,
+                                                         unsigned char v) {}
 
 void rocksdb_options_set_use_direct_reads(rocksdb_options_t* opt,
                                           unsigned char v) {
@@ -2357,21 +2265,11 @@ void rocksdb_options_set_use_adaptive_mutex(
   opt->rep.use_adaptive_mutex = v;
 }
 
-void rocksdb_options_set_wal_bytes_per_sync(
-    rocksdb_options_t* opt, uint64_t v) {
-  opt->rep.wal_bytes_per_sync = v;
-}
-
 void rocksdb_options_set_bytes_per_sync(
     rocksdb_options_t* opt, uint64_t v) {
   opt->rep.bytes_per_sync = v;
 }
 
-void rocksdb_options_set_writable_file_max_buffer_size(rocksdb_options_t* opt,
-                                                       uint64_t v) {
-  opt->rep.writable_file_max_buffer_size = v;
-}
-
 void rocksdb_options_set_allow_concurrent_memtable_write(rocksdb_options_t* opt,
                                                          unsigned char v) {
   opt->rep.allow_concurrent_memtable_write = v;
@@ -2462,7 +2360,7 @@ void rocksdb_options_set_table_cache_numshardbits(
 }
 
 void rocksdb_options_set_table_cache_remove_scan_count_limit(
-    rocksdb_options_t* /*opt*/, int /*v*/) {
+    rocksdb_options_t* opt, int v) {
   // this option is deprecated
 }
 
@@ -2576,9 +2474,8 @@ char *rocksdb_options_statistics_get_string(rocksdb_options_t *opt) {
 }
 
 void rocksdb_options_set_ratelimiter(rocksdb_options_t *opt, rocksdb_ratelimiter_t *limiter) {
-  if (limiter) {
-    opt->rep.rate_limiter = limiter->rep;
-  }
+  opt->rep.rate_limiter.reset(limiter->rep);
+  limiter->rep = nullptr;
 }
 
 rocksdb_ratelimiter_t* rocksdb_ratelimiter_create(
@@ -2586,184 +2483,16 @@ rocksdb_ratelimiter_t* rocksdb_ratelimiter_create(
     int64_t refill_period_us,
     int32_t fairness) {
   rocksdb_ratelimiter_t* rate_limiter = new rocksdb_ratelimiter_t;
-  rate_limiter->rep.reset(
-               NewGenericRateLimiter(rate_bytes_per_sec,
-                                     refill_period_us, fairness));
+  rate_limiter->rep = NewGenericRateLimiter(rate_bytes_per_sec,
+                                            refill_period_us, fairness);
   return rate_limiter;
 }
 
 void rocksdb_ratelimiter_destroy(rocksdb_ratelimiter_t *limiter) {
-  delete limiter;
-}
-
-void rocksdb_set_perf_level(int v) {
-  PerfLevel level = static_cast<PerfLevel>(v);
-  SetPerfLevel(level);
-}
-
-rocksdb_perfcontext_t* rocksdb_perfcontext_create() {
-  rocksdb_perfcontext_t* context = new rocksdb_perfcontext_t;
-  context->rep = rocksdb::get_perf_context();
-  return context;
-}
-
-void rocksdb_perfcontext_reset(rocksdb_perfcontext_t* context) {
-  context->rep->Reset();
-}
-
-char* rocksdb_perfcontext_report(rocksdb_perfcontext_t* context,
-    unsigned char exclude_zero_counters) {
-  return strdup(context->rep->ToString(exclude_zero_counters).c_str());
-}
-
-uint64_t rocksdb_perfcontext_metric(rocksdb_perfcontext_t* context,
-    int metric) {
-  PerfContext* rep = context->rep;
-  switch (metric) {
-    case rocksdb_user_key_comparison_count:
-      return rep->user_key_comparison_count;
-    case rocksdb_block_cache_hit_count:
-      return rep->block_cache_hit_count;
-    case rocksdb_block_read_count:
-      return rep->block_read_count;
-    case rocksdb_block_read_byte:
-      return rep->block_read_byte;
-    case rocksdb_block_read_time:
-      return rep->block_read_time;
-    case rocksdb_block_checksum_time:
-      return rep->block_checksum_time;
-    case rocksdb_block_decompress_time:
-      return rep->block_decompress_time;
-    case rocksdb_get_read_bytes:
-      return rep->get_read_bytes;
-    case rocksdb_multiget_read_bytes:
-      return rep->multiget_read_bytes;
-    case rocksdb_iter_read_bytes:
-      return rep->iter_read_bytes;
-    case rocksdb_internal_key_skipped_count:
-      return rep->internal_key_skipped_count;
-    case rocksdb_internal_delete_skipped_count:
-      return rep->internal_delete_skipped_count;
-    case rocksdb_internal_recent_skipped_count:
-      return rep->internal_recent_skipped_count;
-    case rocksdb_internal_merge_count:
-      return rep->internal_merge_count;
-    case rocksdb_get_snapshot_time:
-      return rep->get_snapshot_time;
-    case rocksdb_get_from_memtable_time:
-      return rep->get_from_memtable_time;
-    case rocksdb_get_from_memtable_count:
-      return rep->get_from_memtable_count;
-    case rocksdb_get_post_process_time:
-      return rep->get_post_process_time;
-    case rocksdb_get_from_output_files_time:
-      return rep->get_from_output_files_time;
-    case rocksdb_seek_on_memtable_time:
-      return rep->seek_on_memtable_time;
-    case rocksdb_seek_on_memtable_count:
-      return rep->seek_on_memtable_count;
-    case rocksdb_next_on_memtable_count:
-      return rep->next_on_memtable_count;
-    case rocksdb_prev_on_memtable_count:
-      return rep->prev_on_memtable_count;
-    case rocksdb_seek_child_seek_time:
-      return rep->seek_child_seek_time;
-    case rocksdb_seek_child_seek_count:
-      return rep->seek_child_seek_count;
-    case rocksdb_seek_min_heap_time:
-      return rep->seek_min_heap_time;
-    case rocksdb_seek_max_heap_time:
-      return rep->seek_max_heap_time;
-    case rocksdb_seek_internal_seek_time:
-      return rep->seek_internal_seek_time;
-    case rocksdb_find_next_user_entry_time:
-      return rep->find_next_user_entry_time;
-    case rocksdb_write_wal_time:
-      return rep->write_wal_time;
-    case rocksdb_write_memtable_time:
-      return rep->write_memtable_time;
-    case rocksdb_write_delay_time:
-      return rep->write_delay_time;
-    case rocksdb_write_pre_and_post_process_time:
-      return rep->write_pre_and_post_process_time;
-    case rocksdb_db_mutex_lock_nanos:
-      return rep->db_mutex_lock_nanos;
-    case rocksdb_db_condition_wait_nanos:
-      return rep->db_condition_wait_nanos;
-    case rocksdb_merge_operator_time_nanos:
-      return rep->merge_operator_time_nanos;
-    case rocksdb_read_index_block_nanos:
-      return rep->read_index_block_nanos;
-    case rocksdb_read_filter_block_nanos:
-      return rep->read_filter_block_nanos;
-    case rocksdb_new_table_block_iter_nanos:
-      return rep->new_table_block_iter_nanos;
-    case rocksdb_new_table_iterator_nanos:
-      return rep->new_table_iterator_nanos;
-    case rocksdb_block_seek_nanos:
-      return rep->block_seek_nanos;
-    case rocksdb_find_table_nanos:
-      return rep->find_table_nanos;
-    case rocksdb_bloom_memtable_hit_count:
-      return rep->bloom_memtable_hit_count;
-    case rocksdb_bloom_memtable_miss_count:
-      return rep->bloom_memtable_miss_count;
-    case rocksdb_bloom_sst_hit_count:
-      return rep->bloom_sst_hit_count;
-    case rocksdb_bloom_sst_miss_count:
-      return rep->bloom_sst_miss_count;
-    case rocksdb_key_lock_wait_time:
-      return rep->key_lock_wait_time;
-    case rocksdb_key_lock_wait_count:
-      return rep->key_lock_wait_count;
-    case rocksdb_env_new_sequential_file_nanos:
-      return rep->env_new_sequential_file_nanos;
-    case rocksdb_env_new_random_access_file_nanos:
-      return rep->env_new_random_access_file_nanos;
-    case rocksdb_env_new_writable_file_nanos:
-      return rep->env_new_writable_file_nanos;
-    case rocksdb_env_reuse_writable_file_nanos:
-      return rep->env_reuse_writable_file_nanos;
-    case rocksdb_env_new_random_rw_file_nanos:
-      return rep->env_new_random_rw_file_nanos;
-    case rocksdb_env_new_directory_nanos:
-      return rep->env_new_directory_nanos;
-    case rocksdb_env_file_exists_nanos:
-      return rep->env_file_exists_nanos;
-    case rocksdb_env_get_children_nanos:
-      return rep->env_get_children_nanos;
-    case rocksdb_env_get_children_file_attributes_nanos:
-      return rep->env_get_children_file_attributes_nanos;
-    case rocksdb_env_delete_file_nanos:
-      return rep->env_delete_file_nanos;
-    case rocksdb_env_create_dir_nanos:
-      return rep->env_create_dir_nanos;
-    case rocksdb_env_create_dir_if_missing_nanos:
-      return rep->env_create_dir_if_missing_nanos;
-    case rocksdb_env_delete_dir_nanos:
-      return rep->env_delete_dir_nanos;
-    case rocksdb_env_get_file_size_nanos:
-      return rep->env_get_file_size_nanos;
-    case rocksdb_env_get_file_modification_time_nanos:
-      return rep->env_get_file_modification_time_nanos;
-    case rocksdb_env_rename_file_nanos:
-      return rep->env_rename_file_nanos;
-    case rocksdb_env_link_file_nanos:
-      return rep->env_link_file_nanos;
-    case rocksdb_env_lock_file_nanos:
-      return rep->env_lock_file_nanos;
-    case rocksdb_env_unlock_file_nanos:
-      return rep->env_unlock_file_nanos;
-    case rocksdb_env_new_logger_nanos:
-      return rep->env_new_logger_nanos;
-    default:
-      break;
+  if (limiter->rep) {
+    delete limiter->rep;
   }
-  return 0;
-}
-
-void rocksdb_perfcontext_destroy(rocksdb_perfcontext_t* context) {
-  delete context;
+  delete limiter;
 }
 
 /*
@@ -2988,18 +2717,6 @@ void rocksdb_readoptions_set_iterate_upper_bound(
   }
 }
 
-void rocksdb_readoptions_set_iterate_lower_bound(
-    rocksdb_readoptions_t *opt,
-    const char* key, size_t keylen) {
-  if (key == nullptr) {
-    opt->lower_bound = Slice();
-    opt->rep.iterate_lower_bound = nullptr;
-  } else {
-    opt->lower_bound = Slice(key, keylen);
-    opt->rep.iterate_lower_bound = &opt->lower_bound;
-  }
-}
-
 void rocksdb_readoptions_set_read_tier(
     rocksdb_readoptions_t* opt, int v) {
   opt->rep.read_tier = static_cast<rocksdb::ReadTier>(v);
@@ -3010,21 +2727,11 @@ void rocksdb_readoptions_set_tailing(
   opt->rep.tailing = v;
 }
 
-void rocksdb_readoptions_set_managed(
-    rocksdb_readoptions_t* opt, unsigned char v) {
-  opt->rep.managed = v;
-}
-
 void rocksdb_readoptions_set_readahead_size(
     rocksdb_readoptions_t* opt, size_t v) {
   opt->rep.readahead_size = v;
 }
 
-void rocksdb_readoptions_set_prefix_same_as_start(
-    rocksdb_readoptions_t* opt, unsigned char v) {
-  opt->rep.prefix_same_as_start = v;
-}
-
 void rocksdb_readoptions_set_pin_data(rocksdb_readoptions_t* opt,
                                       unsigned char v) {
   opt->rep.pin_data = v;
@@ -3035,22 +2742,6 @@ void rocksdb_readoptions_set_total_order_seek(rocksdb_readoptions_t* opt,
   opt->rep.total_order_seek = v;
 }
 
-void rocksdb_readoptions_set_max_skippable_internal_keys(
-    rocksdb_readoptions_t* opt,
-    uint64_t v) {
-  opt->rep.max_skippable_internal_keys = v;
-}
-
-void rocksdb_readoptions_set_background_purge_on_iterator_cleanup(
-    rocksdb_readoptions_t* opt, unsigned char v) {
-  opt->rep.background_purge_on_iterator_cleanup = v;
-}
-
-void rocksdb_readoptions_set_ignore_range_deletions(
-    rocksdb_readoptions_t* opt, unsigned char v) {
-  opt->rep.ignore_range_deletions = v;
-}
-
 rocksdb_writeoptions_t* rocksdb_writeoptions_create() {
   return new rocksdb_writeoptions_t;
 }
@@ -3068,24 +2759,6 @@ void rocksdb_writeoptions_disable_WAL(rocksdb_writeoptions_t* opt, int disable)
   opt->rep.disableWAL = disable;
 }
 
-void rocksdb_writeoptions_set_ignore_missing_column_families(
-    rocksdb_writeoptions_t* opt,
-    unsigned char v) {
-  opt->rep.ignore_missing_column_families = v;
-}
-
-void rocksdb_writeoptions_set_no_slowdown(
-    rocksdb_writeoptions_t* opt,
-    unsigned char v) {
-  opt->rep.no_slowdown = v;
-}
-
-void rocksdb_writeoptions_set_low_pri(
-    rocksdb_writeoptions_t* opt,
-    unsigned char v) {
-  opt->rep.low_pri = v;
-}
-
 rocksdb_compactoptions_t* rocksdb_compactoptions_create() {
   return new rocksdb_compactoptions_t;
 }
@@ -3202,7 +2875,7 @@ rocksdb_sstfilewriter_t* rocksdb_sstfilewriter_create(
 
 rocksdb_sstfilewriter_t* rocksdb_sstfilewriter_create_with_comparator(
     const rocksdb_envoptions_t* env, const rocksdb_options_t* io_options,
-    const rocksdb_comparator_t* /*comparator*/) {
+    const rocksdb_comparator_t* comparator) {
   rocksdb_sstfilewriter_t* writer = new rocksdb_sstfilewriter_t;
   writer->rep = new SstFileWriter(env->rep, io_options->rep);
   return writer;
@@ -3240,7 +2913,7 @@ void rocksdb_sstfilewriter_delete(rocksdb_sstfilewriter_t* writer,
 
 void rocksdb_sstfilewriter_finish(rocksdb_sstfilewriter_t* writer,
                                   char** errptr) {
-  SaveError(errptr, writer->rep->Finish(nullptr));
+  SaveError(errptr, writer->rep->Finish(NULL));
 }
 
 void rocksdb_sstfilewriter_destroy(rocksdb_sstfilewriter_t* writer) {
@@ -3278,12 +2951,6 @@ void rocksdb_ingestexternalfileoptions_set_allow_blocking_flush(
   opt->rep.allow_blocking_flush = allow_blocking_flush;
 }
 
-void rocksdb_ingestexternalfileoptions_set_ingest_behind(
-    rocksdb_ingestexternalfileoptions_t* opt,
-    unsigned char ingest_behind) {
-  opt->rep.ingest_behind = ingest_behind;
-}
-
 void rocksdb_ingestexternalfileoptions_destroy(
     rocksdb_ingestexternalfileoptions_t* opt) {
   delete opt;
@@ -3338,21 +3005,20 @@ void rocksdb_slicetransform_destroy(rocksdb_slicetransform_t* st) {
   delete st;
 }
 
-struct Wrapper : public rocksdb_slicetransform_t {
-  const SliceTransform* rep_;
-  ~Wrapper() { delete rep_; }
-  const char* Name() const override { return rep_->Name(); }
-  Slice Transform(const Slice& src) const override {
-    return rep_->Transform(src);
-  }
-  bool InDomain(const Slice& src) const override {
-    return rep_->InDomain(src);
-  }
-  bool InRange(const Slice& src) const override { return rep_->InRange(src); }
-  static void DoNothing(void*) { }
-};
-
 rocksdb_slicetransform_t* rocksdb_slicetransform_create_fixed_prefix(size_t prefixLen) {
+  struct Wrapper : public rocksdb_slicetransform_t {
+    const SliceTransform* rep_;
+    ~Wrapper() { delete rep_; }
+    const char* Name() const override { return rep_->Name(); }
+    Slice Transform(const Slice& src) const override {
+      return rep_->Transform(src);
+    }
+    bool InDomain(const Slice& src) const override {
+      return rep_->InDomain(src);
+    }
+    bool InRange(const Slice& src) const override { return rep_->InRange(src); }
+    static void DoNothing(void*) { }
+  };
   Wrapper* wrapper = new Wrapper;
   wrapper->rep_ = rocksdb::NewFixedPrefixTransform(prefixLen);
   wrapper->state_ = nullptr;
@@ -3361,6 +3027,19 @@ rocksdb_slicetransform_t* rocksdb_slicetransform_create_fixed_prefix(size_t pref
 }
 
 rocksdb_slicetransform_t* rocksdb_slicetransform_create_noop() {
+  struct Wrapper : public rocksdb_slicetransform_t {
+    const SliceTransform* rep_;
+    ~Wrapper() { delete rep_; }
+    const char* Name() const override { return rep_->Name(); }
+    Slice Transform(const Slice& src) const override {
+      return rep_->Transform(src);
+    }
+    bool InDomain(const Slice& src) const override {
+      return rep_->InDomain(src);
+    }
+    bool InRange(const Slice& src) const override { return rep_->InRange(src); }
+    static void DoNothing(void*) { }
+  };
   Wrapper* wrapper = new Wrapper;
   wrapper->rep_ = rocksdb::NewNoopTransform();
   wrapper->state_ = nullptr;
@@ -3660,14 +3339,6 @@ void rocksdb_transaction_rollback(rocksdb_transaction_t* txn, char** errptr) {
   SaveError(errptr, txn->rep->Rollback());
 }
 
-void rocksdb_transaction_set_savepoint(rocksdb_transaction_t* txn) {
-  txn->rep->SetSavePoint();
-}
-
-void rocksdb_transaction_rollback_to_savepoint(rocksdb_transaction_t* txn, char** errptr) {
-  SaveError(errptr, txn->rep->RollbackToSavePoint());
-}
-
 void rocksdb_transaction_destroy(rocksdb_transaction_t* txn) {
   delete txn->rep;
   delete txn;
@@ -3805,8 +3476,8 @@ void rocksdb_transactiondb_put(rocksdb_transactiondb_t* txn_db,
                                const rocksdb_writeoptions_t* options,
                                const char* key, size_t klen, const char* val,
                                size_t vlen, char** errptr) {
-  SaveError(errptr,
-            txn_db->rep->Put(options->rep, Slice(key, klen), Slice(val, vlen)));
+  SaveError(errptr, txn_db->rep->Put(options->rep, Slice(key, klen), 
+                                     Slice(val, vlen)));
 }
 
 void rocksdb_transactiondb_put_cf(rocksdb_transactiondb_t* txn_db,
@@ -3840,8 +3511,8 @@ void rocksdb_transactiondb_merge(rocksdb_transactiondb_t* txn_db,
                                  const rocksdb_writeoptions_t* options,
                                  const char* key, size_t klen, const char* val,
                                  size_t vlen, char** errptr) {
-  SaveError(errptr, txn_db->rep->Merge(options->rep, Slice(key, klen),
-                                       Slice(val, vlen)));
+  SaveError(errptr,
+    txn_db->rep->Merge(options->rep, Slice(key, klen), Slice(val, vlen)));
 }
 
 // Delete a key inside a transaction
@@ -3879,15 +3550,6 @@ rocksdb_iterator_t* rocksdb_transaction_create_iterator(
   return result;
 }
 
-// Create an iterator inside a transaction with column family
-rocksdb_iterator_t* rocksdb_transaction_create_iterator_cf(
-    rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options,
-    rocksdb_column_family_handle_t* column_family) {
-  rocksdb_iterator_t* result = new rocksdb_iterator_t;
-  result->rep = txn->rep->GetIterator(options->rep, column_family->rep);
-  return result;
-}
-
 // Create an iterator outside a transaction
 rocksdb_iterator_t* rocksdb_transactiondb_create_iterator(
     rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options) {
@@ -3913,7 +3575,8 @@ rocksdb_checkpoint_t* rocksdb_transactiondb_checkpoint_object_create(
 }
 
 rocksdb_optimistictransactiondb_t* rocksdb_optimistictransactiondb_open(
-    const rocksdb_options_t* options, const char* name, char** errptr) {
+    const rocksdb_options_t* options, const char* name,
+    char** errptr) {
   OptimisticTransactionDB* otxn_db;
   if (SaveError(errptr, OptimisticTransactionDB::Open(
                             options->rep, std::string(name), &otxn_db))) {
@@ -3925,56 +3588,6 @@ rocksdb_optimistictransactiondb_t* rocksdb_optimistictransactiondb_open(
   return result;
 }
 
-rocksdb_optimistictransactiondb_t*
-rocksdb_optimistictransactiondb_open_column_families(
-    const rocksdb_options_t* db_options, const char* name,
-    int num_column_families, const char** column_family_names,
-    const rocksdb_options_t** column_family_options,
-    rocksdb_column_family_handle_t** column_family_handles, char** errptr) {
-  std::vector<ColumnFamilyDescriptor> column_families;
-  for (int i = 0; i < num_column_families; i++) {
-    column_families.push_back(ColumnFamilyDescriptor(
-        std::string(column_family_names[i]),
-        ColumnFamilyOptions(column_family_options[i]->rep)));
-  }
-
-  OptimisticTransactionDB* otxn_db;
-  std::vector<ColumnFamilyHandle*> handles;
-  if (SaveError(errptr, OptimisticTransactionDB::Open(
-                            DBOptions(db_options->rep), std::string(name),
-                            column_families, &handles, &otxn_db))) {
-    return nullptr;
-  }
-
-  for (size_t i = 0; i < handles.size(); i++) {
-    rocksdb_column_family_handle_t* c_handle =
-        new rocksdb_column_family_handle_t;
-    c_handle->rep = handles[i];
-    column_family_handles[i] = c_handle;
-  }
-  rocksdb_optimistictransactiondb_t* result =
-      new rocksdb_optimistictransactiondb_t;
-  result->rep = otxn_db;
-  return result;
-}
-
-rocksdb_t* rocksdb_optimistictransactiondb_get_base_db(
-    rocksdb_optimistictransactiondb_t* otxn_db) {
-  DB* base_db = otxn_db->rep->GetBaseDB();
-
-  if (base_db != nullptr) {
-    rocksdb_t* result = new rocksdb_t;
-    result->rep = base_db;
-    return result;
-  }
-
-  return nullptr;
-}
-
-void rocksdb_optimistictransactiondb_close_base_db(rocksdb_t* base_db) {
-  delete base_db;
-}
-
 rocksdb_transaction_t* rocksdb_optimistictransaction_begin(
     rocksdb_optimistictransactiondb_t* otxn_db,
     const rocksdb_writeoptions_t* write_options,
@@ -4010,7 +3623,7 @@ rocksdb_pinnableslice_t* rocksdb_get_pinned(
     if (!s.IsNotFound()) {
       SaveError(errptr, s);
     }
-    return nullptr;
+    return NULL;
   }
   return v;
 }
@@ -4027,7 +3640,7 @@ rocksdb_pinnableslice_t* rocksdb_get_pinned_cf(
     if (!s.IsNotFound()) {
       SaveError(errptr, s);
     }
-    return nullptr;
+    return NULL;
   }
   return v;
 }
@@ -4038,7 +3651,7 @@ const char* rocksdb_pinnableslice_value(const rocksdb_pinnableslice_t* v,
                                         size_t* vlen) {
   if (!v) {
     *vlen = 0;
-    return nullptr;
+    return NULL;
   }
 
   *vlen = v->rep.size();

db/c_test.c

@@ -192,11 +192,10 @@ static void CheckDel(void* ptr, const char* k, size_t klen) {
   (*state)++;
 }
 
-static void CmpDestroy(void* arg) { (void)arg; }
+static void CmpDestroy(void* arg) { }
 
 static int CmpCompare(void* arg, const char* a, size_t alen,
                       const char* b, size_t blen) {
-  (void)arg;
   size_t n = (alen < blen) ? alen : blen;
   int r = memcmp(a, b, n);
   if (r == 0) {
@@ -207,15 +206,13 @@ static int CmpCompare(void* arg, const char* a, size_t alen,
 }
 
 static const char* CmpName(void* arg) {
-  (void)arg;
   return "foo";
 }
 
 // Custom filter policy
 static unsigned char fake_filter_result = 1;
-static void FilterDestroy(void* arg) { (void)arg; }
+static void FilterDestroy(void* arg) { }
 static const char* FilterName(void* arg) {
-  (void)arg;
   return "TestFilter";
 }
 static char* FilterCreate(
@@ -223,10 +220,6 @@ static char* FilterCreate(
     const char* const* key_array, const size_t* key_length_array,
     int num_keys,
     size_t* filter_length) {
-  (void)arg;
-  (void)key_array;
-  (void)key_length_array;
-  (void)num_keys;
   *filter_length = 4;
   char* result = malloc(4);
   memcpy(result, "fake", 4);
@@ -236,30 +229,20 @@ static unsigned char FilterKeyMatch(
     void* arg,
     const char* key, size_t length,
     const char* filter, size_t filter_length) {
-  (void)arg;
-  (void)key;
-  (void)length;
   CheckCondition(filter_length == 4);
   CheckCondition(memcmp(filter, "fake", 4) == 0);
   return fake_filter_result;
 }
 
 // Custom compaction filter
-static void CFilterDestroy(void* arg) { (void)arg; }
-static const char* CFilterName(void* arg) {
-  (void)arg;
-  return "foo";
-}
+static void CFilterDestroy(void* arg) {}
+static const char* CFilterName(void* arg) { return "foo"; }
 static unsigned char CFilterFilter(void* arg, int level, const char* key,
                                    size_t key_length,
                                    const char* existing_value,
                                    size_t value_length, char** new_value,
                                    size_t* new_value_length,
                                    unsigned char* value_changed) {
-  (void)arg;
-  (void)level;
-  (void)existing_value;
-  (void)value_length;
   if (key_length == 3) {
     if (memcmp(key, "bar", key_length) == 0) {
       return 1;
@@ -273,15 +256,10 @@ static unsigned char CFilterFilter(void* arg, int level, const char* key,
   return 0;
 }
 
-static void CFilterFactoryDestroy(void* arg) { (void)arg; }
-static const char* CFilterFactoryName(void* arg) {
-  (void)arg;
-  return "foo";
-}
+static void CFilterFactoryDestroy(void* arg) {}
+static const char* CFilterFactoryName(void* arg) { return "foo"; }
 static rocksdb_compactionfilter_t* CFilterCreate(
     void* arg, rocksdb_compactionfiltercontext_t* context) {
-  (void)arg;
-  (void)context;
   return rocksdb_compactionfilter_create(NULL, CFilterDestroy, CFilterFilter,
                                          CFilterName);
 }
@@ -312,9 +290,8 @@ static rocksdb_t* CheckCompaction(rocksdb_t* db, rocksdb_options_t* options,
 }
 
 // Custom merge operator
-static void MergeOperatorDestroy(void* arg) { (void)arg; }
+static void MergeOperatorDestroy(void* arg) { }
 static const char* MergeOperatorName(void* arg) {
-  (void)arg;
   return "TestMergeOperator";
 }
 static char* MergeOperatorFullMerge(
@@ -324,14 +301,6 @@ static char* MergeOperatorFullMerge(
     const char* const* operands_list, const size_t* operands_list_length,
     int num_operands,
     unsigned char* success, size_t* new_value_length) {
-  (void)arg;
-  (void)key;
-  (void)key_length;
-  (void)existing_value;
-  (void)existing_value_length;
-  (void)operands_list;
-  (void)operands_list_length;
-  (void)num_operands;
   *new_value_length = 4;
   *success = 1;
   char* result = malloc(4);
@@ -344,12 +313,6 @@ static char* MergeOperatorPartialMerge(
     const char* const* operands_list, const size_t* operands_list_length,
     int num_operands,
     unsigned char* success, size_t* new_value_length) {
-  (void)arg;
-  (void)key;
-  (void)key_length;
-  (void)operands_list;
-  (void)operands_list_length;
-  (void)num_operands;
   *new_value_length = 4;
   *success = 1;
   char* result = malloc(4);
@@ -371,20 +334,6 @@ static void CheckTxnGet(
         Free(&val);
 }
 
-static void CheckTxnGetCF(rocksdb_transaction_t* txn,
-                          const rocksdb_readoptions_t* options,
-                          rocksdb_column_family_handle_t* column_family,
-                          const char* key, const char* expected) {
-  char* err = NULL;
-  size_t val_len;
-  char* val;
-  val = rocksdb_transaction_get_cf(txn, options, column_family, key,
-                                   strlen(key), &val_len, &err);
-  CheckNoError(err);
-  CheckEqual(expected, val, val_len);
-  Free(&val);
-}
-
 static void CheckTxnDBGet(
         rocksdb_transactiondb_t* txn_db,
         const rocksdb_readoptions_t* options,
@@ -414,8 +363,6 @@ static void CheckTxnDBGetCF(rocksdb_transactiondb_t* txn_db,
 }
 
 int main(int argc, char** argv) {
-  (void)argc;
-  (void)argv;
   rocksdb_t* db;
   rocksdb_comparator_t* cmp;
   rocksdb_cache_t* cache;
@@ -431,8 +378,6 @@ int main(int argc, char** argv) {
   rocksdb_transactiondb_options_t* txn_db_options;
   rocksdb_transaction_t* txn;
   rocksdb_transaction_options_t* txn_options;
-  rocksdb_optimistictransactiondb_t* otxn_db;
-  rocksdb_optimistictransaction_options_t* otxn_options;
   char* err = NULL;
   int run = -1;
 
@@ -1502,25 +1447,6 @@ int main(int argc, char** argv) {
     CheckNoError(err);
     CheckTxnDBGet(txn_db, roptions, "bar", NULL);
 
-    // save point
-    rocksdb_transaction_put(txn, "foo1", 4, "hi1", 3, &err);
-    rocksdb_transaction_set_savepoint(txn);
-    CheckTxnGet(txn, roptions, "foo1", "hi1");
-    rocksdb_transaction_put(txn, "foo2", 4, "hi2", 3, &err);
-    CheckTxnGet(txn, roptions, "foo2", "hi2");
-
-    // rollback to savepoint
-    rocksdb_transaction_rollback_to_savepoint(txn, &err);
-    CheckNoError(err);
-    CheckTxnGet(txn, roptions, "foo2", NULL);
-    CheckTxnGet(txn, roptions, "foo1", "hi1");
-    CheckTxnDBGet(txn_db, roptions, "foo1", NULL);
-    CheckTxnDBGet(txn_db, roptions, "foo2", NULL);
-    rocksdb_transaction_commit(txn, &err);
-    CheckNoError(err);
-    CheckTxnDBGet(txn_db, roptions, "foo1", "hi1");
-    CheckTxnDBGet(txn_db, roptions, "foo2", NULL);
-
     // Column families.
     rocksdb_column_family_handle_t* cfh;
     cfh = rocksdb_transactiondb_create_column_family(txn_db, options,
@@ -1547,105 +1473,6 @@ int main(int argc, char** argv) {
     rocksdb_transactiondb_options_destroy(txn_db_options);
   }
 
-  StartPhase("optimistic_transactions");
-  {
-    rocksdb_options_t* db_options = rocksdb_options_create();
-    rocksdb_options_set_create_if_missing(db_options, 1);
-    rocksdb_options_set_allow_concurrent_memtable_write(db_options, 1);
-    otxn_db = rocksdb_optimistictransactiondb_open(db_options, dbname, &err);
-    otxn_options = rocksdb_optimistictransaction_options_create();
-    rocksdb_transaction_t* txn1 = rocksdb_optimistictransaction_begin(
-        otxn_db, woptions, otxn_options, NULL);
-    rocksdb_transaction_t* txn2 = rocksdb_optimistictransaction_begin(
-        otxn_db, woptions, otxn_options, NULL);
-    rocksdb_transaction_put(txn1, "key", 3, "value", 5, &err);
-    CheckNoError(err);
-    rocksdb_transaction_put(txn2, "key1", 4, "value1", 6, &err);
-    CheckNoError(err);
-    CheckTxnGet(txn1, roptions, "key", "value");
-    rocksdb_transaction_commit(txn1, &err);
-    CheckNoError(err);
-    rocksdb_transaction_commit(txn2, &err);
-    CheckNoError(err);
-    rocksdb_transaction_destroy(txn1);
-    rocksdb_transaction_destroy(txn2);
-
-    // Check column family
-    db = rocksdb_optimistictransactiondb_get_base_db(otxn_db);
-    rocksdb_put(db, woptions, "key", 3, "value", 5, &err);
-    CheckNoError(err);
-    rocksdb_column_family_handle_t *cfh1, *cfh2;
-    cfh1 = rocksdb_create_column_family(db, db_options, "txn_db_cf1", &err);
-    cfh2 = rocksdb_create_column_family(db, db_options, "txn_db_cf2", &err);
-    txn = rocksdb_optimistictransaction_begin(otxn_db, woptions, otxn_options,
-                                              NULL);
-    rocksdb_transaction_put_cf(txn, cfh1, "key_cf1", 7, "val_cf1", 7, &err);
-    CheckNoError(err);
-    rocksdb_transaction_put_cf(txn, cfh2, "key_cf2", 7, "val_cf2", 7, &err);
-    CheckNoError(err);
-    rocksdb_transaction_commit(txn, &err);
-    CheckNoError(err);
-    txn = rocksdb_optimistictransaction_begin(otxn_db, woptions, otxn_options,
-                                              txn);
-    CheckGetCF(db, roptions, cfh1, "key_cf1", "val_cf1");
-    CheckTxnGetCF(txn, roptions, cfh1, "key_cf1", "val_cf1");
-
-    // Check iterator with column family
-    rocksdb_transaction_put_cf(txn, cfh1, "key1_cf", 7, "val1_cf", 7, &err);
-    CheckNoError(err);
-    rocksdb_iterator_t* iter =
-        rocksdb_transaction_create_iterator_cf(txn, roptions, cfh1);
-    CheckCondition(!rocksdb_iter_valid(iter));
-    rocksdb_iter_seek_to_first(iter);
-    CheckCondition(rocksdb_iter_valid(iter));
-    CheckIter(iter, "key1_cf", "val1_cf");
-    rocksdb_iter_get_error(iter, &err);
-    CheckNoError(err);
-    rocksdb_iter_destroy(iter);
-
-    rocksdb_transaction_destroy(txn);
-    rocksdb_column_family_handle_destroy(cfh1);
-    rocksdb_column_family_handle_destroy(cfh2);
-    rocksdb_optimistictransactiondb_close_base_db(db);
-    rocksdb_optimistictransactiondb_close(otxn_db);
-
-    // Check open optimistic transaction db with column families
-    size_t cf_len;
-    char** column_fams =
-        rocksdb_list_column_families(db_options, dbname, &cf_len, &err);
-    CheckNoError(err);
-    CheckEqual("default", column_fams[0], 7);
-    CheckEqual("txn_db_cf1", column_fams[1], 10);
-    CheckEqual("txn_db_cf2", column_fams[2], 10);
-    CheckCondition(cf_len == 3);
-    rocksdb_list_column_families_destroy(column_fams, cf_len);
-
-    const char* cf_names[3] = {"default", "txn_db_cf1", "txn_db_cf2"};
-    rocksdb_options_t* cf_options = rocksdb_options_create();
-    const rocksdb_options_t* cf_opts[3] = {cf_options, cf_options, cf_options};
-
-    rocksdb_options_set_error_if_exists(cf_options, 0);
-    rocksdb_column_family_handle_t* cf_handles[3];
-    otxn_db = rocksdb_optimistictransactiondb_open_column_families(
-        db_options, dbname, 3, cf_names, cf_opts, cf_handles, &err);
-    CheckNoError(err);
-    rocksdb_transaction_t* txn_cf = rocksdb_optimistictransaction_begin(
-        otxn_db, woptions, otxn_options, NULL);
-    CheckTxnGetCF(txn_cf, roptions, cf_handles[0], "key", "value");
-    CheckTxnGetCF(txn_cf, roptions, cf_handles[1], "key_cf1", "val_cf1");
-    CheckTxnGetCF(txn_cf, roptions, cf_handles[2], "key_cf2", "val_cf2");
-    rocksdb_transaction_destroy(txn_cf);
-    rocksdb_options_destroy(cf_options);
-    rocksdb_column_family_handle_destroy(cf_handles[0]);
-    rocksdb_column_family_handle_destroy(cf_handles[1]);
-    rocksdb_column_family_handle_destroy(cf_handles[2]);
-    rocksdb_optimistictransactiondb_close(otxn_db);
-    rocksdb_destroy_db(db_options, dbname, &err);
-    rocksdb_options_destroy(db_options);
-    rocksdb_optimistictransaction_options_destroy(otxn_options);
-    CheckNoError(err);
-  }
-
   // Simple sanity check that setting memtable rep works.
   StartPhase("memtable_reps");
   {

db/column_family.cc

@@ -31,10 +31,8 @@
 #include "monitoring/thread_status_util.h"
 #include "options/options_helper.h"
 #include "table/block_based_table_factory.h"
-#include "table/merging_iterator.h"
 #include "util/autovector.h"
 #include "util/compression.h"
-#include "util/sst_file_manager_impl.h"
 
 namespace rocksdb {
 
@@ -55,9 +53,6 @@ ColumnFamilyHandleImpl::~ColumnFamilyHandleImpl() {
 #endif  // ROCKSDB_LITE
     // Job id == 0 means that this is not our background process, but rather
     // user thread
-    // Need to hold some shared pointers owned by the initial_cf_options
-    // before final cleaning up finishes.
-    ColumnFamilyOptions initial_cf_options_copy = cfd_->initial_cf_options();
     JobContext job_context(0);
     mutex_->Lock();
     if (cfd_->Unref()) {
@@ -85,7 +80,6 @@ Status ColumnFamilyHandleImpl::GetDescriptor(ColumnFamilyDescriptor* desc) {
   *desc = ColumnFamilyDescriptor(cfd()->GetName(), cfd()->GetLatestCFOptions());
   return Status::OK();
 #else
-  (void)desc;
   return Status::NotSupported();
 #endif  // !ROCKSDB_LITE
 }
@@ -129,22 +123,6 @@ Status CheckCompressionSupported(const ColumnFamilyOptions& cf_options) {
           " is not linked with the binary.");
     }
   }
-  if (cf_options.compression_opts.zstd_max_train_bytes > 0) {
-    if (!CompressionTypeSupported(CompressionType::kZSTD)) {
-      // Dictionary trainer is available since v0.6.1, but ZSTD was marked
-      // stable only since v0.8.0. For now we enable the feature in stable
-      // versions only.
-      return Status::InvalidArgument(
-          "zstd dictionary trainer cannot be used because " +
-          CompressionTypeToString(CompressionType::kZSTD) +
-          " is not linked with the binary.");
-    }
-    if (cf_options.compression_opts.max_dict_bytes == 0) {
-      return Status::InvalidArgument(
-          "The dictionary size limit (`CompressionOptions::max_dict_bytes`) "
-          "should be nonzero if we're using zstd's dictionary generator.");
-    }
-  }
   return Status::OK();
 }
 
@@ -161,28 +139,6 @@ Status CheckConcurrentWritesSupported(const ColumnFamilyOptions& cf_options) {
   return Status::OK();
 }
 
-Status CheckCFPathsSupported(const DBOptions& db_options,
-                             const ColumnFamilyOptions& cf_options) {
-  // More than one cf_paths are supported only in universal
-  // and level compaction styles. This function also checks the case
-  // in which cf_paths is not specified, which results in db_paths
-  // being used.
-  if ((cf_options.compaction_style != kCompactionStyleUniversal) &&
-      (cf_options.compaction_style != kCompactionStyleLevel)) {
-    if (cf_options.cf_paths.size() > 1) {
-      return Status::NotSupported(
-          "More than one CF paths are only supported in "
-          "universal and level compaction styles. ");
-    } else if (cf_options.cf_paths.empty() &&
-               db_options.db_paths.size() > 1) {
-      return Status::NotSupported(
-          "More than one DB paths are only supported in "
-          "universal and level compaction styles. ");
-    }
-  }
-  return Status::OK();
-}
-
 ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
                                     const ColumnFamilyOptions& src) {
   ColumnFamilyOptions result = src;
@@ -301,24 +257,9 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
         result.hard_pending_compaction_bytes_limit;
   }
 
-#ifndef ROCKSDB_LITE
-  // When the DB is stopped, it's possible that there are some .trash files that
-  // were not deleted yet, when we open the DB we will find these .trash files
-  // and schedule them to be deleted (or delete immediately if SstFileManager
-  // was not used)
-  auto sfm = static_cast<SstFileManagerImpl*>(db_options.sst_file_manager.get());
-  for (size_t i = 0; i < result.cf_paths.size(); i++) {
-    DeleteScheduler::CleanupDirectory(db_options.env, sfm, result.cf_paths[i].path);
-  }
-#endif
-
-  if (result.cf_paths.empty()) {
-    result.cf_paths = db_options.db_paths;
-  }
-
   if (result.level_compaction_dynamic_level_bytes) {
     if (result.compaction_style != kCompactionStyleLevel ||
-        result.cf_paths.size() > 1U) {
+        db_options.db_paths.size() > 1U) {
       // 1. level_compaction_dynamic_level_bytes only makes sense for
       //    level-based compaction.
       // 2. we don't yet know how to make both of this feature and multiple
@@ -387,13 +328,12 @@ void SuperVersionUnrefHandle(void* ptr) {
   // When latter happens, we are in ~ColumnFamilyData(), no get should happen as
   // well.
   SuperVersion* sv = static_cast<SuperVersion*>(ptr);
-  bool was_last_ref __attribute__((__unused__));
-  was_last_ref = sv->Unref();
-  // Thread-local SuperVersions can't outlive ColumnFamilyData::super_version_.
-  // This is important because we can't do SuperVersion cleanup here.
-  // That would require locking DB mutex, which would deadlock because
-  // SuperVersionUnrefHandle is called with locked ThreadLocalPtr mutex.
-  assert(!was_last_ref);
+  if (sv->Unref()) {
+    sv->db_mutex->Lock();
+    sv->Cleanup();
+    sv->db_mutex->Unlock();
+    delete sv;
+  }
 }
 }  // anonymous namespace
 
@@ -425,13 +365,11 @@ ColumnFamilyData::ColumnFamilyData(
       next_(nullptr),
       prev_(nullptr),
       log_number_(0),
-      flush_reason_(FlushReason::kOthers),
       column_family_set_(column_family_set),
-      queued_for_flush_(false),
-      queued_for_compaction_(false),
+      pending_flush_(false),
+      pending_compaction_(false),
       prev_compaction_needed_bytes_(0),
-      allow_2pc_(db_options.allow_2pc),
-      last_memtable_id_(0) {
+      allow_2pc_(db_options.allow_2pc) {
   Ref();
 
   // Convert user defined table properties collector factories to internal ones.
@@ -504,8 +442,8 @@ ColumnFamilyData::~ColumnFamilyData() {
 
   // It would be wrong if this ColumnFamilyData is in flush_queue_ or
   // compaction_queue_ and we destroyed it
-  assert(!queued_for_flush_);
-  assert(!queued_for_compaction_);
+  assert(!pending_flush_);
+  assert(!pending_compaction_);
 
   if (super_version_ != nullptr) {
     // Release SuperVersion reference kept in ThreadLocalPtr.
@@ -514,7 +452,7 @@ ColumnFamilyData::~ColumnFamilyData() {
     local_sv_.reset();
     super_version_->db_mutex->Lock();
 
-    bool is_last_reference __attribute__((__unused__));
+    bool is_last_reference __attribute__((unused));
     is_last_reference = super_version_->Unref();
     assert(is_last_reference);
     super_version_->Cleanup();
@@ -525,8 +463,7 @@ ColumnFamilyData::~ColumnFamilyData() {
   if (dummy_versions_ != nullptr) {
     // List must be empty
     assert(dummy_versions_->TEST_Next() == dummy_versions_);
-    bool deleted __attribute__((__unused__));
-    deleted = dummy_versions_->Unref();
+    bool deleted __attribute__((unused)) = dummy_versions_->Unref();
     assert(deleted);
   }
 
@@ -558,9 +495,7 @@ uint64_t ColumnFamilyData::OldestLogToKeep() {
   auto current_log = GetLogNumber();
 
   if (allow_2pc_) {
-    autovector<MemTable*> empty_list;
-    auto imm_prep_log =
-        imm()->PrecomputeMinLogContainingPrepSection(empty_list);
+    auto imm_prep_log = imm()->GetMinLogContainingPrepSection();
     auto mem_prep_log = mem()->GetMinLogContainingPrepSection();
 
     if (imm_prep_log > 0 && imm_prep_log < current_log) {
@@ -678,97 +613,58 @@ int GetL0ThresholdSpeedupCompaction(int level0_file_num_compaction_trigger,
 }
 }  // namespace
 
-std::pair<WriteStallCondition, ColumnFamilyData::WriteStallCause>
-ColumnFamilyData::GetWriteStallConditionAndCause(
-    int num_unflushed_memtables, int num_l0_files,
-    uint64_t num_compaction_needed_bytes,
-    const MutableCFOptions& mutable_cf_options) {
-  if (num_unflushed_memtables >= mutable_cf_options.max_write_buffer_number) {
-    return {WriteStallCondition::kStopped, WriteStallCause::kMemtableLimit};
-  } else if (!mutable_cf_options.disable_auto_compactions &&
-             num_l0_files >= mutable_cf_options.level0_stop_writes_trigger) {
-    return {WriteStallCondition::kStopped, WriteStallCause::kL0FileCountLimit};
-  } else if (!mutable_cf_options.disable_auto_compactions &&
-             mutable_cf_options.hard_pending_compaction_bytes_limit > 0 &&
-             num_compaction_needed_bytes >=
-                 mutable_cf_options.hard_pending_compaction_bytes_limit) {
-    return {WriteStallCondition::kStopped,
-            WriteStallCause::kPendingCompactionBytes};
-  } else if (mutable_cf_options.max_write_buffer_number > 3 &&
-             num_unflushed_memtables >=
-                 mutable_cf_options.max_write_buffer_number - 1) {
-    return {WriteStallCondition::kDelayed, WriteStallCause::kMemtableLimit};
-  } else if (!mutable_cf_options.disable_auto_compactions &&
-             mutable_cf_options.level0_slowdown_writes_trigger >= 0 &&
-             num_l0_files >=
-                 mutable_cf_options.level0_slowdown_writes_trigger) {
-    return {WriteStallCondition::kDelayed, WriteStallCause::kL0FileCountLimit};
-  } else if (!mutable_cf_options.disable_auto_compactions &&
-             mutable_cf_options.soft_pending_compaction_bytes_limit > 0 &&
-             num_compaction_needed_bytes >=
-                 mutable_cf_options.soft_pending_compaction_bytes_limit) {
-    return {WriteStallCondition::kDelayed,
-            WriteStallCause::kPendingCompactionBytes};
-  }
-  return {WriteStallCondition::kNormal, WriteStallCause::kNone};
-}
-
-WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
+void ColumnFamilyData::RecalculateWriteStallConditions(
       const MutableCFOptions& mutable_cf_options) {
-  auto write_stall_condition = WriteStallCondition::kNormal;
   if (current_ != nullptr) {
     auto* vstorage = current_->storage_info();
     auto write_controller = column_family_set_->write_controller_;
     uint64_t compaction_needed_bytes =
         vstorage->estimated_compaction_needed_bytes();
 
-    auto write_stall_condition_and_cause = GetWriteStallConditionAndCause(
-        imm()->NumNotFlushed(), vstorage->l0_delay_trigger_count(),
-        vstorage->estimated_compaction_needed_bytes(), mutable_cf_options);
-    write_stall_condition = write_stall_condition_and_cause.first;
-    auto write_stall_cause = write_stall_condition_and_cause.second;
-
     bool was_stopped = write_controller->IsStopped();
     bool needed_delay = write_controller->NeedsDelay();
 
-    if (write_stall_condition == WriteStallCondition::kStopped &&
-        write_stall_cause == WriteStallCause::kMemtableLimit) {
+    if (imm()->NumNotFlushed() >= mutable_cf_options.max_write_buffer_number) {
       write_controller_token_ = write_controller->GetStopToken();
-      internal_stats_->AddCFStats(InternalStats::MEMTABLE_LIMIT_STOPS, 1);
+      internal_stats_->AddCFStats(InternalStats::MEMTABLE_COMPACTION, 1);
       ROCKS_LOG_WARN(
           ioptions_.info_log,
           "[%s] Stopping writes because we have %d immutable memtables "
           "(waiting for flush), max_write_buffer_number is set to %d",
           name_.c_str(), imm()->NumNotFlushed(),
           mutable_cf_options.max_write_buffer_number);
-    } else if (write_stall_condition == WriteStallCondition::kStopped &&
-               write_stall_cause == WriteStallCause::kL0FileCountLimit) {
+    } else if (!mutable_cf_options.disable_auto_compactions &&
+               vstorage->l0_delay_trigger_count() >=
+                   mutable_cf_options.level0_stop_writes_trigger) {
       write_controller_token_ = write_controller->GetStopToken();
-      internal_stats_->AddCFStats(InternalStats::L0_FILE_COUNT_LIMIT_STOPS, 1);
+      internal_stats_->AddCFStats(InternalStats::LEVEL0_NUM_FILES_TOTAL, 1);
       if (compaction_picker_->IsLevel0CompactionInProgress()) {
         internal_stats_->AddCFStats(
-            InternalStats::LOCKED_L0_FILE_COUNT_LIMIT_STOPS, 1);
+            InternalStats::LEVEL0_NUM_FILES_WITH_COMPACTION, 1);
       }
       ROCKS_LOG_WARN(ioptions_.info_log,
                      "[%s] Stopping writes because we have %d level-0 files",
                      name_.c_str(), vstorage->l0_delay_trigger_count());
-    } else if (write_stall_condition == WriteStallCondition::kStopped &&
-               write_stall_cause == WriteStallCause::kPendingCompactionBytes) {
+    } else if (!mutable_cf_options.disable_auto_compactions &&
+               mutable_cf_options.hard_pending_compaction_bytes_limit > 0 &&
+               compaction_needed_bytes >=
+                   mutable_cf_options.hard_pending_compaction_bytes_limit) {
       write_controller_token_ = write_controller->GetStopToken();
       internal_stats_->AddCFStats(
-          InternalStats::PENDING_COMPACTION_BYTES_LIMIT_STOPS, 1);
+          InternalStats::HARD_PENDING_COMPACTION_BYTES_LIMIT, 1);
       ROCKS_LOG_WARN(
           ioptions_.info_log,
           "[%s] Stopping writes because of estimated pending compaction "
           "bytes %" PRIu64,
           name_.c_str(), compaction_needed_bytes);
-    } else if (write_stall_condition == WriteStallCondition::kDelayed &&
-               write_stall_cause == WriteStallCause::kMemtableLimit) {
+    } else if (mutable_cf_options.max_write_buffer_number > 3 &&
+               imm()->NumNotFlushed() >=
+                   mutable_cf_options.max_write_buffer_number - 1) {
       write_controller_token_ =
           SetupDelay(write_controller, compaction_needed_bytes,
                      prev_compaction_needed_bytes_, was_stopped,
                      mutable_cf_options.disable_auto_compactions);
-      internal_stats_->AddCFStats(InternalStats::MEMTABLE_LIMIT_SLOWDOWNS, 1);
+      internal_stats_->AddCFStats(InternalStats::MEMTABLE_SLOWDOWN, 1);
       ROCKS_LOG_WARN(
           ioptions_.info_log,
           "[%s] Stalling writes because we have %d immutable memtables "
@@ -777,8 +673,10 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
           name_.c_str(), imm()->NumNotFlushed(),
           mutable_cf_options.max_write_buffer_number,
           write_controller->delayed_write_rate());
-    } else if (write_stall_condition == WriteStallCondition::kDelayed &&
-               write_stall_cause == WriteStallCause::kL0FileCountLimit) {
+    } else if (!mutable_cf_options.disable_auto_compactions &&
+               mutable_cf_options.level0_slowdown_writes_trigger >= 0 &&
+               vstorage->l0_delay_trigger_count() >=
+                   mutable_cf_options.level0_slowdown_writes_trigger) {
       // L0 is the last two files from stopping.
       bool near_stop = vstorage->l0_delay_trigger_count() >=
                        mutable_cf_options.level0_stop_writes_trigger - 2;
@@ -786,19 +684,20 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
           SetupDelay(write_controller, compaction_needed_bytes,
                      prev_compaction_needed_bytes_, was_stopped || near_stop,
                      mutable_cf_options.disable_auto_compactions);
-      internal_stats_->AddCFStats(InternalStats::L0_FILE_COUNT_LIMIT_SLOWDOWNS,
-                                  1);
+      internal_stats_->AddCFStats(InternalStats::LEVEL0_SLOWDOWN_TOTAL, 1);
       if (compaction_picker_->IsLevel0CompactionInProgress()) {
         internal_stats_->AddCFStats(
-            InternalStats::LOCKED_L0_FILE_COUNT_LIMIT_SLOWDOWNS, 1);
+            InternalStats::LEVEL0_SLOWDOWN_WITH_COMPACTION, 1);
       }
       ROCKS_LOG_WARN(ioptions_.info_log,
                      "[%s] Stalling writes because we have %d level-0 files "
                      "rate %" PRIu64,
                      name_.c_str(), vstorage->l0_delay_trigger_count(),
                      write_controller->delayed_write_rate());
-    } else if (write_stall_condition == WriteStallCondition::kDelayed &&
-               write_stall_cause == WriteStallCause::kPendingCompactionBytes) {
+    } else if (!mutable_cf_options.disable_auto_compactions &&
+               mutable_cf_options.soft_pending_compaction_bytes_limit > 0 &&
+               vstorage->estimated_compaction_needed_bytes() >=
+                   mutable_cf_options.soft_pending_compaction_bytes_limit) {
       // If the distance to hard limit is less than 1/4 of the gap between soft
       // and
       // hard bytes limit, we think it is near stop and speed up the slowdown.
@@ -815,7 +714,7 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
                      prev_compaction_needed_bytes_, was_stopped || near_stop,
                      mutable_cf_options.disable_auto_compactions);
       internal_stats_->AddCFStats(
-          InternalStats::PENDING_COMPACTION_BYTES_LIMIT_SLOWDOWNS, 1);
+          InternalStats::SOFT_PENDING_COMPACTION_BYTES_LIMIT, 1);
       ROCKS_LOG_WARN(
           ioptions_.info_log,
           "[%s] Stalling writes because of estimated pending compaction "
@@ -823,7 +722,6 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
           name_.c_str(), vstorage->estimated_compaction_needed_bytes(),
           write_controller->delayed_write_rate());
     } else {
-      assert(write_stall_condition == WriteStallCondition::kNormal);
       if (vstorage->l0_delay_trigger_count() >=
           GetL0ThresholdSpeedupCompaction(
               mutable_cf_options.level0_file_num_compaction_trigger,
@@ -871,7 +769,6 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
     }
     prev_compaction_needed_bytes_ = compaction_needed_bytes;
   }
-  return write_stall_condition;
 }
 
 const EnvOptions* ColumnFamilyData::soptions() const {
@@ -890,10 +787,6 @@ uint64_t ColumnFamilyData::GetTotalSstFilesSize() const {
   return VersionSet::GetTotalSstFilesSize(dummy_versions_);
 }
 
-uint64_t ColumnFamilyData::GetLiveSstFilesSize() const {
-  return current_->GetSstFilesSize();
-}
-
 MemTable* ColumnFamilyData::ConstructNewMemtable(
     const MutableCFOptions& mutable_cf_options, SequenceNumber earliest_seq) {
   return new MemTable(internal_comparator_, ioptions_, mutable_cf_options,
@@ -930,80 +823,16 @@ bool ColumnFamilyData::RangeOverlapWithCompaction(
       smallest_user_key, largest_user_key, level);
 }
 
-Status ColumnFamilyData::RangesOverlapWithMemtables(
-    const autovector<Range>& ranges, SuperVersion* super_version,
-    bool* overlap) {
-  assert(overlap != nullptr);
-  *overlap = false;
-  // Create an InternalIterator over all unflushed memtables
-  Arena arena;
-  ReadOptions read_opts;
-  read_opts.total_order_seek = true;
-  MergeIteratorBuilder merge_iter_builder(&internal_comparator_, &arena);
-  merge_iter_builder.AddIterator(
-      super_version->mem->NewIterator(read_opts, &arena));
-  super_version->imm->AddIterators(read_opts, &merge_iter_builder);
-  ScopedArenaIterator memtable_iter(merge_iter_builder.Finish());
-
-  std::vector<InternalIterator*> memtable_range_del_iters;
-  auto* active_range_del_iter =
-      super_version->mem->NewRangeTombstoneIterator(read_opts);
-  if (active_range_del_iter != nullptr) {
-    memtable_range_del_iters.push_back(active_range_del_iter);
-  }
-  super_version->imm->AddRangeTombstoneIterators(read_opts,
-                                                 &memtable_range_del_iters);
-  RangeDelAggregator range_del_agg(internal_comparator_, {} /* snapshots */,
-                                   false /* collapse_deletions */);
-  Status status;
-  {
-    std::unique_ptr<InternalIterator> memtable_range_del_iter(
-        NewMergingIterator(&internal_comparator_,
-                           memtable_range_del_iters.empty()
-                               ? nullptr
-                               : &memtable_range_del_iters[0],
-                           static_cast<int>(memtable_range_del_iters.size())));
-    status = range_del_agg.AddTombstones(std::move(memtable_range_del_iter));
-  }
-  for (size_t i = 0; i < ranges.size() && status.ok() && !*overlap; ++i) {
-    auto* vstorage = super_version->current->storage_info();
-    auto* ucmp = vstorage->InternalComparator()->user_comparator();
-    InternalKey range_start(ranges[i].start, kMaxSequenceNumber,
-                            kValueTypeForSeek);
-    memtable_iter->Seek(range_start.Encode());
-    status = memtable_iter->status();
-    ParsedInternalKey seek_result;
-    if (status.ok()) {
-      if (memtable_iter->Valid() &&
-          !ParseInternalKey(memtable_iter->key(), &seek_result)) {
-        status = Status::Corruption("DB have corrupted keys");
-      }
-    }
-    if (status.ok()) {
-      if (memtable_iter->Valid() &&
-          ucmp->Compare(seek_result.user_key, ranges[i].limit) <= 0) {
-        *overlap = true;
-      } else if (range_del_agg.IsRangeOverlapped(ranges[i].start,
-                                                 ranges[i].limit)) {
-        *overlap = true;
-      }
-    }
-  }
-  return status;
-}
-
 const int ColumnFamilyData::kCompactAllLevels = -1;
 const int ColumnFamilyData::kCompactToBaseLevel = -2;
 
 Compaction* ColumnFamilyData::CompactRange(
     const MutableCFOptions& mutable_cf_options, int input_level,
-    int output_level, uint32_t output_path_id, uint32_t max_subcompactions,
-    const InternalKey* begin, const InternalKey* end,
-    InternalKey** compaction_end, bool* conflict) {
+    int output_level, uint32_t output_path_id, const InternalKey* begin,
+    const InternalKey* end, InternalKey** compaction_end, bool* conflict) {
   auto* result = compaction_picker_->CompactRange(
       GetName(), mutable_cf_options, current_->storage_info(), input_level,
-      output_level, output_path_id, max_subcompactions, begin, end,
-      compaction_end, conflict);
+      output_level, output_path_id, begin, end, compaction_end, conflict);
   if (result != nullptr) {
     result->SetInputVersion(current_);
   }
@@ -1012,13 +841,10 @@ Compaction* ColumnFamilyData::CompactRange(
 
 SuperVersion* ColumnFamilyData::GetReferencedSuperVersion(
     InstrumentedMutex* db_mutex) {
-  SuperVersion* sv = GetThreadLocalSuperVersion(db_mutex);
+  SuperVersion* sv = nullptr;
+  sv = GetThreadLocalSuperVersion(db_mutex);
   sv->Ref();
   if (!ReturnThreadLocalSuperVersion(sv)) {
-    // This Unref() corresponds to the Ref() in GetThreadLocalSuperVersion()
-    // when the thread-local pointer was populated. So, the Ref() earlier in
-    // this function still prevents the returned SuperVersion* from being
-    // deleted out from under the caller.
     sv->Unref();
   }
   return sv;
@@ -1026,6 +852,7 @@ SuperVersion* ColumnFamilyData::GetReferencedSuperVersion(
 
 SuperVersion* ColumnFamilyData::GetThreadLocalSuperVersion(
     InstrumentedMutex* db_mutex) {
+  SuperVersion* sv = nullptr;
   // The SuperVersion is cached in thread local storage to avoid acquiring
   // mutex when SuperVersion does not change since the last use. When a new
   // SuperVersion is installed, the compaction or flush thread cleans up
@@ -1044,7 +871,7 @@ SuperVersion* ColumnFamilyData::GetThreadLocalSuperVersion(
   // should only keep kSVInUse before ReturnThreadLocalSuperVersion call
   // (if no Scrape happens).
   assert(ptr != SuperVersion::kSVInUse);
-  SuperVersion* sv = static_cast<SuperVersion*>(ptr);
+  sv = static_cast<SuperVersion*>(ptr);
   if (sv == SuperVersion::kSVObsolete ||
       sv->version_number != super_version_number_.load()) {
     RecordTick(ioptions_.statistics, NUMBER_SUPERVERSION_ACQUIRES);
@@ -1087,16 +914,15 @@ bool ColumnFamilyData::ReturnThreadLocalSuperVersion(SuperVersion* sv) {
   return false;
 }
 
-void ColumnFamilyData::InstallSuperVersion(
-    SuperVersionContext* sv_context, InstrumentedMutex* db_mutex) {
+SuperVersion* ColumnFamilyData::InstallSuperVersion(
+    SuperVersion* new_superversion, InstrumentedMutex* db_mutex) {
   db_mutex->AssertHeld();
-  return InstallSuperVersion(sv_context, db_mutex, mutable_cf_options_);
+  return InstallSuperVersion(new_superversion, db_mutex, mutable_cf_options_);
 }
 
-void ColumnFamilyData::InstallSuperVersion(
-    SuperVersionContext* sv_context, InstrumentedMutex* db_mutex,
+SuperVersion* ColumnFamilyData::InstallSuperVersion(
+    SuperVersion* new_superversion, InstrumentedMutex* db_mutex,
     const MutableCFOptions& mutable_cf_options) {
-  SuperVersion* new_superversion = sv_context->new_superversion.release();
   new_superversion->db_mutex = db_mutex;
   new_superversion->mutable_cf_options = mutable_cf_options;
   new_superversion->Init(mem_, imm_.current(), current_);
@@ -1104,31 +930,23 @@ void ColumnFamilyData::InstallSuperVersion(
   super_version_ = new_superversion;
   ++super_version_number_;
   super_version_->version_number = super_version_number_;
-  super_version_->write_stall_condition =
-      RecalculateWriteStallConditions(mutable_cf_options);
-
   if (old_superversion != nullptr) {
-    // Reset SuperVersions cached in thread local storage.
-    // This should be done before old_superversion->Unref(). That's to ensure
-    // that local_sv_ never holds the last reference to SuperVersion, since
-    // it has no means to safely do SuperVersion cleanup.
-    ResetThreadLocalSuperVersions();
-
     if (old_superversion->mutable_cf_options.write_buffer_size !=
         mutable_cf_options.write_buffer_size) {
       mem_->UpdateWriteBufferSize(mutable_cf_options.write_buffer_size);
     }
-    if (old_superversion->write_stall_condition !=
-        new_superversion->write_stall_condition) {
-      sv_context->PushWriteStallNotification(
-          old_superversion->write_stall_condition,
-          new_superversion->write_stall_condition, GetName(), ioptions());
-    }
-    if (old_superversion->Unref()) {
-      old_superversion->Cleanup();
-      sv_context->superversions_to_free.push_back(old_superversion);
-    }
   }
+
+  // Reset SuperVersions cached in thread local storage
+  ResetThreadLocalSuperVersions();
+
+  RecalculateWriteStallConditions(mutable_cf_options);
+
+  if (old_superversion != nullptr && old_superversion->Unref()) {
+    old_superversion->Cleanup();
+    return old_superversion;  // will let caller delete outside of mutex
+  }
+  return nullptr;
 }
 
 void ColumnFamilyData::ResetThreadLocalSuperVersions() {
@@ -1140,12 +958,10 @@ void ColumnFamilyData::ResetThreadLocalSuperVersions() {
       continue;
     }
     auto sv = static_cast<SuperVersion*>(ptr);
-    bool was_last_ref __attribute__((__unused__));
-    was_last_ref = sv->Unref();
-    // sv couldn't have been the last reference because
-    // ResetThreadLocalSuperVersions() is called before
-    // unref'ing super_version_.
-    assert(!was_last_ref);
+    if (sv->Unref()) {
+      sv->Cleanup();
+      delete sv;
+    }
   }
 }
 
@@ -1153,9 +969,8 @@ void ColumnFamilyData::ResetThreadLocalSuperVersions() {
 Status ColumnFamilyData::SetOptions(
       const std::unordered_map<std::string, std::string>& options_map) {
   MutableCFOptions new_mutable_cf_options;
-  Status s =
-      GetMutableOptionsFromStrings(mutable_cf_options_, options_map,
-                                   ioptions_.info_log, &new_mutable_cf_options);
+  Status s = GetMutableOptionsFromStrings(mutable_cf_options_, options_map,
+                                          &new_mutable_cf_options);
   if (s.ok()) {
     mutable_cf_options_ = new_mutable_cf_options;
     mutable_cf_options_.RefreshDerivedOptions(ioptions_);
@@ -1164,49 +979,6 @@ Status ColumnFamilyData::SetOptions(
 }
 #endif  // ROCKSDB_LITE
 
-// REQUIRES: DB mutex held
-Env::WriteLifeTimeHint ColumnFamilyData::CalculateSSTWriteHint(int level) {
-  if (initial_cf_options_.compaction_style != kCompactionStyleLevel) {
-    return Env::WLTH_NOT_SET;
-  }
-  if (level == 0) {
-    return Env::WLTH_MEDIUM;
-  }
-  int base_level = current_->storage_info()->base_level();
-
-  // L1: medium, L2: long, ...
-  if (level - base_level >= 2) {
-    return Env::WLTH_EXTREME;
-  }
-  return static_cast<Env::WriteLifeTimeHint>(level - base_level +
-                            static_cast<int>(Env::WLTH_MEDIUM));
-}
-
-Status ColumnFamilyData::AddDirectories() {
-  Status s;
-  assert(data_dirs_.empty());
-  for (auto& p : ioptions_.cf_paths) {
-    std::unique_ptr<Directory> path_directory;
-    s = DBImpl::CreateAndNewDirectory(ioptions_.env, p.path, &path_directory);
-    if (!s.ok()) {
-      return s;
-    }
-    assert(path_directory != nullptr);
-    data_dirs_.emplace_back(path_directory.release());
-  }
-  assert(data_dirs_.size() == ioptions_.cf_paths.size());
-  return s;
-}
-
-Directory* ColumnFamilyData::GetDataDir(size_t path_id) const {
-  if (data_dirs_.empty()) {
-    return nullptr;
-  }
-
-  assert(path_id < data_dirs_.size());
-  return data_dirs_[path_id].get();
-}
-
 ColumnFamilySet::ColumnFamilySet(const std::string& dbname,
                                  const ImmutableDBOptions* db_options,
                                  const EnvOptions& env_options,
@@ -1233,14 +1005,10 @@ ColumnFamilySet::~ColumnFamilySet() {
   while (column_family_data_.size() > 0) {
     // cfd destructor will delete itself from column_family_data_
     auto cfd = column_family_data_.begin()->second;
-    bool last_ref __attribute__((__unused__));
-    last_ref = cfd->Unref();
-    assert(last_ref);
+    cfd->Unref();
     delete cfd;
   }
-  bool dummy_last_ref __attribute__((__unused__));
-  dummy_last_ref = dummy_cfd_->Unref();
-  assert(dummy_last_ref);
+  dummy_cfd_->Unref();
   delete dummy_cfd_;
 }
 

db/column_family.h

@@ -30,7 +30,6 @@ namespace rocksdb {
 
 class Version;
 class VersionSet;
-class VersionStorageInfo;
 class MemTable;
 class MemTableListVersion;
 class CompactionPicker;
@@ -42,7 +41,6 @@ class DBImpl;
 class LogBuffer;
 class InstrumentedMutex;
 class InstrumentedMutexLock;
-struct SuperVersionContext;
 
 extern const double kIncSlowdownRatio;
 
@@ -78,7 +76,7 @@ class ColumnFamilyHandleImpl : public ColumnFamilyHandle {
 class ColumnFamilyHandleInternal : public ColumnFamilyHandleImpl {
  public:
   ColumnFamilyHandleInternal()
-      : ColumnFamilyHandleImpl(nullptr, nullptr, nullptr), internal_cfd_(nullptr) {}
+      : ColumnFamilyHandleImpl(nullptr, nullptr, nullptr) {}
 
   void SetCFD(ColumnFamilyData* _cfd) { internal_cfd_ = _cfd; }
   virtual ColumnFamilyData* cfd() const override { return internal_cfd_; }
@@ -97,7 +95,6 @@ struct SuperVersion {
   MutableCFOptions mutable_cf_options;
   // Version number of the current SuperVersion
   uint64_t version_number;
-  WriteStallCondition write_stall_condition;
 
   InstrumentedMutex* db_mutex;
 
@@ -139,9 +136,6 @@ extern Status CheckCompressionSupported(const ColumnFamilyOptions& cf_options);
 extern Status CheckConcurrentWritesSupported(
     const ColumnFamilyOptions& cf_options);
 
-extern Status CheckCFPathsSupported(const DBOptions& db_options,
-                                    const ColumnFamilyOptions& cf_options);
-
 extern ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
                                            const ColumnFamilyOptions& src);
 // Wrap user defined table proproties collector factories `from cf_options`
@@ -198,7 +192,7 @@ class ColumnFamilyData {
   // *) delete all memory associated with that column family
   // *) delete all the files associated with that column family
   void SetDropped();
-  bool IsDropped() const { return dropped_.load(std::memory_order_relaxed); }
+  bool IsDropped() const { return dropped_; }
 
   // thread-safe
   int NumberLevels() const { return ioptions_.num_levels; }
@@ -206,10 +200,6 @@ class ColumnFamilyData {
   void SetLogNumber(uint64_t log_number) { log_number_ = log_number; }
   uint64_t GetLogNumber() const { return log_number_; }
 
-  void SetFlushReason(FlushReason flush_reason) {
-    flush_reason_ = flush_reason;
-  }
-  FlushReason GetFlushReason() const { return flush_reason_; }
   // thread-safe
   const EnvOptions* soptions() const;
   const ImmutableCFOptions* ioptions() const { return &ioptions_; }
@@ -247,12 +237,7 @@ class ColumnFamilyData {
   void SetCurrent(Version* _current);
   uint64_t GetNumLiveVersions() const;  // REQUIRE: DB mutex held
   uint64_t GetTotalSstFilesSize() const;  // REQUIRE: DB mutex held
-  uint64_t GetLiveSstFilesSize() const;   // REQUIRE: DB mutex held
-  void SetMemtable(MemTable* new_mem) {
-    uint64_t memtable_id = last_memtable_id_.fetch_add(1) + 1;
-    new_mem->SetID(memtable_id);
-    mem_ = new_mem;
-  }
+  void SetMemtable(MemTable* new_mem) { mem_ = new_mem; }
 
   // calculate the oldest log needed for the durability of this column family
   uint64_t OldestLogToKeep();
@@ -278,27 +263,17 @@ class ColumnFamilyData {
                                   const Slice& largest_user_key,
                                   int level) const;
 
-  // Check if the passed ranges overlap with any unflushed memtables
-  // (immutable or mutable).
-  //
-  // @param super_version A referenced SuperVersion that will be held for the
-  //    duration of this function.
-  //
-  // Thread-safe
-  Status RangesOverlapWithMemtables(const autovector<Range>& ranges,
-                                    SuperVersion* super_version, bool* overlap);
-
   // A flag to tell a manual compaction is to compact all levels together
-  // instead of a specific level.
+  // instad of for specific level.
   static const int kCompactAllLevels;
   // A flag to tell a manual compaction's output is base level.
   static const int kCompactToBaseLevel;
   // REQUIRES: DB mutex held
   Compaction* CompactRange(const MutableCFOptions& mutable_cf_options,
                            int input_level, int output_level,
-                           uint32_t output_path_id, uint32_t max_subcompactions,
-                           const InternalKey* begin, const InternalKey* end,
-                           InternalKey** compaction_end, bool* manual_conflict);
+                           uint32_t output_path_id, const InternalKey* begin,
+                           const InternalKey* end, InternalKey** compaction_end,
+                           bool* manual_conflict);
 
   CompactionPicker* compaction_picker() { return compaction_picker_.get(); }
   // thread-safe
@@ -336,53 +311,32 @@ class ColumnFamilyData {
   // As argument takes a pointer to allocated SuperVersion to enable
   // the clients to allocate SuperVersion outside of mutex.
   // IMPORTANT: Only call this from DBImpl::InstallSuperVersion()
-  void InstallSuperVersion(SuperVersionContext* sv_context,
-                           InstrumentedMutex* db_mutex,
-                           const MutableCFOptions& mutable_cf_options);
-  void InstallSuperVersion(SuperVersionContext* sv_context,
-                           InstrumentedMutex* db_mutex);
+  SuperVersion* InstallSuperVersion(SuperVersion* new_superversion,
+                                    InstrumentedMutex* db_mutex,
+                                    const MutableCFOptions& mutable_cf_options);
+  SuperVersion* InstallSuperVersion(SuperVersion* new_superversion,
+                                    InstrumentedMutex* db_mutex);
 
   void ResetThreadLocalSuperVersions();
 
   // Protected by DB mutex
-  void set_queued_for_flush(bool value) { queued_for_flush_ = value; }
-  void set_queued_for_compaction(bool value) { queued_for_compaction_ = value; }
-  bool queued_for_flush() { return queued_for_flush_; }
-  bool queued_for_compaction() { return queued_for_compaction_; }
-
-  enum class WriteStallCause {
-    kNone,
-    kMemtableLimit,
-    kL0FileCountLimit,
-    kPendingCompactionBytes,
-  };
-  static std::pair<WriteStallCondition, WriteStallCause>
-  GetWriteStallConditionAndCause(int num_unflushed_memtables, int num_l0_files,
-                                 uint64_t num_compaction_needed_bytes,
-                                 const MutableCFOptions& mutable_cf_options);
+  void set_pending_flush(bool value) { pending_flush_ = value; }
+  void set_pending_compaction(bool value) { pending_compaction_ = value; }
+  bool pending_flush() { return pending_flush_; }
+  bool pending_compaction() { return pending_compaction_; }
 
   // Recalculate some small conditions, which are changed only during
   // compaction, adding new memtable and/or
   // recalculation of compaction score. These values are used in
   // DBImpl::MakeRoomForWrite function to decide, if it need to make
   // a write stall
-  WriteStallCondition RecalculateWriteStallConditions(
+  void RecalculateWriteStallConditions(
       const MutableCFOptions& mutable_cf_options);
 
   void set_initialized() { initialized_.store(true); }
 
   bool initialized() const { return initialized_.load(); }
 
-  const ColumnFamilyOptions& initial_cf_options() {
-    return initial_cf_options_;
-  }
-
-  Env::WriteLifeTimeHint CalculateSSTWriteHint(int level);
-
-  Status AddDirectories();
-
-  Directory* GetDataDir(size_t path_id) const;
-
  private:
   friend class ColumnFamilySet;
   ColumnFamilyData(uint32_t id, const std::string& name,
@@ -400,7 +354,7 @@ class ColumnFamilyData {
 
   std::atomic<int> refs_;      // outstanding references to ColumnFamilyData
   std::atomic<bool> initialized_;
-  std::atomic<bool> dropped_;  // true if client dropped it
+  bool dropped_;               // true if client dropped it
 
   const InternalKeyComparator internal_comparator_;
   std::vector<std::unique_ptr<IntTblPropCollectorFactory>>
@@ -442,8 +396,6 @@ class ColumnFamilyData {
   // recovered from
   uint64_t log_number_;
 
-  std::atomic<FlushReason> flush_reason_;
-
   // An object that keeps all the compaction stats
   // and picks the next compaction
   std::unique_ptr<CompactionPicker> compaction_picker_;
@@ -453,22 +405,16 @@ class ColumnFamilyData {
   std::unique_ptr<WriteControllerToken> write_controller_token_;
 
   // If true --> this ColumnFamily is currently present in DBImpl::flush_queue_
-  bool queued_for_flush_;
+  bool pending_flush_;
 
   // If true --> this ColumnFamily is currently present in
   // DBImpl::compaction_queue_
-  bool queued_for_compaction_;
+  bool pending_compaction_;
 
   uint64_t prev_compaction_needed_bytes_;
 
   // if the database was opened with 2pc enabled
   bool allow_2pc_;
-
-  // Memtable id to track flush.
-  std::atomic<uint64_t> last_memtable_id_;
-
-  // Directories corresponding to cf_paths.
-  std::vector<std::unique_ptr<Directory>> data_dirs_;
 };
 
 // ColumnFamilySet has interesting thread-safety requirements

db/column_family_test.cc

@@ -69,15 +69,9 @@ class ColumnFamilyTest : public testing::Test {
   }
 
   ~ColumnFamilyTest() {
-    std::vector<ColumnFamilyDescriptor> column_families;
-    for (auto h : handles_) {
-      ColumnFamilyDescriptor cfdescriptor;
-      h->GetDescriptor(&cfdescriptor);
-      column_families.push_back(cfdescriptor);
-    }
     Close();
     rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-    Destroy(column_families);
+    Destroy();
     delete env_;
   }
 
@@ -242,11 +236,9 @@ class ColumnFamilyTest : public testing::Test {
 #endif  // !ROCKSDB_LITE
   }
 
-  void Destroy(const std::vector<ColumnFamilyDescriptor>& column_families =
-                  std::vector<ColumnFamilyDescriptor>()) {
+  void Destroy() {
     Close();
-    ASSERT_OK(DestroyDB(dbname_, Options(db_options_, column_family_options_),
-                        column_families));
+    ASSERT_OK(DestroyDB(dbname_, Options(db_options_, column_family_options_)));
   }
 
   void CreateColumnFamilies(
@@ -299,9 +291,6 @@ class ColumnFamilyTest : public testing::Test {
   }
 
   void PutRandomData(int cf, int num, int key_value_size, bool save = false) {
-    if (cf >= static_cast<int>(keys_.size())) {
-      keys_.resize(cf + 1);
-    }
     for (int i = 0; i < num; ++i) {
       // 10 bytes for key, rest is value
       if (!save) {
@@ -309,7 +298,7 @@ class ColumnFamilyTest : public testing::Test {
                       RandomString(&rnd_, key_value_size - 10)));
       } else {
         std::string key = test::RandomKey(&rnd_, 11);
-        keys_[cf].insert(key);
+        keys_.insert(key);
         ASSERT_OK(Put(cf, key, RandomString(&rnd_, key_value_size - 10)));
       }
     }
@@ -394,9 +383,6 @@ class ColumnFamilyTest : public testing::Test {
   void AssertFilesPerLevel(const std::string& value, int cf) {
 #ifndef ROCKSDB_LITE
     ASSERT_EQ(value, FilesPerLevel(cf));
-#else
-    (void) value;
-    (void) cf;
 #endif
   }
 
@@ -411,8 +397,6 @@ class ColumnFamilyTest : public testing::Test {
   void AssertCountLiveFiles(int expected_value) {
 #ifndef ROCKSDB_LITE
     ASSERT_EQ(expected_value, CountLiveFiles());
-#else
-    (void) expected_value;
 #endif
   }
 
@@ -461,8 +445,6 @@ class ColumnFamilyTest : public testing::Test {
   void AssertCountLiveLogFiles(int value) {
 #ifndef ROCKSDB_LITE  // GetSortedWalFiles is not supported
     ASSERT_EQ(value, CountLiveLogFiles());
-#else
-    (void) value;
 #endif  // !ROCKSDB_LITE
   }
 
@@ -501,15 +483,9 @@ class ColumnFamilyTest : public testing::Test {
     ASSERT_OK(destfile->Close());
   }
 
-  int GetSstFileCount(std::string path) {
-    std::vector<std::string> files;
-    DBTestBase::GetSstFiles(env_, path, &files);
-    return static_cast<int>(files.size());
-  }
-
   std::vector<ColumnFamilyHandle*> handles_;
   std::vector<std::string> names_;
-  std::vector<std::set<std::string>> keys_;
+  std::set<std::string> keys_;
   ColumnFamilyOptions column_family_options_;
   DBOptions db_options_;
   std::string dbname_;
@@ -957,62 +933,62 @@ TEST_F(ColumnFamilyTest, FlushTest) {
 TEST_F(ColumnFamilyTest, LogDeletionTest) {
   db_options_.max_total_wal_size = std::numeric_limits<uint64_t>::max();
   column_family_options_.arena_block_size = 4 * 1024;
-  column_family_options_.write_buffer_size = 128000;  // 128KB
+  column_family_options_.write_buffer_size = 100000;  // 100KB
   Open();
   CreateColumnFamilies({"one", "two", "three", "four"});
   // Each bracket is one log file. if number is in (), it means
   // we don't need it anymore (it's been flushed)
   // []
   AssertCountLiveLogFiles(0);
-  PutRandomData(0, 1, 128);
+  PutRandomData(0, 1, 100);
   // [0]
-  PutRandomData(1, 1, 128);
+  PutRandomData(1, 1, 100);
   // [0, 1]
-  PutRandomData(1, 1000, 128);
+  PutRandomData(1, 1000, 100);
   WaitForFlush(1);
   // [0, (1)] [1]
   AssertCountLiveLogFiles(2);
-  PutRandomData(0, 1, 128);
+  PutRandomData(0, 1, 100);
   // [0, (1)] [0, 1]
   AssertCountLiveLogFiles(2);
-  PutRandomData(2, 1, 128);
+  PutRandomData(2, 1, 100);
   // [0, (1)] [0, 1, 2]
-  PutRandomData(2, 1000, 128);
+  PutRandomData(2, 1000, 100);
   WaitForFlush(2);
   // [0, (1)] [0, 1, (2)] [2]
   AssertCountLiveLogFiles(3);
-  PutRandomData(2, 1000, 128);
+  PutRandomData(2, 1000, 100);
   WaitForFlush(2);
   // [0, (1)] [0, 1, (2)] [(2)] [2]
   AssertCountLiveLogFiles(4);
-  PutRandomData(3, 1, 128);
+  PutRandomData(3, 1, 100);
   // [0, (1)] [0, 1, (2)] [(2)] [2, 3]
-  PutRandomData(1, 1, 128);
+  PutRandomData(1, 1, 100);
   // [0, (1)] [0, 1, (2)] [(2)] [1, 2, 3]
   AssertCountLiveLogFiles(4);
-  PutRandomData(1, 1000, 128);
+  PutRandomData(1, 1000, 100);
   WaitForFlush(1);
   // [0, (1)] [0, (1), (2)] [(2)] [(1), 2, 3] [1]
   AssertCountLiveLogFiles(5);
-  PutRandomData(0, 1000, 128);
+  PutRandomData(0, 1000, 100);
   WaitForFlush(0);
   // [(0), (1)] [(0), (1), (2)] [(2)] [(1), 2, 3] [1, (0)] [0]
   // delete obsolete logs -->
   // [(1), 2, 3] [1, (0)] [0]
   AssertCountLiveLogFiles(3);
-  PutRandomData(0, 1000, 128);
+  PutRandomData(0, 1000, 100);
   WaitForFlush(0);
   // [(1), 2, 3] [1, (0)], [(0)] [0]
   AssertCountLiveLogFiles(4);
-  PutRandomData(1, 1000, 128);
+  PutRandomData(1, 1000, 100);
   WaitForFlush(1);
   // [(1), 2, 3] [(1), (0)] [(0)] [0, (1)] [1]
   AssertCountLiveLogFiles(5);
-  PutRandomData(2, 1000, 128);
+  PutRandomData(2, 1000, 100);
   WaitForFlush(2);
   // [(1), (2), 3] [(1), (0)] [(0)] [0, (1)] [1, (2)], [2]
   AssertCountLiveLogFiles(6);
-  PutRandomData(3, 1000, 128);
+  PutRandomData(3, 1000, 100);
   WaitForFlush(3);
   // [(1), (2), (3)] [(1), (0)] [(0)] [0, (1)] [1, (2)], [2, (3)] [3]
   // delete obsolete logs -->
@@ -1192,14 +1168,13 @@ TEST_F(ColumnFamilyTest, MemtableNotSupportSnapshot) {
 #endif  // !ROCKSDB_LITE
 
 class TestComparator : public Comparator {
-  int Compare(const rocksdb::Slice& /*a*/,
-              const rocksdb::Slice& /*b*/) const override {
+  int Compare(const rocksdb::Slice& a, const rocksdb::Slice& b) const override {
     return 0;
   }
   const char* Name() const override { return "Test"; }
-  void FindShortestSeparator(std::string* /*start*/,
-                             const rocksdb::Slice& /*limit*/) const override {}
-  void FindShortSuccessor(std::string* /*key*/) const override {}
+  void FindShortestSeparator(std::string* start,
+                             const rocksdb::Slice& limit) const override {}
+  void FindShortSuccessor(std::string* key) const override {}
 };
 
 static TestComparator third_comparator;
@@ -1371,7 +1346,7 @@ TEST_F(ColumnFamilyTest, MultipleManualCompactions) {
        {"ColumnFamilyTest::MultiManual:2", "ColumnFamilyTest::MultiManual:5"},
        {"ColumnFamilyTest::MultiManual:2", "ColumnFamilyTest::MultiManual:3"}});
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::BackgroundCompaction:NonTrivial:AfterRun", [&](void* /*arg*/) {
+      "DBImpl::BackgroundCompaction:NonTrivial:AfterRun", [&](void* arg) {
         if (cf_1_1) {
           TEST_SYNC_POINT("ColumnFamilyTest::MultiManual:4");
           cf_1_1 = false;
@@ -1417,8 +1392,8 @@ TEST_F(ColumnFamilyTest, MultipleManualCompactions) {
   CompactAll(2);
   AssertFilesPerLevel("0,1", 2);
   // Compare against saved keys
-  std::set<std::string>::iterator key_iter = keys_[1].begin();
-  while (key_iter != keys_[1].end()) {
+  std::set<std::string>::iterator key_iter = keys_.begin();
+  while (key_iter != keys_.end()) {
     ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
     key_iter++;
   }
@@ -1464,7 +1439,7 @@ TEST_F(ColumnFamilyTest, AutomaticAndManualCompactions) {
        {"ColumnFamilyTest::AutoManual:2", "ColumnFamilyTest::AutoManual:5"},
        {"ColumnFamilyTest::AutoManual:2", "ColumnFamilyTest::AutoManual:3"}});
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::BackgroundCompaction:NonTrivial:AfterRun", [&](void* /*arg*/) {
+      "DBImpl::BackgroundCompaction:NonTrivial:AfterRun", [&](void* arg) {
         if (cf_1_1) {
           cf_1_1 = false;
           TEST_SYNC_POINT("ColumnFamilyTest::AutoManual:4");
@@ -1511,8 +1486,8 @@ TEST_F(ColumnFamilyTest, AutomaticAndManualCompactions) {
   CompactAll(2);
   AssertFilesPerLevel("0,1", 2);
   // Compare against saved keys
-  std::set<std::string>::iterator key_iter = keys_[1].begin();
-  while (key_iter != keys_[1].end()) {
+  std::set<std::string>::iterator key_iter = keys_.begin();
+  while (key_iter != keys_.end()) {
     ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
     key_iter++;
   }
@@ -1565,7 +1540,7 @@ TEST_F(ColumnFamilyTest, ManualAndAutomaticCompactions) {
        {"ColumnFamilyTest::ManualAuto:5", "ColumnFamilyTest::ManualAuto:2"},
        {"ColumnFamilyTest::ManualAuto:2", "ColumnFamilyTest::ManualAuto:3"}});
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::BackgroundCompaction:NonTrivial:AfterRun", [&](void* /*arg*/) {
+      "DBImpl::BackgroundCompaction:NonTrivial:AfterRun", [&](void* arg) {
         if (cf_1_1) {
           TEST_SYNC_POINT("ColumnFamilyTest::ManualAuto:4");
           cf_1_1 = false;
@@ -1607,8 +1582,8 @@ TEST_F(ColumnFamilyTest, ManualAndAutomaticCompactions) {
   CompactAll(2);
   AssertFilesPerLevel("0,1", 2);
   // Compare against saved keys
-  std::set<std::string>::iterator key_iter = keys_[1].begin();
-  while (key_iter != keys_[1].end()) {
+  std::set<std::string>::iterator key_iter = keys_.begin();
+  while (key_iter != keys_.end()) {
     ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
     key_iter++;
   }
@@ -1658,7 +1633,7 @@ TEST_F(ColumnFamilyTest, SameCFManualManualCompactions) {
        {"ColumnFamilyTest::ManualManual:1",
         "ColumnFamilyTest::ManualManual:3"}});
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::BackgroundCompaction:NonTrivial:AfterRun", [&](void* /*arg*/) {
+      "DBImpl::BackgroundCompaction:NonTrivial:AfterRun", [&](void* arg) {
         if (cf_1_1) {
           TEST_SYNC_POINT("ColumnFamilyTest::ManualManual:4");
           cf_1_1 = false;
@@ -1706,8 +1681,8 @@ TEST_F(ColumnFamilyTest, SameCFManualManualCompactions) {
   ASSERT_LE(NumTableFilesAtLevel(0, 1), 2);
 
   // Compare against saved keys
-  std::set<std::string>::iterator key_iter = keys_[1].begin();
-  while (key_iter != keys_[1].end()) {
+  std::set<std::string>::iterator key_iter = keys_.begin();
+  while (key_iter != keys_.end()) {
     ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
     key_iter++;
   }
@@ -1756,7 +1731,7 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactions) {
        {"ColumnFamilyTest::ManualAuto:1", "ColumnFamilyTest::ManualAuto:2"},
        {"ColumnFamilyTest::ManualAuto:1", "ColumnFamilyTest::ManualAuto:3"}});
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::BackgroundCompaction:NonTrivial:AfterRun", [&](void* /*arg*/) {
+      "DBImpl::BackgroundCompaction:NonTrivial:AfterRun", [&](void* arg) {
         if (cf_1_1) {
           TEST_SYNC_POINT("ColumnFamilyTest::ManualAuto:4");
           cf_1_1 = false;
@@ -1796,8 +1771,8 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactions) {
   ASSERT_LE(NumTableFilesAtLevel(0, 1), 2);
 
   // Compare against saved keys
-  std::set<std::string>::iterator key_iter = keys_[1].begin();
-  while (key_iter != keys_[1].end()) {
+  std::set<std::string>::iterator key_iter = keys_.begin();
+  while (key_iter != keys_.end()) {
     ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
     key_iter++;
   }
@@ -1848,7 +1823,7 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactionsLevel) {
         "ColumnFamilyTest::ManualAuto:3"},
        {"ColumnFamilyTest::ManualAuto:1", "ColumnFamilyTest::ManualAuto:3"}});
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::BackgroundCompaction:NonTrivial:AfterRun", [&](void* /*arg*/) {
+      "DBImpl::BackgroundCompaction:NonTrivial:AfterRun", [&](void* arg) {
         if (cf_1_1) {
           TEST_SYNC_POINT("ColumnFamilyTest::ManualAuto:4");
           cf_1_1 = false;
@@ -1886,8 +1861,128 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactionsLevel) {
   AssertFilesPerLevel("0,1", 1);
 
   // Compare against saved keys
-  std::set<std::string>::iterator key_iter = keys_[1].begin();
-  while (key_iter != keys_[1].end()) {
+  std::set<std::string>::iterator key_iter = keys_.begin();
+  while (key_iter != keys_.end()) {
+    ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
+    key_iter++;
+  }
+}
+
+// This test checks for automatic getting a conflict if there is a
+// manual which has not yet been scheduled.
+// The manual compaction waits in NotScheduled
+// We generate more files and then trigger an automatic compaction
+// This will wait because there is an unscheduled manual compaction.
+// Once the conflict is hit, the manual compaction starts and ends
+// Then another automatic will start and end.
+TEST_F(ColumnFamilyTest, SameCFManualAutomaticConflict) {
+  Open();
+  CreateColumnFamilies({"one"});
+  ColumnFamilyOptions default_cf, one;
+  db_options_.max_open_files = 20;  // only 10 files in file cache
+  db_options_.max_background_compactions = 3;
+
+  default_cf.compaction_style = kCompactionStyleLevel;
+  default_cf.num_levels = 3;
+  default_cf.write_buffer_size = 64 << 10;  // 64KB
+  default_cf.target_file_size_base = 30 << 10;
+  default_cf.max_compaction_bytes = default_cf.target_file_size_base * 1100;
+  BlockBasedTableOptions table_options;
+  table_options.no_block_cache = true;
+  default_cf.table_factory.reset(NewBlockBasedTableFactory(table_options));
+
+  one.compaction_style = kCompactionStyleUniversal;
+
+  one.num_levels = 1;
+  // trigger compaction if there are >= 4 files
+  one.level0_file_num_compaction_trigger = 4;
+  one.write_buffer_size = 120000;
+
+  Reopen({default_cf, one});
+  // make sure all background compaction jobs can be scheduled
+  auto stop_token =
+      dbfull()->TEST_write_controler().GetCompactionPressureToken();
+
+  // SETUP column family "one" -- universal style
+  for (int i = 0; i < one.level0_file_num_compaction_trigger - 2; ++i) {
+    PutRandomData(1, 10, 12000, true);
+    PutRandomData(1, 1, 10, true);
+    WaitForFlush(1);
+    AssertFilesPerLevel(ToString(i + 1), 1);
+  }
+  bool cf_1_1 = true;
+  bool cf_1_2 = true;
+  rocksdb::SyncPoint::GetInstance()->LoadDependency(
+      {{"DBImpl::BackgroundCompaction()::Conflict",
+        "ColumnFamilyTest::ManualAutoCon:7"},
+       {"ColumnFamilyTest::ManualAutoCon:9",
+        "ColumnFamilyTest::ManualAutoCon:8"},
+       {"ColumnFamilyTest::ManualAutoCon:2",
+        "ColumnFamilyTest::ManualAutoCon:6"},
+       {"ColumnFamilyTest::ManualAutoCon:4",
+        "ColumnFamilyTest::ManualAutoCon:5"},
+       {"ColumnFamilyTest::ManualAutoCon:1",
+        "ColumnFamilyTest::ManualAutoCon:2"},
+       {"ColumnFamilyTest::ManualAutoCon:1",
+        "ColumnFamilyTest::ManualAutoCon:3"}});
+  rocksdb::SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::BackgroundCompaction:NonTrivial:AfterRun", [&](void* arg) {
+        if (cf_1_1) {
+          TEST_SYNC_POINT("ColumnFamilyTest::ManualAutoCon:4");
+          cf_1_1 = false;
+          TEST_SYNC_POINT("ColumnFamilyTest::ManualAutoCon:3");
+        } else if (cf_1_2) {
+          cf_1_2 = false;
+          TEST_SYNC_POINT("ColumnFamilyTest::ManualAutoCon:2");
+        }
+      });
+  rocksdb::SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::RunManualCompaction:NotScheduled", [&](void* arg) {
+        InstrumentedMutex* mutex = static_cast<InstrumentedMutex*>(arg);
+        mutex->Unlock();
+        TEST_SYNC_POINT("ColumnFamilyTest::ManualAutoCon:9");
+        TEST_SYNC_POINT("ColumnFamilyTest::ManualAutoCon:7");
+        mutex->Lock();
+      });
+
+  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+  rocksdb::port::Thread threads([&] {
+    CompactRangeOptions compact_options;
+    compact_options.exclusive_manual_compaction = false;
+    ASSERT_OK(
+        db_->CompactRange(compact_options, handles_[1], nullptr, nullptr));
+    TEST_SYNC_POINT("ColumnFamilyTest::ManualAutoCon:6");
+  });
+
+  TEST_SYNC_POINT("ColumnFamilyTest::ManualAutoCon:8");
+  WaitForFlush(1);
+
+  // Add more L0 files and force automatic compaction
+  for (int i = 0; i < one.level0_file_num_compaction_trigger; ++i) {
+    PutRandomData(1, 10, 12000, true);
+    PutRandomData(1, 1, 10, true);
+    WaitForFlush(1);
+    AssertFilesPerLevel(ToString(one.level0_file_num_compaction_trigger + i),
+                        1);
+  }
+
+  TEST_SYNC_POINT("ColumnFamilyTest::ManualAutoCon:5");
+  // Add more L0 files and force automatic compaction
+  for (int i = 0; i < one.level0_file_num_compaction_trigger; ++i) {
+    PutRandomData(1, 10, 12000, true);
+    PutRandomData(1, 1, 10, true);
+    WaitForFlush(1);
+  }
+  TEST_SYNC_POINT("ColumnFamilyTest::ManualAutoCon:1");
+
+  threads.join();
+  WaitForCompaction();
+  // VERIFY compaction "one"
+  ASSERT_LE(NumTableFilesAtLevel(0, 1), 3);
+
+  // Compare against saved keys
+  std::set<std::string>::iterator key_iter = keys_.begin();
+  while (key_iter != keys_.end()) {
     ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
     key_iter++;
   }
@@ -1935,7 +2030,7 @@ TEST_F(ColumnFamilyTest, SameCFAutomaticManualCompactions) {
        {"CompactionPicker::CompactRange:Conflict",
         "ColumnFamilyTest::AutoManual:3"}});
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::BackgroundCompaction:NonTrivial:AfterRun", [&](void* /*arg*/) {
+      "DBImpl::BackgroundCompaction:NonTrivial:AfterRun", [&](void* arg) {
         if (cf_1_1) {
           TEST_SYNC_POINT("ColumnFamilyTest::AutoManual:4");
           cf_1_1 = false;
@@ -1975,8 +2070,8 @@ TEST_F(ColumnFamilyTest, SameCFAutomaticManualCompactions) {
   // VERIFY compaction "one"
   AssertFilesPerLevel("1", 1);
   // Compare against saved keys
-  std::set<std::string>::iterator key_iter = keys_[1].begin();
-  while (key_iter != keys_[1].end()) {
+  std::set<std::string>::iterator key_iter = keys_.begin();
+  while (key_iter != keys_.end()) {
     ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
     key_iter++;
   }
@@ -2382,7 +2477,7 @@ TEST_F(ColumnFamilyTest, CreateAndDropRace) {
   auto main_thread_id = std::this_thread::get_id();
 
   rocksdb::SyncPoint::GetInstance()->SetCallBack("PersistRocksDBOptions:start",
-                                                 [&](void* /*arg*/) {
+                                                 [&](void* arg) {
     auto current_thread_id = std::this_thread::get_id();
     // If it's the main thread hitting this sync-point, then it
     // will be blocked until some other thread update the test_stage.
@@ -2395,7 +2490,7 @@ TEST_F(ColumnFamilyTest, CreateAndDropRace) {
   });
 
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "WriteThread::EnterUnbatched:Wait", [&](void* /*arg*/) {
+      "WriteThread::EnterUnbatched:Wait", [&](void* arg) {
         // This means a thread doing DropColumnFamily() is waiting for
         // other thread to finish persisting options.
         // In such case, we update the test_stage to unblock the main thread.
@@ -2815,18 +2910,6 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupTwoColumnFamilies) {
   ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
 }
 
-TEST_F(ColumnFamilyTest, CreateAndDestoryOptions) {
-  std::unique_ptr<ColumnFamilyOptions> cfo(new ColumnFamilyOptions());
-  ColumnFamilyHandle* cfh;
-  Open();
-  ASSERT_OK(db_->CreateColumnFamily(*(cfo.get()), "yoyo", &cfh));
-  cfo.reset();
-  ASSERT_OK(db_->Put(WriteOptions(), cfh, "foo", "bar"));
-  ASSERT_OK(db_->Flush(FlushOptions(), cfh));
-  ASSERT_OK(db_->DropColumnFamily(cfh));
-  ASSERT_OK(db_->DestroyColumnFamilyHandle(cfh));
-}
-
 #ifndef ROCKSDB_LITE
 TEST_F(ColumnFamilyTest, FlushCloseWALFiles) {
   SpecialEnv env(Env::Default());
@@ -3153,77 +3236,6 @@ TEST_F(ColumnFamilyTest, DISABLED_LogTruncationTest) {
   // cleanup
   env_->DeleteDir(backup_logs);
 }
-
-TEST_F(ColumnFamilyTest, DefaultCfPathsTest) {
-  Open();
-  // Leave cf_paths for one column families to be empty.
-  // Files should be generated according to db_paths for that
-  // column family.
-  ColumnFamilyOptions cf_opt1, cf_opt2;
-  cf_opt1.cf_paths.emplace_back(dbname_ + "_one_1",
-                                std::numeric_limits<uint64_t>::max());
-  CreateColumnFamilies({"one", "two"}, {cf_opt1, cf_opt2});
-  Reopen({ColumnFamilyOptions(), cf_opt1, cf_opt2});
-
-  // Fill Column family 1.
-  PutRandomData(1, 100, 100);
-  Flush(1);
-
-  ASSERT_EQ(1, GetSstFileCount(cf_opt1.cf_paths[0].path));
-  ASSERT_EQ(0, GetSstFileCount(dbname_));
-
-  // Fill column family 2
-  PutRandomData(2, 100, 100);
-  Flush(2);
-
-  // SST from Column family 2 should be generated in
-  // db_paths which is dbname_ in this case.
-  ASSERT_EQ(1, GetSstFileCount(dbname_));
-}
-
-TEST_F(ColumnFamilyTest, MultipleCFPathsTest) {
-  Open();
-  // Configure Column family specific paths.
-  ColumnFamilyOptions cf_opt1, cf_opt2;
-  cf_opt1.cf_paths.emplace_back(dbname_ + "_one_1",
-                                std::numeric_limits<uint64_t>::max());
-  cf_opt2.cf_paths.emplace_back(dbname_ + "_two_1",
-                                std::numeric_limits<uint64_t>::max());
-  CreateColumnFamilies({"one", "two"}, {cf_opt1, cf_opt2});
-  Reopen({ColumnFamilyOptions(), cf_opt1, cf_opt2});
-
-  PutRandomData(1, 100, 100, true /* save */);
-  Flush(1);
-
-  // Check that files are generated in appropriate paths.
-  ASSERT_EQ(1, GetSstFileCount(cf_opt1.cf_paths[0].path));
-  ASSERT_EQ(0, GetSstFileCount(dbname_));
-
-  PutRandomData(2, 100, 100, true /* save */);
-  Flush(2);
-
-  ASSERT_EQ(1, GetSstFileCount(cf_opt2.cf_paths[0].path));
-  ASSERT_EQ(0, GetSstFileCount(dbname_));
-
-  // Re-open and verify the keys.
-  Reopen({ColumnFamilyOptions(), cf_opt1, cf_opt2});
-  DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
-  for (int cf = 1; cf != 3; ++cf) {
-    ReadOptions read_options;
-    read_options.readahead_size = 0;
-    auto it = dbi->NewIterator(read_options, handles_[cf]);
-    for (it->SeekToFirst(); it->Valid(); it->Next()) {
-      Slice key(it->key());
-      ASSERT_NE(keys_[cf].end(), keys_[cf].find(key.ToString()));
-    }
-    delete it;
-
-    for (const auto& key : keys_[cf]) {
-      ASSERT_NE("NOT_FOUND", Get(cf, key));
-    }
-  }
-}
-
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/compact_files_test.cc

@@ -37,7 +37,8 @@ class FlushedFileCollector : public EventListener {
   FlushedFileCollector() {}
   ~FlushedFileCollector() {}
 
-  virtual void OnFlushCompleted(DB* /*db*/, const FlushJobInfo& info) override {
+  virtual void OnFlushCompleted(
+      DB* db, const FlushJobInfo& info) override {
     std::lock_guard<std::mutex> lock(mutex_);
     flushed_files_.push_back(info.file_path);
   }
@@ -256,9 +257,9 @@ TEST_F(CompactFilesTest, CapturingPendingFiles) {
 TEST_F(CompactFilesTest, CompactionFilterWithGetSv) {
   class FilterWithGet : public CompactionFilter {
    public:
-    virtual bool Filter(int /*level*/, const Slice& /*key*/,
-                        const Slice& /*value*/, std::string* /*new_value*/,
-                        bool* /*value_changed*/) const override {
+    virtual bool Filter(int level, const Slice& key, const Slice& value,
+                        std::string* new_value,
+                        bool* value_changed) const override {
       if (db_ == nullptr) {
         return true;
       }
@@ -318,7 +319,7 @@ int main(int argc, char** argv) {
 #else
 #include <stdio.h>
 
-int main(int /*argc*/, char** /*argv*/) {
+int main(int argc, char** argv) {
   fprintf(stderr,
           "SKIPPED as DBImpl::CompactFiles is not supported in ROCKSDB_LITE\n");
   return 0;

db/compacted_db_impl.cc

@@ -17,8 +17,7 @@ extern bool SaveValue(void* arg, const ParsedInternalKey& parsed_key,
 
 CompactedDBImpl::CompactedDBImpl(
   const DBOptions& options, const std::string& dbname)
-  : DBImpl(options, dbname), cfd_(nullptr), version_(nullptr),
-    user_comparator_(nullptr) {
+  : DBImpl(options, dbname) {
 }
 
 CompactedDBImpl::~CompactedDBImpl() {
@@ -49,8 +48,8 @@ Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*,
                          GetContext::kNotFound, key, value, nullptr, nullptr,
                          nullptr, nullptr);
   LookupKey lkey(key, kMaxSequenceNumber);
-  files_.files[FindFile(key)].fd.table_reader->Get(options, lkey.internal_key(),
-                                                   &get_context, nullptr);
+  files_.files[FindFile(key)].fd.table_reader->Get(
+      options, lkey.internal_key(), &get_context);
   if (get_context.State() == GetContext::kFound) {
     return Status::OK();
   }
@@ -82,7 +81,7 @@ std::vector<Status> CompactedDBImpl::MultiGet(const ReadOptions& options,
                              GetContext::kNotFound, keys[idx], &pinnable_val,
                              nullptr, nullptr, nullptr, nullptr);
       LookupKey lkey(keys[idx], kMaxSequenceNumber);
-      r->Get(options, lkey.internal_key(), &get_context, nullptr);
+      r->Get(options, lkey.internal_key(), &get_context);
       value.assign(pinnable_val.data(), pinnable_val.size());
       if (get_context.State() == GetContext::kFound) {
         statuses[idx] = Status::OK();
@@ -94,7 +93,6 @@ std::vector<Status> CompactedDBImpl::MultiGet(const ReadOptions& options,
 }
 
 Status CompactedDBImpl::Init(const Options& options) {
-  SuperVersionContext sv_context(/* create_superversion */ true);
   mutex_.Lock();
   ColumnFamilyDescriptor cf(kDefaultColumnFamilyName,
                             ColumnFamilyOptions(options));
@@ -102,10 +100,9 @@ Status CompactedDBImpl::Init(const Options& options) {
   if (s.ok()) {
     cfd_ = reinterpret_cast<ColumnFamilyHandleImpl*>(
               DefaultColumnFamily())->cfd();
-    cfd_->InstallSuperVersion(&sv_context, &mutex_);
+    delete cfd_->InstallSuperVersion(new SuperVersion(), &mutex_);
   }
   mutex_.Unlock();
-  sv_context.Clean();
   if (!s.ok()) {
     return s;
   }

db/compacted_db_impl.h

@@ -32,56 +32,55 @@ class CompactedDBImpl : public DBImpl {
     override;
 
   using DBImpl::Put;
-  virtual Status Put(const WriteOptions& /*options*/,
-                     ColumnFamilyHandle* /*column_family*/,
-                     const Slice& /*key*/, const Slice& /*value*/) override {
+  virtual Status Put(const WriteOptions& options,
+                     ColumnFamilyHandle* column_family, const Slice& key,
+                     const Slice& value) override {
     return Status::NotSupported("Not supported in compacted db mode.");
   }
   using DBImpl::Merge;
-  virtual Status Merge(const WriteOptions& /*options*/,
-                       ColumnFamilyHandle* /*column_family*/,
-                       const Slice& /*key*/, const Slice& /*value*/) override {
+  virtual Status Merge(const WriteOptions& options,
+                       ColumnFamilyHandle* column_family, const Slice& key,
+                       const Slice& value) override {
     return Status::NotSupported("Not supported in compacted db mode.");
   }
   using DBImpl::Delete;
-  virtual Status Delete(const WriteOptions& /*options*/,
-                        ColumnFamilyHandle* /*column_family*/,
-                        const Slice& /*key*/) override {
+  virtual Status Delete(const WriteOptions& options,
+                        ColumnFamilyHandle* column_family,
+                        const Slice& key) override {
     return Status::NotSupported("Not supported in compacted db mode.");
   }
-  virtual Status Write(const WriteOptions& /*options*/,
-                       WriteBatch* /*updates*/) override {
+  virtual Status Write(const WriteOptions& options,
+                       WriteBatch* updates) override {
     return Status::NotSupported("Not supported in compacted db mode.");
   }
   using DBImpl::CompactRange;
-  virtual Status CompactRange(const CompactRangeOptions& /*options*/,
-                              ColumnFamilyHandle* /*column_family*/,
-                              const Slice* /*begin*/,
-                              const Slice* /*end*/) override {
+  virtual Status CompactRange(const CompactRangeOptions& options,
+                              ColumnFamilyHandle* column_family,
+                              const Slice* begin, const Slice* end) override {
     return Status::NotSupported("Not supported in compacted db mode.");
   }
 
   virtual Status DisableFileDeletions() override {
     return Status::NotSupported("Not supported in compacted db mode.");
   }
-  virtual Status EnableFileDeletions(bool /*force*/) override {
+  virtual Status EnableFileDeletions(bool force) override {
     return Status::NotSupported("Not supported in compacted db mode.");
   }
   virtual Status GetLiveFiles(std::vector<std::string>&,
-                              uint64_t* /*manifest_file_size*/,
-                              bool /*flush_memtable*/ = true) override {
+                              uint64_t* manifest_file_size,
+                              bool flush_memtable = true) override {
     return Status::NotSupported("Not supported in compacted db mode.");
   }
   using DBImpl::Flush;
-  virtual Status Flush(const FlushOptions& /*options*/,
-                       ColumnFamilyHandle* /*column_family*/) override {
+  virtual Status Flush(const FlushOptions& options,
+                       ColumnFamilyHandle* column_family) override {
     return Status::NotSupported("Not supported in compacted db mode.");
   }
   using DB::IngestExternalFile;
   virtual Status IngestExternalFile(
-      ColumnFamilyHandle* /*column_family*/,
-      const std::vector<std::string>& /*external_files*/,
-      const IngestExternalFileOptions& /*ingestion_options*/) override {
+      ColumnFamilyHandle* column_family,
+      const std::vector<std::string>& external_files,
+      const IngestExternalFileOptions& ingestion_options) override {
     return Status::NotSupported("Not supported in compacted db mode.");
   }
 

db/compaction.cc

@@ -86,23 +86,35 @@ void Compaction::GetBoundaryKeys(
 bool Compaction::IsBottommostLevel(
     int output_level, VersionStorageInfo* vstorage,
     const std::vector<CompactionInputFiles>& inputs) {
-  int output_l0_idx;
-  if (output_level == 0) {
-    output_l0_idx = 0;
-    for (const auto* file : vstorage->LevelFiles(0)) {
-      if (inputs[0].files.back() == file) {
-        break;
-      }
-      ++output_l0_idx;
-    }
-    assert(static_cast<size_t>(output_l0_idx) < vstorage->LevelFiles(0).size());
-  } else {
-    output_l0_idx = -1;
+  if (inputs[0].level == 0 &&
+      inputs[0].files.back() != vstorage->LevelFiles(0).back()) {
+    return false;
   }
+
   Slice smallest_key, largest_key;
   GetBoundaryKeys(vstorage, inputs, &smallest_key, &largest_key);
-  return !vstorage->RangeMightExistAfterSortedRun(smallest_key, largest_key,
-                                                  output_level, output_l0_idx);
+
+  // Checks whether there are files living beyond the output_level.
+  // If lower levels have files, it checks for overlap between files
+  // if the compaction process and those files.
+  // Bottomlevel optimizations can be made if there are no files in
+  // lower levels or if there is no overlap with the files in
+  // the lower levels.
+  for (int i = output_level + 1; i < vstorage->num_levels(); i++) {
+    // It is not the bottommost level if there are files in higher
+    // levels when the output level is 0 or if there are files in
+    // higher levels which overlap with files to be compacted.
+    // output_level == 0 means that we want it to be considered
+    // s the bottommost level only if the last file on the level
+    // is a part of the files to be compacted - this is verified by
+    // the first if condition in this function
+    if (vstorage->NumLevelFiles(i) > 0 &&
+        (output_level == 0 ||
+         vstorage->OverlapInLevel(i, &smallest_key, &largest_key))) {
+      return false;
+    }
+  }
+  return true;
 }
 
 // test function to validate the functionality of IsBottommostLevel()
@@ -134,7 +146,6 @@ Compaction::Compaction(VersionStorageInfo* vstorage,
                        int _output_level, uint64_t _target_file_size,
                        uint64_t _max_compaction_bytes, uint32_t _output_path_id,
                        CompressionType _compression,
-                       uint32_t _max_subcompactions,
                        std::vector<FileMetaData*> _grandparents,
                        bool _manual_compaction, double _score,
                        bool _deletion_compaction,
@@ -144,7 +155,6 @@ Compaction::Compaction(VersionStorageInfo* vstorage,
       output_level_(_output_level),
       max_output_file_size_(_target_file_size),
       max_compaction_bytes_(_max_compaction_bytes),
-      max_subcompactions_(_max_subcompactions),
       immutable_cf_options_(_immutable_cf_options),
       mutable_cf_options_(_mutable_cf_options),
       input_version_(nullptr),
@@ -165,9 +175,6 @@ Compaction::Compaction(VersionStorageInfo* vstorage,
   if (is_manual_compaction_) {
     compaction_reason_ = CompactionReason::kManualCompaction;
   }
-  if (max_subcompactions_ == 0) {
-    max_subcompactions_ = immutable_cf_options_.max_subcompactions;
-  }
 
 #ifndef NDEBUG
   for (size_t i = 1; i < inputs_.size(); ++i) {
@@ -234,7 +241,7 @@ bool Compaction::IsTrivialMove() const {
 
   // Used in universal compaction, where trivial move can be done if the
   // input files are non overlapping
-  if ((mutable_cf_options_.compaction_options_universal.allow_trivial_move) &&
+  if ((immutable_cf_options_.compaction_options_universal.allow_trivial_move) &&
       (output_level_ != 0)) {
     return is_trivial_move_;
   }
@@ -277,10 +284,10 @@ bool Compaction::KeyNotExistsBeyondOutputLevel(
   assert(input_version_ != nullptr);
   assert(level_ptrs != nullptr);
   assert(level_ptrs->size() == static_cast<size_t>(number_levels_));
-  if (bottommost_level_) {
-    return true;
-  } else if (output_level_ != 0 &&
-             cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
+  if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
+    if (output_level_ == 0) {
+      return false;
+    }
     // Maybe use binary search to find right entry instead of linear search?
     const Comparator* user_cmp = cfd_->user_comparator();
     for (int lvl = output_level_ + 1; lvl < number_levels_; lvl++) {
@@ -300,8 +307,9 @@ bool Compaction::KeyNotExistsBeyondOutputLevel(
       }
     }
     return true;
+  } else {
+    return bottommost_level_;
   }
-  return false;
 }
 
 // Mark (or clear) each file that is being compacted
@@ -410,23 +418,20 @@ void Compaction::Summary(char* output, int len) {
 uint64_t Compaction::OutputFilePreallocationSize() const {
   uint64_t preallocation_size = 0;
 
-  for (const auto& level_files : inputs_) {
-    for (const auto& file : level_files.files) {
-      preallocation_size += file->fd.GetFileSize();
+  if (max_output_file_size_ != port::kMaxUint64 &&
+      (cfd_->ioptions()->compaction_style == kCompactionStyleLevel ||
+       output_level() > 0)) {
+    preallocation_size = max_output_file_size_;
+  } else {
+    for (const auto& level_files : inputs_) {
+      for (const auto& file : level_files.files) {
+        preallocation_size += file->fd.GetFileSize();
+      }
     }
   }
-
-  if (max_output_file_size_ != port::kMaxUint64 &&
-      (immutable_cf_options_.compaction_style == kCompactionStyleLevel ||
-       output_level() > 0)) {
-    preallocation_size = std::min(max_output_file_size_, preallocation_size);
-  }
-
   // Over-estimate slightly so we don't end up just barely crossing
   // the threshold
-  // No point to prellocate more than 1GB.
-  return std::min(uint64_t{1073741824},
-                  preallocation_size + (preallocation_size / 10));
+  return preallocation_size + (preallocation_size / 10);
 }
 
 std::unique_ptr<CompactionFilter> Compaction::CreateCompactionFilter() const {
@@ -447,12 +452,11 @@ bool Compaction::IsOutputLevelEmpty() const {
 }
 
 bool Compaction::ShouldFormSubcompactions() const {
-  if (max_subcompactions_ <= 1 || cfd_ == nullptr) {
+  if (immutable_cf_options_.max_subcompactions <= 1 || cfd_ == nullptr) {
     return false;
   }
   if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
-    return (start_level_ == 0 || is_manual_compaction_) && output_level_ > 0 &&
-           !IsOutputLevelEmpty();
+    return start_level_ == 0 && output_level_ > 0 && !IsOutputLevelEmpty();
   } else if (cfd_->ioptions()->compaction_style == kCompactionStyleUniversal) {
     return number_levels_ > 1 && output_level_ > 0;
   } else {
@@ -473,8 +477,4 @@ uint64_t Compaction::MaxInputFileCreationTime() const {
   return max_creation_time;
 }
 
-int Compaction::GetInputBaseLevel() const {
-  return input_vstorage_->base_level();
-}
-
 }  // namespace rocksdb

db/compaction.h

@@ -40,7 +40,6 @@ class Compaction {
              std::vector<CompactionInputFiles> inputs, int output_level,
              uint64_t target_file_size, uint64_t max_compaction_bytes,
              uint32_t output_path_id, CompressionType compression,
-             uint32_t max_subcompactions,
              std::vector<FileMetaData*> grandparents,
              bool manual_compaction = false, double score = -1,
              bool deletion_compaction = false,
@@ -234,8 +233,6 @@ class Compaction {
 
   Slice GetLargestUserKey() const { return largest_user_key_; }
 
-  int GetInputBaseLevel() const;
-
   CompactionReason compaction_reason() { return compaction_reason_; }
 
   const std::vector<FileMetaData*>& grandparents() const {
@@ -244,8 +241,6 @@ class Compaction {
 
   uint64_t max_compaction_bytes() const { return max_compaction_bytes_; }
 
-  uint32_t max_subcompactions() const { return max_subcompactions_; }
-
   uint64_t MaxInputFileCreationTime() const;
 
  private:
@@ -272,7 +267,6 @@ class Compaction {
   const int output_level_;  // levels to which output files are stored
   uint64_t max_output_file_size_;
   uint64_t max_compaction_bytes_;
-  uint32_t max_subcompactions_;
   const ImmutableCFOptions immutable_cf_options_;
   const MutableCFOptions mutable_cf_options_;
   Version* input_version_;

db/compaction_iterator.cc

@@ -4,59 +4,77 @@
 //  (found in the LICENSE.Apache file in the root directory).
 
 #include "db/compaction_iterator.h"
-
-#include "db/snapshot_checker.h"
-#include "port/likely.h"
 #include "rocksdb/listener.h"
 #include "table/internal_iterator.h"
 
 namespace rocksdb {
 
-CompactionIterator::CompactionIterator(
-    InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
-    SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
-    SequenceNumber earliest_write_conflict_snapshot,
-    const SnapshotChecker* snapshot_checker, Env* env,
-    bool expect_valid_internal_key, RangeDelAggregator* range_del_agg,
-    const Compaction* compaction, const CompactionFilter* compaction_filter,
-    const std::atomic<bool>* shutting_down,
-    const SequenceNumber preserve_deletes_seqnum)
-    : CompactionIterator(
-          input, cmp, merge_helper, last_sequence, snapshots,
-          earliest_write_conflict_snapshot, snapshot_checker, env,
-          expect_valid_internal_key, range_del_agg,
-          std::unique_ptr<CompactionProxy>(
-              compaction ? new CompactionProxy(compaction) : nullptr),
-          compaction_filter, shutting_down, preserve_deletes_seqnum) {}
+#ifndef ROCKSDB_LITE
+CompactionEventListener::CompactionListenerValueType fromInternalValueType(
+    ValueType vt) {
+  switch (vt) {
+    case kTypeDeletion:
+      return CompactionEventListener::CompactionListenerValueType::kDelete;
+    case kTypeValue:
+      return CompactionEventListener::CompactionListenerValueType::kValue;
+    case kTypeMerge:
+      return CompactionEventListener::CompactionListenerValueType::
+          kMergeOperand;
+    case kTypeSingleDeletion:
+      return CompactionEventListener::CompactionListenerValueType::
+          kSingleDelete;
+    case kTypeRangeDeletion:
+      return CompactionEventListener::CompactionListenerValueType::kRangeDelete;
+    case kTypeBlobIndex:
+      return CompactionEventListener::CompactionListenerValueType::kBlobIndex;
+    default:
+      assert(false);
+      return CompactionEventListener::CompactionListenerValueType::kInvalid;
+  }
+}
+#endif  // ROCKSDB_LITE
 
 CompactionIterator::CompactionIterator(
     InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
-    SequenceNumber /*last_sequence*/, std::vector<SequenceNumber>* snapshots,
-    SequenceNumber earliest_write_conflict_snapshot,
-    const SnapshotChecker* snapshot_checker, Env* env,
+    SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
+    SequenceNumber earliest_write_conflict_snapshot, Env* env,
+    bool expect_valid_internal_key, RangeDelAggregator* range_del_agg,
+    const Compaction* compaction, const CompactionFilter* compaction_filter,
+    CompactionEventListener* compaction_listener,
+    const std::atomic<bool>* shutting_down)
+    : CompactionIterator(
+          input, cmp, merge_helper, last_sequence, snapshots,
+          earliest_write_conflict_snapshot, env, expect_valid_internal_key,
+          range_del_agg,
+          std::unique_ptr<CompactionProxy>(
+              compaction ? new CompactionProxy(compaction) : nullptr),
+          compaction_filter, compaction_listener, shutting_down) {}
+
+CompactionIterator::CompactionIterator(
+    InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
+    SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
+    SequenceNumber earliest_write_conflict_snapshot, Env* env,
     bool expect_valid_internal_key, RangeDelAggregator* range_del_agg,
     std::unique_ptr<CompactionProxy> compaction,
     const CompactionFilter* compaction_filter,
-    const std::atomic<bool>* shutting_down,
-    const SequenceNumber preserve_deletes_seqnum)
+    CompactionEventListener* compaction_listener,
+    const std::atomic<bool>* shutting_down)
     : input_(input),
       cmp_(cmp),
       merge_helper_(merge_helper),
       snapshots_(snapshots),
       earliest_write_conflict_snapshot_(earliest_write_conflict_snapshot),
-      snapshot_checker_(snapshot_checker),
       env_(env),
       expect_valid_internal_key_(expect_valid_internal_key),
       range_del_agg_(range_del_agg),
       compaction_(std::move(compaction)),
       compaction_filter_(compaction_filter),
+#ifndef ROCKSDB_LITE
+      compaction_listener_(compaction_listener),
+#endif  // ROCKSDB_LITE
       shutting_down_(shutting_down),
-      preserve_deletes_seqnum_(preserve_deletes_seqnum),
       ignore_snapshots_(false),
-      current_user_key_sequence_(0),
-      current_user_key_snapshot_(0),
-      merge_out_iter_(merge_helper_),
-      current_key_committed_(false) {
+      merge_out_iter_(merge_helper_) {
   assert(compaction_filter_ == nullptr || compaction_ != nullptr);
   bottommost_level_ =
       compaction_ == nullptr ? false : compaction_->bottommost_level();
@@ -113,8 +131,8 @@ void CompactionIterator::Next() {
     if (merge_out_iter_.Valid()) {
       key_ = merge_out_iter_.key();
       value_ = merge_out_iter_.value();
-      bool valid_key __attribute__((__unused__));
-      valid_key =  ParseInternalKey(key_, &ikey_);
+      bool valid_key __attribute__((__unused__)) =
+          ParseInternalKey(key_, &ikey_);
       // MergeUntil stops when it encounters a corrupt key and does not
       // include them in the result, so we expect the keys here to be valid.
       assert(valid_key);
@@ -148,64 +166,6 @@ void CompactionIterator::Next() {
   PrepareOutput();
 }
 
-void CompactionIterator::InvokeFilterIfNeeded(bool* need_skip,
-                                              Slice* skip_until) {
-  if (compaction_filter_ != nullptr &&
-      (ikey_.type == kTypeValue || ikey_.type == kTypeBlobIndex) &&
-      (visible_at_tip_ || ignore_snapshots_ ||
-       ikey_.sequence > latest_snapshot_ ||
-       (snapshot_checker_ != nullptr &&
-        UNLIKELY(!snapshot_checker_->IsInSnapshot(ikey_.sequence,
-                                                  latest_snapshot_))))) {
-    // If the user has specified a compaction filter and the sequence
-    // number is greater than any external snapshot, then invoke the
-    // filter. If the return value of the compaction filter is true,
-    // replace the entry with a deletion marker.
-    CompactionFilter::Decision filter;
-    compaction_filter_value_.clear();
-    compaction_filter_skip_until_.Clear();
-    CompactionFilter::ValueType value_type =
-        ikey_.type == kTypeValue ? CompactionFilter::ValueType::kValue
-                                 : CompactionFilter::ValueType::kBlobIndex;
-    // Hack: pass internal key to BlobIndexCompactionFilter since it needs
-    // to get sequence number.
-    Slice& filter_key = ikey_.type == kTypeValue ? ikey_.user_key : key_;
-    {
-      StopWatchNano timer(env_, true);
-      filter = compaction_filter_->FilterV2(
-          compaction_->level(), filter_key, value_type, value_,
-          &compaction_filter_value_, compaction_filter_skip_until_.rep());
-      iter_stats_.total_filter_time +=
-          env_ != nullptr ? timer.ElapsedNanos() : 0;
-    }
-
-    if (filter == CompactionFilter::Decision::kRemoveAndSkipUntil &&
-        cmp_->Compare(*compaction_filter_skip_until_.rep(), ikey_.user_key) <=
-            0) {
-      // Can't skip to a key smaller than the current one.
-      // Keep the key as per FilterV2 documentation.
-      filter = CompactionFilter::Decision::kKeep;
-    }
-
-    if (filter == CompactionFilter::Decision::kRemove) {
-      // convert the current key to a delete; key_ is pointing into
-      // current_key_ at this point, so updating current_key_ updates key()
-      ikey_.type = kTypeDeletion;
-      current_key_.UpdateInternalKey(ikey_.sequence, kTypeDeletion);
-      // no value associated with delete
-      value_.clear();
-      iter_stats_.num_record_drop_user++;
-    } else if (filter == CompactionFilter::Decision::kChangeValue) {
-      value_ = compaction_filter_value_;
-    } else if (filter == CompactionFilter::Decision::kRemoveAndSkipUntil) {
-      *need_skip = true;
-      compaction_filter_skip_until_.ConvertFromUserKey(kMaxSequenceNumber,
-                                                       kValueTypeForSeek);
-      *skip_until = compaction_filter_skip_until_.Encode();
-    }
-  }
-}
-
 void CompactionIterator::NextFromInput() {
   at_next_ = false;
   valid_ = false;
@@ -260,16 +220,73 @@ void CompactionIterator::NextFromInput() {
       has_outputted_key_ = false;
       current_user_key_sequence_ = kMaxSequenceNumber;
       current_user_key_snapshot_ = 0;
-      current_key_committed_ =
-          (snapshot_checker_ == nullptr ||
-           snapshot_checker_->IsInSnapshot(ikey_.sequence, kMaxSequenceNumber));
 
-      // Apply the compaction filter to the first committed version of the user
-      // key.
-      if (current_key_committed_) {
-        InvokeFilterIfNeeded(&need_skip, &skip_until);
+#ifndef ROCKSDB_LITE
+      if (compaction_listener_) {
+        compaction_listener_->OnCompaction(compaction_->level(), ikey_.user_key,
+                                           fromInternalValueType(ikey_.type),
+                                           value_, ikey_.sequence, true);
+      }
+#endif  // ROCKSDB_LITE
+
+      // apply the compaction filter to the first occurrence of the user key
+      if (compaction_filter_ != nullptr && 
+          (ikey_.type == kTypeValue || ikey_.type == kTypeBlobIndex) &&
+          (visible_at_tip_ || ikey_.sequence > latest_snapshot_ ||
+           ignore_snapshots_)) {
+        // If the user has specified a compaction filter and the sequence
+        // number is greater than any external snapshot, then invoke the
+        // filter. If the return value of the compaction filter is true,
+        // replace the entry with a deletion marker.
+        CompactionFilter::Decision filter;
+        compaction_filter_value_.clear();
+        compaction_filter_skip_until_.Clear();
+        CompactionFilter::ValueType value_type =
+            ikey_.type == kTypeValue ? CompactionFilter::ValueType::kValue
+                                     : CompactionFilter::ValueType::kBlobIndex;
+        {
+          StopWatchNano timer(env_, true);
+          filter = compaction_filter_->FilterV2(
+              compaction_->level(), ikey_.user_key, value_type, value_,
+              &compaction_filter_value_, compaction_filter_skip_until_.rep());
+          iter_stats_.total_filter_time +=
+              env_ != nullptr ? timer.ElapsedNanos() : 0;
+        }
+
+        if (filter == CompactionFilter::Decision::kRemoveAndSkipUntil &&
+            cmp_->Compare(*compaction_filter_skip_until_.rep(),
+                          ikey_.user_key) <= 0) {
+          // Can't skip to a key smaller than the current one.
+          // Keep the key as per FilterV2 documentation.
+          filter = CompactionFilter::Decision::kKeep;
+        }
+
+        if (filter == CompactionFilter::Decision::kRemove) {
+          // convert the current key to a delete; key_ is pointing into
+          // current_key_ at this point, so updating current_key_ updates key()
+          ikey_.type = kTypeDeletion;
+          current_key_.UpdateInternalKey(ikey_.sequence, kTypeDeletion);
+          // no value associated with delete
+          value_.clear();
+          iter_stats_.num_record_drop_user++;
+        } else if (filter == CompactionFilter::Decision::kChangeValue) {
+          value_ = compaction_filter_value_;
+        } else if (filter == CompactionFilter::Decision::kRemoveAndSkipUntil) {
+          need_skip = true;
+          compaction_filter_skip_until_.ConvertFromUserKey(kMaxSequenceNumber,
+                                                           kValueTypeForSeek);
+          skip_until = compaction_filter_skip_until_.Encode();
+        }
       }
     } else {
+#ifndef ROCKSDB_LITE
+      if (compaction_listener_) {
+        compaction_listener_->OnCompaction(compaction_->level(), ikey_.user_key,
+                                           fromInternalValueType(ikey_.type),
+                                           value_, ikey_.sequence, false);
+      }
+#endif  // ROCKSDB_LITE
+
       // Update the current key to reflect the new sequence number/type without
       // copying the user key.
       // TODO(rven): Compaction filter does not process keys in this path
@@ -278,33 +295,13 @@ void CompactionIterator::NextFromInput() {
       current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type);
       key_ = current_key_.GetInternalKey();
       ikey_.user_key = current_key_.GetUserKey();
-
-      // Note that newer version of a key is ordered before older versions. If a
-      // newer version of a key is committed, so as the older version. No need
-      // to query snapshot_checker_ in that case.
-      if (UNLIKELY(!current_key_committed_)) {
-        assert(snapshot_checker_ != nullptr);
-        current_key_committed_ =
-            snapshot_checker_->IsInSnapshot(ikey_.sequence, kMaxSequenceNumber);
-        // Apply the compaction filter to the first committed version of the
-        // user key.
-        if (current_key_committed_) {
-          InvokeFilterIfNeeded(&need_skip, &skip_until);
-        }
-      }
-    }
-
-    if (UNLIKELY(!current_key_committed_)) {
-      assert(snapshot_checker_ != nullptr);
-      valid_ = true;
-      break;
     }
 
     // If there are no snapshots, then this kv affect visibility at tip.
     // Otherwise, search though all existing snapshots to find the earliest
     // snapshot that is affected by this kv.
-    SequenceNumber last_sequence __attribute__((__unused__));
-    last_sequence = current_user_key_sequence_;
+    SequenceNumber last_sequence __attribute__((__unused__)) =
+        current_user_key_sequence_;
     current_user_key_sequence_ = ikey_.sequence;
     SequenceNumber last_snapshot = current_user_key_snapshot_;
     SequenceNumber prev_snapshot = 0;  // 0 means no previous snapshot
@@ -369,10 +366,7 @@ void CompactionIterator::NextFromInput() {
           cmp_->Equal(ikey_.user_key, next_ikey.user_key)) {
         // Check whether the next key belongs to the same snapshot as the
         // SingleDelete.
-        if (prev_snapshot == 0 || next_ikey.sequence > prev_snapshot ||
-            (snapshot_checker_ != nullptr &&
-             UNLIKELY(!snapshot_checker_->IsInSnapshot(next_ikey.sequence,
-                                                       prev_snapshot)))) {
+        if (prev_snapshot == 0 || next_ikey.sequence > prev_snapshot) {
           if (next_ikey.type == kTypeSingleDeletion) {
             // We encountered two SingleDeletes in a row.  This could be due to
             // unexpected user input.
@@ -383,12 +377,8 @@ void CompactionIterator::NextFromInput() {
             // input_->Next().
             ++iter_stats_.num_record_drop_obsolete;
             ++iter_stats_.num_single_del_mismatch;
-          } else if (has_outputted_key_ ||
-                     (ikey_.sequence <= earliest_write_conflict_snapshot_ &&
-                      (snapshot_checker_ == nullptr ||
-                       LIKELY(snapshot_checker_->IsInSnapshot(
-                           ikey_.sequence,
-                           earliest_write_conflict_snapshot_))))) {
+          } else if ((ikey_.sequence <= earliest_write_conflict_snapshot_) ||
+                     has_outputted_key_) {
             // Found a matching value, we can drop the single delete and the
             // value.  It is safe to drop both records since we've already
             // outputted a key in this snapshot, or there is no earlier
@@ -436,9 +426,6 @@ void CompactionIterator::NextFromInput() {
         // comparison, so the value of has_current_user_key does not matter.
         has_current_user_key_ = false;
         if (compaction_ != nullptr && ikey_.sequence <= earliest_snapshot_ &&
-            (snapshot_checker_ == nullptr ||
-             LIKELY(snapshot_checker_->IsInSnapshot(ikey_.sequence,
-                                                    earliest_snapshot_))) &&
             compaction_->KeyNotExistsBeyondOutputLevel(ikey_.user_key,
                                                        &level_ptrs_)) {
           // Key doesn't exist outside of this range.
@@ -472,10 +459,6 @@ void CompactionIterator::NextFromInput() {
       input_->Next();
     } else if (compaction_ != nullptr && ikey_.type == kTypeDeletion &&
                ikey_.sequence <= earliest_snapshot_ &&
-               (snapshot_checker_ == nullptr ||
-                LIKELY(snapshot_checker_->IsInSnapshot(ikey_.sequence,
-                                                       earliest_snapshot_))) &&
-               ikeyNotNeededForIncrementalSnapshot() &&
                compaction_->KeyNotExistsBeyondOutputLevel(ikey_.user_key,
                                                           &level_ptrs_)) {
       // TODO(noetzli): This is the only place where we use compaction_
@@ -492,11 +475,6 @@ void CompactionIterator::NextFromInput() {
       //
       // Note:  Dropping this Delete will not affect TransactionDB
       // write-conflict checking since it is earlier than any snapshot.
-      //
-      // It seems that we can also drop deletion later than earliest snapshot
-      // given that:
-      // (1) The deletion is earlier than earliest_write_conflict_snapshot, and
-      // (2) No value exist earlier than the deletion.
       ++iter_stats_.num_record_drop_obsolete;
       if (!bottommost_level_) {
         ++iter_stats_.num_optimized_del_drop_obsolete;
@@ -526,8 +504,8 @@ void CompactionIterator::NextFromInput() {
         //       These will be correctly set below.
         key_ = merge_out_iter_.key();
         value_ = merge_out_iter_.value();
-        bool valid_key __attribute__((__unused__));
-        valid_key = ParseInternalKey(key_, &ikey_);
+        bool valid_key __attribute__((__unused__)) =
+            ParseInternalKey(key_, &ikey_);
         // MergeUntil stops when it encounters a corrupt key and does not
         // include them in the result, so we expect the keys here to valid.
         assert(valid_key);
@@ -577,18 +555,11 @@ void CompactionIterator::PrepareOutput() {
   // and the earliest snapshot is larger than this seqno
   // and the userkey differs from the last userkey in compaction
   // then we can squash the seqno to zero.
-  //
+
   // This is safe for TransactionDB write-conflict checking since transactions
   // only care about sequence number larger than any active snapshots.
-  //
-  // Can we do the same for levels above bottom level as long as
-  // KeyNotExistsBeyondOutputLevel() return true?
-  if ((compaction_ != nullptr &&
-      !compaction_->allow_ingest_behind()) &&
-      ikeyNotNeededForIncrementalSnapshot() &&
+  if ((compaction_ != nullptr && !compaction_->allow_ingest_behind()) &&
       bottommost_level_ && valid_ && ikey_.sequence <= earliest_snapshot_ &&
-      (snapshot_checker_ == nullptr || LIKELY(snapshot_checker_->IsInSnapshot(
-        ikey_.sequence, earliest_snapshot_))) &&
       ikey_.type != kTypeMerge &&
       !cmp_->Equal(compaction_->GetLargestUserKey(), ikey_.user_key)) {
     assert(ikey_.type != kTypeDeletion && ikey_.type != kTypeSingleDeletion);
@@ -600,11 +571,10 @@ void CompactionIterator::PrepareOutput() {
 inline SequenceNumber CompactionIterator::findEarliestVisibleSnapshot(
     SequenceNumber in, SequenceNumber* prev_snapshot) {
   assert(snapshots_->size());
-  SequenceNumber prev = kMaxSequenceNumber;
+  SequenceNumber prev __attribute__((__unused__)) = kMaxSequenceNumber;
   for (const auto cur : *snapshots_) {
     assert(prev == kMaxSequenceNumber || prev <= cur);
-    if (cur >= in && (snapshot_checker_ == nullptr ||
-                      snapshot_checker_->IsInSnapshot(in, cur))) {
+    if (cur >= in) {
       *prev_snapshot = prev == kMaxSequenceNumber ? 0 : prev;
       return cur;
     }
@@ -615,11 +585,4 @@ inline SequenceNumber CompactionIterator::findEarliestVisibleSnapshot(
   return kMaxSequenceNumber;
 }
 
-// used in 2 places - prevents deletion markers to be dropped if they may be
-// needed and disables seqnum zero-out in PrepareOutput for recent keys.
-inline bool CompactionIterator::ikeyNotNeededForIncrementalSnapshot() {
-  return (!compaction_->preserve_deletes()) ||
-         (ikey_.sequence < preserve_deletes_seqnum_);
-}
-
 }  // namespace rocksdb

db/compaction_iterator.h

@@ -14,12 +14,13 @@
 #include "db/merge_helper.h"
 #include "db/pinned_iterators_manager.h"
 #include "db/range_del_aggregator.h"
-#include "db/snapshot_checker.h"
 #include "options/cf_options.h"
 #include "rocksdb/compaction_filter.h"
 
 namespace rocksdb {
 
+class CompactionEventListener;
+
 class CompactionIterator {
  public:
   // A wrapper around Compaction. Has a much smaller interface, only what
@@ -30,7 +31,7 @@ class CompactionIterator {
         : compaction_(compaction) {}
 
     virtual ~CompactionProxy() = default;
-    virtual int level(size_t /*compaction_input_level*/ = 0) const {
+    virtual int level(size_t compaction_input_level = 0) const {
       return compaction_->level();
     }
     virtual bool KeyNotExistsBeyondOutputLevel(
@@ -47,9 +48,6 @@ class CompactionIterator {
     virtual bool allow_ingest_behind() const {
       return compaction_->immutable_cf_options()->allow_ingest_behind;
     }
-    virtual bool preserve_deletes() const {
-      return compaction_->immutable_cf_options()->preserve_deletes;
-    }
 
    protected:
     CompactionProxy() = default;
@@ -61,27 +59,25 @@ class CompactionIterator {
   CompactionIterator(InternalIterator* input, const Comparator* cmp,
                      MergeHelper* merge_helper, SequenceNumber last_sequence,
                      std::vector<SequenceNumber>* snapshots,
-                     SequenceNumber earliest_write_conflict_snapshot,
-                     const SnapshotChecker* snapshot_checker, Env* env,
+                     SequenceNumber earliest_write_conflict_snapshot, Env* env,
                      bool expect_valid_internal_key,
                      RangeDelAggregator* range_del_agg,
                      const Compaction* compaction = nullptr,
                      const CompactionFilter* compaction_filter = nullptr,
-                     const std::atomic<bool>* shutting_down = nullptr,
-                     const SequenceNumber preserve_deletes_seqnum = 0);
+                     CompactionEventListener* compaction_listener = nullptr,
+                     const std::atomic<bool>* shutting_down = nullptr);
 
   // Constructor with custom CompactionProxy, used for tests.
   CompactionIterator(InternalIterator* input, const Comparator* cmp,
                      MergeHelper* merge_helper, SequenceNumber last_sequence,
                      std::vector<SequenceNumber>* snapshots,
-                     SequenceNumber earliest_write_conflict_snapshot,
-                     const SnapshotChecker* snapshot_checker, Env* env,
+                     SequenceNumber earliest_write_conflict_snapshot, Env* env,
                      bool expect_valid_internal_key,
                      RangeDelAggregator* range_del_agg,
                      std::unique_ptr<CompactionProxy> compaction,
                      const CompactionFilter* compaction_filter = nullptr,
-                     const std::atomic<bool>* shutting_down = nullptr,
-                     const SequenceNumber preserve_deletes_seqnum = 0);
+                     CompactionEventListener* compaction_listener = nullptr,
+                     const std::atomic<bool>* shutting_down = nullptr);
 
   ~CompactionIterator();
 
@@ -115,9 +111,6 @@ class CompactionIterator {
   // compression.
   void PrepareOutput();
 
-  // Invoke compaction filter if needed.
-  void InvokeFilterIfNeeded(bool* need_skip, Slice* skip_until);
-
   // Given a sequence number, return the sequence number of the
   // earliest snapshot that this sequence number is visible in.
   // The snapshots themselves are arranged in ascending order of
@@ -127,24 +120,20 @@ class CompactionIterator {
   inline SequenceNumber findEarliestVisibleSnapshot(
       SequenceNumber in, SequenceNumber* prev_snapshot);
 
-  // Checks whether the currently seen ikey_ is needed for
-  // incremental (differential) snapshot and hence can't be dropped
-  // or seqnum be zero-ed out even if all other conditions for it are met.
-  inline bool ikeyNotNeededForIncrementalSnapshot();
-
   InternalIterator* input_;
   const Comparator* cmp_;
   MergeHelper* merge_helper_;
   const std::vector<SequenceNumber>* snapshots_;
   const SequenceNumber earliest_write_conflict_snapshot_;
-  const SnapshotChecker* const snapshot_checker_;
   Env* env_;
   bool expect_valid_internal_key_;
   RangeDelAggregator* range_del_agg_;
   std::unique_ptr<CompactionProxy> compaction_;
   const CompactionFilter* compaction_filter_;
+#ifndef ROCKSDB_LITE
+  CompactionEventListener* compaction_listener_;
+#endif  // ROCKSDB_LITE
   const std::atomic<bool>* shutting_down_;
-  const SequenceNumber preserve_deletes_seqnum_;
   bool bottommost_level_;
   bool valid_ = false;
   bool visible_at_tip_;
@@ -200,10 +189,6 @@ class CompactionIterator {
   std::vector<size_t> level_ptrs_;
   CompactionIterationStats iter_stats_;
 
-  // Used to avoid purging uncommitted values. The application can specify
-  // uncommitted values by providing a SnapshotChecker object.
-  bool current_key_committed_;
-
   bool IsShuttingDown() {
     // This is a best-effort facility, so memory_order_relaxed is sufficient.
     return shutting_down_ && shutting_down_->load(std::memory_order_relaxed);

db/compaction_iterator_test.cc

@@ -9,25 +9,23 @@
 #include <vector>
 
 #include "port/port.h"
-#include "util/string_util.h"
 #include "util/testharness.h"
 #include "util/testutil.h"
-#include "utilities/merge_operators.h"
 
 namespace rocksdb {
 
 // Expects no merging attempts.
 class NoMergingMergeOp : public MergeOperator {
  public:
-  bool FullMergeV2(const MergeOperationInput& /*merge_in*/,
-                   MergeOperationOutput* /*merge_out*/) const override {
+  bool FullMergeV2(const MergeOperationInput& merge_in,
+                   MergeOperationOutput* merge_out) const override {
     ADD_FAILURE();
     return false;
   }
-  bool PartialMergeMulti(const Slice& /*key*/,
-                         const std::deque<Slice>& /*operand_list*/,
-                         std::string* /*new_value*/,
-                         Logger* /*logger*/) const override {
+  bool PartialMergeMulti(const Slice& key,
+                         const std::deque<Slice>& operand_list,
+                         std::string* new_value,
+                         Logger* logger) const override {
     ADD_FAILURE();
     return false;
   }
@@ -41,9 +39,9 @@ class NoMergingMergeOp : public MergeOperator {
 // Always returns Decition::kRemove.
 class StallingFilter : public CompactionFilter {
  public:
-  Decision FilterV2(int /*level*/, const Slice& key, ValueType /*type*/,
-                    const Slice& /*existing_value*/, std::string* /*new_value*/,
-                    std::string* /*skip_until*/) const override {
+  virtual Decision FilterV2(int level, const Slice& key, ValueType t,
+                            const Slice& existing_value, std::string* new_value,
+                            std::string* skip_until) const override {
     int k = std::atoi(key.ToString().c_str());
     last_seen.store(k);
     while (k >= stall_at.load()) {
@@ -74,18 +72,6 @@ class StallingFilter : public CompactionFilter {
   mutable std::atomic<int> last_seen{0};
 };
 
-// Compaction filter that filter out all keys.
-class FilterAllKeysCompactionFilter : public CompactionFilter {
- public:
-  Decision FilterV2(int /*level*/, const Slice& /*key*/, ValueType /*type*/,
-                    const Slice& /*existing_value*/, std::string* /*new_value*/,
-                    std::string* /*skip_until*/) const override {
-    return Decision::kRemove;
-  }
-
-  const char* Name() const override { return "AllKeysCompactionFilter"; }
-};
-
 class LoggingForwardVectorIterator : public InternalIterator {
  public:
   struct Action {
@@ -126,7 +112,7 @@ class LoggingForwardVectorIterator : public InternalIterator {
                keys_.begin();
   }
 
-  virtual void SeekForPrev(const Slice& /*target*/) override { assert(false); }
+  virtual void SeekForPrev(const Slice& target) override { assert(false); }
 
   virtual void Next() override {
     assert(Valid());
@@ -158,149 +144,71 @@ class FakeCompaction : public CompactionIterator::CompactionProxy {
  public:
   FakeCompaction() = default;
 
-  virtual int level(size_t /*compaction_input_level*/) const override {
-    return 0;
-  }
+  virtual int level(size_t compaction_input_level) const { return 0; }
   virtual bool KeyNotExistsBeyondOutputLevel(
-      const Slice& /*user_key*/,
-      std::vector<size_t>* /*level_ptrs*/) const override {
-    return is_bottommost_level || key_not_exists_beyond_output_level;
+      const Slice& user_key, std::vector<size_t>* level_ptrs) const {
+    return key_not_exists_beyond_output_level;
   }
-  virtual bool bottommost_level() const override { return is_bottommost_level; }
-  virtual int number_levels() const override { return 1; }
-  virtual Slice GetLargestUserKey() const override {
+  virtual bool bottommost_level() const { return false; }
+  virtual int number_levels() const { return 1; }
+  virtual Slice GetLargestUserKey() const {
     return "\xff\xff\xff\xff\xff\xff\xff\xff\xff";
   }
-  virtual bool allow_ingest_behind() const override { return false; }
-
-  virtual bool preserve_deletes() const override { return false; }
+  virtual bool allow_ingest_behind() const { return false; }
 
   bool key_not_exists_beyond_output_level = false;
-
-  bool is_bottommost_level = false;
 };
 
-// A simplifed snapshot checker which assumes each snapshot has a global
-// last visible sequence.
-class TestSnapshotChecker : public SnapshotChecker {
- public:
-  explicit TestSnapshotChecker(
-      SequenceNumber last_committed_sequence,
-      const std::unordered_map<SequenceNumber, SequenceNumber>& snapshots = {})
-      : last_committed_sequence_(last_committed_sequence),
-        snapshots_(snapshots) {}
-
-  bool IsInSnapshot(SequenceNumber seq,
-                    SequenceNumber snapshot_seq) const override {
-    if (snapshot_seq == kMaxSequenceNumber) {
-      return seq <= last_committed_sequence_;
-    }
-    assert(snapshots_.count(snapshot_seq) > 0);
-    return seq <= snapshots_.at(snapshot_seq);
-  }
-
- private:
-  SequenceNumber last_committed_sequence_;
-  // A map of valid snapshot to last visible sequence to the snapshot.
-  std::unordered_map<SequenceNumber, SequenceNumber> snapshots_;
-};
-
-// Test param:
-//   bool: whether to pass snapshot_checker to compaction iterator.
-class CompactionIteratorTest : public testing::TestWithParam<bool> {
+class CompactionIteratorTest : public testing::Test {
  public:
   CompactionIteratorTest()
       : cmp_(BytewiseComparator()), icmp_(cmp_), snapshots_({}) {}
 
-  void InitIterators(
-      const std::vector<std::string>& ks, const std::vector<std::string>& vs,
-      const std::vector<std::string>& range_del_ks,
-      const std::vector<std::string>& range_del_vs,
-      SequenceNumber last_sequence,
-      SequenceNumber last_committed_sequence = kMaxSequenceNumber,
-      MergeOperator* merge_op = nullptr, CompactionFilter* filter = nullptr,
-      bool bottommost_level = false,
-      SequenceNumber earliest_write_conflict_snapshot = kMaxSequenceNumber) {
+  void InitIterators(const std::vector<std::string>& ks,
+                     const std::vector<std::string>& vs,
+                     const std::vector<std::string>& range_del_ks,
+                     const std::vector<std::string>& range_del_vs,
+                     SequenceNumber last_sequence,
+                     MergeOperator* merge_op = nullptr,
+                     CompactionFilter* filter = nullptr) {
     std::unique_ptr<InternalIterator> range_del_iter(
         new test::VectorIterator(range_del_ks, range_del_vs));
     range_del_agg_.reset(new RangeDelAggregator(icmp_, snapshots_));
     ASSERT_OK(range_del_agg_->AddTombstones(std::move(range_del_iter)));
 
     std::unique_ptr<CompactionIterator::CompactionProxy> compaction;
-    if (filter || bottommost_level) {
+    if (filter) {
       compaction_proxy_ = new FakeCompaction();
-      compaction_proxy_->is_bottommost_level = bottommost_level;
       compaction.reset(compaction_proxy_);
     }
-    bool use_snapshot_checker = UseSnapshotChecker() || GetParam();
-    if (use_snapshot_checker || last_committed_sequence < kMaxSequenceNumber) {
-      snapshot_checker_.reset(
-          new TestSnapshotChecker(last_committed_sequence, snapshot_map_));
-    }
-    merge_helper_.reset(
-        new MergeHelper(Env::Default(), cmp_, merge_op, filter, nullptr, false,
-                        0 /*latest_snapshot*/, snapshot_checker_.get(),
-                        0 /*level*/, nullptr /*statistics*/, &shutting_down_));
 
+    merge_helper_.reset(new MergeHelper(Env::Default(), cmp_, merge_op, filter,
+                                        nullptr, false, 0, 0, nullptr,
+                                        &shutting_down_));
     iter_.reset(new LoggingForwardVectorIterator(ks, vs));
     iter_->SeekToFirst();
     c_iter_.reset(new CompactionIterator(
         iter_.get(), cmp_, merge_helper_.get(), last_sequence, &snapshots_,
-        earliest_write_conflict_snapshot, snapshot_checker_.get(),
-        Env::Default(), false, range_del_agg_.get(), std::move(compaction),
-        filter, &shutting_down_));
+        kMaxSequenceNumber, Env::Default(), false, range_del_agg_.get(),
+        std::move(compaction), filter, nullptr, &shutting_down_));
   }
 
-  void AddSnapshot(SequenceNumber snapshot,
-                   SequenceNumber last_visible_seq = kMaxSequenceNumber) {
-    snapshots_.push_back(snapshot);
-    snapshot_map_[snapshot] = last_visible_seq;
-  }
-
-  virtual bool UseSnapshotChecker() const { return false; }
-
-  void RunTest(
-      const std::vector<std::string>& input_keys,
-      const std::vector<std::string>& input_values,
-      const std::vector<std::string>& expected_keys,
-      const std::vector<std::string>& expected_values,
-      SequenceNumber last_committed_seq = kMaxSequenceNumber,
-      MergeOperator* merge_operator = nullptr,
-      CompactionFilter* compaction_filter = nullptr,
-      bool bottommost_level = false,
-      SequenceNumber earliest_write_conflict_snapshot = kMaxSequenceNumber) {
-    InitIterators(input_keys, input_values, {}, {}, kMaxSequenceNumber,
-                  last_committed_seq, merge_operator, compaction_filter,
-                  bottommost_level, earliest_write_conflict_snapshot);
-    c_iter_->SeekToFirst();
-    for (size_t i = 0; i < expected_keys.size(); i++) {
-      std::string info = "i = " + ToString(i);
-      ASSERT_TRUE(c_iter_->Valid()) << info;
-      ASSERT_OK(c_iter_->status()) << info;
-      ASSERT_EQ(expected_keys[i], c_iter_->key().ToString()) << info;
-      ASSERT_EQ(expected_values[i], c_iter_->value().ToString()) << info;
-      c_iter_->Next();
-    }
-    ASSERT_FALSE(c_iter_->Valid());
-  }
+  void AddSnapshot(SequenceNumber snapshot) { snapshots_.push_back(snapshot); }
 
   const Comparator* cmp_;
   const InternalKeyComparator icmp_;
   std::vector<SequenceNumber> snapshots_;
-  // A map of valid snapshot to last visible sequence to the snapshot.
-  std::unordered_map<SequenceNumber, SequenceNumber> snapshot_map_;
   std::unique_ptr<MergeHelper> merge_helper_;
   std::unique_ptr<LoggingForwardVectorIterator> iter_;
   std::unique_ptr<CompactionIterator> c_iter_;
   std::unique_ptr<RangeDelAggregator> range_del_agg_;
-  std::unique_ptr<SnapshotChecker> snapshot_checker_;
   std::atomic<bool> shutting_down_{false};
   FakeCompaction* compaction_proxy_;
 };
 
 // It is possible that the output of the compaction iterator is empty even if
 // the input is not.
-TEST_P(CompactionIteratorTest, EmptyResult) {
+TEST_F(CompactionIteratorTest, EmptyResult) {
   InitIterators({test::KeyStr("a", 5, kTypeSingleDeletion),
                  test::KeyStr("a", 3, kTypeValue)},
                 {"", "val"}, {}, {}, 5);
@@ -310,7 +218,7 @@ TEST_P(CompactionIteratorTest, EmptyResult) {
 
 // If there is a corruption after a single deletion, the corrupted key should
 // be preserved.
-TEST_P(CompactionIteratorTest, CorruptionAfterSingleDeletion) {
+TEST_F(CompactionIteratorTest, CorruptionAfterSingleDeletion) {
   InitIterators({test::KeyStr("a", 5, kTypeSingleDeletion),
                  test::KeyStr("a", 3, kTypeValue, true),
                  test::KeyStr("b", 10, kTypeValue)},
@@ -329,7 +237,7 @@ TEST_P(CompactionIteratorTest, CorruptionAfterSingleDeletion) {
   ASSERT_FALSE(c_iter_->Valid());
 }
 
-TEST_P(CompactionIteratorTest, SimpleRangeDeletion) {
+TEST_F(CompactionIteratorTest, SimpleRangeDeletion) {
   InitIterators({test::KeyStr("morning", 5, kTypeValue),
                  test::KeyStr("morning", 2, kTypeValue),
                  test::KeyStr("night", 3, kTypeValue)},
@@ -345,7 +253,7 @@ TEST_P(CompactionIteratorTest, SimpleRangeDeletion) {
   ASSERT_FALSE(c_iter_->Valid());
 }
 
-TEST_P(CompactionIteratorTest, RangeDeletionWithSnapshots) {
+TEST_F(CompactionIteratorTest, RangeDeletionWithSnapshots) {
   AddSnapshot(10);
   std::vector<std::string> ks1;
   ks1.push_back(test::KeyStr("ma", 28, kTypeRangeDeletion));
@@ -366,11 +274,11 @@ TEST_P(CompactionIteratorTest, RangeDeletionWithSnapshots) {
   ASSERT_FALSE(c_iter_->Valid());
 }
 
-TEST_P(CompactionIteratorTest, CompactionFilterSkipUntil) {
+TEST_F(CompactionIteratorTest, CompactionFilterSkipUntil) {
   class Filter : public CompactionFilter {
-    virtual Decision FilterV2(int /*level*/, const Slice& key, ValueType t,
+    virtual Decision FilterV2(int level, const Slice& key, ValueType t,
                               const Slice& existing_value,
-                              std::string* /*new_value*/,
+                              std::string* new_value,
                               std::string* skip_until) const override {
       std::string k = key.ToString();
       std::string v = existing_value.ToString();
@@ -441,7 +349,7 @@ TEST_P(CompactionIteratorTest, CompactionFilterSkipUntil) {
        test::KeyStr("j", 99, kTypeValue)},
       {"av50", "am45", "bv60", "bv40", "cv35", "dm70", "em71", "fm65", "fm30",
        "fv25", "gv90", "hv91", "im95", "jv99"},
-      {}, {}, kMaxSequenceNumber, kMaxSequenceNumber, &merge_op, &filter);
+      {}, {}, kMaxSequenceNumber, &merge_op, &filter);
 
   // Compaction should output just "a", "e" and "h" keys.
   c_iter_->SeekToFirst();
@@ -476,14 +384,13 @@ TEST_P(CompactionIteratorTest, CompactionFilterSkipUntil) {
   ASSERT_EQ(expected_actions, iter_->log);
 }
 
-TEST_P(CompactionIteratorTest, ShuttingDownInFilter) {
+TEST_F(CompactionIteratorTest, ShuttingDownInFilter) {
   NoMergingMergeOp merge_op;
   StallingFilter filter;
   InitIterators(
       {test::KeyStr("1", 1, kTypeValue), test::KeyStr("2", 2, kTypeValue),
        test::KeyStr("3", 3, kTypeValue), test::KeyStr("4", 4, kTypeValue)},
-      {"v1", "v2", "v3", "v4"}, {}, {}, kMaxSequenceNumber, kMaxSequenceNumber,
-      &merge_op, &filter);
+      {"v1", "v2", "v3", "v4"}, {}, {}, kMaxSequenceNumber, &merge_op, &filter);
   // Don't leave tombstones (kTypeDeletion) for filtered keys.
   compaction_proxy_->key_not_exists_beyond_output_level = true;
 
@@ -514,14 +421,13 @@ TEST_P(CompactionIteratorTest, ShuttingDownInFilter) {
 
 // Same as ShuttingDownInFilter, but shutdown happens during filter call for
 // a merge operand, not for a value.
-TEST_P(CompactionIteratorTest, ShuttingDownInMerge) {
+TEST_F(CompactionIteratorTest, ShuttingDownInMerge) {
   NoMergingMergeOp merge_op;
   StallingFilter filter;
   InitIterators(
       {test::KeyStr("1", 1, kTypeValue), test::KeyStr("2", 2, kTypeMerge),
        test::KeyStr("3", 3, kTypeMerge), test::KeyStr("4", 4, kTypeValue)},
-      {"v1", "v2", "v3", "v4"}, {}, {}, kMaxSequenceNumber, kMaxSequenceNumber,
-      &merge_op, &filter);
+      {"v1", "v2", "v3", "v4"}, {}, {}, kMaxSequenceNumber, &merge_op, &filter);
   compaction_proxy_->key_not_exists_beyond_output_level = true;
 
   std::atomic<bool> seek_done{false};
@@ -549,12 +455,12 @@ TEST_P(CompactionIteratorTest, ShuttingDownInMerge) {
   EXPECT_EQ(2, filter.last_seen.load());
 }
 
-TEST_P(CompactionIteratorTest, SingleMergeOperand) {
+TEST_F(CompactionIteratorTest, SingleMergeOperand) {
   class Filter : public CompactionFilter {
-    virtual Decision FilterV2(int /*level*/, const Slice& key, ValueType t,
+    virtual Decision FilterV2(int level, const Slice& key, ValueType t,
                               const Slice& existing_value,
-                              std::string* /*new_value*/,
-                              std::string* /*skip_until*/) const override {
+                              std::string* new_value,
+                              std::string* skip_until) const override {
       std::string k = key.ToString();
       std::string v = existing_value.ToString();
 
@@ -605,7 +511,7 @@ TEST_P(CompactionIteratorTest, SingleMergeOperand) {
     bool PartialMergeMulti(const Slice& key,
                            const std::deque<Slice>& operand_list,
                            std::string* new_value,
-                           Logger* /*logger*/) const override {
+                           Logger* logger) const override {
       std::string string_key = key.ToString();
       EXPECT_TRUE(string_key == "a" || string_key == "b");
 
@@ -641,7 +547,7 @@ TEST_P(CompactionIteratorTest, SingleMergeOperand) {
        // c should invoke FullMerge due to kTypeValue at the beginning.
        test::KeyStr("c", 90, kTypeMerge), test::KeyStr("c", 80, kTypeValue)},
       {"av1", "bv2", "bv1", "cv2", "cv1"}, {}, {}, kMaxSequenceNumber,
-      kMaxSequenceNumber, &merge_op, &filter);
+      &merge_op, &filter);
 
   c_iter_->SeekToFirst();
   ASSERT_TRUE(c_iter_->Valid());
@@ -654,298 +560,6 @@ TEST_P(CompactionIteratorTest, SingleMergeOperand) {
   ASSERT_EQ("cv1cv2", c_iter_->value().ToString());
 }
 
-// In bottommost level, values earlier than earliest snapshot can be output
-// with sequence = 0.
-TEST_P(CompactionIteratorTest, ZeroOutSequenceAtBottomLevel) {
-  AddSnapshot(1);
-  RunTest({test::KeyStr("a", 1, kTypeValue), test::KeyStr("b", 2, kTypeValue)},
-          {"v1", "v2"},
-          {test::KeyStr("a", 0, kTypeValue), test::KeyStr("b", 2, kTypeValue)},
-          {"v1", "v2"}, kMaxSequenceNumber /*last_commited_seq*/,
-          nullptr /*merge_operator*/, nullptr /*compaction_filter*/,
-          true /*bottommost_level*/);
-}
-
-// In bottommost level, deletions earlier than earliest snapshot can be removed
-// permanently.
-TEST_P(CompactionIteratorTest, RemoveDeletionAtBottomLevel) {
-  AddSnapshot(1);
-  RunTest({test::KeyStr("a", 1, kTypeDeletion),
-           test::KeyStr("b", 2, kTypeDeletion)},
-          {"", ""}, {test::KeyStr("b", 2, kTypeDeletion)}, {""},
-          kMaxSequenceNumber /*last_commited_seq*/, nullptr /*merge_operator*/,
-          nullptr /*compaction_filter*/, true /*bottommost_level*/);
-}
-
-// In bottommost level, single deletions earlier than earliest snapshot can be
-// removed permanently.
-TEST_P(CompactionIteratorTest, RemoveSingleDeletionAtBottomLevel) {
-  AddSnapshot(1);
-  RunTest({test::KeyStr("a", 1, kTypeSingleDeletion),
-           test::KeyStr("b", 2, kTypeSingleDeletion)},
-          {"", ""}, {test::KeyStr("b", 2, kTypeSingleDeletion)}, {""},
-          kMaxSequenceNumber /*last_commited_seq*/, nullptr /*merge_operator*/,
-          nullptr /*compaction_filter*/, true /*bottommost_level*/);
-}
-
-INSTANTIATE_TEST_CASE_P(CompactionIteratorTestInstance, CompactionIteratorTest,
-                        testing::Values(true, false));
-
-// Tests how CompactionIterator work together with SnapshotChecker.
-class CompactionIteratorWithSnapshotCheckerTest
-    : public CompactionIteratorTest {
- public:
-  bool UseSnapshotChecker() const override { return true; }
-};
-
-// Uncommitted keys (keys with seq > last_committed_seq) should be output as-is
-// while committed version of these keys should get compacted as usual.
-
-TEST_F(CompactionIteratorWithSnapshotCheckerTest,
-       PreserveUncommittedKeys_Value) {
-  RunTest(
-      {test::KeyStr("foo", 3, kTypeValue), test::KeyStr("foo", 2, kTypeValue),
-       test::KeyStr("foo", 1, kTypeValue)},
-      {"v3", "v2", "v1"},
-      {test::KeyStr("foo", 3, kTypeValue), test::KeyStr("foo", 2, kTypeValue)},
-      {"v3", "v2"}, 2 /*last_committed_seq*/);
-}
-
-TEST_F(CompactionIteratorWithSnapshotCheckerTest,
-       PreserveUncommittedKeys_Deletion) {
-  RunTest({test::KeyStr("foo", 2, kTypeDeletion),
-           test::KeyStr("foo", 1, kTypeValue)},
-          {"", "v1"},
-          {test::KeyStr("foo", 2, kTypeDeletion),
-           test::KeyStr("foo", 1, kTypeValue)},
-          {"", "v1"}, 1 /*last_committed_seq*/);
-}
-
-TEST_F(CompactionIteratorWithSnapshotCheckerTest,
-       PreserveUncommittedKeys_Merge) {
-  auto merge_op = MergeOperators::CreateStringAppendOperator();
-  RunTest(
-      {test::KeyStr("foo", 3, kTypeMerge), test::KeyStr("foo", 2, kTypeMerge),
-       test::KeyStr("foo", 1, kTypeValue)},
-      {"v3", "v2", "v1"},
-      {test::KeyStr("foo", 3, kTypeMerge), test::KeyStr("foo", 2, kTypeValue)},
-      {"v3", "v1,v2"}, 2 /*last_committed_seq*/, merge_op.get());
-}
-
-TEST_F(CompactionIteratorWithSnapshotCheckerTest,
-       PreserveUncommittedKeys_SingleDelete) {
-  RunTest({test::KeyStr("foo", 2, kTypeSingleDeletion),
-           test::KeyStr("foo", 1, kTypeValue)},
-          {"", "v1"},
-          {test::KeyStr("foo", 2, kTypeSingleDeletion),
-           test::KeyStr("foo", 1, kTypeValue)},
-          {"", "v1"}, 1 /*last_committed_seq*/);
-}
-
-TEST_F(CompactionIteratorWithSnapshotCheckerTest,
-       PreserveUncommittedKeys_BlobIndex) {
-  RunTest({test::KeyStr("foo", 3, kTypeBlobIndex),
-           test::KeyStr("foo", 2, kTypeBlobIndex),
-           test::KeyStr("foo", 1, kTypeBlobIndex)},
-          {"v3", "v2", "v1"},
-          {test::KeyStr("foo", 3, kTypeBlobIndex),
-           test::KeyStr("foo", 2, kTypeBlobIndex)},
-          {"v3", "v2"}, 2 /*last_committed_seq*/);
-}
-
-// Test compaction iterator dedup keys visible to the same snapshot.
-
-TEST_F(CompactionIteratorWithSnapshotCheckerTest, DedupSameSnapshot_Value) {
-  AddSnapshot(2, 1);
-  RunTest(
-      {test::KeyStr("foo", 4, kTypeValue), test::KeyStr("foo", 3, kTypeValue),
-       test::KeyStr("foo", 2, kTypeValue), test::KeyStr("foo", 1, kTypeValue)},
-      {"v4", "v3", "v2", "v1"},
-      {test::KeyStr("foo", 4, kTypeValue), test::KeyStr("foo", 3, kTypeValue),
-       test::KeyStr("foo", 1, kTypeValue)},
-      {"v4", "v3", "v1"}, 3 /*last_committed_seq*/);
-}
-
-TEST_F(CompactionIteratorWithSnapshotCheckerTest, DedupSameSnapshot_Deletion) {
-  AddSnapshot(2, 1);
-  RunTest(
-      {test::KeyStr("foo", 4, kTypeValue),
-       test::KeyStr("foo", 3, kTypeDeletion),
-       test::KeyStr("foo", 2, kTypeValue), test::KeyStr("foo", 1, kTypeValue)},
-      {"v4", "", "v2", "v1"},
-      {test::KeyStr("foo", 4, kTypeValue),
-       test::KeyStr("foo", 3, kTypeDeletion),
-       test::KeyStr("foo", 1, kTypeValue)},
-      {"v4", "", "v1"}, 3 /*last_committed_seq*/);
-}
-
-TEST_F(CompactionIteratorWithSnapshotCheckerTest, DedupSameSnapshot_Merge) {
-  AddSnapshot(2, 1);
-  AddSnapshot(4, 3);
-  auto merge_op = MergeOperators::CreateStringAppendOperator();
-  RunTest(
-      {test::KeyStr("foo", 5, kTypeMerge), test::KeyStr("foo", 4, kTypeMerge),
-       test::KeyStr("foo", 3, kTypeMerge), test::KeyStr("foo", 2, kTypeMerge),
-       test::KeyStr("foo", 1, kTypeValue)},
-      {"v5", "v4", "v3", "v2", "v1"},
-      {test::KeyStr("foo", 5, kTypeMerge), test::KeyStr("foo", 4, kTypeMerge),
-       test::KeyStr("foo", 3, kTypeMerge), test::KeyStr("foo", 1, kTypeValue)},
-      {"v5", "v4", "v2,v3", "v1"}, 4 /*last_committed_seq*/, merge_op.get());
-}
-
-TEST_F(CompactionIteratorWithSnapshotCheckerTest,
-       DedupSameSnapshot_SingleDeletion) {
-  AddSnapshot(2, 1);
-  RunTest(
-      {test::KeyStr("foo", 4, kTypeValue),
-       test::KeyStr("foo", 3, kTypeSingleDeletion),
-       test::KeyStr("foo", 2, kTypeValue), test::KeyStr("foo", 1, kTypeValue)},
-      {"v4", "", "v2", "v1"},
-      {test::KeyStr("foo", 4, kTypeValue), test::KeyStr("foo", 1, kTypeValue)},
-      {"v4", "v1"}, 3 /*last_committed_seq*/);
-}
-
-TEST_F(CompactionIteratorWithSnapshotCheckerTest, DedupSameSnapshot_BlobIndex) {
-  AddSnapshot(2, 1);
-  RunTest({test::KeyStr("foo", 4, kTypeBlobIndex),
-           test::KeyStr("foo", 3, kTypeBlobIndex),
-           test::KeyStr("foo", 2, kTypeBlobIndex),
-           test::KeyStr("foo", 1, kTypeBlobIndex)},
-          {"v4", "v3", "v2", "v1"},
-          {test::KeyStr("foo", 4, kTypeBlobIndex),
-           test::KeyStr("foo", 3, kTypeBlobIndex),
-           test::KeyStr("foo", 1, kTypeBlobIndex)},
-          {"v4", "v3", "v1"}, 3 /*last_committed_seq*/);
-}
-
-// At bottom level, sequence numbers can be zero out, and deletions can be
-// removed, but only when they are visible to earliest snapshot.
-
-TEST_F(CompactionIteratorWithSnapshotCheckerTest,
-       NotZeroOutSequenceIfNotVisibleToEarliestSnapshot) {
-  AddSnapshot(2, 1);
-  RunTest({test::KeyStr("a", 1, kTypeValue), test::KeyStr("b", 2, kTypeValue),
-           test::KeyStr("c", 3, kTypeValue)},
-          {"v1", "v2", "v3"},
-          {test::KeyStr("a", 0, kTypeValue), test::KeyStr("b", 2, kTypeValue),
-           test::KeyStr("c", 3, kTypeValue)},
-          {"v1", "v2", "v3"}, kMaxSequenceNumber /*last_commited_seq*/,
-          nullptr /*merge_operator*/, nullptr /*compaction_filter*/,
-          true /*bottommost_level*/);
-}
-
-TEST_F(CompactionIteratorWithSnapshotCheckerTest,
-       NotRemoveDeletionIfNotVisibleToEarliestSnapshot) {
-  AddSnapshot(2, 1);
-  RunTest(
-      {test::KeyStr("a", 1, kTypeDeletion), test::KeyStr("b", 2, kTypeDeletion),
-       test::KeyStr("c", 3, kTypeDeletion)},
-      {"", "", ""},
-      {test::KeyStr("b", 2, kTypeDeletion),
-       test::KeyStr("c", 3, kTypeDeletion)},
-      {"", ""}, kMaxSequenceNumber /*last_commited_seq*/,
-      nullptr /*merge_operator*/, nullptr /*compaction_filter*/,
-      true /*bottommost_level*/);
-}
-
-TEST_F(CompactionIteratorWithSnapshotCheckerTest,
-       NotRemoveSingleDeletionIfNotVisibleToEarliestSnapshot) {
-  AddSnapshot(2, 1);
-  RunTest({test::KeyStr("a", 1, kTypeSingleDeletion),
-           test::KeyStr("b", 2, kTypeSingleDeletion),
-           test::KeyStr("c", 3, kTypeSingleDeletion)},
-          {"", "", ""},
-          {test::KeyStr("b", 2, kTypeSingleDeletion),
-           test::KeyStr("c", 3, kTypeSingleDeletion)},
-          {"", ""}, kMaxSequenceNumber /*last_commited_seq*/,
-          nullptr /*merge_operator*/, nullptr /*compaction_filter*/,
-          true /*bottommost_level*/);
-}
-
-// Single delete should not cancel out values that not visible to the
-// same set of snapshots
-TEST_F(CompactionIteratorWithSnapshotCheckerTest,
-       SingleDeleteAcrossSnapshotBoundary) {
-  AddSnapshot(2, 1);
-  RunTest({test::KeyStr("a", 2, kTypeSingleDeletion),
-           test::KeyStr("a", 1, kTypeValue)},
-          {"", "v1"},
-          {test::KeyStr("a", 2, kTypeSingleDeletion),
-           test::KeyStr("a", 1, kTypeValue)},
-          {"", "v1"}, 2 /*last_committed_seq*/);
-}
-
-// Single delete should be kept in case it is not visible to the
-// earliest write conflict snapshot. If a single delete is kept for this reason,
-// corresponding value can be trimmed to save space.
-TEST_F(CompactionIteratorWithSnapshotCheckerTest,
-       KeepSingleDeletionForWriteConflictChecking) {
-  AddSnapshot(2, 0);
-  RunTest({test::KeyStr("a", 2, kTypeSingleDeletion),
-           test::KeyStr("a", 1, kTypeValue)},
-          {"", "v1"},
-          {test::KeyStr("a", 2, kTypeSingleDeletion),
-           test::KeyStr("a", 1, kTypeValue)},
-          {"", ""}, 2 /*last_committed_seq*/, nullptr /*merge_operator*/,
-          nullptr /*compaction_filter*/, false /*bottommost_level*/,
-          2 /*earliest_write_conflict_snapshot*/);
-}
-
-// Compaction filter should keep uncommitted key as-is, and
-//   * Convert the latest velue to deletion, and/or
-//   * if latest value is a merge, apply filter to all suequent merges.
-
-TEST_F(CompactionIteratorWithSnapshotCheckerTest, CompactionFilter_Value) {
-  std::unique_ptr<CompactionFilter> compaction_filter(
-      new FilterAllKeysCompactionFilter());
-  RunTest(
-      {test::KeyStr("a", 2, kTypeValue), test::KeyStr("a", 1, kTypeValue),
-       test::KeyStr("b", 3, kTypeValue), test::KeyStr("c", 1, kTypeValue)},
-      {"v2", "v1", "v3", "v4"},
-      {test::KeyStr("a", 2, kTypeValue), test::KeyStr("a", 1, kTypeDeletion),
-       test::KeyStr("b", 3, kTypeValue), test::KeyStr("c", 1, kTypeDeletion)},
-      {"v2", "", "v3", ""}, 1 /*last_committed_seq*/,
-      nullptr /*merge_operator*/, compaction_filter.get());
-}
-
-TEST_F(CompactionIteratorWithSnapshotCheckerTest, CompactionFilter_Deletion) {
-  std::unique_ptr<CompactionFilter> compaction_filter(
-      new FilterAllKeysCompactionFilter());
-  RunTest(
-      {test::KeyStr("a", 2, kTypeDeletion), test::KeyStr("a", 1, kTypeValue)},
-      {"", "v1"},
-      {test::KeyStr("a", 2, kTypeDeletion),
-       test::KeyStr("a", 1, kTypeDeletion)},
-      {"", ""}, 1 /*last_committed_seq*/, nullptr /*merge_operator*/,
-      compaction_filter.get());
-}
-
-TEST_F(CompactionIteratorWithSnapshotCheckerTest,
-       CompactionFilter_PartialMerge) {
-  std::shared_ptr<MergeOperator> merge_op =
-      MergeOperators::CreateStringAppendOperator();
-  std::unique_ptr<CompactionFilter> compaction_filter(
-      new FilterAllKeysCompactionFilter());
-  RunTest({test::KeyStr("a", 3, kTypeMerge), test::KeyStr("a", 2, kTypeMerge),
-           test::KeyStr("a", 1, kTypeMerge)},
-          {"v3", "v2", "v1"}, {test::KeyStr("a", 3, kTypeMerge)}, {"v3"},
-          2 /*last_committed_seq*/, merge_op.get(), compaction_filter.get());
-}
-
-TEST_F(CompactionIteratorWithSnapshotCheckerTest, CompactionFilter_FullMerge) {
-  std::shared_ptr<MergeOperator> merge_op =
-      MergeOperators::CreateStringAppendOperator();
-  std::unique_ptr<CompactionFilter> compaction_filter(
-      new FilterAllKeysCompactionFilter());
-  RunTest(
-      {test::KeyStr("a", 3, kTypeMerge), test::KeyStr("a", 2, kTypeMerge),
-       test::KeyStr("a", 1, kTypeValue)},
-      {"v3", "v2", "v1"},
-      {test::KeyStr("a", 3, kTypeMerge), test::KeyStr("a", 1, kTypeDeletion)},
-      {"v3", ""}, 2 /*last_committed_seq*/, merge_op.get(),
-      compaction_filter.get());
-}
-
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/compaction_job.cc

@@ -38,6 +38,7 @@
 #include "monitoring/iostats_context_imp.h"
 #include "monitoring/perf_context_imp.h"
 #include "monitoring/thread_status_util.h"
+#include "port/likely.h"
 #include "port/port.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
@@ -62,46 +63,6 @@
 
 namespace rocksdb {
 
-const char* GetCompactionReasonString(CompactionReason compaction_reason) {
-  switch (compaction_reason) {
-    case CompactionReason::kUnknown:
-      return "Unknown";
-    case CompactionReason::kLevelL0FilesNum:
-      return "LevelL0FilesNum";
-    case CompactionReason::kLevelMaxLevelSize:
-      return "LevelMaxLevelSize";
-    case CompactionReason::kUniversalSizeAmplification:
-      return "UniversalSizeAmplification";
-    case CompactionReason::kUniversalSizeRatio:
-      return "UniversalSizeRatio";
-    case CompactionReason::kUniversalSortedRunNum:
-      return "UniversalSortedRunNum";
-    case CompactionReason::kFIFOMaxSize:
-      return "FIFOMaxSize";
-    case CompactionReason::kFIFOReduceNumFiles:
-      return "FIFOReduceNumFiles";
-    case CompactionReason::kFIFOTtl:
-      return "FIFOTtl";
-    case CompactionReason::kManualCompaction:
-      return "ManualCompaction";
-    case CompactionReason::kFilesMarkedForCompaction:
-      return "FilesMarkedForCompaction";
-    case CompactionReason::kBottommostFiles:
-      return "BottommostFiles";
-    case CompactionReason::kTtl:
-      return "Ttl";
-    case CompactionReason::kFlush:
-      return "Flush";
-    case CompactionReason::kExternalSstIngestion:
-      return "ExternalSstIngestion";
-    case CompactionReason::kNumOfReasons:
-      // fall through
-    default:
-      assert(false);
-      return "Invalid";
-  }
-}
-
 // Maintains state for each sub-compaction
 struct CompactionJob::SubcompactionState {
   const Compaction* compaction;
@@ -303,29 +264,25 @@ void CompactionJob::AggregateStatistics() {
 
 CompactionJob::CompactionJob(
     int job_id, Compaction* compaction, const ImmutableDBOptions& db_options,
-    const EnvOptions env_options, VersionSet* versions,
-    const std::atomic<bool>* shutting_down,
-    const SequenceNumber preserve_deletes_seqnum, LogBuffer* log_buffer,
+    const EnvOptions& env_options, VersionSet* versions,
+    const std::atomic<bool>* shutting_down, LogBuffer* log_buffer,
     Directory* db_directory, Directory* output_directory, Statistics* stats,
     InstrumentedMutex* db_mutex, Status* db_bg_error,
     std::vector<SequenceNumber> existing_snapshots,
     SequenceNumber earliest_write_conflict_snapshot,
-    const SnapshotChecker* snapshot_checker, std::shared_ptr<Cache> table_cache,
-    EventLogger* event_logger, bool paranoid_file_checks, bool measure_io_stats,
-    const std::string& dbname, CompactionJobStats* compaction_job_stats)
+    std::shared_ptr<Cache> table_cache, EventLogger* event_logger,
+    bool paranoid_file_checks, bool measure_io_stats, const std::string& dbname,
+    CompactionJobStats* compaction_job_stats)
     : job_id_(job_id),
       compact_(new CompactionState(compaction)),
       compaction_job_stats_(compaction_job_stats),
-      compaction_stats_(compaction->compaction_reason(), 1),
+      compaction_stats_(1),
       dbname_(dbname),
       db_options_(db_options),
       env_options_(env_options),
       env_(db_options.env),
-      env_optiosn_for_read_(
-          env_->OptimizeForCompactionTableRead(env_options, db_options_)),
       versions_(versions),
       shutting_down_(shutting_down),
-      preserve_deletes_seqnum_(preserve_deletes_seqnum),
       log_buffer_(log_buffer),
       db_directory_(db_directory),
       output_directory_(output_directory),
@@ -334,13 +291,10 @@ CompactionJob::CompactionJob(
       db_bg_error_(db_bg_error),
       existing_snapshots_(std::move(existing_snapshots)),
       earliest_write_conflict_snapshot_(earliest_write_conflict_snapshot),
-      snapshot_checker_(snapshot_checker),
       table_cache_(std::move(table_cache)),
       event_logger_(event_logger),
-      bottommost_level_(false),
       paranoid_file_checks_(paranoid_file_checks),
-      measure_io_stats_(measure_io_stats),
-      write_hint_(Env::WLTH_NOT_SET) {
+      measure_io_stats_(measure_io_stats) {
   assert(log_buffer_ != nullptr);
   const auto* cfd = compact_->compaction->column_family_data();
   ThreadStatusUtil::SetColumnFamily(cfd, cfd->ioptions()->env,
@@ -354,13 +308,15 @@ CompactionJob::~CompactionJob() {
   ThreadStatusUtil::ResetThreadStatus();
 }
 
-void CompactionJob::ReportStartedCompaction(Compaction* compaction) {
+void CompactionJob::ReportStartedCompaction(
+    Compaction* compaction) {
   const auto* cfd = compact_->compaction->column_family_data();
   ThreadStatusUtil::SetColumnFamily(cfd, cfd->ioptions()->env,
                                     db_options_.enable_thread_tracking);
 
-  ThreadStatusUtil::SetThreadOperationProperty(ThreadStatus::COMPACTION_JOB_ID,
-                                               job_id_);
+  ThreadStatusUtil::SetThreadOperationProperty(
+      ThreadStatus::COMPACTION_JOB_ID,
+      job_id_);
 
   ThreadStatusUtil::SetThreadOperationProperty(
       ThreadStatus::COMPACTION_INPUT_OUTPUT_LEVEL,
@@ -390,7 +346,8 @@ void CompactionJob::ReportStartedCompaction(Compaction* compaction) {
 
   // Set the thread operation after operation properties
   // to ensure GetThreadList() can always show them all together.
-  ThreadStatusUtil::SetThreadOperation(ThreadStatus::OP_COMPACTION);
+  ThreadStatusUtil::SetThreadOperation(
+      ThreadStatus::OP_COMPACTION);
 
   if (compaction_job_stats_) {
     compaction_job_stats_->is_manual_compaction =
@@ -405,11 +362,9 @@ void CompactionJob::Prepare() {
   // Generate file_levels_ for compaction berfore making Iterator
   auto* c = compact_->compaction;
   assert(c->column_family_data() != nullptr);
-  assert(c->column_family_data()->current()->storage_info()->NumLevelFiles(
-             compact_->compaction->level()) > 0);
+  assert(c->column_family_data()->current()->storage_info()
+      ->NumLevelFiles(compact_->compaction->level()) > 0);
 
-  write_hint_ =
-      c->column_family_data()->CalculateSSTWriteHint(c->output_level());
   // Is this compaction producing files at the bottommost level?
   bottommost_level_ = c->bottommost_level();
 
@@ -492,18 +447,14 @@ void CompactionJob::GenSubcompactionBoundaries() {
   }
 
   std::sort(bounds.begin(), bounds.end(),
-            [cfd_comparator](const Slice& a, const Slice& b) -> bool {
-              return cfd_comparator->Compare(ExtractUserKey(a),
-                                             ExtractUserKey(b)) < 0;
-            });
+    [cfd_comparator] (const Slice& a, const Slice& b) -> bool {
+      return cfd_comparator->Compare(ExtractUserKey(a), ExtractUserKey(b)) < 0;
+    });
   // Remove duplicated entries from bounds
-  bounds.erase(
-      std::unique(bounds.begin(), bounds.end(),
-                  [cfd_comparator](const Slice& a, const Slice& b) -> bool {
-                    return cfd_comparator->Compare(ExtractUserKey(a),
-                                                   ExtractUserKey(b)) == 0;
-                  }),
-      bounds.end());
+  bounds.erase(std::unique(bounds.begin(), bounds.end(),
+    [cfd_comparator] (const Slice& a, const Slice& b) -> bool {
+      return cfd_comparator->Compare(ExtractUserKey(a), ExtractUserKey(b)) == 0;
+    }), bounds.end());
 
   // Combine consecutive pairs of boundaries into ranges with an approximate
   // size of data covered by keys in that range
@@ -526,15 +477,12 @@ void CompactionJob::GenSubcompactionBoundaries() {
 
   // Group the ranges into subcompactions
   const double min_file_fill_percent = 4.0 / 5;
-  int base_level = v->storage_info()->base_level();
-  uint64_t max_output_files = static_cast<uint64_t>(std::ceil(
-      sum / min_file_fill_percent /
-      MaxFileSizeForLevel(*(c->mutable_cf_options()), out_lvl,
-          c->immutable_cf_options()->compaction_style, base_level,
-          c->immutable_cf_options()->level_compaction_dynamic_level_bytes)));
+  uint64_t max_output_files = static_cast<uint64_t>(
+      std::ceil(sum / min_file_fill_percent /
+                c->mutable_cf_options()->MaxFileSizeForLevel(out_lvl)));
   uint64_t subcompactions =
       std::min({static_cast<uint64_t>(ranges.size()),
-                static_cast<uint64_t>(c->max_subcompactions()),
+                static_cast<uint64_t>(db_options_.max_subcompactions),
                 max_output_files});
 
   if (subcompactions > 1) {
@@ -610,9 +558,8 @@ Status CompactionJob::Run() {
   TablePropertiesCollection tp;
   for (const auto& state : compact_->sub_compact_states) {
     for (const auto& output : state.outputs) {
-      auto fn =
-          TableFileName(state.compaction->immutable_cf_options()->cf_paths,
-                        output.meta.fd.GetNumber(), output.meta.fd.GetPathId());
+      auto fn = TableFileName(db_options_.db_paths, output.meta.fd.GetNumber(),
+                              output.meta.fd.GetPathId());
       tp[fn] = output.table_properties;
     }
   }
@@ -670,8 +617,7 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) {
       "[%s] compacted to: %s, MB/sec: %.1f rd, %.1f wr, level %d, "
       "files in(%d, %d) out(%d) "
       "MB in(%.1f, %.1f) out(%.1f), read-write-amplify(%.1f) "
-      "write-amplify(%.1f) %s, records in: %" PRIu64
-      ", records dropped: %" PRIu64 " output_compression: %s\n",
+      "write-amplify(%.1f) %s, records in: %d, records dropped: %d\n",
       cfd->GetName().c_str(), vstorage->LevelSummary(&tmp), bytes_read_per_sec,
       bytes_written_per_sec, compact_->compaction->output_level(),
       stats.num_input_files_in_non_output_levels,
@@ -680,23 +626,20 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) {
       stats.bytes_read_output_level / 1048576.0,
       stats.bytes_written / 1048576.0, read_write_amp, write_amp,
       status.ToString().c_str(), stats.num_input_records,
-      stats.num_dropped_records,
-      CompressionTypeToString(compact_->compaction->output_compression())
-          .c_str());
+      stats.num_dropped_records);
 
   UpdateCompactionJobStats(stats);
 
   auto stream = event_logger_->LogToBuffer(log_buffer_);
-  stream << "job" << job_id_ << "event"
-         << "compaction_finished"
+  stream << "job" << job_id_
+         << "event" << "compaction_finished"
          << "compaction_time_micros" << compaction_stats_.micros
          << "output_level" << compact_->compaction->output_level()
          << "num_output_files" << compact_->NumOutputFiles()
-         << "total_output_size" << compact_->total_bytes << "num_input_records"
-         << compact_->num_input_records << "num_output_records"
-         << compact_->num_output_records << "num_subcompactions"
-         << compact_->sub_compact_states.size() << "output_compression"
-         << CompressionTypeToString(compact_->compaction->output_compression());
+         << "total_output_size" << compact_->total_bytes
+         << "num_input_records" << compact_->num_input_records
+         << "num_output_records" << compact_->num_output_records
+         << "num_subcompactions" << compact_->sub_compact_states.size();
 
   if (compaction_job_stats_ != nullptr) {
     stream << "num_single_delete_mismatches"
@@ -731,7 +674,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
   std::unique_ptr<RangeDelAggregator> range_del_agg(
       new RangeDelAggregator(cfd->internal_comparator(), existing_snapshots_));
   std::unique_ptr<InternalIterator> input(versions_->MakeInputIterator(
-      sub_compact->compaction, range_del_agg.get(), env_optiosn_for_read_));
+      sub_compact->compaction, range_del_agg.get()));
 
   AutoThreadOperationStageUpdater stage_updater(
       ThreadStatus::STAGE_COMPACTION_PROCESS_KV);
@@ -756,31 +699,23 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
       sub_compact->compaction->mutable_cf_options();
 
   // To build compression dictionary, we sample the first output file, assuming
-  // it'll reach the maximum length. We optionally pass these samples through
-  // zstd's dictionary trainer, or just use them directly. Then, the dictionary
-  // is used for compressing subsequent output files in the same subcompaction.
-  const bool kUseZstdTrainer =
-      cfd->ioptions()->compression_opts.zstd_max_train_bytes > 0;
-  const size_t kSampleBytes =
-      kUseZstdTrainer ? cfd->ioptions()->compression_opts.zstd_max_train_bytes
-                      : cfd->ioptions()->compression_opts.max_dict_bytes;
+  // it'll reach the maximum length, and then use the dictionary for compressing
+  // subsequent output files. The dictionary may be less than max_dict_bytes if
+  // the first output file's length is less than the maximum.
   const int kSampleLenShift = 6;  // 2^6 = 64-byte samples
   std::set<size_t> sample_begin_offsets;
-  if (bottommost_level_ && kSampleBytes > 0) {
-    const size_t kMaxSamples = kSampleBytes >> kSampleLenShift;
-    const size_t kOutFileLen =
-        static_cast<size_t>(MaxFileSizeForLevel(*mutable_cf_options,
-            compact_->compaction->output_level(),
-            cfd->ioptions()->compaction_style,
-            compact_->compaction->GetInputBaseLevel(),
-            cfd->ioptions()->level_compaction_dynamic_level_bytes));
+  if (bottommost_level_ &&
+      cfd->ioptions()->compression_opts.max_dict_bytes > 0) {
+    const size_t kMaxSamples =
+        cfd->ioptions()->compression_opts.max_dict_bytes >> kSampleLenShift;
+    const size_t kOutFileLen = mutable_cf_options->MaxFileSizeForLevel(
+        compact_->compaction->output_level());
     if (kOutFileLen != port::kMaxSizet) {
       const size_t kOutFileNumSamples = kOutFileLen >> kSampleLenShift;
       Random64 generator{versions_->NewFileNumber()};
       for (size_t i = 0; i < kMaxSamples; ++i) {
-        sample_begin_offsets.insert(
-            static_cast<size_t>(generator.Uniform(kOutFileNumSamples))
-            << kSampleLenShift);
+        sample_begin_offsets.insert(generator.Uniform(kOutFileNumSamples)
+                                    << kSampleLenShift);
       }
     }
   }
@@ -797,8 +732,8 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
       compaction_filter, db_options_.info_log.get(),
       false /* internal key corruption is expected */,
       existing_snapshots_.empty() ? 0 : existing_snapshots_.back(),
-      snapshot_checker_, compact_->compaction->level(),
-      db_options_.statistics.get(), shutting_down_);
+      compact_->compaction->level(), db_options_.statistics.get(),
+      shutting_down_);
 
   TEST_SYNC_POINT("CompactionJob::Run():Inprogress");
 
@@ -812,29 +747,40 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
     input->SeekToFirst();
   }
 
+  // we allow only 1 compaction event listener. Used by blob storage
+  CompactionEventListener* comp_event_listener = nullptr;
+#ifndef ROCKSDB_LITE
+  for (auto& celitr : cfd->ioptions()->listeners) {
+    comp_event_listener = celitr->GetCompactionEventListener();
+    if (comp_event_listener != nullptr) {
+      break;
+    }
+  }
+#endif  // ROCKSDB_LITE
+
   Status status;
   sub_compact->c_iter.reset(new CompactionIterator(
       input.get(), cfd->user_comparator(), &merge, versions_->LastSequence(),
-      &existing_snapshots_, earliest_write_conflict_snapshot_,
-      snapshot_checker_, env_, false, range_del_agg.get(),
-      sub_compact->compaction, compaction_filter, shutting_down_,
-      preserve_deletes_seqnum_));
+      &existing_snapshots_, earliest_write_conflict_snapshot_, env_, false,
+      range_del_agg.get(), sub_compact->compaction, compaction_filter,
+      comp_event_listener, shutting_down_));
   auto c_iter = sub_compact->c_iter.get();
   c_iter->SeekToFirst();
-  if (c_iter->Valid() && sub_compact->compaction->output_level() != 0) {
+  if (c_iter->Valid() &&
+      sub_compact->compaction->output_level() != 0) {
     // ShouldStopBefore() maintains state based on keys processed so far. The
     // compaction loop always calls it on the "next" key, thus won't tell it the
     // first key. So we do that here.
-    sub_compact->ShouldStopBefore(c_iter->key(),
-                                  sub_compact->current_output_file_size);
+    sub_compact->ShouldStopBefore(
+      c_iter->key(), sub_compact->current_output_file_size);
   }
   const auto& c_iter_stats = c_iter->iter_stats();
   auto sample_begin_offset_iter = sample_begin_offsets.cbegin();
-  // data_begin_offset and dict_sample_data are only valid while generating
+  // data_begin_offset and compression_dict are only valid while generating
   // dictionary from the first output file.
   size_t data_begin_offset = 0;
-  std::string dict_sample_data;
-  dict_sample_data.reserve(kSampleBytes);
+  std::string compression_dict;
+  compression_dict.reserve(cfd->ioptions()->compression_opts.max_dict_bytes);
 
   while (status.ok() && !cfd->IsDropped() && c_iter->Valid()) {
     // Invariant: c_iter.status() is guaranteed to be OK if c_iter->Valid()
@@ -906,7 +852,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
             data_elmt_copy_len =
                 data_end_offset - (data_begin_offset + data_elmt_copy_offset);
           }
-          dict_sample_data.append(&data_elmt.data()[data_elmt_copy_offset],
+          compression_dict.append(&data_elmt.data()[data_elmt_copy_offset],
                                   data_elmt_copy_len);
           if (sample_end_offset > data_end_offset) {
             // Didn't finish sample. Try to finish it with the next data_elmt.
@@ -940,8 +886,8 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
     c_iter->Next();
     if (!output_file_ended && c_iter->Valid() &&
         sub_compact->compaction->output_level() != 0 &&
-        sub_compact->ShouldStopBefore(c_iter->key(),
-                                      sub_compact->current_output_file_size) &&
+        sub_compact->ShouldStopBefore(
+          c_iter->key(), sub_compact->current_output_file_size) &&
         sub_compact->builder != nullptr) {
       // (2) this key belongs to the next file. For historical reasons, the
       // iterator status after advancing will be given to
@@ -961,15 +907,9 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
       RecordDroppedKeys(range_del_out_stats,
                         &sub_compact->compaction_job_stats);
       if (sub_compact->outputs.size() == 1) {
-        // Use samples from first output file to create dictionary for
-        // compression of subsequent files.
-        if (kUseZstdTrainer) {
-          sub_compact->compression_dict = ZSTD_TrainDictionary(
-              dict_sample_data, kSampleLenShift,
-              cfd->ioptions()->compression_opts.max_dict_bytes);
-        } else {
-          sub_compact->compression_dict = std::move(dict_sample_data);
-        }
+        // Use dictionary from first output file for compression of subsequent
+        // files.
+        sub_compact->compression_dict = std::move(compression_dict);
       }
     }
   }
@@ -993,8 +933,8 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
   RecordDroppedKeys(c_iter_stats, &sub_compact->compaction_job_stats);
   RecordCompactionIOStats();
 
-  if (status.ok() &&
-      (shutting_down_->load(std::memory_order_relaxed) || cfd->IsDropped())) {
+  if (status.ok() && (shutting_down_->load(std::memory_order_relaxed) ||
+                      cfd->IsDropped())) {
     status = Status::ShutdownInProgress(
         "Database shutdown or Column family drop during compaction");
   }
@@ -1100,7 +1040,6 @@ Status CompactionJob::FinishCompactionOutputFile(
   auto meta = &sub_compact->current_output()->meta;
   if (s.ok()) {
     Slice lower_bound_guard, upper_bound_guard;
-    std::string smallest_user_key;
     const Slice *lower_bound, *upper_bound;
     if (sub_compact->outputs.size() == 1) {
       // For the first output table, include range tombstones before the min key
@@ -1110,8 +1049,7 @@ Status CompactionJob::FinishCompactionOutputFile(
       // For subsequent output tables, only include range tombstones from min
       // key onwards since the previous file was extended to contain range
       // tombstones falling before min key.
-      smallest_user_key = meta->smallest.user_key().ToString(false /*hex*/);
-      lower_bound_guard = Slice(smallest_user_key);
+      lower_bound_guard = meta->smallest.user_key();
       lower_bound = &lower_bound_guard;
     } else {
       lower_bound = nullptr;
@@ -1129,18 +1067,16 @@ Status CompactionJob::FinishCompactionOutputFile(
     range_del_agg->AddToBuilder(sub_compact->builder.get(), lower_bound,
                                 upper_bound, meta, range_del_out_stats,
                                 bottommost_level_);
-    meta->marked_for_compaction = sub_compact->builder->NeedCompact();
   }
   const uint64_t current_entries = sub_compact->builder->NumEntries();
+  meta->marked_for_compaction = sub_compact->builder->NeedCompact();
   if (s.ok()) {
     s = sub_compact->builder->Finish();
   } else {
     sub_compact->builder->Abandon();
   }
   const uint64_t current_bytes = sub_compact->builder->FileSize();
-  if (s.ok()) {
-    meta->fd.file_size = current_bytes;
-  }
+  meta->fd.file_size = current_bytes;
   sub_compact->current_output()->finished = true;
   sub_compact->total_bytes += current_bytes;
 
@@ -1158,9 +1094,8 @@ Status CompactionJob::FinishCompactionOutputFile(
     // If there is nothing to output, no necessary to generate a sst file.
     // This happens when the output level is bottom level, at the same time
     // the sub_compact output nothing.
-    std::string fname =
-        TableFileName(sub_compact->compaction->immutable_cf_options()->cf_paths,
-                      meta->fd.GetNumber(), meta->fd.GetPathId());
+    std::string fname = TableFileName(
+        db_options_.db_paths, meta->fd.GetNumber(), meta->fd.GetPathId());
     env_->DeleteFile(fname);
 
     // Also need to remove the file from outputs, or it will be added to the
@@ -1179,22 +1114,18 @@ Status CompactionJob::FinishCompactionOutputFile(
     // We set for_compaction to false and don't OptimizeForCompactionTableRead
     // here because this is a special case after we finish the table building
     // No matter whether use_direct_io_for_flush_and_compaction is true,
-    // we will regard this verification as user reads since the goal is
+    // we will regrad this verification as user reads since the goal is
     // to cache it here for further user reads
     InternalIterator* iter = cfd->table_cache()->NewIterator(
         ReadOptions(), env_options_, cfd->internal_comparator(), meta->fd,
-        nullptr /* range_del_agg */,
-        sub_compact->compaction->mutable_cf_options()->prefix_extractor.get(),
-        nullptr,
+        nullptr /* range_del_agg */, nullptr,
         cfd->internal_stats()->GetFileReadHist(
             compact_->compaction->output_level()),
-        false, nullptr /* arena */, false /* skip_filters */,
-        compact_->compaction->output_level());
+        false);
     s = iter->status();
 
     if (s.ok() && paranoid_file_checks_) {
-      for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
-      }
+      for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {}
       s = iter->status();
     }
 
@@ -1213,33 +1144,24 @@ Status CompactionJob::FinishCompactionOutputFile(
                      meta->marked_for_compaction ? " (need compaction)" : "");
     }
   }
-  std::string fname;
-  FileDescriptor output_fd;
-  if (meta != nullptr) {
-    fname =
-        TableFileName(sub_compact->compaction->immutable_cf_options()->cf_paths,
-                      meta->fd.GetNumber(), meta->fd.GetPathId());
-    output_fd = meta->fd;
-  } else {
-    fname = "(nil)";
-  }
+  std::string fname = TableFileName(db_options_.db_paths, meta->fd.GetNumber(),
+                                    meta->fd.GetPathId());
   EventHelpers::LogAndNotifyTableFileCreationFinished(
       event_logger_, cfd->ioptions()->listeners, dbname_, cfd->GetName(), fname,
-      job_id_, output_fd, tp, TableFileCreationReason::kCompaction, s);
+      job_id_, meta->fd, tp, TableFileCreationReason::kCompaction, s);
 
 #ifndef ROCKSDB_LITE
   // Report new file to SstFileManagerImpl
   auto sfm =
       static_cast<SstFileManagerImpl*>(db_options_.sst_file_manager.get());
-  if (sfm && meta != nullptr && meta->fd.GetPathId() == 0) {
-    auto fn =
-        TableFileName(sub_compact->compaction->immutable_cf_options()->cf_paths,
-                      meta->fd.GetNumber(), meta->fd.GetPathId());
+  if (sfm && meta->fd.GetPathId() == 0) {
+    auto fn = TableFileName(cfd->ioptions()->db_paths, meta->fd.GetNumber(),
+                            meta->fd.GetPathId());
     sfm->OnAddFile(fn);
     if (sfm->IsMaxAllowedSpaceReached()) {
       // TODO(ajkr): should we return OK() if max space was reached by the final
       // compaction output file (similarly to how flush works when full)?
-      s = Status::NoSpace("Max allowed space was reached");
+      s = Status::IOError("Max allowed space was reached");
       TEST_SYNC_POINT(
           "CompactionJob::FinishCompactionOutputFile:"
           "MaxAllowedSpaceReached");
@@ -1289,7 +1211,7 @@ Status CompactionJob::InstallCompactionResults(
         compaction->InputLevelSummary(&inputs_summary), compact_->total_bytes);
   }
 
-  // Add compaction inputs
+  // Add compaction outputs
   compaction->AddInputDeletions(compact_->compaction->edit());
 
   for (const auto& sub_compact : compact_->sub_compact_states) {
@@ -1319,9 +1241,8 @@ Status CompactionJob::OpenCompactionOutputFile(
   assert(sub_compact->builder == nullptr);
   // no need to lock because VersionSet::next_file_number_ is atomic
   uint64_t file_number = versions_->NewFileNumber();
-  std::string fname =
-      TableFileName(sub_compact->compaction->immutable_cf_options()->cf_paths,
-                    file_number, sub_compact->compaction->output_path_id());
+  std::string fname = TableFileName(db_options_.db_paths, file_number,
+                                    sub_compact->compaction->output_path_id());
   // Fire events.
   ColumnFamilyData* cfd = sub_compact->compaction->column_family_data();
 #ifndef ROCKSDB_LITE
@@ -1331,12 +1252,11 @@ Status CompactionJob::OpenCompactionOutputFile(
 #endif  // !ROCKSDB_LITE
   // Make the output file
   unique_ptr<WritableFile> writable_file;
-#ifndef NDEBUG
-  bool syncpoint_arg = env_options_.use_direct_writes;
+  EnvOptions opt_env_opts =
+      env_->OptimizeForCompactionTableWrite(env_options_, db_options_);
   TEST_SYNC_POINT_CALLBACK("CompactionJob::OpenCompactionOutputFile",
-                           &syncpoint_arg);
-#endif
-  Status s = NewWritableFile(env_, fname, &writable_file, env_options_);
+                           &opt_env_opts.use_direct_writes);
+  Status s = NewWritableFile(env_, fname, &writable_file, opt_env_opts);
   if (!s.ok()) {
     ROCKS_LOG_ERROR(
         db_options_.info_log,
@@ -1359,7 +1279,6 @@ Status CompactionJob::OpenCompactionOutputFile(
 
   sub_compact->outputs.push_back(out);
   writable_file->SetIOPriority(Env::IO_LOW);
-  writable_file->SetWriteLifeTimeHint(write_hint_);
   writable_file->SetPreallocationBlockSize(static_cast<size_t>(
       sub_compact->compaction->OutputFilePreallocationSize()));
   sub_compact->outfile.reset(new WritableFileWriter(
@@ -1375,23 +1294,14 @@ Status CompactionJob::OpenCompactionOutputFile(
       sub_compact->compaction->MaxInputFileCreationTime();
   if (output_file_creation_time == 0) {
     int64_t _current_time = 0;
-    auto status = db_options_.env->GetCurrentTime(&_current_time);
-    // Safe to proceed even if GetCurrentTime fails. So, log and proceed.
-    if (!status.ok()) {
-      ROCKS_LOG_WARN(
-          db_options_.info_log,
-          "Failed to get current time to populate creation_time property. "
-          "Status: %s",
-          status.ToString().c_str());
-    }
+    db_options_.env->GetCurrentTime(&_current_time);  // ignore error
     output_file_creation_time = static_cast<uint64_t>(_current_time);
   }
 
   sub_compact->builder.reset(NewTableBuilder(
-      *cfd->ioptions(), *(sub_compact->compaction->mutable_cf_options()),
-      cfd->internal_comparator(), cfd->int_tbl_prop_collector_factories(),
-      cfd->GetID(), cfd->GetName(), sub_compact->outfile.get(),
-      sub_compact->compaction->output_compression(),
+      *cfd->ioptions(), cfd->internal_comparator(),
+      cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
+      sub_compact->outfile.get(), sub_compact->compaction->output_compression(),
       cfd->ioptions()->compression_opts,
       sub_compact->compaction->output_level(), &sub_compact->compression_dict,
       skip_filters, output_file_creation_time));
@@ -1424,7 +1334,8 @@ void CompactionJob::CleanupCompaction() {
 
 #ifndef ROCKSDB_LITE
 namespace {
-void CopyPrefix(const Slice& src, size_t prefix_length, std::string* dst) {
+void CopyPrefix(
+    const Slice& src, size_t prefix_length, std::string* dst) {
   assert(prefix_length > 0);
   size_t length = src.size() > prefix_length ? prefix_length : src.size();
   dst->assign(src.data(), length);
@@ -1443,11 +1354,13 @@ void CompactionJob::UpdateCompactionStats() {
     if (compaction->level(input_level) != compaction->output_level()) {
       UpdateCompactionInputStatsHelper(
           &compaction_stats_.num_input_files_in_non_output_levels,
-          &compaction_stats_.bytes_read_non_output_levels, input_level);
+          &compaction_stats_.bytes_read_non_output_levels,
+          input_level);
     } else {
       UpdateCompactionInputStatsHelper(
           &compaction_stats_.num_input_files_in_output_level,
-          &compaction_stats_.bytes_read_output_level, input_level);
+          &compaction_stats_.bytes_read_output_level,
+          input_level);
     }
   }
 
@@ -1470,9 +1383,8 @@ void CompactionJob::UpdateCompactionStats() {
   }
 }
 
-void CompactionJob::UpdateCompactionInputStatsHelper(int* num_files,
-                                                     uint64_t* bytes_read,
-                                                     int input_level) {
+void CompactionJob::UpdateCompactionInputStatsHelper(
+    int* num_files, uint64_t* bytes_read, int input_level) {
   const Compaction* compaction = compact_->compaction;
   auto num_input_files = compaction->num_input_files(input_level);
   *num_files += static_cast<int>(num_input_files);
@@ -1493,8 +1405,10 @@ void CompactionJob::UpdateCompactionJobStats(
 
     // input information
     compaction_job_stats_->total_input_bytes =
-        stats.bytes_read_non_output_levels + stats.bytes_read_output_level;
-    compaction_job_stats_->num_input_records = compact_->num_input_records;
+        stats.bytes_read_non_output_levels +
+        stats.bytes_read_output_level;
+    compaction_job_stats_->num_input_records =
+        compact_->num_input_records;
     compaction_job_stats_->num_input_files =
         stats.num_input_files_in_non_output_levels +
         stats.num_input_files_in_output_level;
@@ -1503,20 +1417,21 @@ void CompactionJob::UpdateCompactionJobStats(
 
     // output information
     compaction_job_stats_->total_output_bytes = stats.bytes_written;
-    compaction_job_stats_->num_output_records = compact_->num_output_records;
+    compaction_job_stats_->num_output_records =
+        compact_->num_output_records;
     compaction_job_stats_->num_output_files = stats.num_output_files;
 
     if (compact_->NumOutputFiles() > 0U) {
-      CopyPrefix(compact_->SmallestUserKey(),
-                 CompactionJobStats::kMaxPrefixLength,
-                 &compaction_job_stats_->smallest_output_key_prefix);
-      CopyPrefix(compact_->LargestUserKey(),
-                 CompactionJobStats::kMaxPrefixLength,
-                 &compaction_job_stats_->largest_output_key_prefix);
+      CopyPrefix(
+          compact_->SmallestUserKey(),
+          CompactionJobStats::kMaxPrefixLength,
+          &compaction_job_stats_->smallest_output_key_prefix);
+      CopyPrefix(
+          compact_->LargestUserKey(),
+          CompactionJobStats::kMaxPrefixLength,
+          &compaction_job_stats_->largest_output_key_prefix);
     }
   }
-#else
-  (void)stats;
 #endif  // !ROCKSDB_LITE
 }
 
@@ -1539,9 +1454,7 @@ void CompactionJob::LogCompaction() {
     // build event logger report
     auto stream = event_logger_->Log();
     stream << "job" << job_id_ << "event"
-           << "compaction_started"
-           << "compaction_reason"
-           << GetCompactionReasonString(compaction->compaction_reason());
+           << "compaction_started";
     for (size_t i = 0; i < compaction->num_input_levels(); ++i) {
       stream << ("files_L" + ToString(compaction->level(i)));
       stream.StartArray();

db/compaction_job.h

@@ -30,7 +30,6 @@
 #include "db/write_controller.h"
 #include "db/write_thread.h"
 #include "options/db_options.h"
-#include "options/cf_options.h"
 #include "port/port.h"
 #include "rocksdb/compaction_filter.h"
 #include "rocksdb/compaction_job_stats.h"
@@ -46,28 +45,24 @@
 
 namespace rocksdb {
 
-class Arena;
 class MemTable;
-class SnapshotChecker;
 class TableCache;
 class Version;
 class VersionEdit;
 class VersionSet;
+class Arena;
 
 class CompactionJob {
  public:
   CompactionJob(int job_id, Compaction* compaction,
                 const ImmutableDBOptions& db_options,
-                const EnvOptions env_options, VersionSet* versions,
-                const std::atomic<bool>* shutting_down,
-                const SequenceNumber preserve_deletes_seqnum,
-                LogBuffer* log_buffer,
+                const EnvOptions& env_options, VersionSet* versions,
+                const std::atomic<bool>* shutting_down, LogBuffer* log_buffer,
                 Directory* db_directory, Directory* output_directory,
                 Statistics* stats, InstrumentedMutex* db_mutex,
                 Status* db_bg_error,
                 std::vector<SequenceNumber> existing_snapshots,
                 SequenceNumber earliest_write_conflict_snapshot,
-                const SnapshotChecker* snapshot_checker,
                 std::shared_ptr<Cache> table_cache, EventLogger* event_logger,
                 bool paranoid_file_checks, bool measure_io_stats,
                 const std::string& dbname,
@@ -132,14 +127,11 @@ class CompactionJob {
   // DBImpl state
   const std::string& dbname_;
   const ImmutableDBOptions& db_options_;
-  const EnvOptions env_options_;
+  const EnvOptions& env_options_;
 
   Env* env_;
-  // env_option optimized for compaction table reads
-  EnvOptions env_optiosn_for_read_;
   VersionSet* versions_;
   const std::atomic<bool>* shutting_down_;
-  const SequenceNumber preserve_deletes_seqnum_;
   LogBuffer* log_buffer_;
   Directory* db_directory_;
   Directory* output_directory_;
@@ -157,8 +149,6 @@ class CompactionJob {
   // should make sure not to remove evidence that a write occurred.
   SequenceNumber earliest_write_conflict_snapshot_;
 
-  const SnapshotChecker* const snapshot_checker_;
-
   std::shared_ptr<Cache> table_cache_;
 
   EventLogger* event_logger_;
@@ -170,7 +160,6 @@ class CompactionJob {
   std::vector<Slice> boundaries_;
   // Stores the approx size of keys covered in the range of each subcompaction
   std::vector<uint64_t> sizes_;
-  Env::WriteLifeTimeHint write_hint_;
 };
 
 }  // namespace rocksdb

db/compaction_job_stats_test.cc

@@ -426,7 +426,7 @@ class CompactionJobStatsChecker : public EventListener {
   // Once a compaction completed, this function will verify the returned
   // CompactionJobInfo with the oldest CompactionJobInfo added earlier
   // in "expected_stats_" which has not yet being used for verification.
-  virtual void OnCompactionCompleted(DB* /*db*/, const CompactionJobInfo& ci) {
+  virtual void OnCompactionCompleted(DB *db, const CompactionJobInfo& ci) {
     if (verify_next_comp_io_stats_) {
       ASSERT_GT(ci.stats.file_write_nanos, 0);
       ASSERT_GT(ci.stats.file_range_sync_nanos, 0);
@@ -806,7 +806,7 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
     stats_checker->set_verify_next_comp_io_stats(true);
     std::atomic<bool> first_prepare_write(true);
     rocksdb::SyncPoint::GetInstance()->SetCallBack(
-        "WritableFileWriter::Append:BeforePrepareWrite", [&](void* /*arg*/) {
+        "WritableFileWriter::Append:BeforePrepareWrite", [&](void* arg) {
           if (first_prepare_write.load()) {
             options.env->SleepForMicroseconds(3);
             first_prepare_write.store(false);
@@ -815,7 +815,7 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
 
     std::atomic<bool> first_flush(true);
     rocksdb::SyncPoint::GetInstance()->SetCallBack(
-        "WritableFileWriter::Flush:BeforeAppend", [&](void* /*arg*/) {
+        "WritableFileWriter::Flush:BeforeAppend", [&](void* arg) {
           if (first_flush.load()) {
             options.env->SleepForMicroseconds(3);
             first_flush.store(false);
@@ -824,7 +824,7 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
 
     std::atomic<bool> first_sync(true);
     rocksdb::SyncPoint::GetInstance()->SetCallBack(
-        "WritableFileWriter::SyncInternal:0", [&](void* /*arg*/) {
+        "WritableFileWriter::SyncInternal:0", [&](void* arg) {
           if (first_sync.load()) {
             options.env->SleepForMicroseconds(3);
             first_sync.store(false);
@@ -833,7 +833,7 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
 
     std::atomic<bool> first_range_sync(true);
     rocksdb::SyncPoint::GetInstance()->SetCallBack(
-        "WritableFileWriter::RangeSync:0", [&](void* /*arg*/) {
+        "WritableFileWriter::RangeSync:0", [&](void* arg) {
           if (first_range_sync.load()) {
             options.env->SleepForMicroseconds(3);
             first_range_sync.store(false);
@@ -1034,7 +1034,7 @@ int main(int argc, char** argv) {
 #else
 #include <stdio.h>
 
-int main(int /*argc*/, char** /*argv*/) {
+int main(int argc, char** argv) {
   fprintf(stderr, "SKIPPED, not supported in ROCKSDB_LITE\n");
   return 0;
 }
@@ -1043,5 +1043,5 @@ int main(int /*argc*/, char** /*argv*/) {
 
 #else
 
-int main(int /*argc*/, char** /*argv*/) { return 0; }
+int main(int argc, char** argv) { return 0; }
 #endif  // !defined(IOS_CROSS_COMPILE)

db/compaction_job_test.cc

@@ -76,7 +76,6 @@ class CompactionJobTest : public testing::Test {
                                  table_cache_.get(), &write_buffer_manager_,
                                  &write_controller_)),
         shutting_down_(false),
-        preserve_deletes_seqnum_(0),
         mock_table_factory_(new mock::MockTableFactory()) {
     EXPECT_OK(env_->CreateDirIfMissing(dbname_));
     db_options_.db_paths.emplace_back(dbname_,
@@ -143,8 +142,7 @@ class CompactionJobTest : public testing::Test {
   }
 
   void SetLastSequence(const SequenceNumber sequence_number) {
-    versions_->SetLastAllocatedSequence(sequence_number + 1);
-    versions_->SetLastPublishedSequence(sequence_number + 1);
+    versions_->SetLastToBeWrittenSequence(sequence_number + 1);
     versions_->SetLastSequence(sequence_number + 1);
   }
 
@@ -246,21 +244,18 @@ class CompactionJobTest : public testing::Test {
     Compaction compaction(cfd->current()->storage_info(), *cfd->ioptions(),
                           *cfd->GetLatestMutableCFOptions(),
                           compaction_input_files, 1, 1024 * 1024,
-                          10 * 1024 * 1024, 0, kNoCompression, 0, {}, true);
+                          10 * 1024 * 1024, 0, kNoCompression, {}, true);
     compaction.SetInputVersion(cfd->current());
 
     LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, db_options_.info_log.get());
     mutex_.Lock();
     EventLogger event_logger(db_options_.info_log.get());
-    // TODO(yiwu) add a mock snapshot checker and add test for it.
-    SnapshotChecker* snapshot_checker = nullptr;
-    CompactionJob compaction_job(0, &compaction, db_options_, env_options_,
-                                 versions_.get(), &shutting_down_,
-                                 preserve_deletes_seqnum_, &log_buffer,
-                                 nullptr, nullptr, nullptr, &mutex_, &bg_error_,
-                                 snapshots, earliest_write_conflict_snapshot,
-                                 snapshot_checker, table_cache_, &event_logger,
-                                 false, false, dbname_, &compaction_job_stats_);
+    CompactionJob compaction_job(
+        0, &compaction, db_options_, env_options_, versions_.get(),
+        &shutting_down_, &log_buffer, nullptr, nullptr, nullptr, &mutex_,
+        &bg_error_, snapshots, earliest_write_conflict_snapshot, table_cache_,
+        &event_logger, false, false, dbname_, &compaction_job_stats_);
+
     VerifyInitializationOfCompactionJobStats(compaction_job_stats_);
 
     compaction_job.Prepare();
@@ -296,7 +291,6 @@ class CompactionJobTest : public testing::Test {
   std::unique_ptr<VersionSet> versions_;
   InstrumentedMutex mutex_;
   std::atomic<bool> shutting_down_;
-  SequenceNumber preserve_deletes_seqnum_;
   std::shared_ptr<mock::MockTableFactory> mock_table_factory_;
   CompactionJobStats compaction_job_stats_;
   ColumnFamilyData* cfd_;
@@ -946,7 +940,7 @@ int main(int argc, char** argv) {
 #else
 #include <stdio.h>
 
-int main(int /*argc*/, char** /*argv*/) {
+int main(int argc, char** argv) {
   fprintf(stderr,
           "SKIPPED as CompactionJobStats is not supported in ROCKSDB_LITE\n");
   return 0;

db/compaction_picker.cc

@@ -41,7 +41,7 @@ bool FindIntraL0Compaction(const std::vector<FileMetaData*>& level_files,
                            size_t min_files_to_compact,
                            uint64_t max_compact_bytes_per_del_file,
                            CompactionInputFiles* comp_inputs) {
-  size_t compact_bytes = static_cast<size_t>(level_files[0]->fd.file_size);
+  size_t compact_bytes = level_files[0]->fd.file_size;
   size_t compact_bytes_per_del_file = port::kMaxSizet;
   // compaction range will be [0, span_len).
   size_t span_len;
@@ -59,7 +59,7 @@ bool FindIntraL0Compaction(const std::vector<FileMetaData*>& level_files,
   }
 
   if (span_len >= min_files_to_compact &&
-      compact_bytes_per_del_file < max_compact_bytes_per_del_file) {
+      new_compact_bytes_per_del_file < max_compact_bytes_per_del_file) {
     assert(comp_inputs != nullptr);
     comp_inputs->level = 0;
     for (size_t i = 0; i < span_len; ++i) {
@@ -89,7 +89,7 @@ CompressionType GetCompressionType(const ImmutableCFOptions& ioptions,
   // If bottommost_compression is set and we are compacting to the
   // bottommost level then we should use it.
   if (ioptions.bottommost_compression != kDisableCompressionOption &&
-      level >= (vstorage->num_non_empty_levels() - 1)) {
+      level > base_level && level >= (vstorage->num_non_empty_levels() - 1)) {
     return ioptions.bottommost_compression;
   }
   // If the user has specified a different compression level for each level,
@@ -199,7 +199,7 @@ void CompactionPicker::GetRange(const std::vector<CompactionInputFiles>& inputs,
   assert(initialized);
 }
 
-bool CompactionPicker::ExpandInputsToCleanCut(const std::string& /*cf_name*/,
+bool CompactionPicker::ExpandInputsToCleanCut(const std::string& cf_name,
                                               VersionStorageInfo* vstorage,
                                               CompactionInputFiles* inputs) {
   // This isn't good compaction
@@ -292,18 +292,25 @@ Compaction* CompactionPicker::CompactFiles(
     VersionStorageInfo* vstorage, const MutableCFOptions& mutable_cf_options,
     uint32_t output_path_id) {
   assert(input_files.size());
-  // This compaction output should not overlap with a running compaction as
-  // `SanitizeCompactionInputFiles` should've checked earlier and db mutex
-  // shouldn't have been released since.
-  assert(!FilesRangeOverlapWithCompaction(input_files, output_level));
 
+  // TODO(rven ): we might be able to run concurrent level 0 compaction
+  // if the key ranges of the two compactions do not overlap, but for now
+  // we do not allow it.
+  if ((input_files[0].level == 0) && !level0_compactions_in_progress_.empty()) {
+    return nullptr;
+  }
+  // This compaction output could overlap with a running compaction
+  if (FilesRangeOverlapWithCompaction(input_files, output_level)) {
+    return nullptr;
+  }
   auto c =
       new Compaction(vstorage, ioptions_, mutable_cf_options, input_files,
                      output_level, compact_options.output_file_size_limit,
                      mutable_cf_options.max_compaction_bytes, output_path_id,
-                     compact_options.compression,
-                     compact_options.max_subcompactions,
-                     /* grandparents */ {}, true);
+                     compact_options.compression, /* grandparents */ {}, true);
+
+  // If it's level 0 compaction, make sure we don't execute any other level 0
+  // compactions in parallel
   RegisterCompaction(c);
   return c;
 }
@@ -311,7 +318,7 @@ Compaction* CompactionPicker::CompactFiles(
 Status CompactionPicker::GetCompactionInputsFromFileNumbers(
     std::vector<CompactionInputFiles>* input_files,
     std::unordered_set<uint64_t>* input_set, const VersionStorageInfo* vstorage,
-    const CompactionOptions& /*compact_options*/) const {
+    const CompactionOptions& compact_options) const {
   if (input_set->size() == 0U) {
     return Status::InvalidArgument(
         "Compaction must include at least one file.");
@@ -389,10 +396,7 @@ bool CompactionPicker::SetupOtherInputs(
   assert(output_level_inputs->empty());
   const int input_level = inputs->level;
   const int output_level = output_level_inputs->level;
-  if (input_level == output_level) {
-    // no possibility of conflict
-    return true;
-  }
+  assert(input_level != output_level);
 
   // For now, we only support merging two levels, start level and output level.
   // We need to assert other levels are empty.
@@ -477,7 +481,7 @@ bool CompactionPicker::SetupOtherInputs(
       ROCKS_LOG_INFO(ioptions_.info_log,
                      "[%s] Expanding@%d %" ROCKSDB_PRIszt "+%" ROCKSDB_PRIszt
                      "(%" PRIu64 "+%" PRIu64 " bytes) to %" ROCKSDB_PRIszt
-                     "+%" ROCKSDB_PRIszt " (%" PRIu64 "+%" PRIu64 " bytes)\n",
+                     "+%" ROCKSDB_PRIszt " (%" PRIu64 "+%" PRIu64 "bytes)\n",
                      cf_name.c_str(), input_level, inputs->size(),
                      output_level_inputs->size(), inputs_size,
                      output_level_inputs_size, expanded_inputs.size(),
@@ -506,8 +510,7 @@ void CompactionPicker::GetGrandparents(
 Compaction* CompactionPicker::CompactRange(
     const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
     VersionStorageInfo* vstorage, int input_level, int output_level,
-    uint32_t output_path_id, uint32_t max_subcompactions,
-    const InternalKey* begin, const InternalKey* end,
+    uint32_t output_path_id, const InternalKey* begin, const InternalKey* end,
     InternalKey** compaction_end, bool* manual_conflict) {
   // CompactionPickerFIFO has its own implementation of compact range
   assert(ioptions_.compaction_style != kCompactionStyleFIFO);
@@ -569,12 +572,11 @@ Compaction* CompactionPicker::CompactRange(
 
     Compaction* c = new Compaction(
         vstorage, ioptions_, mutable_cf_options, std::move(inputs),
-        output_level, MaxFileSizeForLevel(mutable_cf_options, output_level,
-            ioptions_.compaction_style),
+        output_level, mutable_cf_options.MaxFileSizeForLevel(output_level),
         /* max_compaction_bytes */ LLONG_MAX, output_path_id,
         GetCompressionType(ioptions_, vstorage, mutable_cf_options,
                            output_level, 1),
-        max_subcompactions, /* grandparents */ {}, /* is manual */ true);
+        /* grandparents */ {}, /* is manual */ true);
     RegisterCompaction(c);
     return c;
   }
@@ -620,7 +622,7 @@ Compaction* CompactionPicker::CompactRange(
       }
     }
   }
-  assert(output_path_id < static_cast<uint32_t>(ioptions_.cf_paths.size()));
+  assert(output_path_id < static_cast<uint32_t>(ioptions_.db_paths.size()));
 
   if (ExpandInputsToCleanCut(cf_name, vstorage, &inputs) == false) {
     // manual compaction is now multi-threaded, so it can
@@ -677,15 +679,11 @@ Compaction* CompactionPicker::CompactRange(
   GetGrandparents(vstorage, inputs, output_level_inputs, &grandparents);
   Compaction* compaction = new Compaction(
       vstorage, ioptions_, mutable_cf_options, std::move(compaction_inputs),
-      output_level,
-      MaxFileSizeForLevel(mutable_cf_options, output_level,
-          ioptions_.compaction_style, vstorage->base_level(),
-          ioptions_.level_compaction_dynamic_level_bytes),
+      output_level, mutable_cf_options.MaxFileSizeForLevel(output_level),
       mutable_cf_options.max_compaction_bytes, output_path_id,
       GetCompressionType(ioptions_, vstorage, mutable_cf_options, output_level,
                          vstorage->base_level()),
-      /* max_subcompactions */ 0, std::move(grandparents),
-      /* is manual compaction */ true);
+      std::move(grandparents), /* is manual compaction */ true);
 
   TEST_SYNC_POINT_CALLBACK("CompactionPicker::CompactRange:Return", compaction);
   RegisterCompaction(compaction);
@@ -732,6 +730,10 @@ Status CompactionPicker::SanitizeCompactionInputFilesForAllLevels(
   auto& levels = cf_meta.levels;
   auto comparator = icmp_->user_comparator();
 
+  // TODO(yhchiang): If there is any input files of L1 or up and there
+  // is at least one L0 files. All L0 files older than the L0 file needs
+  // to be included. Otherwise, it is a false conditoin
+
   // TODO(yhchiang): add is_adjustable to CompactionOptions
 
   // the smallest and largest key of the current compaction input
@@ -792,8 +794,6 @@ Status CompactionPicker::SanitizeCompactionInputFilesForAllLevels(
         }
         last_included++;
       }
-    } else if (output_level > 0) {
-      last_included = static_cast<int>(current_files.size() - 1);
     }
 
     // include all files between the first and the last compaction input files.
@@ -949,12 +949,6 @@ void CompactionPicker::UnregisterCompaction(Compaction* c) {
 
 bool LevelCompactionPicker::NeedsCompaction(
     const VersionStorageInfo* vstorage) const {
-  if (!vstorage->ExpiredTtlFiles().empty()) {
-    return true;
-  }
-  if (!vstorage->BottommostFilesMarkedForCompaction().empty()) {
-    return true;
-  }
   if (!vstorage->FilesMarkedForCompaction().empty()) {
     return true;
   }
@@ -1021,8 +1015,6 @@ class LevelCompactionBuilder {
   // If there is any file marked for compaction, put put it into inputs.
   void PickFilesMarkedForCompaction();
 
-  void PickExpiredTtlFiles();
-
   const std::string& cf_name_;
   VersionStorageInfo* vstorage_;
   CompactionPicker* compaction_picker_;
@@ -1093,42 +1085,6 @@ void LevelCompactionBuilder::PickFilesMarkedForCompaction() {
   start_level_inputs_.files.clear();
 }
 
-void LevelCompactionBuilder::PickExpiredTtlFiles() {
-  if (vstorage_->ExpiredTtlFiles().empty()) {
-    return;
-  }
-
-  auto continuation = [&](std::pair<int, FileMetaData*> level_file) {
-    // If it's being compacted it has nothing to do here.
-    // If this assert() fails that means that some function marked some
-    // files as being_compacted, but didn't call ComputeCompactionScore()
-    assert(!level_file.second->being_compacted);
-    start_level_ = level_file.first;
-    output_level_ =
-        (start_level_ == 0) ? vstorage_->base_level() : start_level_ + 1;
-
-    if ((start_level_ == vstorage_->num_non_empty_levels() - 1) ||
-        (start_level_ == 0 &&
-         !compaction_picker_->level0_compactions_in_progress()->empty())) {
-      return false;
-    }
-
-    start_level_inputs_.files = {level_file.second};
-    start_level_inputs_.level = start_level_;
-    return compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_,
-                                                      &start_level_inputs_);
-  };
-
-  for (auto& level_file : vstorage_->ExpiredTtlFiles()) {
-    if (continuation(level_file)) {
-      // found the compaction!
-      return;
-    }
-  }
-
-  start_level_inputs_.files.clear();
-}
-
 void LevelCompactionBuilder::SetupInitialFiles() {
   // Find the compactions by size on all levels.
   bool skipped_l0_to_base = false;
@@ -1180,40 +1136,11 @@ void LevelCompactionBuilder::SetupInitialFiles() {
   // if we didn't find a compaction, check if there are any files marked for
   // compaction
   if (start_level_inputs_.empty()) {
+    is_manual_ = true;
     parent_index_ = base_index_ = -1;
-
     PickFilesMarkedForCompaction();
     if (!start_level_inputs_.empty()) {
-      is_manual_ = true;
       compaction_reason_ = CompactionReason::kFilesMarkedForCompaction;
-      return;
-    }
-
-    size_t i;
-    for (i = 0; i < vstorage_->BottommostFilesMarkedForCompaction().size();
-         ++i) {
-      auto& level_and_file = vstorage_->BottommostFilesMarkedForCompaction()[i];
-      assert(!level_and_file.second->being_compacted);
-      start_level_inputs_.level = output_level_ = start_level_ =
-          level_and_file.first;
-      start_level_inputs_.files = {level_and_file.second};
-      if (compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_,
-                                                     &start_level_inputs_)) {
-        break;
-      }
-    }
-    if (i == vstorage_->BottommostFilesMarkedForCompaction().size()) {
-      start_level_inputs_.clear();
-    } else {
-      assert(!start_level_inputs_.empty());
-      compaction_reason_ = CompactionReason::kBottommostFiles;
-      return;
-    }
-
-    assert(start_level_inputs_.empty());
-    PickExpiredTtlFiles();
-    if (!start_level_inputs_.empty()) {
-      compaction_reason_ = CompactionReason::kTtl;
     }
   }
 }
@@ -1315,16 +1242,13 @@ Compaction* LevelCompactionBuilder::PickCompaction() {
 Compaction* LevelCompactionBuilder::GetCompaction() {
   auto c = new Compaction(
       vstorage_, ioptions_, mutable_cf_options_, std::move(compaction_inputs_),
-      output_level_,
-      MaxFileSizeForLevel(mutable_cf_options_, output_level_,
-          ioptions_.compaction_style, vstorage_->base_level(),
-          ioptions_.level_compaction_dynamic_level_bytes),
+      output_level_, mutable_cf_options_.MaxFileSizeForLevel(output_level_),
       mutable_cf_options_.max_compaction_bytes,
       GetPathId(ioptions_, mutable_cf_options_, output_level_),
       GetCompressionType(ioptions_, vstorage_, mutable_cf_options_,
                          output_level_, vstorage_->base_level()),
-      /* max_subcompactions */ 0, std::move(grandparents_), is_manual_,
-      start_level_score_, false /* deletion_compaction */, compaction_reason_);
+      std::move(grandparents_), is_manual_, start_level_score_,
+      false /* deletion_compaction */, compaction_reason_);
 
   // If it's level 0 compaction, make sure we don't execute any other level 0
   // compactions in parallel
@@ -1347,47 +1271,32 @@ uint32_t LevelCompactionBuilder::GetPathId(
     const ImmutableCFOptions& ioptions,
     const MutableCFOptions& mutable_cf_options, int level) {
   uint32_t p = 0;
-  assert(!ioptions.cf_paths.empty());
+  assert(!ioptions.db_paths.empty());
 
   // size remaining in the most recent path
-  uint64_t current_path_size = ioptions.cf_paths[0].target_size;
+  uint64_t current_path_size = ioptions.db_paths[0].target_size;
 
   uint64_t level_size;
   int cur_level = 0;
 
-  // max_bytes_for_level_base denotes L1 size.
-  // We estimate L0 size to be the same as L1.
   level_size = mutable_cf_options.max_bytes_for_level_base;
 
   // Last path is the fallback
-  while (p < ioptions.cf_paths.size() - 1) {
+  while (p < ioptions.db_paths.size() - 1) {
     if (level_size <= current_path_size) {
       if (cur_level == level) {
         // Does desired level fit in this path?
         return p;
       } else {
         current_path_size -= level_size;
-        if (cur_level > 0) {
-          if (ioptions.level_compaction_dynamic_level_bytes) {
-            // Currently, level_compaction_dynamic_level_bytes is ignored when
-            // multiple db paths are specified. https://github.com/facebook/
-            // rocksdb/blob/master/db/column_family.cc.
-            // Still, adding this check to avoid accidentally using
-            // max_bytes_for_level_multiplier_additional
-            level_size = static_cast<uint64_t>(
-                level_size * mutable_cf_options.max_bytes_for_level_multiplier);
-          } else {
-            level_size = static_cast<uint64_t>(
-                level_size * mutable_cf_options.max_bytes_for_level_multiplier *
-                mutable_cf_options.MaxBytesMultiplerAdditional(cur_level));
-          }
-        }
+        level_size = static_cast<uint64_t>(
+            level_size * mutable_cf_options.max_bytes_for_level_multiplier);
         cur_level++;
         continue;
       }
     }
     p++;
-    current_path_size = ioptions.cf_paths[p].target_size;
+    current_path_size = ioptions.db_paths[p].target_size;
   }
   return p;
 }
@@ -1512,7 +1421,7 @@ uint64_t GetTotalFilesSize(
 Compaction* FIFOCompactionPicker::PickTTLCompaction(
     const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
     VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
-  assert(mutable_cf_options.compaction_options_fifo.ttl > 0);
+  assert(ioptions_.compaction_options_fifo.ttl > 0);
 
   const int kLevel0 = 0;
   const std::vector<FileMetaData*>& level_files = vstorage->LevelFiles(kLevel0);
@@ -1534,7 +1443,7 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction(
   inputs[0].level = 0;
 
   // avoid underflow
-  if (current_time > mutable_cf_options.compaction_options_fifo.ttl) {
+  if (current_time > ioptions_.compaction_options_fifo.ttl) {
     for (auto ritr = level_files.rbegin(); ritr != level_files.rend(); ++ritr) {
       auto f = *ritr;
       if (f->fd.table_reader != nullptr &&
@@ -1542,8 +1451,8 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction(
         auto creation_time =
             f->fd.table_reader->GetTableProperties()->creation_time;
         if (creation_time == 0 ||
-            creation_time >= (current_time -
-                              mutable_cf_options.compaction_options_fifo.ttl)) {
+            creation_time >=
+                (current_time - ioptions_.compaction_options_fifo.ttl)) {
           break;
         }
         total_size -= f->compensated_file_size;
@@ -1557,8 +1466,7 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction(
   // 2. there are a few files older than ttl, but deleting them will not bring
   //    the total size to be less than max_table_files_size threshold.
   if (inputs[0].files.empty() ||
-      total_size >
-          mutable_cf_options.compaction_options_fifo.max_table_files_size) {
+      total_size > ioptions_.compaction_options_fifo.max_table_files_size) {
     return nullptr;
   }
 
@@ -1572,9 +1480,8 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction(
 
   Compaction* c = new Compaction(
       vstorage, ioptions_, mutable_cf_options, std::move(inputs), 0, 0, 0, 0,
-      kNoCompression, /* max_subcompactions */ 0, {}, /* is manual */ false,
-      vstorage->CompactionScore(0), /* is deletion compaction */ true,
-      CompactionReason::kFIFOTtl);
+      kNoCompression, {}, /* is manual */ false, vstorage->CompactionScore(0),
+      /* is deletion compaction */ true, CompactionReason::kFIFOTtl);
   return c;
 }
 
@@ -1585,46 +1492,34 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction(
   const std::vector<FileMetaData*>& level_files = vstorage->LevelFiles(kLevel0);
   uint64_t total_size = GetTotalFilesSize(level_files);
 
-  if (total_size <=
-          mutable_cf_options.compaction_options_fifo.max_table_files_size ||
+  if (total_size <= ioptions_.compaction_options_fifo.max_table_files_size ||
       level_files.size() == 0) {
     // total size not exceeded
-    if (mutable_cf_options.compaction_options_fifo.allow_compaction &&
+    if (ioptions_.compaction_options_fifo.allow_compaction &&
         level_files.size() > 0) {
       CompactionInputFiles comp_inputs;
-      // try to prevent same files from being compacted multiple times, which
-      // could produce large files that may never TTL-expire. Achieve this by
-      // disallowing compactions with files larger than memtable (inflate its
-      // size by 10% to account for uncompressed L0 files that may have size
-      // slightly greater than memtable size limit).
-      size_t max_compact_bytes_per_del_file =
-          static_cast<size_t>(MultiplyCheckOverflow(
-              static_cast<uint64_t>(mutable_cf_options.write_buffer_size),
-              1.1));
       if (FindIntraL0Compaction(
               level_files,
               mutable_cf_options
-                  .level0_file_num_compaction_trigger /* min_files_to_compact */
-              ,
-              max_compact_bytes_per_del_file, &comp_inputs)) {
+                  .level0_file_num_compaction_trigger /* min_files_to_compact */,
+              mutable_cf_options.write_buffer_size, &comp_inputs)) {
         Compaction* c = new Compaction(
             vstorage, ioptions_, mutable_cf_options, {comp_inputs}, 0,
             16 * 1024 * 1024 /* output file size limit */,
             0 /* max compaction bytes, not applicable */,
-            0 /* output path ID */, mutable_cf_options.compression,
-            0 /* max_subcompactions */, {}, /* is manual */ false,
-            vstorage->CompactionScore(0), /* is deletion compaction */ false,
+            0 /* output path ID */, mutable_cf_options.compression, {},
+            /* is manual */ false, vstorage->CompactionScore(0),
+            /* is deletion compaction */ false,
             CompactionReason::kFIFOReduceNumFiles);
         return c;
       }
     }
 
-    ROCKS_LOG_BUFFER(
-        log_buffer,
-        "[%s] FIFO compaction: nothing to do. Total size %" PRIu64
-        ", max size %" PRIu64 "\n",
-        cf_name.c_str(), total_size,
-        mutable_cf_options.compaction_options_fifo.max_table_files_size);
+    ROCKS_LOG_BUFFER(log_buffer,
+                     "[%s] FIFO compaction: nothing to do. Total size %" PRIu64
+                     ", max size %" PRIu64 "\n",
+                     cf_name.c_str(), total_size,
+                     ioptions_.compaction_options_fifo.max_table_files_size);
     return nullptr;
   }
 
@@ -1651,17 +1546,15 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction(
                      "[%s] FIFO compaction: picking file %" PRIu64
                      " with size %s for deletion",
                      cf_name.c_str(), f->fd.GetNumber(), tmp_fsize);
-    if (total_size <=
-        mutable_cf_options.compaction_options_fifo.max_table_files_size) {
+    if (total_size <= ioptions_.compaction_options_fifo.max_table_files_size) {
       break;
     }
   }
 
   Compaction* c = new Compaction(
       vstorage, ioptions_, mutable_cf_options, std::move(inputs), 0, 0, 0, 0,
-      kNoCompression, /* max_subcompactions */ 0, {}, /* is manual */ false,
-      vstorage->CompactionScore(0), /* is deletion compaction */ true,
-      CompactionReason::kFIFOMaxSize);
+      kNoCompression, {}, /* is manual */ false, vstorage->CompactionScore(0),
+      /* is deletion compaction */ true, CompactionReason::kFIFOMaxSize);
   return c;
 }
 
@@ -1671,7 +1564,7 @@ Compaction* FIFOCompactionPicker::PickCompaction(
   assert(vstorage->num_levels() == 1);
 
   Compaction* c = nullptr;
-  if (mutable_cf_options.compaction_options_fifo.ttl > 0) {
+  if (ioptions_.compaction_options_fifo.ttl > 0) {
     c = PickTTLCompaction(cf_name, mutable_cf_options, vstorage, log_buffer);
   }
   if (c == nullptr) {
@@ -1684,13 +1577,8 @@ Compaction* FIFOCompactionPicker::PickCompaction(
 Compaction* FIFOCompactionPicker::CompactRange(
     const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
     VersionStorageInfo* vstorage, int input_level, int output_level,
-    uint32_t /*output_path_id*/, uint32_t /*max_subcompactions*/,
-    const InternalKey* /*begin*/, const InternalKey* /*end*/,
-    InternalKey** compaction_end, bool* /*manual_conflict*/) {
-#ifdef NDEBUG
-  (void)input_level;
-  (void)output_level;
-#endif
+    uint32_t output_path_id, const InternalKey* begin, const InternalKey* end,
+    InternalKey** compaction_end, bool* manual_conflict) {
   assert(input_level == 0);
   assert(output_level == 0);
   *compaction_end = nullptr;

db/compaction_picker.h

@@ -58,8 +58,7 @@ class CompactionPicker {
   virtual Compaction* CompactRange(
       const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
       VersionStorageInfo* vstorage, int input_level, int output_level,
-      uint32_t output_path_id, uint32_t max_subcompactions,
-      const InternalKey* begin, const InternalKey* end,
+      uint32_t output_path_id, const InternalKey* begin, const InternalKey* end,
       InternalKey** compaction_end, bool* manual_conflict);
 
   // The maximum allowed output level.  Default value is NumberLevels() - 1.
@@ -89,10 +88,6 @@ class CompactionPicker {
 
   // Takes a list of CompactionInputFiles and returns a (manual) Compaction
   // object.
-  //
-  // Caller must provide a set of input files that has been passed through
-  // `SanitizeCompactionInputFiles` earlier. The lock should not be released
-  // between that call and this one.
   Compaction* CompactFiles(const CompactionOptions& compact_options,
                            const std::vector<CompactionInputFiles>& input_files,
                            int output_level, VersionStorageInfo* vstorage,
@@ -239,8 +234,7 @@ class FIFOCompactionPicker : public CompactionPicker {
   virtual Compaction* CompactRange(
       const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
       VersionStorageInfo* vstorage, int input_level, int output_level,
-      uint32_t output_path_id, uint32_t max_subcompactions,
-      const InternalKey* begin, const InternalKey* end,
+      uint32_t output_path_id, const InternalKey* begin, const InternalKey* end,
       InternalKey** compaction_end, bool* manual_conflict) override;
 
   // The maximum allowed output level.  Always returns 0.
@@ -269,30 +263,27 @@ class NullCompactionPicker : public CompactionPicker {
   virtual ~NullCompactionPicker() {}
 
   // Always return "nullptr"
-  Compaction* PickCompaction(const std::string& /*cf_name*/,
-                             const MutableCFOptions& /*mutable_cf_options*/,
-                             VersionStorageInfo* /*vstorage*/,
-                             LogBuffer* /*log_buffer*/) override {
+  Compaction* PickCompaction(const std::string& cf_name,
+                             const MutableCFOptions& mutable_cf_options,
+                             VersionStorageInfo* vstorage,
+                             LogBuffer* log_buffer) override {
     return nullptr;
   }
 
   // Always return "nullptr"
-  Compaction* CompactRange(const std::string& /*cf_name*/,
-                           const MutableCFOptions& /*mutable_cf_options*/,
-                           VersionStorageInfo* /*vstorage*/,
-                           int /*input_level*/, int /*output_level*/,
-                           uint32_t /*output_path_id*/,
-                           uint32_t /*max_subcompactions*/,
-                           const InternalKey* /*begin*/,
-                           const InternalKey* /*end*/,
-                           InternalKey** /*compaction_end*/,
-                           bool* /*manual_conflict*/) override {
+  Compaction* CompactRange(const std::string& cf_name,
+                           const MutableCFOptions& mutable_cf_options,
+                           VersionStorageInfo* vstorage, int input_level,
+                           int output_level, uint32_t output_path_id,
+                           const InternalKey* begin, const InternalKey* end,
+                           InternalKey** compaction_end,
+                           bool* manual_conflict) override {
     return nullptr;
   }
 
   // Always returns false.
   virtual bool NeedsCompaction(
-      const VersionStorageInfo* /*vstorage*/) const override {
+      const VersionStorageInfo* vstorage) const override {
     return false;
   }
 };

db/compaction_picker_test.cc

@@ -20,9 +20,7 @@ namespace rocksdb {
 class CountingLogger : public Logger {
  public:
   using Logger::Logv;
-  virtual void Logv(const char* /*format*/, va_list /*ap*/) override {
-    log_count++;
-  }
+  virtual void Logv(const char* format, va_list ap) override { log_count++; }
   size_t log_count;
 };
 
@@ -59,7 +57,7 @@ class CompactionPickerTest : public testing::Test {
         vstorage_(nullptr) {
     fifo_options_.max_table_files_size = 1;
     mutable_cf_options_.RefreshDerivedOptions(ioptions_);
-    ioptions_.cf_paths.emplace_back("dummy",
+    ioptions_.db_paths.emplace_back("dummy",
                                     std::numeric_limits<uint64_t>::max());
   }
 
@@ -177,8 +175,6 @@ TEST_F(CompactionPickerTest, Level1Trigger) {
 }
 
 TEST_F(CompactionPickerTest, Level1Trigger2) {
-  mutable_cf_options_.target_file_size_base = 10000000000;
-  mutable_cf_options_.RefreshDerivedOptions(ioptions_);
   NewVersionStorage(6, kCompactionStyleLevel);
   Add(1, 66U, "150", "200", 1000000001U);
   Add(1, 88U, "201", "300", 1000000000U);
@@ -195,14 +191,13 @@ TEST_F(CompactionPickerTest, Level1Trigger2) {
   ASSERT_EQ(66U, compaction->input(0, 0)->fd.GetNumber());
   ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber());
   ASSERT_EQ(7U, compaction->input(1, 1)->fd.GetNumber());
-  ASSERT_EQ(uint64_t{1073741824}, compaction->OutputFilePreallocationSize());
 }
 
 TEST_F(CompactionPickerTest, LevelMaxScore) {
   NewVersionStorage(6, kCompactionStyleLevel);
   mutable_cf_options_.target_file_size_base = 10000000;
+  mutable_cf_options_.target_file_size_multiplier = 10;
   mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
-  mutable_cf_options_.RefreshDerivedOptions(ioptions_);
   Add(0, 1U, "150", "200", 1000000U);
   // Level 1 score 1.2
   Add(1, 66U, "150", "200", 6000000U);
@@ -223,9 +218,6 @@ TEST_F(CompactionPickerTest, LevelMaxScore) {
   ASSERT_TRUE(compaction.get() != nullptr);
   ASSERT_EQ(1U, compaction->num_input_files(0));
   ASSERT_EQ(7U, compaction->input(0, 0)->fd.GetNumber());
-  ASSERT_EQ(mutable_cf_options_.target_file_size_base +
-                mutable_cf_options_.target_file_size_base / 10,
-            compaction->OutputFilePreallocationSize());
 }
 
 TEST_F(CompactionPickerTest, NeedsCompactionLevel) {
@@ -445,7 +437,7 @@ TEST_F(CompactionPickerTest, CompactionUniversalIngestBehindReservedLevel) {
 TEST_F(CompactionPickerTest, CannotTrivialMoveUniversal) {
   const uint64_t kFileSize = 100000;
 
-  mutable_cf_options_.compaction_options_universal.allow_trivial_move = true;
+  ioptions_.compaction_options_universal.allow_trivial_move = true;
   NewVersionStorage(1, kCompactionStyleUniversal);
   UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
   // must return false when there's no files.
@@ -476,7 +468,7 @@ TEST_F(CompactionPickerTest, CannotTrivialMoveUniversal) {
 TEST_F(CompactionPickerTest, AllowsTrivialMoveUniversal) {
   const uint64_t kFileSize = 100000;
 
-  mutable_cf_options_.compaction_options_universal.allow_trivial_move = true;
+  ioptions_.compaction_options_universal.allow_trivial_move = true;
   UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
 
   NewVersionStorage(3, kCompactionStyleUniversal);
@@ -504,7 +496,7 @@ TEST_F(CompactionPickerTest, NeedsCompactionFIFO) {
   const uint64_t kMaxSize = kFileSize * kFileCount / 2;
 
   fifo_options_.max_table_files_size = kMaxSize;
-  mutable_cf_options_.compaction_options_fifo = fifo_options_;
+  ioptions_.compaction_options_fifo = fifo_options_;
   FIFOCompactionPicker fifo_compaction_picker(ioptions_, &icmp_);
   UpdateVersionStorageInfo();
   // must return false when there's no files.
@@ -529,10 +521,9 @@ TEST_F(CompactionPickerTest, NeedsCompactionFIFO) {
 TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) {
   NewVersionStorage(6, kCompactionStyleLevel);
   ioptions_.compaction_pri = kMinOverlappingRatio;
-  mutable_cf_options_.target_file_size_base = 100000000000;
+  mutable_cf_options_.target_file_size_base = 10000000;
   mutable_cf_options_.target_file_size_multiplier = 10;
   mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
-  mutable_cf_options_.RefreshDerivedOptions(ioptions_);
 
   Add(2, 6U, "150", "179", 50000000U);
   Add(2, 7U, "180", "220", 50000000U);
@@ -552,8 +543,6 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) {
   ASSERT_EQ(1U, compaction->num_input_files(0));
   // Pick file 8 because it overlaps with 0 files on level 3.
   ASSERT_EQ(8U, compaction->input(0, 0)->fd.GetNumber());
-  // Compaction input size * 1.1
-  ASSERT_GE(uint64_t{55000000}, compaction->OutputFilePreallocationSize());
 }
 
 TEST_F(CompactionPickerTest, CompactionPriMinOverlapping2) {

db/compaction_picker_universal.cc

@@ -35,7 +35,7 @@ namespace {
 // and the index of the file in that level
 
 struct InputFileInfo {
-  InputFileInfo() : f(nullptr), level(0), index(0) {}
+  InputFileInfo() : f(nullptr) {}
 
   FileMetaData* f;
   size_t level;
@@ -204,8 +204,7 @@ void UniversalCompactionPicker::SortedRun::DumpSizeInfo(
 
 std::vector<UniversalCompactionPicker::SortedRun>
 UniversalCompactionPicker::CalculateSortedRuns(
-    const VersionStorageInfo& vstorage, const ImmutableCFOptions& /*ioptions*/,
-    const MutableCFOptions& mutable_cf_options) {
+    const VersionStorageInfo& vstorage, const ImmutableCFOptions& ioptions) {
   std::vector<UniversalCompactionPicker::SortedRun> ret;
   for (FileMetaData* f : vstorage.LevelFiles(0)) {
     ret.emplace_back(0, f, f->fd.GetFileSize(), f->compensated_file_size,
@@ -219,8 +218,7 @@ UniversalCompactionPicker::CalculateSortedRuns(
     for (FileMetaData* f : vstorage.LevelFiles(level)) {
       total_compensated_size += f->compensated_file_size;
       total_size += f->fd.GetFileSize();
-      if (mutable_cf_options.compaction_options_universal.allow_trivial_move ==
-          true) {
+      if (ioptions.compaction_options_universal.allow_trivial_move == true) {
         if (f->being_compacted) {
           being_compacted = f->being_compacted;
         }
@@ -229,8 +227,7 @@ UniversalCompactionPicker::CalculateSortedRuns(
         // non-zero level, all the files should share the same being_compacted
         // value.
         // This assumption is only valid when
-        // mutable_cf_options.compaction_options_universal.allow_trivial_move is
-        // false
+        // ioptions.compaction_options_universal.allow_trivial_move is false
         assert(is_first || f->being_compacted == being_compacted);
       }
       if (is_first) {
@@ -248,13 +245,14 @@ UniversalCompactionPicker::CalculateSortedRuns(
 
 // Universal style of compaction. Pick files that are contiguous in
 // time-range to compact.
+//
 Compaction* UniversalCompactionPicker::PickCompaction(
     const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
     VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
   const int kLevel0 = 0;
   double score = vstorage->CompactionScore(kLevel0);
   std::vector<SortedRun> sorted_runs =
-      CalculateSortedRuns(*vstorage, ioptions_, mutable_cf_options);
+      CalculateSortedRuns(*vstorage, ioptions_);
 
   if (sorted_runs.size() == 0 ||
       sorted_runs.size() <
@@ -281,8 +279,7 @@ Compaction* UniversalCompactionPicker::PickCompaction(
   } else {
     // Size amplification is within limits. Try reducing read
     // amplification while maintaining file size ratios.
-    unsigned int ratio =
-        mutable_cf_options.compaction_options_universal.size_ratio;
+    unsigned int ratio = ioptions_.compaction_options_universal.size_ratio;
 
     if ((c = PickCompactionToReduceSortedRuns(
              cf_name, mutable_cf_options, vstorage, score, ratio, UINT_MAX,
@@ -330,8 +327,7 @@ Compaction* UniversalCompactionPicker::PickCompaction(
     return nullptr;
   }
 
-  if (mutable_cf_options.compaction_options_universal.allow_trivial_move ==
-      true) {
+  if (ioptions_.compaction_options_universal.allow_trivial_move == true) {
     c->set_is_trivial_move(IsInputFilesNonOverlapping(c));
   }
 
@@ -385,8 +381,7 @@ Compaction* UniversalCompactionPicker::PickCompaction(
 }
 
 uint32_t UniversalCompactionPicker::GetPathId(
-    const ImmutableCFOptions& ioptions,
-    const MutableCFOptions& mutable_cf_options, uint64_t file_size) {
+    const ImmutableCFOptions& ioptions, uint64_t file_size) {
   // Two conditions need to be satisfied:
   // (1) the target path needs to be able to hold the file's size
   // (2) Total size left in this and previous paths need to be not
@@ -403,12 +398,12 @@ uint32_t UniversalCompactionPicker::GetPathId(
   // that case. We need to improve it.
   uint64_t accumulated_size = 0;
   uint64_t future_size =
-      file_size *
-      (100 - mutable_cf_options.compaction_options_universal.size_ratio) / 100;
+      file_size * (100 - ioptions.compaction_options_universal.size_ratio) /
+      100;
   uint32_t p = 0;
-  assert(!ioptions.cf_paths.empty());
-  for (; p < ioptions.cf_paths.size() - 1; p++) {
-    uint64_t target_size = ioptions.cf_paths[p].target_size;
+  assert(!ioptions.db_paths.empty());
+  for (; p < ioptions.db_paths.size() - 1; p++) {
+    uint64_t target_size = ioptions.db_paths[p].target_size;
     if (target_size > file_size &&
         accumulated_size + (target_size - file_size) > future_size) {
       return p;
@@ -428,9 +423,9 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSortedRuns(
     unsigned int max_number_of_files_to_compact,
     const std::vector<SortedRun>& sorted_runs, LogBuffer* log_buffer) {
   unsigned int min_merge_width =
-      mutable_cf_options.compaction_options_universal.min_merge_width;
+      ioptions_.compaction_options_universal.min_merge_width;
   unsigned int max_merge_width =
-      mutable_cf_options.compaction_options_universal.max_merge_width;
+      ioptions_.compaction_options_universal.max_merge_width;
 
   const SortedRun* sr = nullptr;
   bool done = false;
@@ -497,7 +492,7 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSortedRuns(
       if (sz < static_cast<double>(succeeding_sr->size)) {
         break;
       }
-      if (mutable_cf_options.compaction_options_universal.stop_style ==
+      if (ioptions_.compaction_options_universal.stop_style ==
           kCompactionStopStyleSimilarSize) {
         // Similar-size stopping rule: also check the last picked file isn't
         // far larger than the next candidate file.
@@ -540,7 +535,7 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSortedRuns(
   // size ratio of compression.
   bool enable_compression = true;
   int ratio_to_compress =
-      mutable_cf_options.compaction_options_universal.compression_size_percent;
+      ioptions_.compaction_options_universal.compression_size_percent;
   if (ratio_to_compress >= 0) {
     uint64_t total_size = 0;
     for (auto& sorted_run : sorted_runs) {
@@ -561,8 +556,7 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSortedRuns(
   for (unsigned int i = 0; i < first_index_after; i++) {
     estimated_total_size += sorted_runs[i].size;
   }
-  uint32_t path_id =
-      GetPathId(ioptions_, mutable_cf_options, estimated_total_size);
+  uint32_t path_id = GetPathId(ioptions_, estimated_total_size);
   int start_level = sorted_runs[start_index].level;
   int output_level;
   if (first_index_after == sorted_runs.size()) {
@@ -603,19 +597,17 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSortedRuns(
 
   CompactionReason compaction_reason;
   if (max_number_of_files_to_compact == UINT_MAX) {
-    compaction_reason = CompactionReason::kUniversalSizeRatio;
-  } else {
     compaction_reason = CompactionReason::kUniversalSortedRunNum;
+  } else {
+    compaction_reason = CompactionReason::kUniversalSizeRatio;
   }
   return new Compaction(
       vstorage, ioptions_, mutable_cf_options, std::move(inputs), output_level,
-      MaxFileSizeForLevel(mutable_cf_options, output_level,
-                          kCompactionStyleUniversal),
-      LLONG_MAX, path_id,
+      mutable_cf_options.MaxFileSizeForLevel(output_level), LLONG_MAX, path_id,
       GetCompressionType(ioptions_, vstorage, mutable_cf_options, start_level,
                          1, enable_compression),
-      /* max_subcompactions */ 0, /* grandparents */ {}, /* is manual */ false,
-      score, false /* deletion_compaction */, compaction_reason);
+      /* grandparents */ {}, /* is manual */ false, score,
+      false /* deletion_compaction */, compaction_reason);
 }
 
 // Look at overall size amplification. If size amplification
@@ -629,8 +621,8 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSizeAmp(
     VersionStorageInfo* vstorage, double score,
     const std::vector<SortedRun>& sorted_runs, LogBuffer* log_buffer) {
   // percentage flexibility while reducing size amplification
-  uint64_t ratio = mutable_cf_options.compaction_options_universal
-                       .max_size_amplification_percent;
+  uint64_t ratio =
+      ioptions_.compaction_options_universal.max_size_amplification_percent;
 
   unsigned int candidate_count = 0;
   uint64_t candidate_size = 0;
@@ -708,8 +700,7 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSizeAmp(
   for (size_t loop = start_index; loop < sorted_runs.size(); loop++) {
     estimated_total_size += sorted_runs[loop].size;
   }
-  uint32_t path_id =
-      GetPathId(ioptions_, mutable_cf_options, estimated_total_size);
+  uint32_t path_id = GetPathId(ioptions_, estimated_total_size);
   int start_level = sorted_runs[start_index].level;
 
   std::vector<CompactionInputFiles> inputs(vstorage->num_levels());
@@ -744,13 +735,12 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSizeAmp(
 
   return new Compaction(
       vstorage, ioptions_, mutable_cf_options, std::move(inputs),
-      output_level, MaxFileSizeForLevel(mutable_cf_options, output_level,
-          kCompactionStyleUniversal),
+      output_level, mutable_cf_options.MaxFileSizeForLevel(output_level),
       /* max_grandparent_overlap_bytes */ LLONG_MAX, path_id,
       GetCompressionType(ioptions_, vstorage, mutable_cf_options,
                          output_level, 1),
-      /* max_subcompactions */ 0, /* grandparents */ {}, /* is manual */ false,
-      score, false /* deletion_compaction */,
+      /* grandparents */ {}, /* is manual */ false, score,
+      false /* deletion_compaction */,
       CompactionReason::kUniversalSizeAmplification);
 }
 }  // namespace rocksdb

db/compaction_picker_universal.h

@@ -80,13 +80,11 @@ class UniversalCompactionPicker : public CompactionPicker {
   bool IsInputFilesNonOverlapping(Compaction* c);
 
   static std::vector<SortedRun> CalculateSortedRuns(
-      const VersionStorageInfo& vstorage, const ImmutableCFOptions& ioptions,
-      const MutableCFOptions& mutable_cf_options);
+      const VersionStorageInfo& vstorage, const ImmutableCFOptions& ioptions);
 
   // Pick a path ID to place a newly generated file, with its estimated file
   // size.
   static uint32_t GetPathId(const ImmutableCFOptions& ioptions,
-                            const MutableCFOptions& mutable_cf_options,
                             uint64_t file_size);
 };
 }  // namespace rocksdb

db/comparator_db_test.cc

@@ -2,7 +2,6 @@
 //  This source code is licensed under both the GPLv2 (found in the
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
-#include <array>
 #include <map>
 #include <string>
 
@@ -189,10 +188,10 @@ class DoubleComparator : public Comparator {
       return -1;
     }
   }
-  virtual void FindShortestSeparator(std::string* /*start*/,
-                                     const Slice& /*limit*/) const override {}
+  virtual void FindShortestSeparator(std::string* start,
+                                     const Slice& limit) const override {}
 
-  virtual void FindShortSuccessor(std::string* /*key*/) const override {}
+  virtual void FindShortSuccessor(std::string* key) const override {}
 };
 
 class HashComparator : public Comparator {
@@ -212,10 +211,10 @@ class HashComparator : public Comparator {
       return -1;
     }
   }
-  virtual void FindShortestSeparator(std::string* /*start*/,
-                                     const Slice& /*limit*/) const override {}
+  virtual void FindShortestSeparator(std::string* start,
+                                     const Slice& limit) const override {}
 
-  virtual void FindShortSuccessor(std::string* /*key*/) const override {}
+  virtual void FindShortSuccessor(std::string* key) const override {}
 };
 
 class TwoStrComparator : public Comparator {
@@ -244,10 +243,10 @@ class TwoStrComparator : public Comparator {
     }
     return a2.compare(b2);
   }
-  virtual void FindShortestSeparator(std::string* /*start*/,
-                                     const Slice& /*limit*/) const override {}
+  virtual void FindShortestSeparator(std::string* start,
+                                     const Slice& limit) const override {}
 
-  virtual void FindShortSuccessor(std::string* /*key*/) const override {}
+  virtual void FindShortSuccessor(std::string* key) const override {}
 };
 }  // namespace
 
@@ -434,145 +433,6 @@ TEST_F(ComparatorDBTest, TwoStrComparator) {
   }
 }
 
-TEST_F(ComparatorDBTest, FindShortestSeparator) {
-  std::string s1 = "abc1xyz";
-  std::string s2 = "abc3xy";
-
-  BytewiseComparator()->FindShortestSeparator(&s1, s2);
-  ASSERT_EQ("abc2", s1);
-
-  s1 = "abc5xyztt";
-
-  ReverseBytewiseComparator()->FindShortestSeparator(&s1, s2);
-  ASSERT_EQ("abc5", s1);
-
-  s1 = "abc3";
-  s2 = "abc2xy";
-  ReverseBytewiseComparator()->FindShortestSeparator(&s1, s2);
-  ASSERT_EQ("abc3", s1);
-
-  s1 = "abc3xyz";
-  s2 = "abc2xy";
-  ReverseBytewiseComparator()->FindShortestSeparator(&s1, s2);
-  ASSERT_EQ("abc3", s1);
-
-  s1 = "abc3xyz";
-  s2 = "abc2";
-  ReverseBytewiseComparator()->FindShortestSeparator(&s1, s2);
-  ASSERT_EQ("abc3", s1);
-
-  std::string old_s1 = s1 = "abc2xy";
-  s2 = "abc2";
-  ReverseBytewiseComparator()->FindShortestSeparator(&s1, s2);
-  ASSERT_TRUE(old_s1 >= s1);
-  ASSERT_TRUE(s1 > s2);
-}
-
-TEST_F(ComparatorDBTest, SeparatorSuccessorRandomizeTest) {
-  // Char list for boundary cases.
-  std::array<unsigned char, 6> char_list{{0, 1, 2, 253, 254, 255}};
-  Random rnd(301);
-
-  for (int attempts = 0; attempts < 1000; attempts++) {
-    uint32_t size1 = rnd.Skewed(4);
-    uint32_t size2;
-
-    if (rnd.OneIn(2)) {
-      // size2 to be random size
-      size2 = rnd.Skewed(4);
-    } else {
-      // size1 is within [-2, +2] of size1
-      int diff = static_cast<int>(rnd.Uniform(5)) - 2;
-      int tmp_size2 = static_cast<int>(size1) + diff;
-      if (tmp_size2 < 0) {
-        tmp_size2 = 0;
-      }
-      size2 = static_cast<uint32_t>(tmp_size2);
-    }
-
-    std::string s1;
-    std::string s2;
-    for (uint32_t i = 0; i < size1; i++) {
-      if (rnd.OneIn(2)) {
-        // Use random byte
-        s1 += static_cast<char>(rnd.Uniform(256));
-      } else {
-        // Use one byte in char_list
-        char c = static_cast<char>(char_list[rnd.Uniform(sizeof(char_list))]);
-        s1 += c;
-      }
-    }
-
-    // First set s2 to be the same as s1, and then modify s2.
-    s2 = s1;
-    s2.resize(size2);
-    // We start from the back of the string
-    if (size2 > 0) {
-      uint32_t pos = size2 - 1;
-      do {
-        if (pos >= size1 || rnd.OneIn(4)) {
-          // For 1/4 chance, use random byte
-          s2[pos] = static_cast<char>(rnd.Uniform(256));
-        } else if (rnd.OneIn(4)) {
-          // In 1/4 chance, stop here.
-          break;
-        } else {
-          // Create a char within [-2, +2] of the matching char of s1.
-          int diff = static_cast<int>(rnd.Uniform(5)) - 2;
-          // char may be signed or unsigned based on platform.
-          int s1_char = static_cast<int>(static_cast<unsigned char>(s1[pos]));
-          int s2_char = s1_char + diff;
-          if (s2_char < 0) {
-            s2_char = 0;
-          }
-          if (s2_char > 255) {
-            s2_char = 255;
-          }
-          s2[pos] = static_cast<char>(s2_char);
-        }
-      } while (pos-- != 0);
-    }
-
-    // Test separators
-    for (int rev = 0; rev < 2; rev++) {
-      if (rev == 1) {
-        // switch s1 and s2
-        std::string t = s1;
-        s1 = s2;
-        s2 = t;
-      }
-      std::string separator = s1;
-      BytewiseComparator()->FindShortestSeparator(&separator, s2);
-      std::string rev_separator = s1;
-      ReverseBytewiseComparator()->FindShortestSeparator(&rev_separator, s2);
-
-      if (s1 == s2) {
-        ASSERT_EQ(s1, separator);
-        ASSERT_EQ(s2, rev_separator);
-      } else if (s1 < s2) {
-        ASSERT_TRUE(s1 <= separator);
-        ASSERT_TRUE(s2 > separator);
-        ASSERT_LE(separator.size(), std::max(s1.size(), s2.size()));
-        ASSERT_EQ(s1, rev_separator);
-      } else {
-        ASSERT_TRUE(s1 >= rev_separator);
-        ASSERT_TRUE(s2 < rev_separator);
-        ASSERT_LE(rev_separator.size(), std::max(s1.size(), s2.size()));
-        ASSERT_EQ(s1, separator);
-      }
-    }
-
-    // Test successors
-    std::string succ = s1;
-    BytewiseComparator()->FindShortSuccessor(&succ);
-    ASSERT_TRUE(succ >= s1);
-
-    succ = s1;
-    ReverseBytewiseComparator()->FindShortSuccessor(&succ);
-    ASSERT_TRUE(succ <= s1);
-  }
-}
-
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/convenience.cc

@@ -19,17 +19,9 @@ void CancelAllBackgroundWork(DB* db, bool wait) {
 }
 
 Status DeleteFilesInRange(DB* db, ColumnFamilyHandle* column_family,
-                          const Slice* begin, const Slice* end,
-                          bool include_end) {
-  RangePtr range(begin, end);
-  return DeleteFilesInRanges(db, column_family, &range, 1, include_end);
-}
-
-Status DeleteFilesInRanges(DB* db, ColumnFamilyHandle* column_family,
-                           const RangePtr* ranges, size_t n,
-                           bool include_end) {
+                          const Slice* begin, const Slice* end) {
   return (static_cast_with_check<DBImpl, DB>(db->GetRootDB()))
-      ->DeleteFilesInRanges(column_family, ranges, n, include_end);
+      ->DeleteFilesInRange(column_family, begin, end);
 }
 
 Status VerifySstFileChecksum(const Options& options,
@@ -50,9 +42,8 @@ Status VerifySstFileChecksum(const Options& options,
   std::unique_ptr<RandomAccessFileReader> file_reader(
       new RandomAccessFileReader(std::move(file), file_path));
   s = ioptions.table_factory->NewTableReader(
-      TableReaderOptions(ioptions, options.prefix_extractor.get(), env_options,
-                         internal_comparator, false /* skip_filters */,
-                         -1 /* level */),
+      TableReaderOptions(ioptions, env_options, internal_comparator,
+                         false /* skip_filters */, -1 /* level */),
       std::move(file_reader), file_size, &table_reader,
       false /* prefetch_index_and_filter_in_cache */);
   if (!s.ok()) {

db/corruption_test.cc

@@ -333,9 +333,9 @@ TEST_F(CorruptionTest, TableFileIndexData) {
   Corrupt(kTableFile, -2000, 500);
   Reopen();
   dbi = reinterpret_cast<DBImpl*>(db_);
-  // one full file may be readable, since only one was corrupted
+  // one full file should be readable, since only one was corrupted
   // the other file should be fully non-readable, since index was corrupted
-  Check(0, 5000);
+  Check(5000, 5000);
   ASSERT_NOK(dbi->VerifyChecksum());
 }
 
@@ -510,7 +510,7 @@ int main(int argc, char** argv) {
 #else
 #include <stdio.h>
 
-int main(int /*argc*/, char** /*argv*/) {
+int main(int argc, char** argv) {
   fprintf(stderr, "SKIPPED as RepairDB() is not supported in ROCKSDB_LITE\n");
   return 0;
 }

db/cuckoo_table_db_test.cc

@@ -241,7 +241,7 @@ TEST_F(CuckooTableDBTest, CompactionIntoMultipleFiles) {
 
   // Write 28 values, each 10016 B ~ 10KB
   for (int idx = 0; idx < 28; ++idx) {
-    ASSERT_OK(Put(Key(idx), std::string(10000, 'a' + char(idx))));
+    ASSERT_OK(Put(Key(idx), std::string(10000, 'a' + idx)));
   }
   dbfull()->TEST_WaitForFlushMemTable();
   ASSERT_EQ("1", FilesPerLevel());
@@ -250,7 +250,7 @@ TEST_F(CuckooTableDBTest, CompactionIntoMultipleFiles) {
                               true /* disallow trivial move */);
   ASSERT_EQ("0,2", FilesPerLevel());
   for (int idx = 0; idx < 28; ++idx) {
-    ASSERT_EQ(std::string(10000, 'a' + char(idx)), Get(Key(idx)));
+    ASSERT_EQ(std::string(10000, 'a' + idx), Get(Key(idx)));
   }
 }
 
@@ -271,14 +271,14 @@ TEST_F(CuckooTableDBTest, SameKeyInsertedInTwoDifferentFilesAndCompacted) {
 
   // Generate one more file in level-0, and should trigger level-0 compaction
   for (int idx = 0; idx < 11; ++idx) {
-    ASSERT_OK(Put(Key(idx), std::string(10000, 'a' + char(idx))));
+    ASSERT_OK(Put(Key(idx), std::string(10000, 'a' + idx)));
   }
   dbfull()->TEST_WaitForFlushMemTable();
   dbfull()->TEST_CompactRange(0, nullptr, nullptr);
 
   ASSERT_EQ("0,1", FilesPerLevel());
   for (int idx = 0; idx < 11; ++idx) {
-    ASSERT_EQ(std::string(10000, 'a' + char(idx)), Get(Key(idx)));
+    ASSERT_EQ(std::string(10000, 'a' + idx), Get(Key(idx)));
   }
 }
 
@@ -333,7 +333,7 @@ int main(int argc, char** argv) {
 #else
 #include <stdio.h>
 
-int main(int /*argc*/, char** /*argv*/) {
+int main(int argc, char** argv) {
   fprintf(stderr, "SKIPPED as Cuckoo table is not supported in ROCKSDB_LITE\n");
   return 0;
 }

db/db_basic_test.cc

@@ -9,7 +9,6 @@
 #include "db/db_test_util.h"
 #include "port/stack_trace.h"
 #include "rocksdb/perf_context.h"
-#include "util/fault_injection_test_env.h"
 #if !defined(ROCKSDB_LITE)
 #include "util/sync_point.h"
 #endif
@@ -42,7 +41,7 @@ TEST_F(DBBasicTest, ReadOnlyDB) {
   Close();
 
   auto options = CurrentOptions();
-  assert(options.env == env_);
+  assert(options.env = env_);
   ASSERT_OK(ReadOnlyReopen(options));
   ASSERT_EQ("v3", Get("foo"));
   ASSERT_EQ("v2", Get("bar"));
@@ -571,19 +570,19 @@ TEST_F(DBBasicTest, CompactBetweenSnapshots) {
 
 TEST_F(DBBasicTest, DBOpen_Options) {
   Options options = CurrentOptions();
-  Close();
-  Destroy(options);
+  std::string dbname = test::TmpDir(env_) + "/db_options_test";
+  ASSERT_OK(DestroyDB(dbname, options));
 
   // Does not exist, and create_if_missing == false: error
   DB* db = nullptr;
   options.create_if_missing = false;
-  Status s = DB::Open(options, dbname_, &db);
+  Status s = DB::Open(options, dbname, &db);
   ASSERT_TRUE(strstr(s.ToString().c_str(), "does not exist") != nullptr);
   ASSERT_TRUE(db == nullptr);
 
   // Does not exist, and create_if_missing == true: OK
   options.create_if_missing = true;
-  s = DB::Open(options, dbname_, &db);
+  s = DB::Open(options, dbname, &db);
   ASSERT_OK(s);
   ASSERT_TRUE(db != nullptr);
 
@@ -593,14 +592,14 @@ TEST_F(DBBasicTest, DBOpen_Options) {
   // Does exist, and error_if_exists == true: error
   options.create_if_missing = false;
   options.error_if_exists = true;
-  s = DB::Open(options, dbname_, &db);
+  s = DB::Open(options, dbname, &db);
   ASSERT_TRUE(strstr(s.ToString().c_str(), "exists") != nullptr);
   ASSERT_TRUE(db == nullptr);
 
   // Does exist, and error_if_exists == false: OK
   options.create_if_missing = true;
   options.error_if_exists = false;
-  s = DB::Open(options, dbname_, &db);
+  s = DB::Open(options, dbname, &db);
   ASSERT_OK(s);
   ASSERT_TRUE(db != nullptr);
 
@@ -848,114 +847,6 @@ TEST_F(DBBasicTest, MmapAndBufferOptions) {
 }
 #endif
 
-class TestEnv : public EnvWrapper {
-  public:
-    explicit TestEnv() : EnvWrapper(Env::Default()),
-                close_count(0) { }
-
-    class TestLogger : public Logger {
-      public:
-        using Logger::Logv;
-        TestLogger(TestEnv *env_ptr) : Logger() { env = env_ptr; }
-        ~TestLogger() {
-          if (!closed_) {
-            CloseHelper();
-          }
-        }
-        virtual void Logv(const char* /*format*/, va_list /*ap*/) override{};
-
-       protected:
-        virtual Status CloseImpl() override {
-          return CloseHelper();
-        }
-      private:
-        Status CloseHelper() {
-          env->CloseCountInc();;
-          return Status::IOError();
-        }
-        TestEnv *env;
-    };
-
-    void CloseCountInc() { close_count++; }
-
-    int GetCloseCount() { return close_count; }
-
-    virtual Status NewLogger(const std::string& /*fname*/,
-                             shared_ptr<Logger>* result) {
-      result->reset(new TestLogger(this));
-      return Status::OK();
-    }
-
-   private:
-    int close_count;
-};
-
-TEST_F(DBBasicTest, DBClose) {
-  Options options = GetDefaultOptions();
-  std::string dbname = test::TmpDir(env_) + "/db_close_test";
-  ASSERT_OK(DestroyDB(dbname, options));
-
-  DB* db = nullptr;
-  TestEnv* env = new TestEnv();
-  options.create_if_missing = true;
-  options.env = env;
-  Status s = DB::Open(options, dbname, &db);
-  ASSERT_OK(s);
-  ASSERT_TRUE(db != nullptr);
-
-  s = db->Close();
-  ASSERT_EQ(env->GetCloseCount(), 1);
-  ASSERT_EQ(s, Status::IOError());
-
-  delete db;
-  ASSERT_EQ(env->GetCloseCount(), 1);
-
-  // Do not call DB::Close() and ensure our logger Close() still gets called
-  s = DB::Open(options, dbname, &db);
-  ASSERT_OK(s);
-  ASSERT_TRUE(db != nullptr);
-  delete db;
-  ASSERT_EQ(env->GetCloseCount(), 2);
-
-  // Provide our own logger and ensure DB::Close() does not close it
-  options.info_log.reset(new TestEnv::TestLogger(env));
-  options.create_if_missing = false;
-  s = DB::Open(options, dbname, &db);
-  ASSERT_OK(s);
-  ASSERT_TRUE(db != nullptr);
-
-  s = db->Close();
-  ASSERT_EQ(s, Status::OK());
-  delete db;
-  ASSERT_EQ(env->GetCloseCount(), 2);
-  options.info_log.reset();
-  ASSERT_EQ(env->GetCloseCount(), 3);
-
-  delete options.env;
-}
-
-TEST_F(DBBasicTest, DBCloseFlushError) {
-  std::unique_ptr<FaultInjectionTestEnv> fault_injection_env(
-      new FaultInjectionTestEnv(Env::Default()));
-  Options options = GetDefaultOptions();
-  options.create_if_missing = true;
-  options.manual_wal_flush = true;
-  options.write_buffer_size=100;
-  options.env = fault_injection_env.get();
-
-  Reopen(options);
-  ASSERT_OK(Put("key1", "value1"));
-  ASSERT_OK(Put("key2", "value2"));
-  ASSERT_OK(dbfull()->TEST_SwitchMemtable());
-  ASSERT_OK(Put("key3", "value3"));
-  fault_injection_env->SetFilesystemActive(false);
-  Status s = dbfull()->Close();
-  fault_injection_env->SetFilesystemActive(true);
-  ASSERT_NE(s, Status::OK());
-
-  Destroy(options);
-}
-
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/db_blob_index_test.cc

@@ -64,8 +64,7 @@ class DBBlobIndexTest : public DBTestBase {
     read_options.snapshot = snapshot;
     PinnableSlice value;
     auto s = dbfull()->GetImpl(read_options, cfh(), key, &value,
-                               nullptr /*value_found*/, nullptr /*callback*/,
-                               is_blob_index);
+                               nullptr /*value_found*/, is_blob_index);
     if (s.IsNotFound()) {
       return "NOT_FOUND";
     }
@@ -89,9 +88,9 @@ class DBBlobIndexTest : public DBTestBase {
   }
 
   ArenaWrappedDBIter* GetBlobIterator() {
-    return dbfull()->NewIteratorImpl(
-        ReadOptions(), cfd(), dbfull()->GetLatestSequenceNumber(),
-        nullptr /*read_callback*/, true /*allow_blob*/);
+    return dbfull()->NewIteratorImpl(ReadOptions(), cfd(),
+                                     dbfull()->GetLatestSequenceNumber(),
+                                     true /*allow_blob*/);
   }
 
   Options GetTestOptions() {

db/db_block_cache_test.cc

@@ -47,7 +47,7 @@ class DBBlockCacheTest : public DBTestBase {
     return options;
   }
 
-  void InitTable(const Options& /*options*/) {
+  void InitTable(const Options& options) {
     std::string value(kValueSize, 'a');
     for (size_t i = 0; i < kNumBlocks; i++) {
       ASSERT_OK(Put(ToString(i), value.c_str()));
@@ -111,31 +111,6 @@ class DBBlockCacheTest : public DBTestBase {
   }
 };
 
-TEST_F(DBBlockCacheTest, IteratorBlockCacheUsage) {
-  ReadOptions read_options;
-  read_options.fill_cache = false;
-  auto table_options = GetTableOptions();
-  auto options = GetOptions(table_options);
-  InitTable(options);
-
-  std::shared_ptr<Cache> cache = NewLRUCache(0, 0, false);
-  table_options.block_cache = cache;
-  options.table_factory.reset(new BlockBasedTableFactory(table_options));
-  Reopen(options);
-  RecordCacheCounters(options);
-
-  std::vector<std::unique_ptr<Iterator>> iterators(kNumBlocks - 1);
-  Iterator* iter = nullptr;
-
-  ASSERT_EQ(0, cache->GetUsage());
-  iter = db_->NewIterator(read_options);
-  iter->Seek(ToString(0));
-  ASSERT_LT(0, cache->GetUsage());
-  delete iter;
-  iter = nullptr;
-  ASSERT_EQ(0, cache->GetUsage());
-}
-
 TEST_F(DBBlockCacheTest, TestWithoutCompressedBlockCache) {
   ReadOptions read_options;
   auto table_options = GetTableOptions();
@@ -305,41 +280,6 @@ TEST_F(DBBlockCacheTest, IndexAndFilterBlocksOfNewTableAddedToCache) {
             TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
 }
 
-// With fill_cache = false, fills up the cache, then iterates over the entire
-// db, verify dummy entries inserted in `BlockBasedTable::NewDataBlockIterator`
-// does not cause heap-use-after-free errors in COMPILE_WITH_ASAN=1 runs
-TEST_F(DBBlockCacheTest, FillCacheAndIterateDB) {
-  ReadOptions read_options;
-  read_options.fill_cache = false;
-  auto table_options = GetTableOptions();
-  auto options = GetOptions(table_options);
-  InitTable(options);
-
-  std::shared_ptr<Cache> cache = NewLRUCache(10, 0, true);
-  table_options.block_cache = cache;
-  options.table_factory.reset(new BlockBasedTableFactory(table_options));
-  Reopen(options);
-  ASSERT_OK(Put("key1", "val1"));
-  ASSERT_OK(Put("key2", "val2"));
-  ASSERT_OK(Flush());
-  ASSERT_OK(Put("key3", "val3"));
-  ASSERT_OK(Put("key4", "val4"));
-  ASSERT_OK(Flush());
-  ASSERT_OK(Put("key5", "val5"));
-  ASSERT_OK(Put("key6", "val6"));
-  ASSERT_OK(Flush());
-
-  Iterator* iter = nullptr;
-
-  iter = db_->NewIterator(read_options);
-  iter->Seek(ToString(0));
-  while (iter->Valid()) {
-    iter->Next();
-  }
-  delete iter;
-  iter = nullptr;
-}
-
 TEST_F(DBBlockCacheTest, IndexAndFilterBlocksStats) {
   Options options = CurrentOptions();
   options.create_if_missing = true;

db/db_bloom_filter_test.cc

@@ -1057,10 +1057,10 @@ TEST_F(DBBloomFilterTest, OptimizeFiltersForHits) {
   int32_t non_trivial_move = 0;
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
       "DBImpl::BackgroundCompaction:TrivialMove",
-      [&](void* /*arg*/) { trivial_move++; });
+      [&](void* arg) { trivial_move++; });
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
       "DBImpl::BackgroundCompaction:NonTrivial",
-      [&](void* /*arg*/) { non_trivial_move++; });
+      [&](void* arg) { non_trivial_move++; });
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 
   CompactRangeOptions compact_options;

db/db_compaction_filter_test.cc

@@ -24,41 +24,11 @@ class DBTestCompactionFilter : public DBTestBase {
   DBTestCompactionFilter() : DBTestBase("/db_compaction_filter_test") {}
 };
 
-// Param variant of DBTestBase::ChangeCompactOptions
-class DBTestCompactionFilterWithCompactParam
-    : public DBTestCompactionFilter,
-      public ::testing::WithParamInterface<DBTestBase::OptionConfig> {
- public:
-  DBTestCompactionFilterWithCompactParam() : DBTestCompactionFilter() {
-    option_config_ = GetParam();
-    Destroy(last_options_);
-    auto options = CurrentOptions();
-    if (option_config_ == kDefault || option_config_ == kUniversalCompaction ||
-        option_config_ == kUniversalCompactionMultiLevel) {
-      options.create_if_missing = true;
-    }
-    if (option_config_ == kLevelSubcompactions ||
-        option_config_ == kUniversalSubcompactions) {
-      assert(options.max_subcompactions > 1);
-    }
-    TryReopen(options);
-  }
-};
-
-INSTANTIATE_TEST_CASE_P(
-    DBTestCompactionFilterWithCompactOption,
-    DBTestCompactionFilterWithCompactParam,
-    ::testing::Values(DBTestBase::OptionConfig::kDefault,
-                      DBTestBase::OptionConfig::kUniversalCompaction,
-                      DBTestBase::OptionConfig::kUniversalCompactionMultiLevel,
-                      DBTestBase::OptionConfig::kLevelSubcompactions,
-                      DBTestBase::OptionConfig::kUniversalSubcompactions));
-
 class KeepFilter : public CompactionFilter {
  public:
-  virtual bool Filter(int /*level*/, const Slice& /*key*/,
-                      const Slice& /*value*/, std::string* /*new_value*/,
-                      bool* /*value_changed*/) const override {
+  virtual bool Filter(int level, const Slice& key, const Slice& value,
+                      std::string* new_value, bool* value_changed) const
+      override {
     cfilter_count++;
     return false;
   }
@@ -68,9 +38,9 @@ class KeepFilter : public CompactionFilter {
 
 class DeleteFilter : public CompactionFilter {
  public:
-  virtual bool Filter(int /*level*/, const Slice& /*key*/,
-                      const Slice& /*value*/, std::string* /*new_value*/,
-                      bool* /*value_changed*/) const override {
+  virtual bool Filter(int level, const Slice& key, const Slice& value,
+                      std::string* new_value, bool* value_changed) const
+      override {
     cfilter_count++;
     return true;
   }
@@ -80,9 +50,9 @@ class DeleteFilter : public CompactionFilter {
 
 class DeleteISFilter : public CompactionFilter {
  public:
-  virtual bool Filter(int /*level*/, const Slice& key, const Slice& /*value*/,
-                      std::string* /*new_value*/,
-                      bool* /*value_changed*/) const override {
+  virtual bool Filter(int level, const Slice& key, const Slice& value,
+                      std::string* new_value,
+                      bool* value_changed) const override {
     cfilter_count++;
     int i = std::stoi(key.ToString());
     if (i > 5 && i <= 105) {
@@ -100,16 +70,14 @@ class DeleteISFilter : public CompactionFilter {
 // zero-padded to length 10.
 class SkipEvenFilter : public CompactionFilter {
  public:
-  virtual Decision FilterV2(int /*level*/, const Slice& key,
-                            ValueType /*value_type*/,
-                            const Slice& /*existing_value*/,
-                            std::string* /*new_value*/,
+  virtual Decision FilterV2(int level, const Slice& key, ValueType value_type,
+                            const Slice& existing_value, std::string* new_value,
                             std::string* skip_until) const override {
     cfilter_count++;
     int i = std::stoi(key.ToString());
     if (i / 10 % 2 == 0) {
       char key_str[100];
-      snprintf(key_str, sizeof(key_str), "%010d", i / 10 * 10 + 10);
+      snprintf(key_str, sizeof(key), "%010d", i / 10 * 10 + 10);
       *skip_until = key_str;
       ++cfilter_skips;
       return Decision::kRemoveAndSkipUntil;
@@ -125,9 +93,9 @@ class SkipEvenFilter : public CompactionFilter {
 class DelayFilter : public CompactionFilter {
  public:
   explicit DelayFilter(DBTestBase* d) : db_test(d) {}
-  virtual bool Filter(int /*level*/, const Slice& /*key*/,
-                      const Slice& /*value*/, std::string* /*new_value*/,
-                      bool* /*value_changed*/) const override {
+  virtual bool Filter(int level, const Slice& key, const Slice& value,
+                      std::string* new_value,
+                      bool* value_changed) const override {
     db_test->env_->addon_time_.fetch_add(1000);
     return true;
   }
@@ -142,9 +110,9 @@ class ConditionalFilter : public CompactionFilter {
  public:
   explicit ConditionalFilter(const std::string* filtered_value)
       : filtered_value_(filtered_value) {}
-  virtual bool Filter(int /*level*/, const Slice& /*key*/, const Slice& value,
-                      std::string* /*new_value*/,
-                      bool* /*value_changed*/) const override {
+  virtual bool Filter(int level, const Slice& key, const Slice& value,
+                      std::string* new_value,
+                      bool* value_changed) const override {
     return value.ToString() == *filtered_value_;
   }
 
@@ -158,9 +126,9 @@ class ChangeFilter : public CompactionFilter {
  public:
   explicit ChangeFilter() {}
 
-  virtual bool Filter(int /*level*/, const Slice& /*key*/,
-                      const Slice& /*value*/, std::string* new_value,
-                      bool* value_changed) const override {
+  virtual bool Filter(int level, const Slice& key, const Slice& value,
+                      std::string* new_value, bool* value_changed) const
+      override {
     assert(new_value != nullptr);
     *new_value = NEW_VALUE;
     *value_changed = true;
@@ -249,7 +217,7 @@ class DelayFilterFactory : public CompactionFilterFactory {
  public:
   explicit DelayFilterFactory(DBTestBase* d) : db_test(d) {}
   virtual std::unique_ptr<CompactionFilter> CreateCompactionFilter(
-      const CompactionFilter::Context& /*context*/) override {
+      const CompactionFilter::Context& context) override {
     return std::unique_ptr<CompactionFilter>(new DelayFilter(db_test));
   }
 
@@ -265,7 +233,7 @@ class ConditionalFilterFactory : public CompactionFilterFactory {
       : filtered_value_(filtered_value.ToString()) {}
 
   virtual std::unique_ptr<CompactionFilter> CreateCompactionFilter(
-      const CompactionFilter::Context& /*context*/) override {
+      const CompactionFilter::Context& context) override {
     return std::unique_ptr<CompactionFilter>(
         new ConditionalFilter(&filtered_value_));
   }
@@ -283,7 +251,7 @@ class ChangeFilterFactory : public CompactionFilterFactory {
   explicit ChangeFilterFactory() {}
 
   virtual std::unique_ptr<CompactionFilter> CreateCompactionFilter(
-      const CompactionFilter::Context& /*context*/) override {
+      const CompactionFilter::Context& context) override {
     return std::unique_ptr<CompactionFilter>(new ChangeFilter());
   }
 
@@ -340,6 +308,7 @@ TEST_F(DBTestCompactionFilter, CompactionFilter) {
     ASSERT_OK(iter->status());
     while (iter->Valid()) {
       ParsedInternalKey ikey(Slice(), 0, kTypeValue);
+      ikey.sequence = -1;
       ASSERT_EQ(ParseInternalKey(iter->key(), &ikey), true);
       total++;
       if (ikey.sequence != 0) {
@@ -471,63 +440,65 @@ TEST_F(DBTestCompactionFilter, CompactionFilterDeletesAll) {
 }
 #endif  // ROCKSDB_LITE
 
-TEST_P(DBTestCompactionFilterWithCompactParam,
-       CompactionFilterWithValueChange) {
-  Options options = CurrentOptions();
-  options.num_levels = 3;
-  options.compaction_filter_factory = std::make_shared<ChangeFilterFactory>();
-  CreateAndReopenWithCF({"pikachu"}, options);
+TEST_F(DBTestCompactionFilter, CompactionFilterWithValueChange) {
+  do {
+    Options options = CurrentOptions();
+    options.num_levels = 3;
+    options.compaction_filter_factory =
+      std::make_shared<ChangeFilterFactory>();
+    CreateAndReopenWithCF({"pikachu"}, options);
 
-  // Write 100K+1 keys, these are written to a few files
-  // in L0. We do this so that the current snapshot points
-  // to the 100001 key.The compaction filter is  not invoked
-  // on keys that are visible via a snapshot because we
-  // anyways cannot delete it.
-  const std::string value(10, 'x');
-  for (int i = 0; i < 100001; i++) {
-    char key[100];
-    snprintf(key, sizeof(key), "B%010d", i);
-    Put(1, key, value);
-  }
+    // Write 100K+1 keys, these are written to a few files
+    // in L0. We do this so that the current snapshot points
+    // to the 100001 key.The compaction filter is  not invoked
+    // on keys that are visible via a snapshot because we
+    // anyways cannot delete it.
+    const std::string value(10, 'x');
+    for (int i = 0; i < 100001; i++) {
+      char key[100];
+      snprintf(key, sizeof(key), "B%010d", i);
+      Put(1, key, value);
+    }
 
-  // push all files to  lower levels
-  ASSERT_OK(Flush(1));
-  if (option_config_ != kUniversalCompactionMultiLevel &&
-      option_config_ != kUniversalSubcompactions) {
-    dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]);
-    dbfull()->TEST_CompactRange(1, nullptr, nullptr, handles_[1]);
-  } else {
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-  }
+    // push all files to  lower levels
+    ASSERT_OK(Flush(1));
+    if (option_config_ != kUniversalCompactionMultiLevel &&
+        option_config_ != kUniversalSubcompactions) {
+      dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]);
+      dbfull()->TEST_CompactRange(1, nullptr, nullptr, handles_[1]);
+    } else {
+      dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
+                             nullptr);
+    }
 
-  // re-write all data again
-  for (int i = 0; i < 100001; i++) {
-    char key[100];
-    snprintf(key, sizeof(key), "B%010d", i);
-    Put(1, key, value);
-  }
+    // re-write all data again
+    for (int i = 0; i < 100001; i++) {
+      char key[100];
+      snprintf(key, sizeof(key), "B%010d", i);
+      Put(1, key, value);
+    }
 
-  // push all files to  lower levels. This should
-  // invoke the compaction filter for all 100000 keys.
-  ASSERT_OK(Flush(1));
-  if (option_config_ != kUniversalCompactionMultiLevel &&
-      option_config_ != kUniversalSubcompactions) {
-    dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]);
-    dbfull()->TEST_CompactRange(1, nullptr, nullptr, handles_[1]);
-  } else {
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-  }
+    // push all files to  lower levels. This should
+    // invoke the compaction filter for all 100000 keys.
+    ASSERT_OK(Flush(1));
+    if (option_config_ != kUniversalCompactionMultiLevel &&
+        option_config_ != kUniversalSubcompactions) {
+      dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]);
+      dbfull()->TEST_CompactRange(1, nullptr, nullptr, handles_[1]);
+    } else {
+      dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
+                             nullptr);
+    }
 
-  // verify that all keys now have the new value that
-  // was set by the compaction process.
-  for (int i = 0; i < 100001; i++) {
-    char key[100];
-    snprintf(key, sizeof(key), "B%010d", i);
-    std::string newvalue = Get(1, key);
-    ASSERT_EQ(newvalue.compare(NEW_VALUE), 0);
-  }
+    // verify that all keys now have the new value that
+    // was set by the compaction process.
+    for (int i = 0; i < 100001; i++) {
+      char key[100];
+      snprintf(key, sizeof(key), "B%010d", i);
+      std::string newvalue = Get(1, key);
+      ASSERT_EQ(newvalue.compare(NEW_VALUE), 0);
+    }
+  } while (ChangeCompactOptions());
 }
 
 TEST_F(DBTestCompactionFilter, CompactionFilterWithMergeOperator) {
@@ -646,6 +617,7 @@ TEST_F(DBTestCompactionFilter, CompactionFilterContextManual) {
     ASSERT_OK(iter->status());
     while (iter->Valid()) {
       ParsedInternalKey ikey(Slice(), 0, kTypeValue);
+      ikey.sequence = -1;
       ASSERT_EQ(ParseInternalKey(iter->key(), &ikey), true);
       total++;
       if (ikey.sequence != 0) {
@@ -767,7 +739,7 @@ TEST_F(DBTestCompactionFilter, CompactionFilterIgnoreSnapshot) {
       iter->Next();
     }
     ASSERT_EQ(count, 6);
-    read_options.snapshot = nullptr;
+    read_options.snapshot = 0;
     std::unique_ptr<Iterator> iter1(db_->NewIterator(read_options));
     iter1->SeekToFirst();
     count = 0;

db/db_dynamic_level_test.cc

@@ -194,7 +194,7 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase2) {
   // Hold compaction jobs to make sure
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
       "CompactionJob::Run():Start",
-      [&](void* /*arg*/) { env_->SleepForMicroseconds(100000); });
+      [&](void* arg) { env_->SleepForMicroseconds(100000); });
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
   ASSERT_OK(dbfull()->SetOptions({
       {"disable_auto_compactions", "true"},
@@ -378,7 +378,7 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBaseInc) {
   int non_trivial = 0;
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
       "DBImpl::BackgroundCompaction:NonTrivial",
-      [&](void* /*arg*/) { non_trivial++; });
+      [&](void* arg) { non_trivial++; });
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 
   Random rnd(301);
@@ -501,8 +501,6 @@ int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 #else
-  (void) argc;
-  (void) argv;
   return 0;
 #endif
 }

db/db_filesnapshot.cc

@@ -57,7 +57,6 @@ Status DBImpl::EnableFileDeletions(bool force) {
       ROCKS_LOG_INFO(immutable_db_options_.info_log, "File Deletions Enabled");
       should_purge_files = true;
       FindObsoleteFiles(&job_context, true);
-      bg_cv_.SignalAll();
     } else {
       ROCKS_LOG_WARN(
           immutable_db_options_.info_log,
@@ -74,7 +73,7 @@ Status DBImpl::EnableFileDeletions(bool force) {
 }
 
 int DBImpl::IsFileDeletionsEnabled() const {
-  return !disable_delete_obsolete_files_;
+  return disable_delete_obsolete_files_;
 }
 
 Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
@@ -93,7 +92,7 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
       }
       cfd->Ref();
       mutex_.Unlock();
-      status = FlushMemTable(cfd, FlushOptions(), FlushReason::kGetLiveFiles);
+      status = FlushMemTable(cfd, FlushOptions());
       TEST_SYNC_POINT("DBImpl::GetLiveFiles:1");
       TEST_SYNC_POINT("DBImpl::GetLiveFiles:2");
       mutex_.Lock();
@@ -142,18 +141,6 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
 }
 
 Status DBImpl::GetSortedWalFiles(VectorLogPtr& files) {
-  {
-    // If caller disabled deletions, this function should return files that are
-    // guaranteed not to be deleted until deletions are re-enabled. We need to
-    // wait for pending purges to finish since WalManager doesn't know which
-    // files are going to be purged. Additional purges won't be scheduled as
-    // long as deletions are disabled (so the below loop must terminate).
-    InstrumentedMutexLock l(&mutex_);
-    while (disable_delete_obsolete_files_ > 0 &&
-           pending_purge_obsolete_files_ > 0) {
-      bg_cv_.Wait();
-    }
-  }
   return wal_manager_.GetSortedWalFiles(files);
 }
 

db/db_flush_test.cc

@@ -72,23 +72,19 @@ TEST_F(DBFlushTest, SyncFail) {
   auto* cfd =
       reinterpret_cast<ColumnFamilyHandleImpl*>(db_->DefaultColumnFamily())
           ->cfd();
+  int refs_before = cfd->current()->TEST_refs();
   FlushOptions flush_options;
   flush_options.wait = false;
   ASSERT_OK(dbfull()->Flush(flush_options));
-  // Flush installs a new super-version. Get the ref count after that.
-  auto current_before = cfd->current();
-  int refs_before = cfd->current()->TEST_refs();
   fault_injection_env->SetFilesystemActive(false);
   TEST_SYNC_POINT("DBFlushTest::SyncFail:1");
   TEST_SYNC_POINT("DBFlushTest::SyncFail:2");
   fault_injection_env->SetFilesystemActive(true);
-  // Now the background job will do the flush; wait for it.
   dbfull()->TEST_WaitForFlushMemTable();
 #ifndef ROCKSDB_LITE
   ASSERT_EQ("", FilesPerLevel());  // flush failed.
 #endif                             // ROCKSDB_LITE
-  // Backgroun flush job should release ref count to current version.
-  ASSERT_EQ(current_before, cfd->current());
+  // Flush job should release ref count to current version.
   ASSERT_EQ(refs_before, cfd->current()->TEST_refs());
   Destroy(options);
 }
@@ -105,7 +101,7 @@ TEST_F(DBFlushTest, FlushInLowPriThreadPool) {
   std::thread::id tid;
   int num_flushes = 0, num_compactions = 0;
   SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::BGWorkFlush", [&](void* /*arg*/) {
+      "DBImpl::BGWorkFlush", [&](void* arg) {
         if (tid == std::thread::id()) {
           tid = std::this_thread::get_id();
         } else {
@@ -114,7 +110,7 @@ TEST_F(DBFlushTest, FlushInLowPriThreadPool) {
         ++num_flushes;
       });
   SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::BGWorkCompaction", [&](void* /*arg*/) {
+      "DBImpl::BGWorkCompaction", [&](void* arg) {
         ASSERT_EQ(tid, std::this_thread::get_id());
         ++num_compactions;
       });
@@ -130,41 +126,6 @@ TEST_F(DBFlushTest, FlushInLowPriThreadPool) {
   ASSERT_EQ(1, num_compactions);
 }
 
-TEST_F(DBFlushTest, ManualFlushWithMinWriteBufferNumberToMerge) {
-  Options options = CurrentOptions();
-  options.write_buffer_size = 100;
-  options.max_write_buffer_number = 4;
-  options.min_write_buffer_number_to_merge = 3;
-  Reopen(options);
-
-  SyncPoint::GetInstance()->LoadDependency(
-      {{"DBImpl::BGWorkFlush",
-        "DBFlushTest::ManualFlushWithMinWriteBufferNumberToMerge:1"},
-       {"DBFlushTest::ManualFlushWithMinWriteBufferNumberToMerge:2",
-        "FlushJob::WriteLevel0Table"}});
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  ASSERT_OK(Put("key1", "value1"));
-
-  port::Thread t([&]() {
-    // The call wait for flush to finish, i.e. with flush_options.wait = true.
-    ASSERT_OK(Flush());
-  });
-
-  // Wait for flush start.
-  TEST_SYNC_POINT("DBFlushTest::ManualFlushWithMinWriteBufferNumberToMerge:1");
-  // Insert a second memtable before the manual flush finish.
-  // At the end of the manual flush job, it will check if further flush
-  // is needed, but it will not trigger flush of the second memtable because
-  // min_write_buffer_number_to_merge is not reached.
-  ASSERT_OK(Put("key2", "value2"));
-  ASSERT_OK(dbfull()->TEST_SwitchMemtable());
-  TEST_SYNC_POINT("DBFlushTest::ManualFlushWithMinWriteBufferNumberToMerge:2");
-
-  // Manual flush should return, without waiting for flush indefinitely.
-  t.join();
-}
-
 TEST_P(DBFlushDirectIOTest, DirectIO) {
   Options options;
   options.create_if_missing = true;
@@ -189,26 +150,6 @@ TEST_P(DBFlushDirectIOTest, DirectIO) {
   delete options.env;
 }
 
-TEST_F(DBFlushTest, FlushError) {
-  Options options;
-  std::unique_ptr<FaultInjectionTestEnv> fault_injection_env(
-      new FaultInjectionTestEnv(env_));
-  options.write_buffer_size = 100;
-  options.max_write_buffer_number = 4;
-  options.min_write_buffer_number_to_merge = 3;
-  options.disable_auto_compactions = true;
-  options.env = fault_injection_env.get();
-  Reopen(options);
-
-  ASSERT_OK(Put("key1", "value1"));
-  ASSERT_OK(Put("key2", "value2"));
-  fault_injection_env->SetFilesystemActive(false);
-  Status s = dbfull()->TEST_SwitchMemtable();
-  fault_injection_env->SetFilesystemActive(true);
-  Destroy(options);
-  ASSERT_NE(s, Status::OK());
-}
-
 INSTANTIATE_TEST_CASE_P(DBFlushDirectIOTest, DBFlushDirectIOTest,
                         testing::Bool());
 

db/db_impl.h

@@ -14,6 +14,7 @@
 #include <limits>
 #include <list>
 #include <map>
+#include <queue>
 #include <set>
 #include <string>
 #include <utility>
@@ -27,10 +28,6 @@
 #include "db/flush_scheduler.h"
 #include "db/internal_stats.h"
 #include "db/log_writer.h"
-#include "db/logs_with_prep_tracker.h"
-#include "db/pre_release_callback.h"
-#include "db/read_callback.h"
-#include "db/snapshot_checker.h"
 #include "db/snapshot_impl.h"
 #include "db/version_edit.h"
 #include "db/wal_manager.h"
@@ -55,13 +52,13 @@
 
 namespace rocksdb {
 
-class Arena;
 class ArenaWrappedDBIter;
 class MemTable;
 class TableCache;
 class Version;
 class VersionEdit;
 class VersionSet;
+class Arena;
 class WriteCallback;
 struct JobContext;
 struct ExternalSstFileInfo;
@@ -69,8 +66,7 @@ struct MemTableInfo;
 
 class DBImpl : public DB {
  public:
-  DBImpl(const DBOptions& options, const std::string& dbname,
-         const bool seq_per_batch = false);
+  DBImpl(const DBOptions& options, const std::string& dbname);
   virtual ~DBImpl();
 
   // Implementations of the DB interface
@@ -103,8 +99,7 @@ class DBImpl : public DB {
   // Note: 'value_found' from KeyMayExist propagates here
   Status GetImpl(const ReadOptions& options, ColumnFamilyHandle* column_family,
                  const Slice& key, PinnableSlice* value,
-                 bool* value_found = nullptr, ReadCallback* callback = nullptr,
-                 bool* is_blob_index = nullptr);
+                 bool* value_found = nullptr, bool* is_blob_index = nullptr);
 
   using DB::MultiGet;
   virtual std::vector<Status> MultiGet(
@@ -147,9 +142,7 @@ class DBImpl : public DB {
   ArenaWrappedDBIter* NewIteratorImpl(const ReadOptions& options,
                                       ColumnFamilyData* cfd,
                                       SequenceNumber snapshot,
-                                      ReadCallback* read_callback,
-                                      bool allow_blob = false,
-                                      bool allow_refresh = true);
+                                      bool allow_blob = false);
 
   virtual const Snapshot* GetSnapshot() override;
   virtual void ReleaseSnapshot(const Snapshot* snapshot) override;
@@ -157,9 +150,9 @@ class DBImpl : public DB {
   virtual bool GetProperty(ColumnFamilyHandle* column_family,
                            const Slice& property, std::string* value) override;
   using DB::GetMapProperty;
-  virtual bool GetMapProperty(
-      ColumnFamilyHandle* column_family, const Slice& property,
-      std::map<std::string, std::string>* value) override;
+  virtual bool GetMapProperty(ColumnFamilyHandle* column_family,
+                              const Slice& property,
+                              std::map<std::string, double>* value) override;
   using DB::GetIntProperty;
   virtual bool GetIntProperty(ColumnFamilyHandle* column_family,
                               const Slice& property, uint64_t* value) override;
@@ -186,9 +179,7 @@ class DBImpl : public DB {
                               ColumnFamilyHandle* column_family,
                               const std::vector<std::string>& input_file_names,
                               const int output_level,
-                              const int output_path_id = -1,
-                              std::vector<std::string>* const output_file_names
-                              = nullptr) override;
+                              const int output_path_id = -1) override;
 
   virtual Status PauseBackgroundWork() override;
   virtual Status ContinueBackgroundWork() override;
@@ -221,19 +212,13 @@ class DBImpl : public DB {
   virtual Status Flush(const FlushOptions& options,
                        ColumnFamilyHandle* column_family) override;
   virtual Status FlushWAL(bool sync) override;
-  bool TEST_WALBufferIsEmpty();
   virtual Status SyncWAL() override;
 
   virtual SequenceNumber GetLatestSequenceNumber() const override;
-  // REQUIRES: joined the main write queue if two_write_queues is disabled, and
-  // the second write queue otherwise.
-  virtual void SetLastPublishedSequence(SequenceNumber seq);
-  // Returns LastSequence in last_seq_same_as_publish_seq_
-  // mode and LastAllocatedSequence otherwise. This is useful when visiblility
-  // depends also on data written to the WAL but not to the memtable.
-  SequenceNumber TEST_GetLastVisibleSequence() const;
 
-  virtual bool SetPreserveDeletesSequenceNumber(SequenceNumber seqnum) override;
+  // Whether there is an active snapshot in range [lower_bound, upper_bound).
+  bool HasActiveSnapshotInRange(SequenceNumber lower_bound,
+                                SequenceNumber upper_bound);
 
 #ifndef ROCKSDB_LITE
   using DB::ResetStats;
@@ -252,9 +237,8 @@ class DBImpl : public DB {
       const TransactionLogIterator::ReadOptions&
           read_options = TransactionLogIterator::ReadOptions()) override;
   virtual Status DeleteFile(std::string name) override;
-  Status DeleteFilesInRanges(ColumnFamilyHandle* column_family,
-                             const RangePtr* ranges, size_t n,
-                             bool include_end = true);
+  Status DeleteFilesInRange(ColumnFamilyHandle* column_family,
+                            const Slice* begin, const Slice* end);
 
   virtual void GetLiveFilesMetaData(
       std::vector<LiveFileMetaData>* metadata) override;
@@ -344,7 +328,6 @@ class DBImpl : public DB {
 
   Status RunManualCompaction(ColumnFamilyData* cfd, int input_level,
                              int output_level, uint32_t output_path_id,
-                             uint32_t max_subcompactions,
                              const Slice* begin, const Slice* end,
                              bool exclusive,
                              bool disallow_trivial_move = false);
@@ -356,10 +339,6 @@ class DBImpl : public DB {
       Arena* arena, RangeDelAggregator* range_del_agg,
       ColumnFamilyHandle* column_family = nullptr);
 
-  LogsWithPrepTracker* logs_with_prep_tracker() {
-    return &logs_with_prep_tracker_;
-  }
-
 #ifndef NDEBUG
   // Extra methods (for testing) that are not in the public DB interface
   // Implemented in db_impl_debug.cc
@@ -369,9 +348,11 @@ class DBImpl : public DB {
                            ColumnFamilyHandle* column_family = nullptr,
                            bool disallow_trivial_move = false);
 
-  void TEST_SwitchWAL();
+  void TEST_HandleWALFull();
 
-  bool TEST_UnableToReleaseOldestLog() { return unable_to_release_oldest_log_; }
+  bool TEST_UnableToFlushOldestLog() {
+    return unable_to_flush_oldest_log_;
+  }
 
   bool TEST_IsLogGettingFlushed() {
     return alive_log_files_.begin()->getting_flushed;
@@ -387,9 +368,7 @@ class DBImpl : public DB {
   Status TEST_WaitForFlushMemTable(ColumnFamilyHandle* column_family = nullptr);
 
   // Wait for any compaction
-  // We add a bool parameter to wait for unscheduledCompactions_ == 0, but this
-  // is only for the special test of CancelledCompactions
-  Status TEST_WaitForCompact(bool waitUnscheduled = false);
+  Status TEST_WaitForCompact();
 
   // Return the maximum overlapping data (in bytes) at next level for any
   // file at a level >= 1.
@@ -399,9 +378,6 @@ class DBImpl : public DB {
   // Return the current manifest file no.
   uint64_t TEST_Current_Manifest_FileNo();
 
-  // Returns the number that'll be assigned to the next file that's created.
-  uint64_t TEST_Current_Next_FileNo();
-
   // get total level0 file size. Only for testing.
   uint64_t TEST_GetLevel0TotalSize();
 
@@ -443,8 +419,6 @@ class DBImpl : public DB {
 
   uint64_t TEST_FindMinLogContainingOutstandingPrep();
   uint64_t TEST_FindMinPrepLogReferencedByMemTable();
-  size_t TEST_PreparedSectionCompletedSize();
-  size_t TEST_LogsWithPrepSize();
 
   int TEST_BGCompactionsAllowed() const;
   int TEST_BGFlushesAllowed() const;
@@ -478,12 +452,10 @@ class DBImpl : public DB {
                          bool no_full_scan = false);
 
   // Diffs the files listed in filenames and those that do not
-  // belong to live files are possibly removed. Also, removes all the
+  // belong to live files are posibly removed. Also, removes all the
   // files in sst_delete_files and log_delete_files.
   // It is not necessary to hold the mutex when invoking this method.
-  // If FindObsoleteFiles() was run, we need to also run
-  // PurgeObsoleteFiles(), even if disable_delete_obsolete_files_ is true
-  void PurgeObsoleteFiles(JobContext& background_contet,
+  void PurgeObsoleteFiles(const JobContext& background_contet,
                           bool schedule_only = false);
 
   void SchedulePurge();
@@ -509,9 +481,6 @@ class DBImpl : public DB {
   // mutex is held.
   SuperVersion* GetAndRefSuperVersion(uint32_t column_family_id);
 
-  // Un-reference the super version and clean it up if it is the last reference.
-  void CleanupSuperVersion(SuperVersion* sv);
-
   // Un-reference the super version and return it to thread local cache if
   // needed. If it is the last reference of the super version. Clean it up
   // after un-referencing it.
@@ -559,20 +528,9 @@ class DBImpl : public DB {
     uint64_t log_number_;
     std::string name_;
     WriteBatch* batch_;
-    // The seq number of the first key in the batch
-    SequenceNumber seq_;
-    // Number of sub-batched. A new sub-batch is created if we txn attempts to
-    // inserts a duplicate key,seq to memtable. This is currently used in
-    // WritePrparedTxn
-    size_t batch_cnt_;
     explicit RecoveredTransaction(const uint64_t log, const std::string& name,
-                                  WriteBatch* batch, SequenceNumber seq,
-                                  size_t batch_cnt)
-        : log_number_(log),
-          name_(name),
-          batch_(batch),
-          seq_(seq),
-          batch_cnt_(batch_cnt) {}
+                                  WriteBatch* batch)
+        : log_number_(log), name_(name), batch_(batch) {}
 
     ~RecoveredTransaction() { delete batch_; }
   };
@@ -594,11 +552,9 @@ class DBImpl : public DB {
   }
 
   void InsertRecoveredTransaction(const uint64_t log, const std::string& name,
-                                  WriteBatch* batch, SequenceNumber seq,
-                                  size_t batch_cnt) {
-    recovered_transactions_[name] =
-        new RecoveredTransaction(log, name, batch, seq, batch_cnt);
-    logs_with_prep_tracker_.MarkLogAsContainingPrepSection(log);
+                                  WriteBatch* batch) {
+    recovered_transactions_[name] = new RecoveredTransaction(log, name, batch);
+    MarkLogAsContainingPrepSection(log);
   }
 
   void DeleteRecoveredTransaction(const std::string& name) {
@@ -606,7 +562,7 @@ class DBImpl : public DB {
     assert(it != recovered_transactions_.end());
     auto* trx = it->second;
     recovered_transactions_.erase(it);
-    logs_with_prep_tracker_.MarkLogAsHavingPrepSectionFlushed(trx->log_number_);
+    MarkLogAsHavingPrepSectionFlushed(trx->log_number_);
     delete trx;
   }
 
@@ -618,36 +574,19 @@ class DBImpl : public DB {
     recovered_transactions_.clear();
   }
 
+  void MarkLogAsHavingPrepSectionFlushed(uint64_t log);
+  void MarkLogAsContainingPrepSection(uint64_t log);
   void AddToLogsToFreeQueue(log::Writer* log_writer) {
     logs_to_free_queue_.push_back(log_writer);
   }
-
-  void SetSnapshotChecker(SnapshotChecker* snapshot_checker);
-
-  // Not thread-safe.
-  void SetRecoverableStatePreReleaseCallback(PreReleaseCallback* callback);
-
   InstrumentedMutex* mutex() { return &mutex_; }
 
   Status NewDB();
 
-  // This is to be used only by internal rocksdb classes.
-  static Status Open(const DBOptions& db_options, const std::string& name,
-                     const std::vector<ColumnFamilyDescriptor>& column_families,
-                     std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
-                     const bool seq_per_batch);
-
-  virtual Status Close() override;
-
-  static Status CreateAndNewDirectory(Env* env, const std::string& dirname,
-                                      std::unique_ptr<Directory>* directory);
-
  protected:
   Env* const env_;
   const std::string dbname_;
   unique_ptr<VersionSet> versions_;
-  // Flag to check whether we allocated and own the info log file
-  bool own_info_log_;
   const DBOptions initial_db_options_;
   const ImmutableDBOptions immutable_db_options_;
   MutableDBOptions mutable_db_options_;
@@ -693,27 +632,10 @@ class DBImpl : public DB {
 
   void EraseThreadStatusDbInfo() const;
 
-  // If disable_memtable is set the application logic must guarantee that the
-  // batch will still be skipped from memtable during the recovery. An excption
-  // to this is seq_per_batch_ mode, in which since each batch already takes one
-  // seq, it is ok for the batch to write to memtable during recovery as long as
-  // it only takes one sequence number: i.e., no duplicate keys.
-  // In WriteCommitted it is guarnateed since disable_memtable is used for
-  // prepare batch which will be written to memtable later during the commit,
-  // and in WritePrepared it is guaranteed since it will be used only for WAL
-  // markers which will never be written to memtable. If the commit marker is
-  // accompanied with CommitTimeWriteBatch that is not written to memtable as
-  // long as it has no duplicate keys, it does not violate the one-seq-per-batch
-  // policy.
-  // batch_cnt is expected to be non-zero in seq_per_batch mode and
-  // indicates the number of sub-patches. A sub-patch is a subset of the write
-  // batch that does not have duplicate keys.
   Status WriteImpl(const WriteOptions& options, WriteBatch* updates,
                    WriteCallback* callback = nullptr,
                    uint64_t* log_used = nullptr, uint64_t log_ref = 0,
-                   bool disable_memtable = false, uint64_t* seq_used = nullptr,
-                   size_t batch_cnt = 0,
-                   PreReleaseCallback* pre_release_callback = nullptr);
+                   bool disable_memtable = false, uint64_t* seq_used = nullptr);
 
   Status PipelinedWriteImpl(const WriteOptions& options, WriteBatch* updates,
                             WriteCallback* callback = nullptr,
@@ -721,54 +643,40 @@ class DBImpl : public DB {
                             bool disable_memtable = false,
                             uint64_t* seq_used = nullptr);
 
-  // batch_cnt is expected to be non-zero in seq_per_batch mode and indicates
-  // the number of sub-patches. A sub-patch is a subset of the write batch that
-  // does not have duplicate keys.
   Status WriteImplWALOnly(const WriteOptions& options, WriteBatch* updates,
                           WriteCallback* callback = nullptr,
                           uint64_t* log_used = nullptr, uint64_t log_ref = 0,
-                          uint64_t* seq_used = nullptr, size_t batch_cnt = 0,
-                          PreReleaseCallback* pre_release_callback = nullptr);
+                          uint64_t* seq_used = nullptr);
 
-  // write cached_recoverable_state_ to memtable if it is not empty
-  // The writer must be the leader in write_thread_ and holding mutex_
-  Status WriteRecoverableState();
-
-  // Actual implementation of Close()
-  Status CloseImpl();
+  uint64_t FindMinLogContainingOutstandingPrep();
+  uint64_t FindMinPrepLogReferencedByMemTable();
 
  private:
   friend class DB;
   friend class InternalStats;
   friend class PessimisticTransaction;
-  friend class TransactionBaseImpl;
   friend class WriteCommittedTxn;
   friend class WritePreparedTxn;
-  friend class WritePreparedTxnDB;
-  friend class WriteBatchWithIndex;
 #ifndef ROCKSDB_LITE
   friend class ForwardIterator;
 #endif
   friend struct SuperVersion;
   friend class CompactedDBImpl;
-  friend class DBTest_ConcurrentFlushWAL_Test;
 #ifndef NDEBUG
   friend class DBTest2_ReadCallbackTest_Test;
-  friend class WriteCallbackTest_WriteWithCallbackTest_Test;
   friend class XFTransactionWriteHandler;
   friend class DBBlobIndexTest;
 #endif
   struct CompactionState;
 
   struct WriteContext {
-    SuperVersionContext superversion_context;
+    autovector<SuperVersion*> superversions_to_free_;
     autovector<MemTable*> memtables_to_free_;
 
-    explicit WriteContext(bool create_superversion = false)
-        : superversion_context(create_superversion) {}
-
     ~WriteContext() {
-      superversion_context.Clean();
+      for (auto& sv : superversions_to_free_) {
+        delete sv;
+      }
       for (auto& m : memtables_to_free_) {
         delete m;
       }
@@ -798,9 +706,9 @@ class DBImpl : public DB {
   // Delete any unneeded files and stale in-memory entries.
   void DeleteObsoleteFiles();
   // Delete obsolete files and log status and information of file deletion
-  void DeleteObsoleteFileImpl(int job_id, const std::string& fname,
-                              const std::string& path_to_sync, FileType type,
-                              uint64_t number);
+  void DeleteObsoleteFileImpl(Status file_deletion_status, int job_id,
+                              const std::string& fname, FileType type,
+                              uint64_t number, uint32_t path_id);
 
   // Background process needs to call
   //     auto x = CaptureCurrentFileNumberInPendingOutputs()
@@ -851,22 +759,17 @@ class DBImpl : public DB {
 
   Status ScheduleFlushes(WriteContext* context);
 
-  Status SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
-                        FlushReason flush_reason = FlushReason::kOthers);
+  Status SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context);
 
   // Force current memtable contents to be flushed.
   Status FlushMemTable(ColumnFamilyData* cfd, const FlushOptions& options,
-                       FlushReason flush_reason, bool writes_stopped = false);
+                       bool writes_stopped = false);
 
-  // Wait for memtable flushed.
-  // If flush_memtable_id is non-null, wait until the memtable with the ID
-  // gets flush. Otherwise, wait until the column family don't have any
-  // memtable pending flush.
-  Status WaitForFlushMemTable(ColumnFamilyData* cfd,
-                              const uint64_t* flush_memtable_id = nullptr);
+  // Wait for memtable flushed
+  Status WaitForFlushMemTable(ColumnFamilyData* cfd);
 
   // REQUIRES: mutex locked
-  Status SwitchWAL(WriteContext* write_context);
+  Status HandleWALFull(WriteContext* write_context);
 
   // REQUIRES: mutex locked
   Status HandleWriteBufferFull(WriteContext* write_context);
@@ -876,8 +779,7 @@ class DBImpl : public DB {
                          WriteContext* write_context);
 
   WriteBatch* MergeBatch(const WriteThread::WriteGroup& write_group,
-                         WriteBatch* tmp_batch, size_t* write_with_wal,
-                         WriteBatch** to_be_cached_state);
+                         WriteBatch* tmp_batch, size_t* write_with_wal);
 
   Status WriteToWAL(const WriteBatch& merged_batch, log::Writer* log_writer,
                     uint64_t* log_used, uint64_t* log_size);
@@ -889,10 +791,10 @@ class DBImpl : public DB {
 
   Status ConcurrentWriteToWAL(const WriteThread::WriteGroup& write_group,
                               uint64_t* log_used, SequenceNumber* last_sequence,
-                              size_t seq_inc);
+                              int total_count);
 
   // Used by WriteImpl to update bg_error_ if paranoid check is enabled.
-  void WriteStatusCheck(const Status& status);
+  void WriteCallbackStatusCheck(const Status& status);
 
   // Used by WriteImpl to update bg_error_ in case of memtable insert error.
   void MemTableInsertStatusCheck(const Status& memtable_insert_status);
@@ -902,7 +804,6 @@ class DBImpl : public DB {
   Status CompactFilesImpl(const CompactionOptions& compact_options,
                           ColumnFamilyData* cfd, Version* version,
                           const std::vector<std::string>& input_file_names,
-                          std::vector<std::string>* const output_file_names,
                           const int output_level, int output_path_id,
                           JobContext* job_context, LogBuffer* log_buffer);
 
@@ -919,10 +820,10 @@ class DBImpl : public DB {
   ColumnFamilyData* GetColumnFamilyDataByName(const std::string& cf_name);
 
   void MaybeScheduleFlushOrCompaction();
-  void SchedulePendingFlush(ColumnFamilyData* cfd, FlushReason flush_reason);
+  void SchedulePendingFlush(ColumnFamilyData* cfd);
   void SchedulePendingCompaction(ColumnFamilyData* cfd);
-  void SchedulePendingPurge(std::string fname, std::string dir_to_sync,
-                            FileType type, uint64_t number, int job_id);
+  void SchedulePendingPurge(std::string fname, FileType type, uint64_t number,
+                            uint32_t path_id, int job_id);
   static void BGWorkCompaction(void* arg);
   // Runs a pre-chosen universal compaction involving bottom level in a
   // separate, bottom-pri thread pool.
@@ -940,9 +841,6 @@ class DBImpl : public DB {
   Status BackgroundFlush(bool* madeProgress, JobContext* job_context,
                          LogBuffer* log_buffer);
 
-  bool EnoughRoomForCompaction(const std::vector<CompactionInputFiles>& inputs,
-                               bool* sfm_bookkeeping, LogBuffer* log_buffer);
-
   void PrintStatistics();
 
   // dump rocksdb.stats to LOG
@@ -961,20 +859,16 @@ class DBImpl : public DB {
   // helper functions for adding and removing from flush & compaction queues
   void AddToCompactionQueue(ColumnFamilyData* cfd);
   ColumnFamilyData* PopFirstFromCompactionQueue();
-  void AddToFlushQueue(ColumnFamilyData* cfd, FlushReason flush_reason);
+  void AddToFlushQueue(ColumnFamilyData* cfd);
   ColumnFamilyData* PopFirstFromFlushQueue();
 
   // helper function to call after some of the logs_ were synced
   void MarkLogsSynced(uint64_t up_to, bool synced_dir, const Status& status);
 
-  SnapshotImpl* GetSnapshotImpl(bool is_write_conflict_boundary);
+  const Snapshot* GetSnapshotImpl(bool is_write_conflict_boundary);
 
   uint64_t GetMaxTotalWalSize() const;
 
-  Directory* GetDataDir(ColumnFamilyData* cfd, size_t path_id) const;
-
-  Status CloseHelper();
-
   // table_cache_ provides its own synchronization
   std::shared_ptr<Cache> table_cache_;
 
@@ -982,12 +876,12 @@ class DBImpl : public DB {
   FileLock* db_lock_;
 
   // In addition to mutex_, log_write_mutex_ protected writes to logs_ and
-  // logfile_number_. With two_write_queues it also protects alive_log_files_,
+  // logfile_number_. With concurrent_prepare it also protects alive_log_files_,
   // and log_empty_. Refer to the definition of each variable below for more
   // details.
   InstrumentedMutex log_write_mutex_;
   // State below is protected by mutex_
-  // With two_write_queues enabled, some of the variables that accessed during
+  // With concurrent_prepare enabled, some of the variables that accessed during
   // WriteToWAL need different synchronization: log_empty_, alive_log_files_,
   // logs_, logfile_number_. Refer to the definition of each variable below for
   // more description.
@@ -1003,10 +897,6 @@ class DBImpl : public DB {
   // (i.e. whenever a flush is done, even if it didn't make any progress)
   // * whenever there is an error in background purge, flush or compaction
   // * whenever num_running_ingest_file_ goes to 0.
-  // * whenever pending_purge_obsolete_files_ goes to 0.
-  // * whenever disable_delete_obsolete_files_ goes to 0.
-  // * whenever SetOptions successfully updates options.
-  // * whenever a column family is dropped.
   InstrumentedCondVar bg_cv_;
   // Writes are protected by locking both mutex_ and log_write_mutex_, and reads
   // must be under either mutex_ or log_write_mutex_. Since after ::Open,
@@ -1014,12 +904,12 @@ class DBImpl : public DB {
   // from the same write_thread_ without any locks.
   uint64_t logfile_number_;
   std::deque<uint64_t>
-      log_recycle_files_;  // a list of log files that we can recycle
+      log_recycle_files;  // a list of log files that we can recycle
   bool log_dir_synced_;
-  // Without two_write_queues, read and writes to log_empty_ are protected by
+  // Without concurrent_prepare, read and writes to log_empty_ are protected by
   // mutex_. Since it is currently updated/read only in write_thread_, it can be
   // accessed from the same write_thread_ without any locks. With
-  // two_write_queues writes, where it can be updated in different threads,
+  // concurrent_prepare writes, where it can be updated in different threads,
   // read and writes are protected by log_write_mutex_ instead. This is to avoid
   // expesnive mutex_ lock during WAL write, which update log_empty_.
   bool log_empty_;
@@ -1044,11 +934,9 @@ class DBImpl : public DB {
       writer = nullptr;
       return w;
     }
-    Status ClearWriter() {
-      Status s = writer->WriteBuffer();
+    void ClearWriter() {
       delete writer;
       writer = nullptr;
-      return s;
     }
 
     uint64_t number;
@@ -1058,10 +946,10 @@ class DBImpl : public DB {
     // true for some prefix of logs_
     bool getting_synced = false;
   };
-  // Without two_write_queues, read and writes to alive_log_files_ are
+  // Without concurrent_prepare, read and writes to alive_log_files_ are
   // protected by mutex_. However since back() is never popped, and push_back()
   // is done only from write_thread_, the same thread can access the item
-  // reffered by back() without mutex_. With two_write_queues_, writes
+  // reffered by back() without mutex_. With concurrent_prepare_, writes
   // are protected by locking both mutex_ and log_write_mutex_, and reads must
   // be under either mutex_ or log_write_mutex_.
   std::deque<LogFileNumberSize> alive_log_files_;
@@ -1081,15 +969,6 @@ class DBImpl : public DB {
   std::deque<LogWriterNumber> logs_;
   // Signaled when getting_synced becomes false for some of the logs_.
   InstrumentedCondVar log_sync_cv_;
-  // This is the app-level state that is written to the WAL but will be used
-  // only during recovery. Using this feature enables not writing the state to
-  // memtable on normal writes and hence improving the throughput. Each new
-  // write of the state will replace the previous state entirely even if the
-  // keys in the two consecuitive states do not overlap.
-  // It is protected by log_write_mutex_ when two_write_queues_ is enabled.
-  // Otherwise only the heaad of write_thread_ can access it.
-  WriteBatch cached_recoverable_state_;
-  std::atomic<bool> cached_recoverable_state_empty_ = {true};
   std::atomic<uint64_t> total_log_size_;
   // only used for dynamically adjusting max_total_wal_size. it is a sum of
   // [write_buffer_size * max_write_buffer_number] over all column families
@@ -1110,7 +989,7 @@ class DBImpl : public DB {
                           const std::string& wal_dir,
                           const std::vector<DbPath>& data_paths);
 
-    Directory* GetDataDir(size_t path_id) const;
+    Directory* GetDataDir(size_t path_id);
 
     Directory* GetWalDir() {
       if (wal_dir_) {
@@ -1125,6 +1004,9 @@ class DBImpl : public DB {
     std::unique_ptr<Directory> db_dir_;
     std::vector<std::unique_ptr<Directory>> data_dirs_;
     std::unique_ptr<Directory> wal_dir_;
+
+    Status CreateAndNewDirectory(Env* env, const std::string& dirname,
+                                 std::unique_ptr<Directory>* directory) const;
   };
 
   Directories directories_;
@@ -1166,13 +1048,13 @@ class DBImpl : public DB {
   // purge_queue_
   struct PurgeFileInfo {
     std::string fname;
-    std::string dir_to_sync;
     FileType type;
     uint64_t number;
+    uint32_t path_id;
     int job_id;
-    PurgeFileInfo(std::string fn, std::string d, FileType t, uint64_t num,
+    PurgeFileInfo(std::string fn, FileType t, uint64_t num, uint32_t pid,
                   int jid)
-        : fname(fn), dir_to_sync(d), type(t), number(num), job_id(jid) {}
+        : fname(fn), type(t), number(num), path_id(pid), job_id(jid) {}
   };
 
   // flush_queue_ and compaction_queue_ hold column families that we need to
@@ -1203,10 +1085,6 @@ class DBImpl : public DB {
   // A queue to store filenames of the files to be purged
   std::deque<PurgeFileInfo> purge_queue_;
 
-  // A vector to store the file numbers that have been assigned to certain
-  // JobContext. Current implementation tracks ssts only.
-  std::vector<uint64_t> files_grabbed_for_purge_;
-
   // A queue to store log writers to close
   std::deque<log::Writer*> logs_to_free_queue_;
   int unscheduled_flushes_;
@@ -1276,10 +1154,6 @@ class DBImpl : public DB {
   // without any synchronization
   int disable_delete_obsolete_files_;
 
-  // Number of times FindObsoleteFiles has found deletable files and the
-  // corresponding call to PurgeObsoleteFiles has not yet finished.
-  int pending_purge_obsolete_files_;
-
   // last time when DeleteObsoleteFiles with full scan was executed. Originaly
   // initialized with startup time.
   uint64_t delete_obsolete_files_last_run_;
@@ -1302,7 +1176,7 @@ class DBImpl : public DB {
   // We must attempt to free the dependent memtables again
   // at a later time after the transaction in the oldest
   // log is fully commited.
-  bool unable_to_release_oldest_log_;
+  bool unable_to_flush_oldest_log_;
 
   static const int KEEP_LOG_FILE_NUM = 1000;
   // MSVC version 1800 still does not have constexpr for ::max()
@@ -1313,9 +1187,6 @@ class DBImpl : public DB {
   // The options to access storage files
   const EnvOptions env_options_;
 
-  // Additonal options for compaction and flush
-  EnvOptions env_options_for_compaction_;
-
   // Number of running IngestExternalFile() calls.
   // REQUIRES: mutex held
   int num_running_ingest_file_;
@@ -1339,15 +1210,27 @@ class DBImpl : public DB {
   // Indicate DB was opened successfully
   bool opened_successfully_;
 
-  LogsWithPrepTracker logs_with_prep_tracker_;
+  // minimum log number still containing prepared data.
+  // this is used by FindObsoleteFiles to determine which
+  // flushed logs we must keep around because they still
+  // contain prepared data which has not been flushed or rolled back
+  std::priority_queue<uint64_t, std::vector<uint64_t>, std::greater<uint64_t>>
+      min_log_with_prep_;
 
-  // Callback for compaction to check if a key is visible to a snapshot.
-  // REQUIRES: mutex held
-  std::unique_ptr<SnapshotChecker> snapshot_checker_;
-
-  // Callback for when the cached_recoverable_state_ is written to memtable
-  // Only to be set during initialization
-  std::unique_ptr<PreReleaseCallback> recoverable_state_pre_release_callback_;
+  // to be used in conjunction with min_log_with_prep_.
+  // once a transaction with data in log L is committed or rolled back
+  // rather than removing the value from the heap we add that value
+  // to prepared_section_completed_ which maps LOG -> instance_count
+  // since a log could contain multiple prepared sections
+  //
+  // when trying to determine the minimum log still active we first
+  // consult min_log_with_prep_. while that root value maps to
+  // a value > 0 in prepared_section_completed_ we decrement the
+  // instance_count for that log and pop the root value in
+  // min_log_with_prep_. This will work the same as a min_heap
+  // where we are deleteing arbitrary elements and the up heaping.
+  std::unordered_map<uint64_t, uint64_t> prepared_section_completed_;
+  std::mutex prep_heap_mutex_;
 
   // No copying allowed
   DBImpl(const DBImpl&);
@@ -1355,15 +1238,18 @@ class DBImpl : public DB {
 
   // Background threads call this function, which is just a wrapper around
   // the InstallSuperVersion() function. Background threads carry
-  // sv_context which can have new_superversion already
+  // job_context which can have new_superversion already
   // allocated.
+  void InstallSuperVersionAndScheduleWorkWrapper(
+      ColumnFamilyData* cfd, JobContext* job_context,
+      const MutableCFOptions& mutable_cf_options);
+
   // All ColumnFamily state changes go through this function. Here we analyze
   // the new state and we schedule background work if we detect that the new
   // state needs flush or compaction.
-  void InstallSuperVersionAndScheduleWork(
-      ColumnFamilyData* cfd, SuperVersionContext* sv_context,
-      const MutableCFOptions& mutable_cf_options,
-      FlushReason flush_reason = FlushReason::kOthers);
+  SuperVersion* InstallSuperVersionAndScheduleWork(
+      ColumnFamilyData* cfd, SuperVersion* new_sv,
+      const MutableCFOptions& mutable_cf_options);
 
 #ifndef ROCKSDB_LITE
   using DB::GetPropertiesOfAllTables;
@@ -1388,42 +1274,12 @@ class DBImpl : public DB {
   bool HaveManualCompaction(ColumnFamilyData* cfd);
   bool MCOverlap(ManualCompactionState* m, ManualCompactionState* m1);
 
-  bool ShouldPurge(uint64_t file_number) const;
-  void MarkAsGrabbedForPurge(uint64_t file_number);
-
   size_t GetWalPreallocateBlockSize(uint64_t write_buffer_size) const;
-  Env::WriteLifeTimeHint CalculateWALWriteHint() {
-    return Env::WLTH_SHORT;
-  }
 
-  // When set, we use a separate queue for writes that dont write to memtable.
-  // In 2PC these are the writes at Prepare phase.
-  const bool two_write_queues_;
+  // When set, we use a seprate queue for writes that dont write to memtable. In
+  // 2PC these are the writes at Prepare phase.
+  const bool concurrent_prepare_;
   const bool manual_wal_flush_;
-  // Increase the sequence number after writing each batch, whether memtable is
-  // disabled for that or not. Otherwise the sequence number is increased after
-  // writing each key into memtable. This implies that when disable_memtable is
-  // set, the seq is not increased at all.
-  //
-  // Default: false
-  const bool seq_per_batch_;
-  // LastSequence also indicates last published sequence visibile to the
-  // readers. Otherwise LastPublishedSequence should be used.
-  const bool last_seq_same_as_publish_seq_;
-  // It indicates that a customized gc algorithm must be used for
-  // flush/compaction and if it is not provided vis SnapshotChecker, we should
-  // disable gc to be safe.
-  const bool use_custom_gc_;
-
-  // Clients must periodically call SetPreserveDeletesSequenceNumber()
-  // to advance this seqnum. Default value is 0 which means ALL deletes are
-  // preserved. Note that this has no effect if DBOptions.preserve_deletes
-  // is set to false.
-  std::atomic<SequenceNumber> preserve_deletes_seqnum_;
-  const bool preserve_deletes_;
-
-  // Flag to check whether Close() has been called on this DB
-  bool closed_;
 };
 
 extern Options SanitizeOptions(const std::string& db,
@@ -1435,25 +1291,6 @@ extern CompressionType GetCompressionFlush(
     const ImmutableCFOptions& ioptions,
     const MutableCFOptions& mutable_cf_options);
 
-// Return the earliest log file to keep after the memtable flush is
-// finalized.
-// `cfd_to_flush` is the column family whose memtable (specified in
-// `memtables_to_flush`) will be flushed and thus will not depend on any WAL
-// file.
-// The function is only applicable to 2pc mode.
-extern uint64_t PrecomputeMinLogNumberToKeep(
-    VersionSet* vset, const ColumnFamilyData& cfd_to_flush,
-    autovector<VersionEdit*> edit_list,
-    const autovector<MemTable*>& memtables_to_flush,
-    LogsWithPrepTracker* prep_tracker);
-
-// `cfd_to_flush` is the column family whose memtable will be flushed and thus
-// will not depend on any WAL file. nullptr means no memtable is being flushed.
-// The function is only applicable to 2pc mode.
-extern uint64_t FindMinPrepLogReferencedByMemTable(
-    VersionSet* vset, const ColumnFamilyData* cfd_to_flush,
-    const autovector<MemTable*>& memtables_to_flush);
-
 // Fix user-supplied options to be reasonable
 template <class T, class V>
 static void ClipToRange(T* ptr, V minvalue, V maxvalue) {

db/db_impl_debug.cc

@@ -19,16 +19,10 @@ uint64_t DBImpl::TEST_GetLevel0TotalSize() {
   return default_cf_handle_->cfd()->current()->storage_info()->NumLevelBytes(0);
 }
 
-void DBImpl::TEST_SwitchWAL() {
+void DBImpl::TEST_HandleWALFull() {
   WriteContext write_context;
   InstrumentedMutexLock l(&mutex_);
-  SwitchWAL(&write_context);
-}
-
-bool DBImpl::TEST_WALBufferIsEmpty() {
-  InstrumentedMutexLock wl(&log_write_mutex_);
-  log::Writer* cur_log_writer = logs_.back().writer;
-  return cur_log_writer->TEST_BufferIsEmpty();
+  HandleWALFull(&write_context);
 }
 
 int64_t DBImpl::TEST_MaxNextLevelOverlappingBytes(
@@ -66,10 +60,6 @@ uint64_t DBImpl::TEST_Current_Manifest_FileNo() {
   return versions_->manifest_file_number();
 }
 
-uint64_t DBImpl::TEST_Current_Next_FileNo() {
-  return versions_->current_next_file_number();
-}
-
 Status DBImpl::TEST_CompactRange(int level, const Slice* begin,
                                  const Slice* end,
                                  ColumnFamilyHandle* column_family,
@@ -86,7 +76,7 @@ Status DBImpl::TEST_CompactRange(int level, const Slice* begin,
        cfd->ioptions()->compaction_style == kCompactionStyleFIFO)
           ? level
           : level + 1;
-  return RunManualCompaction(cfd, level, output_level, 0, 0, begin, end, true,
+  return RunManualCompaction(cfd, level, output_level, 0, begin, end, true,
                              disallow_trivial_move);
 }
 
@@ -109,7 +99,7 @@ Status DBImpl::TEST_FlushMemTable(bool wait, ColumnFamilyHandle* cfh) {
     auto cfhi = reinterpret_cast<ColumnFamilyHandleImpl*>(cfh);
     cfd = cfhi->cfd();
   }
-  return FlushMemTable(cfd, fo, FlushReason::kTest);
+  return FlushMemTable(cfd, fo);
 }
 
 Status DBImpl::TEST_WaitForFlushMemTable(ColumnFamilyHandle* column_family) {
@@ -123,7 +113,7 @@ Status DBImpl::TEST_WaitForFlushMemTable(ColumnFamilyHandle* column_family) {
   return WaitForFlushMemTable(cfd);
 }
 
-Status DBImpl::TEST_WaitForCompact(bool wait_unscheduled) {
+Status DBImpl::TEST_WaitForCompact() {
   // Wait until the compaction completes
 
   // TODO: a bug here. This function actually does not necessarily
@@ -132,8 +122,7 @@ Status DBImpl::TEST_WaitForCompact(bool wait_unscheduled) {
 
   InstrumentedMutexLock l(&mutex_);
   while ((bg_bottom_compaction_scheduled_ || bg_compaction_scheduled_ ||
-          bg_flush_scheduled_ ||
-          (wait_unscheduled && unscheduled_compactions_)) &&
+          bg_flush_scheduled_) &&
          bg_error_.ok()) {
     bg_cv_.Wait();
   }
@@ -190,21 +179,11 @@ Status DBImpl::TEST_GetAllImmutableCFOptions(
 }
 
 uint64_t DBImpl::TEST_FindMinLogContainingOutstandingPrep() {
-  return logs_with_prep_tracker_.FindMinLogContainingOutstandingPrep();
-}
-
-size_t DBImpl::TEST_PreparedSectionCompletedSize() {
-  return logs_with_prep_tracker_.TEST_PreparedSectionCompletedSize();
-}
-
-size_t DBImpl::TEST_LogsWithPrepSize() {
-  return logs_with_prep_tracker_.TEST_LogsWithPrepSize();
+  return FindMinLogContainingOutstandingPrep();
 }
 
 uint64_t DBImpl::TEST_FindMinPrepLogReferencedByMemTable() {
-  autovector<MemTable*> empty_list;
-  return FindMinPrepLogReferencedByMemTable(versions_.get(), nullptr,
-                                            empty_list);
+  return FindMinPrepLogReferencedByMemTable();
 }
 
 Status DBImpl::TEST_GetLatestMutableCFOptions(
@@ -226,13 +205,5 @@ int DBImpl::TEST_BGFlushesAllowed() const {
   return GetBGJobLimits().max_flushes;
 }
 
-SequenceNumber DBImpl::TEST_GetLastVisibleSequence() const {
-  if (last_seq_same_as_publish_seq_) {
-    return versions_->LastSequence();
-  } else {
-    return versions_->LastAllocatedSequence();
-  }
-}
-
 }  // namespace rocksdb
 #endif  // NDEBUG

db/db_impl_experimental.cc

@@ -30,10 +30,10 @@ Status DBImpl::SuggestCompactRange(ColumnFamilyHandle* column_family,
   auto cfd = cfh->cfd();
   InternalKey start_key, end_key;
   if (begin != nullptr) {
-    start_key.SetMinPossibleForUserKey(*begin);
+    start_key.SetMaxPossibleForUserKey(*begin);
   }
   if (end != nullptr) {
-    end_key.SetMaxPossibleForUserKey(*end);
+    end_key.SetMinPossibleForUserKey(*end);
   }
   {
     InstrumentedMutexLock l(&mutex_);
@@ -138,9 +138,8 @@ Status DBImpl::PromoteL0(ColumnFamilyHandle* column_family, int target_level) {
     status = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),
                                     &edit, &mutex_, directories_.GetDbDir());
     if (status.ok()) {
-      InstallSuperVersionAndScheduleWork(
-          cfd, &job_context.superversion_context,
-          *cfd->GetLatestMutableCFOptions());
+      InstallSuperVersionAndScheduleWorkWrapper(
+          cfd, &job_context, *cfd->GetLatestMutableCFOptions());
     }
   }  // lock released here
   LogFlush(immutable_db_options_.info_log);

db/db_impl_files.cc

@@ -12,19 +12,120 @@
 #define __STDC_FORMAT_MACROS
 #endif
 #include <inttypes.h>
-#include <unordered_set>
 #include "db/event_helpers.h"
-#include "db/memtable_list.h"
 #include "util/file_util.h"
 #include "util/sst_file_manager_impl.h"
 
+
 namespace rocksdb {
-uint64_t DBImpl::MinLogNumberToKeep() {
-  if (allow_2pc()) {
-    return versions_->min_log_number_to_keep_2pc();
-  } else {
-    return versions_->MinLogNumberWithUnflushedData();
+uint64_t DBImpl::FindMinPrepLogReferencedByMemTable() {
+  if (!allow_2pc()) {
+    return 0;
   }
+
+  uint64_t min_log = 0;
+
+  // we must look through the memtables for two phase transactions
+  // that have been committed but not yet flushed
+  for (auto loop_cfd : *versions_->GetColumnFamilySet()) {
+    if (loop_cfd->IsDropped()) {
+      continue;
+    }
+
+    auto log = loop_cfd->imm()->GetMinLogContainingPrepSection();
+
+    if (log > 0 && (min_log == 0 || log < min_log)) {
+      min_log = log;
+    }
+
+    log = loop_cfd->mem()->GetMinLogContainingPrepSection();
+
+    if (log > 0 && (min_log == 0 || log < min_log)) {
+      min_log = log;
+    }
+  }
+
+  return min_log;
+}
+
+void DBImpl::MarkLogAsHavingPrepSectionFlushed(uint64_t log) {
+  assert(log != 0);
+  std::lock_guard<std::mutex> lock(prep_heap_mutex_);
+  auto it = prepared_section_completed_.find(log);
+  assert(it != prepared_section_completed_.end());
+  it->second += 1;
+}
+
+void DBImpl::MarkLogAsContainingPrepSection(uint64_t log) {
+  assert(log != 0);
+  std::lock_guard<std::mutex> lock(prep_heap_mutex_);
+  min_log_with_prep_.push(log);
+  auto it = prepared_section_completed_.find(log);
+  if (it == prepared_section_completed_.end()) {
+    prepared_section_completed_[log] = 0;
+  }
+}
+
+uint64_t DBImpl::FindMinLogContainingOutstandingPrep() {
+
+  if (!allow_2pc()) {
+    return 0;
+  }
+
+  std::lock_guard<std::mutex> lock(prep_heap_mutex_);
+  uint64_t min_log = 0;
+
+  // first we look in the prepared heap where we keep
+  // track of transactions that have been prepared (written to WAL)
+  // but not yet committed.
+  while (!min_log_with_prep_.empty()) {
+    min_log = min_log_with_prep_.top();
+
+    auto it = prepared_section_completed_.find(min_log);
+
+    // value was marked as 'deleted' from heap
+    if (it != prepared_section_completed_.end() && it->second > 0) {
+      it->second -= 1;
+      min_log_with_prep_.pop();
+
+      // back to squere one...
+      min_log = 0;
+      continue;
+    } else {
+      // found a valid value
+      break;
+    }
+  }
+
+  return min_log;
+}
+
+uint64_t DBImpl::MinLogNumberToKeep() {
+  uint64_t log_number = versions_->MinLogNumber();
+
+  if (allow_2pc()) {
+    // if are 2pc we must consider logs containing prepared
+    // sections of outstanding transactions.
+    //
+    // We must check min logs with outstanding prep before we check
+    // logs referneces by memtables because a log referenced by the
+    // first data structure could transition to the second under us.
+    //
+    // TODO(horuff): iterating over all column families under db mutex.
+    // should find more optimial solution
+    auto min_log_in_prep_heap = FindMinLogContainingOutstandingPrep();
+
+    if (min_log_in_prep_heap != 0 && min_log_in_prep_heap < log_number) {
+      log_number = min_log_in_prep_heap;
+    }
+
+    auto min_log_refed_by_mem = FindMinPrepLogReferencedByMemTable();
+
+    if (min_log_refed_by_mem != 0 && min_log_refed_by_mem < log_number) {
+      log_number = min_log_refed_by_mem;
+    }
+  }
+  return log_number;
 }
 
 // * Returns the list of live files in 'sst_live'
@@ -47,7 +148,7 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
 
   bool doing_the_full_scan = false;
 
-  // logic for figuring out if we're doing the full scan
+  // logic for figurint out if we're doing the full scan
   if (no_full_scan) {
     doing_the_full_scan = false;
   } else if (force ||
@@ -67,7 +168,7 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
   // threads
   // Since job_context->min_pending_output is set, until file scan finishes,
   // mutex_ cannot be released. Otherwise, we might see no min_pending_output
-  // here but later find newer generated unfinalized files while scanning.
+  // here but later find newer generated unfinalized files while scannint.
   if (!pending_outputs_.empty()) {
     job_context->min_pending_output = *pending_outputs_.begin();
   } else {
@@ -81,65 +182,27 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
                               &job_context->manifest_delete_files,
                               job_context->min_pending_output);
 
-  // Mark the elements in job_context->sst_delete_files as grabbedForPurge
-  // so that other threads calling FindObsoleteFiles with full_scan=true
-  // will not add these files to candidate list for purge.
-  for (const auto& sst_to_del : job_context->sst_delete_files) {
-    MarkAsGrabbedForPurge(sst_to_del.metadata->fd.GetNumber());
-  }
-
   // store the current filenum, lognum, etc
   job_context->manifest_file_number = versions_->manifest_file_number();
   job_context->pending_manifest_file_number =
       versions_->pending_manifest_file_number();
   job_context->log_number = MinLogNumberToKeep();
+
   job_context->prev_log_number = versions_->prev_log_number();
 
   versions_->AddLiveFiles(&job_context->sst_live);
   if (doing_the_full_scan) {
-    InfoLogPrefix info_log_prefix(!immutable_db_options_.db_log_dir.empty(),
-        dbname_);
-    std::vector<std::string> paths;
     for (size_t path_id = 0; path_id < immutable_db_options_.db_paths.size();
          path_id++) {
-      paths.emplace_back(immutable_db_options_.db_paths[path_id].path);
-    }
-
-    // Note that if cf_paths is not specified in the ColumnFamilyOptions
-    // of a particular column family, we use db_paths as the cf_paths
-    // setting. Hence, there can be multiple duplicates of files from db_paths
-    // in the following code. The duplicate are removed while identifying
-    // unique files in PurgeObsoleteFiles.
-    for (auto cfd : *versions_->GetColumnFamilySet()) {
-      for (size_t path_id = 0; path_id < cfd->ioptions()->cf_paths.size();
-           path_id++) {
-        paths.emplace_back(cfd->ioptions()->cf_paths[path_id].path);
-      }
-    }
-
-    for (auto& path : paths) {
       // set of all files in the directory. We'll exclude files that are still
       // alive in the subsequent processings.
       std::vector<std::string> files;
-      env_->GetChildren(path, &files);  // Ignore errors
-      for (const std::string& file : files) {
-        uint64_t number;
-        FileType type;
-        // 1. If we cannot parse the file name, we skip;
-        // 2. If the file with file_number equals number has already been
-        // grabbed for purge by another compaction job, or it has already been
-        // schedule for purge, we also skip it if we
-        // are doing full scan in order to avoid double deletion of the same
-        // file under race conditions. See
-        // https://github.com/facebook/rocksdb/issues/3573
-        if (!ParseFileName(file, &number, info_log_prefix.prefix, &type) ||
-            !ShouldPurge(number)) {
-          continue;
-        }
-
+      env_->GetChildren(immutable_db_options_.db_paths[path_id].path,
+                        &files);  // Ignore errors
+      for (std::string file : files) {
         // TODO(icanadi) clean up this mess to avoid having one-off "/" prefixes
         job_context->full_scan_candidate_files.emplace_back(
-            "/" + file, path);
+            "/" + file, static_cast<uint32_t>(path_id));
       }
     }
 
@@ -148,9 +211,8 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
       std::vector<std::string> log_files;
       env_->GetChildren(immutable_db_options_.wal_dir,
                         &log_files);  // Ignore errors
-      for (const std::string& log_file : log_files) {
-        job_context->full_scan_candidate_files.emplace_back(log_file,
-            immutable_db_options_.wal_dir);
+      for (std::string log_file : log_files) {
+        job_context->full_scan_candidate_files.emplace_back(log_file, 0);
       }
     }
     // Add info log files in db_log_dir
@@ -159,9 +221,8 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
       std::vector<std::string> info_log_files;
       // Ignore errors
       env_->GetChildren(immutable_db_options_.db_log_dir, &info_log_files);
-      for (std::string& log_file : info_log_files) {
-        job_context->full_scan_candidate_files.emplace_back(log_file,
-            immutable_db_options_.db_log_dir);
+      for (std::string log_file : info_log_files) {
+        job_context->full_scan_candidate_files.emplace_back(log_file, 0);
       }
     }
   }
@@ -175,11 +236,11 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
     while (alive_log_files_.begin()->number < min_log_number) {
       auto& earliest = *alive_log_files_.begin();
       if (immutable_db_options_.recycle_log_file_num >
-          log_recycle_files_.size()) {
+          log_recycle_files.size()) {
         ROCKS_LOG_INFO(immutable_db_options_.info_log,
                        "adding log %" PRIu64 " to recycle list\n",
                        earliest.number);
-        log_recycle_files_.push_back(earliest.number);
+        log_recycle_files.push_back(earliest.number);
       } else {
         job_context->log_delete_files.push_back(earliest.number);
       }
@@ -189,11 +250,11 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
       }
       job_context->size_log_to_delete += earliest.size;
       total_log_size_ -= earliest.size;
-      if (two_write_queues_) {
+      if (concurrent_prepare_) {
         log_write_mutex_.Lock();
       }
       alive_log_files_.pop_front();
-      if (two_write_queues_) {
+      if (concurrent_prepare_) {
         log_write_mutex_.Unlock();
       }
       // Current log should always stay alive since it can't have
@@ -220,11 +281,8 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
   // We're just cleaning up for DB::Write().
   assert(job_context->logs_to_free.empty());
   job_context->logs_to_free = logs_to_free_;
-  job_context->log_recycle_files.assign(log_recycle_files_.begin(),
-                                        log_recycle_files_.end());
-  if (job_context->HaveSomethingToDelete()) {
-    ++pending_purge_obsolete_files_;
-  }
+  job_context->log_recycle_files.assign(log_recycle_files.begin(),
+                                        log_recycle_files.end());
   logs_to_free_.clear();
 }
 
@@ -236,24 +294,21 @@ bool CompareCandidateFile(const JobContext::CandidateFileInfo& first,
   } else if (first.file_name < second.file_name) {
     return false;
   } else {
-    return (first.file_path > second.file_path);
+    return (first.path_id > second.path_id);
   }
 }
 };  // namespace
 
 // Delete obsolete files and log status and information of file deletion
-void DBImpl::DeleteObsoleteFileImpl(int job_id, const std::string& fname,
-                                    const std::string& path_to_sync,
-                                    FileType type, uint64_t number) {
-  Status file_deletion_status;
+void DBImpl::DeleteObsoleteFileImpl(Status file_deletion_status, int job_id,
+                                    const std::string& fname, FileType type,
+                                    uint64_t number, uint32_t path_id) {
   if (type == kTableFile) {
     file_deletion_status =
-        DeleteSSTFile(&immutable_db_options_, fname, path_to_sync);
+        DeleteSSTFile(&immutable_db_options_, fname, path_id);
   } else {
     file_deletion_status = env_->DeleteFile(fname);
   }
-  TEST_SYNC_POINT_CALLBACK("DBImpl::DeleteObsoleteFileImpl:AfterDeletion",
-      &file_deletion_status);
   if (file_deletion_status.ok()) {
     ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
                     "[JOB %d] Delete %s type=%d #%" PRIu64 " -- %s\n", job_id,
@@ -280,16 +335,19 @@ void DBImpl::DeleteObsoleteFileImpl(int job_id, const std::string& fname,
 }
 
 // Diffs the files listed in filenames and those that do not
-// belong to live files are possibly removed. Also, removes all the
+// belong to live files are posibly removed. Also, removes all the
 // files in sst_delete_files and log_delete_files.
 // It is not necessary to hold the mutex when invoking this method.
-void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
-  TEST_SYNC_POINT("DBImpl::PurgeObsoleteFiles:Begin");
+void DBImpl::PurgeObsoleteFiles(const JobContext& state, bool schedule_only) {
   // we'd better have sth to delete
   assert(state.HaveSomethingToDelete());
 
-  // FindObsoleteFiles() should've populated this so nonzero
-  assert(state.manifest_file_number != 0);
+  // this checks if FindObsoleteFiles() was run before. If not, don't do
+  // PurgeObsoleteFiles(). If FindObsoleteFiles() was run, we need to also
+  // run PurgeObsoleteFiles(), even if disable_delete_obsolete_files_ is true
+  if (state.manifest_file_number == 0) {
+    return;
+  }
 
   // Now, convert live list to an unordered map, WITHOUT mutex held;
   // set is slow.
@@ -306,23 +364,23 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
       state.log_delete_files.size() + state.manifest_delete_files.size());
   // We may ignore the dbname when generating the file names.
   const char* kDumbDbName = "";
-  for (auto& file : state.sst_delete_files) {
+  for (auto file : state.sst_delete_files) {
     candidate_files.emplace_back(
-        MakeTableFileName(kDumbDbName, file.metadata->fd.GetNumber()), file.path);
-    if (file.metadata->table_reader_handle) {
-      table_cache_->Release(file.metadata->table_reader_handle);
+        MakeTableFileName(kDumbDbName, file->fd.GetNumber()),
+        file->fd.GetPathId());
+    if (file->table_reader_handle) {
+      table_cache_->Release(file->table_reader_handle);
     }
-    file.DeleteMetadata();
+    delete file;
   }
 
   for (auto file_num : state.log_delete_files) {
     if (file_num > 0) {
-      candidate_files.emplace_back(LogFileName(kDumbDbName, file_num),
-          immutable_db_options_.wal_dir);
+      candidate_files.emplace_back(LogFileName(kDumbDbName, file_num), 0);
     }
   }
   for (const auto& filename : state.manifest_delete_files) {
-    candidate_files.emplace_back(filename, dbname_);
+    candidate_files.emplace_back(filename, 0);
   }
 
   // dedup state.candidate_files so we don't try to delete the same
@@ -345,9 +403,9 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
   std::vector<std::string> old_info_log_files;
   InfoLogPrefix info_log_prefix(!immutable_db_options_.db_log_dir.empty(),
                                 dbname_);
-  std::unordered_set<uint64_t> files_to_del;
   for (const auto& candidate_file : candidate_files) {
     std::string to_delete = candidate_file.file_name;
+    uint32_t path_id = candidate_file.path_id;
     uint64_t number;
     FileType type;
     // Ignore file if we cannot recognize it.
@@ -373,9 +431,6 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
         // DontDeletePendingOutputs fail
         keep = (sst_live_map.find(number) != sst_live_map.end()) ||
                number >= state.min_pending_output;
-        if (!keep) {
-          files_to_del.insert(number);
-        }
         break;
       case kTempFile:
         // Any temp files that are currently being written to must
@@ -411,16 +466,13 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
     }
 
     std::string fname;
-    std::string dir_to_sync;
     if (type == kTableFile) {
       // evict from cache
       TableCache::Evict(table_cache_.get(), number);
-      fname = MakeTableFileName(candidate_file.file_path, number);
-      dir_to_sync = candidate_file.file_path;
+      fname = TableFileName(immutable_db_options_.db_paths, number, path_id);
     } else {
-      dir_to_sync =
-          (type == kLogFile) ? immutable_db_options_.wal_dir : dbname_;
-      fname = dir_to_sync + "/" + to_delete;
+      fname = ((type == kLogFile) ? immutable_db_options_.wal_dir : dbname_) +
+              "/" + to_delete;
     }
 
 #ifndef ROCKSDB_LITE
@@ -434,25 +486,13 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
     Status file_deletion_status;
     if (schedule_only) {
       InstrumentedMutexLock guard_lock(&mutex_);
-      SchedulePendingPurge(fname, dir_to_sync, type, number, state.job_id);
+      SchedulePendingPurge(fname, type, number, path_id, state.job_id);
     } else {
-      DeleteObsoleteFileImpl(state.job_id, fname, dir_to_sync, type, number);
+      DeleteObsoleteFileImpl(file_deletion_status, state.job_id, fname, type,
+                             number, path_id);
     }
   }
 
-  {
-    // After purging obsolete files, remove them from files_grabbed_for_purge_.
-    // Use a temporary vector to perform bulk deletion via swap.
-    InstrumentedMutexLock guard_lock(&mutex_);
-    std::vector<uint64_t> tmp;
-    for (auto fn : files_grabbed_for_purge_) {
-      if (files_to_del.count(fn) == 0) {
-        tmp.emplace_back(fn);
-      }
-    }
-    files_grabbed_for_purge_.swap(tmp);
-  }
-
   // Delete old info log files.
   size_t old_info_log_file_count = old_info_log_files.size();
   if (old_info_log_file_count != 0 &&
@@ -491,13 +531,6 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
   wal_manager_.PurgeObsoleteWALFiles();
 #endif  // ROCKSDB_LITE
   LogFlush(immutable_db_options_.info_log);
-  InstrumentedMutexLock l(&mutex_);
-  --pending_purge_obsolete_files_;
-  assert(pending_purge_obsolete_files_ >= 0);
-  if (pending_purge_obsolete_files_ == 0) {
-    bg_cv_.SignalAll();
-  }
-  TEST_SYNC_POINT("DBImpl::PurgeObsoleteFiles:End");
 }
 
 void DBImpl::DeleteObsoleteFiles() {
@@ -512,95 +545,4 @@ void DBImpl::DeleteObsoleteFiles() {
   job_context.Clean();
   mutex_.Lock();
 }
-
-uint64_t FindMinPrepLogReferencedByMemTable(
-    VersionSet* vset, const ColumnFamilyData* cfd_to_flush,
-    const autovector<MemTable*>& memtables_to_flush) {
-  uint64_t min_log = 0;
-
-  // we must look through the memtables for two phase transactions
-  // that have been committed but not yet flushed
-  for (auto loop_cfd : *vset->GetColumnFamilySet()) {
-    if (loop_cfd->IsDropped() || loop_cfd == cfd_to_flush) {
-      continue;
-    }
-
-    auto log = loop_cfd->imm()->PrecomputeMinLogContainingPrepSection(
-        memtables_to_flush);
-
-    if (log > 0 && (min_log == 0 || log < min_log)) {
-      min_log = log;
-    }
-
-    log = loop_cfd->mem()->GetMinLogContainingPrepSection();
-
-    if (log > 0 && (min_log == 0 || log < min_log)) {
-      min_log = log;
-    }
-  }
-
-  return min_log;
-}
-
-uint64_t PrecomputeMinLogNumberToKeep(
-    VersionSet* vset, const ColumnFamilyData& cfd_to_flush,
-    autovector<VersionEdit*> edit_list,
-    const autovector<MemTable*>& memtables_to_flush,
-    LogsWithPrepTracker* prep_tracker) {
-  assert(vset != nullptr);
-  assert(prep_tracker != nullptr);
-  // Calculate updated min_log_number_to_keep
-  // Since the function should only be called in 2pc mode, log number in
-  // the version edit should be sufficient.
-
-  // Precompute the min log number containing unflushed data for the column
-  // family being flushed (`cfd_to_flush`).
-  uint64_t cf_min_log_number_to_keep = 0;
-  for (auto& e : edit_list) {
-    if (e->has_log_number()) {
-      cf_min_log_number_to_keep =
-          std::max(cf_min_log_number_to_keep, e->log_number());
-    }
-  }
-  if (cf_min_log_number_to_keep == 0) {
-    // No version edit contains information on log number. The log number
-    // for this column family should stay the same as it is.
-    cf_min_log_number_to_keep = cfd_to_flush.GetLogNumber();
-  }
-
-  // Get min log number containing unflushed data for other column families.
-  uint64_t min_log_number_to_keep =
-      vset->PreComputeMinLogNumberWithUnflushedData(&cfd_to_flush);
-  if (cf_min_log_number_to_keep != 0) {
-    min_log_number_to_keep =
-        std::min(cf_min_log_number_to_keep, min_log_number_to_keep);
-  }
-
-  // if are 2pc we must consider logs containing prepared
-  // sections of outstanding transactions.
-  //
-  // We must check min logs with outstanding prep before we check
-  // logs references by memtables because a log referenced by the
-  // first data structure could transition to the second under us.
-  //
-  // TODO: iterating over all column families under db mutex.
-  // should find more optimal solution
-  auto min_log_in_prep_heap =
-      prep_tracker->FindMinLogContainingOutstandingPrep();
-
-  if (min_log_in_prep_heap != 0 &&
-      min_log_in_prep_heap < min_log_number_to_keep) {
-    min_log_number_to_keep = min_log_in_prep_heap;
-  }
-
-  uint64_t min_log_refed_by_mem = FindMinPrepLogReferencedByMemTable(
-      vset, &cfd_to_flush, memtables_to_flush);
-
-  if (min_log_refed_by_mem != 0 &&
-      min_log_refed_by_mem < min_log_number_to_keep) {
-    min_log_number_to_keep = min_log_refed_by_mem;
-  }
-  return min_log_number_to_keep;
-}
-
 }  // namespace rocksdb

db/db_impl_open.cc

@@ -106,14 +106,14 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src) {
     result.db_paths.emplace_back(dbname, std::numeric_limits<uint64_t>::max());
   }
 
-  if (result.use_direct_reads &&
+  if (result.use_direct_io_for_flush_and_compaction &&
       result.compaction_readahead_size == 0) {
     TEST_SYNC_POINT_CALLBACK("SanitizeOptions:direct_io", nullptr);
     result.compaction_readahead_size = 1024 * 1024 * 2;
   }
 
   if (result.compaction_readahead_size > 0 ||
-      result.use_direct_reads) {
+      result.use_direct_io_for_flush_and_compaction) {
     result.new_table_reader_for_compaction_inputs = true;
   }
 
@@ -124,17 +124,6 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src) {
     result.avoid_flush_during_recovery = false;
   }
 
-#ifndef ROCKSDB_LITE
-  // When the DB is stopped, it's possible that there are some .trash files that
-  // were not deleted yet, when we open the DB we will find these .trash files
-  // and schedule them to be deleted (or delete immediately if SstFileManager
-  // was not used)
-  auto sfm = static_cast<SstFileManagerImpl*>(result.sst_file_manager.get());
-  for (size_t i = 0; i < result.db_paths.size(); i++) {
-    DeleteScheduler::CleanupDirectory(result.env, sfm, result.db_paths[i].path);
-  }
-#endif
-
   return result;
 }
 
@@ -163,23 +152,28 @@ static Status ValidateOptions(
     if (s.ok() && db_options.allow_concurrent_memtable_write) {
       s = CheckConcurrentWritesSupported(cfd.options);
     }
-    if (s.ok()) {
-      s = CheckCFPathsSupported(db_options, cfd.options);
-    }
     if (!s.ok()) {
       return s;
     }
-
-    if (cfd.options.ttl > 0 || cfd.options.compaction_options_fifo.ttl > 0) {
+    if (db_options.db_paths.size() > 1) {
+      if ((cfd.options.compaction_style != kCompactionStyleUniversal) &&
+          (cfd.options.compaction_style != kCompactionStyleLevel)) {
+        return Status::NotSupported(
+            "More than one DB paths are only supported in "
+            "universal and level compaction styles. ");
+      }
+    }
+    if (cfd.options.compaction_options_fifo.ttl > 0) {
       if (db_options.max_open_files != -1) {
         return Status::NotSupported(
-            "TTL is only supported when files are always "
+            "FIFO Compaction with TTL is only supported when files are always "
             "kept open (set max_open_files = -1). ");
       }
       if (cfd.options.table_factory->Name() !=
           BlockBasedTableFactory().Name()) {
         return Status::NotSupported(
-            "TTL is only supported in Block-Based Table format. ");
+            "FIFO Compaction with TTL is only supported in "
+            "Block-Based Table format. ");
       }
     }
   }
@@ -249,9 +243,9 @@ Status DBImpl::NewDB() {
   return s;
 }
 
-Status DBImpl::CreateAndNewDirectory(
+Status DBImpl::Directories::CreateAndNewDirectory(
     Env* env, const std::string& dirname,
-    std::unique_ptr<Directory>* directory) {
+    std::unique_ptr<Directory>* directory) const {
   // We call CreateDirIfMissing() as the directory may already exist (if we
   // are reopening a DB), when this happens we don't want creating the
   // directory to cause an error. However, we need to check if creating the
@@ -269,12 +263,12 @@ Status DBImpl::CreateAndNewDirectory(
 Status DBImpl::Directories::SetDirectories(
     Env* env, const std::string& dbname, const std::string& wal_dir,
     const std::vector<DbPath>& data_paths) {
-  Status s = DBImpl::CreateAndNewDirectory(env, dbname, &db_dir_);
+  Status s = CreateAndNewDirectory(env, dbname, &db_dir_);
   if (!s.ok()) {
     return s;
   }
   if (!wal_dir.empty() && dbname != wal_dir) {
-    s = DBImpl::CreateAndNewDirectory(env, wal_dir, &wal_dir_);
+    s = CreateAndNewDirectory(env, wal_dir, &wal_dir_);
     if (!s.ok()) {
       return s;
     }
@@ -287,7 +281,7 @@ Status DBImpl::Directories::SetDirectories(
       data_dirs_.emplace_back(nullptr);
     } else {
       std::unique_ptr<Directory> path_directory;
-      s = DBImpl::CreateAndNewDirectory(env, db_path, &path_directory);
+      s = CreateAndNewDirectory(env, db_path, &path_directory);
       if (!s.ok()) {
         return s;
       }
@@ -351,43 +345,12 @@ Status DBImpl::Recover(
       assert(s.IsIOError());
       return s;
     }
-    // Verify compatibility of env_options_ and filesystem
-    {
-      unique_ptr<RandomAccessFile> idfile;
-      EnvOptions customized_env(env_options_);
-      customized_env.use_direct_reads |=
-          immutable_db_options_.use_direct_io_for_flush_and_compaction;
-      s = env_->NewRandomAccessFile(IdentityFileName(dbname_), &idfile,
-                                    customized_env);
-      if (!s.ok()) {
-        const char* error_msg = s.ToString().c_str();
-        // Check if unsupported Direct I/O is the root cause
-        customized_env.use_direct_reads = false;
-        s = env_->NewRandomAccessFile(IdentityFileName(dbname_), &idfile,
-                                      customized_env);
-        if (s.ok()) {
-          return Status::InvalidArgument(
-              "Direct I/O is not supported by the specified DB.");
-        } else {
-          return Status::InvalidArgument(
-              "Found options incompatible with filesystem", error_msg);
-        }
-      }
-    }
   }
 
   Status s = versions_->Recover(column_families, read_only);
   if (immutable_db_options_.paranoid_checks && s.ok()) {
     s = CheckConsistency();
   }
-  if (s.ok() && !read_only) {
-    for (auto cfd : *versions_->GetColumnFamilySet()) {
-      s = cfd->AddDirectories();
-      if (!s.ok()) {
-        return s;
-      }
-    }
-  }
   if (s.ok()) {
     SequenceNumber next_sequence(kMaxSequenceNumber);
     default_cf_handle_ = new ColumnFamilyHandleImpl(
@@ -530,19 +493,11 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
   bool stop_replay_by_wal_filter = false;
   bool stop_replay_for_corruption = false;
   bool flushed = false;
-  uint64_t corrupted_log_number = kMaxSequenceNumber;
   for (auto log_number : log_numbers) {
-    if (log_number < versions_->min_log_number_to_keep_2pc()) {
-      ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                     "Skipping log #%" PRIu64
-                     " since it is older than min log to keep #%" PRIu64,
-                     log_number, versions_->min_log_number_to_keep_2pc());
-      continue;
-    }
     // The previous incarnation may not have written any MANIFEST
     // records after allocating this log number.  So we manually
     // update the file number allocation counter in VersionSet.
-    versions_->MarkFileNumberUsed(log_number);
+    versions_->MarkFileNumberUsedDuringRecovery(log_number);
     // Open the log file
     std::string fname = LogFileName(immutable_db_options_.wal_dir, log_number);
 
@@ -719,7 +674,7 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
       status = WriteBatchInternal::InsertInto(
           &batch, column_family_memtables_.get(), &flush_scheduler_, true,
           log_number, this, false /* concurrent_memtable_writes */,
-          next_sequence, &has_valid_writes, seq_per_batch_);
+          next_sequence, &has_valid_writes);
       MaybeIgnoreError(&status);
       if (!status.ok()) {
         // We are treating this as a failure while reading since we read valid
@@ -756,12 +711,6 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
     }
 
     if (!status.ok()) {
-      if (status.IsNotSupported()) {
-        // We should not treat NotSupported as corruption. It is rather a clear
-        // sign that we are processing a WAL that is produced by an incompatible
-        // version of the code.
-        return status;
-      }
       if (immutable_db_options_.wal_recovery_mode ==
           WALRecoveryMode::kSkipAnyCorruptedRecords) {
         // We should ignore all errors unconditionally
@@ -771,7 +720,6 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
         // We should ignore the error but not continue replaying
         status = Status::OK();
         stop_replay_for_corruption = true;
-        corrupted_log_number = log_number;
         ROCKS_LOG_INFO(immutable_db_options_.info_log,
                        "Point in time recovered to log #%" PRIu64
                        " seq #%" PRIu64,
@@ -789,30 +737,10 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
     auto last_sequence = *next_sequence - 1;
     if ((*next_sequence != kMaxSequenceNumber) &&
         (versions_->LastSequence() <= last_sequence)) {
-      versions_->SetLastAllocatedSequence(last_sequence);
-      versions_->SetLastPublishedSequence(last_sequence);
+      versions_->SetLastToBeWrittenSequence(last_sequence);
       versions_->SetLastSequence(last_sequence);
     }
   }
-  // Compare the corrupted log number to all columnfamily's current log number.
-  // Abort Open() if any column family's log number is greater than
-  // the corrupted log number, which means CF contains data beyond the point of
-  // corruption. This could during PIT recovery when the WAL is corrupted and
-  // some (but not all) CFs are flushed
-  if (stop_replay_for_corruption == true &&
-      (immutable_db_options_.wal_recovery_mode ==
-           WALRecoveryMode::kPointInTimeRecovery ||
-       immutable_db_options_.wal_recovery_mode ==
-           WALRecoveryMode::kTolerateCorruptedTailRecords)) {
-    for (auto cfd : *versions_->GetColumnFamilySet()) {
-      if (cfd->GetLogNumber() > corrupted_log_number) {
-        ROCKS_LOG_ERROR(immutable_db_options_.info_log,
-                        "Column family inconsistency: SST file contains data"
-                        " beyond the point of corruption.");
-        return Status::Corruption("SST file is ahead of WALs");
-      }
-    }
-  }
 
   // True if there's any data in the WALs; if not, we can skip re-processing
   // them later
@@ -868,7 +796,7 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
       // not actually used. that is because VersionSet assumes
       // VersionSet::next_file_number_ always to be strictly greater than any
       // log number
-      versions_->MarkFileNumberUsed(max_log_number + 1);
+      versions_->MarkFileNumberUsedDuringRecovery(max_log_number + 1);
       status = versions_->LogAndApply(
           cfd, *cfd->GetLatestMutableCFOptions(), edit, &mutex_);
       if (!status.ok()) {
@@ -881,13 +809,13 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
   if (data_seen && !flushed) {
     // Mark these as alive so they'll be considered for deletion later by
     // FindObsoleteFiles()
-    if (two_write_queues_) {
+    if (concurrent_prepare_) {
       log_write_mutex_.Lock();
     }
     for (auto log_number : log_numbers) {
       alive_log_files_.push_back(LogFileNumberSize(log_number));
     }
-    if (two_write_queues_) {
+    if (concurrent_prepare_) {
       log_write_mutex_.Unlock();
     }
   }
@@ -929,28 +857,26 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
     const uint64_t current_time = static_cast<uint64_t>(_current_time);
 
     {
-      auto write_hint = cfd->CalculateSSTWriteHint(0);
       mutex_.Unlock();
 
       SequenceNumber earliest_write_conflict_snapshot;
       std::vector<SequenceNumber> snapshot_seqs =
           snapshots_.GetAll(&earliest_write_conflict_snapshot);
-      auto snapshot_checker = snapshot_checker_.get();
-      if (use_custom_gc_ && snapshot_checker == nullptr) {
-        snapshot_checker = DisableGCSnapshotChecker::Instance();
-      }
+
+      EnvOptions optimized_env_options =
+          env_->OptimizeForCompactionTableWrite(env_options_, immutable_db_options_);
       s = BuildTable(
           dbname_, env_, *cfd->ioptions(), mutable_cf_options,
-          env_options_for_compaction_, cfd->table_cache(), iter.get(),
+          optimized_env_options, cfd->table_cache(), iter.get(),
           std::unique_ptr<InternalIterator>(mem->NewRangeTombstoneIterator(ro)),
           &meta, cfd->internal_comparator(),
           cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
-          snapshot_seqs, earliest_write_conflict_snapshot, snapshot_checker,
+          snapshot_seqs, earliest_write_conflict_snapshot,
           GetCompressionFlush(*cfd->ioptions(), mutable_cf_options),
           cfd->ioptions()->compression_opts, paranoid_file_checks,
           cfd->internal_stats(), TableFileCreationReason::kRecovery,
           &event_logger_, job_id, Env::IO_HIGH, nullptr /* table_properties */,
-          -1 /* level */, current_time, write_hint);
+          -1 /* level */, current_time);
       LogFlush(immutable_db_options_.info_log);
       ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
                       "[%s] [WriteLevel0TableForRecovery]"
@@ -972,7 +898,7 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
                   meta.marked_for_compaction);
   }
 
-  InternalStats::CompactionStats stats(CompactionReason::kFlush, 1);
+  InternalStats::CompactionStats stats(1);
   stats.micros = env_->NowMicros() - start_micros;
   stats.bytes_written = meta.fd.GetFileSize();
   stats.num_output_files = 1;
@@ -1003,15 +929,6 @@ Status DB::Open(const Options& options, const std::string& dbname, DB** dbptr) {
 Status DB::Open(const DBOptions& db_options, const std::string& dbname,
                 const std::vector<ColumnFamilyDescriptor>& column_families,
                 std::vector<ColumnFamilyHandle*>* handles, DB** dbptr) {
-  const bool seq_per_batch = true;
-  return DBImpl::Open(db_options, dbname, column_families, handles, dbptr,
-                      !seq_per_batch);
-}
-
-Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
-                    const std::vector<ColumnFamilyDescriptor>& column_families,
-                    std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
-                    const bool seq_per_batch) {
   Status s = SanitizeOptionsByTable(db_options, column_families);
   if (!s.ok()) {
     return s;
@@ -1031,20 +948,11 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
         std::max(max_write_buffer_size, cf.options.write_buffer_size);
   }
 
-  DBImpl* impl = new DBImpl(db_options, dbname, seq_per_batch);
+  DBImpl* impl = new DBImpl(db_options, dbname);
   s = impl->env_->CreateDirIfMissing(impl->immutable_db_options_.wal_dir);
   if (s.ok()) {
-    std::vector<std::string> paths;
-    for (auto& db_path : impl->immutable_db_options_.db_paths) {
-      paths.emplace_back(db_path.path);
-    }
-    for (auto& cf : column_families) {
-      for (auto& cf_path : cf.options.cf_paths) {
-        paths.emplace_back(cf_path.path);
-      }
-    }
-    for (auto& path : paths) {
-      s = impl->env_->CreateDirIfMissing(path);
+    for (auto db_path : impl->immutable_db_options_.db_paths) {
+      s = impl->env_->CreateDirIfMissing(db_path.path);
       if (!s.ok()) {
         break;
       }
@@ -1062,7 +970,6 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
     return s;
   }
   impl->mutex_.Lock();
-  auto write_hint = impl->CalculateWALWriteHint();
   // Handles create_if_missing, error_if_exists
   s = impl->Recover(column_families);
   if (s.ok()) {
@@ -1078,7 +985,6 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
         LogFileName(impl->immutable_db_options_.wal_dir, new_log_number),
         &lfile, opt_env_options);
     if (s.ok()) {
-      lfile->SetWriteLifeTimeHint(write_hint);
       lfile->SetPreallocationBlockSize(
           impl->GetWalPreallocateBlockSize(max_write_buffer_size));
       {
@@ -1090,8 +996,7 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
             new_log_number,
             new log::Writer(
                 std::move(file_writer), new_log_number,
-                impl->immutable_db_options_.recycle_log_file_num > 0,
-                impl->immutable_db_options_.manual_wal_flush));
+                impl->immutable_db_options_.recycle_log_file_num > 0));
       }
 
       // set column family handles
@@ -1122,18 +1027,16 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
       }
     }
     if (s.ok()) {
-      SuperVersionContext sv_context(/* create_superversion */ true);
       for (auto cfd : *impl->versions_->GetColumnFamilySet()) {
-        impl->InstallSuperVersionAndScheduleWork(
-            cfd, &sv_context, *cfd->GetLatestMutableCFOptions());
+        delete impl->InstallSuperVersionAndScheduleWork(
+            cfd, nullptr, *cfd->GetLatestMutableCFOptions());
       }
-      sv_context.Clean();
-      if (impl->two_write_queues_) {
+      if (impl->concurrent_prepare_) {
         impl->log_write_mutex_.Lock();
       }
       impl->alive_log_files_.push_back(
           DBImpl::LogFileNumberSize(impl->logfile_number_));
-      if (impl->two_write_queues_) {
+      if (impl->concurrent_prepare_) {
         impl->log_write_mutex_.Unlock();
       }
       impl->DeleteObsoleteFiles();
@@ -1188,28 +1091,17 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
       impl->immutable_db_options_.sst_file_manager.get());
   if (s.ok() && sfm) {
     // Notify SstFileManager about all sst files that already exist in
-    // db_paths[0] and cf_paths[0] when the DB is opened.
-    std::vector<std::string> paths;
-    paths.emplace_back(impl->immutable_db_options_.db_paths[0].path);
-    for (auto& cf : column_families) {
-      if (!cf.options.cf_paths.empty()) {
-        paths.emplace_back(cf.options.cf_paths[0].path);
-      }
-    }
-    // Remove duplicate paths.
-    std::sort(paths.begin(), paths.end());
-    paths.erase(std::unique(paths.begin(), paths.end()), paths.end());
-    for (auto& path : paths) {
-      std::vector<std::string> existing_files;
-      impl->immutable_db_options_.env->GetChildren(path, &existing_files);
-      for (auto& file_name : existing_files) {
-        uint64_t file_number;
-        FileType file_type;
-        std::string file_path = path + "/" + file_name;
-        if (ParseFileName(file_name, &file_number, &file_type) &&
-            file_type == kTableFile) {
-          sfm->OnAddFile(file_path);
-        }
+    // db_paths[0] when the DB is opened.
+    auto& db_path = impl->immutable_db_options_.db_paths[0];
+    std::vector<std::string> existing_files;
+    impl->immutable_db_options_.env->GetChildren(db_path.path, &existing_files);
+    for (auto& file_name : existing_files) {
+      uint64_t file_number;
+      FileType file_type;
+      std::string file_path = db_path.path + "/" + file_name;
+      if (ParseFileName(file_name, &file_number, &file_type) &&
+          file_type == kTableFile) {
+        sfm->OnAddFile(file_path);
       }
     }
   }
@@ -1218,9 +1110,6 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
   if (s.ok()) {
     ROCKS_LOG_INFO(impl->immutable_db_options_.info_log, "DB pointer %p", impl);
     LogFlush(impl->immutable_db_options_.info_log);
-    assert(impl->TEST_WALBufferIsEmpty());
-    // If the assert above fails then we need to FlushWAL before returning
-    // control back to the user.
     if (!persist_options_status.ok()) {
       s = Status::IOError(
           "DB::Open() failed --- Unable to persist Options file",

db/db_impl_readonly.cc

@@ -24,7 +24,8 @@ DBImplReadOnly::DBImplReadOnly(const DBOptions& db_options,
   LogFlush(immutable_db_options_.info_log);
 }
 
-DBImplReadOnly::~DBImplReadOnly() {}
+DBImplReadOnly::~DBImplReadOnly() {
+}
 
 // Implementations of the DB interface
 Status DBImplReadOnly::Get(const ReadOptions& read_options,
@@ -56,15 +57,14 @@ Iterator* DBImplReadOnly::NewIterator(const ReadOptions& read_options,
   auto cfd = cfh->cfd();
   SuperVersion* super_version = cfd->GetSuperVersion()->Ref();
   SequenceNumber latest_snapshot = versions_->LastSequence();
-  ReadCallback* read_callback = nullptr;  // No read callback provided.
   auto db_iter = NewArenaWrappedDbIterator(
-      env_, read_options, *cfd->ioptions(), super_version->mutable_cf_options,
+      env_, read_options, *cfd->ioptions(),
       (read_options.snapshot != nullptr
            ? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
                  ->number_
            : latest_snapshot),
       super_version->mutable_cf_options.max_sequential_skip_in_iterations,
-      super_version->version_number, read_callback);
+      super_version->version_number);
   auto internal_iter =
       NewInternalIterator(read_options, cfd, super_version, db_iter->GetArena(),
                           db_iter->GetRangeDelAggregator());
@@ -76,7 +76,6 @@ Status DBImplReadOnly::NewIterators(
     const ReadOptions& read_options,
     const std::vector<ColumnFamilyHandle*>& column_families,
     std::vector<Iterator*>* iterators) {
-  ReadCallback* read_callback = nullptr;  // No read callback provided.
   if (iterators == nullptr) {
     return Status::InvalidArgument("iterators not allowed to be nullptr");
   }
@@ -88,13 +87,13 @@ Status DBImplReadOnly::NewIterators(
     auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(cfh)->cfd();
     auto* sv = cfd->GetSuperVersion()->Ref();
     auto* db_iter = NewArenaWrappedDbIterator(
-        env_, read_options, *cfd->ioptions(), sv->mutable_cf_options,
+        env_, read_options, *cfd->ioptions(),
         (read_options.snapshot != nullptr
              ? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
                    ->number_
              : latest_snapshot),
         sv->mutable_cf_options.max_sequential_skip_in_iterations,
-        sv->version_number, read_callback);
+        sv->version_number);
     auto* internal_iter =
         NewInternalIterator(read_options, cfd, sv, db_iter->GetArena(),
                             db_iter->GetRangeDelAggregator());
@@ -106,7 +105,7 @@ Status DBImplReadOnly::NewIterators(
 }
 
 Status DB::OpenForReadOnly(const Options& options, const std::string& dbname,
-                           DB** dbptr, bool /*error_if_log_file_exist*/) {
+                           DB** dbptr, bool error_if_log_file_exist) {
   *dbptr = nullptr;
 
   // Try to first open DB as fully compacted DB
@@ -141,7 +140,6 @@ Status DB::OpenForReadOnly(
   *dbptr = nullptr;
   handles->clear();
 
-  SuperVersionContext sv_context(/* create_superversion */ true);
   DBImplReadOnly* impl = new DBImplReadOnly(db_options, dbname);
   impl->mutex_.Lock();
   Status s = impl->Recover(column_families, true /* read only */,
@@ -160,12 +158,10 @@ Status DB::OpenForReadOnly(
   }
   if (s.ok()) {
     for (auto cfd : *impl->versions_->GetColumnFamilySet()) {
-      sv_context.NewSuperVersion();
-      cfd->InstallSuperVersion(&sv_context, &impl->mutex_);
+      delete cfd->InstallSuperVersion(new SuperVersion(), &impl->mutex_);
     }
   }
   impl->mutex_.Unlock();
-  sv_context.Clean();
   if (s.ok()) {
     *dbptr = impl;
     for (auto* h : *handles) {
@@ -182,21 +178,20 @@ Status DB::OpenForReadOnly(
   return s;
 }
 
-#else   // !ROCKSDB_LITE
+#else  // !ROCKSDB_LITE
 
-Status DB::OpenForReadOnly(const Options& /*options*/,
-                           const std::string& /*dbname*/, DB** /*dbptr*/,
-                           bool /*error_if_log_file_exist*/) {
+Status DB::OpenForReadOnly(const Options& options, const std::string& dbname,
+                           DB** dbptr, bool error_if_log_file_exist) {
   return Status::NotSupported("Not supported in ROCKSDB_LITE.");
 }
 
 Status DB::OpenForReadOnly(
-    const DBOptions& /*db_options*/, const std::string& /*dbname*/,
-    const std::vector<ColumnFamilyDescriptor>& /*column_families*/,
-    std::vector<ColumnFamilyHandle*>* /*handles*/, DB** /*dbptr*/,
-    bool /*error_if_log_file_exist*/) {
+    const DBOptions& db_options, const std::string& dbname,
+    const std::vector<ColumnFamilyDescriptor>& column_families,
+    std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
+    bool error_if_log_file_exist) {
   return Status::NotSupported("Not supported in ROCKSDB_LITE.");
 }
 #endif  // !ROCKSDB_LITE
 
-}  // namespace rocksdb
+}   // namespace rocksdb

db/db_impl_readonly.h

@@ -36,49 +36,46 @@ class DBImplReadOnly : public DBImpl {
       std::vector<Iterator*>* iterators) override;
 
   using DBImpl::Put;
-  virtual Status Put(const WriteOptions& /*options*/,
-                     ColumnFamilyHandle* /*column_family*/,
-                     const Slice& /*key*/, const Slice& /*value*/) override {
+  virtual Status Put(const WriteOptions& options,
+                     ColumnFamilyHandle* column_family, const Slice& key,
+                     const Slice& value) override {
     return Status::NotSupported("Not supported operation in read only mode.");
   }
   using DBImpl::Merge;
-  virtual Status Merge(const WriteOptions& /*options*/,
-                       ColumnFamilyHandle* /*column_family*/,
-                       const Slice& /*key*/, const Slice& /*value*/) override {
+  virtual Status Merge(const WriteOptions& options,
+                       ColumnFamilyHandle* column_family, const Slice& key,
+                       const Slice& value) override {
     return Status::NotSupported("Not supported operation in read only mode.");
   }
   using DBImpl::Delete;
-  virtual Status Delete(const WriteOptions& /*options*/,
-                        ColumnFamilyHandle* /*column_family*/,
-                        const Slice& /*key*/) override {
+  virtual Status Delete(const WriteOptions& options,
+                        ColumnFamilyHandle* column_family,
+                        const Slice& key) override {
     return Status::NotSupported("Not supported operation in read only mode.");
   }
   using DBImpl::SingleDelete;
-  virtual Status SingleDelete(const WriteOptions& /*options*/,
-                              ColumnFamilyHandle* /*column_family*/,
-                              const Slice& /*key*/) override {
+  virtual Status SingleDelete(const WriteOptions& options,
+                              ColumnFamilyHandle* column_family,
+                              const Slice& key) override {
     return Status::NotSupported("Not supported operation in read only mode.");
   }
-  virtual Status Write(const WriteOptions& /*options*/,
-                       WriteBatch* /*updates*/) override {
+  virtual Status Write(const WriteOptions& options,
+                       WriteBatch* updates) override {
     return Status::NotSupported("Not supported operation in read only mode.");
   }
   using DBImpl::CompactRange;
-  virtual Status CompactRange(const CompactRangeOptions& /*options*/,
-                              ColumnFamilyHandle* /*column_family*/,
-                              const Slice* /*begin*/,
-                              const Slice* /*end*/) override {
+  virtual Status CompactRange(const CompactRangeOptions& options,
+                              ColumnFamilyHandle* column_family,
+                              const Slice* begin, const Slice* end) override {
     return Status::NotSupported("Not supported operation in read only mode.");
   }
 
   using DBImpl::CompactFiles;
   virtual Status CompactFiles(
-      const CompactionOptions& /*compact_options*/,
-      ColumnFamilyHandle* /*column_family*/,
-      const std::vector<std::string>& /*input_file_names*/,
-      const int /*output_level*/, const int /*output_path_id*/ = -1,
-      std::vector<std::string>* const /*output_file_names*/ = nullptr
-      ) override {
+      const CompactionOptions& compact_options,
+      ColumnFamilyHandle* column_family,
+      const std::vector<std::string>& input_file_names,
+      const int output_level, const int output_path_id = -1) override {
     return Status::NotSupported("Not supported operation in read only mode.");
   }
 
@@ -86,18 +83,18 @@ class DBImplReadOnly : public DBImpl {
     return Status::NotSupported("Not supported operation in read only mode.");
   }
 
-  virtual Status EnableFileDeletions(bool /*force*/) override {
+  virtual Status EnableFileDeletions(bool force) override {
     return Status::NotSupported("Not supported operation in read only mode.");
   }
   virtual Status GetLiveFiles(std::vector<std::string>&,
-                              uint64_t* /*manifest_file_size*/,
-                              bool /*flush_memtable*/ = true) override {
+                              uint64_t* manifest_file_size,
+                              bool flush_memtable = true) override {
     return Status::NotSupported("Not supported operation in read only mode.");
   }
 
   using DBImpl::Flush;
-  virtual Status Flush(const FlushOptions& /*options*/,
-                       ColumnFamilyHandle* /*column_family*/) override {
+  virtual Status Flush(const FlushOptions& options,
+                       ColumnFamilyHandle* column_family) override {
     return Status::NotSupported("Not supported operation in read only mode.");
   }
 
@@ -108,9 +105,9 @@ class DBImplReadOnly : public DBImpl {
 
   using DB::IngestExternalFile;
   virtual Status IngestExternalFile(
-      ColumnFamilyHandle* /*column_family*/,
-      const std::vector<std::string>& /*external_files*/,
-      const IngestExternalFileOptions& /*ingestion_options*/) override {
+      ColumnFamilyHandle* column_family,
+      const std::vector<std::string>& external_files,
+      const IngestExternalFileOptions& ingestion_options) override {
     return Status::NotSupported("Not supported operation in read only mode.");
   }
 

db/db_impl_write.cc

@@ -45,11 +45,6 @@ Status DBImpl::SingleDelete(const WriteOptions& write_options,
   return DB::SingleDelete(write_options, column_family, key);
 }
 
-void DBImpl::SetRecoverableStatePreReleaseCallback(
-    PreReleaseCallback* callback) {
-  recoverable_state_pre_release_callback_.reset(callback);
-}
-
 Status DBImpl::Write(const WriteOptions& write_options, WriteBatch* my_batch) {
   return WriteImpl(write_options, my_batch, nullptr, nullptr);
 }
@@ -62,34 +57,17 @@ Status DBImpl::WriteWithCallback(const WriteOptions& write_options,
 }
 #endif  // ROCKSDB_LITE
 
-// The main write queue. This is the only write queue that updates LastSequence.
-// When using one write queue, the same sequence also indicates the last
-// published sequence.
 Status DBImpl::WriteImpl(const WriteOptions& write_options,
                          WriteBatch* my_batch, WriteCallback* callback,
                          uint64_t* log_used, uint64_t log_ref,
-                         bool disable_memtable, uint64_t* seq_used,
-                         size_t batch_cnt,
-                         PreReleaseCallback* pre_release_callback) {
-  assert(!seq_per_batch_ || batch_cnt != 0);
+                         bool disable_memtable, uint64_t* seq_used) {
   if (my_batch == nullptr) {
     return Status::Corruption("Batch is nullptr!");
   }
-  if (write_options.sync && write_options.disableWAL) {
-    return Status::InvalidArgument("Sync writes has to enable WAL.");
-  }
-  if (two_write_queues_ && immutable_db_options_.enable_pipelined_write) {
+  if (concurrent_prepare_ && immutable_db_options_.enable_pipelined_write) {
     return Status::NotSupported(
         "pipelined_writes is not compatible with concurrent prepares");
   }
-  if (seq_per_batch_ && immutable_db_options_.enable_pipelined_write) {
-    // TODO(yiwu): update pipeline write with seq_per_batch and batch_cnt
-    return Status::NotSupported(
-        "pipelined_writes is not compatible with seq_per_batch");
-  }
-  // Otherwise IsLatestPersistentState optimization does not make sense
-  assert(!WriteBatchInternal::IsLatestPersistentState(my_batch) ||
-         disable_memtable);
 
   Status status;
   if (write_options.low_pri) {
@@ -99,9 +77,9 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
     }
   }
 
-  if (two_write_queues_ && disable_memtable) {
+  if (concurrent_prepare_ && disable_memtable) {
     return WriteImplWALOnly(write_options, my_batch, callback, log_used,
-                            log_ref, seq_used, batch_cnt, pre_release_callback);
+                            log_ref, seq_used);
   }
 
   if (immutable_db_options_.enable_pipelined_write) {
@@ -111,7 +89,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
 
   PERF_TIMER_GUARD(write_pre_and_post_process_time);
   WriteThread::Writer w(write_options, my_batch, callback, log_ref,
-                        disable_memtable, batch_cnt, pre_release_callback);
+                        disable_memtable);
 
   if (!write_options.disableWAL) {
     RecordTick(stats_, WRITE_WITH_WAL);
@@ -122,35 +100,19 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
   write_thread_.JoinBatchGroup(&w);
   if (w.state == WriteThread::STATE_PARALLEL_MEMTABLE_WRITER) {
     // we are a non-leader in a parallel group
+    PERF_TIMER_GUARD(write_memtable_time);
 
     if (w.ShouldWriteToMemtable()) {
-      PERF_TIMER_STOP(write_pre_and_post_process_time);
-      PERF_TIMER_GUARD(write_memtable_time);
-
       ColumnFamilyMemTablesImpl column_family_memtables(
           versions_->GetColumnFamilySet());
       w.status = WriteBatchInternal::InsertInto(
           &w, w.sequence, &column_family_memtables, &flush_scheduler_,
           write_options.ignore_missing_column_families, 0 /*log_number*/, this,
-          true /*concurrent_memtable_writes*/, seq_per_batch_, w.batch_cnt);
-
-      PERF_TIMER_START(write_pre_and_post_process_time);
+          true /*concurrent_memtable_writes*/);
     }
 
     if (write_thread_.CompleteParallelMemTableWriter(&w)) {
       // we're responsible for exit batch group
-      for (auto* writer : *(w.write_group)) {
-        if (!writer->CallbackFailed() && writer->pre_release_callback) {
-          assert(writer->sequence != kMaxSequenceNumber);
-          Status ws = writer->pre_release_callback->Callback(writer->sequence,
-                                                             disable_memtable);
-          if (!ws.ok()) {
-            status = ws;
-            break;
-          }
-        }
-      }
-      // TODO(myabandeh): propagate status to write_group
       auto last_sequence = w.write_group->last_sequence;
       versions_->SetLastSequence(last_sequence);
       MemTableInsertStatusCheck(w.status);
@@ -182,25 +144,19 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
   WriteThread::WriteGroup write_group;
   bool in_parallel_group = false;
   uint64_t last_sequence = kMaxSequenceNumber;
-  if (!two_write_queues_) {
+  if (!concurrent_prepare_) {
     last_sequence = versions_->LastSequence();
   }
 
   mutex_.Lock();
 
-  bool need_log_sync = write_options.sync;
+  bool need_log_sync = !write_options.disableWAL && write_options.sync;
   bool need_log_dir_sync = need_log_sync && !log_dir_synced_;
-  if (!two_write_queues_ || !disable_memtable) {
+  if (!concurrent_prepare_ || !disable_memtable) {
     // With concurrent writes we do preprocess only in the write thread that
     // also does write to memtable to avoid sync issue on shared data structure
     // with the other thread
-
-    // PreprocessWrite does its own perf timing.
-    PERF_TIMER_STOP(write_pre_and_post_process_time);
-
     status = PreprocessWrite(write_options, &need_log_sync, &write_context);
-
-    PERF_TIMER_START(write_pre_and_post_process_time);
   }
   log::Writer* log_writer = logs_.back().writer;
 
@@ -211,7 +167,6 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
   // and protects against concurrent loggers and concurrent writes
   // into memtables
 
-  TEST_SYNC_POINT("DBImpl::WriteImpl:BeforeLeaderEnters");
   last_batch_group_size_ =
       write_thread_.EnterAsBatchGroupLeader(&w, &write_group);
 
@@ -229,12 +184,10 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
     // more than once to a particular key.
     bool parallel = immutable_db_options_.allow_concurrent_memtable_write &&
                     write_group.size > 1;
-    size_t total_count = 0;
-    size_t valid_batches = 0;
-    size_t total_byte_size = 0;
+    int total_count = 0;
+    uint64_t total_byte_size = 0;
     for (auto* writer : write_group) {
       if (writer->CheckCallback(this)) {
-        valid_batches += writer->batch_cnt;
         if (writer->ShouldWriteToMemtable()) {
           total_count += WriteBatchInternal::Count(writer->batch);
           parallel = parallel && !writer->batch->HasMerge();
@@ -244,15 +197,8 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
             total_byte_size, WriteBatchInternal::ByteSize(writer->batch));
       }
     }
-    // Note about seq_per_batch_: either disableWAL is set for the entire write
-    // group or not. In either case we inc seq for each write batch with no
-    // failed callback. This means that there could be a batch with
-    // disalbe_memtable in between; although we do not write this batch to
-    // memtable it still consumes a seq. Otherwise, if !seq_per_batch_, we inc
-    // the seq per valid written key to mem.
-    size_t seq_inc = seq_per_batch_ ? valid_batches : total_count;
 
-    const bool concurrent_update = two_write_queues_;
+    const bool concurrent_update = concurrent_prepare_;
     // Update stats while we are an exclusive group leader, so we know
     // that nobody else can be writing to these particular stats.
     // We're optimistic, updating the stats before we successfully
@@ -280,7 +226,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
 
     PERF_TIMER_STOP(write_pre_and_post_process_time);
 
-    if (!two_write_queues_) {
+    if (!concurrent_prepare_) {
       if (status.ok() && !write_options.disableWAL) {
         PERF_TIMER_GUARD(write_wal_time);
         status = WriteToWAL(write_group, log_writer, log_used, need_log_sync,
@@ -289,43 +235,32 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
     } else {
       if (status.ok() && !write_options.disableWAL) {
         PERF_TIMER_GUARD(write_wal_time);
-        // LastAllocatedSequence is increased inside WriteToWAL under
+        // LastToBeWrittenSequence is increased inside WriteToWAL under
         // wal_write_mutex_ to ensure ordered events in WAL
         status = ConcurrentWriteToWAL(write_group, log_used, &last_sequence,
-                                      seq_inc);
+                                      total_count);
       } else {
         // Otherwise we inc seq number for memtable writes
-        last_sequence = versions_->FetchAddLastAllocatedSequence(seq_inc);
+        last_sequence = versions_->FetchAddLastToBeWrittenSequence(total_count);
       }
     }
     assert(last_sequence != kMaxSequenceNumber);
     const SequenceNumber current_sequence = last_sequence + 1;
-    last_sequence += seq_inc;
+    last_sequence += total_count;
 
     if (status.ok()) {
       PERF_TIMER_GUARD(write_memtable_time);
 
       if (!parallel) {
-        // w.sequence will be set inside InsertInto
         w.status = WriteBatchInternal::InsertInto(
             write_group, current_sequence, column_family_memtables_.get(),
             &flush_scheduler_, write_options.ignore_missing_column_families,
-            0 /*recovery_log_number*/, this, parallel, seq_per_batch_);
+            0 /*recovery_log_number*/, this);
       } else {
         SequenceNumber next_sequence = current_sequence;
-        // Note: the logic for advancing seq here must be consistent with the
-        // logic in WriteBatchInternal::InsertInto(write_group...) as well as
-        // with WriteBatchInternal::InsertInto(write_batch...) that is called on
-        // the merged batch during recovery from the WAL.
         for (auto* writer : write_group) {
-          if (writer->CallbackFailed()) {
-            continue;
-          }
-          writer->sequence = next_sequence;
-          if (seq_per_batch_) {
-            assert(writer->batch_cnt);
-            next_sequence += writer->batch_cnt;
-          } else if (writer->ShouldWriteToMemtable()) {
+          if (writer->ShouldWriteToMemtable()) {
+            writer->sequence = next_sequence;
             next_sequence += WriteBatchInternal::Count(writer->batch);
           }
         }
@@ -344,28 +279,27 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
           w.status = WriteBatchInternal::InsertInto(
               &w, w.sequence, &column_family_memtables, &flush_scheduler_,
               write_options.ignore_missing_column_families, 0 /*log_number*/,
-              this, true /*concurrent_memtable_writes*/, seq_per_batch_,
-              w.batch_cnt);
+              this, true /*concurrent_memtable_writes*/);
+        }
+        if (seq_used != nullptr) {
+          *seq_used = w.sequence;
         }
-      }
-      if (seq_used != nullptr) {
-        *seq_used = w.sequence;
       }
     }
   }
   PERF_TIMER_START(write_pre_and_post_process_time);
 
   if (!w.CallbackFailed()) {
-    WriteStatusCheck(status);
+    WriteCallbackStatusCheck(status);
   }
 
   if (need_log_sync) {
     mutex_.Lock();
     MarkLogsSynced(logfile_number_, need_log_dir_sync, status);
     mutex_.Unlock();
-    // Requesting sync with two_write_queues_ is expected to be very rare. We
-    // hence provide a simple implementation that is not necessarily efficient.
-    if (two_write_queues_) {
+    // Requesting sync with concurrent_prepare_ is expected to be very rare. We
+    // hance provide a simple implementation that is not necessarily efficient.
+    if (concurrent_prepare_) {
       if (manual_wal_flush_) {
         status = FlushWAL(true);
       } else {
@@ -382,17 +316,6 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
   }
   if (should_exit_batch_group) {
     if (status.ok()) {
-      for (auto* writer : write_group) {
-        if (!writer->CallbackFailed() && writer->pre_release_callback) {
-          assert(writer->sequence != kMaxSequenceNumber);
-          Status ws = writer->pre_release_callback->Callback(writer->sequence,
-                                                             disable_memtable);
-          if (!ws.ok()) {
-            status = ws;
-            break;
-          }
-        }
-      }
       versions_->SetLastSequence(last_sequence);
     }
     MemTableInsertStatusCheck(w.status);
@@ -425,10 +348,7 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
     mutex_.Lock();
     bool need_log_sync = !write_options.disableWAL && write_options.sync;
     bool need_log_dir_sync = need_log_sync && !log_dir_synced_;
-    // PreprocessWrite does its own perf timing.
-    PERF_TIMER_STOP(write_pre_and_post_process_time);
     w.status = PreprocessWrite(write_options, &need_log_sync, &write_context);
-    PERF_TIMER_START(write_pre_and_post_process_time);
     log::Writer* log_writer = logs_.back().writer;
     mutex_.Unlock();
 
@@ -465,7 +385,6 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
     RecordTick(stats_, NUMBER_KEYS_WRITTEN, total_count);
     stats->AddDBStats(InternalStats::BYTES_WRITTEN, total_byte_size);
     RecordTick(stats_, BYTES_WRITTEN, total_byte_size);
-    MeasureTime(stats_, BYTES_PER_WRITE, total_byte_size);
 
     PERF_TIMER_STOP(write_pre_and_post_process_time);
 
@@ -483,7 +402,7 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
     }
 
     if (!w.CallbackFailed()) {
-      WriteStatusCheck(w.status);
+      WriteCallbackStatusCheck(w.status);
     }
 
     if (need_log_sync) {
@@ -507,7 +426,7 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
       memtable_write_group.status = WriteBatchInternal::InsertInto(
           memtable_write_group, w.sequence, column_family_memtables_.get(),
           &flush_scheduler_, write_options.ignore_missing_column_families,
-          0 /*log_number*/, this, seq_per_batch_);
+          0 /*log_number*/, this);
       versions_->SetLastSequence(memtable_write_group.last_sequence);
       write_thread_.ExitAsMemTableWriter(&w, memtable_write_group);
     }
@@ -535,19 +454,17 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
   return w.FinalStatus();
 }
 
-// The 2nd write queue. If enabled it will be used only for WAL-only writes.
-// This is the only queue that updates LastPublishedSequence which is only
-// applicable in a two-queue setting.
 Status DBImpl::WriteImplWALOnly(const WriteOptions& write_options,
                                 WriteBatch* my_batch, WriteCallback* callback,
                                 uint64_t* log_used, uint64_t log_ref,
-                                uint64_t* seq_used, size_t batch_cnt,
-                                PreReleaseCallback* pre_release_callback) {
+                                uint64_t* seq_used) {
   Status status;
   PERF_TIMER_GUARD(write_pre_and_post_process_time);
   WriteThread::Writer w(write_options, my_batch, callback, log_ref,
-                        true /* disable_memtable */, batch_cnt,
-                        pre_release_callback);
+                        true /* disable_memtable */);
+  if (write_options.disableWAL) {
+    return status;
+  }
   RecordTick(stats_, WRITE_WITH_WAL);
   StopWatch write_sw(env_, immutable_db_options_.statistics.get(), DB_WRITE);
 
@@ -571,7 +488,7 @@ Status DBImpl::WriteImplWALOnly(const WriteOptions& write_options,
   // Note: no need to update last_batch_group_size_ here since the batch writes
   // to WAL only
 
-  size_t total_byte_size = 0;
+  uint64_t total_byte_size = 0;
   for (auto* writer : write_group) {
     if (writer->CheckCallback(this)) {
       total_byte_size = WriteBatchInternal::AppendedByteSize(
@@ -601,39 +518,19 @@ Status DBImpl::WriteImplWALOnly(const WriteOptions& write_options,
   PERF_TIMER_STOP(write_pre_and_post_process_time);
 
   PERF_TIMER_GUARD(write_wal_time);
-  // LastAllocatedSequence is increased inside WriteToWAL under
+  // LastToBeWrittenSequence is increased inside WriteToWAL under
   // wal_write_mutex_ to ensure ordered events in WAL
-  size_t seq_inc = 0 /* total_count */;
-  if (seq_per_batch_) {
-    size_t total_batch_cnt = 0;
-    for (auto* writer : write_group) {
-      assert(writer->batch_cnt);
-      total_batch_cnt += writer->batch_cnt;
-    }
-    seq_inc = total_batch_cnt;
-  }
-  if (!write_options.disableWAL) {
-    status =
-        ConcurrentWriteToWAL(write_group, log_used, &last_sequence, seq_inc);
-  } else {
-    // Otherwise we inc seq number to do solely the seq allocation
-    last_sequence = versions_->FetchAddLastAllocatedSequence(seq_inc);
-  }
+  status = ConcurrentWriteToWAL(write_group, log_used, &last_sequence,
+                                0 /*total_count*/);
   auto curr_seq = last_sequence + 1;
   for (auto* writer : write_group) {
-    if (writer->CallbackFailed()) {
-      continue;
+    if (writer->CheckCallback(this)) {
+      writer->sequence = curr_seq;
+      curr_seq += WriteBatchInternal::Count(writer->batch);
     }
-    writer->sequence = curr_seq;
-    if (seq_per_batch_) {
-      assert(writer->batch_cnt);
-      curr_seq += writer->batch_cnt;
-    }
-    // else seq advances only by memtable writes
   }
   if (status.ok() && write_options.sync) {
-    assert(!write_options.disableWAL);
-    // Requesting sync with two_write_queues_ is expected to be very rare. We
+    // Requesting sync with concurrent_prepare_ is expected to be very rare. We
     // hance provide a simple implementation that is not necessarily efficient.
     if (manual_wal_flush_) {
       status = FlushWAL(true);
@@ -644,21 +541,7 @@ Status DBImpl::WriteImplWALOnly(const WriteOptions& write_options,
   PERF_TIMER_START(write_pre_and_post_process_time);
 
   if (!w.CallbackFailed()) {
-    WriteStatusCheck(status);
-  }
-  if (status.ok()) {
-    for (auto* writer : write_group) {
-      if (!writer->CallbackFailed() && writer->pre_release_callback) {
-        assert(writer->sequence != kMaxSequenceNumber);
-        const bool DISABLE_MEMTABLE = true;
-        Status ws = writer->pre_release_callback->Callback(writer->sequence,
-                                                           DISABLE_MEMTABLE);
-        if (!ws.ok()) {
-          status = ws;
-          break;
-        }
-      }
-    }
+    WriteCallbackStatusCheck(status);
   }
   nonmem_write_thread_.ExitAsBatchGroupLeader(write_group, status);
   if (status.ok()) {
@@ -670,7 +553,7 @@ Status DBImpl::WriteImplWALOnly(const WriteOptions& write_options,
   return status;
 }
 
-void DBImpl::WriteStatusCheck(const Status& status) {
+void DBImpl::WriteCallbackStatusCheck(const Status& status) {
   // Is setting bg_error_ enough here?  This will at least stop
   // compaction and fail any further writes.
   if (immutable_db_options_.paranoid_checks && !status.ok() &&
@@ -718,13 +601,11 @@ Status DBImpl::PreprocessWrite(const WriteOptions& write_options,
   assert(write_context != nullptr && need_log_sync != nullptr);
   Status status;
 
-  PERF_TIMER_GUARD(write_scheduling_flushes_compactions_time);
-
   assert(!single_column_family_mode_ ||
          versions_->GetColumnFamilySet()->NumberOfColumnFamilies() == 1);
   if (UNLIKELY(status.ok() && !single_column_family_mode_ &&
                total_log_size_ > GetMaxTotalWalSize())) {
-    status = SwitchWAL(write_context);
+    status = HandleWALFull(write_context);
   }
 
   if (UNLIKELY(status.ok() && write_buffer_manager_->ShouldFlush())) {
@@ -737,26 +618,21 @@ Status DBImpl::PreprocessWrite(const WriteOptions& write_options,
   }
 
   if (UNLIKELY(status.ok() && !bg_error_.ok())) {
-    status = bg_error_;
+    return bg_error_;
   }
 
   if (UNLIKELY(status.ok() && !flush_scheduler_.Empty())) {
     status = ScheduleFlushes(write_context);
   }
 
-  PERF_TIMER_STOP(write_scheduling_flushes_compactions_time);
-  PERF_TIMER_GUARD(write_pre_and_post_process_time);
-
   if (UNLIKELY(status.ok() && (write_controller_.IsStopped() ||
                                write_controller_.NeedsDelay()))) {
-    PERF_TIMER_STOP(write_pre_and_post_process_time);
     PERF_TIMER_GUARD(write_delay_time);
     // We don't know size of curent batch so that we always use the size
     // for previous one. It might create a fairness issue that expiration
     // might happen for smaller writes but larger writes can go through.
     // Can optimize it if it is an issue.
     status = DelayWrite(last_batch_group_size_, write_options);
-    PERF_TIMER_START(write_pre_and_post_process_time);
   }
 
   if (status.ok() && *need_log_sync) {
@@ -787,24 +663,18 @@ Status DBImpl::PreprocessWrite(const WriteOptions& write_options,
 }
 
 WriteBatch* DBImpl::MergeBatch(const WriteThread::WriteGroup& write_group,
-                               WriteBatch* tmp_batch, size_t* write_with_wal,
-                               WriteBatch** to_be_cached_state) {
+                               WriteBatch* tmp_batch, size_t* write_with_wal) {
   assert(write_with_wal != nullptr);
   assert(tmp_batch != nullptr);
-  assert(*to_be_cached_state == nullptr);
   WriteBatch* merged_batch = nullptr;
   *write_with_wal = 0;
   auto* leader = write_group.leader;
-  assert(!leader->disable_wal);  // Same holds for all in the batch group
-  if (write_group.size == 1 && !leader->CallbackFailed() &&
+  if (write_group.size == 1 && leader->ShouldWriteToWAL() &&
       leader->batch->GetWalTerminationPoint().is_cleared()) {
     // we simply write the first WriteBatch to WAL if the group only
     // contains one batch, that batch should be written to the WAL,
     // and the batch is not wanting to be truncated
     merged_batch = leader->batch;
-    if (WriteBatchInternal::IsLatestPersistentState(merged_batch)) {
-      *to_be_cached_state = merged_batch;
-    }
     *write_with_wal = 1;
   } else {
     // WAL needs all of the batches flattened into a single batch.
@@ -812,13 +682,9 @@ WriteBatch* DBImpl::MergeBatch(const WriteThread::WriteGroup& write_group,
     // interface
     merged_batch = tmp_batch;
     for (auto writer : write_group) {
-      if (!writer->CallbackFailed()) {
+      if (writer->ShouldWriteToWAL()) {
         WriteBatchInternal::Append(merged_batch, writer->batch,
                                    /*WAL_only*/ true);
-        if (WriteBatchInternal::IsLatestPersistentState(writer->batch)) {
-          // We only need to cache the last of such write batch
-          *to_be_cached_state = writer->batch;
-        }
         (*write_with_wal)++;
       }
     }
@@ -826,7 +692,7 @@ WriteBatch* DBImpl::MergeBatch(const WriteThread::WriteGroup& write_group,
   return merged_batch;
 }
 
-// When two_write_queues_ is disabled, this function is called from the only
+// When concurrent_prepare_ is disabled, this function is called from the only
 // write thread. Otherwise this must be called holding log_write_mutex_.
 Status DBImpl::WriteToWAL(const WriteBatch& merged_batch,
                           log::Writer* log_writer, uint64_t* log_used,
@@ -834,21 +700,7 @@ Status DBImpl::WriteToWAL(const WriteBatch& merged_batch,
   assert(log_size != nullptr);
   Slice log_entry = WriteBatchInternal::Contents(&merged_batch);
   *log_size = log_entry.size();
-  // When two_write_queues_ WriteToWAL has to be protected from concurretn calls
-  // from the two queues anyway and log_write_mutex_ is already held. Otherwise
-  // if manual_wal_flush_ is enabled we need to protect log_writer->AddRecord
-  // from possible concurrent calls via the FlushWAL by the application.
-  const bool needs_locking = manual_wal_flush_ && !two_write_queues_;
-  // Due to performance cocerns of missed branch prediction penalize the new
-  // manual_wal_flush_ feature (by UNLIKELY) instead of the more common case
-  // when we do not need any locking.
-  if (UNLIKELY(needs_locking)) {
-    log_write_mutex_.Lock();
-  }
   Status status = log_writer->AddRecord(log_entry);
-  if (UNLIKELY(needs_locking)) {
-    log_write_mutex_.Unlock();
-  }
   if (log_used != nullptr) {
     *log_used = logfile_number_;
   }
@@ -866,12 +718,9 @@ Status DBImpl::WriteToWAL(const WriteThread::WriteGroup& write_group,
                           SequenceNumber sequence) {
   Status status;
 
-  assert(!write_group.leader->disable_wal);
-  // Same holds for all in the batch group
   size_t write_with_wal = 0;
-  WriteBatch* to_be_cached_state = nullptr;
-  WriteBatch* merged_batch = MergeBatch(write_group, &tmp_batch_,
-                                        &write_with_wal, &to_be_cached_state);
+  WriteBatch* merged_batch =
+      MergeBatch(write_group, &tmp_batch_, &write_with_wal);
   if (merged_batch == write_group.leader->batch) {
     write_group.leader->log_used = logfile_number_;
   } else if (write_with_wal > 1) {
@@ -884,10 +733,6 @@ Status DBImpl::WriteToWAL(const WriteThread::WriteGroup& write_group,
 
   uint64_t log_size;
   status = WriteToWAL(*merged_batch, log_writer, log_used, &log_size);
-  if (to_be_cached_state) {
-    cached_recoverable_state_ = *to_be_cached_state;
-      cached_recoverable_state_empty_ = false;
-  }
 
   if (status.ok() && need_log_sync) {
     StopWatch sw(env_, stats_, WAL_FILE_SYNC_MICROS);
@@ -932,16 +777,13 @@ Status DBImpl::WriteToWAL(const WriteThread::WriteGroup& write_group,
 Status DBImpl::ConcurrentWriteToWAL(const WriteThread::WriteGroup& write_group,
                                     uint64_t* log_used,
                                     SequenceNumber* last_sequence,
-                                    size_t seq_inc) {
+                                    int total_count) {
   Status status;
 
-  assert(!write_group.leader->disable_wal);
-  // Same holds for all in the batch group
   WriteBatch tmp_batch;
   size_t write_with_wal = 0;
-  WriteBatch* to_be_cached_state = nullptr;
   WriteBatch* merged_batch =
-      MergeBatch(write_group, &tmp_batch, &write_with_wal, &to_be_cached_state);
+      MergeBatch(write_group, &tmp_batch, &write_with_wal);
 
   // We need to lock log_write_mutex_ since logs_ and alive_log_files might be
   // pushed back concurrently
@@ -953,17 +795,13 @@ Status DBImpl::ConcurrentWriteToWAL(const WriteThread::WriteGroup& write_group,
       writer->log_used = logfile_number_;
     }
   }
-  *last_sequence = versions_->FetchAddLastAllocatedSequence(seq_inc);
+  *last_sequence = versions_->FetchAddLastToBeWrittenSequence(total_count);
   auto sequence = *last_sequence + 1;
   WriteBatchInternal::SetSequence(merged_batch, sequence);
 
   log::Writer* log_writer = logs_.back().writer;
   uint64_t log_size;
   status = WriteToWAL(*merged_batch, log_writer, log_used, &log_size);
-  if (to_be_cached_state) {
-    cached_recoverable_state_ = *to_be_cached_state;
-      cached_recoverable_state_empty_ = false;
-  }
   log_write_mutex_.Unlock();
 
   if (status.ok()) {
@@ -978,53 +816,7 @@ Status DBImpl::ConcurrentWriteToWAL(const WriteThread::WriteGroup& write_group,
   return status;
 }
 
-Status DBImpl::WriteRecoverableState() {
-  mutex_.AssertHeld();
-  if (!cached_recoverable_state_empty_) {
-    bool dont_care_bool;
-    SequenceNumber next_seq;
-    if (two_write_queues_) {
-      log_write_mutex_.Lock();
-    }
-    SequenceNumber seq;
-    if (two_write_queues_) {
-      seq = versions_->FetchAddLastAllocatedSequence(0);
-    } else {
-      seq = versions_->LastSequence();
-    }
-    WriteBatchInternal::SetSequence(&cached_recoverable_state_, seq + 1);
-    auto status = WriteBatchInternal::InsertInto(
-        &cached_recoverable_state_, column_family_memtables_.get(),
-        &flush_scheduler_, true, 0 /*recovery_log_number*/, this,
-        false /* concurrent_memtable_writes */, &next_seq, &dont_care_bool,
-        seq_per_batch_);
-    auto last_seq = next_seq - 1;
-    if (two_write_queues_) {
-      versions_->FetchAddLastAllocatedSequence(last_seq - seq);
-      versions_->SetLastPublishedSequence(last_seq);
-    }
-    versions_->SetLastSequence(last_seq);
-    if (two_write_queues_) {
-      log_write_mutex_.Unlock();
-    }
-    if (status.ok() && recoverable_state_pre_release_callback_) {
-      const bool DISABLE_MEMTABLE = true;
-      for (uint64_t sub_batch_seq = seq + 1;
-           sub_batch_seq < next_seq && status.ok(); sub_batch_seq++) {
-        status = recoverable_state_pre_release_callback_->Callback(
-            sub_batch_seq, !DISABLE_MEMTABLE);
-      }
-    }
-    if (status.ok()) {
-      cached_recoverable_state_.Clear();
-      cached_recoverable_state_empty_ = true;
-    }
-    return status;
-  }
-  return Status::OK();
-}
-
-Status DBImpl::SwitchWAL(WriteContext* write_context) {
+Status DBImpl::HandleWALFull(WriteContext* write_context) {
   mutex_.AssertHeld();
   assert(write_context != nullptr);
   Status status;
@@ -1034,34 +826,28 @@ Status DBImpl::SwitchWAL(WriteContext* write_context) {
   }
 
   auto oldest_alive_log = alive_log_files_.begin()->number;
-  bool flush_wont_release_oldest_log = false;
-  if (allow_2pc()) {
-    auto oldest_log_with_uncommited_prep =
-        logs_with_prep_tracker_.FindMinLogContainingOutstandingPrep();
+  auto oldest_log_with_uncommited_prep = FindMinLogContainingOutstandingPrep();
 
-    assert(oldest_log_with_uncommited_prep == 0 ||
-           oldest_log_with_uncommited_prep >= oldest_alive_log);
-    if (oldest_log_with_uncommited_prep > 0 &&
-        oldest_log_with_uncommited_prep == oldest_alive_log) {
-      if (unable_to_release_oldest_log_) {
+  if (allow_2pc() &&
+      oldest_log_with_uncommited_prep > 0 &&
+      oldest_log_with_uncommited_prep <= oldest_alive_log) {
+    if (unable_to_flush_oldest_log_) {
         // we already attempted to flush all column families dependent on
-        // the oldest alive log but the log still contained uncommited
-        // transactions so there is still nothing that we can do.
+        // the oldest alive log but the log still contained uncommited transactions.
+        // the oldest alive log STILL contains uncommited transaction so there
+        // is still nothing that we can do.
         return status;
-      } else {
-        ROCKS_LOG_WARN(
-            immutable_db_options_.info_log,
-            "Unable to release oldest log due to uncommited transaction");
-        unable_to_release_oldest_log_ = true;
-        flush_wont_release_oldest_log = true;
-      }
+    } else {
+      ROCKS_LOG_WARN(
+          immutable_db_options_.info_log,
+          "Unable to release oldest log due to uncommited transaction");
+      unable_to_flush_oldest_log_ = true;
     }
-  }
-  if (!flush_wont_release_oldest_log) {
+  } else {
     // we only mark this log as getting flushed if we have successfully
     // flushed all data in this log. If this log contains outstanding prepared
     // transactions then we cannot flush this log until those transactions are commited.
-    unable_to_release_oldest_log_ = false;
+    unable_to_flush_oldest_log_ = false;
     alive_log_files_.begin()->getting_flushed = true;
   }
 
@@ -1082,7 +868,7 @@ Status DBImpl::SwitchWAL(WriteContext* write_context) {
         break;
       }
       cfd->imm()->FlushRequested();
-      SchedulePendingFlush(cfd, FlushReason::kWriteBufferManager);
+      SchedulePendingFlush(cfd);
     }
   }
   MaybeScheduleFlushOrCompaction();
@@ -1125,11 +911,10 @@ Status DBImpl::HandleWriteBufferFull(WriteContext* write_context) {
     }
   }
   if (cfd_picked != nullptr) {
-    status = SwitchMemtable(cfd_picked, write_context,
-                            FlushReason::kWriteBufferFull);
+    status = SwitchMemtable(cfd_picked, write_context);
     if (status.ok()) {
       cfd_picked->imm()->FlushRequested();
-      SchedulePendingFlush(cfd_picked, FlushReason::kWriteBufferFull);
+      SchedulePendingFlush(cfd_picked);
       MaybeScheduleFlushOrCompaction();
     }
   }
@@ -1216,7 +1001,6 @@ Status DBImpl::ThrottleLowPriWritesIfNeeded(const WriteOptions& write_options,
       // is that in case the write is heavy, low pri writes may never have
       // a chance to run. Now we guarantee we are still slowly making
       // progress.
-      PERF_TIMER_GUARD(write_delay_time);
       write_controller_.low_pri_rate_limiter()->Request(
           my_batch->GetDataSize(), Env::IO_HIGH, nullptr /* stats */,
           RateLimiter::OpType::kWrite);
@@ -1228,7 +1012,7 @@ Status DBImpl::ThrottleLowPriWritesIfNeeded(const WriteOptions& write_options,
 Status DBImpl::ScheduleFlushes(WriteContext* context) {
   ColumnFamilyData* cfd;
   while ((cfd = flush_scheduler_.TakeNextColumnFamily()) != nullptr) {
-    auto status = SwitchMemtable(cfd, context, FlushReason::kWriteBufferFull);
+    auto status = SwitchMemtable(cfd, context);
     if (cfd->Unref()) {
       delete cfd;
     }
@@ -1240,7 +1024,7 @@ Status DBImpl::ScheduleFlushes(WriteContext* context) {
 }
 
 #ifndef ROCKSDB_LITE
-void DBImpl::NotifyOnMemTableSealed(ColumnFamilyData* /*cfd*/,
+void DBImpl::NotifyOnMemTableSealed(ColumnFamilyData* cfd,
                                     const MemTableInfo& mem_table_info) {
   if (immutable_db_options_.listeners.size() == 0U) {
     return;
@@ -1257,11 +1041,10 @@ void DBImpl::NotifyOnMemTableSealed(ColumnFamilyData* /*cfd*/,
 
 // REQUIRES: mutex_ is held
 // REQUIRES: this thread is currently at the front of the writer queue
-Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
-                              FlushReason flush_reason) {
+Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
   mutex_.AssertHeld();
   WriteThread::Writer nonmem_w;
-  if (two_write_queues_) {
+  if (concurrent_prepare_) {
     // SwitchMemtable is a rare event. To simply the reasoning, we make sure
     // that there is no concurrent thread writing to WAL.
     nonmem_write_thread_.EnterUnbatched(&nonmem_w, &mutex_);
@@ -1271,13 +1054,6 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
   log::Writer* new_log = nullptr;
   MemTable* new_mem = nullptr;
 
-  // Recoverable state is persisted in WAL. After memtable switch, WAL might
-  // be deleted, so we write the state to memtable to be persisted as well.
-  Status s = WriteRecoverableState();
-  if (!s.ok()) {
-    return s;
-  }
-
   // In case of pipelined write is enabled, wait for all pending memtable
   // writers.
   if (immutable_db_options_.enable_pipelined_write) {
@@ -1287,21 +1063,22 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
   // Attempt to switch to a new memtable and trigger flush of old.
   // Do this without holding the dbmutex lock.
   assert(versions_->prev_log_number() == 0);
-  if (two_write_queues_) {
+  if (concurrent_prepare_) {
     log_write_mutex_.Lock();
   }
   bool creating_new_log = !log_empty_;
-  if (two_write_queues_) {
+  if (concurrent_prepare_) {
     log_write_mutex_.Unlock();
   }
   uint64_t recycle_log_number = 0;
   if (creating_new_log && immutable_db_options_.recycle_log_file_num &&
-      !log_recycle_files_.empty()) {
-    recycle_log_number = log_recycle_files_.front();
-    log_recycle_files_.pop_front();
+      !log_recycle_files.empty()) {
+    recycle_log_number = log_recycle_files.front();
+    log_recycle_files.pop_front();
   }
   uint64_t new_log_number =
       creating_new_log ? versions_->NewFileNumber() : logfile_number_;
+  SuperVersion* new_superversion = nullptr;
   const MutableCFOptions mutable_cf_options = *cfd->GetLatestMutableCFOptions();
 
   // Set memtable_info for memtable sealed callback
@@ -1320,8 +1097,8 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
       BuildDBOptions(immutable_db_options_, mutable_db_options_);
   const auto preallocate_block_size =
     GetWalPreallocateBlockSize(mutable_cf_options.write_buffer_size);
-  auto write_hint = CalculateWALWriteHint();
   mutex_.Unlock();
+  Status s;
   {
     if (creating_new_log) {
       EnvOptions opt_env_opt =
@@ -1346,7 +1123,6 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
         // use preallocate_block_size instead
         // of calling GetWalPreallocateBlockSize()
         lfile->SetPreallocationBlockSize(preallocate_block_size);
-        lfile->SetWriteLifeTimeHint(write_hint);
         unique_ptr<WritableFileWriter> file_writer(
             new WritableFileWriter(std::move(lfile), opt_env_opt));
         new_log = new log::Writer(
@@ -1358,7 +1134,7 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
     if (s.ok()) {
       SequenceNumber seq = versions_->LastSequence();
       new_mem = cfd->ConstructNewMemtable(mutable_cf_options, seq);
-      context->superversion_context.NewSuperVersion();
+      new_superversion = new SuperVersion();
     }
 
 #ifndef ROCKSDB_LITE
@@ -1373,7 +1149,17 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
                  ". Immutable memtables: %d.\n",
                  cfd->GetName().c_str(), new_log_number, num_imm_unflushed);
   mutex_.Lock();
-  if (s.ok() && creating_new_log) {
+  if (!s.ok()) {
+    // how do we fail if we're not creating new log?
+    assert(creating_new_log);
+    assert(!new_mem);
+    assert(!new_log);
+    if (concurrent_prepare_) {
+      nonmem_write_thread_.ExitUnbatched(&nonmem_w);
+    }
+    return s;
+  }
+  if (creating_new_log) {
     log_write_mutex_.Lock();
     logfile_number_ = new_log_number;
     assert(new_log != nullptr);
@@ -1382,31 +1168,12 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
     if (!logs_.empty()) {
       // Alway flush the buffer of the last log before switching to a new one
       log::Writer* cur_log_writer = logs_.back().writer;
-      s = cur_log_writer->WriteBuffer();
-      if (!s.ok()) {
-        ROCKS_LOG_WARN(immutable_db_options_.info_log,
-                       "[%s] Failed to switch from #%" PRIu64 " to #%" PRIu64
-                       "  WAL file -- %s\n",
-                       cfd->GetName().c_str(), cur_log_writer->get_log_number(),
-                       new_log_number);
-      }
+      cur_log_writer->WriteBuffer();
     }
     logs_.emplace_back(logfile_number_, new_log);
     alive_log_files_.push_back(LogFileNumberSize(logfile_number_));
     log_write_mutex_.Unlock();
   }
-
-  if (!s.ok()) {
-    // how do we fail if we're not creating new log?
-    assert(creating_new_log);
-    assert(!new_mem);
-    assert(!new_log);
-    if (two_write_queues_) {
-      nonmem_write_thread_.ExitUnbatched(&nonmem_w);
-    }
-    return s;
-  }
-
   for (auto loop_cfd : *versions_->GetColumnFamilySet()) {
     // all this is just optimization to delete logs that
     // are no longer needed -- if CF is empty, that means it
@@ -1425,9 +1192,9 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
   cfd->imm()->Add(cfd->mem(), &context->memtables_to_free_);
   new_mem->Ref();
   cfd->SetMemtable(new_mem);
-  InstallSuperVersionAndScheduleWork(cfd, &context->superversion_context,
-                                     mutable_cf_options, flush_reason);
-  if (two_write_queues_) {
+  context->superversions_to_free_.push_back(InstallSuperVersionAndScheduleWork(
+      cfd, new_superversion, mutable_cf_options));
+  if (concurrent_prepare_) {
     nonmem_write_thread_.ExitUnbatched(&nonmem_w);
   }
   return s;
@@ -1435,13 +1202,11 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
 
 size_t DBImpl::GetWalPreallocateBlockSize(uint64_t write_buffer_size) const {
   mutex_.AssertHeld();
-  size_t bsize = static_cast<size_t>(
-    write_buffer_size / 10 + write_buffer_size);
+  size_t bsize = write_buffer_size / 10 + write_buffer_size;
   // Some users might set very high write_buffer_size and rely on
   // max_total_wal_size or other parameters to control the WAL size.
   if (mutable_db_options_.max_total_wal_size > 0) {
-    bsize = std::min<size_t>(bsize, static_cast<size_t>(
-      mutable_db_options_.max_total_wal_size));
+    bsize = std::min<size_t>(bsize, mutable_db_options_.max_total_wal_size);
   }
   if (immutable_db_options_.db_write_buffer_size > 0) {
     bsize = std::min<size_t>(bsize, immutable_db_options_.db_write_buffer_size);
@@ -1463,10 +1228,7 @@ Status DB::Put(const WriteOptions& opt, ColumnFamilyHandle* column_family,
   // 8 bytes are taken by header, 4 bytes for count, 1 byte for type,
   // and we allocate 11 extra bytes for key length, as well as value length.
   WriteBatch batch(key.size() + value.size() + 24);
-  Status s = batch.Put(column_family, key, value);
-  if (!s.ok()) {
-    return s;
-  }
+  batch.Put(column_family, key, value);
   return Write(opt, &batch);
 }
 
@@ -1495,10 +1257,7 @@ Status DB::DeleteRange(const WriteOptions& opt,
 Status DB::Merge(const WriteOptions& opt, ColumnFamilyHandle* column_family,
                  const Slice& key, const Slice& value) {
   WriteBatch batch;
-  Status s = batch.Merge(column_family, key, value);
-  if (!s.ok()) {
-    return s;
-  }
+  batch.Merge(column_family, key, value);
   return Write(opt, &batch);
 }
 }  // namespace rocksdb

db/db_iter.h

@@ -30,12 +30,10 @@ class InternalIterator;
 // into appropriate user keys.
 extern Iterator* NewDBIterator(Env* env, const ReadOptions& read_options,
                                const ImmutableCFOptions& cf_options,
-                               const MutableCFOptions& mutable_cf_options,
                                const Comparator* user_key_comparator,
                                InternalIterator* internal_iter,
                                const SequenceNumber& sequence,
                                uint64_t max_sequential_skip_in_iterations,
-                               ReadCallback* read_callback,
                                bool allow_blob = false);
 
 // A wrapper iterator which wraps DB Iterator and the arena, with which the DB
@@ -72,18 +70,15 @@ class ArenaWrappedDBIter : public Iterator {
 
   void Init(Env* env, const ReadOptions& read_options,
             const ImmutableCFOptions& cf_options,
-            const MutableCFOptions& mutable_cf_options,
             const SequenceNumber& sequence,
             uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
-            ReadCallback* read_callback, bool allow_blob, bool allow_refresh);
+            bool allow_blob);
 
   void StoreRefreshInfo(const ReadOptions& read_options, DBImpl* db_impl,
-                        ColumnFamilyData* cfd, ReadCallback* read_callback,
-                        bool allow_blob) {
+                        ColumnFamilyData* cfd, bool allow_blob) {
     read_options_ = read_options;
     db_impl_ = db_impl;
     cfd_ = cfd;
-    read_callback_ = read_callback;
     allow_blob_ = allow_blob;
   }
 
@@ -94,9 +89,7 @@ class ArenaWrappedDBIter : public Iterator {
   ColumnFamilyData* cfd_ = nullptr;
   DBImpl* db_impl_ = nullptr;
   ReadOptions read_options_;
-  ReadCallback* read_callback_;
   bool allow_blob_ = false;
-  bool allow_refresh_ = true;
 };
 
 // Generate the arena wrapped iterator class.
@@ -104,10 +97,9 @@ class ArenaWrappedDBIter : public Iterator {
 // be supported.
 extern ArenaWrappedDBIter* NewArenaWrappedDbIterator(
     Env* env, const ReadOptions& read_options,
-    const ImmutableCFOptions& cf_options,
-    const MutableCFOptions& mutable_cf_options, const SequenceNumber& sequence,
+    const ImmutableCFOptions& cf_options, const SequenceNumber& sequence,
     uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
-    ReadCallback* read_callback, DBImpl* db_impl = nullptr,
-    ColumnFamilyData* cfd = nullptr, bool allow_blob = false,
-    bool allow_refresh = true);
+    DBImpl* db_impl = nullptr, ColumnFamilyData* cfd = nullptr,
+    bool allow_blob = false);
+
 }  // namespace rocksdb

db/db_iter_stress_test.cc

@@ -1,654 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/db_iter.h"
-#include "db/dbformat.h"
-#include "rocksdb/comparator.h"
-#include "rocksdb/options.h"
-#include "rocksdb/slice.h"
-#include "util/random.h"
-#include "util/string_util.h"
-#include "util/testharness.h"
-#include "utilities/merge_operators.h"
-
-#ifdef GFLAGS
-
-#include "util/gflags_compat.h"
-
-using GFLAGS_NAMESPACE::ParseCommandLineFlags;
-
-DEFINE_bool(verbose, false,
-            "Print huge, detailed trace. Intended for debugging failures.");
-
-#else
-
-void ParseCommandLineFlags(int*, char***, bool) {}
-bool FLAGS_verbose = false;
-
-#endif
-
-namespace rocksdb {
-
-class DBIteratorStressTest : public testing::Test {
- public:
-  Env* env_;
-
-  DBIteratorStressTest() : env_(Env::Default()) {}
-};
-
-namespace {
-
-struct Entry {
-  std::string key;
-  ValueType type;  // kTypeValue, kTypeDeletion, kTypeMerge
-  uint64_t sequence;
-  std::string ikey;  // internal key, made from `key`, `sequence` and `type`
-  std::string value;
-  // If false, we'll pretend that this entry doesn't exist.
-  bool visible = true;
-
-  bool operator<(const Entry& e) const {
-    if (key != e.key) return key < e.key;
-    return std::tie(sequence, type) > std::tie(e.sequence, e.type);
-  }
-};
-
-struct Data {
-  std::vector<Entry> entries;
-
-  // Indices in `entries` with `visible` = false.
-  std::vector<size_t> hidden;
-  // Keys of entries whose `visible` changed since the last seek of iterators.
-  std::set<std::string> recently_touched_keys;
-};
-
-struct StressTestIterator : public InternalIterator {
-  Data* data;
-  Random64* rnd;
-  InternalKeyComparator cmp;
-
-  // Each operation will return error with this probability...
-  double error_probability = 0;
-  // ... and add/remove entries with this probability.
-  double mutation_probability = 0;
-  // The probability of adding vs removing entries will be chosen so that the
-  // amount of removed entries stays somewhat close to this number.
-  double target_hidden_fraction = 0;
-  // If true, print all mutations to stdout for debugging.
-  bool trace = false;
-
-  int iter = -1;
-  Status status_;
-
-  StressTestIterator(Data* _data, Random64* _rnd, const Comparator* _cmp)
-      : data(_data), rnd(_rnd), cmp(_cmp) {}
-
-  bool Valid() const override {
-    if (iter >= 0 && iter < (int)data->entries.size()) {
-      assert(status_.ok());
-      return true;
-    }
-    return false;
-  }
-
-  Status status() const override { return status_; }
-
-  bool MaybeFail() {
-    if (rnd->Next() >=
-        std::numeric_limits<uint64_t>::max() * error_probability) {
-      return false;
-    }
-    if (rnd->Next() % 2) {
-      status_ = Status::Incomplete("test");
-    } else {
-      status_ = Status::IOError("test");
-    }
-    if (trace) {
-      std::cout << "injecting " << status_.ToString() << std::endl;
-    }
-    iter = -1;
-    return true;
-  }
-
-  void MaybeMutate() {
-    if (rnd->Next() >=
-        std::numeric_limits<uint64_t>::max() * mutation_probability) {
-      return;
-    }
-    do {
-      // If too many entries are hidden, hide less, otherwise hide more.
-      double hide_probability =
-          data->hidden.size() > data->entries.size() * target_hidden_fraction
-              ? 1. / 3
-              : 2. / 3;
-      if (data->hidden.empty()) {
-        hide_probability = 1;
-      }
-      bool do_hide =
-          rnd->Next() < std::numeric_limits<uint64_t>::max() * hide_probability;
-      if (do_hide) {
-        // Hide a random entry.
-        size_t idx = rnd->Next() % data->entries.size();
-        Entry& e = data->entries[idx];
-        if (e.visible) {
-          if (trace) {
-            std::cout << "hiding idx " << idx << std::endl;
-          }
-          e.visible = false;
-          data->hidden.push_back(idx);
-          data->recently_touched_keys.insert(e.key);
-        } else {
-          // Already hidden. Let's go unhide something instead, just because
-          // it's easy and it doesn't really matter what we do.
-          do_hide = false;
-        }
-      }
-      if (!do_hide) {
-        // Unhide a random entry.
-        size_t hi = rnd->Next() % data->hidden.size();
-        size_t idx = data->hidden[hi];
-        if (trace) {
-          std::cout << "unhiding idx " << idx << std::endl;
-        }
-        Entry& e = data->entries[idx];
-        assert(!e.visible);
-        e.visible = true;
-        data->hidden[hi] = data->hidden.back();
-        data->hidden.pop_back();
-        data->recently_touched_keys.insert(e.key);
-      }
-    } while (rnd->Next() % 3 != 0);  // do 3 mutations on average
-  }
-
-  void SkipForward() {
-    while (iter < (int)data->entries.size() && !data->entries[iter].visible) {
-      ++iter;
-    }
-  }
-  void SkipBackward() {
-    while (iter >= 0 && !data->entries[iter].visible) {
-      --iter;
-    }
-  }
-
-  void SeekToFirst() override {
-    if (MaybeFail()) return;
-    MaybeMutate();
-
-    status_ = Status::OK();
-    iter = 0;
-    SkipForward();
-  }
-  void SeekToLast() override {
-    if (MaybeFail()) return;
-    MaybeMutate();
-
-    status_ = Status::OK();
-    iter = (int)data->entries.size() - 1;
-    SkipBackward();
-  }
-
-  void Seek(const Slice& target) override {
-    if (MaybeFail()) return;
-    MaybeMutate();
-
-    status_ = Status::OK();
-    // Binary search.
-    auto it = std::partition_point(
-        data->entries.begin(), data->entries.end(),
-        [&](const Entry& e) { return cmp.Compare(e.ikey, target) < 0; });
-    iter = (int)(it - data->entries.begin());
-    SkipForward();
-  }
-  void SeekForPrev(const Slice& target) override {
-    if (MaybeFail()) return;
-    MaybeMutate();
-
-    status_ = Status::OK();
-    // Binary search.
-    auto it = std::partition_point(
-        data->entries.begin(), data->entries.end(),
-        [&](const Entry& e) { return cmp.Compare(e.ikey, target) <= 0; });
-    iter = (int)(it - data->entries.begin());
-    --iter;
-    SkipBackward();
-  }
-
-  void Next() override {
-    assert(Valid());
-    if (MaybeFail()) return;
-    MaybeMutate();
-    ++iter;
-    SkipForward();
-  }
-  void Prev() override {
-    assert(Valid());
-    if (MaybeFail()) return;
-    MaybeMutate();
-    --iter;
-    SkipBackward();
-  }
-
-  Slice key() const override {
-    assert(Valid());
-    return data->entries[iter].ikey;
-  }
-  Slice value() const override {
-    assert(Valid());
-    return data->entries[iter].value;
-  }
-
-  bool IsKeyPinned() const override { return true; }
-  bool IsValuePinned() const override { return true; }
-};
-
-// A small reimplementation of DBIter, supporting only some of the features,
-// and doing everything in O(log n).
-// Skips all keys that are in recently_touched_keys.
-struct ReferenceIterator {
-  Data* data;
-  uint64_t sequence;  // ignore entries with sequence number below this
-
-  bool valid = false;
-  std::string key;
-  std::string value;
-
-  ReferenceIterator(Data* _data, uint64_t _sequence)
-      : data(_data), sequence(_sequence) {}
-
-  bool Valid() const { return valid; }
-
-  // Finds the first entry with key
-  // greater/less/greater-or-equal/less-or-equal than `key`, depending on
-  // arguments: if `skip`, inequality is strict; if `forward`, it's
-  // greater/greater-or-equal, otherwise less/less-or-equal.
-  // Sets `key` to the result.
-  // If no such key exists, returns false. Doesn't check `visible`.
-  bool FindNextKey(bool skip, bool forward) {
-    valid = false;
-    auto it = std::partition_point(data->entries.begin(), data->entries.end(),
-                                   [&](const Entry& e) {
-                                     if (forward != skip) {
-                                       return e.key < key;
-                                     } else {
-                                       return e.key <= key;
-                                     }
-                                   });
-    if (forward) {
-      if (it != data->entries.end()) {
-        key = it->key;
-        return true;
-      }
-    } else {
-      if (it != data->entries.begin()) {
-        --it;
-        key = it->key;
-        return true;
-      }
-    }
-    return false;
-  }
-
-  bool FindValueForCurrentKey() {
-    if (data->recently_touched_keys.count(key)) {
-      return false;
-    }
-
-    // Find the first entry for the key. The caller promises that it exists.
-    auto it = std::partition_point(data->entries.begin(), data->entries.end(),
-                                   [&](const Entry& e) {
-                                     if (e.key != key) {
-                                       return e.key < key;
-                                     }
-                                     return e.sequence > sequence;
-                                   });
-
-    // Find the first visible entry.
-    for (;; ++it) {
-      if (it == data->entries.end()) {
-        return false;
-      }
-      Entry& e = *it;
-      if (e.key != key) {
-        return false;
-      }
-      assert(e.sequence <= sequence);
-      if (!e.visible) continue;
-      if (e.type == kTypeDeletion) {
-        return false;
-      }
-      if (e.type == kTypeValue) {
-        value = e.value;
-        valid = true;
-        return true;
-      }
-      assert(e.type == kTypeMerge);
-      break;
-    }
-
-    // Collect merge operands.
-    std::vector<Slice> operands;
-    for (; it != data->entries.end(); ++it) {
-      Entry& e = *it;
-      if (e.key != key) {
-        break;
-      }
-      assert(e.sequence <= sequence);
-      if (!e.visible) continue;
-      if (e.type == kTypeDeletion) {
-        break;
-      }
-      operands.push_back(e.value);
-      if (e.type == kTypeValue) {
-        break;
-      }
-    }
-
-    // Do a merge.
-    value = operands.back().ToString();
-    for (int i = (int)operands.size() - 2; i >= 0; --i) {
-      value.append(",");
-      value.append(operands[i].data(), operands[i].size());
-    }
-
-    valid = true;
-    return true;
-  }
-
-  // Start at `key` and move until we encounter a valid value.
-  // `forward` defines the direction of movement.
-  // If `skip` is true, we're looking for key not equal to `key`.
-  void DoTheThing(bool skip, bool forward) {
-    while (FindNextKey(skip, forward) && !FindValueForCurrentKey()) {
-      skip = true;
-    }
-  }
-
-  void Seek(const Slice& target) {
-    key = target.ToString();
-    DoTheThing(false, true);
-  }
-  void SeekForPrev(const Slice& target) {
-    key = target.ToString();
-    DoTheThing(false, false);
-  }
-  void SeekToFirst() { Seek(""); }
-  void SeekToLast() {
-    key = data->entries.back().key;
-    DoTheThing(false, false);
-  }
-  void Next() {
-    assert(Valid());
-    DoTheThing(true, true);
-  }
-  void Prev() {
-    assert(Valid());
-    DoTheThing(true, false);
-  }
-};
-
-}  // namespace
-
-// Use an internal iterator that sometimes returns errors and sometimes
-// adds/removes entries on the fly. Do random operations on a DBIter and
-// check results.
-// TODO: can be improved for more coverage:
-//   * Override IsKeyPinned() and IsValuePinned() to actually use
-//     PinnedIteratorManager and check that there's no use-after free.
-//   * Try different combinations of prefix_extractor, total_order_seek,
-//     prefix_same_as_start, iterate_lower_bound, iterate_upper_bound.
-TEST_F(DBIteratorStressTest, StressTest) {
-  // We use a deterministic RNG, and everything happens in a single thread.
-  Random64 rnd(826909345792864532ll);
-
-  auto gen_key = [&](int max_key) {
-    assert(max_key > 0);
-    int len = 0;
-    int a = max_key;
-    while (a) {
-      a /= 10;
-      ++len;
-    }
-    std::string s = ToString(rnd.Next() % static_cast<uint64_t>(max_key));
-    s.insert(0, len - (int)s.size(), '0');
-    return s;
-  };
-
-  Options options;
-  options.merge_operator = MergeOperators::CreateFromStringId("stringappend");
-  ReadOptions ropt;
-
-  size_t num_matching = 0;
-  size_t num_at_end = 0;
-  size_t num_not_ok = 0;
-  size_t num_recently_removed = 0;
-
-  // Number of iterations for each combination of parameters
-  // (there are ~250 of those).
-  // Tweak this to change the test run time.
-  // As of the time of writing, the test takes ~4 seconds for value of 5000.
-  const int num_iterations = 5000;
-  // Enable this to print all the operations for debugging.
-  bool trace = FLAGS_verbose;
-
-  for (int num_entries : {5, 10, 100}) {
-    for (double key_space : {0.1, 1.0, 3.0}) {
-      for (ValueType prevalent_entry_type :
-           {kTypeValue, kTypeDeletion, kTypeMerge}) {
-        for (double error_probability : {0.01, 0.1}) {
-          for (double mutation_probability : {0.01, 0.5}) {
-            for (double target_hidden_fraction : {0.1, 0.5}) {
-              std::string trace_str =
-                  "entries: " + ToString(num_entries) +
-                  ", key_space: " + ToString(key_space) +
-                  ", error_probability: " + ToString(error_probability) +
-                  ", mutation_probability: " + ToString(mutation_probability) +
-                  ", target_hidden_fraction: " +
-                  ToString(target_hidden_fraction);
-              SCOPED_TRACE(trace_str);
-              if (trace) {
-                std::cout << trace_str << std::endl;
-              }
-
-              // Generate data.
-              Data data;
-              int max_key = (int)(num_entries * key_space) + 1;
-              for (int i = 0; i < num_entries; ++i) {
-                Entry e;
-                e.key = gen_key(max_key);
-                if (rnd.Next() % 10 != 0) {
-                  e.type = prevalent_entry_type;
-                } else {
-                  const ValueType types[] = {kTypeValue, kTypeDeletion,
-                                             kTypeMerge};
-                  e.type =
-                      types[rnd.Next() % (sizeof(types) / sizeof(types[0]))];
-                }
-                e.sequence = i;
-                e.value = "v" + ToString(i);
-                ParsedInternalKey internal_key(e.key, e.sequence, e.type);
-                AppendInternalKey(&e.ikey, internal_key);
-
-                data.entries.push_back(e);
-              }
-              std::sort(data.entries.begin(), data.entries.end());
-              if (trace) {
-                std::cout << "entries:";
-                for (size_t i = 0; i < data.entries.size(); ++i) {
-                  Entry& e = data.entries[i];
-                  std::cout
-                      << "\n  idx " << i << ": \"" << e.key << "\": \""
-                      << e.value << "\" seq: " << e.sequence << " type: "
-                      << (e.type == kTypeValue
-                              ? "val"
-                              : e.type == kTypeDeletion ? "del" : "merge");
-                }
-                std::cout << std::endl;
-              }
-
-              std::unique_ptr<Iterator> db_iter;
-              std::unique_ptr<ReferenceIterator> ref_iter;
-              for (int iteration = 0; iteration < num_iterations; ++iteration) {
-                SCOPED_TRACE(iteration);
-                // Create a new iterator every ~30 operations.
-                if (db_iter == nullptr || rnd.Next() % 30 == 0) {
-                  uint64_t sequence = rnd.Next() % (data.entries.size() + 2);
-                  ref_iter.reset(new ReferenceIterator(&data, sequence));
-                  if (trace) {
-                    std::cout << "new iterator, seq: " << sequence << std::endl;
-                  }
-
-                  auto internal_iter =
-                      new StressTestIterator(&data, &rnd, BytewiseComparator());
-                  internal_iter->error_probability = error_probability;
-                  internal_iter->mutation_probability = mutation_probability;
-                  internal_iter->target_hidden_fraction =
-                      target_hidden_fraction;
-                  internal_iter->trace = trace;
-                  db_iter.reset(NewDBIterator(
-                      env_, ropt, ImmutableCFOptions(options),
-                      MutableCFOptions(options), BytewiseComparator(),
-                      internal_iter, sequence,
-                      options.max_sequential_skip_in_iterations,
-                      nullptr /*read_callback*/));
-                }
-
-                // Do a random operation. It's important to do it on ref_it
-                // later than on db_iter to make sure ref_it sees the correct
-                // recently_touched_keys.
-                std::string old_key;
-                bool forward = rnd.Next() % 2 > 0;
-                // Do Next()/Prev() ~90% of the time.
-                bool seek = !ref_iter->Valid() || rnd.Next() % 10 == 0;
-                if (trace) {
-                  std::cout << iteration << ": ";
-                }
-
-                if (!seek) {
-                  assert(db_iter->Valid());
-                  old_key = ref_iter->key;
-                  if (trace) {
-                    std::cout << (forward ? "Next" : "Prev") << std::endl;
-                  }
-
-                  if (forward) {
-                    db_iter->Next();
-                    ref_iter->Next();
-                  } else {
-                    db_iter->Prev();
-                    ref_iter->Prev();
-                  }
-                } else {
-                  data.recently_touched_keys.clear();
-                  // Do SeekToFirst less often than Seek.
-                  if (rnd.Next() % 4 == 0) {
-                    if (trace) {
-                      std::cout << (forward ? "SeekToFirst" : "SeekToLast")
-                                << std::endl;
-                    }
-
-                    if (forward) {
-                      old_key = "";
-                      db_iter->SeekToFirst();
-                      ref_iter->SeekToFirst();
-                    } else {
-                      old_key = data.entries.back().key;
-                      db_iter->SeekToLast();
-                      ref_iter->SeekToLast();
-                    }
-                  } else {
-                    old_key = gen_key(max_key);
-                    if (trace) {
-                      std::cout << (forward ? "Seek" : "SeekForPrev") << " \""
-                                << old_key << '"' << std::endl;
-                    }
-                    if (forward) {
-                      db_iter->Seek(old_key);
-                      ref_iter->Seek(old_key);
-                    } else {
-                      db_iter->SeekForPrev(old_key);
-                      ref_iter->SeekForPrev(old_key);
-                    }
-                  }
-                }
-
-                // Check the result.
-                if (db_iter->Valid()) {
-                  ASSERT_TRUE(db_iter->status().ok());
-                  if (data.recently_touched_keys.count(
-                          db_iter->key().ToString())) {
-                    // Ended on a key that may have been mutated during the
-                    // operation. Reference iterator skips such keys, so we
-                    // can't check the exact result.
-
-                    // Check that the key moved in the right direction.
-                    if (forward) {
-                      if (seek)
-                        ASSERT_GE(db_iter->key().ToString(), old_key);
-                      else
-                        ASSERT_GT(db_iter->key().ToString(), old_key);
-                    } else {
-                      if (seek)
-                        ASSERT_LE(db_iter->key().ToString(), old_key);
-                      else
-                        ASSERT_LT(db_iter->key().ToString(), old_key);
-                    }
-
-                    if (ref_iter->Valid()) {
-                      // Check that DBIter didn't miss any non-mutated key.
-                      if (forward) {
-                        ASSERT_LT(db_iter->key().ToString(), ref_iter->key);
-                      } else {
-                        ASSERT_GT(db_iter->key().ToString(), ref_iter->key);
-                      }
-                    }
-                    // Tell the next iteration of the loop to reseek the
-                    // iterators.
-                    ref_iter->valid = false;
-
-                    ++num_recently_removed;
-                  } else {
-                    ASSERT_TRUE(ref_iter->Valid());
-                    ASSERT_EQ(ref_iter->key, db_iter->key().ToString());
-                    ASSERT_EQ(ref_iter->value, db_iter->value());
-                    ++num_matching;
-                  }
-                } else if (db_iter->status().ok()) {
-                  ASSERT_FALSE(ref_iter->Valid());
-                  ++num_at_end;
-                } else {
-                  // Non-ok status. Nothing to check here.
-                  // Tell the next iteration of the loop to reseek the
-                  // iterators.
-                  ref_iter->valid = false;
-                  ++num_not_ok;
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  }
-
-  // Check that all cases were hit many times.
-  EXPECT_GT(num_matching, 10000);
-  EXPECT_GT(num_at_end, 10000);
-  EXPECT_GT(num_not_ok, 10000);
-  EXPECT_GT(num_recently_removed, 10000);
-
-  std::cout << "stats:\n  exact matches: " << num_matching
-            << "\n  end reached: " << num_at_end
-            << "\n  non-ok status: " << num_not_ok
-            << "\n  mutated on the fly: " << num_recently_removed << std::endl;
-}
-
-}  // namespace rocksdb
-
-int main(int argc, char** argv) {
-  ::testing::InitGoogleTest(&argc, argv);
-  ParseCommandLineFlags(&argc, &argv, true);
-  return RUN_ALL_TESTS();
-}

db/db_log_iter_test.cc

@@ -289,8 +289,6 @@ int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 #else
-  (void) argc;
-  (void) argv;
   return 0;
 #endif
 }

db/db_memtable_test.cc

@@ -28,11 +28,13 @@ class MockMemTableRep : public MemTableRep {
     return rep_->Allocate(len, buf);
   }
 
-  virtual void Insert(KeyHandle handle) override { rep_->Insert(handle); }
+  virtual void Insert(KeyHandle handle) override {
+    return rep_->Insert(handle);
+  }
 
   virtual void InsertWithHint(KeyHandle handle, void** hint) override {
     num_insert_with_hint_++;
-    EXPECT_NE(nullptr, hint);
+    ASSERT_NE(nullptr, hint);
     last_hint_in_ = *hint;
     rep_->InsertWithHint(handle, hint);
     last_hint_out_ = *hint;
@@ -119,7 +121,7 @@ class TestPrefixExtractor : public SliceTransform {
     return separator(key) != nullptr;
   }
 
-  virtual bool InRange(const Slice& /*key*/) const override { return false; }
+  virtual bool InRange(const Slice& key) const override { return false; }
 
  private:
   const char* separator(const Slice& key) const {
@@ -127,84 +129,6 @@ class TestPrefixExtractor : public SliceTransform {
   }
 };
 
-// Test that ::Add properly returns false when inserting duplicate keys
-TEST_F(DBMemTableTest, DuplicateSeq) {
-  SequenceNumber seq = 123;
-  std::string value;
-  Status s;
-  MergeContext merge_context;
-  Options options;
-  InternalKeyComparator ikey_cmp(options.comparator);
-  RangeDelAggregator range_del_agg(ikey_cmp, {} /* snapshots */);
-
-  // Create a MemTable
-  InternalKeyComparator cmp(BytewiseComparator());
-  auto factory = std::make_shared<SkipListFactory>();
-  options.memtable_factory = factory;
-  ImmutableCFOptions ioptions(options);
-  WriteBufferManager wb(options.db_write_buffer_size);
-  MemTable* mem = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb,
-                               kMaxSequenceNumber, 0 /* column_family_id */);
-
-  // Write some keys and make sure it returns false on duplicates
-  bool res;
-  res = mem->Add(seq, kTypeValue, "key", "value2");
-  ASSERT_TRUE(res);
-  res = mem->Add(seq, kTypeValue, "key", "value2");
-  ASSERT_FALSE(res);
-  // Changing the type should still cause the duplicatae key
-  res = mem->Add(seq, kTypeMerge, "key", "value2");
-  ASSERT_FALSE(res);
-  // Changing the seq number will make the key fresh
-  res = mem->Add(seq + 1, kTypeMerge, "key", "value2");
-  ASSERT_TRUE(res);
-  // Test with different types for duplicate keys
-  res = mem->Add(seq, kTypeDeletion, "key", "");
-  ASSERT_FALSE(res);
-  res = mem->Add(seq, kTypeSingleDeletion, "key", "");
-  ASSERT_FALSE(res);
-
-  // Test the duplicate keys under stress
-  for (int i = 0; i < 10000; i++) {
-    bool insert_dup = i % 10 == 1;
-    if (!insert_dup) {
-      seq++;
-    }
-    res = mem->Add(seq, kTypeValue, "foo", "value" + ToString(seq));
-    if (insert_dup) {
-      ASSERT_FALSE(res);
-    } else {
-      ASSERT_TRUE(res);
-    }
-  }
-  delete mem;
-
-  // Test with InsertWithHint
-  options.memtable_insert_with_hint_prefix_extractor.reset(
-      new TestPrefixExtractor());  // which uses _ to extract the prefix
-  ioptions = ImmutableCFOptions(options);
-  mem = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb,
-                     kMaxSequenceNumber, 0 /* column_family_id */);
-  // Insert a duplicate key with _ in it
-  res = mem->Add(seq, kTypeValue, "key_1", "value");
-  ASSERT_TRUE(res);
-  res = mem->Add(seq, kTypeValue, "key_1", "value");
-  ASSERT_FALSE(res);
-  delete mem;
-
-  // Test when InsertConcurrently will be invoked
-  options.allow_concurrent_memtable_write = true;
-  ioptions = ImmutableCFOptions(options);
-  mem = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb,
-                     kMaxSequenceNumber, 0 /* column_family_id */);
-  MemTablePostProcessInfo post_process_info;
-  res = mem->Add(seq, kTypeValue, "key", "value", true, &post_process_info);
-  ASSERT_TRUE(res);
-  res = mem->Add(seq, kTypeValue, "key", "value", true, &post_process_info);
-  ASSERT_FALSE(res);
-  delete mem;
-}
-
 TEST_F(DBMemTableTest, InsertWithHint) {
   Options options;
   options.allow_concurrent_memtable_write = false;

db/db_merge_operator_test.cc

@@ -8,125 +8,16 @@
 #include "db/db_test_util.h"
 #include "db/forward_iterator.h"
 #include "port/stack_trace.h"
-#include "rocksdb/merge_operator.h"
 #include "utilities/merge_operators.h"
-#include "utilities/merge_operators/string_append/stringappend2.h"
 
 namespace rocksdb {
 
-class TestReadCallback : public ReadCallback {
- public:
-  TestReadCallback(SnapshotChecker* snapshot_checker,
-                   SequenceNumber snapshot_seq)
-      : snapshot_checker_(snapshot_checker), snapshot_seq_(snapshot_seq) {}
-
-  bool IsCommitted(SequenceNumber seq) override {
-    return snapshot_checker_->IsInSnapshot(seq, snapshot_seq_);
-  }
-
- private:
-  SnapshotChecker* snapshot_checker_;
-  SequenceNumber snapshot_seq_;
-};
-
 // Test merge operator functionality.
 class DBMergeOperatorTest : public DBTestBase {
  public:
   DBMergeOperatorTest() : DBTestBase("/db_merge_operator_test") {}
-
-  std::string GetWithReadCallback(SnapshotChecker* snapshot_checker,
-                                  const Slice& key,
-                                  const Snapshot* snapshot = nullptr) {
-    SequenceNumber seq = snapshot == nullptr ? db_->GetLatestSequenceNumber()
-                                             : snapshot->GetSequenceNumber();
-    TestReadCallback read_callback(snapshot_checker, seq);
-    ReadOptions read_opt;
-    read_opt.snapshot = snapshot;
-    PinnableSlice value;
-    Status s =
-        dbfull()->GetImpl(read_opt, db_->DefaultColumnFamily(), key, &value,
-                          nullptr /*value_found*/, &read_callback);
-    if (!s.ok()) {
-      return s.ToString();
-    }
-    return value.ToString();
-  }
 };
 
-TEST_F(DBMergeOperatorTest, LimitMergeOperands) {
-  class LimitedStringAppendMergeOp : public StringAppendTESTOperator {
-   public:
-    LimitedStringAppendMergeOp(int limit, char delim)
-        : StringAppendTESTOperator(delim), limit_(limit) {}
-
-    const char* Name() const override {
-      return "DBMergeOperatorTest::LimitedStringAppendMergeOp";
-    }
-
-    bool ShouldMerge(const std::vector<Slice>& operands) const override {
-      if (operands.size() > 0 && limit_ > 0 && operands.size() >= limit_) {
-        return true;
-      }
-      return false;
-    }
-
-   private:
-    size_t limit_ = 0;
-  };
-
-  Options options;
-  options.create_if_missing = true;
-  // Use only the latest two merge operands.
-  options.merge_operator =
-      std::make_shared<LimitedStringAppendMergeOp>(2, ',');
-  options.env = env_;
-  Reopen(options);
-  // All K1 values are in memtable.
-  ASSERT_OK(Merge("k1", "a"));
-  ASSERT_OK(Merge("k1", "b"));
-  ASSERT_OK(Merge("k1", "c"));
-  ASSERT_OK(Merge("k1", "d"));
-  std::string value;
-  ASSERT_TRUE(db_->Get(ReadOptions(), "k1", &value).ok());
-  // Make sure that only the latest two merge operands are used. If this was
-  // not the case the value would be "a,b,c,d".
-  ASSERT_EQ(value, "c,d");
-
-  // All K2 values are flushed to L0 into a single file.
-  ASSERT_OK(Merge("k2", "a"));
-  ASSERT_OK(Merge("k2", "b"));
-  ASSERT_OK(Merge("k2", "c"));
-  ASSERT_OK(Merge("k2", "d"));
-  ASSERT_OK(Flush());
-  ASSERT_TRUE(db_->Get(ReadOptions(), "k2", &value).ok());
-  ASSERT_EQ(value, "c,d");
-
-  // All K3 values are flushed and are in different files.
-  ASSERT_OK(Merge("k3", "ab"));
-  ASSERT_OK(Flush());
-  ASSERT_OK(Merge("k3", "bc"));
-  ASSERT_OK(Flush());
-  ASSERT_OK(Merge("k3", "cd"));
-  ASSERT_OK(Flush());
-  ASSERT_OK(Merge("k3", "de"));
-  ASSERT_TRUE(db_->Get(ReadOptions(), "k3", &value).ok());
-  ASSERT_EQ(value, "cd,de");
-
-  // All K4 values are in different levels
-  ASSERT_OK(Merge("k4", "ab"));
-  ASSERT_OK(Flush());
-  MoveFilesToLevel(4);
-  ASSERT_OK(Merge("k4", "bc"));
-  ASSERT_OK(Flush());
-  MoveFilesToLevel(3);
-  ASSERT_OK(Merge("k4", "cd"));
-  ASSERT_OK(Flush());
-  MoveFilesToLevel(1);
-  ASSERT_OK(Merge("k4", "de"));
-  ASSERT_TRUE(db_->Get(ReadOptions(), "k4", &value).ok());
-  ASSERT_EQ(value, "cd,de");
-}
-
 TEST_F(DBMergeOperatorTest, MergeErrorOnRead) {
   Options options;
   options.create_if_missing = true;
@@ -166,33 +57,16 @@ TEST_F(DBMergeOperatorTest, MergeErrorOnIteration) {
   ASSERT_OK(Merge("k1", "v1"));
   ASSERT_OK(Merge("k1", "corrupted"));
   ASSERT_OK(Put("k2", "v2"));
-  auto* iter = db_->NewIterator(ReadOptions());
-  iter->Seek("k1");
-  ASSERT_FALSE(iter->Valid());
-  ASSERT_TRUE(iter->status().IsCorruption());
-  delete iter;
-  iter = db_->NewIterator(ReadOptions());
-  iter->Seek("k2");
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_OK(iter->status());
-  iter->Prev();
-  ASSERT_FALSE(iter->Valid());
-  ASSERT_TRUE(iter->status().IsCorruption());
-  delete iter;
+  VerifyDBFromMap({{"k1", ""}, {"k2", "v2"}}, nullptr, false,
+                  {{"k1", Status::Corruption()}});
   VerifyDBInternal({{"k1", "corrupted"}, {"k1", "v1"}, {"k2", "v2"}});
 
   DestroyAndReopen(options);
   ASSERT_OK(Merge("k1", "v1"));
   ASSERT_OK(Put("k2", "v2"));
   ASSERT_OK(Merge("k2", "corrupted"));
-  iter = db_->NewIterator(ReadOptions());
-  iter->Seek("k1");
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_OK(iter->status());
-  iter->Next();
-  ASSERT_FALSE(iter->Valid());
-  ASSERT_TRUE(iter->status().IsCorruption());
-  delete iter;
+  VerifyDBFromMap({{"k1", "v1"}, {"k2", ""}}, nullptr, false,
+                  {{"k2", Status::Corruption()}});
   VerifyDBInternal({{"k1", "v1"}, {"k2", "corrupted"}, {"k2", "v2"}});
 }
 
@@ -284,7 +158,7 @@ TEST_P(MergeOperatorPinningTest, Randomized) {
     Random rnd(301);
     std::map<std::string, std::string> true_data;
 
-    const int kTotalMerges = 5000;
+    const int kTotalMerges = 10000;
     // Every key gets ~10 operands
     const int kKeyRange = kTotalMerges / 10;
     const int kOperandSize = 20;
@@ -482,152 +356,8 @@ TEST_P(MergeOperatorPinningTest, TailingIterator) {
   writer_thread.join();
   reader_thread.join();
 }
-
-TEST_F(DBMergeOperatorTest, TailingIteratorMemtableUnrefedBySomeoneElse) {
-  Options options = CurrentOptions();
-  options.merge_operator = MergeOperators::CreateStringAppendOperator();
-  DestroyAndReopen(options);
-
-  // Overview of the test:
-  //  * There are two merge operands for the same key: one in an sst file,
-  //    another in a memtable.
-  //  * Seek a tailing iterator to this key.
-  //  * As part of the seek, the iterator will:
-  //      (a) first visit the operand in the memtable and tell ForwardIterator
-  //          to pin this operand, then
-  //      (b) move on to the operand in the sst file, then pass both operands
-  //          to merge operator.
-  //  * The memtable may get flushed and unreferenced by another thread between
-  //    (a) and (b). The test simulates it by flushing the memtable inside a
-  //    SyncPoint callback located between (a) and (b).
-  //  * In this case it's ForwardIterator's responsibility to keep the memtable
-  //    pinned until (b) is complete. There used to be a bug causing
-  //    ForwardIterator to not pin it in some circumstances. This test
-  //    reproduces it.
-
-  db_->Merge(WriteOptions(), "key", "sst");
-  db_->Flush(FlushOptions()); // Switch to SuperVersion A
-  db_->Merge(WriteOptions(), "key", "memtable");
-
-  // Pin SuperVersion A
-  std::unique_ptr<Iterator> someone_else(db_->NewIterator(ReadOptions()));
-
-  bool pushed_first_operand = false;
-  bool stepped_to_next_operand = false;
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "DBIter::MergeValuesNewToOld:PushedFirstOperand", [&](void*) {
-        EXPECT_FALSE(pushed_first_operand);
-        pushed_first_operand = true;
-        db_->Flush(FlushOptions()); // Switch to SuperVersion B
-      });
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "DBIter::MergeValuesNewToOld:SteppedToNextOperand", [&](void*) {
-        EXPECT_FALSE(stepped_to_next_operand);
-        stepped_to_next_operand = true;
-        someone_else.reset(); // Unpin SuperVersion A
-      });
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-
-  ReadOptions ro;
-  ro.tailing = true;
-  std::unique_ptr<Iterator> iter(db_->NewIterator(ro));
-  iter->Seek("key");
-
-  ASSERT_TRUE(iter->status().ok());
-  ASSERT_TRUE(iter->Valid());
-  EXPECT_EQ(std::string("sst,memtable"), iter->value().ToString());
-  EXPECT_TRUE(pushed_first_operand);
-  EXPECT_TRUE(stepped_to_next_operand);
-}
 #endif  // ROCKSDB_LITE
 
-TEST_F(DBMergeOperatorTest, SnapshotCheckerAndReadCallback) {
-  Options options = CurrentOptions();
-  options.merge_operator = MergeOperators::CreateStringAppendOperator();
-  DestroyAndReopen(options);
-
-  class TestSnapshotChecker : public SnapshotChecker {
-    bool IsInSnapshot(SequenceNumber seq,
-                      SequenceNumber snapshot_seq) const override {
-      switch (snapshot_seq) {
-        case 0:
-          return seq == 0;
-        case 1:
-          return seq <= 1;
-        case 2:
-          // seq = 2 not visible to snapshot with seq = 2
-          return seq <= 1;
-        case 3:
-          return seq <= 3;
-        case 4:
-          // seq = 4 not visible to snpahost with seq = 4
-          return seq <= 3;
-        default:
-          // seq >=4 is uncommitted
-          return seq <= 4;
-      };
-    }
-  };
-  TestSnapshotChecker* snapshot_checker = new TestSnapshotChecker();
-  dbfull()->SetSnapshotChecker(snapshot_checker);
-
-  std::string value;
-  ASSERT_OK(Merge("foo", "v1"));
-  ASSERT_EQ(1, db_->GetLatestSequenceNumber());
-  ASSERT_EQ("v1", GetWithReadCallback(snapshot_checker, "foo"));
-  ASSERT_OK(Merge("foo", "v2"));
-  ASSERT_EQ(2, db_->GetLatestSequenceNumber());
-  // v2 is not visible to latest snapshot, which has seq = 2.
-  ASSERT_EQ("v1", GetWithReadCallback(snapshot_checker, "foo"));
-  // Take a snapshot with seq = 2.
-  const Snapshot* snapshot1 = db_->GetSnapshot();
-  ASSERT_EQ(2, snapshot1->GetSequenceNumber());
-  // v2 is not visible to snapshot1, which has seq = 2
-  ASSERT_EQ("v1", GetWithReadCallback(snapshot_checker, "foo", snapshot1));
-
-  // Verify flush doesn't alter the result.
-  ASSERT_OK(Flush());
-  ASSERT_EQ("v1", GetWithReadCallback(snapshot_checker, "foo", snapshot1));
-  ASSERT_EQ("v1", GetWithReadCallback(snapshot_checker, "foo"));
-
-  ASSERT_OK(Merge("foo", "v3"));
-  ASSERT_EQ(3, db_->GetLatestSequenceNumber());
-  ASSERT_EQ("v1,v2,v3", GetWithReadCallback(snapshot_checker, "foo"));
-  ASSERT_OK(Merge("foo", "v4"));
-  ASSERT_EQ(4, db_->GetLatestSequenceNumber());
-  // v4 is not visible to latest snapshot, which has seq = 4.
-  ASSERT_EQ("v1,v2,v3", GetWithReadCallback(snapshot_checker, "foo"));
-  const Snapshot* snapshot2 = db_->GetSnapshot();
-  ASSERT_EQ(4, snapshot2->GetSequenceNumber());
-  // v4 is not visible to snapshot2, which has seq = 4.
-  ASSERT_EQ("v1,v2,v3",
-            GetWithReadCallback(snapshot_checker, "foo", snapshot2));
-
-  // Verify flush doesn't alter the result.
-  ASSERT_OK(Flush());
-  ASSERT_EQ("v1", GetWithReadCallback(snapshot_checker, "foo", snapshot1));
-  ASSERT_EQ("v1,v2,v3",
-            GetWithReadCallback(snapshot_checker, "foo", snapshot2));
-  ASSERT_EQ("v1,v2,v3", GetWithReadCallback(snapshot_checker, "foo"));
-
-  ASSERT_OK(Merge("foo", "v5"));
-  ASSERT_EQ(5, db_->GetLatestSequenceNumber());
-  // v5 is uncommitted
-  ASSERT_EQ("v1,v2,v3,v4", GetWithReadCallback(snapshot_checker, "foo"));
-
-  // full manual compaction.
-  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
-
-  // Verify compaction doesn't alter the result.
-  ASSERT_EQ("v1", GetWithReadCallback(snapshot_checker, "foo", snapshot1));
-  ASSERT_EQ("v1,v2,v3",
-            GetWithReadCallback(snapshot_checker, "foo", snapshot2));
-  ASSERT_EQ("v1,v2,v3,v4", GetWithReadCallback(snapshot_checker, "foo"));
-
-  db_->ReleaseSnapshot(snapshot1);
-  db_->ReleaseSnapshot(snapshot2);
-}
-
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/db_options_test.cc

@@ -117,150 +117,6 @@ TEST_F(DBOptionsTest, GetLatestCFOptions) {
             GetMutableCFOptionsMap(dbfull()->GetOptions(handles_[1])));
 }
 
-TEST_F(DBOptionsTest, SetBytesPerSync) {
-  const size_t kValueSize = 1024 * 1024;  // 1MB
-  Options options;
-  options.create_if_missing = true;
-  options.bytes_per_sync = 1024 * 1024;
-  options.use_direct_reads = false;
-  options.write_buffer_size = 400 * kValueSize;
-  options.disable_auto_compactions = true;
-  options.compression = kNoCompression;
-  options.env = env_;
-  Reopen(options);
-  int counter = 0;
-  int low_bytes_per_sync = 0;
-  int i = 0;
-  const std::string kValue(kValueSize, 'v');
-  ASSERT_EQ(options.bytes_per_sync, dbfull()->GetDBOptions().bytes_per_sync);
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "WritableFileWriter::RangeSync:0", [&](void* /*arg*/) {
-        counter++;
-      });
-
-  WriteOptions write_opts;
-  // should sync approximately 40MB/1MB ~= 40 times.
-  for (i = 0; i < 40; i++) {
-    Put(Key(i), kValue, write_opts);
-  }
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-  ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-  low_bytes_per_sync = counter;
-  ASSERT_GT(low_bytes_per_sync, 35);
-  ASSERT_LT(low_bytes_per_sync, 45);
-
-  counter = 0;
-  // 8388608 = 8 * 1024 * 1024
-  ASSERT_OK(dbfull()->SetDBOptions({{"bytes_per_sync", "8388608"}}));
-  ASSERT_EQ(8388608, dbfull()->GetDBOptions().bytes_per_sync);
-  // should sync approximately 40MB*2/8MB ~= 10 times.
-  // data will be 40*2MB because of previous Puts too.
-  for (i = 0; i < 40; i++) {
-    Put(Key(i), kValue, write_opts);
-  }
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-  ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
-  ASSERT_GT(counter, 5);
-  ASSERT_LT(counter, 15);
-
-  // Redundant assert. But leaving it here just to get the point across that
-  // low_bytes_per_sync > counter.
-  ASSERT_GT(low_bytes_per_sync, counter);
-}
-
-TEST_F(DBOptionsTest, SetWalBytesPerSync) {
-  const size_t kValueSize = 1024 * 1024 * 3;
-  Options options;
-  options.create_if_missing = true;
-  options.wal_bytes_per_sync = 512;
-  options.write_buffer_size = 100 * kValueSize;
-  options.disable_auto_compactions = true;
-  options.compression = kNoCompression;
-  options.env = env_;
-  Reopen(options);
-  ASSERT_EQ(512, dbfull()->GetDBOptions().wal_bytes_per_sync);
-  int counter = 0;
-  int low_bytes_per_sync = 0;
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "WritableFileWriter::RangeSync:0", [&](void* /*arg*/) {
-        counter++;
-      });
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-  const std::string kValue(kValueSize, 'v');
-  int i = 0;
-  for (; i < 10; i++) {
-    Put(Key(i), kValue);
-  }
-  // Do not flush. If we flush here, SwitchWAL will reuse old WAL file since its
-  // empty and will not get the new wal_bytes_per_sync value.
-  low_bytes_per_sync = counter;
-  //5242880 = 1024 * 1024 * 5
-  ASSERT_OK(dbfull()->SetDBOptions({{"wal_bytes_per_sync", "5242880"}}));
-  ASSERT_EQ(5242880, dbfull()->GetDBOptions().wal_bytes_per_sync);
-  counter = 0;
-  i = 0;
-  for (; i < 10; i++) {
-    Put(Key(i), kValue);
-  }
-  ASSERT_GT(counter, 0);
-  ASSERT_GT(low_bytes_per_sync, 0);
-  ASSERT_GT(low_bytes_per_sync, counter);
-}
-
-TEST_F(DBOptionsTest, WritableFileMaxBufferSize) {
-  Options options;
-  options.create_if_missing = true;
-  options.writable_file_max_buffer_size = 1024 * 1024;
-  options.level0_file_num_compaction_trigger = 3;
-  options.max_manifest_file_size = 1;
-  options.env = env_;
-  int buffer_size = 1024 * 1024;
-  Reopen(options);
-  ASSERT_EQ(buffer_size,
-            dbfull()->GetDBOptions().writable_file_max_buffer_size);
-
-  std::atomic<int> match_cnt(0);
-  std::atomic<int> unmatch_cnt(0);
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "WritableFileWriter::WritableFileWriter:0", [&](void* arg) {
-        int value = static_cast<int>(reinterpret_cast<uintptr_t>(arg));
-        if (value == buffer_size) {
-          match_cnt++;
-        } else {
-          unmatch_cnt++;
-        }
-      });
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-  int i = 0;
-  for (; i < 3; i++) {
-    ASSERT_OK(Put("foo", ToString(i)));
-    ASSERT_OK(Put("bar", ToString(i)));
-    Flush();
-  }
-  dbfull()->TEST_WaitForCompact();
-  ASSERT_EQ(unmatch_cnt, 0);
-  ASSERT_GE(match_cnt, 11);
-
-  ASSERT_OK(
-      dbfull()->SetDBOptions({{"writable_file_max_buffer_size", "524288"}}));
-  buffer_size = 512 * 1024;
-  match_cnt = 0;
-  unmatch_cnt = 0;  // SetDBOptions() will create a WriteableFileWriter
-
-  ASSERT_EQ(buffer_size,
-            dbfull()->GetDBOptions().writable_file_max_buffer_size);
-  i = 0;
-  for (; i < 3; i++) {
-    ASSERT_OK(Put("foo", ToString(i)));
-    ASSERT_OK(Put("bar", ToString(i)));
-    Flush();
-  }
-  dbfull()->TEST_WaitForCompact();
-  ASSERT_EQ(unmatch_cnt, 0);
-  ASSERT_GE(match_cnt, 11);
-}
-
 TEST_F(DBOptionsTest, SetOptionsAndReopen) {
   Random rnd(1044);
   auto rand_opts = GetRandomizedMutableCFOptionsMap(&rnd);
@@ -519,13 +375,8 @@ static void assert_candidate_files_empty(DBImpl* dbfull, const bool empty) {
   JobContext job_context(0);
   dbfull->FindObsoleteFiles(&job_context, false);
   ASSERT_EQ(empty, job_context.full_scan_candidate_files.empty());
-  dbfull->TEST_UnlockMutex();
-  if (job_context.HaveSomethingToDelete()) {
-    // fulfill the contract of FindObsoleteFiles by calling PurgeObsoleteFiles
-    // afterwards; otherwise the test may hang on shutdown
-    dbfull->PurgeObsoleteFiles(job_context);
-  }
   job_context.Clean();
+  dbfull->TEST_UnlockMutex();
 }
 
 TEST_F(DBOptionsTest, DeleteObsoleteFilesPeriodChange) {
@@ -590,140 +441,6 @@ TEST_F(DBOptionsTest, SanitizeDelayedWriteRate) {
   ASSERT_EQ(31 * 1024 * 1024, dbfull()->GetDBOptions().delayed_write_rate);
 }
 
-TEST_F(DBOptionsTest, SetFIFOCompactionOptions) {
-  Options options;
-  options.compaction_style = kCompactionStyleFIFO;
-  options.write_buffer_size = 10 << 10;  // 10KB
-  options.arena_block_size = 4096;
-  options.compression = kNoCompression;
-  options.create_if_missing = true;
-  options.compaction_options_fifo.allow_compaction = false;
-  env_->time_elapse_only_sleep_ = false;
-  options.env = env_;
-
-  // Test dynamically changing compaction_options_fifo.ttl
-  env_->addon_time_.store(0);
-  options.compaction_options_fifo.ttl = 1 * 60 * 60;  // 1 hour
-  ASSERT_OK(TryReopen(options));
-
-  Random rnd(301);
-  for (int i = 0; i < 10; i++) {
-    // Generate and flush a file about 10KB.
-    for (int j = 0; j < 10; j++) {
-      ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
-    }
-    Flush();
-  }
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
-  ASSERT_EQ(NumTableFilesAtLevel(0), 10);
-
-  // Add 61 seconds to the time.
-  env_->addon_time_.fetch_add(61);
-
-  // No files should be compacted as ttl is set to 1 hour.
-  ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.ttl, 3600);
-  dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
-  ASSERT_EQ(NumTableFilesAtLevel(0), 10);
-
-  // Set ttl to 1 minute. So all files should get deleted.
-  ASSERT_OK(dbfull()->SetOptions({{"compaction_options_fifo", "{ttl=60;}"}}));
-  ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.ttl, 60);
-  dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
-  ASSERT_EQ(NumTableFilesAtLevel(0), 0);
-
-  // Test dynamically changing compaction_options_fifo.max_table_files_size
-  env_->addon_time_.store(0);
-  options.compaction_options_fifo.max_table_files_size = 500 << 10;  // 00KB
-  options.compaction_options_fifo.ttl = 0;
-  DestroyAndReopen(options);
-
-  for (int i = 0; i < 10; i++) {
-    // Generate and flush a file about 10KB.
-    for (int j = 0; j < 10; j++) {
-      ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
-    }
-    Flush();
-  }
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
-  ASSERT_EQ(NumTableFilesAtLevel(0), 10);
-
-  // No files should be compacted as max_table_files_size is set to 500 KB.
-  ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size,
-            500 << 10);
-  dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
-  ASSERT_EQ(NumTableFilesAtLevel(0), 10);
-
-  // Set max_table_files_size to 12 KB. So only 1 file should remain now.
-  ASSERT_OK(dbfull()->SetOptions(
-      {{"compaction_options_fifo", "{max_table_files_size=12288;}"}}));
-  ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size,
-            12 << 10);
-  dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
-  ASSERT_EQ(NumTableFilesAtLevel(0), 1);
-
-  // Test dynamically changing compaction_options_fifo.allow_compaction
-  options.compaction_options_fifo.max_table_files_size = 500 << 10;  // 500KB
-  options.compaction_options_fifo.ttl = 0;
-  options.compaction_options_fifo.allow_compaction = false;
-  options.level0_file_num_compaction_trigger = 6;
-  DestroyAndReopen(options);
-
-  for (int i = 0; i < 10; i++) {
-    // Generate and flush a file about 10KB.
-    for (int j = 0; j < 10; j++) {
-      ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
-    }
-    Flush();
-  }
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
-  ASSERT_EQ(NumTableFilesAtLevel(0), 10);
-
-  // No files should be compacted as max_table_files_size is set to 500 KB and
-  // allow_compaction is false
-  ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction,
-            false);
-  dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
-  ASSERT_EQ(NumTableFilesAtLevel(0), 10);
-
-  // Set allow_compaction to true. So number of files should be between 1 and 5.
-  ASSERT_OK(dbfull()->SetOptions(
-      {{"compaction_options_fifo", "{allow_compaction=true;}"}}));
-  ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction,
-            true);
-  dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
-  ASSERT_GE(NumTableFilesAtLevel(0), 1);
-  ASSERT_LE(NumTableFilesAtLevel(0), 5);
-}
-
-TEST_F(DBOptionsTest, CompactionReadaheadSizeChange) {
-  SpecialEnv env(env_);
-  Options options;
-  options.env = &env;
-
-  options.compaction_readahead_size = 0;
-  options.new_table_reader_for_compaction_inputs = true;
-  options.level0_file_num_compaction_trigger = 2;
-  const std::string kValue(1024, 'v');
-  Reopen(options);
-
-  ASSERT_EQ(0, dbfull()->GetDBOptions().compaction_readahead_size);
-  ASSERT_OK(dbfull()->SetDBOptions({{"compaction_readahead_size", "256"}}));
-  ASSERT_EQ(256, dbfull()->GetDBOptions().compaction_readahead_size);
-  for (int i = 0; i < 1024; i++) {
-    Put(Key(i), kValue);
-  }
-  Flush();
-  for (int i = 0; i < 1024 * 2; i++) {
-    Put(Key(i), kValue);
-  }
-  Flush();
-  dbfull()->TEST_WaitForCompact();
-  ASSERT_EQ(256, env_->compaction_readahead_size_);
-  Close();
-}
 #endif  // ROCKSDB_LITE
 
 }  // namespace rocksdb

db/db_properties_test.cc

@@ -14,7 +14,6 @@
 
 #include "db/db_test_util.h"
 #include "port/stack_trace.h"
-#include "rocksdb/listener.h"
 #include "rocksdb/options.h"
 #include "rocksdb/perf_context.h"
 #include "rocksdb/perf_level.h"
@@ -69,27 +68,27 @@ TEST_F(DBPropertiesTest, Empty) {
     ASSERT_OK(db_->DisableFileDeletions());
     ASSERT_TRUE(
         dbfull()->GetProperty("rocksdb.is-file-deletions-enabled", &num));
-    ASSERT_EQ("0", num);
+    ASSERT_EQ("1", num);
 
     ASSERT_OK(db_->DisableFileDeletions());
     ASSERT_TRUE(
         dbfull()->GetProperty("rocksdb.is-file-deletions-enabled", &num));
-    ASSERT_EQ("0", num);
+    ASSERT_EQ("2", num);
 
     ASSERT_OK(db_->DisableFileDeletions());
     ASSERT_TRUE(
         dbfull()->GetProperty("rocksdb.is-file-deletions-enabled", &num));
-    ASSERT_EQ("0", num);
+    ASSERT_EQ("3", num);
 
     ASSERT_OK(db_->EnableFileDeletions(false));
     ASSERT_TRUE(
         dbfull()->GetProperty("rocksdb.is-file-deletions-enabled", &num));
-    ASSERT_EQ("0", num);
+    ASSERT_EQ("2", num);
 
     ASSERT_OK(db_->EnableFileDeletions());
     ASSERT_TRUE(
         dbfull()->GetProperty("rocksdb.is-file-deletions-enabled", &num));
-    ASSERT_EQ("1", num);
+    ASSERT_EQ("0", num);
   } while (ChangeOptions());
 }
 
@@ -986,9 +985,8 @@ class CountingUserTblPropCollector : public TablePropertiesCollector {
     return Status::OK();
   }
 
-  Status AddUserKey(const Slice& /*user_key*/, const Slice& /*value*/,
-                    EntryType /*type*/, SequenceNumber /*seq*/,
-                    uint64_t /*file_size*/) override {
+  Status AddUserKey(const Slice& user_key, const Slice& value, EntryType type,
+                    SequenceNumber seq, uint64_t file_size) override {
     ++count_;
     return Status::OK();
   }
@@ -1029,9 +1027,8 @@ class CountingDeleteTabPropCollector : public TablePropertiesCollector {
  public:
   const char* Name() const override { return "CountingDeleteTabPropCollector"; }
 
-  Status AddUserKey(const Slice& /*user_key*/, const Slice& /*value*/,
-                    EntryType type, SequenceNumber /*seq*/,
-                    uint64_t /*file_size*/) override {
+  Status AddUserKey(const Slice& user_key, const Slice& value, EntryType type,
+                    SequenceNumber seq, uint64_t file_size) override {
     if (type == kEntryDelete) {
       num_deletes_++;
     }
@@ -1058,7 +1055,7 @@ class CountingDeleteTabPropCollectorFactory
     : public TablePropertiesCollectorFactory {
  public:
   virtual TablePropertiesCollector* CreateTablePropertiesCollector(
-      TablePropertiesCollectorFactory::Context /*context*/) override {
+      TablePropertiesCollectorFactory::Context context) override {
     return new CountingDeleteTabPropCollector();
   }
   const char* Name() const override {
@@ -1386,157 +1383,6 @@ TEST_F(DBPropertiesTest, EstimateOldestKeyTime) {
   Close();
 }
 
-TEST_F(DBPropertiesTest, SstFilesSize) {
-  struct TestListener : public EventListener {
-    void OnCompactionCompleted(DB* db,
-                               const CompactionJobInfo& /*info*/) override {
-      assert(callback_triggered == false);
-      assert(size_before_compaction > 0);
-      callback_triggered = true;
-      uint64_t total_sst_size = 0;
-      uint64_t live_sst_size = 0;
-      bool ok = db->GetIntProperty(DB::Properties::kTotalSstFilesSize,
-                                   &total_sst_size);
-      ASSERT_TRUE(ok);
-      // total_sst_size include files before and after compaction.
-      ASSERT_GT(total_sst_size, size_before_compaction);
-      ok =
-          db->GetIntProperty(DB::Properties::kLiveSstFilesSize, &live_sst_size);
-      ASSERT_TRUE(ok);
-      // live_sst_size only include files after compaction.
-      ASSERT_GT(live_sst_size, 0);
-      ASSERT_LT(live_sst_size, size_before_compaction);
-    }
-
-    uint64_t size_before_compaction = 0;
-    bool callback_triggered = false;
-  };
-  std::shared_ptr<TestListener> listener = std::make_shared<TestListener>();
-
-  Options options;
-  options.disable_auto_compactions = true;
-  options.listeners.push_back(listener);
-  Reopen(options);
-
-  for (int i = 0; i < 10; i++) {
-    ASSERT_OK(Put("key" + ToString(i), std::string(1000, 'v')));
-  }
-  ASSERT_OK(Flush());
-  for (int i = 0; i < 5; i++) {
-    ASSERT_OK(Delete("key" + ToString(i)));
-  }
-  ASSERT_OK(Flush());
-  uint64_t sst_size;
-  bool ok = db_->GetIntProperty(DB::Properties::kTotalSstFilesSize, &sst_size);
-  ASSERT_TRUE(ok);
-  ASSERT_GT(sst_size, 0);
-  listener->size_before_compaction = sst_size;
-  // Compact to clean all keys and trigger listener.
-  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
-  ASSERT_TRUE(listener->callback_triggered);
-}
-
-TEST_F(DBPropertiesTest, BlockCacheProperties) {
-  Options options;
-  uint64_t value;
-
-  // Block cache properties are not available for tables other than
-  // block-based table.
-  options.table_factory.reset(NewPlainTableFactory());
-  Reopen(options);
-  ASSERT_FALSE(
-      db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value));
-  ASSERT_FALSE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value));
-  ASSERT_FALSE(
-      db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value));
-
-  options.table_factory.reset(NewCuckooTableFactory());
-  Reopen(options);
-  ASSERT_FALSE(
-      db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value));
-  ASSERT_FALSE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value));
-  ASSERT_FALSE(
-      db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value));
-
-  // Block cache properties are not available if block cache is not used.
-  BlockBasedTableOptions table_options;
-  table_options.no_block_cache = true;
-  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-  Reopen(options);
-  ASSERT_FALSE(
-      db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value));
-  ASSERT_FALSE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value));
-  ASSERT_FALSE(
-      db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value));
-
-  // Test with empty block cache.
-  constexpr size_t kCapacity = 100;
-  auto block_cache = NewLRUCache(kCapacity, 0 /*num_shard_bits*/);
-  table_options.block_cache = block_cache;
-  table_options.no_block_cache = false;
-  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-  Reopen(options);
-  ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value));
-  ASSERT_EQ(kCapacity, value);
-  ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value));
-  ASSERT_EQ(0, value);
-  ASSERT_TRUE(
-      db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value));
-  ASSERT_EQ(0, value);
-
-  // Insert unpinned item to the cache and check size.
-  constexpr size_t kSize1 = 50;
-  block_cache->Insert("item1", nullptr /*value*/, kSize1, nullptr /*deleter*/);
-  ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value));
-  ASSERT_EQ(kCapacity, value);
-  ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value));
-  ASSERT_EQ(kSize1, value);
-  ASSERT_TRUE(
-      db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value));
-  ASSERT_EQ(0, value);
-
-  // Insert pinned item to the cache and check size.
-  constexpr size_t kSize2 = 30;
-  Cache::Handle* item2 = nullptr;
-  block_cache->Insert("item2", nullptr /*value*/, kSize2, nullptr /*deleter*/,
-                      &item2);
-  ASSERT_NE(nullptr, item2);
-  ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value));
-  ASSERT_EQ(kCapacity, value);
-  ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value));
-  ASSERT_EQ(kSize1 + kSize2, value);
-  ASSERT_TRUE(
-      db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value));
-  ASSERT_EQ(kSize2, value);
-
-  // Insert another pinned item to make the cache over-sized.
-  constexpr size_t kSize3 = 80;
-  Cache::Handle* item3 = nullptr;
-  block_cache->Insert("item3", nullptr /*value*/, kSize3, nullptr /*deleter*/,
-                      &item3);
-  ASSERT_NE(nullptr, item2);
-  ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value));
-  ASSERT_EQ(kCapacity, value);
-  ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value));
-  // Item 1 is evicted.
-  ASSERT_EQ(kSize2 + kSize3, value);
-  ASSERT_TRUE(
-      db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value));
-  ASSERT_EQ(kSize2 + kSize3, value);
-
-  // Check size after release.
-  block_cache->Release(item2);
-  block_cache->Release(item3);
-  ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value));
-  ASSERT_EQ(kCapacity, value);
-  ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value));
-  // item2 will be evicted, while item3 remain in cache after release.
-  ASSERT_EQ(kSize3, value);
-  ASSERT_TRUE(
-      db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value));
-  ASSERT_EQ(0, value);
-}
-
 #endif  // ROCKSDB_LITE
 }  // namespace rocksdb
 

db/db_range_del_test.cc

@@ -433,8 +433,8 @@ TEST_F(DBRangeDelTest, ValidUniversalSubcompactionBoundaries) {
       reinterpret_cast<ColumnFamilyHandleImpl*>(db_->DefaultColumnFamily())
           ->cfd(),
       1 /* input_level */, 2 /* output_level */, 0 /* output_path_id */,
-      0 /* max_subcompactions */, nullptr /* begin */, nullptr /* end */,
-      true /* exclusive */, true /* disallow_trivial_move */));
+      nullptr /* begin */, nullptr /* end */, true /* exclusive */,
+      true /* disallow_trivial_move */));
 }
 #endif  // ROCKSDB_LITE
 

db/db_sst_test.cc

@@ -20,37 +20,6 @@ class DBSSTTest : public DBTestBase {
   DBSSTTest() : DBTestBase("/db_sst_test") {}
 };
 
-#ifndef ROCKSDB_LITE
-// A class which remembers the name of each flushed file.
-class FlushedFileCollector : public EventListener {
- public:
-  FlushedFileCollector() {}
-  ~FlushedFileCollector() {}
-
-  virtual void OnFlushCompleted(DB* /*db*/, const FlushJobInfo& info) override {
-    std::lock_guard<std::mutex> lock(mutex_);
-    flushed_files_.push_back(info.file_path);
-  }
-
-  std::vector<std::string> GetFlushedFiles() {
-    std::lock_guard<std::mutex> lock(mutex_);
-    std::vector<std::string> result;
-    for (auto fname : flushed_files_) {
-      result.push_back(fname);
-    }
-    return result;
-  }
-  void ClearFlushedFiles() {
-    std::lock_guard<std::mutex> lock(mutex_);
-    flushed_files_.clear();
-  }
-
- private:
-  std::vector<std::string> flushed_files_;
-  std::mutex mutex_;
-};
-#endif  // ROCKSDB_LITE
-
 TEST_F(DBSSTTest, DontDeletePendingOutputs) {
   Options options;
   options.env = env_;
@@ -103,7 +72,7 @@ TEST_F(DBSSTTest, SSTsWithLdbSuffixHandling) {
   ASSERT_GT(num_files, 0);
 
   std::vector<std::string> filenames;
-  GetSstFiles(env_, dbname_, &filenames);
+  GetSstFiles(dbname_, &filenames);
   int num_ldb_files = 0;
   for (size_t i = 0; i < filenames.size(); ++i) {
     if (i & 1) {
@@ -262,11 +231,11 @@ TEST_F(DBSSTTest, DBWithSstFileManager) {
   int files_deleted = 0;
   int files_moved = 0;
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "SstFileManagerImpl::OnAddFile", [&](void* /*arg*/) { files_added++; });
+      "SstFileManagerImpl::OnAddFile", [&](void* arg) { files_added++; });
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "SstFileManagerImpl::OnDeleteFile", [&](void* /*arg*/) { files_deleted++; });
+      "SstFileManagerImpl::OnDeleteFile", [&](void* arg) { files_deleted++; });
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "SstFileManagerImpl::OnMoveFile", [&](void* /*arg*/) { files_moved++; });
+      "SstFileManagerImpl::OnMoveFile", [&](void* arg) { files_moved++; });
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 
   Options options = CurrentOptions();
@@ -358,14 +327,15 @@ TEST_F(DBSSTTest, RateLimitedDelete) {
   options.disable_auto_compactions = true;
   options.env = env_;
 
+  std::string trash_dir = test::TmpDir(env_) + "/trash";
   int64_t rate_bytes_per_sec = 1024 * 10;  // 10 Kbs / Sec
   Status s;
   options.sst_file_manager.reset(
-      NewSstFileManager(env_, nullptr, "", 0, false, &s, 0));
+      NewSstFileManager(env_, nullptr, trash_dir, 0, false, &s));
   ASSERT_OK(s);
   options.sst_file_manager->SetDeleteRateBytesPerSecond(rate_bytes_per_sec);
   auto sfm = static_cast<SstFileManagerImpl*>(options.sst_file_manager.get());
-  sfm->delete_scheduler()->SetMaxTrashDBRatio(1.1);
+  sfm->delete_scheduler()->TEST_SetMaxTrashDBRatio(1.1);
 
   ASSERT_OK(TryReopen(options));
   // Create 4 files in L0
@@ -407,30 +377,6 @@ TEST_F(DBSSTTest, RateLimitedDelete) {
   rocksdb::SyncPoint::GetInstance()->DisableProcessing();
 }
 
-TEST_F(DBSSTTest, OpenDBWithExistingTrash) {
-  Options options = CurrentOptions();
-
-  options.sst_file_manager.reset(
-      NewSstFileManager(env_, nullptr, "", 1024 * 1024 /* 1 MB/sec */));
-  auto sfm = static_cast<SstFileManagerImpl*>(options.sst_file_manager.get());
-
-  Destroy(last_options_);
-
-  // Add some trash files to the db directory so the DB can clean them up
-  env_->CreateDirIfMissing(dbname_);
-  ASSERT_OK(WriteStringToFile(env_, "abc", dbname_ + "/" + "001.sst.trash"));
-  ASSERT_OK(WriteStringToFile(env_, "abc", dbname_ + "/" + "002.sst.trash"));
-  ASSERT_OK(WriteStringToFile(env_, "abc", dbname_ + "/" + "003.sst.trash"));
-
-  // Reopen the DB and verify that it deletes existing trash files
-  ASSERT_OK(TryReopen(options));
-  sfm->WaitForEmptyTrash();
-  ASSERT_NOK(env_->FileExists(dbname_ + "/" + "001.sst.trash"));
-  ASSERT_NOK(env_->FileExists(dbname_ + "/" + "002.sst.trash"));
-  ASSERT_NOK(env_->FileExists(dbname_ + "/" + "003.sst.trash"));
-}
-
-
 // Create a DB with 2 db_paths, and generate multiple files in the 2
 // db_paths using CompactRangeOptions, make sure that files that were
 // deleted from first db_path were deleted using DeleteScheduler and
@@ -439,12 +385,7 @@ TEST_F(DBSSTTest, DeleteSchedulerMultipleDBPaths) {
   int bg_delete_file = 0;
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
       "DeleteScheduler::DeleteTrashFile:DeleteFile",
-      [&](void* /*arg*/) { bg_delete_file++; });
-  // The deletion scheduler sometimes skips marking file as trash according to
-  // a heuristic. In that case the deletion will go through the below SyncPoint.
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "DeleteScheduler::DeleteFile",
-      [&](void* /*arg*/) { bg_delete_file++; });
+      [&](void* arg) { bg_delete_file++; });
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 
   Options options = CurrentOptions();
@@ -453,14 +394,14 @@ TEST_F(DBSSTTest, DeleteSchedulerMultipleDBPaths) {
   options.db_paths.emplace_back(dbname_ + "_2", 1024 * 100);
   options.env = env_;
 
+  std::string trash_dir = test::TmpDir(env_) + "/trash";
   int64_t rate_bytes_per_sec = 1024 * 1024;  // 1 Mb / Sec
   Status s;
-  options.sst_file_manager.reset(
-      NewSstFileManager(env_, nullptr, "", rate_bytes_per_sec, false, &s,
-                        /* max_trash_db_ratio= */ 1.1));
-
+  options.sst_file_manager.reset(NewSstFileManager(
+      env_, nullptr, trash_dir, rate_bytes_per_sec, false, &s));
   ASSERT_OK(s);
   auto sfm = static_cast<SstFileManagerImpl*>(options.sst_file_manager.get());
+  sfm->delete_scheduler()->TEST_SetMaxTrashDBRatio(1.1);
 
   DestroyAndReopen(options);
 
@@ -497,15 +438,13 @@ TEST_F(DBSSTTest, DeleteSchedulerMultipleDBPaths) {
   sfm->WaitForEmptyTrash();
   ASSERT_EQ(bg_delete_file, 8);
 
-  // Compaction will delete both files and regenerate a file in L1 in second
-  // db path. The deleted files should still be cleaned up via delete scheduler.
   compact_options.bottommost_level_compaction =
       BottommostLevelCompaction::kForce;
   ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr));
   ASSERT_EQ("0,1", FilesPerLevel(0));
 
   sfm->WaitForEmptyTrash();
-  ASSERT_EQ(bg_delete_file, 10);
+  ASSERT_EQ(bg_delete_file, 8);
 
   rocksdb::SyncPoint::GetInstance()->DisableProcessing();
 }
@@ -514,15 +453,16 @@ TEST_F(DBSSTTest, DestroyDBWithRateLimitedDelete) {
   int bg_delete_file = 0;
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
       "DeleteScheduler::DeleteTrashFile:DeleteFile",
-      [&](void* /*arg*/) { bg_delete_file++; });
+      [&](void* arg) { bg_delete_file++; });
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 
   Status s;
   Options options = CurrentOptions();
   options.disable_auto_compactions = true;
   options.env = env_;
+  std::string trash_dir = test::TmpDir(env_) + "/trash";
   options.sst_file_manager.reset(
-      NewSstFileManager(env_, nullptr, "", 0, false, &s, 0));
+      NewSstFileManager(env_, nullptr, trash_dir, 0, false, &s));
   ASSERT_OK(s);
   DestroyAndReopen(options);
 
@@ -540,7 +480,7 @@ TEST_F(DBSSTTest, DestroyDBWithRateLimitedDelete) {
   auto sfm = static_cast<SstFileManagerImpl*>(options.sst_file_manager.get());
 
   sfm->SetDeleteRateBytesPerSecond(1024 * 1024);
-  sfm->delete_scheduler()->SetMaxTrashDBRatio(1.1);
+  sfm->delete_scheduler()->TEST_SetMaxTrashDBRatio(1.1);
   ASSERT_OK(DestroyDB(dbname_, options));
   sfm->WaitForEmptyTrash();
   // We have deleted the 4 sst files in the delete_scheduler
@@ -576,127 +516,6 @@ TEST_F(DBSSTTest, DBWithMaxSpaceAllowed) {
   ASSERT_NOK(Flush());
 }
 
-TEST_F(DBSSTTest, CancellingCompactionsWorks) {
-  std::shared_ptr<SstFileManager> sst_file_manager(NewSstFileManager(env_));
-  auto sfm = static_cast<SstFileManagerImpl*>(sst_file_manager.get());
-
-  Options options = CurrentOptions();
-  options.sst_file_manager = sst_file_manager;
-  options.level0_file_num_compaction_trigger = 2;
-  options.statistics = CreateDBStatistics();
-  DestroyAndReopen(options);
-
-  int completed_compactions = 0;
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::BackgroundCompaction():CancelledCompaction", [&](void* /*arg*/) {
-        sfm->SetMaxAllowedSpaceUsage(0);
-        ASSERT_EQ(sfm->GetCompactionsReservedSize(), 0);
-      });
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::BackgroundCompaction:NonTrivial:AfterRun",
-      [&](void* /*arg*/) { completed_compactions++; });
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-
-  Random rnd(301);
-
-  // Generate a file containing 10 keys.
-  for (int i = 0; i < 10; i++) {
-    ASSERT_OK(Put(Key(i), RandomString(&rnd, 50)));
-  }
-  ASSERT_OK(Flush());
-  uint64_t total_file_size = 0;
-  auto files_in_db = GetAllSSTFiles(&total_file_size);
-  // Set the maximum allowed space usage to the current total size
-  sfm->SetMaxAllowedSpaceUsage(2 * total_file_size + 1);
-
-  // Generate another file to trigger compaction.
-  for (int i = 0; i < 10; i++) {
-    ASSERT_OK(Put(Key(i), RandomString(&rnd, 50)));
-  }
-  ASSERT_OK(Flush());
-  dbfull()->TEST_WaitForCompact(true);
-
-  // Because we set a callback in CancelledCompaction, we actually
-  // let the compaction run
-  ASSERT_GT(completed_compactions, 0);
-  ASSERT_EQ(sfm->GetCompactionsReservedSize(), 0);
-  // Make sure the stat is bumped
-  ASSERT_GT(dbfull()->immutable_db_options().statistics.get()->getTickerCount(COMPACTION_CANCELLED), 0);
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-}
-
-TEST_F(DBSSTTest, CancellingManualCompactionsWorks) {
-  std::shared_ptr<SstFileManager> sst_file_manager(NewSstFileManager(env_));
-  auto sfm = static_cast<SstFileManagerImpl*>(sst_file_manager.get());
-
-  Options options = CurrentOptions();
-  options.sst_file_manager = sst_file_manager;
-  options.statistics = CreateDBStatistics();
-
-  FlushedFileCollector* collector = new FlushedFileCollector();
-  options.listeners.emplace_back(collector);
-
-  DestroyAndReopen(options);
-
-  Random rnd(301);
-
-  // Generate a file containing 10 keys.
-  for (int i = 0; i < 10; i++) {
-    ASSERT_OK(Put(Key(i), RandomString(&rnd, 50)));
-  }
-  ASSERT_OK(Flush());
-  uint64_t total_file_size = 0;
-  auto files_in_db = GetAllSSTFiles(&total_file_size);
-  // Set the maximum allowed space usage to the current total size
-  sfm->SetMaxAllowedSpaceUsage(2 * total_file_size + 1);
-
-  // Generate another file to trigger compaction.
-  for (int i = 0; i < 10; i++) {
-    ASSERT_OK(Put(Key(i), RandomString(&rnd, 50)));
-  }
-  ASSERT_OK(Flush());
-
-  // OK, now trigger a manual compaction
-  dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
-
-  // Wait for manual compaction to get scheduled and finish
-  dbfull()->TEST_WaitForCompact(true);
-
-  ASSERT_EQ(sfm->GetCompactionsReservedSize(), 0);
-  // Make sure the stat is bumped
-  ASSERT_EQ(dbfull()->immutable_db_options().statistics.get()->getTickerCount(
-                COMPACTION_CANCELLED),
-            1);
-
-  // Now make sure CompactFiles also gets cancelled
-  auto l0_files = collector->GetFlushedFiles();
-  dbfull()->CompactFiles(rocksdb::CompactionOptions(), l0_files, 0);
-
-  // Wait for manual compaction to get scheduled and finish
-  dbfull()->TEST_WaitForCompact(true);
-
-  ASSERT_EQ(dbfull()->immutable_db_options().statistics.get()->getTickerCount(
-                COMPACTION_CANCELLED),
-            2);
-  ASSERT_EQ(sfm->GetCompactionsReservedSize(), 0);
-
-  // Now let the flush through and make sure GetCompactionsReservedSize
-  // returns to normal
-  sfm->SetMaxAllowedSpaceUsage(0);
-  int completed_compactions = 0;
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "CompactFilesImpl:End", [&](void* /*arg*/) { completed_compactions++; });
-
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-  dbfull()->CompactFiles(rocksdb::CompactionOptions(), l0_files, 0);
-  dbfull()->TEST_WaitForCompact(true);
-
-  ASSERT_EQ(sfm->GetCompactionsReservedSize(), 0);
-  ASSERT_GT(completed_compactions, 0);
-
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-}
-
 TEST_F(DBSSTTest, DBWithMaxSpaceAllowedRandomized) {
   // This test will set a maximum allowed space for the DB, then it will
   // keep filling the DB until the limit is reached and bg_error_ is set.
@@ -725,15 +544,9 @@ TEST_F(DBSSTTest, DBWithMaxSpaceAllowedRandomized) {
         estimate_multiplier++;  // used in the main loop assert
       });
 
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::BackgroundCompaction():CancelledCompaction", [&](void* arg) {
-        bool* enough_room = static_cast<bool*>(arg);
-        *enough_room = true;
-      });
-
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
       "CompactionJob::FinishCompactionOutputFile:MaxAllowedSpaceReached",
-      [&](void* /*arg*/) {
+      [&](void* arg) {
         bg_error_set = true;
         GetAllSSTFiles(&total_sst_files_size);
         reached_max_space_on_compaction++;