release 5.6.2

Summary:
`KeyNotExistsBeyondOutputLevel` didn't consider L0 files' key-ranges. So if a key only was covered by older L0 files' key-ranges, we would incorrectly drop deletions of that key. This PR just skips the deletion-dropping optimization when output level is L0.
Closes https://github.com/facebook/rocksdb/pull/2726

Differential Revision: D5617286

Pulled By: ajkr

fbshipit-source-id: 4bff1396b06d49a828ba4542f249191052915bce

.deprecated_arcconfig

@@ -0,0 +1,17 @@
+{
+  "project_id" : "rocksdb",
+  "conduit_uri" : "https://phabricator.fb.com/api/",
+  "copyright_holder" : "Facebook",
+  "load" : [
+    "arcanist_util"
+  ],
+  "lint.engine" : "FacebookFbcodeLintEngine",
+  "lint.engine.single.linter" : "FbcodeCppLinter",
+  "unit.engine" : "FacebookFbcodeUnitTestEngine",
+  "arcanist_configuration" : "FacebookArcanistConfiguration",
+  "base" : "git:HEAD^, hg:.^",
+  "git.default-relative-commit" : "HEAD^",
+  "git:arc.feature.start.default" : "origin/master",
+  "arc.feature.start.default" : "master",
+  "history.immutable" : false
+}

.gitignore

@@ -39,6 +39,7 @@ coverage/COVERAGE_REPORT
 .gdbhistory
 .gdb_history
 package/
+.phutil_module_cache
 unity.a
 tags
 etags

.travis.yml

@@ -57,11 +57,11 @@ before_script:
 
 script:
   - ${CXX} --version
-  - if [ "${TEST_GROUP}" == 'platform_dependent' ]; then ccache -C && OPT=-DTRAVIS V=1 ROCKSDBTESTS_END=db_block_cache_test make -j4 all_but_some_tests check_some; fi
+  - if [ "${TEST_GROUP}" == 'platform_dependent' ]; then OPT=-DTRAVIS V=1 make -j4 all; OPT=-DTRAVIS V=1 ROCKSDBTESTS_END=db_block_cache_test make -j4 check_some; fi
   - if [ "${TEST_GROUP}" == '1' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=db_block_cache_test ROCKSDBTESTS_END=comparator_db_test make -j4 check_some; fi
   - if [ "${TEST_GROUP}" == '2' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=comparator_db_test make -j4 check_some; fi
   - if [ "${JOB_NAME}" == 'java_test' ]; then OPT=-DTRAVIS V=1 make clean jclean && make rocksdbjava jtest; fi
-  - if [ "${JOB_NAME}" == 'lite_build' ]; then OPT="-DTRAVIS -DROCKSDB_LITE" V=1 make -j4 static_lib tools; fi
+  - if [ "${JOB_NAME}" == 'lite_build' ]; then OPT="-DTRAVIS -DROCKSDB_LITE" V=1 make -j4 static_lib; fi
   - if [ "${JOB_NAME}" == 'examples' ]; then OPT=-DTRAVIS V=1 make -j4 static_lib; cd examples; make -j4; fi
   - if [ "${JOB_NAME}" == 'cmake' ]; then mkdir build && cd build && cmake .. && make -j4 rocksdb; fi
   - if [ "${JOB_NAME}" == 'cmake-mingw' ]; then mkdir build && cd build && cmake .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb; fi

CMakeLists.txt

@@ -41,10 +41,10 @@ endif()
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/modules/")
 
-option(WITH_JEMALLOC "build with JeMalloc" OFF)
 if(MSVC)
   include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc)
 else()
+  option(WITH_JEMALLOC "build with JeMalloc" OFF)
   if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
     # FreeBSD has jemaloc as default malloc
     # but it does not have all the jemalloc files in include/...
@@ -56,7 +56,6 @@ else()
       include_directories(${JEMALLOC_INCLUDE_DIR})
     endif()
   endif()
-
   option(WITH_SNAPPY "build with SNAPPY" OFF)
   if(WITH_SNAPPY)
     find_package(snappy REQUIRED)
@@ -64,38 +63,6 @@ else()
     include_directories(${SNAPPY_INCLUDE_DIR})
     list(APPEND THIRDPARTY_LIBS ${SNAPPY_LIBRARIES})
   endif()
-
-  option(WITH_ZLIB "build with zlib" OFF)
-  if(WITH_ZLIB)
-    find_package(zlib REQUIRED)
-    add_definitions(-DZLIB)
-    include_directories(${ZLIB_INCLUDE_DIR})
-    list(APPEND THIRDPARTY_LIBS ${ZLIB_LIBRARIES})
-  endif()
-
-  option(WITH_BZ2 "build with bzip2" OFF)
-  if(WITH_BZ2)
-    find_package(bzip2 REQUIRED)
-    add_definitions(-DBZIP2)
-    include_directories(${BZIP2_INCLUDE_DIR})
-    list(APPEND THIRDPARTY_LIBS ${BZIP2_LIBRARIES})
-  endif()
-
-  option(WITH_LZ4 "build with lz4" OFF)
-  if(WITH_LZ4)
-    find_package(lz4 REQUIRED)
-    add_definitions(-DLZ4)
-    include_directories(${LZ4_INCLUDE_DIR})
-    list(APPEND THIRDPARTY_LIBS ${LZ4_LIBRARIES})
-  endif()
-
-  option(WITH_ZSTD "build with zstd" OFF)
-  if(WITH_ZSTD)
-    find_package(zstd REQUIRED)
-    add_definitions(-DZSTD)
-    include_directories(${ZSTD_INCLUDE_DIR})
-    list(APPEND THIRDPARTY_LIBS ${ZSTD_LIBRARIES})
-  endif()
 endif()
 
 if(WIN32)
@@ -148,31 +115,6 @@ if(WIN32 AND MSVC)
   endif()
 endif()
 
-set(BUILD_VERSION_CC ${CMAKE_BINARY_DIR}/build_version.cc)
-configure_file(util/build_version.cc.in ${BUILD_VERSION_CC} @ONLY)
-add_library(build_version OBJECT ${BUILD_VERSION_CC})
-target_include_directories(build_version PRIVATE
-  ${CMAKE_CURRENT_SOURCE_DIR}/util)
-if(MSVC)
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W3 /wd4127 /wd4800 /wd4996 /wd4351")
-else()
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wshadow -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers")
-  if(MINGW)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format")
-  endif()
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
-  if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -fno-omit-frame-pointer")
-    include(CheckCXXCompilerFlag)
-    CHECK_CXX_COMPILER_FLAG("-momit-leaf-frame-pointer" HAVE_OMIT_LEAF_FRAME_POINTER)
-    if(HAVE_OMIT_LEAF_FRAME_POINTER)
-      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -momit-leaf-frame-pointer")
-    endif()
-  endif()
-endif()
-
 option(PORTABLE "build a portable binary" OFF)
 option(FORCE_SSE42 "force building with SSE4.2, even when PORTABLE=ON" OFF)
 if(PORTABLE)
@@ -216,6 +158,32 @@ if(HAVE_THREAD_LOCAL)
   add_definitions(-DROCKSDB_SUPPORT_THREAD_LOCAL)
 endif()
 
+set(BUILD_VERSION_CC ${CMAKE_BINARY_DIR}/build_version.cc)
+configure_file(util/build_version.cc.in ${BUILD_VERSION_CC} @ONLY)
+add_library(build_version OBJECT ${BUILD_VERSION_CC})
+target_include_directories(build_version PRIVATE
+  ${CMAKE_CURRENT_SOURCE_DIR}/util)
+if(MSVC)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W3 /wd4127 /wd4800 /wd4996 /wd4351")
+else()
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wshadow -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers")
+  if(MINGW)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format")
+  endif()
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+  if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
+    add_definitions(-DNDEBUG)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -fno-omit-frame-pointer")
+    include(CheckCXXCompilerFlag)
+    CHECK_CXX_COMPILER_FLAG("-momit-leaf-frame-pointer" HAVE_OMIT_LEAF_FRAME_POINTER)
+    if(HAVE_OMIT_LEAF_FRAME_POINTER)
+      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -momit-leaf-frame-pointer")
+    endif()
+  endif()
+endif()
+
 option(FAIL_ON_WARNINGS "Treat compile warnings as errors" ON)
 if(FAIL_ON_WARNINGS)
   if(MSVC)
@@ -325,7 +293,6 @@ endif()
 option(WITH_FALLOCATE "build with fallocate" ON)
 
 if(WITH_FALLOCATE)
-  set(CMAKE_REQUIRED_FLAGS ${CMAKE_C_FLAGS})
   include(CheckCSourceCompiles)
   CHECK_C_SOURCE_COMPILES("
 #include <fcntl.h>
@@ -413,7 +380,6 @@ set(SOURCES
         db/write_thread.cc
         env/env.cc
         env/env_chroot.cc
-        env/env_encryption.cc
         env/env_hdfs.cc
         env/mock_env.cc
         memtable/alloc_tracker.cc
@@ -510,15 +476,12 @@ set(SOURCES
         utilities/backupable/backupable_db.cc
         utilities/blob_db/blob_db.cc
         utilities/blob_db/blob_db_impl.cc
+        utilities/blob_db/blob_db_options_impl.cc
         utilities/blob_db/blob_dump_tool.cc
         utilities/blob_db/blob_file.cc
         utilities/blob_db/blob_log_reader.cc
         utilities/blob_db/blob_log_writer.cc
         utilities/blob_db/blob_log_format.cc
-        utilities/blob_db/ttl_extractor.cc
-        utilities/cassandra/cassandra_compaction_filter.cc
-        utilities/cassandra/format.cc
-        utilities/cassandra/merge_operator.cc
         utilities/checkpoint/checkpoint_impl.cc
         utilities/col_buf_decoder.cc
         utilities/col_buf_encoder.cc
@@ -537,6 +500,8 @@ set(SOURCES
         utilities/memory/memory_util.cc
         utilities/merge_operators/max.cc
         utilities/merge_operators/put.cc
+        utilities/merge_operators/cassandra/format.cc
+        utilities/merge_operators/cassandra/merge_operator.cc
         utilities/merge_operators/string_append/stringappend.cc
         utilities/merge_operators/string_append/stringappend2.cc
         utilities/merge_operators/uint64add.cc
@@ -552,14 +517,13 @@ set(SOURCES
         utilities/spatialdb/spatial_db.cc
         utilities/table_properties_collectors/compact_on_deletion_collector.cc
         utilities/transactions/optimistic_transaction_db_impl.cc
-        utilities/transactions/optimistic_transaction.cc
+        utilities/transactions/optimistic_transaction_impl.cc
         utilities/transactions/transaction_base.cc
-        utilities/transactions/pessimistic_transaction_db.cc
+        utilities/transactions/transaction_db_impl.cc
         utilities/transactions/transaction_db_mutex_impl.cc
-        utilities/transactions/pessimistic_transaction.cc
+        utilities/transactions/transaction_impl.cc
         utilities/transactions/transaction_lock_mgr.cc
         utilities/transactions/transaction_util.cc
-        utilities/transactions/write_prepared_txn.cc
         utilities/ttl/db_ttl_impl.cc
         utilities/write_batch_with_index/write_batch_with_index.cc
         utilities/write_batch_with_index/write_batch_with_index_internal.cc
@@ -574,12 +538,6 @@ if(WIN32)
     port/win/win_logger.cc
     port/win/win_thread.cc
     port/win/xpress_win.cc)
-	
-if(WITH_JEMALLOC)
-  list(APPEND SOURCES
-    port/win/win_jemalloc.cc)
-endif()
-	
 else()
   list(APPEND SOURCES
     port/port_posix.cc
@@ -641,72 +599,7 @@ else()
   message(STATUS "JNI library is disabled")
 endif()
 
-# Installation and packaging
-if(WIN32)
-  option(ROCKSDB_INSTALL_ON_WINDOWS "Enable install target on Windows" OFF)
-endif()
-if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS)
-  if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
-    if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
-      # Change default installation prefix on Linux to /usr
-      set(CMAKE_INSTALL_PREFIX /usr CACHE PATH "Install path prefix, prepended onto install directories." FORCE)
-    endif()
-  endif()
-
-  include(GNUInstallDirs)
-  include(CMakePackageConfigHelpers)
-
-  set(package_config_destination ${CMAKE_INSTALL_LIBDIR}/cmake/rocksdb)
-
-  configure_package_config_file(
-    ${CMAKE_SOURCE_DIR}/cmake/RocksDBConfig.cmake.in RocksDBConfig.cmake
-    INSTALL_DESTINATION ${package_config_destination}
-  )
-
-  write_basic_package_version_file(
-    RocksDBConfigVersion.cmake
-    VERSION ${ROCKSDB_VERSION}
-    COMPATIBILITY SameMajorVersion
-  )
-
-  install(DIRECTORY include/rocksdb COMPONENT devel DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
-
-  install(
-    TARGETS ${ROCKSDB_STATIC_LIB}
-    EXPORT RocksDBTargets
-    COMPONENT devel
-    ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
-    INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
-  )
-
-  install(
-    TARGETS ${ROCKSDB_SHARED_LIB}
-    EXPORT RocksDBTargets
-    COMPONENT runtime
-    RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
-    LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
-    INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
-  )
-
-  install(
-    EXPORT RocksDBTargets
-    COMPONENT devel
-    DESTINATION ${package_config_destination}
-    NAMESPACE RocksDB::
-  )
-
-  install(
-    FILES
-    ${CMAKE_CURRENT_BINARY_DIR}/RocksDBConfig.cmake
-    ${CMAKE_CURRENT_BINARY_DIR}/RocksDBConfigVersion.cmake
-    COMPONENT devel
-    DESTINATION ${package_config_destination}
-  )
-endif()
-
-option(WITH_TESTS "build with tests" ON)
-if(WITH_TESTS)
-  set(TESTS
+set(TESTS
         cache/cache_test.cc
         cache/lru_cache_test.cc
         db/column_family_test.cc
@@ -719,7 +612,6 @@ if(WITH_TESTS)
         db/corruption_test.cc
         db/cuckoo_table_db_test.cc
         db/db_basic_test.cc
-        db/db_blob_index_test.cc
         db/db_block_cache_test.cc
         db/db_bloom_filter_test.cc
         db/db_compaction_filter_test.cc
@@ -785,7 +677,6 @@ if(WITH_TESTS)
         options/options_test.cc
         table/block_based_filter_block_test.cc
         table/block_test.cc
-        table/cleanable_test.cc
         table/cuckoo_table_builder_test.cc
         table/cuckoo_table_reader_test.cc
         table/full_filter_block_test.cc
@@ -814,10 +705,6 @@ if(WITH_TESTS)
         util/thread_local_test.cc
         utilities/backupable/backupable_db_test.cc
         utilities/blob_db/blob_db_test.cc
-        utilities/cassandra/cassandra_functional_test.cc
-        utilities/cassandra/cassandra_format_test.cc
-        utilities/cassandra/cassandra_row_merge_test.cc
-        utilities/cassandra/cassandra_serialize_test.cc
         utilities/checkpoint/checkpoint_test.cc
         utilities/column_aware_encoding_test.cc
         utilities/date_tiered/date_tiered_test.cc
@@ -826,6 +713,10 @@ if(WITH_TESTS)
         utilities/geodb/geodb_test.cc
         utilities/lua/rocks_lua_test.cc
         utilities/memory/memory_test.cc
+        utilities/merge_operators/cassandra/cassandra_merge_test.cc
+        utilities/merge_operators/cassandra/cassandra_format_test.cc
+        utilities/merge_operators/cassandra/cassandra_row_merge_test.cc
+        utilities/merge_operators/cassandra/cassandra_serialize_test.cc
         utilities/merge_operators/string_append/stringappend_test.cc
         utilities/object_registry_test.cc
         utilities/option_change_migration/option_change_migration_test.cc
@@ -834,94 +725,98 @@ if(WITH_TESTS)
         utilities/persistent_cache/persistent_cache_test.cc
         utilities/redis/redis_lists_test.cc
         utilities/spatialdb/spatial_db_test.cc
-        utilities/simulator_cache/sim_cache_test.cc
         utilities/table_properties_collectors/compact_on_deletion_collector_test.cc
         utilities/transactions/optimistic_transaction_test.cc
         utilities/transactions/transaction_test.cc
         utilities/ttl/ttl_test.cc
         utilities/write_batch_with_index/write_batch_with_index_test.cc
-  )
-  if(WITH_LIBRADOS)
-    list(APPEND TESTS utilities/env_librados_test.cc)
-  endif()
+)
+if(WITH_LIBRADOS)
+  list(APPEND TESTS utilities/env_librados_test.cc)
+endif()
 
-  set(BENCHMARKS
-    cache/cache_bench.cc
-    memtable/memtablerep_bench.cc
-    tools/db_bench.cc
-    table/table_reader_bench.cc
-    utilities/column_aware_encoding_exp.cc
-    utilities/persistent_cache/hash_table_bench.cc)
-  add_library(testharness OBJECT util/testharness.cc)
-  foreach(sourcefile ${BENCHMARKS})
+set(BENCHMARKS
+  cache/cache_bench.cc
+  memtable/memtablerep_bench.cc
+  tools/db_bench.cc
+  table/table_reader_bench.cc
+  utilities/column_aware_encoding_exp.cc
+  utilities/persistent_cache/hash_table_bench.cc)
+add_library(testharness OBJECT util/testharness.cc)
+foreach(sourcefile ${BENCHMARKS})
+  get_filename_component(exename ${sourcefile} NAME_WE)
+  add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile}
+    $<TARGET_OBJECTS:testharness>)
+  target_link_libraries(${exename}${ARTIFACT_SUFFIX} gtest ${LIBS})
+endforeach(sourcefile ${BENCHMARKS})
+
+# For test util library that is build only in DEBUG mode
+# and linked to tests. Add test only code that is not #ifdefed for Release here.
+set(TESTUTIL_SOURCE
+    db/db_test_util.cc
+    monitoring/thread_status_updater_debug.cc
+    table/mock_table.cc
+    util/fault_injection_test_env.cc
+    utilities/merge_operators/cassandra/test_utils.cc
+)
+# test utilities are only build in debug
+enable_testing()
+add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND})
+set(TESTUTILLIB testutillib${ARTIFACT_SUFFIX})
+add_library(${TESTUTILLIB} STATIC ${TESTUTIL_SOURCE})
+if(MSVC)
+  set_target_properties(${TESTUTILLIB} PROPERTIES COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/testutillib${ARTIFACT_SUFFIX}.pdb")
+endif()
+set_target_properties(${TESTUTILLIB}
+      PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
+      EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
+      EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
+      )
+
+# Tests are excluded from Release builds
+set(TEST_EXES ${TESTS})
+
+foreach(sourcefile ${TEST_EXES})
     get_filename_component(exename ${sourcefile} NAME_WE)
     add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile}
       $<TARGET_OBJECTS:testharness>)
-    target_link_libraries(${exename}${ARTIFACT_SUFFIX} gtest ${LIBS})
-  endforeach(sourcefile ${BENCHMARKS})
-
-  # For test util library that is build only in DEBUG mode
-  # and linked to tests. Add test only code that is not #ifdefed for Release here.
-  set(TESTUTIL_SOURCE
-      db/db_test_util.cc
-      monitoring/thread_status_updater_debug.cc
-      table/mock_table.cc
-      util/fault_injection_test_env.cc
-      utilities/cassandra/test_utils.cc
-  )
-  # test utilities are only build in debug
-  enable_testing()
-  add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND})
-  set(TESTUTILLIB testutillib${ARTIFACT_SUFFIX})
-  add_library(${TESTUTILLIB} STATIC ${TESTUTIL_SOURCE})
-  if(MSVC)
-    set_target_properties(${TESTUTILLIB} PROPERTIES COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/testutillib${ARTIFACT_SUFFIX}.pdb")
-  endif()
-  set_target_properties(${TESTUTILLIB}
-        PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
-        EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
-        EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
-        )
-
-  # Tests are excluded from Release builds
-  set(TEST_EXES ${TESTS})
-
-  foreach(sourcefile ${TEST_EXES})
-      get_filename_component(exename ${sourcefile} NAME_WE)
-      add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile}
-        $<TARGET_OBJECTS:testharness>)
-      set_target_properties(${exename}${ARTIFACT_SUFFIX}
-        PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
-        EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
-        EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
-        )
-      target_link_libraries(${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} gtest ${LIBS})
-      if(NOT "${exename}" MATCHES "db_sanity_test")
-        add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
-        add_dependencies(check ${exename}${ARTIFACT_SUFFIX})
-      endif()
-  endforeach(sourcefile ${TEST_EXES})
-
-  # C executables must link to a shared object
-  set(C_TESTS db/c_test.c)
-  set(C_TEST_EXES ${C_TESTS})
-
-  foreach(sourcefile ${C_TEST_EXES})
-      string(REPLACE ".c" "" exename ${sourcefile})
-      string(REGEX REPLACE "^((.+)/)+" "" exename ${exename})
-      add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile})
-      set_target_properties(${exename}${ARTIFACT_SUFFIX}
-        PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
-        EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
-        EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
-        )
-      target_link_libraries(${exename}${ARTIFACT_SUFFIX} ${ROCKSDB_IMPORT_LIB} testutillib${ARTIFACT_SUFFIX})
+    set_target_properties(${exename}${ARTIFACT_SUFFIX}
+      PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
+      EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
+      EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
+      )
+    target_link_libraries(${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} gtest ${LIBS})
+    if(NOT "${exename}" MATCHES "db_sanity_test")
       add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
       add_dependencies(check ${exename}${ARTIFACT_SUFFIX})
-  endforeach(sourcefile ${C_TEST_EXES})
-endif()
+    endif()
+endforeach(sourcefile ${TEST_EXES})
 
-option(WITH_TOOLS "build with tools" ON)
-if(WITH_TOOLS)
-  add_subdirectory(tools)
+# C executables must link to a shared object
+set(C_TESTS db/c_test.c)
+set(C_TEST_EXES ${C_TESTS})
+
+foreach(sourcefile ${C_TEST_EXES})
+    string(REPLACE ".c" "" exename ${sourcefile})
+    string(REGEX REPLACE "^((.+)/)+" "" exename ${exename})
+    add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile})
+    set_target_properties(${exename}${ARTIFACT_SUFFIX}
+      PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
+      EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
+      EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
+      )
+    target_link_libraries(${exename}${ARTIFACT_SUFFIX} ${ROCKSDB_IMPORT_LIB} testutillib${ARTIFACT_SUFFIX})
+    add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
+    add_dependencies(check ${exename}${ARTIFACT_SUFFIX})
+endforeach(sourcefile ${C_TEST_EXES})
+add_subdirectory(tools)
+
+# Installation and packaging for Linux
+if(NOT WIN32)
+install(TARGETS ${ROCKSDB_STATIC_LIB} COMPONENT devel ARCHIVE DESTINATION lib64)
+install(TARGETS ${ROCKSDB_SHARED_LIB} COMPONENT runtime DESTINATION lib64)
+install(DIRECTORY "${PROJECT_SOURCE_DIR}/include/rocksdb/"
+        COMPONENT devel
+        DESTINATION include/rocksdb)
+set(CMAKE_INSTALL_PREFIX /usr)
 endif()

HISTORY.md

@@ -1,54 +1,11 @@
 # Rocksdb Change Log
-## 5.8.8 (12/6/2017)
+## 5.6.2 (08/12/2017)
 ### Bug Fixes
-* Fix possible corruption to LSM structure when `DeleteFilesInRange()` deletes a subset of files spanned by a `DeleteRange()` marker.
-
-## 5.8.7 (11/28/2017)
-### Bug Fixes
-* Fix IOError on WAL write doesn't propagate to write group follower
-
-## 5.8.6 (11/20/2017)
-### Bug Fixes
-* Fixed aligned_alloc issues with Windows.
-
-## 5.8.1 (10/23/2017)
-### New Features
-* Add a new db property "rocksdb.estimate-oldest-key-time" to return oldest data timestamp. The property is available only for FIFO compaction with compaction_options_fifo.allow_compaction = false.
-
-## 5.8.0 (08/30/2017)
-### Public API Change
-* Users of `Statistics::getHistogramString()` will see fewer histogram buckets and different bucket endpoints.
-* `Slice::compare` and BytewiseComparator `Compare` no longer accept `Slice`s containing nullptr.
-* `Transaction::Get` and `Transaction::GetForUpdate` variants with `PinnableSlice` added.
-
-### New Features
-* Add Iterator::Refresh(), which allows users to update the iterator state so that they can avoid some initialization costs of recreating iterators.
-* Replace dynamic_cast<> (except unit test) so people can choose to build with RTTI off. With make, release mode is by default built with -fno-rtti and debug mode is built without it. Users can override it by setting USE_RTTI=0 or 1.
-* Universal compactions including the bottom level can be executed in a dedicated thread pool. This alleviates head-of-line blocking in the compaction queue, which cause write stalling, particularly in multi-instance use cases. Users can enable this feature via `Env::SetBackgroundThreads(N, Env::Priority::BOTTOM)`, where `N > 0`.
-* Allow merge operator to be called even with a single merge operand during compactions, by appropriately overriding `MergeOperator::AllowSingleOperand`.
-* Add `DB::VerifyChecksum()`, which verifies the checksums in all SST files in a running DB.
-* Block-based table support for disabling checksums by setting `BlockBasedTableOptions::checksum = kNoChecksum`.
-
-### Bug Fixes
-* Fix wrong latencies in `rocksdb.db.get.micros`, `rocksdb.db.write.micros`, and `rocksdb.sst.read.micros`.
 * Fix incorrect dropping of deletions during intra-L0 compaction.
-* Fix transient reappearance of keys covered by range deletions when memtable prefix bloom filter is enabled.
-* Fix potentially wrong file smallest key when range deletions separated by snapshot are written together.
-
-## 5.7.0 (07/13/2017)
-### Public API Change
-* DB property "rocksdb.sstables" now prints keys in hex form.
-
-### New Features
-* Measure estimated number of reads per file. The information can be accessed through DB::GetColumnFamilyMetaData or "rocksdb.sstables" DB property.
-* RateLimiter support for throttling background reads, or throttling the sum of background reads and writes. This can give more predictable I/O usage when compaction reads more data than it writes, e.g., due to lots of deletions.
-* [Experimental] FIFO compaction with TTL support. It can be enabled by setting CompactionOptionsFIFO.ttl > 0.
-* Introduce `EventListener::OnBackgroundError()` callback. Users can implement it to be notified of errors causing the DB to enter read-only mode, and optionally override them.
-* Partitioned Index/Filters exiting the experimental mode. To enable partitioned indexes set index_type to kTwoLevelIndexSearch and to further enable partitioned filters set partition_filters to true. To configure the partition size set metadata_block_size.
-
 
+## 5.6.1 (07/25/2017)
 ### Bug Fixes
-* Fix discarding empty compaction output files when `DeleteRange()` is used together with subcompactions.
+* Fix lite build.
 
 ## 5.6.0 (06/06/2017)
 ### Public API Change

INSTALL.md

@@ -32,7 +32,6 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
 * You can link RocksDB with following compression libraries:
   - [zlib](http://www.zlib.net/) - a library for data compression.
   - [bzip2](http://www.bzip.org/) - a library for data compression.
-  - [lz4](https://github.com/lz4/lz4) - a library for extremely fast data compression.
   - [snappy](http://google.github.io/snappy/) - a library for fast
       data compression.
   - [zstandard](http://www.zstd.net) - Fast real-time compression
@@ -54,7 +53,6 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
       `sudo apt-get install libsnappy-dev`.
     * Install zlib. Try: `sudo apt-get install zlib1g-dev`.
     * Install bzip2: `sudo apt-get install libbz2-dev`.
-    * Install lz4: `sudo apt-get install liblz4-dev`.
     * Install zstandard: `sudo apt-get install libzstd-dev`.
 
 * **Linux - CentOS / RHEL**
@@ -83,10 +81,6 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
 
               sudo yum install bzip2 bzip2-devel
 
-    * Install lz4:
-
-              sudo yum install lz4-devel
-
     * Install ASAN (optional for debugging):
 
               sudo yum install libasan
@@ -113,8 +107,6 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
 * **Windows**:
   * For building with MS Visual Studio 13 you will need Update 4 installed.
   * Read and follow the instructions at CMakeLists.txt
-  * Or install via [vcpkg](https://github.com/microsoft/vcpkg) 
-       * run `vcpkg install rocksdb`
 
 * **AIX 6.1**
     * Install AIX Toolbox rpms with gcc

LANGUAGE-BINDINGS.md

@@ -9,8 +9,5 @@ This is the list of all known third-party language bindings for RocksDB. If some
 * Haskell - https://hackage.haskell.org/package/rocksdb-haskell
 * PHP - https://github.com/Photonios/rocksdb-php
 * C# - https://github.com/warrenfalk/rocksdb-sharp
-* Rust
-    * https://github.com/spacejam/rust-rocksdb
-    * https://github.com/bh1xuw/rust-rocks
+* Rust - https://github.com/spacejam/rust-rocksdb
 * D programming language - https://github.com/b1naryth1ef/rocksdb
-* Erlang - https://gitlab.com/barrel-db/erlang-rocksdb

Makefile

@@ -101,19 +101,7 @@ endif
 ifeq ($(DEBUG_LEVEL),0)
 OPT += -DNDEBUG
 DISABLE_WARNING_AS_ERROR=1
-
-ifneq ($(USE_RTTI), 1)
-	CXXFLAGS += -fno-rtti
 else
-	CXXFLAGS += -DROCKSDB_USE_RTTI
-endif
-else
-ifneq ($(USE_RTTI), 0)
-	CXXFLAGS += -DROCKSDB_USE_RTTI
-else
-	CXXFLAGS += -fno-rtti
-endif
-
 $(warning Warning: Compiling in debug mode. Don't use the resulting binary in production)
 endif
 
@@ -232,10 +220,6 @@ ifndef DISABLE_JEMALLOC
 		PLATFORM_CXXFLAGS += -DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE
 		PLATFORM_CCFLAGS  += -DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE
 	endif
-	ifdef WITH_JEMALLOC_FLAG
-		PLATFORM_LDFLAGS += -ljemalloc
-		JAVA_LDFLAGS += -ljemalloc
-	endif
 	EXEC_LDFLAGS := $(JEMALLOC_LIB) $(EXEC_LDFLAGS)
 	PLATFORM_CXXFLAGS += $(JEMALLOC_INCLUDE)
 	PLATFORM_CCFLAGS += $(JEMALLOC_INCLUDE)
@@ -339,7 +323,6 @@ EXPOBJECTS = $(EXP_LIB_SOURCES:.cc=.o) $(LIBOBJECTS) $(TESTUTIL)
 
 TESTS = \
 	db_basic_test \
-	db_encryption_test \
 	db_test2 \
 	external_sst_file_basic_test \
 	auto_roll_logger_test \
@@ -360,7 +343,6 @@ TESTS = \
 	db_wal_test \
 	db_block_cache_test \
 	db_test \
-	db_blob_index_test \
 	db_bloom_filter_test \
 	db_iter_test \
 	db_log_iter_test \
@@ -422,7 +404,7 @@ TESTS = \
 	write_buffer_manager_test \
 	stringappend_test \
 	cassandra_format_test \
-	cassandra_functional_test \
+	cassandra_merge_test \
 	cassandra_row_merge_test \
 	cassandra_serialize_test \
 	ttl_test \
@@ -592,8 +574,6 @@ endif  # PLATFORM_SHARED_EXT
 
 all: $(LIBRARY) $(BENCHMARKS) tools tools_lib test_libs $(TESTS)
 
-all_but_some_tests: $(LIBRARY) $(BENCHMARKS) tools tools_lib test_libs $(SUBSET)
-
 static_lib: $(LIBRARY)
 
 shared_lib: $(SHARED)
@@ -797,8 +777,8 @@ ldb_tests: ldb
 crash_test: whitebox_crash_test blackbox_crash_test
 
 blackbox_crash_test: db_stress
-	python -u tools/db_crashtest.py --simple blackbox $(CRASH_TEST_EXT_ARGS)
-	python -u tools/db_crashtest.py blackbox $(CRASH_TEST_EXT_ARGS)
+	python -u tools/db_crashtest.py --simple blackbox
+	python -u tools/db_crashtest.py blackbox
 
 ifeq ($(CRASH_TEST_KILL_ODD),)
   CRASH_TEST_KILL_ODD=888887
@@ -806,9 +786,9 @@ endif
 
 whitebox_crash_test: db_stress
 	python -u tools/db_crashtest.py --simple whitebox --random_kill_odd \
-      $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
+      $(CRASH_TEST_KILL_ODD)
 	python -u tools/db_crashtest.py whitebox  --random_kill_odd \
-      $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
+      $(CRASH_TEST_KILL_ODD)
 
 asan_check:
 	$(MAKE) clean
@@ -1019,16 +999,16 @@ option_change_migration_test: utilities/option_change_migration/option_change_mi
 stringappend_test: utilities/merge_operators/string_append/stringappend_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-cassandra_format_test: utilities/cassandra/cassandra_format_test.o utilities/cassandra/test_utils.o $(LIBOBJECTS) $(TESTHARNESS)
+cassandra_format_test: utilities/merge_operators/cassandra/cassandra_format_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-cassandra_functional_test: utilities/cassandra/cassandra_functional_test.o utilities/cassandra/test_utils.o $(LIBOBJECTS) $(TESTHARNESS)
+cassandra_merge_test: utilities/merge_operators/cassandra/cassandra_merge_test.o utilities/merge_operators/cassandra/test_utils.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-cassandra_row_merge_test: utilities/cassandra/cassandra_row_merge_test.o utilities/cassandra/test_utils.o $(LIBOBJECTS) $(TESTHARNESS)
+cassandra_row_merge_test: utilities/merge_operators/cassandra/cassandra_row_merge_test.o utilities/merge_operators/cassandra/test_utils.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-cassandra_serialize_test: utilities/cassandra/cassandra_serialize_test.o $(LIBOBJECTS) $(TESTHARNESS)
+cassandra_serialize_test: utilities/merge_operators/cassandra/cassandra_serialize_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
 redis_test: utilities/redis/redis_lists_test.o $(LIBOBJECTS) $(TESTHARNESS)
@@ -1055,18 +1035,12 @@ slice_transform_test: util/slice_transform_test.o $(LIBOBJECTS) $(TESTHARNESS)
 db_basic_test: db/db_basic_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-db_encryption_test: db/db_encryption_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 db_test: db/db_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
 db_test2: db/db_test2.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-db_blob_index_test: db/db_blob_index_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 db_block_cache_test: db/db_block_cache_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 

TARGETS

@@ -6,6 +6,8 @@ REPO_PATH = TARGETS_PATH[(TARGETS_PATH.find('fbcode/') + len('fbcode/')):] + "/"
 BUCK_BINS = "buck-out/gen/" + REPO_PATH
 TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh"
 rocksdb_compiler_flags = [
+  "-msse",
+  "-msse4.2",
   "-fno-builtin-memcmp",
   "-DROCKSDB_PLATFORM_POSIX",
   "-DROCKSDB_LIB_IO_POSIX",
@@ -14,8 +16,8 @@ rocksdb_compiler_flags = [
   "-DROCKSDB_RANGESYNC_PRESENT",
   "-DROCKSDB_SCHED_GETCPU_PRESENT",
   "-DROCKSDB_SUPPORT_THREAD_LOCAL",
+  "-DHAVE_SSE42",
   "-DOS_LINUX",
-  "-DROCKSDB_UBSAN_RUN",
   # Flags to enable libs we include
   "-DSNAPPY",
   "-DZLIB",
@@ -47,10 +49,6 @@ rocksdb_preprocessor_flags = [
   "-I" + REPO_PATH,
 ]
 
-rocksdb_arch_preprocessor_flags = {
-  "x86_64": ["-DHAVE_SSE42"],
-}
-
 cpp_library(
     name = "rocksdb_lib",
     headers = AutoHeaders.RECURSIVE_GLOB,
@@ -112,7 +110,6 @@ cpp_library(
       "db/write_thread.cc",
       "env/env.cc",
       "env/env_chroot.cc",
-      "env/env_encryption.cc",
       "env/env_hdfs.cc",
       "env/env_posix.cc",
       "env/io_posix.cc",
@@ -209,14 +206,11 @@ cpp_library(
       "utilities/backupable/backupable_db.cc",
       "utilities/blob_db/blob_db.cc",
       "utilities/blob_db/blob_db_impl.cc",
+      "utilities/blob_db/blob_db_options_impl.cc",
       "utilities/blob_db/blob_file.cc",
       "utilities/blob_db/blob_log_reader.cc",
       "utilities/blob_db/blob_log_writer.cc",
       "utilities/blob_db/blob_log_format.cc",
-      "utilities/blob_db/ttl_extractor.cc",
-      "utilities/cassandra/cassandra_compaction_filter.cc",
-      "utilities/cassandra/format.cc",
-      "utilities/cassandra/merge_operator.cc",
       "utilities/checkpoint/checkpoint_impl.cc",
       "utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc",
       "utilities/convenience/info_log_finder.cc",
@@ -231,6 +225,8 @@ cpp_library(
       "utilities/leveldb_options/leveldb_options.cc",
       "utilities/lua/rocks_lua_compaction_filter.cc",
       "utilities/memory/memory_util.cc",
+      "utilities/merge_operators/cassandra/format.cc",
+      "utilities/merge_operators/cassandra/merge_operator.cc",
       "utilities/merge_operators/max.cc",
       "utilities/merge_operators/put.cc",
       "utilities/merge_operators/string_append/stringappend.cc",
@@ -248,14 +244,13 @@ cpp_library(
       "utilities/spatialdb/spatial_db.cc",
       "utilities/table_properties_collectors/compact_on_deletion_collector.cc",
       "utilities/transactions/optimistic_transaction_db_impl.cc",
-      "utilities/transactions/optimistic_transaction.cc",
+      "utilities/transactions/optimistic_transaction_impl.cc",
       "utilities/transactions/transaction_base.cc",
-      "utilities/transactions/pessimistic_transaction_db.cc",
+      "utilities/transactions/transaction_db_impl.cc",
       "utilities/transactions/transaction_db_mutex_impl.cc",
-      "utilities/transactions/pessimistic_transaction.cc",
+      "utilities/transactions/transaction_impl.cc",
       "utilities/transactions/transaction_lock_mgr.cc",
       "utilities/transactions/transaction_util.cc",
-      "utilities/transactions/write_prepared_txn.cc",
       "utilities/ttl/db_ttl_impl.cc",
       "utilities/write_batch_with_index/write_batch_with_index.cc",
       "utilities/write_batch_with_index/write_batch_with_index_internal.cc",
@@ -266,7 +261,6 @@ cpp_library(
     ],
     deps = [],
     preprocessor_flags = rocksdb_preprocessor_flags,
-    arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
     compiler_flags = rocksdb_compiler_flags,
     external_deps = rocksdb_external_deps,
 )
@@ -280,14 +274,13 @@ cpp_library(
       "util/testharness.cc",
       "util/testutil.cc",
       "db/db_test_util.cc",
-      "utilities/cassandra/test_utils.cc",
+      "utilities/merge_operators/cassandra/test_utils.cc",
       "utilities/col_buf_encoder.cc",
       "utilities/col_buf_decoder.cc",
       "utilities/column_aware_encoding_util.cc",
     ],
     deps = [":rocksdb_lib"],
     preprocessor_flags = rocksdb_preprocessor_flags,
-    arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
     compiler_flags = rocksdb_compiler_flags,
     external_deps = rocksdb_external_deps,
 )
@@ -301,7 +294,6 @@ cpp_library(
     ],
     deps = [":rocksdb_lib"],
     preprocessor_flags = rocksdb_preprocessor_flags,
-    arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
     compiler_flags = rocksdb_compiler_flags,
     external_deps = rocksdb_external_deps,
 )
@@ -312,7 +304,6 @@ cpp_library(
     srcs = ["env/env_basic_test.cc"],
     deps = [":rocksdb_test_lib"],
     preprocessor_flags = rocksdb_preprocessor_flags,
-    arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
     compiler_flags = rocksdb_compiler_flags,
     external_deps = rocksdb_external_deps,
 )
@@ -333,16 +324,16 @@ ROCKS_TESTS = [['arena_test', 'util/arena_test.cc', 'serial'],
  ['c_test', 'db/c_test.c', 'serial'],
  ['cache_test', 'cache/cache_test.cc', 'serial'],
  ['cassandra_format_test',
-  'utilities/cassandra/cassandra_format_test.cc',
+  'utilities/merge_operators/cassandra/cassandra_format_test.cc',
   'serial'],
- ['cassandra_functional_test',
-  'utilities/cassandra/cassandra_functional_test.cc',
+ ['cassandra_merge_test',
+  'utilities/merge_operators/cassandra/cassandra_merge_test.cc',
   'serial'],
  ['cassandra_row_merge_test',
-  'utilities/cassandra/cassandra_row_merge_test.cc',
+  'utilities/merge_operators/cassandra/cassandra_row_merge_test.cc',
   'serial'],
  ['cassandra_serialize_test',
-  'utilities/cassandra/cassandra_serialize_test.cc',
+  'utilities/merge_operators/cassandra/cassandra_serialize_test.cc',
   'serial'],
  ['checkpoint_test', 'utilities/checkpoint/checkpoint_test.cc', 'serial'],
  ['cleanable_test', 'table/cleanable_test.cc', 'serial'],
@@ -367,7 +358,6 @@ ROCKS_TESTS = [['arena_test', 'util/arena_test.cc', 'serial'],
  ['cuckoo_table_reader_test', 'table/cuckoo_table_reader_test.cc', 'serial'],
  ['date_tiered_test', 'utilities/date_tiered/date_tiered_test.cc', 'serial'],
  ['db_basic_test', 'db/db_basic_test.cc', 'serial'],
- ['db_blob_index_test', 'db/db_blob_index_test.cc', 'serial'],
  ['db_block_cache_test', 'db/db_block_cache_test.cc', 'serial'],
  ['db_bloom_filter_test', 'db/db_bloom_filter_test.cc', 'serial'],
  ['db_compaction_filter_test', 'db/db_compaction_filter_test.cc', 'parallel'],
@@ -509,7 +499,6 @@ for test_cfg in ROCKS_TESTS:
       srcs = [test_cc],
       deps = [":rocksdb_test_lib"],
       preprocessor_flags = rocksdb_preprocessor_flags,
-      arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
       compiler_flags = rocksdb_compiler_flags,
       external_deps = rocksdb_external_deps,
     )

USERS.md

@@ -76,10 +76,7 @@ quasardb uses a heavily tuned RocksDB as its persistence layer.
 [Dgraph](https://github.com/dgraph-io/dgraph) is an open-source, scalable, distributed, low latency, high throughput Graph database .They use RocksDB to store state locally on a machine.
 
 ## Uber
-[Uber](http://eng.uber.com/cherami/) uses RocksDB as a durable and scalable task queue.
+[Uber](http://eng.uber.com/cherami/) uses RocksDB as a durable and and scalable task queue.
 
 ## 360 Pika
 [360](http://www.360.cn/) [Pika](https://github.com/Qihoo360/pika) is a nosql compatible with redis. With the huge amount of data stored, redis may suffer for a capacity bottleneck, and pika was born for solving it. It has widely been widely used in many company
-
-## LzLabs
-LzLabs is using RocksDB as a storage engine in their multi-database distributed framework to store application configuration and user data.

appveyor.yml

@@ -3,7 +3,7 @@ image: Visual Studio 2015
 before_build:
 - md %APPVEYOR_BUILD_FOLDER%\build
 - cd %APPVEYOR_BUILD_FOLDER%\build
-- cmake -G "Visual Studio 14 2015 Win64" -DOPTDBG=1 -DXPRESS=1 -DPORTABLE=1 ..
+- cmake -G "Visual Studio 14 2015 Win64" -DOPTDBG=1 -DXPRESS=1 ..
 - cd ..
 build:
   project: build\rocksdb.sln

arcanist_util/INTERNAL_ONLY_DIR

@@ -0,0 +1,2 @@
+arcanist_util are only used internaly, If you want to change it please check
+<internal_rocksdb_repo>/arcanist_util

buckifier/rocks_test_runner.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 # Create a tmp directory for the test to use
 TEST_DIR=$(mktemp -d /dev/shm/fbcode_rocksdb_XXXXXXX)
 TEST_TMPDIR="$TEST_DIR" $@ && rm -rf "$TEST_DIR"

buckifier/targets_cfg.py

@@ -6,10 +6,12 @@ rocksdb_target_header = """
 import os
 
 TARGETS_PATH = os.path.dirname(__file__)
-REPO_PATH = "rocksdb/src/"
+REPO_PATH = TARGETS_PATH[(TARGETS_PATH.find('fbcode/') + len('fbcode/')):] + "/"
 BUCK_BINS = "buck-out/gen/" + REPO_PATH
 TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh"
 rocksdb_compiler_flags = [
+  "-msse",
+  "-msse4.2",
   "-fno-builtin-memcmp",
   "-DROCKSDB_PLATFORM_POSIX",
   "-DROCKSDB_LIB_IO_POSIX",
@@ -18,6 +20,7 @@ rocksdb_compiler_flags = [
   "-DROCKSDB_RANGESYNC_PRESENT",
   "-DROCKSDB_SCHED_GETCPU_PRESENT",
   "-DROCKSDB_SUPPORT_THREAD_LOCAL",
+  "-DHAVE_SSE42",
   "-DOS_LINUX",
   # Flags to enable libs we include
   "-DSNAPPY",
@@ -49,10 +52,6 @@ rocksdb_preprocessor_flags = [
   "-I" + REPO_PATH + "include/",
   "-I" + REPO_PATH,
 ]
-
-rocksdb_arch_preprocessor_flags = {
-  "x86_64": ["-DHAVE_SSE42"],
-}
 """
 
 
@@ -63,7 +62,6 @@ cpp_library(
     srcs = [%s],
     deps = [%s],
     preprocessor_flags = rocksdb_preprocessor_flags,
-    arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
     compiler_flags = rocksdb_compiler_flags,
     external_deps = rocksdb_external_deps,
 )
@@ -75,7 +73,6 @@ cpp_binary(
   srcs = [%s],
   deps = [%s],
   preprocessor_flags = rocksdb_preprocessor_flags,
-  arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
   compiler_flags = rocksdb_compiler_flags,
   external_deps = rocksdb_external_deps,
 )
@@ -98,7 +95,6 @@ for test_cfg in ROCKS_TESTS:
       srcs = [test_cc],
       deps = [":rocksdb_test_lib"],
       preprocessor_flags = rocksdb_preprocessor_flags,
-      arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
       compiler_flags = rocksdb_compiler_flags,
       external_deps = rocksdb_external_deps,
     )

build_tools/RocksDBCommonHelper.php

@@ -1,377 +0,0 @@
-<?php
-// Copyright 2004-present Facebook. All Rights Reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-// Name of the environment variables which need to be set by the entity which
-// triggers continuous runs so that code at the end of the file gets executed
-// and Sandcastle run starts.
-define("ENV_POST_RECEIVE_HOOK", "POST_RECEIVE_HOOK");
-define("ENV_HTTPS_APP_VALUE", "HTTPS_APP_VALUE");
-define("ENV_HTTPS_TOKEN_VALUE", "HTTPS_TOKEN_VALUE");
-
-define("PRIMARY_TOKEN_FILE", '/home/krad/.sandcastle');
-define("CONT_RUN_ALIAS", "leveldb");
-
-//////////////////////////////////////////////////////////////////////
-/*  Run tests in sandcastle */
-function postURL($diffID, $url) {
-  assert(strlen($diffID) > 0);
-  assert(is_numeric($diffID));
-  assert(strlen($url) > 0);
-
-  $cmd_args = array(
-    'diff_id' => (int)$diffID,
-    'name' => sprintf(
-      'click here for sandcastle tests for D%d',
-      (int)$diffID
-    ),
-    'link' => $url
-  );
-  $cmd = 'echo ' . escapeshellarg(json_encode($cmd_args))
-         . ' | arc call-conduit differential.updateunitresults';
-
-  shell_exec($cmd);
-}
-
-function buildUpdateTestStatusCmd($diffID, $test, $status) {
-  assert(strlen($diffID) > 0);
-  assert(is_numeric($diffID));
-  assert(strlen($test) > 0);
-  assert(strlen($status) > 0);
-
-  $cmd_args = array(
-    'diff_id' => (int)$diffID,
-    'name' => $test,
-    'result' => $status
-  );
-
-  $cmd = 'echo ' . escapeshellarg(json_encode($cmd_args))
-         . ' | arc call-conduit differential.updateunitresults';
-
-  return $cmd;
-}
-
-function updateTestStatus($diffID, $test) {
-  assert(strlen($diffID) > 0);
-  assert(is_numeric($diffID));
-  assert(strlen($test) > 0);
-
-  shell_exec(buildUpdateTestStatusCmd($diffID, $test, "waiting"));
-}
-
-function getSteps($applyDiff, $diffID, $username, $test) {
-  assert(strlen($username) > 0);
-  assert(strlen($test) > 0);
-
-  if ($applyDiff) {
-    assert(strlen($diffID) > 0);
-    assert(is_numeric($diffID));
-
-    $arcrc_content = (PHP_OS == "Darwin" ?
-        exec("cat ~/.arcrc | gzip -f | base64") :
-            exec("cat ~/.arcrc | gzip -f | base64 -w0"));
-    assert(strlen($arcrc_content) > 0);
-
-    // Sandcastle machines don't have arc setup. We copy the user certificate
-    // and authenticate using that in Sandcastle.
-    $setup = array(
-      "name" => "Setup arcrc",
-      "shell" => "echo " . escapeshellarg($arcrc_content) . " | base64 --decode"
-                 . " | gzip -d > ~/.arcrc",
-      "user" => "root"
-    );
-
-    // arc demands certain permission on its config.
-    // also fix the sticky bit issue in sandcastle
-    $fix_permission = array(
-      "name" => "Fix environment",
-      "shell" => "chmod 600 ~/.arcrc && chmod +t /dev/shm",
-      "user" => "root"
-    );
-
-    // Construct the steps in the order of execution.
-    $steps[] = $setup;
-    $steps[] = $fix_permission;
-  }
-
-  // fbcode is a sub-repo. We cannot patch until we add it to ignore otherwise
-  // Git thinks it is an uncommited change.
-  $fix_git_ignore = array(
-    "name" => "Fix git ignore",
-    "shell" => "echo fbcode >> .git/info/exclude",
-    "user" => "root"
-  );
-
-  // This fixes "FATAL: ThreadSanitizer can not mmap the shadow memory"
-  // Source:
-  // https://github.com/google/sanitizers/wiki/ThreadSanitizerCppManual#FAQ
-  $fix_kernel_issue = array(
-    "name" => "Fix kernel issue with tsan",
-    "shell" => "echo 2 >/proc/sys/kernel/randomize_va_space",
-    "user" => "root"
-  );
-
-  $steps[] = $fix_git_ignore;
-  $steps[] = $fix_kernel_issue;
-
-  // This will be the command used to execute particular type of tests.
-  $cmd = "";
-
-  if ($applyDiff) {
-    // Patch the code (keep your fingures crossed).
-    $patch = array(
-      "name" => "Patch " . $diffID,
-      "shell" => "arc --arcrc-file ~/.arcrc "
-                  . "patch --nocommit --diff " . escapeshellarg($diffID),
-      "user" => "root"
-    );
-
-    $steps[] = $patch;
-
-    updateTestStatus($diffID, $test);
-    $cmd = buildUpdateTestStatusCmd($diffID, $test, "running") . "; ";
-  }
-
-  // Run the actual command.
-  $cmd = $cmd . "J=$(nproc) ./build_tools/precommit_checker.py " .
-           escapeshellarg($test) . "; exit_code=$?; ";
-
-  if ($applyDiff) {
-    $cmd = $cmd . "([[ \$exit_code -eq 0 ]] &&"
-                . buildUpdateTestStatusCmd($diffID, $test, "pass") . ")"
-                . "||" . buildUpdateTestStatusCmd($diffID, $test, "fail")
-                . "; ";
-  }
-
-  // shell command to sort the tests based on exit code and print
-  // the output of the log files.
-  $cat_sorted_logs = "
-    while read code log_file;
-      do echo \"################ cat \$log_file [exit_code : \$code] ################\";
-      cat \$log_file;
-    done < <(tail -n +2 LOG | sort -k7,7n -k4,4gr | awk '{print \$7,\$NF}')";
-
-  // Shell command to cat all log files
-  $cat_all_logs = "for f in `ls t/!(run-*)`; do echo \$f;cat \$f; done";
-
-  // If LOG file exist use it to cat log files sorted by exit code, otherwise
-  // cat everything
-  $logs_cmd = "if [ -f LOG ]; then {$cat_sorted_logs}; else {$cat_all_logs}; fi";
-
-  $cmd = $cmd . " cat /tmp/precommit-check.log"
-              . "; shopt -s extglob; {$logs_cmd}"
-              . "; shopt -u extglob; [[ \$exit_code -eq 0 ]]";
-  assert(strlen($cmd) > 0);
-
-  $run_test = array(
-    "name" => "Run " . $test,
-    "shell" => $cmd,
-    "user" => "root",
-    "parser" => "python build_tools/error_filter.py " . escapeshellarg($test),
-  );
-
-  $steps[] = $run_test;
-
-  if ($applyDiff) {
-    // Clean up the user arc config we are using.
-    $cleanup = array(
-      "name" => "Arc cleanup",
-      "shell" => "rm -f ~/.arcrc",
-      "user" => "root"
-    );
-
-    $steps[] = $cleanup;
-  }
-
-  assert(count($steps) > 0);
-  return $steps;
-}
-
-function getSandcastleConfig() {
-  $sandcastle_config = array();
-
-  $cwd = getcwd();
-  $cwd_token_file = "{$cwd}/.sandcastle";
-  // This is a case when we're executed from a continuous run. Fetch the values
-  // from the environment.
-  if (getenv(ENV_POST_RECEIVE_HOOK)) {
-    $sandcastle_config[0] = getenv(ENV_HTTPS_APP_VALUE);
-    $sandcastle_config[1] = getenv(ENV_HTTPS_TOKEN_VALUE);
-  } else {
-    // This is a typical `[p]arc diff` case. Fetch the values from the specific
-    // configuration files.
-    for ($i = 0; $i < 50; $i++) {
-      if (file_exists(PRIMARY_TOKEN_FILE) ||
-          file_exists($cwd_token_file)) {
-        break;
-      }
-      // If we failed to fetch the tokens, sleep for 0.2 second and try again
-      usleep(200000);
-    }
-    assert(file_exists(PRIMARY_TOKEN_FILE) ||
-           file_exists($cwd_token_file));
-
-    // Try the primary location first, followed by a secondary.
-    if (file_exists(PRIMARY_TOKEN_FILE)) {
-      $cmd = 'cat ' . PRIMARY_TOKEN_FILE;
-    } else {
-      $cmd = 'cat ' . escapeshellarg($cwd_token_file);
-    }
-
-    assert(strlen($cmd) > 0);
-    $sandcastle_config = explode(':', rtrim(shell_exec($cmd)));
-  }
-
-  // In this case be very explicit about the implications.
-  if (count($sandcastle_config) != 2) {
-    echo "Sandcastle configuration files don't contain valid information " .
-         "or the necessary environment variables aren't defined. Unable " .
-         "to validate the code changes.";
-    exit(1);
-  }
-
-  assert(strlen($sandcastle_config[0]) > 0);
-  assert(strlen($sandcastle_config[1]) > 0);
-  assert(count($sandcastle_config) > 0);
-
-  return $sandcastle_config;
-}
-
-// This function can be called either from `[p]arc diff` command or during
-// the Git post-receive hook.
- function startTestsInSandcastle($applyDiff, $workflow, $diffID) {
-  // Default options don't terminate on failure, but that's what we want. In
-  // the current case we use assertions intentionally as "terminate on failure
-  // invariants".
-  assert_options(ASSERT_BAIL, true);
-
-  // In case of a diff we'll send notificatios to the author. Else it'll go to
-  // the entire team because failures indicate that build quality has regressed.
-  $username = $applyDiff ? exec("whoami") : CONT_RUN_ALIAS;
-  assert(strlen($username) > 0);
-
-  if ($applyDiff) {
-    assert($workflow);
-    assert(strlen($diffID) > 0);
-    assert(is_numeric($diffID));
-  }
-
-  // List of tests we want to run in Sandcastle.
-  $tests = array("unit", "unit_non_shm", "unit_481", "clang_unit", "tsan",
-                 "asan", "lite_test", "valgrind", "release", "release_481",
-                 "clang_release", "clang_analyze", "code_cov",
-                 "java_build", "no_compression", "unity", "ubsan");
-
-  $send_email_template = array(
-    'type' => 'email',
-    'triggers' => array('fail'),
-    'emails' => array($username . '@fb.com'),
-  );
-
-  // Construct a job definition for each test and add it to the master plan.
-  foreach ($tests as $test) {
-    $stepName = "RocksDB diff " . $diffID . " test " . $test;
-
-    if (!$applyDiff) {
-      $stepName = "RocksDB continuous integration test " . $test;
-    }
-
-    $arg[] = array(
-      "name" => $stepName,
-      "report" => array($send_email_template),
-      "steps" => getSteps($applyDiff, $diffID, $username, $test)
-    );
-  }
-
-  // We cannot submit the parallel execution master plan to Sandcastle and
-  // need supply the job plan as a determinator. So we construct a small job
-  // that will spit out the master job plan which Sandcastle will parse and
-  // execute. Why compress the job definitions? Otherwise we run over the max
-  // string size.
-  $cmd = "echo " . base64_encode(json_encode($arg))
-         . (PHP_OS == "Darwin" ?
-             " | gzip -f | base64" :
-                 " | gzip -f | base64 -w0");
-  assert(strlen($cmd) > 0);
-
-  $arg_encoded = shell_exec($cmd);
-  assert(strlen($arg_encoded) > 0);
-
-  $runName = "Run diff " . $diffID . "for user " . $username;
-
-  if (!$applyDiff) {
-    $runName = "RocksDB continuous integration build and test run";
-  }
-
-  $command = array(
-    "name" => $runName,
-    "steps" => array()
-  );
-
-  $command["steps"][] = array(
-    "name" => "Generate determinator",
-    "shell" => "echo " . $arg_encoded . " | base64 --decode | gzip -d"
-               . " | base64 --decode",
-    "determinator" => true,
-    "user" => "root"
-  );
-
-  // Submit to Sandcastle.
-  $url = 'https://interngraph.intern.facebook.com/sandcastle/create';
-
-  $job = array(
-    'command' => 'SandcastleUniversalCommand',
-    'args' => $command,
-    'capabilities' => array(
-      'vcs' => 'rocksdb-int-git',
-      'type' => 'lego',
-    ),
-    'hash' => 'origin/master',
-    'user' => $username,
-    'alias' => 'rocksdb-precommit',
-    'tags' => array('rocksdb'),
-    'description' => 'Rocksdb precommit job',
-  );
-
-  // Fetch the configuration necessary to submit a successful HTTPS request.
-  $sandcastle_config = getSandcastleConfig();
-
-  $app = $sandcastle_config[0];
-  $token = $sandcastle_config[1];
-
-  $cmd = 'curl -s -k '
-          . ' -F app=' . escapeshellarg($app)
-          . ' -F token=' . escapeshellarg($token)
-          . ' -F job=' . escapeshellarg(json_encode($job))
-          .' ' . escapeshellarg($url);
-
-  $output = shell_exec($cmd);
-  assert(strlen($output) > 0);
-
-  // Extract Sandcastle URL from the response.
-  preg_match('/url": "(.+)"/', $output, $sandcastle_url);
-
-  assert(count($sandcastle_url) > 0, "Unable to submit Sandcastle request.");
-  assert(strlen($sandcastle_url[1]) > 0, "Unable to extract Sandcastle URL.");
-
-  if ($applyDiff) {
-    echo "\nSandcastle URL: " . $sandcastle_url[1] . "\n";
-    // Ask Phabricator to display it on the diff UI.
-    postURL($diffID, $sandcastle_url[1]);
-  } else {
-    echo "Continuous integration started Sandcastle tests. You can look at ";
-    echo "the progress at:\n" . $sandcastle_url[1] . "\n";
-  }
-}
-
-// Continuous run cript will set the environment variable and based on that
-// we'll trigger the execution of tests in Sandcastle. In that case we don't
-// need to apply any diffs and there's no associated workflow either.
-if (getenv(ENV_POST_RECEIVE_HOOK)) {
-  startTestsInSandcastle(
-    false /* $applyDiff */,
-    NULL /* $workflow */,
-    NULL /* $diffID */);
-}

build_tools/build_detect_platform

@@ -51,13 +51,11 @@ if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
     FBCODE_BUILD="true"
     # If we're compiling with TSAN we need pic build
     PIC_BUILD=$COMPILE_WITH_TSAN
-    if [ -n "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
-      # we need this to build with MySQL. Don't use for other purposes.
-      source "$PWD/build_tools/fbcode_config4.8.1.sh"
-    elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_5xx" ]; then
+    if [ -z "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
       source "$PWD/build_tools/fbcode_config.sh"
     else
-      source "$PWD/build_tools/fbcode_config_platform007.sh"
+      # we need this to build with MySQL. Don't use for other purposes.
+      source "$PWD/build_tools/fbcode_config4.8.1.sh"
     fi
 fi
 
@@ -319,11 +317,9 @@ EOF
     # Test whether jemalloc is available
     if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null -ljemalloc \
       2>/dev/null; then
-        # This will enable some preprocessor identifiers in the Makefile
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ljemalloc"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -ljemalloc"
         JEMALLOC=1
-        # JEMALLOC can be enabled either using the flag (like here) or by
-        # providing direct link to the jemalloc library
-        WITH_JEMALLOC_FLAG=1
     else
         # jemalloc is not available. Let's try tcmalloc
         if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null \
@@ -460,7 +456,7 @@ elif test -z "$PORTABLE"; then
   fi
 fi
 
-$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
+$CXX $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
   #include <cstdint>
   #include <nmmintrin.h>
   int main() {
@@ -521,14 +517,7 @@ echo "ROCKSDB_PATCH=$ROCKSDB_PATCH" >> "$OUTPUT"
 echo "CLANG_SCAN_BUILD=$CLANG_SCAN_BUILD" >> "$OUTPUT"
 echo "CLANG_ANALYZER=$CLANG_ANALYZER" >> "$OUTPUT"
 echo "PROFILING_FLAGS=$PROFILING_FLAGS" >> "$OUTPUT"
-# This will enable some related identifiers for the preprocessor
 if test -n "$JEMALLOC"; then
   echo "JEMALLOC=1" >> "$OUTPUT"
 fi
-# Indicates that jemalloc should be enabled using -ljemalloc flag
-# The alternative is to porvide a direct link to the library via JEMALLOC_LIB
-# and JEMALLOC_INCLUDE
-if test -n "$WITH_JEMALLOC_FLAG"; then
-  echo "WITH_JEMALLOC_FLAG=$WITH_JEMALLOC_FLAG" >> "$OUTPUT"
-fi
 echo "LUA_PATH=$LUA_PATH" >> "$OUTPUT"

build_tools/cont_integration.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 #
 # Copyright (c) 2016, Facebook. All rights reserved.
 #
@@ -67,7 +67,7 @@ function update_repo_status {
 #
 
 # Path to the determinator from the root of the RocksDB repo.
-CONTRUN_DETERMINATOR=./build_tools/RocksDBCommonHelper.php
+CONTRUN_DETERMINATOR=./arcanist_util/config/RocksDBCommonHelper.php
 
 # Value of the previous commit.
 PREV_COMMIT=

build_tools/dependencies_platform007.sh

@@ -1,18 +0,0 @@
-GCC_BASE=/mnt/gvfs/third-party2/gcc/6e8e715624fd15256a7970073387793dfcf79b46/7.x/centos7-native/b2ef2b6
-CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/ef37e1faa1c29782abfac1ae65a291b9b7966f6d/stable/centos7-native/c9f9104
-LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/c67031f0f739ac61575a061518d6ef5038f99f90/7.x/platform007/5620abc
-GLIBC_BASE=/mnt/gvfs/third-party2/glibc/60d6f124a78798b73944f5ba87c2306ae3460153/2.26/platform007/f259413
-SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/platform007/ca4da3d
-ZLIB_BASE=/mnt/gvfs/third-party2/zlib/22c2d65676fb7c23cfa797c4f6937f38b026f3cf/1.2.8/platform007/ca4da3d
-BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/platform007/ca4da3d
-LZ4_BASE=/mnt/gvfs/third-party2/lz4/907b498203d297947f3bb70b9466f47e100f1873/r131/platform007/ca4da3d
-ZSTD_BASE=/mnt/gvfs/third-party2/zstd/3ee276cbacfad3074e3f07bf826ac47f06970f4e/1.3.5/platform007/15a3614
-GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/platform007/ca4da3d
-JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/9c910d36d6235cc40e8ff559358f1833452300ca/master/platform007/5b0f53e
-NUMA_BASE=/mnt/gvfs/third-party2/numa/9cbf2460284c669ed19c3ccb200a71f7dd7e53c7/2.0.11/platform007/ca4da3d
-LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/bf3d7497fe4e6d007354f0adffa16ce3003f8338/1.3/platform007/6f3e0a9
-TBB_BASE=/mnt/gvfs/third-party2/tbb/ff4e0b093534704d8abab678a4fd7f5ea7b094c7/2018_U5/platform007/ca4da3d
-KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/b5c4a61a5c483ba24722005ae07895971a2ac707/fb/platform007/da39a3e
-BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/92ff90349e2f43ea0a8246d8b1cf17b6869013e3/2.29.1/centos7-native/da39a3e
-VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/f3f697a28122e6bcd513273dd9c1ff23852fc59f/3.13.0/platform007/ca4da3d
-LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.3.4/platform007/5007832

build_tools/dockerbuild.sh

@@ -1,2 +1,2 @@
-#!/usr/bin/env bash
+#!/bin/bash
 docker run -v $PWD:/rocks -w /rocks buildpack-deps make

build_tools/fbcode_config_platform007.sh

@@ -1,157 +0,0 @@
-#!/bin/sh
-#
-# Set environment variables so that we can compile rocksdb using
-# fbcode settings.  It uses the latest g++ and clang compilers and also
-# uses jemalloc
-# Environment variables that change the behavior of this script:
-# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included
-
-
-BASEDIR=`dirname $BASH_SOURCE`
-source "$BASEDIR/dependencies_platform007.sh"
-
-CFLAGS=""
-
-# libgcc
-LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/7.3.0"
-LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
-
-# glibc
-GLIBC_INCLUDE="$GLIBC_BASE/include"
-GLIBC_LIBS=" -L $GLIBC_BASE/lib"
-
-# snappy
-SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
-if test -z $PIC_BUILD; then
-  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
-else
-  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a"
-fi
-CFLAGS+=" -DSNAPPY"
-
-if test -z $PIC_BUILD; then
-  # location of zlib headers and libraries
-  ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
-  ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
-  CFLAGS+=" -DZLIB"
-
-  # location of bzip headers and libraries
-  BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
-  BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a"
-  CFLAGS+=" -DBZIP2"
-
-  LZ4_INCLUDE=" -I $LZ4_BASE/include/"
-  LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
-  CFLAGS+=" -DLZ4"
-fi
-
-ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
-if test -z $PIC_BUILD; then
-  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
-else
-  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a"
-fi
-CFLAGS+=" -DZSTD"
-
-# location of gflags headers and libraries
-GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
-if test -z $PIC_BUILD; then
-  GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
-else
-  GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
-fi
-CFLAGS+=" -DGFLAGS=gflags"
-
-# location of jemalloc
-JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
-JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a"
-
-if test -z $PIC_BUILD; then
-  # location of numa
-  NUMA_INCLUDE=" -I $NUMA_BASE/include/"
-  NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
-  CFLAGS+=" -DNUMA"
-
-  # location of libunwind
-  LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
-fi
-
-# location of TBB
-TBB_INCLUDE=" -isystem $TBB_BASE/include/"
-if test -z $PIC_BUILD; then
-  TBB_LIBS="$TBB_BASE/lib/libtbb.a"
-else
-  TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a"
-fi
-CFLAGS+=" -DTBB"
-
-# use Intel SSE support for checksum calculations
-export USE_SSE=1
-export PORTABLE=1
-
-BINUTILS="$BINUTILS_BASE/bin"
-AR="$BINUTILS/ar"
-
-DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE"
-
-STDLIBS="-L $GCC_BASE/lib64"
-
-CLANG_BIN="$CLANG_BASE/bin"
-CLANG_LIB="$CLANG_BASE/lib"
-CLANG_SRC="$CLANG_BASE/../../src"
-
-CLANG_ANALYZER="$CLANG_BIN/clang++"
-CLANG_SCAN_BUILD="$CLANG_SRC/llvm/tools/clang/tools/scan-build/bin/scan-build"
-
-if [ -z "$USE_CLANG" ]; then
-  # gcc
-  CC="$GCC_BASE/bin/gcc"
-  CXX="$GCC_BASE/bin/g++"
-
-  CFLAGS+=" -B$BINUTILS/gold"
-  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
-  CFLAGS+=" -isystem $GLIBC_INCLUDE"
-  JEMALLOC=1
-else
-  # clang
-  CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
-  CC="$CLANG_BIN/clang"
-  CXX="$CLANG_BIN/clang++"
-
-  KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
-
-  CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x "
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x/x86_64-facebook-linux "
-  CFLAGS+=" -isystem $GLIBC_INCLUDE"
-  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
-  CFLAGS+=" -isystem $CLANG_INCLUDE"
-  CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
-  CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
-  CFLAGS+=" -Wno-expansion-to-defined "
-  CXXFLAGS="-nostdinc++"
-fi
-
-CFLAGS+=" $DEPS_INCLUDE"
-CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
-CXXFLAGS+=" $CFLAGS"
-
-EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
-EXEC_LDFLAGS+=" -B$BINUTILS/gold"
-EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform007/lib/ld.so"
-EXEC_LDFLAGS+=" $LIBUNWIND"
-EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform007/lib"
-# required by libtbb
-EXEC_LDFLAGS+=" -ldl"
-
-PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
-
-EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS"
-
-VALGRIND_VER="$VALGRIND_BASE/bin/"
-
-# lua not supported because it's on track for deprecation, I think
-LUA_PATH=
-LUA_LIB=
-
-export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB

build_tools/format-diff.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 # If clang_format_diff.py command is not specfied, we assume we are able to
 # access directly without any path.
 if [ -z $CLANG_FORMAT_DIFF ]

build_tools/regression_build_test.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 set -e
 

build_tools/rocksdb-lego-determinator

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 # This script is executed by Sandcastle
 # to determine next steps to run
 
@@ -85,9 +85,8 @@ NON_SHM="TMPD=/tmp/rocksdb_test_tmp"
 GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1"
 ASAN="COMPILE_WITH_ASAN=1"
 CLANG="USE_CLANG=1"
-# in gcc-5 there are known problems with TSAN like https://gcc.gnu.org/bugzilla/show_bug.cgi?id=71090.
-# using platform007 gives us gcc-8 or higher which has that bug fixed.
-TSAN="ROCKSDB_FBCODE_BUILD_WITH_PLATFORM007=1 COMPILE_WITH_TSAN=1"
+LITE="OPT=\"-DROCKSDB_LITE -g\""
+TSAN="COMPILE_WITH_TSAN=1"
 UBSAN="COMPILE_WITH_UBSAN=1"
 DISABLE_JEMALLOC="DISABLE_JEMALLOC=1"
 HTTP_PROXY="https_proxy=http://fwdproxy.29.prn1:8080 http_proxy=http://fwdproxy.29.prn1:8080 ftp_proxy=http://fwdproxy.29.prn1:8080"
@@ -131,6 +130,35 @@ DISABLE_COMMANDS="[
     }
 ]"
 
+#
+# RocksDB unit test in parallel
+# Currently we always have noise in our parallel runs. This job is to help
+# manage the noise
+#
+PARALLEL_UNIT_TEST_COMMANDS="[
+    {
+        'name':'Rocksdb Parallel Unit Test',
+        'oncall':'$ONCALL',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Build and test RocksDB debug version',
+                'shell':'$DEBUG make -j$(nproc) all && $SHM make check > /dev/null 2>&1 || $CONTRUN_NAME=punit_check $TASK_CREATION_TOOL || cat t/log-*',
+                'user':'root',
+                $PARSER
+            },
+            $CLEANUP_ENV,
+            {
+                'name':'Build and test RocksDB debug version under gcc-4.8.1',
+                'shell':'$GCC_481 $DEBUG make -j$(nproc) all && $SHM make check > /dev/null 2>&1 || $CONTRUN_NAME=punit_check_gcc481 $TASK_CREATION_TOOL || cat t/log-*',
+                'user':'root',
+                $PARSER
+            },
+        ],
+        $REPORT
+    }
+]"
+
 #
 # RocksDB unit test
 #
@@ -344,7 +372,27 @@ LITE_BUILD_COMMANDS="[
             $CLEANUP_ENV,
             {
                 'name':'Build RocksDB debug version',
-                'shell':'$LITE make J=1 all check || $CONTRUN_NAME=lite $TASK_CREATION_TOOL',
+                'shell':'$LITE make J=1 static_lib || $CONTRUN_NAME=lite_static_lib $TASK_CREATION_TOOL',
+                'user':'root',
+                $PARSER
+            },
+        ],
+        $REPORT
+    }
+]"
+
+#
+# RocksDB lite tests
+#
+LITE_UNIT_TEST_COMMANDS="[
+    {
+        'name':'Rocksdb Lite Unit Test',
+        'oncall':'$ONCALL',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Build RocksDB debug version',
+                'shell':'$SHM $LITE make J=1 check || $CONTRUN_NAME=lite_check $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -545,7 +593,7 @@ TSAN_CRASH_TEST_COMMANDS="[
             {
                 'name':'Compile and run',
                 'timeout': 86400,
-                'shell':'set -o pipefail && $SHM $DEBUG $TSAN CRASH_TEST_KILL_ODD=1887 CRASH_TEST_EXT_ARGS=--log2_keys_per_lock=22  make J=1 crash_test || $CONTRUN_NAME=tsan_crash_test $TASK_CREATION_TOOL',
+                'shell':'set -o pipefail && $SHM $DEBUG $TSAN CRASH_TEST_KILL_ODD=1887 make J=1 crash_test || $CONTRUN_NAME=tsan_crash_test $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -564,7 +612,39 @@ run_format_compatible()
   rm -rf /dev/shm/rocksdb
   mkdir /dev/shm/rocksdb
 
-  tools/check_format_compatible.sh
+  echo '
+  if [ -e "build_tools/build_detect_platform" ]
+  then
+    sed "s/tcmalloc/nothingnothingnothing/g" build_tools/build_detect_platform > $TEST_TMPDIR/temp_build_file
+    rm -rf build_tools/build_detect_platform
+    cp $TEST_TMPDIR/temp_build_file build_tools/build_detect_platform
+    chmod +x build_tools/build_detect_platform
+  fi
+
+  if [ -e "build_detect_platform" ]
+  then
+    sed "s/tcmalloc/nothingnothingnothing/g" build_detect_platform > $TEST_TMPDIR/temp_build_file
+    rm -rf build_detect_platform
+    cp $TEST_TMPDIR/temp_build_file build_detect_platform
+    chmod +x build_detect_platform
+  fi
+
+  make ldb -j32
+
+  if [ -e "build_detect_platform" ]
+  then
+    git checkout -- build_detect_platform
+  fi
+
+  if [ -e "build_tools/build_detect_platform" ]
+  then
+    git checkout -- build_tools/build_detect_platform
+  fi
+  ' > temp_build_ldb.sh
+
+  sed "s/make ldb -j32/source temp_build_ldb.sh/g" tools/check_format_compatible.sh > tools/temp_check_format_compatible.sh
+  chmod +x tools/temp_check_format_compatible.sh
+  tools/temp_check_format_compatible.sh
 }
 
 FORMAT_COMPATIBLE_COMMANDS="[
@@ -696,6 +776,9 @@ JAVA_BUILD_TEST_COMMANDS="[
 
 
 case $1 in
+  punit)
+    echo $PARALLEL_UNIT_TEST_COMMANDS
+    ;;
   unit)
     echo $UNIT_TEST_COMMANDS
     ;;
@@ -729,6 +812,9 @@ case $1 in
   lite)
     echo $LITE_BUILD_COMMANDS
     ;;
+  lite_test)
+    echo $LITE_UNIT_TEST_COMMANDS
+    ;;
   stress_crash)
     echo $STRESS_CRASH_TEST_COMMANDS
     ;;

build_tools/update_dependencies.sh

@@ -52,45 +52,6 @@ function get_lib_base()
   log_variable $__res_var
 }
 
-###########################################################
-#                platform007 dependencies                 #
-###########################################################
-
-OUTPUT="$BASEDIR/dependencies_platform007.sh"
-
-rm -f "$OUTPUT"
-touch "$OUTPUT"
-
-echo "Writing dependencies to $OUTPUT"
-
-# Compilers locations
-GCC_BASE=`readlink -f $TP2_LATEST/gcc/7.x/centos7-native/*/`
-CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos7-native/*/`
-
-log_variable GCC_BASE
-log_variable CLANG_BASE
-
-# Libraries locations
-get_lib_base libgcc     7.x     platform007
-get_lib_base glibc      2.26    platform007
-get_lib_base snappy     LATEST  platform007
-get_lib_base zlib       LATEST  platform007
-get_lib_base bzip2      LATEST  platform007
-get_lib_base lz4        LATEST  platform007
-get_lib_base zstd       LATEST  platform007
-get_lib_base gflags     LATEST  platform007
-get_lib_base jemalloc   LATEST  platform007
-get_lib_base numa       LATEST  platform007
-get_lib_base libunwind  LATEST  platform007
-get_lib_base tbb        LATEST  platform007
-
-get_lib_base kernel-headers fb platform007
-get_lib_base binutils   LATEST centos7-native
-get_lib_base valgrind   LATEST platform007
-get_lib_base lua        5.3.4  platform007
-
-git diff $OUTPUT
-
 ###########################################################
 #                   5.x dependencies                      #
 ###########################################################

cache/clock_cache.cc

@@ -27,11 +27,6 @@ std::shared_ptr<Cache> NewClockCache(size_t capacity, int num_shard_bits,
 #include <atomic>
 #include <deque>
 
-// "tbb/concurrent_hash_map.h" requires RTTI if exception is enabled.
-// Disable it so users can chooose to disable RTTI.
-#ifndef ROCKSDB_USE_RTTI
-#define TBB_USE_EXCEPTIONS 0
-#endif
 #include "tbb/concurrent_hash_map.h"
 
 #include "cache/sharded_cache.h"

cache/lru_cache.cc

@@ -22,7 +22,7 @@
 
 namespace rocksdb {
 
-LRUHandleTable::LRUHandleTable() : list_(nullptr), length_(0), elems_(0) {
+LRUHandleTable::LRUHandleTable() : length_(0), elems_(0), list_(nullptr) {
   Resize();
 }
 
@@ -100,7 +100,7 @@ void LRUHandleTable::Resize() {
 }
 
 LRUCacheShard::LRUCacheShard()
-    : high_pri_pool_usage_(0), usage_(0), lru_usage_(0) {
+    : usage_(0), lru_usage_(0), high_pri_pool_usage_(0) {
   // Make empty circular linked list
   lru_.next = &lru_;
   lru_.prev = &lru_;
@@ -157,16 +157,6 @@ void LRUCacheShard::TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri) {
   *lru_low_pri = lru_low_pri_;
 }
 
-size_t LRUCacheShard::TEST_GetLRUSize() {
-  LRUHandle* lru_handle = lru_.next;
-  size_t lru_size = 0;
-  while (lru_handle != &lru_) {
-    lru_size++;
-    lru_handle = lru_handle->next;
-  }
-  return lru_size;
-}
-
 void LRUCacheShard::LRU_Remove(LRUHandle* e) {
   assert(e->next != nullptr);
   assert(e->prev != nullptr);
@@ -233,22 +223,6 @@ void LRUCacheShard::EvictFromLRU(size_t charge,
   }
 }
 
-void* LRUCacheShard::operator new(size_t size) {
-  return port::cacheline_aligned_alloc(size);
-}
-
-void* LRUCacheShard::operator new[](size_t size) {
-  return port::cacheline_aligned_alloc(size);
-}
-
-void LRUCacheShard::operator delete(void *memblock) {
-  port::cacheline_aligned_free(memblock);
-}
-
-void LRUCacheShard::operator delete[](void* memblock) {
-  port::cacheline_aligned_free(memblock);
-}
-
 void LRUCacheShard::SetCapacity(size_t capacity) {
   autovector<LRUHandle*> last_reference_list;
   {
@@ -464,11 +438,11 @@ std::string LRUCacheShard::GetPrintableOptions() const {
 LRUCache::LRUCache(size_t capacity, int num_shard_bits,
                    bool strict_capacity_limit, double high_pri_pool_ratio)
     : ShardedCache(capacity, num_shard_bits, strict_capacity_limit) {
-  num_shards_ = 1 << num_shard_bits;
-  shards_ = new LRUCacheShard[num_shards_];
+  int num_shards = 1 << num_shard_bits;
+  shards_ = new LRUCacheShard[num_shards];
   SetCapacity(capacity);
   SetStrictCapacityLimit(strict_capacity_limit);
-  for (int i = 0; i < num_shards_; i++) {
+  for (int i = 0; i < num_shards; i++) {
     shards_[i].SetHighPriorityPoolRatio(high_pri_pool_ratio);
   }
 }
@@ -495,20 +469,7 @@ uint32_t LRUCache::GetHash(Handle* handle) const {
   return reinterpret_cast<const LRUHandle*>(handle)->hash;
 }
 
-void LRUCache::DisownData() {
-// Do not drop data if compile with ASAN to suppress leak warning.
-#ifndef __SANITIZE_ADDRESS__
-  shards_ = nullptr;
-#endif  // !__SANITIZE_ADDRESS__
-}
-
-size_t LRUCache::TEST_GetLRUSize() {
-  size_t lru_size_of_all_shards = 0;
-  for (int i = 0; i < num_shards_; i++) {
-    lru_size_of_all_shards += shards_[i].TEST_GetLRUSize();
-  }
-  return lru_size_of_all_shards;
-}
+void LRUCache::DisownData() { shards_ = nullptr; }
 
 std::shared_ptr<Cache> NewLRUCache(size_t capacity, int num_shard_bits,
                                    bool strict_capacity_limit,

cache/lru_cache.h

@@ -148,13 +148,13 @@ class LRUHandleTable {
 
   // The table consists of an array of buckets where each bucket is
   // a linked list of cache entries that hash into the bucket.
-  LRUHandle** list_;
   uint32_t length_;
   uint32_t elems_;
+  LRUHandle** list_;
 };
 
 // A single shard of sharded cache.
-class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard : public CacheShard {
+class LRUCacheShard : public CacheShard {
  public:
   LRUCacheShard();
   virtual ~LRUCacheShard();
@@ -198,19 +198,6 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard : public CacheShard {
 
   void TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri);
 
-  //  Retrieves number of elements in LRU, for unit test purpose only
-  //  not threadsafe
-  size_t TEST_GetLRUSize();
-
-  // Overloading to aligned it to cache line size
-  void* operator new(size_t);
-
-  void* operator new[](size_t);
-
-  void operator delete(void *);
-
-  void operator delete[](void*);
-
  private:
   void LRU_Remove(LRUHandle* e);
   void LRU_Insert(LRUHandle* e);
@@ -232,6 +219,12 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard : public CacheShard {
   // Initialized before use.
   size_t capacity_;
 
+  // Memory size for entries residing in the cache
+  size_t usage_;
+
+  // Memory size for entries residing only in the LRU list
+  size_t lru_usage_;
+
   // Memory size for entries in high-pri pool.
   size_t high_pri_pool_usage_;
 
@@ -245,6 +238,11 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard : public CacheShard {
   // Remember the value to avoid recomputing each time.
   double high_pri_pool_capacity_;
 
+  // mutex_ protects the following state.
+  // We don't count mutex_ as the cache's internal state so semantically we
+  // don't mind mutex_ invoking the non-const actions.
+  mutable port::Mutex mutex_;
+
   // Dummy head of LRU list.
   // lru.prev is newest entry, lru.next is oldest entry.
   // LRU contains items which can be evicted, ie reference only by cache
@@ -253,29 +251,7 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard : public CacheShard {
   // Pointer to head of low-pri pool in LRU list.
   LRUHandle* lru_low_pri_;
 
-  // ------------^^^^^^^^^^^^^-----------
-  // Not frequently modified data members
-  // ------------------------------------
-  //
-  // We separate data members that are updated frequently from the ones that
-  // are not frequently updated so that they don't share the same cache line
-  // which will lead into false cache sharing
-  //
-  // ------------------------------------
-  // Frequently modified data members
-  // ------------vvvvvvvvvvvvv-----------
   LRUHandleTable table_;
-
-  // Memory size for entries residing in the cache
-  size_t usage_;
-
-  // Memory size for entries residing only in the LRU list
-  size_t lru_usage_;
-
-  // mutex_ protects the following state.
-  // We don't count mutex_ as the cache's internal state so semantically we
-  // don't mind mutex_ invoking the non-const actions.
-  mutable port::Mutex mutex_;
 };
 
 class LRUCache : public ShardedCache {
@@ -291,12 +267,8 @@ class LRUCache : public ShardedCache {
   virtual uint32_t GetHash(Handle* handle) const override;
   virtual void DisownData() override;
 
-  //  Retrieves number of elements in LRU, for unit test purpose only
-  size_t TEST_GetLRUSize();
-
  private:
   LRUCacheShard* shards_;
-  int num_shards_ = 0;
 };
 
 }  // namespace rocksdb

cache/lru_cache_test.cc

@@ -17,16 +17,7 @@ class LRUCacheTest : public testing::Test {
   ~LRUCacheTest() {}
 
   void NewCache(size_t capacity, double high_pri_pool_ratio = 0.0) {
-    cache_.reset(
-#if defined(_MSC_VER)
-#pragma warning(push)
-#pragma warning(disable: 4316) // We've validated the alignment with the new operators
-#endif
-      new LRUCacheShard()
-#if defined(_MSC_VER)
-#pragma warning(pop)
-#endif
-    );
+    cache_.reset(new LRUCacheShard());
     cache_->SetCapacity(capacity);
     cache_->SetStrictCapacityLimit(false);
     cache_->SetHighPriorityPoolRatio(high_pri_pool_ratio);

cmake/RocksDBConfig.cmake.in

@@ -1,3 +0,0 @@
-@PACKAGE_INIT@
-include("${CMAKE_CURRENT_LIST_DIR}/RocksDBTargets.cmake")
-check_required_components(RocksDB)

cmake/modules/Findbzip2.cmake

@@ -1,21 +0,0 @@
-# - Find Bzip2
-# Find the bzip2 compression library and includes
-#
-# BZIP2_INCLUDE_DIR - where to find bzlib.h, etc.
-# BZIP2_LIBRARIES - List of libraries when using bzip2.
-# BZIP2_FOUND - True if bzip2 found.
-
-find_path(BZIP2_INCLUDE_DIR
-  NAMES bzlib.h
-  HINTS ${BZIP2_ROOT_DIR}/include)
-
-find_library(BZIP2_LIBRARIES
-  NAMES bz2
-  HINTS ${BZIP2_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(bzip2 DEFAULT_MSG BZIP2_LIBRARIES BZIP2_INCLUDE_DIR)
-
-mark_as_advanced(
-  BZIP2_LIBRARIES
-  BZIP2_INCLUDE_DIR)

cmake/modules/Findlz4.cmake

@@ -1,21 +0,0 @@
-# - Find Lz4
-# Find the lz4 compression library and includes
-#
-# LZ4_INCLUDE_DIR - where to find lz4.h, etc.
-# LZ4_LIBRARIES - List of libraries when using lz4.
-# LZ4_FOUND - True if lz4 found.
-
-find_path(LZ4_INCLUDE_DIR
-  NAMES lz4.h
-  HINTS ${LZ4_ROOT_DIR}/include)
-
-find_library(LZ4_LIBRARIES
-  NAMES lz4
-  HINTS ${LZ4_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(lz4 DEFAULT_MSG LZ4_LIBRARIES LZ4_INCLUDE_DIR)
-
-mark_as_advanced(
-  LZ4_LIBRARIES
-  LZ4_INCLUDE_DIR)

cmake/modules/Findzlib.cmake

@@ -1,21 +0,0 @@
-# - Find zlib
-# Find the zlib compression library and includes
-#
-# ZLIB_INCLUDE_DIR - where to find zlib.h, etc.
-# ZLIB_LIBRARIES - List of libraries when using zlib.
-# ZLIB_FOUND - True if zlib found.
-
-find_path(ZLIB_INCLUDE_DIR
-  NAMES zlib.h
-  HINTS ${ZLIB_ROOT_DIR}/include)
-
-find_library(ZLIB_LIBRARIES
-  NAMES z
-  HINTS ${ZLIB_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(zlib DEFAULT_MSG ZLIB_LIBRARIES ZLIB_INCLUDE_DIR)
-
-mark_as_advanced(
-  ZLIB_LIBRARIES
-  ZLIB_INCLUDE_DIR)

cmake/modules/Findzstd.cmake

@@ -1,21 +0,0 @@
-# - Find zstd
-# Find the zstd compression library and includes
-#
-# ZSTD_INCLUDE_DIR - where to find zstd.h, etc.
-# ZSTD_LIBRARIES - List of libraries when using zstd.
-# ZSTD_FOUND - True if zstd found.
-
-find_path(ZSTD_INCLUDE_DIR
-  NAMES zstd.h
-  HINTS ${ZSTD_ROOT_DIR}/include)
-
-find_library(ZSTD_LIBRARIES
-  NAMES zstd
-  HINTS ${ZSTD_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(zstd DEFAULT_MSG ZSTD_LIBRARIES ZSTD_INCLUDE_DIR)
-
-mark_as_advanced(
-  ZSTD_LIBRARIES
-  ZSTD_INCLUDE_DIR)

coverage/coverage_test.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 # Exit on error.
 set -e

db/builder.cc

@@ -45,17 +45,17 @@ TableBuilder* NewTableBuilder(
         int_tbl_prop_collector_factories,
     uint32_t column_family_id, const std::string& column_family_name,
     WritableFileWriter* file, const CompressionType compression_type,
-    const CompressionOptions& compression_opts, int level,
-    const std::string* compression_dict, const bool skip_filters,
-    const uint64_t creation_time, const uint64_t oldest_key_time) {
+    const CompressionOptions& compression_opts,
+    int level,
+    const std::string* compression_dict, const bool skip_filters) {
   assert((column_family_id ==
           TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
          column_family_name.empty());
   return ioptions.table_factory->NewTableBuilder(
-      TableBuilderOptions(
-          ioptions, internal_comparator, int_tbl_prop_collector_factories,
-          compression_type, compression_opts, compression_dict, skip_filters,
-          column_family_name, level, creation_time, oldest_key_time),
+      TableBuilderOptions(ioptions, internal_comparator,
+                          int_tbl_prop_collector_factories, compression_type,
+                          compression_opts, compression_dict, skip_filters,
+                          column_family_name, level),
       column_family_id, file);
 }
 
@@ -74,8 +74,7 @@ Status BuildTable(
     const CompressionOptions& compression_opts, bool paranoid_file_checks,
     InternalStats* internal_stats, TableFileCreationReason reason,
     EventLogger* event_logger, int job_id, const Env::IOPriority io_priority,
-    TableProperties* table_properties, int level, const uint64_t creation_time,
-    const uint64_t oldest_key_time) {
+    TableProperties* table_properties, int level) {
   assert((column_family_id ==
           TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
          column_family_name.empty());
@@ -120,11 +119,11 @@ Status BuildTable(
 
       file_writer.reset(new WritableFileWriter(std::move(file), env_options,
                                                ioptions.statistics));
+
       builder = NewTableBuilder(
           ioptions, internal_comparator, int_tbl_prop_collector_factories,
           column_family_id, column_family_name, file_writer.get(), compression,
-          compression_opts, level, nullptr /* compression_dict */,
-          false /* skip_filters */, creation_time, oldest_key_time);
+          compression_opts, level);
     }
 
     MergeHelper merge(env, internal_comparator.user_comparator(),

db/builder.h

@@ -48,10 +48,10 @@ TableBuilder* NewTableBuilder(
         int_tbl_prop_collector_factories,
     uint32_t column_family_id, const std::string& column_family_name,
     WritableFileWriter* file, const CompressionType compression_type,
-    const CompressionOptions& compression_opts, int level,
+    const CompressionOptions& compression_opts,
+    int level,
     const std::string* compression_dict = nullptr,
-    const bool skip_filters = false, const uint64_t creation_time = 0,
-    const uint64_t oldest_key_time = 0);
+    const bool skip_filters = false);
 
 // Build a Table file from the contents of *iter.  The generated file
 // will be named according to number specified in meta. On success, the rest of
@@ -77,7 +77,6 @@ extern Status BuildTable(
     InternalStats* internal_stats, TableFileCreationReason reason,
     EventLogger* event_logger = nullptr, int job_id = 0,
     const Env::IOPriority io_priority = Env::IO_HIGH,
-    TableProperties* table_properties = nullptr, int level = -1,
-    const uint64_t creation_time = 0, const uint64_t oldest_key_time = 0);
+    TableProperties* table_properties = nullptr, int level = -1);
 
 }  // namespace rocksdb

db/c.cc

@@ -36,7 +36,6 @@
 #include "utilities/merge_operators.h"
 #include "rocksdb/utilities/transaction.h"
 #include "rocksdb/utilities/transaction_db.h"
-#include "rocksdb/utilities/optimistic_transaction_db.h"
 #include "rocksdb/utilities/checkpoint.h"
 
 using rocksdb::BytewiseComparator;
@@ -53,7 +52,6 @@ using rocksdb::CompressionType;
 using rocksdb::WALRecoveryMode;
 using rocksdb::DB;
 using rocksdb::DBOptions;
-using rocksdb::DbPath;
 using rocksdb::Env;
 using rocksdb::EnvOptions;
 using rocksdb::InfoLogLevel;
@@ -96,8 +94,6 @@ using rocksdb::PinnableSlice;
 using rocksdb::TransactionDBOptions;
 using rocksdb::TransactionDB;
 using rocksdb::TransactionOptions;
-using rocksdb::OptimisticTransactionDB;
-using rocksdb::OptimisticTransactionOptions;
 using rocksdb::Transaction;
 using rocksdb::Checkpoint;
 
@@ -156,12 +152,6 @@ struct rocksdb_transaction_t {
 struct rocksdb_checkpoint_t {
   Checkpoint* rep;
 };
-struct rocksdb_optimistictransactiondb_t {
-  OptimisticTransactionDB* rep;
-};
-struct rocksdb_optimistictransaction_options_t {
-  OptimisticTransactionOptions rep;
-};
 
 struct rocksdb_compactionfiltercontext_t {
   CompactionFilter::Context rep;
@@ -392,10 +382,6 @@ struct rocksdb_mergeoperator_t : public MergeOperator {
   }
 };
 
-struct rocksdb_dbpath_t {
-  DbPath rep;
-};
-
 struct rocksdb_env_t {
   Env* rep;
   bool is_default;
@@ -1386,24 +1372,23 @@ void rocksdb_writebatch_put_log_data(
   b->rep.PutLogData(Slice(blob, len));
 }
 
-class H : public WriteBatch::Handler {
- public:
-  void* state_;
-  void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
-  void (*deleted_)(void*, const char* k, size_t klen);
-  virtual void Put(const Slice& key, const Slice& value) override {
-    (*put_)(state_, key.data(), key.size(), value.data(), value.size());
-  }
-  virtual void Delete(const Slice& key) override {
-    (*deleted_)(state_, key.data(), key.size());
-  }
-};
-
 void rocksdb_writebatch_iterate(
     rocksdb_writebatch_t* b,
     void* state,
     void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen),
     void (*deleted)(void*, const char* k, size_t klen)) {
+  class H : public WriteBatch::Handler {
+   public:
+    void* state_;
+    void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
+    void (*deleted_)(void*, const char* k, size_t klen);
+    virtual void Put(const Slice& key, const Slice& value) override {
+      (*put_)(state_, key.data(), key.size(), value.data(), value.size());
+    }
+    virtual void Delete(const Slice& key) override {
+      (*deleted_)(state_, key.data(), key.size());
+    }
+  };
   H handler;
   handler.state_ = state;
   handler.put_ = put;
@@ -1648,6 +1633,18 @@ void rocksdb_writebatch_wi_iterate(
     void* state,
     void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen),
     void (*deleted)(void*, const char* k, size_t klen)) {
+  class H : public WriteBatch::Handler {
+   public:
+    void* state_;
+    void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
+    void (*deleted_)(void*, const char* k, size_t klen);
+    virtual void Put(const Slice& key, const Slice& value) override {
+      (*put_)(state_, key.data(), key.size(), value.data(), value.size());
+    }
+    virtual void Delete(const Slice& key) override {
+      (*deleted_)(state_, key.data(), key.size());
+    }
+  };
   H handler;
   handler.state_ = state;
   handler.put_ = put;
@@ -2012,16 +2009,6 @@ void rocksdb_options_set_paranoid_checks(
   opt->rep.paranoid_checks = v;
 }
 
-void rocksdb_options_set_db_paths(rocksdb_options_t* opt, 
-                                  const rocksdb_dbpath_t** dbpath_values, 
-                                  size_t num_paths) {
-  std::vector<DbPath> db_paths(num_paths);
-  for (size_t i = 0; i < num_paths; ++i) {
-    db_paths[i] = dbpath_values[i]->rep;
-  }
-  opt->rep.db_paths = db_paths;
-}
-
 void rocksdb_options_set_env(rocksdb_options_t* opt, rocksdb_env_t* env) {
   opt->rep.env = (env ? env->rep : nullptr);
 }
@@ -2100,10 +2087,6 @@ void rocksdb_options_enable_statistics(rocksdb_options_t* opt) {
   opt->rep.statistics = rocksdb::CreateDBStatistics();
 }
 
-void rocksdb_options_set_skip_stats_update_on_db_open(rocksdb_options_t* opt, unsigned char val) {
-  opt->rep.skip_stats_update_on_db_open = val;
-}
-
 void rocksdb_options_set_num_levels(rocksdb_options_t* opt, int n) {
   opt->rep.num_levels = n;
 }
@@ -2806,17 +2789,6 @@ size_t rocksdb_cache_get_pinned_usage(rocksdb_cache_t* cache) {
   return cache->rep->GetPinnedUsage();
 }
 
-rocksdb_dbpath_t* rocksdb_dbpath_create(const char* path, uint64_t target_size) {
-  rocksdb_dbpath_t* result = new rocksdb_dbpath_t;
-  result->rep.path = std::string(path);
-  result->rep.target_size = target_size;
-  return result;
-}
-
-void rocksdb_dbpath_destroy(rocksdb_dbpath_t* dbpath) {
-  delete dbpath;
-}
-
 rocksdb_env_t* rocksdb_create_default_env() {
   rocksdb_env_t* result = new rocksdb_env_t;
   result->rep = Env::Default();
@@ -2994,21 +2966,20 @@ void rocksdb_slicetransform_destroy(rocksdb_slicetransform_t* st) {
   delete st;
 }
 
-struct Wrapper : public rocksdb_slicetransform_t {
-  const SliceTransform* rep_;
-  ~Wrapper() { delete rep_; }
-  const char* Name() const override { return rep_->Name(); }
-  Slice Transform(const Slice& src) const override {
-    return rep_->Transform(src);
-  }
-  bool InDomain(const Slice& src) const override {
-    return rep_->InDomain(src);
-  }
-  bool InRange(const Slice& src) const override { return rep_->InRange(src); }
-  static void DoNothing(void*) { }
-};
-
 rocksdb_slicetransform_t* rocksdb_slicetransform_create_fixed_prefix(size_t prefixLen) {
+  struct Wrapper : public rocksdb_slicetransform_t {
+    const SliceTransform* rep_;
+    ~Wrapper() { delete rep_; }
+    const char* Name() const override { return rep_->Name(); }
+    Slice Transform(const Slice& src) const override {
+      return rep_->Transform(src);
+    }
+    bool InDomain(const Slice& src) const override {
+      return rep_->InDomain(src);
+    }
+    bool InRange(const Slice& src) const override { return rep_->InRange(src); }
+    static void DoNothing(void*) { }
+  };
   Wrapper* wrapper = new Wrapper;
   wrapper->rep_ = rocksdb::NewFixedPrefixTransform(prefixLen);
   wrapper->state_ = nullptr;
@@ -3017,6 +2988,19 @@ rocksdb_slicetransform_t* rocksdb_slicetransform_create_fixed_prefix(size_t pref
 }
 
 rocksdb_slicetransform_t* rocksdb_slicetransform_create_noop() {
+  struct Wrapper : public rocksdb_slicetransform_t {
+    const SliceTransform* rep_;
+    ~Wrapper() { delete rep_; }
+    const char* Name() const override { return rep_->Name(); }
+    Slice Transform(const Slice& src) const override {
+      return rep_->Transform(src);
+    }
+    bool InDomain(const Slice& src) const override {
+      return rep_->InDomain(src);
+    }
+    bool InRange(const Slice& src) const override { return rep_->InRange(src); }
+    static void DoNothing(void*) { }
+  };
   Wrapper* wrapper = new Wrapper;
   wrapper->rep_ = rocksdb::NewNoopTransform();
   wrapper->state_ = nullptr;
@@ -3239,32 +3223,6 @@ void rocksdb_transaction_options_set_max_write_batch_size(
   opt->rep.max_write_batch_size = size;
 }
 
-rocksdb_optimistictransaction_options_t*
-rocksdb_optimistictransaction_options_create() {
-  return new rocksdb_optimistictransaction_options_t;
-}
-
-void rocksdb_optimistictransaction_options_destroy(
-    rocksdb_optimistictransaction_options_t* opt) {
-  delete opt;
-}
-
-void rocksdb_optimistictransaction_options_set_set_snapshot(
-    rocksdb_optimistictransaction_options_t* opt, unsigned char v) {
-  opt->rep.set_snapshot = v;
-}
-
-rocksdb_column_family_handle_t* rocksdb_transactiondb_create_column_family(
-    rocksdb_transactiondb_t* txn_db,
-    const rocksdb_options_t* column_family_options,
-    const char* column_family_name, char** errptr) {
-  rocksdb_column_family_handle_t* handle = new rocksdb_column_family_handle_t;
-  SaveError(errptr, txn_db->rep->CreateColumnFamily(
-                        ColumnFamilyOptions(column_family_options->rep),
-                        std::string(column_family_name), &(handle->rep)));
-  return handle;
-}
-
 rocksdb_transactiondb_t* rocksdb_transactiondb_open(
     const rocksdb_options_t* options,
     const rocksdb_transactiondb_options_t* txn_db_options, const char* name,
@@ -3321,14 +3279,7 @@ void rocksdb_transaction_destroy(rocksdb_transaction_t* txn) {
   delete txn;
 }
 
-const rocksdb_snapshot_t* rocksdb_transaction_get_snapshot(
-    rocksdb_transaction_t* txn) {
-  rocksdb_snapshot_t* result = new rocksdb_snapshot_t;
-  result->rep = txn->rep->GetSnapshot();
-  return result;
-}
-
-// Read a key inside a transaction
+//Read a key inside a transaction
 char* rocksdb_transaction_get(rocksdb_transaction_t* txn,
                               const rocksdb_readoptions_t* options,
                               const char* key, size_t klen, size_t* vlen,
@@ -3348,49 +3299,6 @@ char* rocksdb_transaction_get(rocksdb_transaction_t* txn,
   return result;
 }
 
-char* rocksdb_transaction_get_cf(rocksdb_transaction_t* txn,
-                                 const rocksdb_readoptions_t* options,
-                                 rocksdb_column_family_handle_t* column_family,
-                                 const char* key, size_t klen, size_t* vlen,
-                                 char** errptr) {
-  char* result = nullptr;
-  std::string tmp;
-  Status s =
-      txn->rep->Get(options->rep, column_family->rep, Slice(key, klen), &tmp);
-  if (s.ok()) {
-    *vlen = tmp.size();
-    result = CopyString(tmp);
-  } else {
-    *vlen = 0;
-    if (!s.IsNotFound()) {
-      SaveError(errptr, s);
-    }
-  }
-  return result;
-}
-
-// Read a key inside a transaction
-char* rocksdb_transaction_get_for_update(rocksdb_transaction_t* txn,
-                                         const rocksdb_readoptions_t* options,
-                                         const char* key, size_t klen,
-                                         size_t* vlen, unsigned char exclusive,
-                                         char** errptr) {
-  char* result = nullptr;
-  std::string tmp;
-  Status s =
-      txn->rep->GetForUpdate(options->rep, Slice(key, klen), &tmp, exclusive);
-  if (s.ok()) {
-    *vlen = tmp.size();
-    result = CopyString(tmp);
-  } else {
-    *vlen = 0;
-    if (!s.IsNotFound()) {
-      SaveError(errptr, s);
-    }
-  }
-  return result;
-}
-
 // Read a key outside a transaction
 char* rocksdb_transactiondb_get(
     rocksdb_transactiondb_t* txn_db,
@@ -3413,26 +3321,6 @@ char* rocksdb_transactiondb_get(
   return result;
 }
 
-char* rocksdb_transactiondb_get_cf(
-    rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options,
-    rocksdb_column_family_handle_t* column_family, const char* key,
-    size_t keylen, size_t* vallen, char** errptr) {
-  char* result = nullptr;
-  std::string tmp;
-  Status s = txn_db->rep->Get(options->rep, column_family->rep,
-                              Slice(key, keylen), &tmp);
-  if (s.ok()) {
-    *vallen = tmp.size();
-    result = CopyString(tmp);
-  } else {
-    *vallen = 0;
-    if (!s.IsNotFound()) {
-      SaveError(errptr, s);
-    }
-  }
-  return result;
-}
-
 // Put a key inside a transaction
 void rocksdb_transaction_put(rocksdb_transaction_t* txn, const char* key,
                              size_t klen, const char* val, size_t vlen,
@@ -3440,56 +3328,13 @@ void rocksdb_transaction_put(rocksdb_transaction_t* txn, const char* key,
   SaveError(errptr, txn->rep->Put(Slice(key, klen), Slice(val, vlen)));
 }
 
-void rocksdb_transaction_put_cf(rocksdb_transaction_t* txn,
-                                rocksdb_column_family_handle_t* column_family,
-                                const char* key, size_t klen, const char* val,
-                                size_t vlen, char** errptr) {
-  SaveError(errptr, txn->rep->Put(column_family->rep, Slice(key, klen),
-                                  Slice(val, vlen)));
-}
-
-// Put a key outside a transaction
+//Put a key outside a transaction
 void rocksdb_transactiondb_put(rocksdb_transactiondb_t* txn_db,
                                const rocksdb_writeoptions_t* options,
                                const char* key, size_t klen, const char* val,
                                size_t vlen, char** errptr) {
-  SaveError(errptr, txn_db->rep->Put(options->rep, Slice(key, klen), 
-                                     Slice(val, vlen)));
-}
-
-void rocksdb_transactiondb_put_cf(rocksdb_transactiondb_t* txn_db,
-                                  const rocksdb_writeoptions_t* options,
-                                  rocksdb_column_family_handle_t* column_family,
-                                  const char* key, size_t keylen,
-                                  const char* val, size_t vallen,
-                                  char** errptr) {
-  SaveError(errptr, txn_db->rep->Put(options->rep, column_family->rep,
-                                     Slice(key, keylen), Slice(val, vallen)));
-}
-
-// Write batch into transaction db
-void rocksdb_transactiondb_write(
-        rocksdb_transactiondb_t* db,
-        const rocksdb_writeoptions_t* options,
-        rocksdb_writebatch_t* batch,
-        char** errptr) {
-  SaveError(errptr, db->rep->Write(options->rep, &batch->rep));
-}
-
-// Merge a key inside a transaction
-void rocksdb_transaction_merge(rocksdb_transaction_t* txn, const char* key,
-                               size_t klen, const char* val, size_t vlen,
-                               char** errptr) {
-  SaveError(errptr, txn->rep->Merge(Slice(key, klen), Slice(val, vlen)));
-}
-
-// Merge a key outside a transaction
-void rocksdb_transactiondb_merge(rocksdb_transactiondb_t* txn_db,
-                                 const rocksdb_writeoptions_t* options,
-                                 const char* key, size_t klen, const char* val,
-                                 size_t vlen, char** errptr) {
   SaveError(errptr,
-    txn_db->rep->Merge(options->rep, Slice(key, klen), Slice(val, vlen)));
+            txn_db->rep->Put(options->rep, Slice(key, klen), Slice(val, vlen)));
 }
 
 // Delete a key inside a transaction
@@ -3498,12 +3343,6 @@ void rocksdb_transaction_delete(rocksdb_transaction_t* txn, const char* key,
   SaveError(errptr, txn->rep->Delete(Slice(key, klen)));
 }
 
-void rocksdb_transaction_delete_cf(
-    rocksdb_transaction_t* txn, rocksdb_column_family_handle_t* column_family,
-    const char* key, size_t klen, char** errptr) {
-  SaveError(errptr, txn->rep->Delete(column_family->rep, Slice(key, klen)));
-}
-
 // Delete a key outside a transaction
 void rocksdb_transactiondb_delete(rocksdb_transactiondb_t* txn_db,
                                   const rocksdb_writeoptions_t* options,
@@ -3511,14 +3350,6 @@ void rocksdb_transactiondb_delete(rocksdb_transactiondb_t* txn_db,
   SaveError(errptr, txn_db->rep->Delete(options->rep, Slice(key, klen)));
 }
 
-void rocksdb_transactiondb_delete_cf(
-    rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options,
-    rocksdb_column_family_handle_t* column_family, const char* key,
-    size_t keylen, char** errptr) {
-  SaveError(errptr, txn_db->rep->Delete(options->rep, column_family->rep,
-                                        Slice(key, keylen)));
-}
-
 // Create an iterator inside a transaction
 rocksdb_iterator_t* rocksdb_transaction_create_iterator(
     rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options) {
@@ -3527,14 +3358,6 @@ rocksdb_iterator_t* rocksdb_transaction_create_iterator(
   return result;
 }
 
-// Create an iterator outside a transaction
-rocksdb_iterator_t* rocksdb_transactiondb_create_iterator(
-    rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options) {
-  rocksdb_iterator_t* result = new rocksdb_iterator_t;
-  result->rep = txn_db->rep->NewIterator(options->rep);
-  return result;
-}
-
 void rocksdb_transactiondb_close(rocksdb_transactiondb_t* txn_db) {
   delete txn_db->rep;
   delete txn_db;
@@ -3551,42 +3374,6 @@ rocksdb_checkpoint_t* rocksdb_transactiondb_checkpoint_object_create(
   return result;
 }
 
-rocksdb_optimistictransactiondb_t* rocksdb_optimistictransactiondb_open(
-    const rocksdb_options_t* options, const char* name,
-    char** errptr) {
-  OptimisticTransactionDB* otxn_db;
-  if (SaveError(errptr, OptimisticTransactionDB::Open(
-                            options->rep, std::string(name), &otxn_db))) {
-    return nullptr;
-  }
-  rocksdb_optimistictransactiondb_t* result =
-      new rocksdb_optimistictransactiondb_t;
-  result->rep = otxn_db;
-  return result;
-}
-
-rocksdb_transaction_t* rocksdb_optimistictransaction_begin(
-    rocksdb_optimistictransactiondb_t* otxn_db,
-    const rocksdb_writeoptions_t* write_options,
-    const rocksdb_optimistictransaction_options_t* otxn_options,
-    rocksdb_transaction_t* old_txn) {
-  if (old_txn == nullptr) {
-    rocksdb_transaction_t* result = new rocksdb_transaction_t;
-    result->rep = otxn_db->rep->BeginTransaction(write_options->rep,
-                                                 otxn_options->rep, nullptr);
-    return result;
-  }
-  old_txn->rep = otxn_db->rep->BeginTransaction(
-      write_options->rep, otxn_options->rep, old_txn->rep);
-  return old_txn;
-}
-
-void rocksdb_optimistictransactiondb_close(
-    rocksdb_optimistictransactiondb_t* otxn_db) {
-  delete otxn_db->rep;
-  delete otxn_db;
-}
-
 void rocksdb_free(void* ptr) { free(ptr); }
 
 rocksdb_pinnableslice_t* rocksdb_get_pinned(

db/c_test.c

@@ -41,7 +41,6 @@ static char dbname[200];
 static char sstfilename[200];
 static char dbbackupname[200];
 static char dbcheckpointname[200];
-static char dbpathname[200];
 
 static void StartPhase(const char* name) {
   fprintf(stderr, "=== Test %s\n", name);
@@ -348,25 +347,10 @@ static void CheckTxnDBGet(
         Free(&val);
 }
 
-static void CheckTxnDBGetCF(rocksdb_transactiondb_t* txn_db,
-                            const rocksdb_readoptions_t* options,
-                            rocksdb_column_family_handle_t* column_family,
-                            const char* key, const char* expected) {
-  char* err = NULL;
-  size_t val_len;
-  char* val;
-  val = rocksdb_transactiondb_get_cf(txn_db, options, column_family, key,
-                                     strlen(key), &val_len, &err);
-  CheckNoError(err);
-  CheckEqual(expected, val, val_len);
-  Free(&val);
-}
-
 int main(int argc, char** argv) {
   rocksdb_t* db;
   rocksdb_comparator_t* cmp;
   rocksdb_cache_t* cache;
-  rocksdb_dbpath_t *dbpath;
   rocksdb_env_t* env;
   rocksdb_options_t* options;
   rocksdb_compactoptions_t* coptions;
@@ -401,14 +385,8 @@ int main(int argc, char** argv) {
            GetTempDir(),
            ((int)geteuid()));
 
-  snprintf(dbpathname, sizeof(dbpathname),
-           "%s/rocksdb_c_test-%d-dbpath",
-           GetTempDir(),
-           ((int) geteuid()));
-
   StartPhase("create_objects");
   cmp = rocksdb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
-  dbpath = rocksdb_dbpath_create(dbpathname, 1024 * 1024);
   env = rocksdb_create_default_env();
   cache = rocksdb_cache_create_lru(100000);
 
@@ -1379,18 +1357,6 @@ int main(int argc, char** argv) {
     CheckNoError(err);
     CheckTxnDBGet(txn_db, roptions, "foo", NULL);
 
-    // write batch into TransactionDB
-    rocksdb_writebatch_t* wb = rocksdb_writebatch_create();
-    rocksdb_writebatch_put(wb, "foo", 3, "a", 1);
-    rocksdb_writebatch_clear(wb);
-    rocksdb_writebatch_put(wb, "bar", 3, "b", 1);
-    rocksdb_writebatch_put(wb, "box", 3, "c", 1);
-    rocksdb_writebatch_delete(wb, "bar", 3);
-    rocksdb_transactiondb_write(txn_db, woptions, wb, &err);
-    rocksdb_writebatch_destroy(wb);
-    CheckTxnDBGet(txn_db, roptions, "box", "c");
-    CheckNoError(err);
-
     // begin a transaction
     txn = rocksdb_transaction_begin(txn_db, woptions, txn_options, NULL);
     // put
@@ -1447,23 +1413,6 @@ int main(int argc, char** argv) {
     CheckNoError(err);
     CheckTxnDBGet(txn_db, roptions, "bar", NULL);
 
-    // Column families.
-    rocksdb_column_family_handle_t* cfh;
-    cfh = rocksdb_transactiondb_create_column_family(txn_db, options,
-                                                     "txn_db_cf", &err);
-    CheckNoError(err);
-
-    rocksdb_transactiondb_put_cf(txn_db, woptions, cfh, "cf_foo", 6, "cf_hello",
-                                 8, &err);
-    CheckNoError(err);
-    CheckTxnDBGetCF(txn_db, roptions, cfh, "cf_foo", "cf_hello");
-
-    rocksdb_transactiondb_delete_cf(txn_db, woptions, cfh, "cf_foo", 6, &err);
-    CheckNoError(err);
-    CheckTxnDBGetCF(txn_db, roptions, cfh, "cf_foo", NULL);
-
-    rocksdb_column_family_handle_destroy(cfh);
-
     // close and destroy
     rocksdb_transaction_destroy(txn);
     rocksdb_transactiondb_close(txn_db);
@@ -1491,18 +1440,6 @@ int main(int argc, char** argv) {
     CheckNoError(err);
   }
 
-  // Simple sanity check that options setting db_paths work.
-  StartPhase("open_db_paths");
-  {
-    rocksdb_close(db);
-    rocksdb_destroy_db(options, dbname, &err);
-
-    const rocksdb_dbpath_t* paths[1] = {dbpath};
-    rocksdb_options_set_db_paths(options, paths, 1);
-    db = rocksdb_open(options, dbname, &err);
-    CheckNoError(err);
-  }
-  
   StartPhase("cleanup");
   rocksdb_close(db);
   rocksdb_options_destroy(options);
@@ -1512,7 +1449,6 @@ int main(int argc, char** argv) {
   rocksdb_compactoptions_destroy(coptions);
   rocksdb_cache_destroy(cache);
   rocksdb_comparator_destroy(cmp);
-  rocksdb_dbpath_destroy(dbpath);
   rocksdb_env_destroy(env);
 
   fprintf(stderr, "PASS\n");

db/column_family.cc

@@ -353,8 +353,8 @@ ColumnFamilyData::ColumnFamilyData(
       initial_cf_options_(SanitizeOptions(db_options, cf_options)),
       ioptions_(db_options, initial_cf_options_),
       mutable_cf_options_(initial_cf_options_),
-      is_delete_range_supported_(
-          cf_options.table_factory->IsDeleteRangeSupported()),
+      is_delete_range_supported_(strcmp(cf_options.table_factory->Name(),
+                                        BlockBasedTableFactory().Name()) == 0),
       write_buffer_manager_(write_buffer_manager),
       mem_(nullptr),
       imm_(ioptions_.min_write_buffer_number_to_merge,
@@ -930,13 +930,6 @@ SuperVersion* ColumnFamilyData::InstallSuperVersion(
   super_version_ = new_superversion;
   ++super_version_number_;
   super_version_->version_number = super_version_number_;
-  if (old_superversion != nullptr) {
-    if (old_superversion->mutable_cf_options.write_buffer_size !=
-        mutable_cf_options.write_buffer_size) {
-      mem_->UpdateWriteBufferSize(mutable_cf_options.write_buffer_size);
-    }
-  }
-
   // Reset SuperVersions cached in thread local storage
   ResetThreadLocalSuperVersions();
 

db/column_family_test.cc

@@ -302,7 +302,6 @@ class ColumnFamilyTest : public testing::Test {
         ASSERT_OK(Put(cf, key, RandomString(&rnd_, key_value_size - 10)));
       }
     }
-    db_->FlushWAL(false);
   }
 
 #ifndef ROCKSDB_LITE  // TEST functions in DB are not supported in lite
@@ -582,7 +581,6 @@ TEST_P(FlushEmptyCFTestWithParam, FlushEmptyCFTest) {
   Flush(0);
   ASSERT_OK(Put(1, "bar", "v3"));  // seqID 4
   ASSERT_OK(Put(1, "foo", "v4"));  // seqID 5
-  db_->FlushWAL(false);
 
   // Preserve file system state up to here to simulate a crash condition.
   fault_env->SetFilesystemActive(false);
@@ -645,7 +643,6 @@ TEST_P(FlushEmptyCFTestWithParam, FlushEmptyCFTest2) {
   // Write to log file D
   ASSERT_OK(Put(1, "bar", "v4"));  // seqID 7
   ASSERT_OK(Put(1, "bar", "v5"));  // seqID 8
-  db_->FlushWAL(false);
   // Preserve file system state up to here to simulate a crash condition.
   fault_env->SetFilesystemActive(false);
   std::vector<std::string> names;

db/compaction.cc

@@ -169,7 +169,6 @@ Compaction::Compaction(VersionStorageInfo* vstorage,
       bottommost_level_(IsBottommostLevel(output_level_, vstorage, inputs_)),
       is_full_compaction_(IsFullCompaction(vstorage, inputs_)),
       is_manual_compaction_(_manual_compaction),
-      is_trivial_move_(false),
       compaction_reason_(_compaction_reason) {
   MarkFilesBeingCompacted(true);
   if (is_manual_compaction_) {
@@ -284,32 +283,32 @@ bool Compaction::KeyNotExistsBeyondOutputLevel(
   assert(input_version_ != nullptr);
   assert(level_ptrs != nullptr);
   assert(level_ptrs->size() == static_cast<size_t>(number_levels_));
-  if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
-    if (output_level_ == 0) {
-      return false;
-    }
-    // Maybe use binary search to find right entry instead of linear search?
-    const Comparator* user_cmp = cfd_->user_comparator();
-    for (int lvl = output_level_ + 1; lvl < number_levels_; lvl++) {
-      const std::vector<FileMetaData*>& files =
-          input_vstorage_->LevelFiles(lvl);
-      for (; level_ptrs->at(lvl) < files.size(); level_ptrs->at(lvl)++) {
-        auto* f = files[level_ptrs->at(lvl)];
-        if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) {
-          // We've advanced far enough
-          if (user_cmp->Compare(user_key, f->smallest.user_key()) >= 0) {
-            // Key falls in this file's range, so definitely
-            // exists beyond output level
-            return false;
-          }
-          break;
-        }
-      }
-    }
-    return true;
-  } else {
+  assert(cfd_->ioptions()->compaction_style != kCompactionStyleFIFO);
+  if (cfd_->ioptions()->compaction_style == kCompactionStyleUniversal) {
     return bottommost_level_;
   }
+  if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel &&
+      output_level_ == 0) {
+    return false;
+  }
+  // Maybe use binary search to find right entry instead of linear search?
+  const Comparator* user_cmp = cfd_->user_comparator();
+  for (int lvl = output_level_ + 1; lvl < number_levels_; lvl++) {
+    const std::vector<FileMetaData*>& files = input_vstorage_->LevelFiles(lvl);
+    for (; level_ptrs->at(lvl) < files.size(); level_ptrs->at(lvl)++) {
+      auto* f = files[level_ptrs->at(lvl)];
+      if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) {
+        // We've advanced far enough
+        if (user_cmp->Compare(user_key, f->smallest.user_key()) >= 0) {
+          // Key falls in this file's range, so definitely
+          // exists beyond output level
+          return false;
+        }
+        break;
+      }
+    }
+  }
+  return true;
 }
 
 // Mark (or clear) each file that is being compacted
@@ -464,17 +463,4 @@ bool Compaction::ShouldFormSubcompactions() const {
   }
 }
 
-uint64_t Compaction::MaxInputFileCreationTime() const {
-  uint64_t max_creation_time = 0;
-  for (const auto& file : inputs_[0].files) {
-    if (file->fd.table_reader != nullptr &&
-        file->fd.table_reader->GetTableProperties() != nullptr) {
-      uint64_t creation_time =
-          file->fd.table_reader->GetTableProperties()->creation_time;
-      max_creation_time = std::max(max_creation_time, creation_time);
-    }
-  }
-  return max_creation_time;
-}
-
 }  // namespace rocksdb

db/compaction.h

@@ -99,8 +99,7 @@ class Compaction {
   // input level.
   // REQUIREMENT: "compaction_input_level" must be >= 0 and
   //              < "input_levels()"
-  const std::vector<FileMetaData*>* inputs(
-      size_t compaction_input_level) const {
+  const std::vector<FileMetaData*>* inputs(size_t compaction_input_level) {
     assert(compaction_input_level < inputs_.size());
     return &inputs_[compaction_input_level].files;
   }
@@ -241,8 +240,6 @@ class Compaction {
 
   uint64_t max_compaction_bytes() const { return max_compaction_bytes_; }
 
-  uint64_t MaxInputFileCreationTime() const;
-
  private:
   // mark (or clear) all files that are being compacted
   void MarkFilesBeingCompacted(bool mark_as_compacted);

db/compaction_iteration_stats.h

@@ -16,8 +16,6 @@ struct CompactionIterationStats {
   int64_t num_record_drop_obsolete = 0;
   int64_t num_record_drop_range_del = 0;
   int64_t num_range_del_drop_obsolete = 0;
-  // Deletions obsoleted before bottom level due to file gap optimization.
-  int64_t num_optimized_del_drop_obsolete = 0;
   uint64_t total_filter_time = 0;
 
   // Input statistics

db/compaction_iterator.cc

@@ -25,8 +25,6 @@ CompactionEventListener::CompactionListenerValueType fromInternalValueType(
           kSingleDelete;
     case kTypeRangeDeletion:
       return CompactionEventListener::CompactionListenerValueType::kRangeDelete;
-    case kTypeBlobIndex:
-      return CompactionEventListener::CompactionListenerValueType::kBlobIndex;
     default:
       assert(false);
       return CompactionEventListener::CompactionListenerValueType::kInvalid;
@@ -113,7 +111,6 @@ void CompactionIterator::ResetRecordCounts() {
   iter_stats_.num_record_drop_obsolete = 0;
   iter_stats_.num_record_drop_range_del = 0;
   iter_stats_.num_range_del_drop_obsolete = 0;
-  iter_stats_.num_optimized_del_drop_obsolete = 0;
 }
 
 void CompactionIterator::SeekToFirst() {
@@ -230,8 +227,7 @@ void CompactionIterator::NextFromInput() {
 #endif  // ROCKSDB_LITE
 
       // apply the compaction filter to the first occurrence of the user key
-      if (compaction_filter_ != nullptr && 
-          (ikey_.type == kTypeValue || ikey_.type == kTypeBlobIndex) &&
+      if (compaction_filter_ != nullptr && ikey_.type == kTypeValue &&
           (visible_at_tip_ || ikey_.sequence > latest_snapshot_ ||
            ignore_snapshots_)) {
         // If the user has specified a compaction filter and the sequence
@@ -241,13 +237,11 @@ void CompactionIterator::NextFromInput() {
         CompactionFilter::Decision filter;
         compaction_filter_value_.clear();
         compaction_filter_skip_until_.Clear();
-        CompactionFilter::ValueType value_type =
-            ikey_.type == kTypeValue ? CompactionFilter::ValueType::kValue
-                                     : CompactionFilter::ValueType::kBlobIndex;
         {
           StopWatchNano timer(env_, true);
           filter = compaction_filter_->FilterV2(
-              compaction_->level(), ikey_.user_key, value_type, value_,
+              compaction_->level(), ikey_.user_key,
+              CompactionFilter::ValueType::kValue, value_,
               &compaction_filter_value_, compaction_filter_skip_until_.rep());
           iter_stats_.total_filter_time +=
               env_ != nullptr ? timer.ElapsedNanos() : 0;
@@ -432,9 +426,6 @@ void CompactionIterator::NextFromInput() {
           // Can compact out this SingleDelete.
           ++iter_stats_.num_record_drop_obsolete;
           ++iter_stats_.num_single_del_fallthru;
-          if (!bottommost_level_) {
-            ++iter_stats_.num_optimized_del_drop_obsolete;
-          }
         } else {
           // Output SingleDelete
           valid_ = true;
@@ -476,9 +467,6 @@ void CompactionIterator::NextFromInput() {
       // Note:  Dropping this Delete will not affect TransactionDB
       // write-conflict checking since it is earlier than any snapshot.
       ++iter_stats_.num_record_drop_obsolete;
-      if (!bottommost_level_) {
-        ++iter_stats_.num_optimized_del_drop_obsolete;
-      }
       input_->Next();
     } else if (ikey_.type == kTypeMerge) {
       if (!merge_helper_->HasOperator()) {

db/compaction_iterator_test.cc

@@ -455,111 +455,6 @@ TEST_F(CompactionIteratorTest, ShuttingDownInMerge) {
   EXPECT_EQ(2, filter.last_seen.load());
 }
 
-TEST_F(CompactionIteratorTest, SingleMergeOperand) {
-  class Filter : public CompactionFilter {
-    virtual Decision FilterV2(int level, const Slice& key, ValueType t,
-                              const Slice& existing_value,
-                              std::string* new_value,
-                              std::string* skip_until) const override {
-      std::string k = key.ToString();
-      std::string v = existing_value.ToString();
-
-      // See InitIterators() call below for the sequence of keys and their
-      // filtering decisions. Here we closely assert that compaction filter is
-      // called with the expected keys and only them, and with the right values.
-      if (k == "a") {
-        EXPECT_EQ(ValueType::kMergeOperand, t);
-        EXPECT_EQ("av1", v);
-        return Decision::kKeep;
-      } else if (k == "b") {
-        EXPECT_EQ(ValueType::kMergeOperand, t);
-        return Decision::kKeep;
-      } else if (k == "c") {
-        return Decision::kKeep;
-      }
-
-      ADD_FAILURE();
-      return Decision::kKeep;
-    }
-
-    const char* Name() const override {
-      return "CompactionIteratorTest.SingleMergeOperand::Filter";
-    }
-  };
-
-  class SingleMergeOp : public MergeOperator {
-   public:
-    bool FullMergeV2(const MergeOperationInput& merge_in,
-                     MergeOperationOutput* merge_out) const override {
-      // See InitIterators() call below for why "c" is the only key for which
-      // FullMergeV2 should be called.
-      EXPECT_EQ("c", merge_in.key.ToString());
-
-      std::string temp_value;
-      if (merge_in.existing_value != nullptr) {
-        temp_value = merge_in.existing_value->ToString();
-      }
-
-      for (auto& operand : merge_in.operand_list) {
-        temp_value.append(operand.ToString());
-      }
-      merge_out->new_value = temp_value;
-
-      return true;
-    }
-
-    bool PartialMergeMulti(const Slice& key,
-                           const std::deque<Slice>& operand_list,
-                           std::string* new_value,
-                           Logger* logger) const override {
-      std::string string_key = key.ToString();
-      EXPECT_TRUE(string_key == "a" || string_key == "b");
-
-      if (string_key == "a") {
-        EXPECT_EQ(1, operand_list.size());
-      } else if (string_key == "b") {
-        EXPECT_EQ(2, operand_list.size());
-      }
-
-      std::string temp_value;
-      for (auto& operand : operand_list) {
-        temp_value.append(operand.ToString());
-      }
-      swap(temp_value, *new_value);
-
-      return true;
-    }
-
-    const char* Name() const override {
-      return "CompactionIteratorTest SingleMergeOp";
-    }
-
-    bool AllowSingleOperand() const override { return true; }
-  };
-
-  SingleMergeOp merge_op;
-  Filter filter;
-  InitIterators(
-      // a should invoke PartialMergeMulti with a single merge operand.
-      {test::KeyStr("a", 50, kTypeMerge),
-       // b should invoke PartialMergeMulti with two operands.
-       test::KeyStr("b", 70, kTypeMerge), test::KeyStr("b", 60, kTypeMerge),
-       // c should invoke FullMerge due to kTypeValue at the beginning.
-       test::KeyStr("c", 90, kTypeMerge), test::KeyStr("c", 80, kTypeValue)},
-      {"av1", "bv2", "bv1", "cv2", "cv1"}, {}, {}, kMaxSequenceNumber,
-      &merge_op, &filter);
-
-  c_iter_->SeekToFirst();
-  ASSERT_TRUE(c_iter_->Valid());
-  ASSERT_EQ(test::KeyStr("a", 50, kTypeMerge), c_iter_->key().ToString());
-  ASSERT_EQ("av1", c_iter_->value().ToString());
-  c_iter_->Next();
-  ASSERT_TRUE(c_iter_->Valid());
-  ASSERT_EQ("bv1bv2", c_iter_->value().ToString());
-  c_iter_->Next();
-  ASSERT_EQ("cv1cv2", c_iter_->value().ToString());
-}
-
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/compaction_job.cc

@@ -594,9 +594,6 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) {
 
   double read_write_amp = 0.0;
   double write_amp = 0.0;
-  double bytes_read_per_sec = 0;
-  double bytes_written_per_sec = 0;
-
   if (stats.bytes_read_non_output_levels > 0) {
     read_write_amp = (stats.bytes_written + stats.bytes_read_output_level +
                       stats.bytes_read_non_output_levels) /
@@ -604,22 +601,17 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) {
     write_amp = stats.bytes_written /
                 static_cast<double>(stats.bytes_read_non_output_levels);
   }
-  if (stats.micros > 0) {
-    bytes_read_per_sec =
-        (stats.bytes_read_non_output_levels + stats.bytes_read_output_level) /
-        static_cast<double>(stats.micros);
-    bytes_written_per_sec =
-        stats.bytes_written / static_cast<double>(stats.micros);
-  }
-
   ROCKS_LOG_BUFFER(
       log_buffer_,
       "[%s] compacted to: %s, MB/sec: %.1f rd, %.1f wr, level %d, "
       "files in(%d, %d) out(%d) "
       "MB in(%.1f, %.1f) out(%.1f), read-write-amplify(%.1f) "
       "write-amplify(%.1f) %s, records in: %d, records dropped: %d\n",
-      cfd->GetName().c_str(), vstorage->LevelSummary(&tmp), bytes_read_per_sec,
-      bytes_written_per_sec, compact_->compaction->output_level(),
+      cfd->GetName().c_str(), vstorage->LevelSummary(&tmp),
+      (stats.bytes_read_non_output_levels + stats.bytes_read_output_level) /
+          static_cast<double>(stats.micros),
+      stats.bytes_written / static_cast<double>(stats.micros),
+      compact_->compaction->output_level(),
       stats.num_input_files_in_non_output_levels,
       stats.num_input_files_in_output_level, stats.num_output_files,
       stats.bytes_read_non_output_levels / 1048576.0,
@@ -1014,10 +1006,6 @@ void CompactionJob::RecordDroppedKeys(
     RecordTick(stats_, COMPACTION_RANGE_DEL_DROP_OBSOLETE,
                c_iter_stats.num_range_del_drop_obsolete);
   }
-  if (c_iter_stats.num_optimized_del_drop_obsolete > 0) {
-    RecordTick(stats_, COMPACTION_OPTIMIZED_DEL_DROP_OBSOLETE,
-               c_iter_stats.num_optimized_del_drop_obsolete);
-  }
 }
 
 Status CompactionJob::FinishCompactionOutputFile(
@@ -1035,6 +1023,7 @@ Status CompactionJob::FinishCompactionOutputFile(
   uint64_t output_number = sub_compact->current_output()->meta.fd.GetNumber();
   assert(output_number != 0);
 
+  TableProperties table_properties;
   // Check for iterator errors
   Status s = input_status;
   auto meta = &sub_compact->current_output()->meta;
@@ -1090,23 +1079,6 @@ Status CompactionJob::FinishCompactionOutputFile(
   }
   sub_compact->outfile.reset();
 
-  if (s.ok() && current_entries == 0) {
-    // If there is nothing to output, no necessary to generate a sst file.
-    // This happens when the output level is bottom level, at the same time
-    // the sub_compact output nothing.
-    std::string fname = TableFileName(
-        db_options_.db_paths, meta->fd.GetNumber(), meta->fd.GetPathId());
-    env_->DeleteFile(fname);
-
-    // Also need to remove the file from outputs, or it will be added to the
-    // VersionEdit.
-    assert(!sub_compact->outputs.empty());
-    sub_compact->outputs.pop_back();
-    sub_compact->builder.reset();
-    sub_compact->current_output_file_size = 0;
-    return s;
-  }
-
   ColumnFamilyData* cfd = sub_compact->compaction->column_family_data();
   TableProperties tp;
   if (s.ok() && current_entries > 0) {
@@ -1159,22 +1131,12 @@ Status CompactionJob::FinishCompactionOutputFile(
                             meta->fd.GetPathId());
     sfm->OnAddFile(fn);
     if (sfm->IsMaxAllowedSpaceReached()) {
-      // TODO(ajkr): should we return OK() if max space was reached by the final
-      // compaction output file (similarly to how flush works when full)?
-      s = Status::IOError("Max allowed space was reached");
-      TEST_SYNC_POINT(
-          "CompactionJob::FinishCompactionOutputFile:"
-          "MaxAllowedSpaceReached");
       InstrumentedMutexLock l(db_mutex_);
       if (db_bg_error_->ok()) {
-        Status new_bg_error = s;
-        // may temporarily unlock and lock the mutex.
-        EventHelpers::NotifyOnBackgroundError(
-            cfd->ioptions()->listeners, BackgroundErrorReason::kCompaction,
-            &new_bg_error, db_mutex_);
-        if (!new_bg_error.ok()) {
-          *db_bg_error_ = new_bg_error;
-        }
+        s = Status::IOError("Max allowed space was reached");
+        *db_bg_error_ = s;
+        TEST_SYNC_POINT(
+            "CompactionJob::FinishCompactionOutputFile:MaxAllowedSpaceReached");
       }
     }
   }
@@ -1289,22 +1251,14 @@ Status CompactionJob::OpenCompactionOutputFile(
   // data is going to be found
   bool skip_filters =
       cfd->ioptions()->optimize_filters_for_hits && bottommost_level_;
-
-  uint64_t output_file_creation_time =
-      sub_compact->compaction->MaxInputFileCreationTime();
-  if (output_file_creation_time == 0) {
-    int64_t _current_time = 0;
-    db_options_.env->GetCurrentTime(&_current_time);  // ignore error
-    output_file_creation_time = static_cast<uint64_t>(_current_time);
-  }
-
   sub_compact->builder.reset(NewTableBuilder(
       *cfd->ioptions(), cfd->internal_comparator(),
       cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
       sub_compact->outfile.get(), sub_compact->compaction->output_compression(),
       cfd->ioptions()->compression_opts,
-      sub_compact->compaction->output_level(), &sub_compact->compression_dict,
-      skip_filters, output_file_creation_time));
+      sub_compact->compaction->output_level(),
+      &sub_compact->compression_dict,
+      skip_filters));
   LogFlush(db_options_.info_log);
   return s;
 }

db/compaction_job_test.cc

@@ -142,7 +142,6 @@ class CompactionJobTest : public testing::Test {
   }
 
   void SetLastSequence(const SequenceNumber sequence_number) {
-    versions_->SetLastToBeWrittenSequence(sequence_number + 1);
     versions_->SetLastSequence(sequence_number + 1);
   }
 
@@ -338,24 +337,6 @@ TEST_F(CompactionJobTest, SimpleDeletion) {
   RunCompaction({files}, expected_results);
 }
 
-TEST_F(CompactionJobTest, OutputNothing) {
-  NewDB();
-
-  auto file1 = mock::MakeMockFile({{KeyStr("a", 1U, kTypeValue), "val"}});
-
-  AddMockFile(file1);
-
-  auto file2 = mock::MakeMockFile({{KeyStr("a", 2U, kTypeDeletion), ""}});
-
-  AddMockFile(file2);
-
-  auto expected_results = mock::MakeMockFile();
-
-  SetLastSequence(4U);
-  auto files = cfd_->current()->storage_info()->LevelFiles(0);
-  RunCompaction({files}, expected_results);
-}
-
 TEST_F(CompactionJobTest, SimpleOverwrite) {
   NewDB();
 

db/compaction_picker.cc

@@ -234,6 +234,11 @@ bool CompactionPicker::ExpandInputsToCleanCut(const std::string& cf_name,
   // If, after the expansion, there are files that are already under
   // compaction, then we must drop/cancel this compaction.
   if (AreFilesInCompaction(inputs->files)) {
+    ROCKS_LOG_WARN(
+        ioptions_.info_log,
+        "[%s] ExpandWhileOverlapping() failure because some of the necessary"
+        " compaction input files are currently being compacted.",
+        cf_name.c_str());
     return false;
   }
   return true;
@@ -1101,7 +1106,11 @@ void LevelCompactionBuilder::SetupInitialFiles() {
       }
       output_level_ =
           (start_level_ == 0) ? vstorage_->base_level() : start_level_ + 1;
-      if (PickFileToCompact()) {
+      if (PickFileToCompact() &&
+          compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_,
+                                                     &start_level_inputs_) &&
+          !compaction_picker_->FilesRangeOverlapWithCompaction(
+              {start_level_inputs_}, output_level_)) {
         // found the compaction!
         if (start_level_ == 0) {
           // L0 score = `num L0 files` / `level0_file_num_compaction_trigger`
@@ -1323,10 +1332,12 @@ bool LevelCompactionBuilder::PickFileToCompact() {
   const std::vector<FileMetaData*>& level_files =
       vstorage_->LevelFiles(start_level_);
 
-  unsigned int cmp_idx;
-  for (cmp_idx = vstorage_->NextCompactionIndex(start_level_);
-       cmp_idx < file_size.size(); cmp_idx++) {
-    int index = file_size[cmp_idx];
+  // record the first file that is not yet compacted
+  int nextIndex = -1;
+
+  for (unsigned int i = vstorage_->NextCompactionIndex(start_level_);
+       i < file_size.size(); i++) {
+    int index = file_size[i];
     auto* f = level_files[index];
 
     // do not pick a file to compact if it is being compacted
@@ -1335,42 +1346,27 @@ bool LevelCompactionBuilder::PickFileToCompact() {
       continue;
     }
 
-    start_level_inputs_.files.push_back(f);
-    start_level_inputs_.level = start_level_;
-    if (!compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_,
-                                                    &start_level_inputs_) ||
-        compaction_picker_->FilesRangeOverlapWithCompaction(
-            {start_level_inputs_}, output_level_)) {
-      // A locked (pending compaction) input-level file was pulled in due to
-      // user-key overlap.
-      start_level_inputs_.clear();
-      continue;
+    // remember the startIndex for the next call to PickCompaction
+    if (nextIndex == -1) {
+      nextIndex = i;
     }
 
-    // Now that input level is fully expanded, we check whether any output files
-    // are locked due to pending compaction.
-    //
-    // Note we rely on ExpandInputsToCleanCut() to tell us whether any output-
-    // level files are locked, not just the extra ones pulled in for user-key
-    // overlap.
-    InternalKey smallest, largest;
-    compaction_picker_->GetRange(start_level_inputs_, &smallest, &largest);
-    CompactionInputFiles output_level_inputs;
-    output_level_inputs.level = output_level_;
-    vstorage_->GetOverlappingInputs(output_level_, &smallest, &largest,
-                                    &output_level_inputs.files);
-    if (!output_level_inputs.empty() &&
-        !compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_,
-                                                    &output_level_inputs)) {
-      start_level_inputs_.clear();
+    // Do not pick this file if its parents at level+1 are being compacted.
+    // Maybe we can avoid redoing this work in SetupOtherInputs
+    parent_index_ = -1;
+    if (compaction_picker_->IsRangeInCompaction(vstorage_, &f->smallest,
+                                                &f->largest, output_level_,
+                                                &parent_index_)) {
       continue;
     }
+    start_level_inputs_.files.push_back(f);
+    start_level_inputs_.level = start_level_;
     base_index_ = index;
     break;
   }
 
   // store where to start the iteration in the next call to PickCompaction
-  vstorage_->SetNextCompactionIndex(start_level_, cmp_idx);
+  vstorage_->SetNextCompactionIndex(start_level_, nextIndex);
 
   return start_level_inputs_.size() > 0;
 }
@@ -1407,90 +1403,16 @@ bool FIFOCompactionPicker::NeedsCompaction(
   return vstorage->CompactionScore(kLevel0) >= 1;
 }
 
-namespace {
-uint64_t GetTotalFilesSize(
-    const std::vector<FileMetaData*>& files) {
+Compaction* FIFOCompactionPicker::PickCompaction(
+    const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
+    VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
+  assert(vstorage->num_levels() == 1);
+  const int kLevel0 = 0;
+  const std::vector<FileMetaData*>& level_files = vstorage->LevelFiles(kLevel0);
   uint64_t total_size = 0;
-  for (const auto& f : files) {
-    total_size += f->fd.file_size;
+  for (const auto& file : level_files) {
+    total_size += file->fd.file_size;
   }
-  return total_size;
-}
-}  // anonymous namespace
-
-Compaction* FIFOCompactionPicker::PickTTLCompaction(
-    const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-    VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
-  assert(ioptions_.compaction_options_fifo.ttl > 0);
-
-  const int kLevel0 = 0;
-  const std::vector<FileMetaData*>& level_files = vstorage->LevelFiles(kLevel0);
-  uint64_t total_size = GetTotalFilesSize(level_files);
-
-  int64_t _current_time;
-  auto status = ioptions_.env->GetCurrentTime(&_current_time);
-  if (!status.ok()) {
-    ROCKS_LOG_BUFFER(log_buffer,
-                     "[%s] FIFO compaction: Couldn't get current time: %s. "
-                     "Not doing compactions based on TTL. ",
-                     cf_name.c_str(), status.ToString().c_str());
-    return nullptr;
-  }
-  const uint64_t current_time = static_cast<uint64_t>(_current_time);
-
-  std::vector<CompactionInputFiles> inputs;
-  inputs.emplace_back();
-  inputs[0].level = 0;
-
-  // avoid underflow
-  if (current_time > ioptions_.compaction_options_fifo.ttl) {
-    for (auto ritr = level_files.rbegin(); ritr != level_files.rend(); ++ritr) {
-      auto f = *ritr;
-      if (f->fd.table_reader != nullptr &&
-          f->fd.table_reader->GetTableProperties() != nullptr) {
-        auto creation_time =
-            f->fd.table_reader->GetTableProperties()->creation_time;
-        if (creation_time == 0 ||
-            creation_time >=
-                (current_time - ioptions_.compaction_options_fifo.ttl)) {
-          break;
-        }
-        total_size -= f->compensated_file_size;
-        inputs[0].files.push_back(f);
-      }
-    }
-  }
-
-  // Return a nullptr and proceed to size-based FIFO compaction if:
-  // 1. there are no files older than ttl OR
-  // 2. there are a few files older than ttl, but deleting them will not bring
-  //    the total size to be less than max_table_files_size threshold.
-  if (inputs[0].files.empty() ||
-      total_size > ioptions_.compaction_options_fifo.max_table_files_size) {
-    return nullptr;
-  }
-
-  for (const auto& f : inputs[0].files) {
-    ROCKS_LOG_BUFFER(log_buffer,
-                     "[%s] FIFO compaction: picking file %" PRIu64
-                     " with creation time %" PRIu64 " for deletion",
-                     cf_name.c_str(), f->fd.GetNumber(),
-                     f->fd.table_reader->GetTableProperties()->creation_time);
-  }
-
-  Compaction* c = new Compaction(
-      vstorage, ioptions_, mutable_cf_options, std::move(inputs), 0, 0, 0, 0,
-      kNoCompression, {}, /* is manual */ false, vstorage->CompactionScore(0),
-      /* is deletion compaction */ true, CompactionReason::kFIFOTtl);
-  return c;
-}
-
-Compaction* FIFOCompactionPicker::PickSizeCompaction(
-    const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-    VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
-  const int kLevel0 = 0;
-  const std::vector<FileMetaData*>& level_files = vstorage->LevelFiles(kLevel0);
-  uint64_t total_size = GetTotalFilesSize(level_files);
 
   if (total_size <= ioptions_.compaction_options_fifo.max_table_files_size ||
       level_files.size() == 0) {
@@ -1511,6 +1433,7 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction(
             /* is manual */ false, vstorage->CompactionScore(0),
             /* is deletion compaction */ false,
             CompactionReason::kFIFOReduceNumFiles);
+        RegisterCompaction(c);
         return c;
       }
     }
@@ -1535,41 +1458,24 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction(
   std::vector<CompactionInputFiles> inputs;
   inputs.emplace_back();
   inputs[0].level = 0;
-
+  // delete old files (FIFO)
   for (auto ritr = level_files.rbegin(); ritr != level_files.rend(); ++ritr) {
     auto f = *ritr;
     total_size -= f->compensated_file_size;
     inputs[0].files.push_back(f);
     char tmp_fsize[16];
     AppendHumanBytes(f->fd.GetFileSize(), tmp_fsize, sizeof(tmp_fsize));
-    ROCKS_LOG_BUFFER(log_buffer,
-                     "[%s] FIFO compaction: picking file %" PRIu64
-                     " with size %s for deletion",
+    ROCKS_LOG_BUFFER(log_buffer, "[%s] FIFO compaction: picking file %" PRIu64
+                                 " with size %s for deletion",
                      cf_name.c_str(), f->fd.GetNumber(), tmp_fsize);
     if (total_size <= ioptions_.compaction_options_fifo.max_table_files_size) {
       break;
     }
   }
-
   Compaction* c = new Compaction(
       vstorage, ioptions_, mutable_cf_options, std::move(inputs), 0, 0, 0, 0,
       kNoCompression, {}, /* is manual */ false, vstorage->CompactionScore(0),
       /* is deletion compaction */ true, CompactionReason::kFIFOMaxSize);
-  return c;
-}
-
-Compaction* FIFOCompactionPicker::PickCompaction(
-    const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-    VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
-  assert(vstorage->num_levels() == 1);
-
-  Compaction* c = nullptr;
-  if (ioptions_.compaction_options_fifo.ttl > 0) {
-    c = PickTTLCompaction(cf_name, mutable_cf_options, vstorage, log_buffer);
-  }
-  if (c == nullptr) {
-    c = PickSizeCompaction(cf_name, mutable_cf_options, vstorage, log_buffer);
-  }
   RegisterCompaction(c);
   return c;
 }

db/compaction_picker.h

@@ -242,17 +242,6 @@ class FIFOCompactionPicker : public CompactionPicker {
 
   virtual bool NeedsCompaction(
       const VersionStorageInfo* vstorage) const override;
-
- private:
-  Compaction* PickTTLCompaction(const std::string& cf_name,
-                                const MutableCFOptions& mutable_cf_options,
-                                VersionStorageInfo* version,
-                                LogBuffer* log_buffer);
-
-  Compaction* PickSizeCompaction(const std::string& cf_name,
-                                 const MutableCFOptions& mutable_cf_options,
-                                 VersionStorageInfo* version,
-                                 LogBuffer* log_buffer);
 };
 
 class NullCompactionPicker : public CompactionPicker {

db/compaction_picker_test.cc

@@ -512,7 +512,7 @@ TEST_F(CompactionPickerTest, NeedsCompactionFIFO) {
         kFileSize, 0, i * 100, i * 100 + 99);
     current_size += kFileSize;
     UpdateVersionStorageInfo();
-    ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()),
+    ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()),
               vstorage_->CompactionScore(0) >= 1);
   }
 }
@@ -852,80 +852,6 @@ TEST_F(CompactionPickerTest, OverlappingUserKeys9) {
   ASSERT_EQ(8U, compaction->input(1, 1)->fd.GetNumber());
 }
 
-TEST_F(CompactionPickerTest, OverlappingUserKeys10) {
-  // Locked file encountered when pulling in extra input-level files with same
-  // user keys. Verify we pick the next-best file from the same input level.
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.max_compaction_bytes = 100000000000u;
-
-  // file_number 2U is largest and thus first choice. But it overlaps with
-  // file_number 1U which is being compacted. So instead we pick the next-
-  // biggest file, 3U, which is eligible for compaction.
-  Add(1 /* level */, 1U /* file_number */, "100" /* smallest */,
-      "150" /* largest */, 1U /* file_size */);
-  file_map_[1U].first->being_compacted = true;
-  Add(1 /* level */, 2U /* file_number */, "150" /* smallest */,
-      "200" /* largest */, 1000000000U /* file_size */, 0 /* smallest_seq */,
-      0 /* largest_seq */);
-  Add(1 /* level */, 3U /* file_number */, "201" /* smallest */,
-      "250" /* largest */, 900000000U /* file_size */);
-  Add(2 /* level */, 4U /* file_number */, "100" /* smallest */,
-      "150" /* largest */, 1U /* file_size */);
-  Add(2 /* level */, 5U /* file_number */, "151" /* smallest */,
-      "200" /* largest */, 1U /* file_size */);
-  Add(2 /* level */, 6U /* file_number */, "201" /* smallest */,
-      "250" /* largest */, 1U /* file_size */);
-
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(2U, compaction->num_input_levels());
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(1U, compaction->num_input_files(1));
-  ASSERT_EQ(3U, compaction->input(0, 0)->fd.GetNumber());
-  ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber());
-}
-
-TEST_F(CompactionPickerTest, OverlappingUserKeys11) {
-  // Locked file encountered when pulling in extra output-level files with same
-  // user keys. Expected to skip that compaction and pick the next-best choice.
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.max_compaction_bytes = 100000000000u;
-
-  // score(L1) = 3.7
-  // score(L2) = 1.85
-  // There is no eligible file in L1 to compact since both candidates pull in
-  // file_number 5U, which overlaps with a file pending compaction (6U). The
-  // first eligible compaction is from L2->L3.
-  Add(1 /* level */, 2U /* file_number */, "151" /* smallest */,
-      "200" /* largest */, 1000000000U /* file_size */);
-  Add(1 /* level */, 3U /* file_number */, "201" /* smallest */,
-      "250" /* largest */, 1U /* file_size */);
-  Add(2 /* level */, 4U /* file_number */, "100" /* smallest */,
-      "149" /* largest */, 5000000000U /* file_size */);
-  Add(2 /* level */, 5U /* file_number */, "150" /* smallest */,
-      "201" /* largest */, 1U /* file_size */);
-  Add(2 /* level */, 6U /* file_number */, "201" /* smallest */,
-      "249" /* largest */, 1U /* file_size */, 0 /* smallest_seq */,
-      0 /* largest_seq */);
-  file_map_[6U].first->being_compacted = true;
-  Add(3 /* level */, 7U /* file_number */, "100" /* smallest */,
-      "149" /* largest */, 1U /* file_size */);
-
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(2U, compaction->num_input_levels());
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(1U, compaction->num_input_files(1));
-  ASSERT_EQ(4U, compaction->input(0, 0)->fd.GetNumber());
-  ASSERT_EQ(7U, compaction->input(1, 0)->fd.GetNumber());
-}
-
 TEST_F(CompactionPickerTest, NotScheduleL1IfL0WithHigherPri1) {
   NewVersionStorage(6, kCompactionStyleLevel);
   mutable_cf_options_.level0_file_num_compaction_trigger = 2;
@@ -1390,49 +1316,6 @@ TEST_F(CompactionPickerTest, IsTrivialMoveOff) {
   ASSERT_FALSE(compaction->IsTrivialMove());
 }
 
-TEST_F(CompactionPickerTest, CacheNextCompactionIndex) {
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.max_compaction_bytes = 100000000000u;
-
-  Add(1 /* level */, 1U /* file_number */, "100" /* smallest */,
-      "149" /* largest */, 1000000000U /* file_size */);
-  file_map_[1U].first->being_compacted = true;
-  Add(1 /* level */, 2U /* file_number */, "150" /* smallest */,
-      "199" /* largest */, 900000000U /* file_size */);
-  Add(1 /* level */, 3U /* file_number */, "200" /* smallest */,
-      "249" /* largest */, 800000000U /* file_size */);
-  Add(1 /* level */, 4U /* file_number */, "250" /* smallest */,
-      "299" /* largest */, 700000000U /* file_size */);
-  Add(2 /* level */, 5U /* file_number */, "150" /* smallest */,
-      "199" /* largest */, 1U /* file_size */);
-  file_map_[5U].first->being_compacted = true;
-
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(1U, compaction->num_input_levels());
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(0U, compaction->num_input_files(1));
-  ASSERT_EQ(3U, compaction->input(0, 0)->fd.GetNumber());
-  ASSERT_EQ(2, vstorage_->NextCompactionIndex(1 /* level */));
-
-  compaction.reset(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(1U, compaction->num_input_levels());
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(0U, compaction->num_input_files(1));
-  ASSERT_EQ(4U, compaction->input(0, 0)->fd.GetNumber());
-  ASSERT_EQ(3, vstorage_->NextCompactionIndex(1 /* level */));
-
-  compaction.reset(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() == nullptr);
-  ASSERT_EQ(4, vstorage_->NextCompactionIndex(1 /* level */));
-}
-
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/compaction_picker_universal.cc

@@ -373,7 +373,6 @@ Compaction* UniversalCompactionPicker::PickCompaction(
               c->inputs(0)->size());
 
   RegisterCompaction(c);
-  vstorage->ComputeCompactionScore(ioptions_, mutable_cf_options);
 
   TEST_SYNC_POINT_CALLBACK("UniversalCompactionPicker::PickCompaction:Return",
                            c);

db/convenience.cc

@@ -9,50 +9,19 @@
 #include "rocksdb/convenience.h"
 
 #include "db/db_impl.h"
-#include "util/cast_util.h"
 
 namespace rocksdb {
 
 void CancelAllBackgroundWork(DB* db, bool wait) {
-  (static_cast_with_check<DBImpl, DB>(db->GetRootDB()))
-      ->CancelAllBackgroundWork(wait);
+  (dynamic_cast<DBImpl*>(db->GetRootDB()))->CancelAllBackgroundWork(wait);
 }
 
 Status DeleteFilesInRange(DB* db, ColumnFamilyHandle* column_family,
                           const Slice* begin, const Slice* end) {
-  return (static_cast_with_check<DBImpl, DB>(db->GetRootDB()))
+  return (dynamic_cast<DBImpl*>(db->GetRootDB()))
       ->DeleteFilesInRange(column_family, begin, end);
 }
 
-Status VerifySstFileChecksum(const Options& options,
-                             const EnvOptions& env_options,
-                             const std::string& file_path) {
-  unique_ptr<RandomAccessFile> file;
-  uint64_t file_size;
-  InternalKeyComparator internal_comparator(options.comparator);
-  ImmutableCFOptions ioptions(options);
-
-  Status s = ioptions.env->NewRandomAccessFile(file_path, &file, env_options);
-  if (s.ok()) {
-    s = ioptions.env->GetFileSize(file_path, &file_size);
-  } else {
-    return s;
-  }
-  unique_ptr<TableReader> table_reader;
-  std::unique_ptr<RandomAccessFileReader> file_reader(
-      new RandomAccessFileReader(std::move(file), file_path));
-  s = ioptions.table_factory->NewTableReader(
-      TableReaderOptions(ioptions, env_options, internal_comparator,
-                         false /* skip_filters */, -1 /* level */),
-      std::move(file_reader), file_size, &table_reader,
-      false /* prefetch_index_and_filter_in_cache */);
-  if (!s.ok()) {
-    return s;
-  }
-  s = table_reader->VerifyChecksum();
-  return s;
-}
-
 }  // namespace rocksdb
 
 #endif  // ROCKSDB_LITE

db/corruption_test.cc

@@ -20,7 +20,6 @@
 #include "db/log_format.h"
 #include "db/version_set.h"
 #include "rocksdb/cache.h"
-#include "rocksdb/convenience.h"
 #include "rocksdb/env.h"
 #include "rocksdb/table.h"
 #include "rocksdb/write_batch.h"
@@ -153,7 +152,7 @@ class CorruptionTest : public testing::Test {
     struct stat sbuf;
     if (stat(fname.c_str(), &sbuf) != 0) {
       const char* msg = strerror(errno);
-      FAIL() << fname << ": " << msg;
+      ASSERT_TRUE(false) << fname << ": " << msg;
     }
 
     if (offset < 0) {
@@ -180,9 +179,6 @@ class CorruptionTest : public testing::Test {
     }
     s = WriteStringToFile(Env::Default(), contents, fname);
     ASSERT_TRUE(s.ok()) << s.ToString();
-    Options options;
-    EnvOptions env_options;
-    ASSERT_NOK(VerifySstFileChecksum(options, env_options, fname));
   }
 
   void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) {
@@ -217,7 +213,7 @@ class CorruptionTest : public testing::Test {
         return;
       }
     }
-    FAIL() << "no file found at level";
+    ASSERT_TRUE(false) << "no file found at level";
   }
 
 
@@ -316,7 +312,6 @@ TEST_F(CorruptionTest, TableFile) {
 
   Corrupt(kTableFile, 100, 1);
   Check(99, 99);
-  ASSERT_NOK(dbi->VerifyChecksum());
 }
 
 TEST_F(CorruptionTest, TableFileIndexData) {
@@ -332,11 +327,9 @@ TEST_F(CorruptionTest, TableFileIndexData) {
   // corrupt an index block of an entire file
   Corrupt(kTableFile, -2000, 500);
   Reopen();
-  dbi = reinterpret_cast<DBImpl*>(db_);
   // one full file should be readable, since only one was corrupted
   // the other file should be fully non-readable, since index was corrupted
   Check(5000, 5000);
-  ASSERT_NOK(dbi->VerifyChecksum());
 }
 
 TEST_F(CorruptionTest, MissingDescriptor) {
@@ -396,12 +389,10 @@ TEST_F(CorruptionTest, CompactionInputError) {
 
   Corrupt(kTableFile, 100, 1);
   Check(9, 9);
-  ASSERT_NOK(dbi->VerifyChecksum());
 
   // Force compactions by writing lots of values
   Build(10000);
   Check(10000, 10000);
-  ASSERT_NOK(dbi->VerifyChecksum());
 }
 
 TEST_F(CorruptionTest, CompactionInputErrorParanoid) {
@@ -433,7 +424,6 @@ TEST_F(CorruptionTest, CompactionInputErrorParanoid) {
 
   CorruptTableFileAtLevel(0, 100, 1);
   Check(9, 9);
-  ASSERT_NOK(dbi->VerifyChecksum());
 
   // Write must eventually fail because of corrupted table
   Status s;
@@ -455,7 +445,6 @@ TEST_F(CorruptionTest, UnrelatedKeys) {
   DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
   dbi->TEST_FlushMemTable();
   Corrupt(kTableFile, 100, 1);
-  ASSERT_NOK(dbi->VerifyChecksum());
 
   std::string tmp1, tmp2;
   ASSERT_OK(db_->Put(WriteOptions(), Key(1000, &tmp1), Value(1000, &tmp2)));

db/db_basic_test.cc

@@ -175,8 +175,6 @@ TEST_F(DBBasicTest, LevelLimitReopen) {
   int i = 0;
   while (NumTableFilesAtLevel(2, 1) == 0) {
     ASSERT_OK(Put(1, Key(i++), value));
-    dbfull()->TEST_WaitForFlushMemTable();
-    dbfull()->TEST_WaitForCompact();
   }
 
   options.num_levels = 1;
@@ -360,6 +358,7 @@ TEST_F(DBBasicTest, FLUSH) {
     WriteOptions writeOpt = WriteOptions();
     writeOpt.disableWAL = true;
     SetPerfLevel(kEnableTime);
+    ;
     ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v1"));
     // this will now also flush the last 2 writes
     ASSERT_OK(Flush(1));
@@ -368,7 +367,6 @@ TEST_F(DBBasicTest, FLUSH) {
     get_perf_context()->Reset();
     Get(1, "foo");
     ASSERT_TRUE((int)get_perf_context()->get_from_output_files_time > 0);
-    ASSERT_EQ(2, (int)get_perf_context()->get_read_bytes);
 
     ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
     ASSERT_EQ("v1", Get(1, "foo"));
@@ -725,7 +723,6 @@ TEST_F(DBBasicTest, FlushOneColumnFamily) {
 TEST_F(DBBasicTest, MultiGetSimple) {
   do {
     CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
-    SetPerfLevel(kEnableCount);
     ASSERT_OK(Put(1, "k1", "v1"));
     ASSERT_OK(Put(1, "k2", "v2"));
     ASSERT_OK(Put(1, "k3", "v3"));
@@ -739,15 +736,12 @@ TEST_F(DBBasicTest, MultiGetSimple) {
     std::vector<std::string> values(20, "Temporary data to be overwritten");
     std::vector<ColumnFamilyHandle*> cfs(keys.size(), handles_[1]);
 
-    get_perf_context()->Reset();
     std::vector<Status> s = db_->MultiGet(ReadOptions(), cfs, keys, &values);
     ASSERT_EQ(values.size(), keys.size());
     ASSERT_EQ(values[0], "v1");
     ASSERT_EQ(values[1], "v2");
     ASSERT_EQ(values[2], "v3");
     ASSERT_EQ(values[4], "v5");
-    // four kv pairs * two bytes per value
-    ASSERT_EQ(8, (int)get_perf_context()->multiget_read_bytes);
 
     ASSERT_OK(s[0]);
     ASSERT_OK(s[1]);
@@ -755,7 +749,6 @@ TEST_F(DBBasicTest, MultiGetSimple) {
     ASSERT_TRUE(s[3].IsNotFound());
     ASSERT_OK(s[4]);
     ASSERT_TRUE(s[5].IsNotFound());
-    SetPerfLevel(kDisable);
   } while (ChangeCompactOptions());
 }
 
@@ -792,30 +785,36 @@ TEST_F(DBBasicTest, MultiGetEmpty) {
 TEST_F(DBBasicTest, ChecksumTest) {
   BlockBasedTableOptions table_options;
   Options options = CurrentOptions();
-  // change when new checksum type added
-  int max_checksum = static_cast<int>(kxxHash);
-  const int kNumPerFile = 2;
 
-  // generate one table with each type of checksum
-  for (int i = 0; i <= max_checksum; ++i) {
-    table_options.checksum = static_cast<ChecksumType>(i);
-    options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-    Reopen(options);
-    for (int j = 0; j < kNumPerFile; ++j) {
-      ASSERT_OK(Put(Key(i * kNumPerFile + j), Key(i * kNumPerFile + j)));
-    }
-    ASSERT_OK(Flush());
-  }
+  table_options.checksum = kCRC32c;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  Reopen(options);
+  ASSERT_OK(Put("a", "b"));
+  ASSERT_OK(Put("c", "d"));
+  ASSERT_OK(Flush());  // table with crc checksum
 
-  // verify data with each type of checksum
-  for (int i = 0; i <= kxxHash; ++i) {
-    table_options.checksum = static_cast<ChecksumType>(i);
-    options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-    Reopen(options);
-    for (int j = 0; j < (max_checksum + 1) * kNumPerFile; ++j) {
-      ASSERT_EQ(Key(j), Get(Key(j)));
-    }
-  }
+  table_options.checksum = kxxHash;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  Reopen(options);
+  ASSERT_OK(Put("e", "f"));
+  ASSERT_OK(Put("g", "h"));
+  ASSERT_OK(Flush());  // table with xxhash checksum
+
+  table_options.checksum = kCRC32c;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  Reopen(options);
+  ASSERT_EQ("b", Get("a"));
+  ASSERT_EQ("d", Get("c"));
+  ASSERT_EQ("f", Get("e"));
+  ASSERT_EQ("h", Get("g"));
+
+  table_options.checksum = kCRC32c;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  Reopen(options);
+  ASSERT_EQ("b", Get("a"));
+  ASSERT_EQ("d", Get("c"));
+  ASSERT_EQ("f", Get("e"));
+  ASSERT_EQ("h", Get("g"));
 }
 
 // On Windows you can have either memory mapped file or a file
@@ -823,9 +822,6 @@ TEST_F(DBBasicTest, ChecksumTest) {
 // sense to run
 #ifndef OS_WIN
 TEST_F(DBBasicTest, MmapAndBufferOptions) {
-  if (!IsMemoryMappedAccessSupported()) {
-    return;
-  }
   Options options = CurrentOptions();
 
   options.use_direct_reads = true;

db/db_blob_index_test.cc

@@ -1,409 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include <functional>
-#include <string>
-#include <utility>
-#include <vector>
-
-#include "db/column_family.h"
-#include "db/db_iter.h"
-#include "db/db_test_util.h"
-#include "db/dbformat.h"
-#include "db/write_batch_internal.h"
-#include "port/port.h"
-#include "port/stack_trace.h"
-#include "util/string_util.h"
-#include "utilities/merge_operators.h"
-
-namespace rocksdb {
-
-// kTypeBlobIndex is a value type used by BlobDB only. The base rocksdb
-// should accept the value type on write, and report not supported value
-// for reads, unless caller request for it explicitly. The base rocksdb
-// doesn't understand format of actual blob index (the value).
-class DBBlobIndexTest : public DBTestBase {
- public:
-  enum Tier {
-    kMemtable = 0,
-    kImmutableMemtables = 1,
-    kL0SstFile = 2,
-    kLnSstFile = 3,
-  };
-  const std::vector<Tier> kAllTiers = {Tier::kMemtable,
-                                       Tier::kImmutableMemtables,
-                                       Tier::kL0SstFile, Tier::kLnSstFile};
-
-  DBBlobIndexTest() : DBTestBase("/db_blob_index_test") {}
-
-  ColumnFamilyHandle* cfh() { return dbfull()->DefaultColumnFamily(); }
-
-  ColumnFamilyData* cfd() {
-    return reinterpret_cast<ColumnFamilyHandleImpl*>(cfh())->cfd();
-  }
-
-  Status PutBlobIndex(WriteBatch* batch, const Slice& key,
-                      const Slice& blob_index) {
-    return WriteBatchInternal::PutBlobIndex(batch, cfd()->GetID(), key,
-                                            blob_index);
-  }
-
-  Status Write(WriteBatch* batch) {
-    return dbfull()->Write(WriteOptions(), batch);
-  }
-
-  std::string GetImpl(const Slice& key, bool* is_blob_index = nullptr,
-                      const Snapshot* snapshot = nullptr) {
-    ReadOptions read_options;
-    read_options.snapshot = snapshot;
-    PinnableSlice value;
-    auto s = dbfull()->GetImpl(read_options, cfh(), key, &value,
-                               nullptr /*value_found*/, is_blob_index);
-    if (s.IsNotFound()) {
-      return "NOT_FOUND";
-    }
-    if (s.IsNotSupported()) {
-      return "NOT_SUPPORTED";
-    }
-    if (!s.ok()) {
-      return s.ToString();
-    }
-    return value.ToString();
-  }
-
-  std::string GetBlobIndex(const Slice& key,
-                           const Snapshot* snapshot = nullptr) {
-    bool is_blob_index = false;
-    std::string value = GetImpl(key, &is_blob_index, snapshot);
-    if (!is_blob_index) {
-      return "NOT_BLOB";
-    }
-    return value;
-  }
-
-  ArenaWrappedDBIter* GetBlobIterator() {
-    return dbfull()->NewIteratorImpl(ReadOptions(), cfd(),
-                                     dbfull()->GetLatestSequenceNumber(),
-                                     true /*allow_blob*/);
-  }
-
-  Options GetTestOptions() {
-    Options options;
-    options.create_if_missing = true;
-    options.num_levels = 2;
-    options.disable_auto_compactions = true;
-    // Disable auto flushes.
-    options.max_write_buffer_number = 10;
-    options.min_write_buffer_number_to_merge = 10;
-    options.merge_operator = MergeOperators::CreateStringAppendOperator();
-    return options;
-  }
-
-  void MoveDataTo(Tier tier) {
-    switch (tier) {
-      case Tier::kMemtable:
-        break;
-      case Tier::kImmutableMemtables:
-        ASSERT_OK(dbfull()->TEST_SwitchMemtable());
-        break;
-      case Tier::kL0SstFile:
-        ASSERT_OK(Flush());
-        break;
-      case Tier::kLnSstFile:
-        ASSERT_OK(Flush());
-        ASSERT_OK(Put("a", "dummy"));
-        ASSERT_OK(Put("z", "dummy"));
-        ASSERT_OK(Flush());
-        ASSERT_OK(
-            dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
-#ifndef ROCKSDB_LITE
-        ASSERT_EQ("0,1", FilesPerLevel());
-#endif  // !ROCKSDB_LITE
-        break;
-    }
-  }
-};
-
-// Should be able to write kTypeBlobIndex to memtables and SST files.
-TEST_F(DBBlobIndexTest, Write) {
-  for (auto tier : kAllTiers) {
-    DestroyAndReopen(GetTestOptions());
-    for (int i = 1; i <= 5; i++) {
-      std::string index = ToString(i);
-      WriteBatch batch;
-      ASSERT_OK(PutBlobIndex(&batch, "key" + index, "blob" + index));
-      ASSERT_OK(Write(&batch));
-    }
-    MoveDataTo(tier);
-    for (int i = 1; i <= 5; i++) {
-      std::string index = ToString(i);
-      ASSERT_EQ("blob" + index, GetBlobIndex("key" + index));
-    }
-  }
-}
-
-// Get should be able to return blob index if is_blob_index is provided,
-// otherwise return Status::NotSupported status.
-TEST_F(DBBlobIndexTest, Get) {
-  for (auto tier : kAllTiers) {
-    DestroyAndReopen(GetTestOptions());
-    WriteBatch batch;
-    ASSERT_OK(batch.Put("key", "value"));
-    ASSERT_OK(PutBlobIndex(&batch, "blob_key", "blob_index"));
-    ASSERT_OK(Write(&batch));
-    MoveDataTo(tier);
-    // Verify normal value
-    bool is_blob_index = false;
-    PinnableSlice value;
-    ASSERT_EQ("value", Get("key"));
-    ASSERT_EQ("value", GetImpl("key"));
-    ASSERT_EQ("value", GetImpl("key", &is_blob_index));
-    ASSERT_FALSE(is_blob_index);
-    // Verify blob index
-    ASSERT_TRUE(Get("blob_key", &value).IsNotSupported());
-    ASSERT_EQ("NOT_SUPPORTED", GetImpl("blob_key"));
-    ASSERT_EQ("blob_index", GetImpl("blob_key", &is_blob_index));
-    ASSERT_TRUE(is_blob_index);
-  }
-}
-
-// Get should NOT return Status::NotSupported if blob index is updated with
-// a normal value.
-TEST_F(DBBlobIndexTest, Updated) {
-  for (auto tier : kAllTiers) {
-    DestroyAndReopen(GetTestOptions());
-    WriteBatch batch;
-    for (int i = 0; i < 10; i++) {
-      ASSERT_OK(PutBlobIndex(&batch, "key" + ToString(i), "blob_index"));
-    }
-    ASSERT_OK(Write(&batch));
-    // Avoid blob values from being purged.
-    const Snapshot* snapshot = dbfull()->GetSnapshot();
-    ASSERT_OK(Put("key1", "new_value"));
-    ASSERT_OK(Merge("key2", "a"));
-    ASSERT_OK(Merge("key2", "b"));
-    ASSERT_OK(Merge("key2", "c"));
-    ASSERT_OK(Delete("key3"));
-    ASSERT_OK(SingleDelete("key4"));
-    ASSERT_OK(Delete("key5"));
-    ASSERT_OK(Merge("key5", "a"));
-    ASSERT_OK(Merge("key5", "b"));
-    ASSERT_OK(Merge("key5", "c"));
-    ASSERT_OK(dbfull()->DeleteRange(WriteOptions(), cfh(), "key6", "key9"));
-    MoveDataTo(tier);
-    for (int i = 0; i < 10; i++) {
-      ASSERT_EQ("blob_index", GetBlobIndex("key" + ToString(i), snapshot));
-    }
-    ASSERT_EQ("new_value", Get("key1"));
-    ASSERT_EQ("NOT_SUPPORTED", GetImpl("key2"));
-    ASSERT_EQ("NOT_FOUND", Get("key3"));
-    ASSERT_EQ("NOT_FOUND", Get("key4"));
-    ASSERT_EQ("a,b,c", GetImpl("key5"));
-    for (int i = 6; i < 9; i++) {
-      ASSERT_EQ("NOT_FOUND", Get("key" + ToString(i)));
-    }
-    ASSERT_EQ("blob_index", GetBlobIndex("key9"));
-    dbfull()->ReleaseSnapshot(snapshot);
-  }
-}
-
-// Iterator should get blob value if allow_blob flag is set,
-// otherwise return Status::NotSupported status.
-TEST_F(DBBlobIndexTest, Iterate) {
-  const std::vector<std::vector<ValueType>> data = {
-      /*00*/ {kTypeValue},
-      /*01*/ {kTypeBlobIndex},
-      /*02*/ {kTypeValue},
-      /*03*/ {kTypeBlobIndex, kTypeValue},
-      /*04*/ {kTypeValue},
-      /*05*/ {kTypeValue, kTypeBlobIndex},
-      /*06*/ {kTypeValue},
-      /*07*/ {kTypeDeletion, kTypeBlobIndex},
-      /*08*/ {kTypeValue},
-      /*09*/ {kTypeSingleDeletion, kTypeBlobIndex},
-      /*10*/ {kTypeValue},
-      /*11*/ {kTypeMerge, kTypeMerge, kTypeMerge, kTypeBlobIndex},
-      /*12*/ {kTypeValue},
-      /*13*/
-      {kTypeMerge, kTypeMerge, kTypeMerge, kTypeDeletion, kTypeBlobIndex},
-      /*14*/ {kTypeValue},
-      /*15*/ {kTypeBlobIndex},
-      /*16*/ {kTypeValue},
-  };
-
-  auto get_key = [](int index) {
-    char buf[20];
-    snprintf(buf, sizeof(buf), "%02d", index);
-    return "key" + std::string(buf);
-  };
-
-  auto get_value = [&](int index, int version) {
-    return get_key(index) + "_value" + ToString(version);
-  };
-
-  auto check_iterator = [&](Iterator* iterator, Status::Code expected_status,
-                            const Slice& expected_value) {
-    ASSERT_EQ(expected_status, iterator->status().code());
-    if (expected_status == Status::kOk) {
-      ASSERT_TRUE(iterator->Valid());
-      ASSERT_EQ(expected_value, iterator->value());
-    } else {
-      ASSERT_FALSE(iterator->Valid());
-    }
-  };
-
-  auto create_normal_iterator = [&]() -> Iterator* {
-    return dbfull()->NewIterator(ReadOptions());
-  };
-
-  auto create_blob_iterator = [&]() -> Iterator* { return GetBlobIterator(); };
-
-  auto check_is_blob = [&](bool is_blob) {
-    return [is_blob](Iterator* iterator) {
-      ASSERT_EQ(is_blob,
-                reinterpret_cast<ArenaWrappedDBIter*>(iterator)->IsBlob());
-    };
-  };
-
-  auto verify = [&](int index, Status::Code expected_status,
-                    const Slice& forward_value, const Slice& backward_value,
-                    std::function<Iterator*()> create_iterator,
-                    std::function<void(Iterator*)> extra_check = nullptr) {
-    // Seek
-    auto* iterator = create_iterator();
-    ASSERT_OK(iterator->Refresh());
-    iterator->Seek(get_key(index));
-    check_iterator(iterator, expected_status, forward_value);
-    if (extra_check) {
-      extra_check(iterator);
-    }
-    delete iterator;
-
-    // Next
-    iterator = create_iterator();
-    ASSERT_OK(iterator->Refresh());
-    iterator->Seek(get_key(index - 1));
-    ASSERT_TRUE(iterator->Valid());
-    iterator->Next();
-    check_iterator(iterator, expected_status, forward_value);
-    if (extra_check) {
-      extra_check(iterator);
-    }
-    delete iterator;
-
-    // SeekForPrev
-    iterator = create_iterator();
-    ASSERT_OK(iterator->Refresh());
-    iterator->SeekForPrev(get_key(index));
-    check_iterator(iterator, expected_status, backward_value);
-    if (extra_check) {
-      extra_check(iterator);
-    }
-    delete iterator;
-
-    // Prev
-    iterator = create_iterator();
-    iterator->Seek(get_key(index + 1));
-    ASSERT_TRUE(iterator->Valid());
-    iterator->Prev();
-    check_iterator(iterator, expected_status, backward_value);
-    if (extra_check) {
-      extra_check(iterator);
-    }
-    delete iterator;
-  };
-
-  for (auto tier : {Tier::kMemtable} /*kAllTiers*/) {
-    // Avoid values from being purged.
-    std::vector<const Snapshot*> snapshots;
-    DestroyAndReopen(GetTestOptions());
-
-    // fill data
-    for (int i = 0; i < static_cast<int>(data.size()); i++) {
-      for (int j = static_cast<int>(data[i].size()) - 1; j >= 0; j--) {
-        std::string key = get_key(i);
-        std::string value = get_value(i, j);
-        WriteBatch batch;
-        switch (data[i][j]) {
-          case kTypeValue:
-            ASSERT_OK(Put(key, value));
-            break;
-          case kTypeDeletion:
-            ASSERT_OK(Delete(key));
-            break;
-          case kTypeSingleDeletion:
-            ASSERT_OK(SingleDelete(key));
-            break;
-          case kTypeMerge:
-            ASSERT_OK(Merge(key, value));
-            break;
-          case kTypeBlobIndex:
-            ASSERT_OK(PutBlobIndex(&batch, key, value));
-            ASSERT_OK(Write(&batch));
-            break;
-          default:
-            assert(false);
-        };
-      }
-      snapshots.push_back(dbfull()->GetSnapshot());
-    }
-    ASSERT_OK(
-        dbfull()->DeleteRange(WriteOptions(), cfh(), get_key(15), get_key(16)));
-    snapshots.push_back(dbfull()->GetSnapshot());
-    MoveDataTo(tier);
-
-    // Normal iterator
-    verify(1, Status::kNotSupported, "", "", create_normal_iterator);
-    verify(3, Status::kNotSupported, "", "", create_normal_iterator);
-    verify(5, Status::kOk, get_value(5, 0), get_value(5, 0),
-           create_normal_iterator);
-    verify(7, Status::kOk, get_value(8, 0), get_value(6, 0),
-           create_normal_iterator);
-    verify(9, Status::kOk, get_value(10, 0), get_value(8, 0),
-           create_normal_iterator);
-    verify(11, Status::kNotSupported, "", "", create_normal_iterator);
-    verify(13, Status::kOk,
-           get_value(13, 2) + "," + get_value(13, 1) + "," + get_value(13, 0),
-           get_value(13, 2) + "," + get_value(13, 1) + "," + get_value(13, 0),
-           create_normal_iterator);
-    verify(15, Status::kOk, get_value(16, 0), get_value(14, 0),
-           create_normal_iterator);
-
-    // Iterator with blob support
-    verify(1, Status::kOk, get_value(1, 0), get_value(1, 0),
-           create_blob_iterator, check_is_blob(true));
-    verify(3, Status::kOk, get_value(3, 0), get_value(3, 0),
-           create_blob_iterator, check_is_blob(true));
-    verify(5, Status::kOk, get_value(5, 0), get_value(5, 0),
-           create_blob_iterator, check_is_blob(false));
-    verify(7, Status::kOk, get_value(8, 0), get_value(6, 0),
-           create_blob_iterator, check_is_blob(false));
-    verify(9, Status::kOk, get_value(10, 0), get_value(8, 0),
-           create_blob_iterator, check_is_blob(false));
-    verify(11, Status::kNotSupported, "", "", create_blob_iterator);
-    verify(13, Status::kOk,
-           get_value(13, 2) + "," + get_value(13, 1) + "," + get_value(13, 0),
-           get_value(13, 2) + "," + get_value(13, 1) + "," + get_value(13, 0),
-           create_blob_iterator, check_is_blob(false));
-    verify(15, Status::kOk, get_value(16, 0), get_value(14, 0),
-           create_blob_iterator, check_is_blob(false));
-
-    for (auto* snapshot : snapshots) {
-      dbfull()->ReleaseSnapshot(snapshot);
-    }
-  }
-}
-
-}  // namespace rocksdb
-
-int main(int argc, char** argv) {
-  rocksdb::port::InstallStackTraceHandler();
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}

db/db_block_cache_test.cc

@@ -497,7 +497,7 @@ TEST_F(DBBlockCacheTest, CompressedCache) {
         options.compression = kNoCompression;
         break;
       default:
-        FAIL();
+        ASSERT_TRUE(false);
     }
     CreateAndReopenWithCF({"pikachu"}, options);
     // default column family doesn't have block cache
@@ -560,7 +560,7 @@ TEST_F(DBBlockCacheTest, CompressedCache) {
         ASSERT_EQ(TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_HIT), 0);
         break;
       default:
-        FAIL();
+        ASSERT_TRUE(false);
     }
 
     options.create_if_missing = true;

db/db_compaction_test.cc

@@ -1439,60 +1439,6 @@ TEST_F(DBCompactionTest, DeleteFileRange) {
   ASSERT_GT(old_num_files, new_num_files);
 }
 
-TEST_F(DBCompactionTest, DeleteFileRangeFileEndpointsOverlapBug) {
-  // regression test for #2833: groups of files whose user-keys overlap at the
-  // endpoints could be split by `DeleteFilesInRange`. This caused old data to
-  // reappear, either because a new version of the key was removed, or a range
-  // deletion was partially dropped. It could also cause non-overlapping
-  // invariant to be violated if the files dropped by DeleteFilesInRange were
-  // a subset of files that a range deletion spans.
-  const int kNumL0Files = 2;
-  const int kValSize = 8 << 10;  // 8KB
-  Options options = CurrentOptions();
-  options.level0_file_num_compaction_trigger = kNumL0Files;
-  options.target_file_size_base = 1 << 10;  // 1KB
-  DestroyAndReopen(options);
-
-  // The snapshot prevents key 1 from having its old version dropped. The low
-  // `target_file_size_base` ensures two keys will be in each output file.
-  const Snapshot* snapshot = nullptr;
-  Random rnd(301);
-  // The value indicates which flush the key belonged to, which is enough
-  // for us to determine the keys' relative ages. After L0 flushes finish,
-  // files look like:
-  //
-  // File 0: 0 -> vals[0], 1 -> vals[0]
-  // File 1:               1 -> vals[1], 2 -> vals[1]
-  //
-  // Then L0->L1 compaction happens, which outputs keys as follows:
-  //
-  // File 0: 0 -> vals[0], 1 -> vals[1]
-  // File 1:               1 -> vals[0], 2 -> vals[1]
-  //
-  // DeleteFilesInRange shouldn't be allowed to drop just file 0, as that
-  // would cause `1 -> vals[0]` (an older key) to reappear.
-  std::string vals[kNumL0Files];
-  for (int i = 0; i < kNumL0Files; ++i) {
-    vals[i] = RandomString(&rnd, kValSize);
-    Put(Key(i), vals[i]);
-    Put(Key(i + 1), vals[i]);
-    Flush();
-    if (i == 0) {
-      snapshot = db_->GetSnapshot();
-    }
-  }
-  dbfull()->TEST_WaitForCompact();
-
-  // Verify `DeleteFilesInRange` can't drop only file 0 which would cause
-  // "1 -> vals[0]" to reappear.
-  Slice begin = Key(0);
-  Slice end = Key(1);
-  ASSERT_OK(DeleteFilesInRange(db_, db_->DefaultColumnFamily(), &begin, &end));
-  ASSERT_EQ(vals[1], Get(Key(1)));
-
-  db_->ReleaseSnapshot(snapshot);
-}
-
 TEST_P(DBCompactionTestWithParam, TrivialMoveToLastLevelWithFiles) {
   int32_t trivial_move = 0;
   int32_t non_trivial_move = 0;
@@ -1971,7 +1917,7 @@ TEST_F(DBCompactionTest, ManualAutoRace) {
 
   // The auto compaction is scheduled but waited until here
   TEST_SYNC_POINT("DBCompactionTest::ManualAutoRace:1");
-  // The auto compaction will wait until the manual compaction is registerd
+  // The auto compaction will wait until the the manual compaction is registerd
   // before processing so that it will be cancelled.
   dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr, nullptr);
   ASSERT_EQ("0,1", FilesPerLevel(1));
@@ -2738,46 +2684,6 @@ TEST_P(DBCompactionTestWithParam, IntraL0CompactionDoesNotObsoleteDeletions) {
   ASSERT_TRUE(db_->Get(roptions, Key(0), &result).IsNotFound());
 }
 
-TEST_F(DBCompactionTest, OptimizedDeletionObsoleting) {
-  // Deletions can be dropped when compacted to non-last level if they fall
-  // outside the lower-level files' key-ranges.
-  const int kNumL0Files = 4;
-  Options options = CurrentOptions();
-  options.level0_file_num_compaction_trigger = kNumL0Files;
-  options.statistics = rocksdb::CreateDBStatistics();
-  DestroyAndReopen(options);
-
-  // put key 1 and 3 in separate L1, L2 files.
-  // So key 0, 2, and 4+ fall outside these levels' key-ranges.
-  for (int level = 2; level >= 1; --level) {
-    for (int i = 0; i < 2; ++i) {
-      Put(Key(2 * i + 1), "val");
-      Flush();
-    }
-    MoveFilesToLevel(level);
-    ASSERT_EQ(2, NumTableFilesAtLevel(level));
-  }
-
-  // Delete keys in range [1, 4]. These L0 files will be compacted with L1:
-  // - Tombstones for keys 2 and 4 can be dropped early.
-  // - Tombstones for keys 1 and 3 must be kept due to L2 files' key-ranges.
-  for (int i = 0; i < kNumL0Files; ++i) {
-    Put(Key(0), "val");  // sentinel to prevent trivial move
-    Delete(Key(i + 1));
-    Flush();
-  }
-  dbfull()->TEST_WaitForCompact();
-
-  for (int i = 0; i < kNumL0Files; ++i) {
-    std::string value;
-    ASSERT_TRUE(db_->Get(ReadOptions(), Key(i + 1), &value).IsNotFound());
-  }
-  ASSERT_EQ(2, options.statistics->getTickerCount(
-                   COMPACTION_OPTIMIZED_DEL_DROP_OBSOLETE));
-  ASSERT_EQ(2,
-            options.statistics->getTickerCount(COMPACTION_KEY_DROP_OBSOLETE));
-}
-
 INSTANTIATE_TEST_CASE_P(DBCompactionTestWithParam, DBCompactionTestWithParam,
                         ::testing::Values(std::make_tuple(1, true),
                                           std::make_tuple(1, false),

db/db_encryption_test.cc

@@ -1,94 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-#include "db/db_test_util.h"
-#include "port/stack_trace.h"
-#include "rocksdb/perf_context.h"
-#if !defined(ROCKSDB_LITE)
-#include "util/sync_point.h"
-#endif
-#include <iostream>
-#include <string>
-
-namespace rocksdb {
-
-class DBEncryptionTest : public DBTestBase {
- public:
-  DBEncryptionTest() : DBTestBase("/db_encryption_test") {}
-};
-
-#ifndef ROCKSDB_LITE
-
-TEST_F(DBEncryptionTest, CheckEncrypted) {
-  ASSERT_OK(Put("foo567", "v1.fetdq"));
-  ASSERT_OK(Put("bar123", "v2.dfgkjdfghsd"));
-  Close();
-
-  // Open all files and look for the values we've put in there.
-  // They should not be found if encrypted, otherwise
-  // they should be found.
-  std::vector<std::string> fileNames;
-  auto status = env_->GetChildren(dbname_, &fileNames);
-  ASSERT_OK(status);
-
-  auto defaultEnv = Env::Default();
-  int hits = 0;
-  for (auto it = fileNames.begin() ; it != fileNames.end(); ++it) {
-    if ((*it == "..") || (*it == ".")) {
-      continue;
-    }
-    auto filePath = dbname_ + "/" + *it;
-    unique_ptr<SequentialFile> seqFile;
-    auto envOptions = EnvOptions(CurrentOptions());
-    status = defaultEnv->NewSequentialFile(filePath, &seqFile, envOptions);
-    ASSERT_OK(status);
-
-    uint64_t fileSize;
-    status = defaultEnv->GetFileSize(filePath, &fileSize);
-    ASSERT_OK(status);
-
-    std::string scratch;
-    scratch.reserve(fileSize);
-    Slice data;
-    status = seqFile->Read(fileSize, &data, (char*)scratch.data());
-    ASSERT_OK(status);
-
-    if (data.ToString().find("foo567") != std::string::npos) {
-      hits++; 
-      //std::cout << "Hit in " << filePath << "\n";
-    }
-    if (data.ToString().find("v1.fetdq") != std::string::npos) {
-      hits++; 
-      //std::cout << "Hit in " << filePath << "\n";
-    }
-    if (data.ToString().find("bar123") != std::string::npos) {
-      hits++; 
-      //std::cout << "Hit in " << filePath << "\n";
-    }
-    if (data.ToString().find("v2.dfgkjdfghsd") != std::string::npos) {
-      hits++; 
-      //std::cout << "Hit in " << filePath << "\n";
-    }
-    if (data.ToString().find("dfgk") != std::string::npos) {
-      hits++; 
-      //std::cout << "Hit in " << filePath << "\n";
-    }
-  }
-  if (encrypted_env_) {
-    ASSERT_EQ(hits, 0);
-  } else {
-    ASSERT_GE(hits, 4);
-  }
-}
-
-#endif // ROCKSDB_LITE
-
-}  // namespace rocksdb
-
-int main(int argc, char** argv) {
-  rocksdb::port::InstallStackTraceHandler();
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}

db/db_impl.cc

@@ -67,7 +67,6 @@
 #include "port/port.h"
 #include "rocksdb/cache.h"
 #include "rocksdb/compaction_filter.h"
-#include "rocksdb/convenience.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "rocksdb/merge_operator.h"
@@ -81,7 +80,6 @@
 #include "table/merging_iterator.h"
 #include "table/table_builder.h"
 #include "table/two_level_iterator.h"
-#include "tools/sst_dump_tool_imp.h"
 #include "util/auto_roll_logger.h"
 #include "util/autovector.h"
 #include "util/build_version.h"
@@ -160,7 +158,6 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
       is_snapshot_supported_(true),
       write_buffer_manager_(immutable_db_options_.write_buffer_manager.get()),
       write_thread_(immutable_db_options_),
-      nonmem_write_thread_(immutable_db_options_),
       write_controller_(mutable_db_options_.delayed_write_rate),
       // Use delayed_write_rate as a base line to determine the initial
       // low pri write rate limit. It may be adjusted later.
@@ -170,7 +167,6 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
       last_batch_group_size_(0),
       unscheduled_flushes_(0),
       unscheduled_compactions_(0),
-      bg_bottom_compaction_scheduled_(0),
       bg_compaction_scheduled_(0),
       num_running_compactions_(0),
       bg_flush_scheduled_(0),
@@ -191,9 +187,7 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
       bg_work_paused_(0),
       bg_compaction_paused_(0),
       refitting_level_(false),
-      opened_successfully_(false),
-      concurrent_prepare_(options.concurrent_prepare),
-      manual_wal_flush_(options.manual_wal_flush) {
+      opened_successfully_(false) {
   env_->GetAbsolutePath(dbname, &db_absolute_path_);
 
   // Reserve ten files or so for other uses and give the rest to TableCache.
@@ -245,8 +239,7 @@ void DBImpl::CancelAllBackgroundWork(bool wait) {
     return;
   }
   // Wait for background work to finish
-  while (bg_bottom_compaction_scheduled_ || bg_compaction_scheduled_ ||
-         bg_flush_scheduled_) {
+  while (bg_compaction_scheduled_ || bg_flush_scheduled_) {
     bg_cv_.Wait();
   }
 }
@@ -256,18 +249,15 @@ DBImpl::~DBImpl() {
   // marker. After this we do a variant of the waiting and unschedule work
   // (to consider: moving all the waiting into CancelAllBackgroundWork(true))
   CancelAllBackgroundWork(false);
-  int bottom_compactions_unscheduled =
-      env_->UnSchedule(this, Env::Priority::BOTTOM);
   int compactions_unscheduled = env_->UnSchedule(this, Env::Priority::LOW);
   int flushes_unscheduled = env_->UnSchedule(this, Env::Priority::HIGH);
   mutex_.Lock();
-  bg_bottom_compaction_scheduled_ -= bottom_compactions_unscheduled;
   bg_compaction_scheduled_ -= compactions_unscheduled;
   bg_flush_scheduled_ -= flushes_unscheduled;
 
   // Wait for background work to finish
-  while (bg_bottom_compaction_scheduled_ || bg_compaction_scheduled_ ||
-         bg_flush_scheduled_ || bg_purge_scheduled_) {
+  while (bg_compaction_scheduled_ || bg_flush_scheduled_ ||
+         bg_purge_scheduled_) {
     TEST_SYNC_POINT("DBImpl::~DBImpl:WaitJob");
     bg_cv_.Wait();
   }
@@ -620,26 +610,6 @@ int DBImpl::FindMinimumEmptyLevelFitting(ColumnFamilyData* cfd,
   return minimum_level;
 }
 
-Status DBImpl::FlushWAL(bool sync) {
-  {
-    // We need to lock log_write_mutex_ since logs_ might change concurrently
-    InstrumentedMutexLock wl(&log_write_mutex_);
-    log::Writer* cur_log_writer = logs_.back().writer;
-    auto s = cur_log_writer->WriteBuffer();
-    if (!s.ok()) {
-      ROCKS_LOG_ERROR(immutable_db_options_.info_log, "WAL flush error %s",
-                      s.ToString().c_str());
-    }
-    if (!sync) {
-      ROCKS_LOG_DEBUG(immutable_db_options_.info_log, "FlushWAL sync=false");
-      return s;
-    }
-  }
-  // sync = true
-  ROCKS_LOG_DEBUG(immutable_db_options_.info_log, "FlushWAL sync=true");
-  return SyncWAL();
-}
-
 Status DBImpl::SyncWAL() {
   autovector<log::Writer*, 1> logs_to_sync;
   bool need_log_dir_sync;
@@ -678,7 +648,6 @@ Status DBImpl::SyncWAL() {
     need_log_dir_sync = !log_dir_synced_;
   }
 
-  TEST_SYNC_POINT("DBWALTest::SyncWALNotWaitWrite:1");
   RecordTick(stats_, WAL_FILE_SYNCED);
   Status status;
   for (log::Writer* log : logs_to_sync) {
@@ -690,7 +659,6 @@ Status DBImpl::SyncWAL() {
   if (status.ok() && need_log_dir_sync) {
     status = directories_.GetWalDir()->Fsync();
   }
-  TEST_SYNC_POINT("DBWALTest::SyncWALNotWaitWrite:2");
 
   TEST_SYNC_POINT("DBImpl::SyncWAL:BeforeMarkLogsSynced:1");
   {
@@ -909,8 +877,7 @@ Status DBImpl::Get(const ReadOptions& read_options,
 
 Status DBImpl::GetImpl(const ReadOptions& read_options,
                        ColumnFamilyHandle* column_family, const Slice& key,
-                       PinnableSlice* pinnable_val, bool* value_found,
-                       bool* is_blob_index) {
+                       PinnableSlice* pinnable_val, bool* value_found) {
   assert(pinnable_val != nullptr);
   StopWatch sw(env_, stats_, DB_GET);
   PERF_TIMER_GUARD(get_snapshot_time);
@@ -960,13 +927,13 @@ Status DBImpl::GetImpl(const ReadOptions& read_options,
   bool done = false;
   if (!skip_memtable) {
     if (sv->mem->Get(lkey, pinnable_val->GetSelf(), &s, &merge_context,
-                     &range_del_agg, read_options, is_blob_index)) {
+                     &range_del_agg, read_options)) {
       done = true;
       pinnable_val->PinSelf();
       RecordTick(stats_, MEMTABLE_HIT);
     } else if ((s.ok() || s.IsMergeInProgress()) &&
                sv->imm->Get(lkey, pinnable_val->GetSelf(), &s, &merge_context,
-                            &range_del_agg, read_options, is_blob_index)) {
+                            &range_del_agg, read_options)) {
       done = true;
       pinnable_val->PinSelf();
       RecordTick(stats_, MEMTABLE_HIT);
@@ -978,8 +945,7 @@ Status DBImpl::GetImpl(const ReadOptions& read_options,
   if (!done) {
     PERF_TIMER_GUARD(get_from_output_files_time);
     sv->current->Get(read_options, lkey, pinnable_val, &s, &merge_context,
-                     &range_del_agg, value_found, nullptr, nullptr,
-                     is_blob_index);
+                     &range_del_agg, value_found);
     RecordTick(stats_, MEMTABLE_MISS);
   }
 
@@ -992,7 +958,6 @@ Status DBImpl::GetImpl(const ReadOptions& read_options,
     size_t size = pinnable_val->size();
     RecordTick(stats_, BYTES_READ, size);
     MeasureTime(stats_, BYTES_PER_READ, size);
-    PERF_COUNTER_ADD(get_read_bytes, size);
   }
   return s;
 }
@@ -1120,7 +1085,6 @@ std::vector<Status> DBImpl::MultiGet(
   RecordTick(stats_, NUMBER_MULTIGET_KEYS_READ, num_keys);
   RecordTick(stats_, NUMBER_MULTIGET_BYTES_READ, bytes_read);
   MeasureTime(stats_, BYTES_PER_MULTIGET, bytes_read);
-  PERF_COUNTER_ADD(multiget_read_bytes, bytes_read);
   PERF_TIMER_STOP(get_post_process_time);
 
   return stat_list;
@@ -1326,11 +1290,6 @@ Status DBImpl::DropColumnFamilyImpl(ColumnFamilyHandle* column_family) {
                                  &edit, &mutex_);
       write_thread_.ExitUnbatched(&w);
     }
-    if (s.ok()) {
-      auto* mutable_cf_options = cfd->GetLatestMutableCFOptions();
-      max_total_in_memory_state_ -= mutable_cf_options->write_buffer_size *
-                                    mutable_cf_options->max_write_buffer_number;
-    }
 
     if (!cf_support_snapshot) {
       // Dropped Column Family doesn't support snapshot. Need to recalculate
@@ -1352,6 +1311,9 @@ Status DBImpl::DropColumnFamilyImpl(ColumnFamilyHandle* column_family) {
     // later inside db_mutex.
     EraseThreadStatusCfInfo(cfd);
     assert(cfd->IsDropped());
+    auto* mutable_cf_options = cfd->GetLatestMutableCFOptions();
+    max_total_in_memory_state_ -= mutable_cf_options->write_buffer_size *
+                                  mutable_cf_options->max_write_buffer_number;
     ROCKS_LOG_INFO(immutable_db_options_.info_log,
                    "Dropped column family with id %u\n", cfd->GetID());
   } else {
@@ -1415,83 +1377,77 @@ Iterator* DBImpl::NewIterator(const ReadOptions& read_options,
     return NewDBIterator(
         env_, read_options, *cfd->ioptions(), cfd->user_comparator(), iter,
         kMaxSequenceNumber,
-        sv->mutable_cf_options.max_sequential_skip_in_iterations);
+        sv->mutable_cf_options.max_sequential_skip_in_iterations,
+        sv->version_number);
 #endif
   } else {
     SequenceNumber latest_snapshot = versions_->LastSequence();
+    SuperVersion* sv = cfd->GetReferencedSuperVersion(&mutex_);
+
     auto snapshot =
         read_options.snapshot != nullptr
-            ? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
-                  ->number_
+            ? reinterpret_cast<const SnapshotImpl*>(
+                read_options.snapshot)->number_
             : latest_snapshot;
-    return NewIteratorImpl(read_options, cfd, snapshot);
+
+    // Try to generate a DB iterator tree in continuous memory area to be
+    // cache friendly. Here is an example of result:
+    // +-------------------------------+
+    // |                               |
+    // | ArenaWrappedDBIter            |
+    // |  +                            |
+    // |  +---> Inner Iterator   ------------+
+    // |  |                            |     |
+    // |  |    +-- -- -- -- -- -- -- --+     |
+    // |  +--- | Arena                 |     |
+    // |       |                       |     |
+    // |          Allocated Memory:    |     |
+    // |       |   +-------------------+     |
+    // |       |   | DBIter            | <---+
+    // |           |  +                |
+    // |       |   |  +-> iter_  ------------+
+    // |       |   |                   |     |
+    // |       |   +-------------------+     |
+    // |       |   | MergingIterator   | <---+
+    // |           |  +                |
+    // |       |   |  +->child iter1  ------------+
+    // |       |   |  |                |          |
+    // |           |  +->child iter2  ----------+ |
+    // |       |   |  |                |        | |
+    // |       |   |  +->child iter3  --------+ | |
+    // |           |                   |      | | |
+    // |       |   +-------------------+      | | |
+    // |       |   | Iterator1         | <--------+
+    // |       |   +-------------------+      | |
+    // |       |   | Iterator2         | <------+
+    // |       |   +-------------------+      |
+    // |       |   | Iterator3         | <----+
+    // |       |   +-------------------+
+    // |       |                       |
+    // +-------+-----------------------+
+    //
+    // ArenaWrappedDBIter inlines an arena area where all the iterators in
+    // the iterator tree are allocated in the order of being accessed when
+    // querying.
+    // Laying out the iterators in the order of being accessed makes it more
+    // likely that any iterator pointer is close to the iterator it points to so
+    // that they are likely to be in the same cache line and/or page.
+    ArenaWrappedDBIter* db_iter = NewArenaWrappedDbIterator(
+        env_, read_options, *cfd->ioptions(), cfd->user_comparator(), snapshot,
+        sv->mutable_cf_options.max_sequential_skip_in_iterations,
+        sv->version_number);
+
+    InternalIterator* internal_iter =
+        NewInternalIterator(read_options, cfd, sv, db_iter->GetArena(),
+                            db_iter->GetRangeDelAggregator());
+    db_iter->SetIterUnderDBIter(internal_iter);
+
+    return db_iter;
   }
   // To stop compiler from complaining
   return nullptr;
 }
 
-ArenaWrappedDBIter* DBImpl::NewIteratorImpl(const ReadOptions& read_options,
-                                            ColumnFamilyData* cfd,
-                                            SequenceNumber snapshot,
-                                            bool allow_blob) {
-  SuperVersion* sv = cfd->GetReferencedSuperVersion(&mutex_);
-
-  // Try to generate a DB iterator tree in continuous memory area to be
-  // cache friendly. Here is an example of result:
-  // +-------------------------------+
-  // |                               |
-  // | ArenaWrappedDBIter            |
-  // |  +                            |
-  // |  +---> Inner Iterator   ------------+
-  // |  |                            |     |
-  // |  |    +-- -- -- -- -- -- -- --+     |
-  // |  +--- | Arena                 |     |
-  // |       |                       |     |
-  // |          Allocated Memory:    |     |
-  // |       |   +-------------------+     |
-  // |       |   | DBIter            | <---+
-  // |           |  +                |
-  // |       |   |  +-> iter_  ------------+
-  // |       |   |                   |     |
-  // |       |   +-------------------+     |
-  // |       |   | MergingIterator   | <---+
-  // |           |  +                |
-  // |       |   |  +->child iter1  ------------+
-  // |       |   |  |                |          |
-  // |           |  +->child iter2  ----------+ |
-  // |       |   |  |                |        | |
-  // |       |   |  +->child iter3  --------+ | |
-  // |           |                   |      | | |
-  // |       |   +-------------------+      | | |
-  // |       |   | Iterator1         | <--------+
-  // |       |   +-------------------+      | |
-  // |       |   | Iterator2         | <------+
-  // |       |   +-------------------+      |
-  // |       |   | Iterator3         | <----+
-  // |       |   +-------------------+
-  // |       |                       |
-  // +-------+-----------------------+
-  //
-  // ArenaWrappedDBIter inlines an arena area where all the iterators in
-  // the iterator tree are allocated in the order of being accessed when
-  // querying.
-  // Laying out the iterators in the order of being accessed makes it more
-  // likely that any iterator pointer is close to the iterator it points to so
-  // that they are likely to be in the same cache line and/or page.
-  ArenaWrappedDBIter* db_iter = NewArenaWrappedDbIterator(
-      env_, read_options, *cfd->ioptions(), snapshot,
-      sv->mutable_cf_options.max_sequential_skip_in_iterations,
-      sv->version_number, ((read_options.snapshot != nullptr) ? nullptr : this),
-      cfd, allow_blob);
-
-  InternalIterator* internal_iter =
-      NewInternalIterator(read_options, cfd, sv, db_iter->GetArena(),
-                          db_iter->GetRangeDelAggregator());
-  db_iter->SetIterUnderDBIter(internal_iter);
-
-  return db_iter;
-}
-
 Status DBImpl::NewIterators(
     const ReadOptions& read_options,
     const std::vector<ColumnFamilyHandle*>& column_families,
@@ -1530,21 +1486,33 @@ Status DBImpl::NewIterators(
       iterators->push_back(NewDBIterator(
           env_, read_options, *cfd->ioptions(), cfd->user_comparator(), iter,
           kMaxSequenceNumber,
-          sv->mutable_cf_options.max_sequential_skip_in_iterations));
+          sv->mutable_cf_options.max_sequential_skip_in_iterations,
+          sv->version_number));
     }
 #endif
   } else {
     SequenceNumber latest_snapshot = versions_->LastSequence();
-    auto snapshot =
-        read_options.snapshot != nullptr
-            ? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
-                  ->number_
-            : latest_snapshot;
 
     for (size_t i = 0; i < column_families.size(); ++i) {
       auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(
           column_families[i])->cfd();
-      iterators->push_back(NewIteratorImpl(read_options, cfd, snapshot));
+      SuperVersion* sv = cfd->GetReferencedSuperVersion(&mutex_);
+
+      auto snapshot =
+          read_options.snapshot != nullptr
+              ? reinterpret_cast<const SnapshotImpl*>(
+                  read_options.snapshot)->number_
+              : latest_snapshot;
+
+      ArenaWrappedDBIter* db_iter = NewArenaWrappedDbIterator(
+          env_, read_options, *cfd->ioptions(), cfd->user_comparator(),
+          snapshot, sv->mutable_cf_options.max_sequential_skip_in_iterations,
+          sv->version_number);
+      InternalIterator* internal_iter =
+          NewInternalIterator(read_options, cfd, sv, db_iter->GetArena(),
+                              db_iter->GetRangeDelAggregator());
+      db_iter->SetIterUnderDBIter(internal_iter);
+      iterators->push_back(db_iter);
     }
   }
 
@@ -1583,10 +1551,12 @@ void DBImpl::ReleaseSnapshot(const Snapshot* s) {
   delete casted_s;
 }
 
-bool DBImpl::HasActiveSnapshotInRange(SequenceNumber lower_bound,
-                                      SequenceNumber upper_bound) {
+bool DBImpl::HasActiveSnapshotLaterThanSN(SequenceNumber sn) {
   InstrumentedMutexLock l(&mutex_);
-  return snapshots_.HasSnapshotInRange(lower_bound, upper_bound);
+  if (snapshots_.empty()) {
+    return false;
+  }
+  return (snapshots_.newest()->GetSequenceNumber() > sn);
 }
 
 #ifndef ROCKSDB_LITE
@@ -2062,19 +2032,25 @@ Status DBImpl::DeleteFilesInRange(ColumnFamilyHandle* column_family,
         end_key = &end_storage;
       }
 
-      vstorage->GetCleanInputsWithinInterval(i, begin_key, end_key,
-                                             &level_files, -1 /* hint_index */,
-                                             nullptr /* file_index */);
+      vstorage->GetOverlappingInputs(i, begin_key, end_key, &level_files, -1,
+                                     nullptr, false);
       FileMetaData* level_file;
       for (uint32_t j = 0; j < level_files.size(); j++) {
         level_file = level_files[j];
-        if (level_file->being_compacted) {
-          continue;
+        if (((begin == nullptr) ||
+             (cfd->internal_comparator().user_comparator()->Compare(
+                  level_file->smallest.user_key(), *begin) >= 0)) &&
+            ((end == nullptr) ||
+             (cfd->internal_comparator().user_comparator()->Compare(
+                  level_file->largest.user_key(), *end) <= 0))) {
+          if (level_file->being_compacted) {
+            continue;
+          }
+          edit.SetColumnFamily(cfd->GetID());
+          edit.DeleteFile(i, level_file->fd.GetNumber());
+          deleted_files.push_back(level_file);
+          level_file->being_compacted = true;
         }
-        edit.SetColumnFamily(cfd->GetID());
-        edit.DeleteFile(i, level_file->fd.GetNumber());
-        deleted_files.push_back(level_file);
-        level_file->being_compacted = true;
       }
     }
     if (edit.GetDeletedFiles().empty()) {
@@ -2292,7 +2268,7 @@ Status DestroyDB(const std::string& dbname, const Options& options) {
     // Delete log files in the WAL dir
     for (const auto& file : walDirFiles) {
       if (ParseFileName(file, &number, &type) && type == kLogFile) {
-        Status del = env->DeleteFile(LogFileName(soptions.wal_dir, number));
+        Status del = env->DeleteFile(soptions.wal_dir + "/" + file);
         if (result.ok() && !del.ok()) {
           result = del;
         }
@@ -2520,8 +2496,7 @@ SequenceNumber DBImpl::GetEarliestMemTableSequenceNumber(SuperVersion* sv,
 #ifndef ROCKSDB_LITE
 Status DBImpl::GetLatestSequenceForKey(SuperVersion* sv, const Slice& key,
                                        bool cache_only, SequenceNumber* seq,
-                                       bool* found_record_for_key,
-                                       bool* is_blob_index) {
+                                       bool* found_record_for_key) {
   Status s;
   MergeContext merge_context;
   RangeDelAggregator range_del_agg(sv->mem->GetInternalKeyComparator(),
@@ -2536,7 +2511,7 @@ Status DBImpl::GetLatestSequenceForKey(SuperVersion* sv, const Slice& key,
 
   // Check if there is a record for this key in the latest memtable
   sv->mem->Get(lkey, nullptr, &s, &merge_context, &range_del_agg, seq,
-               read_options, is_blob_index);
+               read_options);
 
   if (!(s.ok() || s.IsNotFound() || s.IsMergeInProgress())) {
     // unexpected error reading memtable.
@@ -2555,7 +2530,7 @@ Status DBImpl::GetLatestSequenceForKey(SuperVersion* sv, const Slice& key,
 
   // Check if there is a record for this key in the immutable memtables
   sv->imm->Get(lkey, nullptr, &s, &merge_context, &range_del_agg, seq,
-               read_options, is_blob_index);
+               read_options);
 
   if (!(s.ok() || s.IsNotFound() || s.IsMergeInProgress())) {
     // unexpected error reading memtable.
@@ -2574,7 +2549,7 @@ Status DBImpl::GetLatestSequenceForKey(SuperVersion* sv, const Slice& key,
 
   // Check if there is a record for this key in the immutable memtables
   sv->imm->GetFromHistory(lkey, nullptr, &s, &merge_context, &range_del_agg,
-                          seq, read_options, is_blob_index);
+                          seq, read_options);
 
   if (!(s.ok() || s.IsNotFound() || s.IsMergeInProgress())) {
     // unexpected error reading memtable.
@@ -2598,7 +2573,7 @@ Status DBImpl::GetLatestSequenceForKey(SuperVersion* sv, const Slice& key,
     // Check tables
     sv->current->Get(read_options, lkey, nullptr, &s, &merge_context,
                      &range_del_agg, nullptr /* value_found */,
-                     found_record_for_key, seq, is_blob_index);
+                     found_record_for_key, seq);
 
     if (!(s.ok() || s.IsNotFound() || s.IsMergeInProgress())) {
       // unexpected error reading SST files
@@ -2657,13 +2632,9 @@ Status DBImpl::IngestExternalFile(
     InstrumentedMutexLock l(&mutex_);
     TEST_SYNC_POINT("DBImpl::AddFile:MutexLock");
 
-    // Stop writes to the DB by entering both write threads
+    // Stop writes to the DB
     WriteThread::Writer w;
     write_thread_.EnterUnbatched(&w, &mutex_);
-    WriteThread::Writer nonmem_w;
-    if (concurrent_prepare_) {
-      nonmem_write_thread_.EnterUnbatched(&nonmem_w, &mutex_);
-    }
 
     num_running_ingest_file_++;
 
@@ -2704,9 +2675,6 @@ Status DBImpl::IngestExternalFile(
     }
 
     // Resume writes to the DB
-    if (concurrent_prepare_) {
-      nonmem_write_thread_.ExitUnbatched(&nonmem_w);
-    }
     write_thread_.ExitUnbatched(&w);
 
     // Update stats
@@ -2735,54 +2703,6 @@ Status DBImpl::IngestExternalFile(
   return status;
 }
 
-Status DBImpl::VerifyChecksum() {
-  Status s;
-  Options options;
-  EnvOptions env_options;
-  std::vector<ColumnFamilyData*> cfd_list;
-  {
-    InstrumentedMutexLock l(&mutex_);
-    for (auto cfd : *versions_->GetColumnFamilySet()) {
-      if (!cfd->IsDropped() && cfd->initialized()) {
-        cfd->Ref();
-        cfd_list.push_back(cfd);
-      }
-    }
-  }
-  std::vector<SuperVersion*> sv_list;
-  for (auto cfd : cfd_list) {
-    sv_list.push_back(cfd->GetReferencedSuperVersion(&mutex_));
-  }
-  for (auto& sv : sv_list) {
-    VersionStorageInfo* vstorage = sv->current->storage_info();
-    for (int i = 0; i < vstorage->num_non_empty_levels() && s.ok(); i++) {
-      for (size_t j = 0; j < vstorage->LevelFilesBrief(i).num_files && s.ok();
-           j++) {
-        const auto& fd = vstorage->LevelFilesBrief(i).files[j].fd;
-        std::string fname = TableFileName(immutable_db_options_.db_paths,
-                                          fd.GetNumber(), fd.GetPathId());
-        s = rocksdb::VerifySstFileChecksum(options, env_options, fname);
-      }
-    }
-    if (!s.ok()) {
-      break;
-    }
-  }
-  {
-    InstrumentedMutexLock l(&mutex_);
-    for (auto sv : sv_list) {
-      if (sv && sv->Unref()) {
-        sv->Cleanup();
-        delete sv;
-      }
-    }
-    for (auto cfd : cfd_list) {
-        cfd->Unref();
-    }
-  }
-  return s;
-}
-
 void DBImpl::NotifyOnExternalFileIngested(
     ColumnFamilyData* cfd, const ExternalSstFileIngestionJob& ingestion_job) {
 #ifndef ROCKSDB_LITE

db/db_impl.h

@@ -52,7 +52,6 @@
 
 namespace rocksdb {
 
-class ArenaWrappedDBIter;
 class MemTable;
 class TableCache;
 class Version;
@@ -94,13 +93,6 @@ class DBImpl : public DB {
   virtual Status Get(const ReadOptions& options,
                      ColumnFamilyHandle* column_family, const Slice& key,
                      PinnableSlice* value) override;
-
-  // Function that Get and KeyMayExist call with no_io true or false
-  // Note: 'value_found' from KeyMayExist propagates here
-  Status GetImpl(const ReadOptions& options, ColumnFamilyHandle* column_family,
-                 const Slice& key, PinnableSlice* value,
-                 bool* value_found = nullptr, bool* is_blob_index = nullptr);
-
   using DB::MultiGet;
   virtual std::vector<Status> MultiGet(
       const ReadOptions& options,
@@ -131,7 +123,6 @@ class DBImpl : public DB {
                            ColumnFamilyHandle* column_family, const Slice& key,
                            std::string* value,
                            bool* value_found = nullptr) override;
-
   using DB::NewIterator;
   virtual Iterator* NewIterator(const ReadOptions& options,
                                 ColumnFamilyHandle* column_family) override;
@@ -139,11 +130,6 @@ class DBImpl : public DB {
       const ReadOptions& options,
       const std::vector<ColumnFamilyHandle*>& column_families,
       std::vector<Iterator*>* iterators) override;
-  ArenaWrappedDBIter* NewIteratorImpl(const ReadOptions& options,
-                                      ColumnFamilyData* cfd,
-                                      SequenceNumber snapshot,
-                                      bool allow_blob = false);
-
   virtual const Snapshot* GetSnapshot() override;
   virtual void ReleaseSnapshot(const Snapshot* snapshot) override;
   using DB::GetProperty;
@@ -211,14 +197,11 @@ class DBImpl : public DB {
   using DB::Flush;
   virtual Status Flush(const FlushOptions& options,
                        ColumnFamilyHandle* column_family) override;
-  virtual Status FlushWAL(bool sync) override;
   virtual Status SyncWAL() override;
 
   virtual SequenceNumber GetLatestSequenceNumber() const override;
 
-  // Whether there is an active snapshot in range [lower_bound, upper_bound).
-  bool HasActiveSnapshotInRange(SequenceNumber lower_bound,
-                                SequenceNumber upper_bound);
+  bool HasActiveSnapshotLaterThanSN(SequenceNumber sn);
 
 #ifndef ROCKSDB_LITE
   using DB::ResetStats;
@@ -251,11 +234,11 @@ class DBImpl : public DB {
       ColumnFamilyHandle* column_family,
       ColumnFamilyMetaData* metadata) override;
 
+  // experimental API
   Status SuggestCompactRange(ColumnFamilyHandle* column_family,
-                             const Slice* begin, const Slice* end) override;
+                             const Slice* begin, const Slice* end);
 
-  Status PromoteL0(ColumnFamilyHandle* column_family,
-                   int target_level) override;
+  Status PromoteL0(ColumnFamilyHandle* column_family, int target_level);
 
   // Similar to Write() but will call the callback once on the single write
   // thread to determine whether it is safe to perform the write.
@@ -301,8 +284,7 @@ class DBImpl : public DB {
   // TODO(andrewkr): this API need to be aware of range deletion operations
   Status GetLatestSequenceForKey(SuperVersion* sv, const Slice& key,
                                  bool cache_only, SequenceNumber* seq,
-                                 bool* found_record_for_key,
-                                 bool* is_blob_index = nullptr);
+                                 bool* found_record_for_key);
 
   using DB::IngestExternalFile;
   virtual Status IngestExternalFile(
@@ -310,8 +292,6 @@ class DBImpl : public DB {
       const std::vector<std::string>& external_files,
       const IngestExternalFileOptions& ingestion_options) override;
 
-  virtual Status VerifyChecksum() override;
-
 #endif  // ROCKSDB_LITE
 
   // Similar to GetSnapshot(), but also lets the db know that this snapshot
@@ -358,8 +338,6 @@ class DBImpl : public DB {
     return alive_log_files_.begin()->getting_flushed;
   }
 
-  Status TEST_SwitchMemtable(ColumnFamilyData* cfd = nullptr);
-
   // Force current memtable contents to be flushed.
   Status TEST_FlushMemTable(bool wait = true,
                             ColumnFamilyHandle* cfh = nullptr);
@@ -515,12 +493,6 @@ class DBImpl : public DB {
 
   const WriteController& write_controller() { return write_controller_; }
 
-  InternalIterator* NewInternalIterator(const ReadOptions&,
-                                        ColumnFamilyData* cfd,
-                                        SuperVersion* super_version,
-                                        Arena* arena,
-                                        RangeDelAggregator* range_del_agg);
-
   // hollow transactions shell used for recovery.
   // these will then be passed to TransactionDB so that
   // locks can be reacquired before writing can resume.
@@ -579,7 +551,6 @@ class DBImpl : public DB {
   void AddToLogsToFreeQueue(log::Writer* log_writer) {
     logs_to_free_queue_.push_back(log_writer);
   }
-  InstrumentedMutex* mutex() { return &mutex_; }
 
   Status NewDB();
 
@@ -594,6 +565,12 @@ class DBImpl : public DB {
   std::unordered_map<std::string, RecoveredTransaction*>
       recovered_transactions_;
 
+  InternalIterator* NewInternalIterator(const ReadOptions&,
+                                        ColumnFamilyData* cfd,
+                                        SuperVersion* super_version,
+                                        Arena* arena,
+                                        RangeDelAggregator* range_del_agg);
+
   // Except in DB::Open(), WriteOptionsFile can only be called when:
   // Persist options to options file.
   // If need_mutex_lock = false, the method will lock DB mutex.
@@ -635,18 +612,12 @@ class DBImpl : public DB {
   Status WriteImpl(const WriteOptions& options, WriteBatch* updates,
                    WriteCallback* callback = nullptr,
                    uint64_t* log_used = nullptr, uint64_t log_ref = 0,
-                   bool disable_memtable = false, uint64_t* seq_used = nullptr);
+                   bool disable_memtable = false);
 
   Status PipelinedWriteImpl(const WriteOptions& options, WriteBatch* updates,
                             WriteCallback* callback = nullptr,
                             uint64_t* log_used = nullptr, uint64_t log_ref = 0,
-                            bool disable_memtable = false,
-                            uint64_t* seq_used = nullptr);
-
-  Status WriteImplWALOnly(const WriteOptions& options, WriteBatch* updates,
-                          WriteCallback* callback = nullptr,
-                          uint64_t* log_used = nullptr, uint64_t log_ref = 0,
-                          uint64_t* seq_used = nullptr);
+                            bool disable_memtable = false);
 
   uint64_t FindMinLogContainingOutstandingPrep();
   uint64_t FindMinPrepLogReferencedByMemTable();
@@ -654,18 +625,14 @@ class DBImpl : public DB {
  private:
   friend class DB;
   friend class InternalStats;
-  friend class PessimisticTransaction;
-  friend class WriteCommittedTxn;
-  friend class WritePreparedTxn;
+  friend class TransactionImpl;
 #ifndef ROCKSDB_LITE
   friend class ForwardIterator;
 #endif
   friend struct SuperVersion;
   friend class CompactedDBImpl;
 #ifndef NDEBUG
-  friend class DBTest2_ReadCallbackTest_Test;
   friend class XFTransactionWriteHandler;
-  friend class DBBlobIndexTest;
 #endif
   struct CompactionState;
 
@@ -683,7 +650,6 @@ class DBImpl : public DB {
     }
   };
 
-  struct PrepickedCompaction;
   struct PurgeFileInfo;
 
   // Recover the descriptor from persistent storage.  May do a significant
@@ -778,23 +744,12 @@ class DBImpl : public DB {
   Status PreprocessWrite(const WriteOptions& write_options, bool* need_log_sync,
                          WriteContext* write_context);
 
-  WriteBatch* MergeBatch(const WriteThread::WriteGroup& write_group,
-                         WriteBatch* tmp_batch, size_t* write_with_wal);
-
-  Status WriteToWAL(const WriteBatch& merged_batch, log::Writer* log_writer,
-                    uint64_t* log_used, uint64_t* log_size);
-
   Status WriteToWAL(const WriteThread::WriteGroup& write_group,
-                    log::Writer* log_writer, uint64_t* log_used,
-                    bool need_log_sync, bool need_log_dir_sync,
-                    SequenceNumber sequence);
-
-  Status ConcurrentWriteToWAL(const WriteThread::WriteGroup& write_group,
-                              uint64_t* log_used, SequenceNumber* last_sequence,
-                              int total_count);
+                    log::Writer* log_writer, bool need_log_sync,
+                    bool need_log_dir_sync, SequenceNumber sequence);
 
   // Used by WriteImpl to update bg_error_ if paranoid check is enabled.
-  void WriteCallbackStatusCheck(const Status& status);
+  void ParanoidCheck(const Status& status);
 
   // Used by WriteImpl to update bg_error_ in case of memtable insert error.
   void MemTableInsertStatusCheck(const Status& memtable_insert_status);
@@ -825,19 +780,14 @@ class DBImpl : public DB {
   void SchedulePendingPurge(std::string fname, FileType type, uint64_t number,
                             uint32_t path_id, int job_id);
   static void BGWorkCompaction(void* arg);
-  // Runs a pre-chosen universal compaction involving bottom level in a
-  // separate, bottom-pri thread pool.
-  static void BGWorkBottomCompaction(void* arg);
   static void BGWorkFlush(void* db);
   static void BGWorkPurge(void* arg);
   static void UnscheduleCallback(void* arg);
-  void BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction,
-                                Env::Priority bg_thread_pri);
+  void BackgroundCallCompaction(void* arg);
   void BackgroundCallFlush();
   void BackgroundCallPurge();
   Status BackgroundCompaction(bool* madeProgress, JobContext* job_context,
-                              LogBuffer* log_buffer,
-                              PrepickedCompaction* prepicked_compaction);
+                              LogBuffer* log_buffer, void* m = 0);
   Status BackgroundFlush(bool* madeProgress, JobContext* job_context,
                          LogBuffer* log_buffer);
 
@@ -875,16 +825,11 @@ class DBImpl : public DB {
   // Lock over the persistent DB state.  Non-nullptr iff successfully acquired.
   FileLock* db_lock_;
 
-  // In addition to mutex_, log_write_mutex_ protected writes to logs_ and
-  // logfile_number_. With concurrent_prepare it also protects alive_log_files_,
-  // and log_empty_. Refer to the definition of each variable below for more
-  // details.
-  InstrumentedMutex log_write_mutex_;
+  // The mutex for options file related operations.
+  // NOTE: should never acquire options_file_mutex_ and mutex_ at the
+  //       same time.
+  InstrumentedMutex options_files_mutex_;
   // State below is protected by mutex_
-  // With concurrent_prepare enabled, some of the variables that accessed during
-  // WriteToWAL need different synchronization: log_empty_, alive_log_files_,
-  // logs_, logfile_number_. Refer to the definition of each variable below for
-  // more description.
   mutable InstrumentedMutex mutex_;
 
   std::atomic<bool> shutting_down_;
@@ -898,20 +843,10 @@ class DBImpl : public DB {
   // * whenever there is an error in background purge, flush or compaction
   // * whenever num_running_ingest_file_ goes to 0.
   InstrumentedCondVar bg_cv_;
-  // Writes are protected by locking both mutex_ and log_write_mutex_, and reads
-  // must be under either mutex_ or log_write_mutex_. Since after ::Open,
-  // logfile_number_ is currently updated only in write_thread_, it can be read
-  // from the same write_thread_ without any locks.
   uint64_t logfile_number_;
   std::deque<uint64_t>
       log_recycle_files;  // a list of log files that we can recycle
   bool log_dir_synced_;
-  // Without concurrent_prepare, read and writes to log_empty_ are protected by
-  // mutex_. Since it is currently updated/read only in write_thread_, it can be
-  // accessed from the same write_thread_ without any locks. With
-  // concurrent_prepare writes, where it can be updated in different threads,
-  // read and writes are protected by log_write_mutex_ instead. This is to avoid
-  // expesnive mutex_ lock during WAL write, which update log_empty_.
   bool log_empty_;
   ColumnFamilyHandleImpl* default_cf_handle_;
   InternalStats* default_cf_internal_stats_;
@@ -946,26 +881,14 @@ class DBImpl : public DB {
     // true for some prefix of logs_
     bool getting_synced = false;
   };
-  // Without concurrent_prepare, read and writes to alive_log_files_ are
-  // protected by mutex_. However since back() is never popped, and push_back()
-  // is done only from write_thread_, the same thread can access the item
-  // reffered by back() without mutex_. With concurrent_prepare_, writes
-  // are protected by locking both mutex_ and log_write_mutex_, and reads must
-  // be under either mutex_ or log_write_mutex_.
   std::deque<LogFileNumberSize> alive_log_files_;
   // Log files that aren't fully synced, and the current log file.
   // Synchronization:
-  //  - push_back() is done from write_thread_ with locked mutex_ and
-  //  log_write_mutex_
-  //  - pop_front() is done from any thread with locked mutex_ and
-  //  log_write_mutex_
-  //  - reads are done with either locked mutex_ or log_write_mutex_
+  //  - push_back() is done from write thread with locked mutex_,
+  //  - pop_front() is done from any thread with locked mutex_,
   //  - back() and items with getting_synced=true are not popped,
-  //  - The same thread that sets getting_synced=true will reset it.
-  //  - it follows that the object referred by back() can be safely read from
-  //  the write_thread_ without using mutex
-  //  - it follows that the items with getting_synced=true can be safely read
-  //  from the same thread that has set getting_synced=true
+  //  - it follows that write thread with unlocked mutex_ can safely access
+  //    back() and items with getting_synced=true.
   std::deque<LogWriterNumber> logs_;
   // Signaled when getting_synced becomes false for some of the logs_.
   InstrumentedCondVar log_sync_cv_;
@@ -1014,10 +937,8 @@ class DBImpl : public DB {
   WriteBufferManager* write_buffer_manager_;
 
   WriteThread write_thread_;
+
   WriteBatch tmp_batch_;
-  // The write thread when the writers have no memtable write. This will be used
-  // in 2PC to batch the prepares separately from the serial commit.
-  WriteThread nonmem_write_thread_;
 
   WriteController write_controller_;
 
@@ -1025,8 +946,6 @@ class DBImpl : public DB {
 
   // Size of the last batch group. In slowdown mode, next write needs to
   // sleep if it uses up the quota.
-  // Note: This is to protect memtable and compaction. If the batch only writes
-  // to the WAL its size need not to be included in this.
   uint64_t last_batch_group_size_;
 
   FlushScheduler flush_scheduler_;
@@ -1090,10 +1009,6 @@ class DBImpl : public DB {
   int unscheduled_flushes_;
   int unscheduled_compactions_;
 
-  // count how many background compactions are running or have been scheduled in
-  // the BOTTOM pool
-  int bg_bottom_compaction_scheduled_;
-
   // count how many background compactions are running or have been scheduled
   int bg_compaction_scheduled_;
 
@@ -1110,7 +1025,7 @@ class DBImpl : public DB {
   int bg_purge_scheduled_;
 
   // Information for a manual compaction
-  struct ManualCompactionState {
+  struct ManualCompaction {
     ColumnFamilyData* cfd;
     int input_level;
     int output_level;
@@ -1126,21 +1041,13 @@ class DBImpl : public DB {
     InternalKey* manual_end;      // how far we are compacting
     InternalKey tmp_storage;      // Used to keep track of compaction progress
     InternalKey tmp_storage1;     // Used to keep track of compaction progress
-  };
-  struct PrepickedCompaction {
-    // background compaction takes ownership of `compaction`.
     Compaction* compaction;
-    // caller retains ownership of `manual_compaction_state` as it is reused
-    // across background compactions.
-    ManualCompactionState* manual_compaction_state;  // nullptr if non-manual
   };
-  std::deque<ManualCompactionState*> manual_compaction_dequeue_;
+  std::deque<ManualCompaction*> manual_compaction_dequeue_;
 
   struct CompactionArg {
-    // caller retains ownership of `db`.
     DBImpl* db;
-    // background compaction takes ownership of `prepicked_compaction`.
-    PrepickedCompaction* prepicked_compaction;
+    ManualCompaction* m;
   };
 
   // Have we encountered a background error in paranoid mode?
@@ -1262,24 +1169,25 @@ class DBImpl : public DB {
 
 #endif  // ROCKSDB_LITE
 
+  // Function that Get and KeyMayExist call with no_io true or false
+  // Note: 'value_found' from KeyMayExist propagates here
+  Status GetImpl(const ReadOptions& options, ColumnFamilyHandle* column_family,
+                 const Slice& key, PinnableSlice* value,
+                 bool* value_found = nullptr);
+
   bool GetIntPropertyInternal(ColumnFamilyData* cfd,
                               const DBPropertyInfo& property_info,
                               bool is_locked, uint64_t* value);
 
   bool HasPendingManualCompaction();
   bool HasExclusiveManualCompaction();
-  void AddManualCompaction(ManualCompactionState* m);
-  void RemoveManualCompaction(ManualCompactionState* m);
-  bool ShouldntRunManualCompaction(ManualCompactionState* m);
+  void AddManualCompaction(ManualCompaction* m);
+  void RemoveManualCompaction(ManualCompaction* m);
+  bool ShouldntRunManualCompaction(ManualCompaction* m);
   bool HaveManualCompaction(ColumnFamilyData* cfd);
-  bool MCOverlap(ManualCompactionState* m, ManualCompactionState* m1);
+  bool MCOverlap(ManualCompaction* m, ManualCompaction* m1);
 
   size_t GetWalPreallocateBlockSize(uint64_t write_buffer_size) const;
-
-  // When set, we use a seprate queue for writes that dont write to memtable. In
-  // 2PC these are the writes at Prepare phase.
-  const bool concurrent_prepare_;
-  const bool manual_wal_flush_;
 };
 
 extern Options SanitizeOptions(const std::string& db,

db/db_impl_compaction_flush.cc

@@ -14,7 +14,6 @@
 #include <inttypes.h>
 
 #include "db/builder.h"
-#include "db/event_helpers.h"
 #include "monitoring/iostats_context_imp.h"
 #include "monitoring/perf_context_imp.h"
 #include "monitoring/thread_status_updater.h"
@@ -60,14 +59,7 @@ Status DBImpl::SyncClosedLogs(JobContext* job_context) {
     // "number < current_log_number".
     MarkLogsSynced(current_log_number - 1, true, s);
     if (!s.ok()) {
-      Status new_bg_error = s;
-      // may temporarily unlock and lock the mutex.
-      EventHelpers::NotifyOnBackgroundError(immutable_db_options_.listeners,
-                                            BackgroundErrorReason::kFlush,
-                                            &new_bg_error, &mutex_);
-      if (!new_bg_error.ok()) {
-        bg_error_ = new_bg_error;
-      }
+      bg_error_ = s;
       TEST_SYNC_POINT("DBImpl::SyncClosedLogs:Failed");
       return s;
     }
@@ -142,16 +134,9 @@ Status DBImpl::FlushMemTableToOutputFile(
 
   if (!s.ok() && !s.IsShutdownInProgress() &&
       immutable_db_options_.paranoid_checks && bg_error_.ok()) {
-    Status new_bg_error = s;
-    // may temporarily unlock and lock the mutex.
-    EventHelpers::NotifyOnBackgroundError(immutable_db_options_.listeners,
-                                          BackgroundErrorReason::kFlush,
-                                          &new_bg_error, &mutex_);
-    if (!new_bg_error.ok()) {
-      // if a bad error happened (not ShutdownInProgress), paranoid_checks is
-      // true, and the error isn't handled by callback, mark DB read-only
-      bg_error_ = new_bg_error;
-    }
+    // if a bad error happened (not ShutdownInProgress) and paranoid_checks is
+    // true, mark DB read-only
+    bg_error_ = s;
   }
   if (s.ok()) {
 #ifndef ROCKSDB_LITE
@@ -166,17 +151,10 @@ Status DBImpl::FlushMemTableToOutputFile(
           immutable_db_options_.db_paths[0].path, file_meta.fd.GetNumber());
       sfm->OnAddFile(file_path);
       if (sfm->IsMaxAllowedSpaceReached() && bg_error_.ok()) {
-        Status new_bg_error = Status::IOError("Max allowed space was reached");
+        bg_error_ = Status::IOError("Max allowed space was reached");
         TEST_SYNC_POINT_CALLBACK(
             "DBImpl::FlushMemTableToOutputFile:MaxAllowedSpaceReached",
-            &new_bg_error);
-        // may temporarily unlock and lock the mutex.
-        EventHelpers::NotifyOnBackgroundError(immutable_db_options_.listeners,
-                                              BackgroundErrorReason::kFlush,
-                                              &new_bg_error, &mutex_);
-        if (!new_bg_error.ok()) {
-          bg_error_ = new_bg_error;
-        }
+            &bg_error_);
       }
     }
 #endif  // ROCKSDB_LITE
@@ -587,14 +565,7 @@ Status DBImpl::CompactFilesImpl(
                    c->column_family_data()->GetName().c_str(),
                    job_context->job_id, status.ToString().c_str());
     if (immutable_db_options_.paranoid_checks && bg_error_.ok()) {
-      Status new_bg_error = status;
-      // may temporarily unlock and lock the mutex.
-      EventHelpers::NotifyOnBackgroundError(immutable_db_options_.listeners,
-                                            BackgroundErrorReason::kCompaction,
-                                            &new_bg_error, &mutex_);
-      if (!new_bg_error.ok()) {
-        bg_error_ = new_bg_error;
-      }
+      bg_error_ = status;
     }
   }
 
@@ -612,8 +583,7 @@ Status DBImpl::CompactFilesImpl(
 Status DBImpl::PauseBackgroundWork() {
   InstrumentedMutexLock guard_lock(&mutex_);
   bg_compaction_paused_++;
-  while (bg_bottom_compaction_scheduled_ > 0 || bg_compaction_scheduled_ > 0 ||
-         bg_flush_scheduled_ > 0) {
+  while (bg_compaction_scheduled_ > 0 || bg_flush_scheduled_ > 0) {
     bg_cv_.Wait();
   }
   bg_work_paused_++;
@@ -809,7 +779,7 @@ Status DBImpl::RunManualCompaction(ColumnFamilyData* cfd, int input_level,
 
   bool scheduled = false;
   bool manual_conflict = false;
-  ManualCompactionState manual;
+  ManualCompaction manual;
   manual.cfd = cfd;
   manual.input_level = input_level;
   manual.output_level = output_level;
@@ -859,8 +829,7 @@ Status DBImpl::RunManualCompaction(ColumnFamilyData* cfd, int input_level,
   AddManualCompaction(&manual);
   TEST_SYNC_POINT_CALLBACK("DBImpl::RunManualCompaction:NotScheduled", &mutex_);
   if (exclusive) {
-    while (bg_bottom_compaction_scheduled_ > 0 ||
-           bg_compaction_scheduled_ > 0) {
+    while (bg_compaction_scheduled_ > 0) {
       TEST_SYNC_POINT("DBImpl::RunManualCompaction:WaitScheduled");
       ROCKS_LOG_INFO(
           immutable_db_options_.info_log,
@@ -880,14 +849,14 @@ Status DBImpl::RunManualCompaction(ColumnFamilyData* cfd, int input_level,
   while (!manual.done) {
     assert(HasPendingManualCompaction());
     manual_conflict = false;
-    Compaction* compaction;
     if (ShouldntRunManualCompaction(&manual) || (manual.in_progress == true) ||
         scheduled ||
-        ((manual.manual_end = &manual.tmp_storage1) &&
-         ((compaction = manual.cfd->CompactRange(
-               *manual.cfd->GetLatestMutableCFOptions(), manual.input_level,
-               manual.output_level, manual.output_path_id, manual.begin,
-               manual.end, &manual.manual_end, &manual_conflict)) == nullptr) &&
+        ((manual.manual_end = &manual.tmp_storage1)&&(
+             (manual.compaction = manual.cfd->CompactRange(
+                  *manual.cfd->GetLatestMutableCFOptions(), manual.input_level,
+                  manual.output_level, manual.output_path_id, manual.begin,
+                  manual.end, &manual.manual_end, &manual_conflict)) ==
+             nullptr) &&
          manual_conflict)) {
       // exclusive manual compactions should not see a conflict during
       // CompactRange
@@ -900,16 +869,14 @@ Status DBImpl::RunManualCompaction(ColumnFamilyData* cfd, int input_level,
         manual.incomplete = false;
       }
     } else if (!scheduled) {
-      if (compaction == nullptr) {
+      if (manual.compaction == nullptr) {
         manual.done = true;
         bg_cv_.SignalAll();
         continue;
       }
       ca = new CompactionArg;
       ca->db = this;
-      ca->prepicked_compaction = new PrepickedCompaction;
-      ca->prepicked_compaction->manual_compaction_state = &manual;
-      ca->prepicked_compaction->compaction = compaction;
+      ca->m = &manual;
       manual.incomplete = false;
       bg_compaction_scheduled_++;
       env_->Schedule(&DBImpl::BGWorkCompaction, ca, Env::Priority::LOW, this,
@@ -1051,7 +1018,7 @@ void DBImpl::MaybeScheduleFlushOrCompaction() {
          unscheduled_compactions_ > 0) {
     CompactionArg* ca = new CompactionArg;
     ca->db = this;
-    ca->prepicked_compaction = nullptr;
+    ca->m = nullptr;
     bg_compaction_scheduled_++;
     unscheduled_compactions_--;
     env_->Schedule(&DBImpl::BGWorkCompaction, ca, Env::Priority::LOW, this,
@@ -1156,23 +1123,7 @@ void DBImpl::BGWorkCompaction(void* arg) {
   delete reinterpret_cast<CompactionArg*>(arg);
   IOSTATS_SET_THREAD_POOL_ID(Env::Priority::LOW);
   TEST_SYNC_POINT("DBImpl::BGWorkCompaction");
-  auto prepicked_compaction =
-      static_cast<PrepickedCompaction*>(ca.prepicked_compaction);
-  reinterpret_cast<DBImpl*>(ca.db)->BackgroundCallCompaction(
-      prepicked_compaction, Env::Priority::LOW);
-  delete prepicked_compaction;
-}
-
-void DBImpl::BGWorkBottomCompaction(void* arg) {
-  CompactionArg ca = *(static_cast<CompactionArg*>(arg));
-  delete static_cast<CompactionArg*>(arg);
-  IOSTATS_SET_THREAD_POOL_ID(Env::Priority::BOTTOM);
-  TEST_SYNC_POINT("DBImpl::BGWorkBottomCompaction");
-  auto* prepicked_compaction = ca.prepicked_compaction;
-  assert(prepicked_compaction && prepicked_compaction->compaction &&
-         !prepicked_compaction->manual_compaction_state);
-  ca.db->BackgroundCallCompaction(prepicked_compaction, Env::Priority::BOTTOM);
-  delete prepicked_compaction;
+  reinterpret_cast<DBImpl*>(ca.db)->BackgroundCallCompaction(ca.m);
 }
 
 void DBImpl::BGWorkPurge(void* db) {
@@ -1185,11 +1136,8 @@ void DBImpl::BGWorkPurge(void* db) {
 void DBImpl::UnscheduleCallback(void* arg) {
   CompactionArg ca = *(reinterpret_cast<CompactionArg*>(arg));
   delete reinterpret_cast<CompactionArg*>(arg);
-  if (ca.prepicked_compaction != nullptr) {
-    if (ca.prepicked_compaction->compaction != nullptr) {
-      delete ca.prepicked_compaction->compaction;
-    }
-    delete ca.prepicked_compaction;
+  if ((ca.m != nullptr) && (ca.m->compaction != nullptr)) {
+    delete ca.m->compaction;
   }
   TEST_SYNC_POINT("DBImpl::UnscheduleCallback");
 }
@@ -1249,6 +1197,7 @@ Status DBImpl::BackgroundFlush(bool* made_progress, JobContext* job_context,
 void DBImpl::BackgroundCallFlush() {
   bool made_progress = false;
   JobContext job_context(next_job_id_.fetch_add(1), true);
+  assert(bg_flush_scheduled_);
 
   TEST_SYNC_POINT("DBImpl::BackgroundCallFlush:start");
 
@@ -1256,7 +1205,6 @@ void DBImpl::BackgroundCallFlush() {
                        immutable_db_options_.info_log.get());
   {
     InstrumentedMutexLock l(&mutex_);
-    assert(bg_flush_scheduled_);
     num_running_flushes_++;
 
     auto pending_outputs_inserted_elem =
@@ -1316,9 +1264,9 @@ void DBImpl::BackgroundCallFlush() {
   }
 }
 
-void DBImpl::BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction,
-                                      Env::Priority bg_thread_pri) {
+void DBImpl::BackgroundCallCompaction(void* arg) {
   bool made_progress = false;
+  ManualCompaction* m = reinterpret_cast<ManualCompaction*>(arg);
   JobContext job_context(next_job_id_.fetch_add(1), true);
   TEST_SYNC_POINT("BackgroundCallCompaction:0");
   MaybeDumpStats();
@@ -1336,11 +1284,9 @@ void DBImpl::BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction,
     auto pending_outputs_inserted_elem =
         CaptureCurrentFileNumberInPendingOutputs();
 
-    assert((bg_thread_pri == Env::Priority::BOTTOM &&
-            bg_bottom_compaction_scheduled_) ||
-           (bg_thread_pri == Env::Priority::LOW && bg_compaction_scheduled_));
-    Status s = BackgroundCompaction(&made_progress, &job_context, &log_buffer,
-                                    prepicked_compaction);
+    assert(bg_compaction_scheduled_);
+    Status s =
+        BackgroundCompaction(&made_progress, &job_context, &log_buffer, m);
     TEST_SYNC_POINT("BackgroundCallCompaction:1");
     if (!s.ok() && !s.IsShutdownInProgress()) {
       // Wait a little bit before retrying background compaction in
@@ -1386,24 +1332,17 @@ void DBImpl::BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction,
 
     assert(num_running_compactions_ > 0);
     num_running_compactions_--;
-    if (bg_thread_pri == Env::Priority::LOW) {
-      bg_compaction_scheduled_--;
-    } else {
-      assert(bg_thread_pri == Env::Priority::BOTTOM);
-      bg_bottom_compaction_scheduled_--;
-    }
+    bg_compaction_scheduled_--;
 
     versions_->GetColumnFamilySet()->FreeDeadColumnFamilies();
 
     // See if there's more work to be done
     MaybeScheduleFlushOrCompaction();
-    if (made_progress ||
-        (bg_compaction_scheduled_ == 0 &&
-         bg_bottom_compaction_scheduled_ == 0) ||
+    if (made_progress || bg_compaction_scheduled_ == 0 ||
         HasPendingManualCompaction()) {
       // signal if
       // * made_progress -- need to wakeup DelayWrite
-      // * bg_{bottom,}_compaction_scheduled_ == 0 -- need to wakeup ~DBImpl
+      // * bg_compaction_scheduled_ == 0 -- need to wakeup ~DBImpl
       // * HasPendingManualCompaction -- need to wakeup RunManualCompaction
       // If none of this is true, there is no need to signal since nobody is
       // waiting for it
@@ -1418,23 +1357,14 @@ void DBImpl::BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction,
 
 Status DBImpl::BackgroundCompaction(bool* made_progress,
                                     JobContext* job_context,
-                                    LogBuffer* log_buffer,
-                                    PrepickedCompaction* prepicked_compaction) {
-  ManualCompactionState* manual_compaction =
-      prepicked_compaction == nullptr
-          ? nullptr
-          : prepicked_compaction->manual_compaction_state;
+                                    LogBuffer* log_buffer, void* arg) {
+  ManualCompaction* manual_compaction =
+      reinterpret_cast<ManualCompaction*>(arg);
   *made_progress = false;
   mutex_.AssertHeld();
   TEST_SYNC_POINT("DBImpl::BackgroundCompaction:Start");
 
   bool is_manual = (manual_compaction != nullptr);
-  unique_ptr<Compaction> c;
-  if (prepicked_compaction != nullptr &&
-      prepicked_compaction->compaction != nullptr) {
-    c.reset(prepicked_compaction->compaction);
-  }
-  bool is_prepicked = is_manual || c;
 
   // (manual_compaction->in_progress == false);
   bool trivial_move_disallowed =
@@ -1451,6 +1381,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
       manual_compaction->status = status;
       manual_compaction->done = true;
       manual_compaction->in_progress = false;
+      delete manual_compaction->compaction;
       manual_compaction = nullptr;
     }
     return status;
@@ -1461,11 +1392,13 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
     manual_compaction->in_progress = true;
   }
 
+  unique_ptr<Compaction> c;
   // InternalKey manual_end_storage;
   // InternalKey* manual_end = &manual_end_storage;
   if (is_manual) {
-    ManualCompactionState* m = manual_compaction;
+    ManualCompaction* m = manual_compaction;
     assert(m->in_progress);
+    c.reset(std::move(m->compaction));
     if (!c) {
       m->done = true;
       m->manual_end = nullptr;
@@ -1487,7 +1420,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
                ? "(end)"
                : m->manual_end->DebugString().c_str()));
     }
-  } else if (!is_prepicked && !compaction_queue_.empty()) {
+  } else if (!compaction_queue_.empty()) {
     if (HaveManualCompaction(compaction_queue_.front())) {
       // Can't compact right now, but try again later
       TEST_SYNC_POINT("DBImpl::BackgroundCompaction()::Conflict");
@@ -1639,28 +1572,6 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
 
     // Clear Instrument
     ThreadStatusUtil::ResetThreadStatus();
-  } else if (c->column_family_data()->ioptions()->compaction_style ==
-                 kCompactionStyleUniversal &&
-             !is_prepicked && c->output_level() > 0 &&
-             c->output_level() ==
-                 c->column_family_data()
-                     ->current()
-                     ->storage_info()
-                     ->MaxOutputLevel(
-                         immutable_db_options_.allow_ingest_behind) &&
-             env_->GetBackgroundThreads(Env::Priority::BOTTOM) > 0) {
-    // Forward universal compactions involving last level to the bottom pool
-    // if it exists, such that long-running compactions can't block short-
-    // lived ones, like L0->L0s.
-    TEST_SYNC_POINT("DBImpl::BackgroundCompaction:ForwardToBottomPriPool");
-    CompactionArg* ca = new CompactionArg;
-    ca->db = this;
-    ca->prepicked_compaction = new PrepickedCompaction;
-    ca->prepicked_compaction->compaction = c.release();
-    ca->prepicked_compaction->manual_compaction_state = nullptr;
-    ++bg_bottom_compaction_scheduled_;
-    env_->Schedule(&DBImpl::BGWorkBottomCompaction, ca, Env::Priority::BOTTOM,
-                   this, &DBImpl::UnscheduleCallback);
   } else {
     int output_level  __attribute__((unused)) = c->output_level();
     TEST_SYNC_POINT_CALLBACK("DBImpl::BackgroundCompaction:NonTrivial",
@@ -1712,19 +1623,12 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
     ROCKS_LOG_WARN(immutable_db_options_.info_log, "Compaction error: %s",
                    status.ToString().c_str());
     if (immutable_db_options_.paranoid_checks && bg_error_.ok()) {
-      Status new_bg_error = status;
-      // may temporarily unlock and lock the mutex.
-      EventHelpers::NotifyOnBackgroundError(immutable_db_options_.listeners,
-                                            BackgroundErrorReason::kCompaction,
-                                            &new_bg_error, &mutex_);
-      if (!new_bg_error.ok()) {
-        bg_error_ = new_bg_error;
-      }
+      bg_error_ = status;
     }
   }
 
   if (is_manual) {
-    ManualCompactionState* m = manual_compaction;
+    ManualCompaction* m = manual_compaction;
     if (!status.ok()) {
       m->status = status;
       m->done = true;
@@ -1767,13 +1671,13 @@ bool DBImpl::HasPendingManualCompaction() {
   return (!manual_compaction_dequeue_.empty());
 }
 
-void DBImpl::AddManualCompaction(DBImpl::ManualCompactionState* m) {
+void DBImpl::AddManualCompaction(DBImpl::ManualCompaction* m) {
   manual_compaction_dequeue_.push_back(m);
 }
 
-void DBImpl::RemoveManualCompaction(DBImpl::ManualCompactionState* m) {
+void DBImpl::RemoveManualCompaction(DBImpl::ManualCompaction* m) {
   // Remove from queue
-  std::deque<ManualCompactionState*>::iterator it =
+  std::deque<ManualCompaction*>::iterator it =
       manual_compaction_dequeue_.begin();
   while (it != manual_compaction_dequeue_.end()) {
     if (m == (*it)) {
@@ -1786,17 +1690,16 @@ void DBImpl::RemoveManualCompaction(DBImpl::ManualCompactionState* m) {
   return;
 }
 
-bool DBImpl::ShouldntRunManualCompaction(ManualCompactionState* m) {
+bool DBImpl::ShouldntRunManualCompaction(ManualCompaction* m) {
   if (num_running_ingest_file_ > 0) {
     // We need to wait for other IngestExternalFile() calls to finish
     // before running a manual compaction.
     return true;
   }
   if (m->exclusive) {
-    return (bg_bottom_compaction_scheduled_ > 0 ||
-            bg_compaction_scheduled_ > 0);
+    return (bg_compaction_scheduled_ > 0);
   }
-  std::deque<ManualCompactionState*>::iterator it =
+  std::deque<ManualCompaction*>::iterator it =
       manual_compaction_dequeue_.begin();
   bool seen = false;
   while (it != manual_compaction_dequeue_.end()) {
@@ -1817,7 +1720,7 @@ bool DBImpl::ShouldntRunManualCompaction(ManualCompactionState* m) {
 
 bool DBImpl::HaveManualCompaction(ColumnFamilyData* cfd) {
   // Remove from priority queue
-  std::deque<ManualCompactionState*>::iterator it =
+  std::deque<ManualCompaction*>::iterator it =
       manual_compaction_dequeue_.begin();
   while (it != manual_compaction_dequeue_.end()) {
     if ((*it)->exclusive) {
@@ -1825,7 +1728,7 @@ bool DBImpl::HaveManualCompaction(ColumnFamilyData* cfd) {
     }
     if ((cfd == (*it)->cfd) && (!((*it)->in_progress || (*it)->done))) {
       // Allow automatic compaction if manual compaction is
-      // in progress
+      // is in progress
       return true;
     }
     it++;
@@ -1835,7 +1738,7 @@ bool DBImpl::HaveManualCompaction(ColumnFamilyData* cfd) {
 
 bool DBImpl::HasExclusiveManualCompaction() {
   // Remove from priority queue
-  std::deque<ManualCompactionState*>::iterator it =
+  std::deque<ManualCompaction*>::iterator it =
       manual_compaction_dequeue_.begin();
   while (it != manual_compaction_dequeue_.end()) {
     if ((*it)->exclusive) {
@@ -1846,7 +1749,7 @@ bool DBImpl::HasExclusiveManualCompaction() {
   return false;
 }
 
-bool DBImpl::MCOverlap(ManualCompactionState* m, ManualCompactionState* m1) {
+bool DBImpl::MCOverlap(ManualCompaction* m, ManualCompaction* m1) {
   if ((m->exclusive) || (m1->exclusive)) {
     return true;
   }

db/db_impl_debug.cc

@@ -80,15 +80,6 @@ Status DBImpl::TEST_CompactRange(int level, const Slice* begin,
                              disallow_trivial_move);
 }
 
-Status DBImpl::TEST_SwitchMemtable(ColumnFamilyData* cfd) {
-  WriteContext write_context;
-  InstrumentedMutexLock l(&mutex_);
-  if (cfd == nullptr) {
-    cfd = default_cf_handle_->cfd();
-  }
-  return SwitchMemtable(cfd, &write_context);
-}
-
 Status DBImpl::TEST_FlushMemTable(bool wait, ColumnFamilyHandle* cfh) {
   FlushOptions fo;
   fo.wait = wait;
@@ -121,9 +112,7 @@ Status DBImpl::TEST_WaitForCompact() {
   // OR flush to finish.
 
   InstrumentedMutexLock l(&mutex_);
-  while ((bg_bottom_compaction_scheduled_ || bg_compaction_scheduled_ ||
-          bg_flush_scheduled_) &&
-         bg_error_.ok()) {
+  while ((bg_compaction_scheduled_ || bg_flush_scheduled_) && bg_error_.ok()) {
     bg_cv_.Wait();
   }
   return bg_error_;

db/db_impl_files.cc

@@ -250,13 +250,7 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
       }
       job_context->size_log_to_delete += earliest.size;
       total_log_size_ -= earliest.size;
-      if (concurrent_prepare_) {
-        log_write_mutex_.Lock();
-      }
       alive_log_files_.pop_front();
-      if (concurrent_prepare_) {
-        log_write_mutex_.Unlock();
-      }
       // Current log should always stay alive since it can't have
       // number < MinLogNumber().
       assert(alive_log_files_.size());
@@ -269,10 +263,7 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
         continue;
       }
       logs_to_free_.push_back(log.ReleaseWriter());
-      {
-        InstrumentedMutexLock wl(&log_write_mutex_);
-        logs_.pop_front();
-      }
+      logs_.pop_front();
     }
     // Current log cannot be obsolete.
     assert(!logs_.empty());
@@ -368,9 +359,6 @@ void DBImpl::PurgeObsoleteFiles(const JobContext& state, bool schedule_only) {
     candidate_files.emplace_back(
         MakeTableFileName(kDumbDbName, file->fd.GetNumber()),
         file->fd.GetPathId());
-    if (file->table_reader_handle) {
-      table_cache_->Release(file->table_reader_handle);
-    }
     delete file;
   }
 

db/db_impl_open.cc

@@ -16,7 +16,6 @@
 #include "db/builder.h"
 #include "options/options_helper.h"
 #include "rocksdb/wal_filter.h"
-#include "table/block_based_table_factory.h"
 #include "util/rate_limiter.h"
 #include "util/sst_file_manager_impl.h"
 #include "util/sync_point.h"
@@ -163,19 +162,6 @@ static Status ValidateOptions(
             "universal and level compaction styles. ");
       }
     }
-    if (cfd.options.compaction_options_fifo.ttl > 0) {
-      if (db_options.max_open_files != -1) {
-        return Status::NotSupported(
-            "FIFO Compaction with TTL is only supported when files are always "
-            "kept open (set max_open_files = -1). ");
-      }
-      if (cfd.options.table_factory->Name() !=
-          BlockBasedTableFactory().Name()) {
-        return Status::NotSupported(
-            "FIFO Compaction with TTL is only supported in "
-            "Block-Based Table format. ");
-      }
-    }
   }
 
   if (db_options.db_paths.size() > 4) {
@@ -737,7 +723,6 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
     auto last_sequence = *next_sequence - 1;
     if ((*next_sequence != kMaxSequenceNumber) &&
         (versions_->LastSequence() <= last_sequence)) {
-      versions_->SetLastToBeWrittenSequence(last_sequence);
       versions_->SetLastSequence(last_sequence);
     }
   }
@@ -809,15 +794,9 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
   if (data_seen && !flushed) {
     // Mark these as alive so they'll be considered for deletion later by
     // FindObsoleteFiles()
-    if (concurrent_prepare_) {
-      log_write_mutex_.Lock();
-    }
     for (auto log_number : log_numbers) {
       alive_log_files_.push_back(LogFileNumberSize(log_number));
     }
-    if (concurrent_prepare_) {
-      log_write_mutex_.Unlock();
-    }
   }
 
   event_logger_.Log() << "job" << job_id << "event"
@@ -851,11 +830,6 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
         *cfd->GetLatestMutableCFOptions();
     bool paranoid_file_checks =
         cfd->GetLatestMutableCFOptions()->paranoid_file_checks;
-
-    int64_t _current_time = 0;
-    env_->GetCurrentTime(&_current_time);  // ignore error
-    const uint64_t current_time = static_cast<uint64_t>(_current_time);
-
     {
       mutex_.Unlock();
 
@@ -875,8 +849,7 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
           GetCompressionFlush(*cfd->ioptions(), mutable_cf_options),
           cfd->ioptions()->compression_opts, paranoid_file_checks,
           cfd->internal_stats(), TableFileCreationReason::kRecovery,
-          &event_logger_, job_id, Env::IO_HIGH, nullptr /* table_properties */,
-          -1 /* level */, current_time);
+          &event_logger_, job_id);
       LogFlush(immutable_db_options_.info_log);
       ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
                       "[%s] [WriteLevel0TableForRecovery]"
@@ -987,17 +960,14 @@ Status DB::Open(const DBOptions& db_options, const std::string& dbname,
     if (s.ok()) {
       lfile->SetPreallocationBlockSize(
           impl->GetWalPreallocateBlockSize(max_write_buffer_size));
-      {
-        InstrumentedMutexLock wl(&impl->log_write_mutex_);
-        impl->logfile_number_ = new_log_number;
-        unique_ptr<WritableFileWriter> file_writer(
-            new WritableFileWriter(std::move(lfile), opt_env_options));
-        impl->logs_.emplace_back(
-            new_log_number,
-            new log::Writer(
-                std::move(file_writer), new_log_number,
-                impl->immutable_db_options_.recycle_log_file_num > 0));
-      }
+      impl->logfile_number_ = new_log_number;
+      unique_ptr<WritableFileWriter> file_writer(
+          new WritableFileWriter(std::move(lfile), opt_env_options));
+      impl->logs_.emplace_back(
+          new_log_number,
+          new log::Writer(
+              std::move(file_writer), new_log_number,
+              impl->immutable_db_options_.recycle_log_file_num > 0));
 
       // set column family handles
       for (auto cf : column_families) {
@@ -1031,14 +1001,8 @@ Status DB::Open(const DBOptions& db_options, const std::string& dbname,
         delete impl->InstallSuperVersionAndScheduleWork(
             cfd, nullptr, *cfd->GetLatestMutableCFOptions());
       }
-      if (impl->concurrent_prepare_) {
-        impl->log_write_mutex_.Lock();
-      }
       impl->alive_log_files_.push_back(
           DBImpl::LogFileNumberSize(impl->logfile_number_));
-      if (impl->concurrent_prepare_) {
-        impl->log_write_mutex_.Unlock();
-      }
       impl->DeleteObsoleteFiles();
       s = impl->directories_.GetDbDir()->Fsync();
     }

db/db_impl_readonly.cc

@@ -3,6 +3,7 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
 
+
 #include "db/db_impl_readonly.h"
 
 #include "db/compacted_db_impl.h"
@@ -58,7 +59,7 @@ Iterator* DBImplReadOnly::NewIterator(const ReadOptions& read_options,
   SuperVersion* super_version = cfd->GetSuperVersion()->Ref();
   SequenceNumber latest_snapshot = versions_->LastSequence();
   auto db_iter = NewArenaWrappedDbIterator(
-      env_, read_options, *cfd->ioptions(),
+      env_, read_options, *cfd->ioptions(), cfd->user_comparator(),
       (read_options.snapshot != nullptr
            ? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
                  ->number_
@@ -87,7 +88,7 @@ Status DBImplReadOnly::NewIterators(
     auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(cfh)->cfd();
     auto* sv = cfd->GetSuperVersion()->Ref();
     auto* db_iter = NewArenaWrappedDbIterator(
-        env_, read_options, *cfd->ioptions(),
+        env_, read_options, *cfd->ioptions(), cfd->user_comparator(),
         (read_options.snapshot != nullptr
              ? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
                    ->number_

db/db_impl_write.cc

@@ -12,7 +12,6 @@
 #define __STDC_FORMAT_MACROS
 #endif
 #include <inttypes.h>
-#include "db/event_helpers.h"
 #include "monitoring/perf_context_imp.h"
 #include "options/options_helper.h"
 #include "util/sync_point.h"
@@ -60,14 +59,10 @@ Status DBImpl::WriteWithCallback(const WriteOptions& write_options,
 Status DBImpl::WriteImpl(const WriteOptions& write_options,
                          WriteBatch* my_batch, WriteCallback* callback,
                          uint64_t* log_used, uint64_t log_ref,
-                         bool disable_memtable, uint64_t* seq_used) {
+                         bool disable_memtable) {
   if (my_batch == nullptr) {
     return Status::Corruption("Batch is nullptr!");
   }
-  if (concurrent_prepare_ && immutable_db_options_.enable_pipelined_write) {
-    return Status::NotSupported(
-        "pipelined_writes is not compatible with concurrent prepares");
-  }
 
   Status status;
   if (write_options.low_pri) {
@@ -77,14 +72,9 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
     }
   }
 
-  if (concurrent_prepare_ && disable_memtable) {
-    return WriteImplWALOnly(write_options, my_batch, callback, log_used,
-                            log_ref, seq_used);
-  }
-
   if (immutable_db_options_.enable_pipelined_write) {
     return PipelinedWriteImpl(write_options, my_batch, callback, log_used,
-                              log_ref, disable_memtable, seq_used);
+                              log_ref, disable_memtable);
   }
 
   PERF_TIMER_GUARD(write_pre_and_post_process_time);
@@ -127,9 +117,6 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
     if (log_used != nullptr) {
       *log_used = w.log_used;
     }
-    if (seq_used != nullptr) {
-      *seq_used = w.sequence;
-    }
     // write is complete and leader has updated sequence
     return w.FinalStatus();
   }
@@ -143,22 +130,14 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
   WriteContext write_context;
   WriteThread::WriteGroup write_group;
   bool in_parallel_group = false;
-  uint64_t last_sequence = kMaxSequenceNumber;
-  if (!concurrent_prepare_) {
-    last_sequence = versions_->LastSequence();
-  }
+  uint64_t last_sequence = versions_->LastSequence();
 
   mutex_.Lock();
 
   bool need_log_sync = !write_options.disableWAL && write_options.sync;
   bool need_log_dir_sync = need_log_sync && !log_dir_synced_;
-  if (!concurrent_prepare_ || !disable_memtable) {
-    // With concurrent writes we do preprocess only in the write thread that
-    // also does write to memtable to avoid sync issue on shared data structure
-    // with the other thread
-    status = PreprocessWrite(write_options, &need_log_sync, &write_context);
-  }
-  log::Writer* log_writer = logs_.back().writer;
+  status = PreprocessWrite(write_options, &need_log_sync, &write_context);
+  log::Writer* cur_log_writer = logs_.back().writer;
 
   mutex_.Unlock();
 
@@ -198,24 +177,24 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
       }
     }
 
-    const bool concurrent_update = concurrent_prepare_;
+    const SequenceNumber current_sequence = last_sequence + 1;
+    last_sequence += total_count;
+
     // Update stats while we are an exclusive group leader, so we know
     // that nobody else can be writing to these particular stats.
     // We're optimistic, updating the stats before we successfully
     // commit.  That lets us release our leader status early.
     auto stats = default_cf_internal_stats_;
-    stats->AddDBStats(InternalStats::NUMBER_KEYS_WRITTEN, total_count,
-                      concurrent_update);
+    stats->AddDBStats(InternalStats::NUMBER_KEYS_WRITTEN, total_count);
     RecordTick(stats_, NUMBER_KEYS_WRITTEN, total_count);
-    stats->AddDBStats(InternalStats::BYTES_WRITTEN, total_byte_size,
-                      concurrent_update);
+    stats->AddDBStats(InternalStats::BYTES_WRITTEN, total_byte_size);
     RecordTick(stats_, BYTES_WRITTEN, total_byte_size);
-    stats->AddDBStats(InternalStats::WRITE_DONE_BY_SELF, 1, concurrent_update);
+    stats->AddDBStats(InternalStats::WRITE_DONE_BY_SELF, 1);
     RecordTick(stats_, WRITE_DONE_BY_SELF);
     auto write_done_by_other = write_group.size - 1;
     if (write_done_by_other > 0) {
-      stats->AddDBStats(InternalStats::WRITE_DONE_BY_OTHER, write_done_by_other,
-                        concurrent_update);
+      stats->AddDBStats(InternalStats::WRITE_DONE_BY_OTHER,
+                        write_done_by_other);
       RecordTick(stats_, WRITE_DONE_BY_OTHER, write_done_by_other);
     }
     MeasureTime(stats_, BYTES_PER_WRITE, total_byte_size);
@@ -226,27 +205,14 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
 
     PERF_TIMER_STOP(write_pre_and_post_process_time);
 
-    if (!concurrent_prepare_) {
-      if (status.ok() && !write_options.disableWAL) {
-        PERF_TIMER_GUARD(write_wal_time);
-        status = WriteToWAL(write_group, log_writer, log_used, need_log_sync,
-                            need_log_dir_sync, last_sequence + 1);
-      }
-    } else {
-      if (status.ok() && !write_options.disableWAL) {
-        PERF_TIMER_GUARD(write_wal_time);
-        // LastToBeWrittenSequence is increased inside WriteToWAL under
-        // wal_write_mutex_ to ensure ordered events in WAL
-        status = ConcurrentWriteToWAL(write_group, log_used, &last_sequence,
-                                      total_count);
-      } else {
-        // Otherwise we inc seq number for memtable writes
-        last_sequence = versions_->FetchAddLastToBeWrittenSequence(total_count);
+    if (status.ok() && !write_options.disableWAL) {
+      PERF_TIMER_GUARD(write_wal_time);
+      status = WriteToWAL(write_group, cur_log_writer, need_log_sync,
+                          need_log_dir_sync, current_sequence);
+      if (log_used != nullptr) {
+        *log_used = logfile_number_;
       }
     }
-    assert(last_sequence != kMaxSequenceNumber);
-    const SequenceNumber current_sequence = last_sequence + 1;
-    last_sequence += total_count;
 
     if (status.ok()) {
       PERF_TIMER_GUARD(write_memtable_time);
@@ -281,31 +247,19 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
               write_options.ignore_missing_column_families, 0 /*log_number*/,
               this, true /*concurrent_memtable_writes*/);
         }
-        if (seq_used != nullptr) {
-          *seq_used = w.sequence;
-        }
       }
     }
   }
   PERF_TIMER_START(write_pre_and_post_process_time);
 
   if (!w.CallbackFailed()) {
-    WriteCallbackStatusCheck(status);
+    ParanoidCheck(status);
   }
 
   if (need_log_sync) {
     mutex_.Lock();
     MarkLogsSynced(logfile_number_, need_log_dir_sync, status);
     mutex_.Unlock();
-    // Requesting sync with concurrent_prepare_ is expected to be very rare. We
-    // hance provide a simple implementation that is not necessarily efficient.
-    if (concurrent_prepare_) {
-      if (manual_wal_flush_) {
-        status = FlushWAL(true);
-      } else {
-        status = SyncWAL();
-      }
-    }
   }
 
   bool should_exit_batch_group = true;
@@ -315,11 +269,9 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
     should_exit_batch_group = write_thread_.CompleteParallelMemTableWriter(&w);
   }
   if (should_exit_batch_group) {
-    if (status.ok()) {
-      versions_->SetLastSequence(last_sequence);
-    }
+    versions_->SetLastSequence(last_sequence);
     MemTableInsertStatusCheck(w.status);
-    write_thread_.ExitAsBatchGroupLeader(write_group, status);
+    write_thread_.ExitAsBatchGroupLeader(write_group, w.status);
   }
 
   if (status.ok()) {
@@ -331,7 +283,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
 Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
                                   WriteBatch* my_batch, WriteCallback* callback,
                                   uint64_t* log_used, uint64_t log_ref,
-                                  bool disable_memtable, uint64_t* seq_used) {
+                                  bool disable_memtable) {
   PERF_TIMER_GUARD(write_pre_and_post_process_time);
   StopWatch write_sw(env_, immutable_db_options_.statistics.get(), DB_WRITE);
 
@@ -349,7 +301,7 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
     bool need_log_sync = !write_options.disableWAL && write_options.sync;
     bool need_log_dir_sync = need_log_sync && !log_dir_synced_;
     w.status = PreprocessWrite(write_options, &need_log_sync, &write_context);
-    log::Writer* log_writer = logs_.back().writer;
+    log::Writer* cur_log_writer = logs_.back().writer;
     mutex_.Unlock();
 
     // This can set non-OK status if callback fail.
@@ -397,12 +349,12 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
                           wal_write_group.size - 1);
         RecordTick(stats_, WRITE_DONE_BY_OTHER, wal_write_group.size - 1);
       }
-      w.status = WriteToWAL(wal_write_group, log_writer, log_used,
-                            need_log_sync, need_log_dir_sync, current_sequence);
+      w.status = WriteToWAL(wal_write_group, cur_log_writer, need_log_sync,
+                            need_log_dir_sync, current_sequence);
     }
 
     if (!w.CallbackFailed()) {
-      WriteCallbackStatusCheck(w.status);
+      ParanoidCheck(w.status);
     }
 
     if (need_log_sync) {
@@ -446,128 +398,23 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
       write_thread_.ExitAsMemTableWriter(&w, *w.write_group);
     }
   }
-  if (seq_used != nullptr) {
-    *seq_used = w.sequence;
-  }
 
   assert(w.state == WriteThread::STATE_COMPLETED);
+  if (log_used != nullptr) {
+    *log_used = w.log_used;
+  }
+
   return w.FinalStatus();
 }
 
-Status DBImpl::WriteImplWALOnly(const WriteOptions& write_options,
-                                WriteBatch* my_batch, WriteCallback* callback,
-                                uint64_t* log_used, uint64_t log_ref,
-                                uint64_t* seq_used) {
-  Status status;
-  PERF_TIMER_GUARD(write_pre_and_post_process_time);
-  WriteThread::Writer w(write_options, my_batch, callback, log_ref,
-                        true /* disable_memtable */);
-  if (write_options.disableWAL) {
-    return status;
-  }
-  RecordTick(stats_, WRITE_WITH_WAL);
-  StopWatch write_sw(env_, immutable_db_options_.statistics.get(), DB_WRITE);
-
-  nonmem_write_thread_.JoinBatchGroup(&w);
-  assert(w.state != WriteThread::STATE_PARALLEL_MEMTABLE_WRITER);
-  if (w.state == WriteThread::STATE_COMPLETED) {
-    if (log_used != nullptr) {
-      *log_used = w.log_used;
-    }
-    if (seq_used != nullptr) {
-      *seq_used = w.sequence;
-    }
-    return w.FinalStatus();
-  }
-  // else we are the leader of the write batch group
-  assert(w.state == WriteThread::STATE_GROUP_LEADER);
-  WriteContext write_context;
-  WriteThread::WriteGroup write_group;
-  uint64_t last_sequence;
-  nonmem_write_thread_.EnterAsBatchGroupLeader(&w, &write_group);
-  // Note: no need to update last_batch_group_size_ here since the batch writes
-  // to WAL only
-
-  uint64_t total_byte_size = 0;
-  for (auto* writer : write_group) {
-    if (writer->CheckCallback(this)) {
-      total_byte_size = WriteBatchInternal::AppendedByteSize(
-          total_byte_size, WriteBatchInternal::ByteSize(writer->batch));
-    }
-  }
-
-  const bool concurrent_update = true;
-  // Update stats while we are an exclusive group leader, so we know
-  // that nobody else can be writing to these particular stats.
-  // We're optimistic, updating the stats before we successfully
-  // commit.  That lets us release our leader status early.
-  auto stats = default_cf_internal_stats_;
-  stats->AddDBStats(InternalStats::BYTES_WRITTEN, total_byte_size,
-                    concurrent_update);
-  RecordTick(stats_, BYTES_WRITTEN, total_byte_size);
-  stats->AddDBStats(InternalStats::WRITE_DONE_BY_SELF, 1, concurrent_update);
-  RecordTick(stats_, WRITE_DONE_BY_SELF);
-  auto write_done_by_other = write_group.size - 1;
-  if (write_done_by_other > 0) {
-    stats->AddDBStats(InternalStats::WRITE_DONE_BY_OTHER, write_done_by_other,
-                      concurrent_update);
-    RecordTick(stats_, WRITE_DONE_BY_OTHER, write_done_by_other);
-  }
-  MeasureTime(stats_, BYTES_PER_WRITE, total_byte_size);
-
-  PERF_TIMER_STOP(write_pre_and_post_process_time);
-
-  PERF_TIMER_GUARD(write_wal_time);
-  // LastToBeWrittenSequence is increased inside WriteToWAL under
-  // wal_write_mutex_ to ensure ordered events in WAL
-  status = ConcurrentWriteToWAL(write_group, log_used, &last_sequence,
-                                0 /*total_count*/);
-  auto curr_seq = last_sequence + 1;
-  for (auto* writer : write_group) {
-    if (writer->CheckCallback(this)) {
-      writer->sequence = curr_seq;
-      curr_seq += WriteBatchInternal::Count(writer->batch);
-    }
-  }
-  if (status.ok() && write_options.sync) {
-    // Requesting sync with concurrent_prepare_ is expected to be very rare. We
-    // hance provide a simple implementation that is not necessarily efficient.
-    if (manual_wal_flush_) {
-      status = FlushWAL(true);
-    } else {
-      status = SyncWAL();
-    }
-  }
-  PERF_TIMER_START(write_pre_and_post_process_time);
-
-  if (!w.CallbackFailed()) {
-    WriteCallbackStatusCheck(status);
-  }
-  nonmem_write_thread_.ExitAsBatchGroupLeader(write_group, status);
-  if (status.ok()) {
-    status = w.FinalStatus();
-  }
-  if (seq_used != nullptr) {
-    *seq_used = w.sequence;
-  }
-  return status;
-}
-
-void DBImpl::WriteCallbackStatusCheck(const Status& status) {
+void DBImpl::ParanoidCheck(const Status& status) {
   // Is setting bg_error_ enough here?  This will at least stop
   // compaction and fail any further writes.
   if (immutable_db_options_.paranoid_checks && !status.ok() &&
       !status.IsBusy() && !status.IsIncomplete()) {
     mutex_.Lock();
     if (bg_error_.ok()) {
-      Status new_bg_error = status;
-      // may temporarily unlock and lock the mutex.
-      EventHelpers::NotifyOnBackgroundError(immutable_db_options_.listeners,
-                                            BackgroundErrorReason::kWriteCallback,
-                                            &new_bg_error, &mutex_);
-      if (!new_bg_error.ok()) {
-        bg_error_ = new_bg_error;  // stop compaction & fail any further writes
-      }
+      bg_error_ = status;  // stop compaction & fail any further writes
     }
     mutex_.Unlock();
   }
@@ -582,14 +429,7 @@ void DBImpl::MemTableInsertStatusCheck(const Status& status) {
   if (!status.ok()) {
     mutex_.Lock();
     assert(bg_error_.ok());
-    Status new_bg_error = status;
-    // may temporarily unlock and lock the mutex.
-    EventHelpers::NotifyOnBackgroundError(immutable_db_options_.listeners,
-                                          BackgroundErrorReason::kMemTable,
-                                          &new_bg_error, &mutex_);
-    if (!new_bg_error.ok()) {
-      bg_error_ = new_bg_error;  // stop compaction & fail any further writes
-    }
+    bg_error_ = status;
     mutex_.Unlock();
   }
 }
@@ -662,12 +502,13 @@ Status DBImpl::PreprocessWrite(const WriteOptions& write_options,
   return status;
 }
 
-WriteBatch* DBImpl::MergeBatch(const WriteThread::WriteGroup& write_group,
-                               WriteBatch* tmp_batch, size_t* write_with_wal) {
-  assert(write_with_wal != nullptr);
-  assert(tmp_batch != nullptr);
+Status DBImpl::WriteToWAL(const WriteThread::WriteGroup& write_group,
+                          log::Writer* log_writer, bool need_log_sync,
+                          bool need_log_dir_sync, SequenceNumber sequence) {
+  Status status;
+
   WriteBatch* merged_batch = nullptr;
-  *write_with_wal = 0;
+  size_t write_with_wal = 0;
   auto* leader = write_group.leader;
   if (write_group.size == 1 && leader->ShouldWriteToWAL() &&
       leader->batch->GetWalTerminationPoint().is_cleared()) {
@@ -675,64 +516,31 @@ WriteBatch* DBImpl::MergeBatch(const WriteThread::WriteGroup& write_group,
     // contains one batch, that batch should be written to the WAL,
     // and the batch is not wanting to be truncated
     merged_batch = leader->batch;
-    *write_with_wal = 1;
+    leader->log_used = logfile_number_;
+    write_with_wal = 1;
   } else {
     // WAL needs all of the batches flattened into a single batch.
     // We could avoid copying here with an iov-like AddRecord
     // interface
-    merged_batch = tmp_batch;
+    merged_batch = &tmp_batch_;
     for (auto writer : write_group) {
       if (writer->ShouldWriteToWAL()) {
         WriteBatchInternal::Append(merged_batch, writer->batch,
                                    /*WAL_only*/ true);
-        (*write_with_wal)++;
+        write_with_wal++;
       }
-    }
-  }
-  return merged_batch;
-}
-
-// When concurrent_prepare_ is disabled, this function is called from the only
-// write thread. Otherwise this must be called holding log_write_mutex_.
-Status DBImpl::WriteToWAL(const WriteBatch& merged_batch,
-                          log::Writer* log_writer, uint64_t* log_used,
-                          uint64_t* log_size) {
-  assert(log_size != nullptr);
-  Slice log_entry = WriteBatchInternal::Contents(&merged_batch);
-  *log_size = log_entry.size();
-  Status status = log_writer->AddRecord(log_entry);
-  if (log_used != nullptr) {
-    *log_used = logfile_number_;
-  }
-  total_log_size_ += log_entry.size();
-  // TODO(myabandeh): it might be unsafe to access alive_log_files_.back() here
-  // since alive_log_files_ might be modified concurrently
-  alive_log_files_.back().AddSize(log_entry.size());
-  log_empty_ = false;
-  return status;
-}
-
-Status DBImpl::WriteToWAL(const WriteThread::WriteGroup& write_group,
-                          log::Writer* log_writer, uint64_t* log_used,
-                          bool need_log_sync, bool need_log_dir_sync,
-                          SequenceNumber sequence) {
-  Status status;
-
-  size_t write_with_wal = 0;
-  WriteBatch* merged_batch =
-      MergeBatch(write_group, &tmp_batch_, &write_with_wal);
-  if (merged_batch == write_group.leader->batch) {
-    write_group.leader->log_used = logfile_number_;
-  } else if (write_with_wal > 1) {
-    for (auto writer : write_group) {
       writer->log_used = logfile_number_;
     }
   }
 
   WriteBatchInternal::SetSequence(merged_batch, sequence);
 
-  uint64_t log_size;
-  status = WriteToWAL(*merged_batch, log_writer, log_used, &log_size);
+  Slice log_entry = WriteBatchInternal::Contents(merged_batch);
+  status = log_writer->AddRecord(log_entry);
+  total_log_size_ += log_entry.size();
+  alive_log_files_.back().AddSize(log_entry.size());
+  log_empty_ = false;
+  uint64_t log_size = log_entry.size();
 
   if (status.ok() && need_log_sync) {
     StopWatch sw(env_, stats_, WAL_FILE_SYNC_MICROS);
@@ -774,48 +582,6 @@ Status DBImpl::WriteToWAL(const WriteThread::WriteGroup& write_group,
   return status;
 }
 
-Status DBImpl::ConcurrentWriteToWAL(const WriteThread::WriteGroup& write_group,
-                                    uint64_t* log_used,
-                                    SequenceNumber* last_sequence,
-                                    int total_count) {
-  Status status;
-
-  WriteBatch tmp_batch;
-  size_t write_with_wal = 0;
-  WriteBatch* merged_batch =
-      MergeBatch(write_group, &tmp_batch, &write_with_wal);
-
-  // We need to lock log_write_mutex_ since logs_ and alive_log_files might be
-  // pushed back concurrently
-  log_write_mutex_.Lock();
-  if (merged_batch == write_group.leader->batch) {
-    write_group.leader->log_used = logfile_number_;
-  } else if (write_with_wal > 1) {
-    for (auto writer : write_group) {
-      writer->log_used = logfile_number_;
-    }
-  }
-  *last_sequence = versions_->FetchAddLastToBeWrittenSequence(total_count);
-  auto sequence = *last_sequence + 1;
-  WriteBatchInternal::SetSequence(merged_batch, sequence);
-
-  log::Writer* log_writer = logs_.back().writer;
-  uint64_t log_size;
-  status = WriteToWAL(*merged_batch, log_writer, log_used, &log_size);
-  log_write_mutex_.Unlock();
-
-  if (status.ok()) {
-    const bool concurrent = true;
-    auto stats = default_cf_internal_stats_;
-    stats->AddDBStats(InternalStats::WAL_FILE_BYTES, log_size, concurrent);
-    RecordTick(stats_, WAL_FILE_BYTES, log_size);
-    stats->AddDBStats(InternalStats::WRITE_WITH_WAL, write_with_wal,
-                      concurrent);
-    RecordTick(stats_, WRITE_WITH_WAL, write_with_wal);
-  }
-  return status;
-}
-
 Status DBImpl::HandleWALFull(WriteContext* write_context) {
   mutex_.AssertHeld();
   assert(write_context != nullptr);
@@ -1001,9 +767,8 @@ Status DBImpl::ThrottleLowPriWritesIfNeeded(const WriteOptions& write_options,
       // is that in case the write is heavy, low pri writes may never have
       // a chance to run. Now we guarantee we are still slowly making
       // progress.
-      write_controller_.low_pri_rate_limiter()->Request(
-          my_batch->GetDataSize(), Env::IO_HIGH, nullptr /* stats */,
-          RateLimiter::OpType::kWrite);
+      write_controller_.low_pri_rate_limiter()->Request(my_batch->GetDataSize(),
+                                                        Env::IO_HIGH, nullptr);
     }
   }
   return Status::OK();
@@ -1043,13 +808,6 @@ void DBImpl::NotifyOnMemTableSealed(ColumnFamilyData* cfd,
 // REQUIRES: this thread is currently at the front of the writer queue
 Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
   mutex_.AssertHeld();
-  WriteThread::Writer nonmem_w;
-  if (concurrent_prepare_) {
-    // SwitchMemtable is a rare event. To simply the reasoning, we make sure
-    // that there is no concurrent thread writing to WAL.
-    nonmem_write_thread_.EnterUnbatched(&nonmem_w, &mutex_);
-  }
-
   unique_ptr<WritableFile> lfile;
   log::Writer* new_log = nullptr;
   MemTable* new_mem = nullptr;
@@ -1063,13 +821,7 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
   // Attempt to switch to a new memtable and trigger flush of old.
   // Do this without holding the dbmutex lock.
   assert(versions_->prev_log_number() == 0);
-  if (concurrent_prepare_) {
-    log_write_mutex_.Lock();
-  }
   bool creating_new_log = !log_empty_;
-  if (concurrent_prepare_) {
-    log_write_mutex_.Unlock();
-  }
   uint64_t recycle_log_number = 0;
   if (creating_new_log && immutable_db_options_.recycle_log_file_num &&
       !log_recycle_files.empty()) {
@@ -1081,7 +833,7 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
   SuperVersion* new_superversion = nullptr;
   const MutableCFOptions mutable_cf_options = *cfd->GetLatestMutableCFOptions();
 
-  // Set memtable_info for memtable sealed callback
+  // Set current_memtble_info for memtable sealed callback
 #ifndef ROCKSDB_LITE
   MemTableInfo memtable_info;
   memtable_info.cf_name = cfd->GetName();
@@ -1125,9 +877,9 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
         lfile->SetPreallocationBlockSize(preallocate_block_size);
         unique_ptr<WritableFileWriter> file_writer(
             new WritableFileWriter(std::move(lfile), opt_env_opt));
-        new_log = new log::Writer(
-            std::move(file_writer), new_log_number,
-            immutable_db_options_.recycle_log_file_num > 0, manual_wal_flush_);
+        new_log =
+            new log::Writer(std::move(file_writer), new_log_number,
+                            immutable_db_options_.recycle_log_file_num > 0);
       }
     }
 
@@ -1154,25 +906,15 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
     assert(creating_new_log);
     assert(!new_mem);
     assert(!new_log);
-    if (concurrent_prepare_) {
-      nonmem_write_thread_.ExitUnbatched(&nonmem_w);
-    }
     return s;
   }
   if (creating_new_log) {
-    log_write_mutex_.Lock();
     logfile_number_ = new_log_number;
     assert(new_log != nullptr);
     log_empty_ = true;
     log_dir_synced_ = false;
-    if (!logs_.empty()) {
-      // Alway flush the buffer of the last log before switching to a new one
-      log::Writer* cur_log_writer = logs_.back().writer;
-      cur_log_writer->WriteBuffer();
-    }
     logs_.emplace_back(logfile_number_, new_log);
     alive_log_files_.push_back(LogFileNumberSize(logfile_number_));
-    log_write_mutex_.Unlock();
   }
   for (auto loop_cfd : *versions_->GetColumnFamilySet()) {
     // all this is just optimization to delete logs that
@@ -1194,9 +936,6 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
   cfd->SetMemtable(new_mem);
   context->superversions_to_free_.push_back(InstallSuperVersionAndScheduleWork(
       cfd, new_superversion, mutable_cf_options));
-  if (concurrent_prepare_) {
-    nonmem_write_thread_.ExitUnbatched(&nonmem_w);
-  }
   return s;
 }
 

db/db_io_failure_test.cc

@@ -204,6 +204,7 @@ TEST_F(DBIOFailureTest, ManifestWriteError) {
     ASSERT_EQ("bar2", Get("foo2"));
   }
 }
+#endif  // ROCKSDB_LITE
 
 TEST_F(DBIOFailureTest, PutFailsParanoid) {
   // Test the following:
@@ -249,316 +250,7 @@ TEST_F(DBIOFailureTest, PutFailsParanoid) {
   // the next put should NOT fail
   ASSERT_TRUE(s.ok());
 }
-#if !(defined NDEBUG) || !defined(OS_WIN)
-TEST_F(DBIOFailureTest, FlushSstRangeSyncError) {
-  Options options = CurrentOptions();
-  options.env = env_;
-  options.create_if_missing = true;
-  options.error_if_exists = false;
-  options.paranoid_checks = true;
-  options.write_buffer_size = 256 * 1024 * 1024;
-  options.writable_file_max_buffer_size = 128 * 1024;
-  options.bytes_per_sync = 128 * 1024;
-  options.level0_file_num_compaction_trigger = 4;
-  options.memtable_factory.reset(new SpecialSkipListFactory(10));
-  BlockBasedTableOptions table_options;
-  table_options.filter_policy.reset(NewBloomFilterPolicy(10));
-  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
 
-  DestroyAndReopen(options);
-  CreateAndReopenWithCF({"pikachu"}, options);
-  Status s;
-
-  std::atomic<int> range_sync_called(0);
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "SpecialEnv::SStableFile::RangeSync", [&](void* arg) {
-        if (range_sync_called.fetch_add(1) == 0) {
-          Status* st = static_cast<Status*>(arg);
-          *st = Status::IOError("range sync dummy error");
-        }
-      });
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-
-  Random rnd(301);
-  std::string rnd_str =
-      RandomString(&rnd, static_cast<int>(options.bytes_per_sync / 2));
-  std::string rnd_str_512kb = RandomString(&rnd, 512 * 1024);
-
-  ASSERT_OK(Put(1, "foo", "bar"));
-  // First 1MB doesn't get range synced
-  ASSERT_OK(Put(1, "foo0_0", rnd_str_512kb));
-  ASSERT_OK(Put(1, "foo0_1", rnd_str_512kb));
-  ASSERT_OK(Put(1, "foo1_1", rnd_str));
-  ASSERT_OK(Put(1, "foo1_2", rnd_str));
-  ASSERT_OK(Put(1, "foo1_3", rnd_str));
-  ASSERT_OK(Put(1, "foo2", "bar"));
-  ASSERT_OK(Put(1, "foo3_1", rnd_str));
-  ASSERT_OK(Put(1, "foo3_2", rnd_str));
-  ASSERT_OK(Put(1, "foo3_3", rnd_str));
-  ASSERT_OK(Put(1, "foo4", "bar"));
-  dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
-
-  // Following writes should fail as flush failed.
-  ASSERT_NOK(Put(1, "foo2", "bar3"));
-  ASSERT_EQ("bar", Get(1, "foo"));
-
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-  ASSERT_GE(1, range_sync_called.load());
-
-  ReopenWithColumnFamilies({"default", "pikachu"}, options);
-  ASSERT_EQ("bar", Get(1, "foo"));
-}
-
-TEST_F(DBIOFailureTest, CompactSstRangeSyncError) {
-  Options options = CurrentOptions();
-  options.env = env_;
-  options.create_if_missing = true;
-  options.error_if_exists = false;
-  options.paranoid_checks = true;
-  options.write_buffer_size = 256 * 1024 * 1024;
-  options.writable_file_max_buffer_size = 128 * 1024;
-  options.bytes_per_sync = 128 * 1024;
-  options.level0_file_num_compaction_trigger = 2;
-  options.target_file_size_base = 256 * 1024 * 1024;
-  options.disable_auto_compactions = true;
-  BlockBasedTableOptions table_options;
-  table_options.filter_policy.reset(NewBloomFilterPolicy(10));
-  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-  DestroyAndReopen(options);
-  CreateAndReopenWithCF({"pikachu"}, options);
-  Status s;
-
-  Random rnd(301);
-  std::string rnd_str =
-      RandomString(&rnd, static_cast<int>(options.bytes_per_sync / 2));
-  std::string rnd_str_512kb = RandomString(&rnd, 512 * 1024);
-
-  ASSERT_OK(Put(1, "foo", "bar"));
-  // First 1MB doesn't get range synced
-  ASSERT_OK(Put(1, "foo0_0", rnd_str_512kb));
-  ASSERT_OK(Put(1, "foo0_1", rnd_str_512kb));
-  ASSERT_OK(Put(1, "foo1_1", rnd_str));
-  ASSERT_OK(Put(1, "foo1_2", rnd_str));
-  ASSERT_OK(Put(1, "foo1_3", rnd_str));
-  Flush(1);
-  ASSERT_OK(Put(1, "foo", "bar"));
-  ASSERT_OK(Put(1, "foo3_1", rnd_str));
-  ASSERT_OK(Put(1, "foo3_2", rnd_str));
-  ASSERT_OK(Put(1, "foo3_3", rnd_str));
-  ASSERT_OK(Put(1, "foo4", "bar"));
-  Flush(1);
-  dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
-
-  std::atomic<int> range_sync_called(0);
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "SpecialEnv::SStableFile::RangeSync", [&](void* arg) {
-        if (range_sync_called.fetch_add(1) == 0) {
-          Status* st = static_cast<Status*>(arg);
-          *st = Status::IOError("range sync dummy error");
-        }
-      });
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-
-  ASSERT_OK(dbfull()->SetOptions(handles_[1],
-                                 {
-                                     {"disable_auto_compactions", "false"},
-                                 }));
-  dbfull()->TEST_WaitForCompact();
-
-  // Following writes should fail as flush failed.
-  ASSERT_NOK(Put(1, "foo2", "bar3"));
-  ASSERT_EQ("bar", Get(1, "foo"));
-
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-  ASSERT_GE(1, range_sync_called.load());
-
-  ReopenWithColumnFamilies({"default", "pikachu"}, options);
-  ASSERT_EQ("bar", Get(1, "foo"));
-}
-
-TEST_F(DBIOFailureTest, FlushSstCloseError) {
-  Options options = CurrentOptions();
-  options.env = env_;
-  options.create_if_missing = true;
-  options.error_if_exists = false;
-  options.paranoid_checks = true;
-  options.level0_file_num_compaction_trigger = 4;
-  options.memtable_factory.reset(new SpecialSkipListFactory(2));
-
-  DestroyAndReopen(options);
-  CreateAndReopenWithCF({"pikachu"}, options);
-  Status s;
-  std::atomic<int> close_called(0);
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "SpecialEnv::SStableFile::Close", [&](void* arg) {
-        if (close_called.fetch_add(1) == 0) {
-          Status* st = static_cast<Status*>(arg);
-          *st = Status::IOError("close dummy error");
-        }
-      });
-
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-
-  ASSERT_OK(Put(1, "foo", "bar"));
-  ASSERT_OK(Put(1, "foo1", "bar1"));
-  ASSERT_OK(Put(1, "foo", "bar2"));
-  dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
-
-  // Following writes should fail as flush failed.
-  ASSERT_NOK(Put(1, "foo2", "bar3"));
-  ASSERT_EQ("bar2", Get(1, "foo"));
-  ASSERT_EQ("bar1", Get(1, "foo1"));
-
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-
-  ReopenWithColumnFamilies({"default", "pikachu"}, options);
-  ASSERT_EQ("bar2", Get(1, "foo"));
-  ASSERT_EQ("bar1", Get(1, "foo1"));
-}
-
-TEST_F(DBIOFailureTest, CompactionSstCloseError) {
-  Options options = CurrentOptions();
-  options.env = env_;
-  options.create_if_missing = true;
-  options.error_if_exists = false;
-  options.paranoid_checks = true;
-  options.level0_file_num_compaction_trigger = 2;
-  options.disable_auto_compactions = true;
-
-  DestroyAndReopen(options);
-  CreateAndReopenWithCF({"pikachu"}, options);
-  Status s;
-
-  ASSERT_OK(Put(1, "foo", "bar"));
-  ASSERT_OK(Put(1, "foo2", "bar"));
-  Flush(1);
-  ASSERT_OK(Put(1, "foo", "bar2"));
-  ASSERT_OK(Put(1, "foo2", "bar"));
-  Flush(1);
-  ASSERT_OK(Put(1, "foo", "bar3"));
-  ASSERT_OK(Put(1, "foo2", "bar"));
-  Flush(1);
-  dbfull()->TEST_WaitForCompact();
-
-  std::atomic<int> close_called(0);
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "SpecialEnv::SStableFile::Close", [&](void* arg) {
-        if (close_called.fetch_add(1) == 0) {
-          Status* st = static_cast<Status*>(arg);
-          *st = Status::IOError("close dummy error");
-        }
-      });
-
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-  ASSERT_OK(dbfull()->SetOptions(handles_[1],
-                                 {
-                                     {"disable_auto_compactions", "false"},
-                                 }));
-  dbfull()->TEST_WaitForCompact();
-
-  // Following writes should fail as compaction failed.
-  ASSERT_NOK(Put(1, "foo2", "bar3"));
-  ASSERT_EQ("bar3", Get(1, "foo"));
-
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-
-  ReopenWithColumnFamilies({"default", "pikachu"}, options);
-  ASSERT_EQ("bar3", Get(1, "foo"));
-}
-
-TEST_F(DBIOFailureTest, FlushSstSyncError) {
-  Options options = CurrentOptions();
-  options.env = env_;
-  options.create_if_missing = true;
-  options.error_if_exists = false;
-  options.paranoid_checks = true;
-  options.use_fsync = false;
-  options.level0_file_num_compaction_trigger = 4;
-  options.memtable_factory.reset(new SpecialSkipListFactory(2));
-
-  DestroyAndReopen(options);
-  CreateAndReopenWithCF({"pikachu"}, options);
-  Status s;
-  std::atomic<int> sync_called(0);
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "SpecialEnv::SStableFile::Sync", [&](void* arg) {
-        if (sync_called.fetch_add(1) == 0) {
-          Status* st = static_cast<Status*>(arg);
-          *st = Status::IOError("sync dummy error");
-        }
-      });
-
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-
-  ASSERT_OK(Put(1, "foo", "bar"));
-  ASSERT_OK(Put(1, "foo1", "bar1"));
-  ASSERT_OK(Put(1, "foo", "bar2"));
-  dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
-
-  // Following writes should fail as flush failed.
-  ASSERT_NOK(Put(1, "foo2", "bar3"));
-  ASSERT_EQ("bar2", Get(1, "foo"));
-  ASSERT_EQ("bar1", Get(1, "foo1"));
-
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-
-  ReopenWithColumnFamilies({"default", "pikachu"}, options);
-  ASSERT_EQ("bar2", Get(1, "foo"));
-  ASSERT_EQ("bar1", Get(1, "foo1"));
-}
-
-TEST_F(DBIOFailureTest, CompactionSstSyncError) {
-  Options options = CurrentOptions();
-  options.env = env_;
-  options.create_if_missing = true;
-  options.error_if_exists = false;
-  options.paranoid_checks = true;
-  options.level0_file_num_compaction_trigger = 2;
-  options.disable_auto_compactions = true;
-  options.use_fsync = false;
-
-  DestroyAndReopen(options);
-  CreateAndReopenWithCF({"pikachu"}, options);
-  Status s;
-
-  ASSERT_OK(Put(1, "foo", "bar"));
-  ASSERT_OK(Put(1, "foo2", "bar"));
-  Flush(1);
-  ASSERT_OK(Put(1, "foo", "bar2"));
-  ASSERT_OK(Put(1, "foo2", "bar"));
-  Flush(1);
-  ASSERT_OK(Put(1, "foo", "bar3"));
-  ASSERT_OK(Put(1, "foo2", "bar"));
-  Flush(1);
-  dbfull()->TEST_WaitForCompact();
-
-  std::atomic<int> sync_called(0);
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "SpecialEnv::SStableFile::Sync", [&](void* arg) {
-        if (sync_called.fetch_add(1) == 0) {
-          Status* st = static_cast<Status*>(arg);
-          *st = Status::IOError("close dummy error");
-        }
-      });
-
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-  ASSERT_OK(dbfull()->SetOptions(handles_[1],
-                                 {
-                                     {"disable_auto_compactions", "false"},
-                                 }));
-  dbfull()->TEST_WaitForCompact();
-
-  // Following writes should fail as compaction failed.
-  ASSERT_NOK(Put(1, "foo2", "bar3"));
-  ASSERT_EQ("bar3", Get(1, "foo"));
-
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-
-  ReopenWithColumnFamilies({"default", "pikachu"}, options);
-  ASSERT_EQ("bar3", Get(1, "foo"));
-}
-#endif  // !(defined NDEBUG) || !defined(OS_WIN)
-#endif  // ROCKSDB_LITE
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/db_iter.cc

@@ -8,6 +8,8 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
 #include "db/db_iter.h"
+#include <stdexcept>
+#include <deque>
 #include <string>
 #include <limits>
 
@@ -16,6 +18,7 @@
 #include "db/merge_helper.h"
 #include "db/pinned_iterators_manager.h"
 #include "monitoring/perf_context_imp.h"
+#include "port/port.h"
 #include "rocksdb/env.h"
 #include "rocksdb/iterator.h"
 #include "rocksdb/merge_operator.h"
@@ -83,7 +86,6 @@ class DBIter: public Iterator {
       RecordTick(global_statistics, NUMBER_DB_PREV, prev_count_);
       RecordTick(global_statistics, NUMBER_DB_PREV_FOUND, prev_found_count_);
       RecordTick(global_statistics, ITER_BYTES_READ, bytes_read_);
-      PERF_COUNTER_ADD(iter_read_bytes, bytes_read_);
       ResetCounters();
     }
 
@@ -99,12 +101,12 @@ class DBIter: public Iterator {
     uint64_t bytes_read_;
   };
 
-  DBIter(Env* _env, const ReadOptions& read_options,
+  DBIter(Env* env, const ReadOptions& read_options,
          const ImmutableCFOptions& cf_options, const Comparator* cmp,
          InternalIterator* iter, SequenceNumber s, bool arena_mode,
-         uint64_t max_sequential_skip_in_iterations, bool allow_blob)
+         uint64_t max_sequential_skip_in_iterations, uint64_t version_number)
       : arena_mode_(arena_mode),
-        env_(_env),
+        env_(env),
         logger_(cf_options.info_log),
         user_comparator_(cmp),
         merge_operator_(cf_options.merge_operator),
@@ -114,13 +116,13 @@ class DBIter: public Iterator {
         valid_(false),
         current_entry_is_merged_(false),
         statistics_(cf_options.statistics),
+        version_number_(version_number),
         iterate_upper_bound_(read_options.iterate_upper_bound),
         prefix_same_as_start_(read_options.prefix_same_as_start),
         pin_thru_lifetime_(read_options.pin_data),
         total_order_seek_(read_options.total_order_seek),
         range_del_agg_(cf_options.internal_comparator, s,
-                       true /* collapse_deletions */),
-        allow_blob_(allow_blob) {
+                       true /* collapse_deletions */) {
     RecordTick(statistics_, NO_ITERATORS);
     prefix_extractor_ = cf_options.prefix_extractor;
     max_skip_ = max_sequential_skip_in_iterations;
@@ -178,10 +180,6 @@ class DBIter: public Iterator {
       return status_;
     }
   }
-  bool IsBlob() const {
-    assert(valid_ && (allow_blob_ || !is_blob_));
-    return is_blob_;
-  }
 
   virtual Status GetProperty(std::string prop_name,
                              std::string* prop) override {
@@ -190,7 +188,10 @@ class DBIter: public Iterator {
     }
     if (prop_name == "rocksdb.iterator.super-version-number") {
       // First try to pass the value returned from inner iterator.
-      return iter_->GetProperty(prop_name, prop);
+      if (!iter_->GetProperty(prop_name, prop).ok()) {
+        *prop = ToString(version_number_);
+      }
+      return Status::OK();
     } else if (prop_name == "rocksdb.iterator.is-key-pinned") {
       if (valid_) {
         *prop = (pin_thru_lifetime_ && saved_key_.IsKeyPinned()) ? "1" : "0";
@@ -208,9 +209,6 @@ class DBIter: public Iterator {
   virtual void SeekForPrev(const Slice& target) override;
   virtual void SeekToFirst() override;
   virtual void SeekToLast() override;
-  Env* env() { return env_; }
-  void set_sequence(uint64_t s) { sequence_ = s; }
-  void set_valid(bool v) { valid_ = v; }
 
  private:
   void ReverseToForward();
@@ -262,7 +260,7 @@ class DBIter: public Iterator {
   const Comparator* const user_comparator_;
   const MergeOperator* const merge_operator_;
   InternalIterator* iter_;
-  SequenceNumber sequence_;
+  SequenceNumber const sequence_;
 
   Status status_;
   IterKey saved_key_;
@@ -276,6 +274,7 @@ class DBIter: public Iterator {
   uint64_t max_skip_;
   uint64_t max_skippable_internal_keys_;
   uint64_t num_internal_keys_skipped_;
+  uint64_t version_number_;
   const Slice* iterate_upper_bound_;
   IterKey prefix_start_buf_;
   Slice prefix_start_key_;
@@ -289,8 +288,6 @@ class DBIter: public Iterator {
   RangeDelAggregator range_del_agg_;
   LocalStatistics local_stats_;
   PinnedIteratorsManager pinned_iters_mgr_;
-  bool allow_blob_;
-  bool is_blob_;
 
   // No copying allowed
   DBIter(const DBIter&);
@@ -380,8 +377,6 @@ void DBIter::FindNextUserEntryInternal(bool skipping, bool prefix_check) {
   //  - none of the above  : saved_key_ can contain anything, it doesn't matter.
   uint64_t num_skipped = 0;
 
-  is_blob_ = false;
-
   do {
     ParsedInternalKey ikey;
 
@@ -426,7 +421,6 @@ void DBIter::FindNextUserEntryInternal(bool skipping, bool prefix_check) {
             PERF_COUNTER_ADD(internal_delete_skipped_count, 1);
             break;
           case kTypeValue:
-          case kTypeBlobIndex:
             saved_key_.SetUserKey(
                 ikey.user_key,
                 !iter_->IsKeyPinned() || !pin_thru_lifetime_ /* copy */);
@@ -438,18 +432,6 @@ void DBIter::FindNextUserEntryInternal(bool skipping, bool prefix_check) {
               skipping = true;
               num_skipped = 0;
               PERF_COUNTER_ADD(internal_delete_skipped_count, 1);
-            } else if (ikey.type == kTypeBlobIndex) {
-              if (!allow_blob_) {
-                ROCKS_LOG_ERROR(logger_, "Encounter unexpected blob index.");
-                status_ = Status::NotSupported(
-                    "Encounter unexpected blob index. Please open DB with "
-                    "rocksdb::blob_db::BlobDB instead.");
-                valid_ = false;
-              } else {
-                is_blob_ = true;
-                valid_ = true;
-              }
-              return;
             } else {
               valid_ = true;
               return;
@@ -591,18 +573,6 @@ void DBIter::MergeValuesNewToOld() {
       merge_context_.PushOperand(iter_->value(),
                                  iter_->IsValuePinned() /* operand_pinned */);
       PERF_COUNTER_ADD(internal_merge_count, 1);
-    } else if (kTypeBlobIndex == ikey.type) {
-      if (!allow_blob_) {
-        ROCKS_LOG_ERROR(logger_, "Encounter unexpected blob index.");
-        status_ = Status::NotSupported(
-            "Encounter unexpected blob index. Please open DB with "
-            "rocksdb::blob_db::BlobDB instead.");
-      } else {
-        status_ =
-            Status::NotSupported("Blob DB does not support merge operator.");
-      }
-      valid_ = false;
-      return;
     } else {
       assert(false);
     }
@@ -672,7 +642,6 @@ void DBIter::ReverseToBackward() {
     while (iter_->Valid() &&
            user_comparator_->Compare(ikey.user_key, saved_key_.GetUserKey()) >
                0) {
-      assert(ikey.sequence != kMaxSequenceNumber);
       if (ikey.sequence > sequence_) {
         PERF_COUNTER_ADD(internal_recent_skipped_count, 1);
       } else {
@@ -709,6 +678,7 @@ void DBIter::PrevInternal() {
         !iter_->IsKeyPinned() || !pin_thru_lifetime_ /* copy */);
 
     if (FindValueForCurrentKey()) {
+      valid_ = true;
       if (!iter_->Valid()) {
         return;
       }
@@ -775,7 +745,6 @@ bool DBIter::FindValueForCurrentKey() {
     last_key_entry_type = ikey.type;
     switch (last_key_entry_type) {
       case kTypeValue:
-      case kTypeBlobIndex:
         if (range_del_agg_.ShouldDelete(
                 ikey,
                 RangeDelAggregator::RangePositioningMode::kBackwardTraversal)) {
@@ -821,7 +790,6 @@ bool DBIter::FindValueForCurrentKey() {
   }
 
   Status s;
-  is_blob_ = false;
   switch (last_key_entry_type) {
     case kTypeDeletion:
     case kTypeSingleDeletion:
@@ -837,18 +805,6 @@ bool DBIter::FindValueForCurrentKey() {
             merge_operator_, saved_key_.GetUserKey(), nullptr,
             merge_context_.GetOperands(), &saved_value_, logger_, statistics_,
             env_, &pinned_value_, true);
-      } else if (last_not_merge_type == kTypeBlobIndex) {
-        if (!allow_blob_) {
-          ROCKS_LOG_ERROR(logger_, "Encounter unexpected blob index.");
-          status_ = Status::NotSupported(
-              "Encounter unexpected blob index. Please open DB with "
-              "rocksdb::blob_db::BlobDB instead.");
-        } else {
-          status_ =
-              Status::NotSupported("Blob DB does not support merge operator.");
-        }
-        valid_ = false;
-        return true;
       } else {
         assert(last_not_merge_type == kTypeValue);
         s = MergeHelper::TimedFullMerge(
@@ -860,17 +816,6 @@ bool DBIter::FindValueForCurrentKey() {
     case kTypeValue:
       // do nothing - we've already has value in saved_value_
       break;
-    case kTypeBlobIndex:
-      if (!allow_blob_) {
-        ROCKS_LOG_ERROR(logger_, "Encounter unexpected blob index.");
-        status_ = Status::NotSupported(
-            "Encounter unexpected blob index. Please open DB with "
-            "rocksdb::blob_db::BlobDB instead.");
-        valid_ = false;
-        return true;
-      }
-      is_blob_ = true;
-      break;
     default:
       assert(false);
       break;
@@ -904,15 +849,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
     valid_ = false;
     return false;
   }
-  if (ikey.type == kTypeBlobIndex && !allow_blob_) {
-    ROCKS_LOG_ERROR(logger_, "Encounter unexpected blob index.");
-    status_ = Status::NotSupported(
-        "Encounter unexpected blob index. Please open DB with "
-        "rocksdb::blob_db::BlobDB instead.");
-    valid_ = false;
-    return true;
-  }
-  if (ikey.type == kTypeValue || ikey.type == kTypeBlobIndex) {
+  if (ikey.type == kTypeValue) {
     assert(iter_->IsValuePinned());
     pinned_value_ = iter_->value();
     valid_ = true;
@@ -1017,7 +954,6 @@ void DBIter::FindPrevUserKey() {
         ++num_skipped;
       }
     }
-    assert(ikey.sequence != kMaxSequenceNumber);
     if (ikey.sequence > sequence_) {
       PERF_COUNTER_ADD(internal_recent_skipped_count, 1);
     } else {
@@ -1078,7 +1014,6 @@ void DBIter::Seek(const Slice& target) {
       if (valid_) {
         RecordTick(statistics_, NUMBER_DB_SEEK_FOUND);
         RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
-        PERF_COUNTER_ADD(iter_read_bytes, key().size() + value().size());
       }
     }
   } else {
@@ -1121,7 +1056,6 @@ void DBIter::SeekForPrev(const Slice& target) {
       if (valid_) {
         RecordTick(statistics_, NUMBER_DB_SEEK_FOUND);
         RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
-        PERF_COUNTER_ADD(iter_read_bytes, key().size() + value().size());
       }
     }
   } else {
@@ -1160,7 +1094,6 @@ void DBIter::SeekToFirst() {
       if (valid_) {
         RecordTick(statistics_, NUMBER_DB_SEEK_FOUND);
         RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
-        PERF_COUNTER_ADD(iter_read_bytes, key().size() + value().size());
       }
     }
   } else {
@@ -1208,7 +1141,6 @@ void DBIter::SeekToLast() {
     if (valid_) {
       RecordTick(statistics_, NUMBER_DB_SEEK_FOUND);
       RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
-      PERF_COUNTER_ADD(iter_read_bytes, key().size() + value().size());
     }
   }
   if (valid_ && prefix_extractor_ && prefix_same_as_start_) {
@@ -1224,15 +1156,17 @@ Iterator* NewDBIterator(Env* env, const ReadOptions& read_options,
                         InternalIterator* internal_iter,
                         const SequenceNumber& sequence,
                         uint64_t max_sequential_skip_in_iterations,
-                        bool allow_blob) {
+                        uint64_t version_number) {
   DBIter* db_iter = new DBIter(
       env, read_options, cf_options, user_key_comparator, internal_iter,
-      sequence, false, max_sequential_skip_in_iterations, allow_blob);
+      sequence, false, max_sequential_skip_in_iterations, version_number);
   return db_iter;
 }
 
 ArenaWrappedDBIter::~ArenaWrappedDBIter() { db_iter_->~DBIter(); }
 
+void ArenaWrappedDBIter::SetDBIter(DBIter* iter) { db_iter_ = iter; }
+
 RangeDelAggregator* ArenaWrappedDBIter::GetRangeDelAggregator() {
   return db_iter_->GetRangeDelAggregator();
 }
@@ -1255,70 +1189,28 @@ inline void ArenaWrappedDBIter::Prev() { db_iter_->Prev(); }
 inline Slice ArenaWrappedDBIter::key() const { return db_iter_->key(); }
 inline Slice ArenaWrappedDBIter::value() const { return db_iter_->value(); }
 inline Status ArenaWrappedDBIter::status() const { return db_iter_->status(); }
-bool ArenaWrappedDBIter::IsBlob() const { return db_iter_->IsBlob(); }
 inline Status ArenaWrappedDBIter::GetProperty(std::string prop_name,
                                               std::string* prop) {
-  if (prop_name == "rocksdb.iterator.super-version-number") {
-    // First try to pass the value returned from inner iterator.
-    if (!db_iter_->GetProperty(prop_name, prop).ok()) {
-      *prop = ToString(sv_number_);
-    }
-    return Status::OK();
-  }
   return db_iter_->GetProperty(prop_name, prop);
 }
-
-void ArenaWrappedDBIter::Init(Env* env, const ReadOptions& read_options,
-                              const ImmutableCFOptions& cf_options,
-                              const SequenceNumber& sequence,
-                              uint64_t max_sequential_skip_in_iteration,
-                              uint64_t version_number, bool allow_blob) {
-  auto mem = arena_.AllocateAligned(sizeof(DBIter));
-  db_iter_ = new (mem)
-      DBIter(env, read_options, cf_options, cf_options.user_comparator, nullptr,
-             sequence, true, max_sequential_skip_in_iteration, allow_blob);
-  sv_number_ = version_number;
-}
-
-Status ArenaWrappedDBIter::Refresh() {
-  if (cfd_ == nullptr || db_impl_ == nullptr) {
-    return Status::NotSupported("Creating renew iterator is not allowed.");
-  }
-  assert(db_iter_ != nullptr);
-  SequenceNumber latest_seq = db_impl_->GetLatestSequenceNumber();
-  uint64_t cur_sv_number = cfd_->GetSuperVersionNumber();
-  if (sv_number_ != cur_sv_number) {
-    Env* env = db_iter_->env();
-    db_iter_->~DBIter();
-    arena_.~Arena();
-    new (&arena_) Arena();
-
-    SuperVersion* sv = cfd_->GetReferencedSuperVersion(db_impl_->mutex());
-    Init(env, read_options_, *(cfd_->ioptions()), latest_seq,
-         sv->mutable_cf_options.max_sequential_skip_in_iterations,
-         cur_sv_number, allow_blob_);
-
-    InternalIterator* internal_iter = db_impl_->NewInternalIterator(
-        read_options_, cfd_, sv, &arena_, db_iter_->GetRangeDelAggregator());
-    SetIterUnderDBIter(internal_iter);
-  } else {
-    db_iter_->set_sequence(latest_seq);
-    db_iter_->set_valid(false);
-  }
-  return Status::OK();
+void ArenaWrappedDBIter::RegisterCleanup(CleanupFunction function, void* arg1,
+                                         void* arg2) {
+  db_iter_->RegisterCleanup(function, arg1, arg2);
 }
 
 ArenaWrappedDBIter* NewArenaWrappedDbIterator(
     Env* env, const ReadOptions& read_options,
-    const ImmutableCFOptions& cf_options, const SequenceNumber& sequence,
-    uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
-    DBImpl* db_impl, ColumnFamilyData* cfd, bool allow_blob) {
+    const ImmutableCFOptions& cf_options, const Comparator* user_key_comparator,
+    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations,
+    uint64_t version_number) {
   ArenaWrappedDBIter* iter = new ArenaWrappedDBIter();
-  iter->Init(env, read_options, cf_options, sequence,
-             max_sequential_skip_in_iterations, version_number, allow_blob);
-  if (db_impl != nullptr && cfd != nullptr) {
-    iter->StoreRefreshInfo(read_options, db_impl, cfd, allow_blob);
-  }
+  Arena* arena = iter->GetArena();
+  auto mem = arena->AllocateAligned(sizeof(DBIter));
+  DBIter* db_iter = new (mem)
+      DBIter(env, read_options, cf_options, user_key_comparator, nullptr,
+             sequence, true, max_sequential_skip_in_iterations, version_number);
+
+  iter->SetDBIter(db_iter);
 
   return iter;
 }

db/db_iter.h

@@ -10,7 +10,6 @@
 #pragma once
 #include <stdint.h>
 #include <string>
-#include "db/db_impl.h"
 #include "db/dbformat.h"
 #include "db/range_del_aggregator.h"
 #include "options/cf_options.h"
@@ -34,7 +33,7 @@ extern Iterator* NewDBIterator(Env* env, const ReadOptions& read_options,
                                InternalIterator* internal_iter,
                                const SequenceNumber& sequence,
                                uint64_t max_sequential_skip_in_iterations,
-                               bool allow_blob = false);
+                               uint64_t version_number);
 
 // A wrapper iterator which wraps DB Iterator and the arena, with which the DB
 // iterator is supposed be allocated. This class is used as an entry point of
@@ -50,6 +49,10 @@ class ArenaWrappedDBIter : public Iterator {
   virtual Arena* GetArena() { return &arena_; }
   virtual RangeDelAggregator* GetRangeDelAggregator();
 
+  // Set the DB Iterator to be wrapped
+
+  virtual void SetDBIter(DBIter* iter);
+
   // Set the internal iterator wrapped inside the DB Iterator. Usually it is
   // a merging iterator.
   virtual void SetIterUnderDBIter(InternalIterator* iter);
@@ -63,43 +66,20 @@ class ArenaWrappedDBIter : public Iterator {
   virtual Slice key() const override;
   virtual Slice value() const override;
   virtual Status status() const override;
-  virtual Status Refresh() override;
-  bool IsBlob() const;
 
+  void RegisterCleanup(CleanupFunction function, void* arg1, void* arg2);
   virtual Status GetProperty(std::string prop_name, std::string* prop) override;
 
-  void Init(Env* env, const ReadOptions& read_options,
-            const ImmutableCFOptions& cf_options,
-            const SequenceNumber& sequence,
-            uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
-            bool allow_blob);
-
-  void StoreRefreshInfo(const ReadOptions& read_options, DBImpl* db_impl,
-                        ColumnFamilyData* cfd, bool allow_blob) {
-    read_options_ = read_options;
-    db_impl_ = db_impl;
-    cfd_ = cfd;
-    allow_blob_ = allow_blob;
-  }
-
  private:
   DBIter* db_iter_;
   Arena arena_;
-  uint64_t sv_number_;
-  ColumnFamilyData* cfd_ = nullptr;
-  DBImpl* db_impl_ = nullptr;
-  ReadOptions read_options_;
-  bool allow_blob_ = false;
 };
 
 // Generate the arena wrapped iterator class.
-// `db_impl` and `cfd` are used for reneweal. If left null, renewal will not
-// be supported.
 extern ArenaWrappedDBIter* NewArenaWrappedDbIterator(
     Env* env, const ReadOptions& read_options,
-    const ImmutableCFOptions& cf_options, const SequenceNumber& sequence,
-    uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
-    DBImpl* db_impl = nullptr, ColumnFamilyData* cfd = nullptr,
-    bool allow_blob = false);
+    const ImmutableCFOptions& cf_options, const Comparator* user_key_comparator,
+    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations,
+    uint64_t version_number);
 
 }  // namespace rocksdb

db/db_iter_test.cc

@@ -193,7 +193,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
     ReadOptions ro;
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));
 
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
@@ -225,7 +225,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
     ReadOptions ro;
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));
 
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
@@ -251,7 +251,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));
 
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
@@ -283,7 +283,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));
 
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
@@ -318,7 +318,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));
 
     db_iter->SeekToLast();
     ASSERT_TRUE(!db_iter->Valid());
@@ -347,7 +347,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      7, options.max_sequential_skip_in_iterations));
+                      7, options.max_sequential_skip_in_iterations, 0));
 
     SetPerfLevel(kEnableCount);
     ASSERT_TRUE(GetPerfLevel() == kEnableCount);
@@ -384,7 +384,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      4, options.max_sequential_skip_in_iterations));
+                      4, options.max_sequential_skip_in_iterations, 0));
 
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
@@ -409,7 +409,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));
 
     db_iter->SeekToLast();
     ASSERT_TRUE(!db_iter->Valid());
@@ -431,7 +431,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));
 
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
@@ -466,7 +466,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      7, options.max_sequential_skip_in_iterations));
+                      7, options.max_sequential_skip_in_iterations, 0));
 
     SetPerfLevel(kEnableCount);
     ASSERT_TRUE(GetPerfLevel() == kEnableCount);
@@ -495,7 +495,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
     ReadOptions ro;
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));
 
     db_iter->SeekToFirst();
     ASSERT_TRUE(db_iter->Valid());
@@ -537,7 +537,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
     ReadOptions ro;
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      2, options.max_sequential_skip_in_iterations));
+                      2, options.max_sequential_skip_in_iterations, 0));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "b");
@@ -568,7 +568,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
     ReadOptions ro;
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "c");
@@ -597,7 +597,7 @@ TEST_F(DBIteratorTest, DBIteratorEmpty) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      0, options.max_sequential_skip_in_iterations));
+                      0, options.max_sequential_skip_in_iterations, 0));
     db_iter->SeekToLast();
     ASSERT_TRUE(!db_iter->Valid());
   }
@@ -608,7 +608,7 @@ TEST_F(DBIteratorTest, DBIteratorEmpty) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      0, options.max_sequential_skip_in_iterations));
+                      0, options.max_sequential_skip_in_iterations, 0));
     db_iter->SeekToFirst();
     ASSERT_TRUE(!db_iter->Valid());
   }
@@ -630,7 +630,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkipCountSkips) {
 
   std::unique_ptr<Iterator> db_iter(NewDBIterator(
       env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 2, options.max_sequential_skip_in_iterations));
+      internal_iter, 2, options.max_sequential_skip_in_iterations, 0));
   db_iter->SeekToLast();
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), "c");
@@ -673,7 +673,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
       options.statistics = rocksdb::CreateDBStatistics();
       std::unique_ptr<Iterator> db_iter(NewDBIterator(
           env_, ro, cf_options, BytewiseComparator(), internal_iter, i + 2,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));
       db_iter->SeekToLast();
       ASSERT_TRUE(db_iter->Valid());
 
@@ -708,7 +708,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
 
       std::unique_ptr<Iterator> db_iter(NewDBIterator(
           env_, ro, cf_options, BytewiseComparator(), internal_iter, i + 2,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));
       db_iter->SeekToLast();
       ASSERT_TRUE(db_iter->Valid());
 
@@ -736,7 +736,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
 
       std::unique_ptr<Iterator> db_iter(NewDBIterator(
           env_, ro, cf_options, BytewiseComparator(), internal_iter, 202,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));
       db_iter->SeekToLast();
       ASSERT_TRUE(db_iter->Valid());
 
@@ -768,7 +768,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
       internal_iter->Finish();
       std::unique_ptr<Iterator> db_iter(NewDBIterator(
           env_, ro, cf_options, BytewiseComparator(), internal_iter, i,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));
       db_iter->SeekToLast();
       ASSERT_TRUE(!db_iter->Valid());
 
@@ -784,7 +784,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
     internal_iter->Finish();
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      200, options.max_sequential_skip_in_iterations));
+                      200, options.max_sequential_skip_in_iterations, 0));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "c");
@@ -818,7 +818,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
 
       std::unique_ptr<Iterator> db_iter(NewDBIterator(
           env_, ro, cf_options, BytewiseComparator(), internal_iter, i + 2,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));
       db_iter->SeekToLast();
       ASSERT_TRUE(db_iter->Valid());
 
@@ -852,7 +852,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
 
       std::unique_ptr<Iterator> db_iter(NewDBIterator(
           env_, ro, cf_options, BytewiseComparator(), internal_iter, i + 2,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));
       db_iter->SeekToLast();
       ASSERT_TRUE(db_iter->Valid());
 
@@ -900,7 +900,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
     ro.max_skippable_internal_keys = 0;
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));
 
     db_iter->SeekToFirst();
     ASSERT_TRUE(db_iter->Valid());
@@ -946,7 +946,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
     ro.max_skippable_internal_keys = 2;
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));
 
     db_iter->SeekToFirst();
     ASSERT_TRUE(db_iter->Valid());
@@ -990,7 +990,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
     ro.max_skippable_internal_keys = 2;
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));
 
     db_iter->SeekToFirst();
     ASSERT_TRUE(db_iter->Valid());
@@ -1028,7 +1028,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
     ro.max_skippable_internal_keys = 2;
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));
 
     db_iter->SeekToFirst();
     ASSERT_TRUE(db_iter->Valid());
@@ -1063,7 +1063,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
     ro.max_skippable_internal_keys = 2;
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));
 
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
@@ -1093,7 +1093,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
     ro.max_skippable_internal_keys = 2;
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));
 
     db_iter->SeekToFirst();
     ASSERT_TRUE(db_iter->Valid());
@@ -1130,7 +1130,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
     ro.max_skippable_internal_keys = 2;
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));
 
     db_iter->SeekToFirst();
     ASSERT_TRUE(db_iter->Valid());
@@ -1167,7 +1167,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
       ro.max_skippable_internal_keys = i;
       std::unique_ptr<Iterator> db_iter(NewDBIterator(
           env_, ro, cf_options, BytewiseComparator(), internal_iter, 2 * i + 1,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));
 
       db_iter->SeekToFirst();
       ASSERT_TRUE(db_iter->Valid());
@@ -1220,7 +1220,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
       ro.max_skippable_internal_keys = i;
       std::unique_ptr<Iterator> db_iter(NewDBIterator(
           env_, ro, cf_options, BytewiseComparator(), internal_iter, 2 * i + 1,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));
 
       db_iter->SeekToFirst();
       ASSERT_TRUE(db_iter->Valid());
@@ -1258,7 +1258,7 @@ TEST_F(DBIteratorTest, DBIterator1) {
 
   std::unique_ptr<Iterator> db_iter(NewDBIterator(
       env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 1, options.max_sequential_skip_in_iterations));
+      internal_iter, 1, options.max_sequential_skip_in_iterations, 0));
   db_iter->SeekToFirst();
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1285,7 +1285,7 @@ TEST_F(DBIteratorTest, DBIterator2) {
 
   std::unique_ptr<Iterator> db_iter(NewDBIterator(
       env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 0, options.max_sequential_skip_in_iterations));
+      internal_iter, 0, options.max_sequential_skip_in_iterations, 0));
   db_iter->SeekToFirst();
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1309,7 +1309,7 @@ TEST_F(DBIteratorTest, DBIterator3) {
 
   std::unique_ptr<Iterator> db_iter(NewDBIterator(
       env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 2, options.max_sequential_skip_in_iterations));
+      internal_iter, 2, options.max_sequential_skip_in_iterations, 0));
   db_iter->SeekToFirst();
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1333,7 +1333,7 @@ TEST_F(DBIteratorTest, DBIterator4) {
 
   std::unique_ptr<Iterator> db_iter(NewDBIterator(
       env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 4, options.max_sequential_skip_in_iterations));
+      internal_iter, 4, options.max_sequential_skip_in_iterations, 0));
   db_iter->SeekToFirst();
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1365,7 +1365,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      0, options.max_sequential_skip_in_iterations));
+                      0, options.max_sequential_skip_in_iterations, 0));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1387,7 +1387,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      1, options.max_sequential_skip_in_iterations));
+                      1, options.max_sequential_skip_in_iterations, 0));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1409,7 +1409,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      2, options.max_sequential_skip_in_iterations));
+                      2, options.max_sequential_skip_in_iterations, 0));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1431,7 +1431,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      3, options.max_sequential_skip_in_iterations));
+                      3, options.max_sequential_skip_in_iterations, 0));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1453,7 +1453,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      4, options.max_sequential_skip_in_iterations));
+                      4, options.max_sequential_skip_in_iterations, 0));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1475,7 +1475,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      5, options.max_sequential_skip_in_iterations));
+                      5, options.max_sequential_skip_in_iterations, 0));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1497,7 +1497,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      6, options.max_sequential_skip_in_iterations));
+                      6, options.max_sequential_skip_in_iterations, 0));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1517,7 +1517,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
     internal_iter->Finish();
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));
     db_iter->Seek("b");
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "b");
@@ -1546,7 +1546,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      0, options.max_sequential_skip_in_iterations));
+                      0, options.max_sequential_skip_in_iterations, 0));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1568,7 +1568,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      1, options.max_sequential_skip_in_iterations));
+                      1, options.max_sequential_skip_in_iterations, 0));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1590,7 +1590,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      2, options.max_sequential_skip_in_iterations));
+                      2, options.max_sequential_skip_in_iterations, 0));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1612,7 +1612,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      3, options.max_sequential_skip_in_iterations));
+                      3, options.max_sequential_skip_in_iterations, 0));
     db_iter->SeekToLast();
     ASSERT_TRUE(!db_iter->Valid());
   }
@@ -1630,7 +1630,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      4, options.max_sequential_skip_in_iterations));
+                      4, options.max_sequential_skip_in_iterations, 0));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1652,7 +1652,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      5, options.max_sequential_skip_in_iterations));
+                      5, options.max_sequential_skip_in_iterations, 0));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1674,7 +1674,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      6, options.max_sequential_skip_in_iterations));
+                      6, options.max_sequential_skip_in_iterations, 0));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1715,7 +1715,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      0, options.max_sequential_skip_in_iterations));
+                      0, options.max_sequential_skip_in_iterations, 0));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1749,7 +1749,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      2, options.max_sequential_skip_in_iterations));
+                      2, options.max_sequential_skip_in_iterations, 0));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
 
@@ -1789,7 +1789,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      4, options.max_sequential_skip_in_iterations));
+                      4, options.max_sequential_skip_in_iterations, 0));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
 
@@ -1829,7 +1829,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      5, options.max_sequential_skip_in_iterations));
+                      5, options.max_sequential_skip_in_iterations, 0));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
 
@@ -1874,7 +1874,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      6, options.max_sequential_skip_in_iterations));
+                      6, options.max_sequential_skip_in_iterations, 0));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
 
@@ -1920,7 +1920,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      7, options.max_sequential_skip_in_iterations));
+                      7, options.max_sequential_skip_in_iterations, 0));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
 
@@ -1960,7 +1960,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      9, options.max_sequential_skip_in_iterations));
+                      9, options.max_sequential_skip_in_iterations, 0));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
 
@@ -2006,7 +2006,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      13, options.max_sequential_skip_in_iterations));
+                      13, options.max_sequential_skip_in_iterations, 0));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
 
@@ -2053,7 +2053,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
 
     std::unique_ptr<Iterator> db_iter(
         NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      14, options.max_sequential_skip_in_iterations));
+                      14, options.max_sequential_skip_in_iterations, 0));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
 
@@ -2083,7 +2083,7 @@ TEST_F(DBIteratorTest, DBIterator8) {
 
   std::unique_ptr<Iterator> db_iter(NewDBIterator(
       env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 10, options.max_sequential_skip_in_iterations));
+      internal_iter, 10, options.max_sequential_skip_in_iterations, 0));
   db_iter->SeekToLast();
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), "b");
@@ -2113,7 +2113,7 @@ TEST_F(DBIteratorTest, DBIterator9) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-        internal_iter, 10, options.max_sequential_skip_in_iterations));
+        internal_iter, 10, options.max_sequential_skip_in_iterations, 0));
 
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
@@ -2179,7 +2179,7 @@ TEST_F(DBIteratorTest, DBIterator10) {
 
   std::unique_ptr<Iterator> db_iter(NewDBIterator(
       env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 10, options.max_sequential_skip_in_iterations));
+      internal_iter, 10, options.max_sequential_skip_in_iterations, 0));
 
   db_iter->Seek("c");
   ASSERT_TRUE(db_iter->Valid());
@@ -2218,7 +2218,7 @@ TEST_F(DBIteratorTest, SeekToLastOccurrenceSeq0) {
 
   std::unique_ptr<Iterator> db_iter(
       NewDBIterator(env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-                    internal_iter, 10, 0 /* force seek */));
+                    internal_iter, 10, 0 /* force seek */, 0));
   db_iter->SeekToFirst();
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -2246,7 +2246,7 @@ TEST_F(DBIteratorTest, DBIterator11) {
 
   std::unique_ptr<Iterator> db_iter(NewDBIterator(
       env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 1, options.max_sequential_skip_in_iterations));
+      internal_iter, 1, options.max_sequential_skip_in_iterations, 0));
   db_iter->SeekToFirst();
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -2272,7 +2272,7 @@ TEST_F(DBIteratorTest, DBIterator12) {
 
   std::unique_ptr<Iterator> db_iter(
       NewDBIterator(env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-                    internal_iter, 10, 0));
+                    internal_iter, 10, 0, 0));
   db_iter->SeekToLast();
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), "c");
@@ -2309,7 +2309,7 @@ TEST_F(DBIteratorTest, DBIterator13) {
 
   std::unique_ptr<Iterator> db_iter(
       NewDBIterator(env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-                    internal_iter, 2, 3));
+                    internal_iter, 2, 3, 0));
   db_iter->Seek("b");
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), key);
@@ -2337,7 +2337,7 @@ TEST_F(DBIteratorTest, DBIterator14) {
 
   std::unique_ptr<Iterator> db_iter(
       NewDBIterator(env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-                    internal_iter, 4, 1));
+                    internal_iter, 4, 1, 0));
   db_iter->Seek("b");
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), "b");
@@ -2376,7 +2376,7 @@ class DBIterWithMergeIterTest : public testing::Test {
     db_iter_.reset(NewDBIterator(env_, ro_, ImmutableCFOptions(options_),
                                  BytewiseComparator(), merge_iter,
                                  8 /* read data earlier than seqId 8 */,
-                                 3 /* max iterators before reseek */));
+                                 3 /* max iterators before reseek */, 0));
   }
 
   Env* env_;

db/db_iterator_test.cc

@@ -1719,15 +1719,12 @@ TEST_F(DBIteratorTest, IteratorWithLocalStatistics) {
 
   std::vector<port::Thread> threads;
   std::function<void()> reader_func_next = [&]() {
-    SetPerfLevel(kEnableCount);
-    get_perf_context()->Reset();
     Iterator* iter = db_->NewIterator(ReadOptions());
 
     iter->SeekToFirst();
     // Seek will bump ITER_BYTES_READ
-    uint64_t bytes = 0;
-    bytes += iter->key().size();
-    bytes += iter->value().size();
+    total_bytes += iter->key().size();
+    total_bytes += iter->value().size();
     while (true) {
       iter->Next();
       total_next++;
@@ -1736,25 +1733,20 @@ TEST_F(DBIteratorTest, IteratorWithLocalStatistics) {
         break;
       }
       total_next_found++;
-      bytes += iter->key().size();
-      bytes += iter->value().size();
+      total_bytes += iter->key().size();
+      total_bytes += iter->value().size();
     }
 
     delete iter;
-    ASSERT_EQ(bytes, get_perf_context()->iter_read_bytes);
-    SetPerfLevel(kDisable);
-    total_bytes += bytes;
   };
 
   std::function<void()> reader_func_prev = [&]() {
-    SetPerfLevel(kEnableCount);
     Iterator* iter = db_->NewIterator(ReadOptions());
 
     iter->SeekToLast();
     // Seek will bump ITER_BYTES_READ
-    uint64_t bytes = 0;
-    bytes += iter->key().size();
-    bytes += iter->value().size();
+    total_bytes += iter->key().size();
+    total_bytes += iter->value().size();
     while (true) {
       iter->Prev();
       total_prev++;
@@ -1763,14 +1755,11 @@ TEST_F(DBIteratorTest, IteratorWithLocalStatistics) {
         break;
       }
       total_prev_found++;
-      bytes += iter->key().size();
-      bytes += iter->value().size();
+      total_bytes += iter->key().size();
+      total_bytes += iter->value().size();
     }
 
     delete iter;
-    ASSERT_EQ(bytes, get_perf_context()->iter_read_bytes);
-    SetPerfLevel(kDisable);
-    total_bytes += bytes;
   };
 
   for (int i = 0; i < 10; i++) {
@@ -1920,65 +1909,6 @@ TEST_F(DBIteratorTest, DBIteratorSkipRecentDuplicatesTest) {
                  NUMBER_OF_RESEEKS_IN_ITERATION));
 }
 
-TEST_F(DBIteratorTest, Refresh) {
-  ASSERT_OK(Put("x", "y"));
-
-  std::unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions()));
-  iter->Seek(Slice("a"));
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("x")), 0);
-  iter->Next();
-  ASSERT_FALSE(iter->Valid());
-
-  ASSERT_OK(Put("c", "d"));
-
-  iter->Seek(Slice("a"));
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("x")), 0);
-  iter->Next();
-  ASSERT_FALSE(iter->Valid());
-
-  iter->Refresh();
-
-  iter->Seek(Slice("a"));
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("c")), 0);
-  iter->Next();
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("x")), 0);
-  iter->Next();
-  ASSERT_FALSE(iter->Valid());
-
-  dbfull()->Flush(FlushOptions());
-
-  ASSERT_OK(Put("m", "n"));
-
-  iter->Seek(Slice("a"));
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("c")), 0);
-  iter->Next();
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("x")), 0);
-  iter->Next();
-  ASSERT_FALSE(iter->Valid());
-
-  iter->Refresh();
-
-  iter->Seek(Slice("a"));
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("c")), 0);
-  iter->Next();
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("m")), 0);
-  iter->Next();
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("x")), 0);
-  iter->Next();
-  ASSERT_FALSE(iter->Valid());
-
-  iter.reset();
-}
-
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/db_log_iter_test.cc

@@ -116,7 +116,6 @@ TEST_F(DBTestXactLogIterator, TransactionLogIteratorRace) {
       dbfull()->Flush(FlushOptions());
       Put("key4", DummyString(1024));
       ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 4U);
-      dbfull()->FlushWAL(false);
 
       {
         auto iter = OpenTransactionLogIter(0);
@@ -133,7 +132,6 @@ TEST_F(DBTestXactLogIterator, TransactionLogIteratorRace) {
 
       // "key5" would be written in a new memtable and log
       Put("key5", DummyString(1024));
-      dbfull()->FlushWAL(false);
       {
         // this iter would miss "key4" if not fixed
         auto iter = OpenTransactionLogIter(0);
@@ -183,7 +181,6 @@ TEST_F(DBTestXactLogIterator, TransactionLogIteratorCorruptedLog) {
       Put("key"+ToString(i), DummyString(10));
     }
     dbfull()->Flush(FlushOptions());
-    dbfull()->FlushWAL(false);
     // Corrupt this log to create a gap
     rocksdb::VectorLogPtr wal_files;
     ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
@@ -197,7 +194,6 @@ TEST_F(DBTestXactLogIterator, TransactionLogIteratorCorruptedLog) {
 
     // Insert a new entry to a new log file
     Put("key1025", DummyString(10));
-    dbfull()->FlushWAL(false);
     // Try to read from the beginning. Should stop before the gap and read less
     // than 1025 entries
     auto iter = OpenTransactionLogIter(0);

db/db_options_test.cc

@@ -414,7 +414,6 @@ TEST_F(DBOptionsTest, DeleteObsoleteFilesPeriodChange) {
 TEST_F(DBOptionsTest, MaxOpenFilesChange) {
   SpecialEnv env(env_);
   Options options;
-  options.env = CurrentOptions().env;
   options.max_open_files = -1;
 
   Reopen(options);

db/db_properties_test.cc

@@ -1309,80 +1309,6 @@ TEST_F(DBPropertiesTest, EstimateNumKeysUnderflow) {
   ASSERT_EQ(0, num_keys);
 }
 
-TEST_F(DBPropertiesTest, EstimateOldestKeyTime) {
-  std::unique_ptr<MockTimeEnv> mock_env(new MockTimeEnv(Env::Default()));
-  uint64_t oldest_key_time = 0;
-  Options options;
-  options.env = mock_env.get();
-
-  // "rocksdb.estimate-oldest-key-time" only available to fifo compaction.
-  mock_env->set_current_time(100);
-  for (auto compaction : {kCompactionStyleLevel, kCompactionStyleUniversal,
-                          kCompactionStyleNone}) {
-    options.compaction_style = compaction;
-    options.create_if_missing = true;
-    DestroyAndReopen(options);
-    ASSERT_OK(Put("foo", "bar"));
-    ASSERT_FALSE(dbfull()->GetIntProperty(
-        DB::Properties::kEstimateOldestKeyTime, &oldest_key_time));
-  }
-
-  options.compaction_style = kCompactionStyleFIFO;
-  options.compaction_options_fifo.ttl = 300;
-  options.compaction_options_fifo.allow_compaction = false;
-  DestroyAndReopen(options);
-
-  mock_env->set_current_time(100);
-  ASSERT_OK(Put("k1", "v1"));
-  ASSERT_TRUE(dbfull()->GetIntProperty(DB::Properties::kEstimateOldestKeyTime,
-                                       &oldest_key_time));
-  ASSERT_EQ(100, oldest_key_time);
-  ASSERT_OK(Flush());
-  ASSERT_EQ("1", FilesPerLevel());
-  ASSERT_TRUE(dbfull()->GetIntProperty(DB::Properties::kEstimateOldestKeyTime,
-                                       &oldest_key_time));
-  ASSERT_EQ(100, oldest_key_time);
-
-  mock_env->set_current_time(200);
-  ASSERT_OK(Put("k2", "v2"));
-  ASSERT_OK(Flush());
-  ASSERT_EQ("2", FilesPerLevel());
-  ASSERT_TRUE(dbfull()->GetIntProperty(DB::Properties::kEstimateOldestKeyTime,
-                                       &oldest_key_time));
-  ASSERT_EQ(100, oldest_key_time);
-
-  mock_env->set_current_time(300);
-  ASSERT_OK(Put("k3", "v3"));
-  ASSERT_OK(Flush());
-  ASSERT_EQ("3", FilesPerLevel());
-  ASSERT_TRUE(dbfull()->GetIntProperty(DB::Properties::kEstimateOldestKeyTime,
-                                       &oldest_key_time));
-  ASSERT_EQ(100, oldest_key_time);
-
-  mock_env->set_current_time(450);
-  ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
-  ASSERT_EQ("2", FilesPerLevel());
-  ASSERT_TRUE(dbfull()->GetIntProperty(DB::Properties::kEstimateOldestKeyTime,
-                                       &oldest_key_time));
-  ASSERT_EQ(200, oldest_key_time);
-
-  mock_env->set_current_time(550);
-  ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
-  ASSERT_EQ("1", FilesPerLevel());
-  ASSERT_TRUE(dbfull()->GetIntProperty(DB::Properties::kEstimateOldestKeyTime,
-                                       &oldest_key_time));
-  ASSERT_EQ(300, oldest_key_time);
-
-  mock_env->set_current_time(650);
-  ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
-  ASSERT_EQ("", FilesPerLevel());
-  ASSERT_FALSE(dbfull()->GetIntProperty(DB::Properties::kEstimateOldestKeyTime,
-                                        &oldest_key_time));
-
-  // Close before mock_env destructs.
-  Close();
-}
-
 #endif  // ROCKSDB_LITE
 }  // namespace rocksdb
 

db/db_range_del_test.cc

@@ -27,9 +27,6 @@ class DBRangeDelTest : public DBTestBase {
 // ROCKSDB_LITE
 #ifndef ROCKSDB_LITE
 TEST_F(DBRangeDelTest, NonBlockBasedTableNotSupported) {
-  if (!IsMemoryMappedAccessSupported()) {
-    return;
-  }
   Options opts = CurrentOptions();
   opts.table_factory.reset(new PlainTableFactory());
   opts.prefix_extractor.reset(NewNoopTransform());
@@ -819,183 +816,8 @@ TEST_F(DBRangeDelTest, TailingIteratorRangeTombstoneUnsupported) {
   }
   db_->ReleaseSnapshot(snapshot);
 }
-
 #endif  // !ROCKSDB_UBSAN_RUN
 
-TEST_F(DBRangeDelTest, SubcompactionHasEmptyDedicatedRangeDelFile) {
-  const int kNumFiles = 2, kNumKeysPerFile = 4;
-  Options options = CurrentOptions();
-  options.compression = kNoCompression;
-  options.disable_auto_compactions = true;
-  options.level0_file_num_compaction_trigger = kNumFiles;
-  options.max_subcompactions = 2;
-  options.num_levels = 2;
-  options.target_file_size_base = 4096;
-  Reopen(options);
-
-  // need a L1 file for subcompaction to be triggered
-  ASSERT_OK(
-      db_->Put(WriteOptions(), db_->DefaultColumnFamily(), Key(0), "val"));
-  ASSERT_OK(db_->Flush(FlushOptions()));
-  MoveFilesToLevel(1);
-
-  // put enough keys to fill up the first subcompaction, and later range-delete
-  // them so that the first subcompaction outputs no key-values. In that case
-  // it'll consider making an SST file dedicated to range deletions.
-  for (int i = 0; i < kNumKeysPerFile; ++i) {
-    ASSERT_OK(db_->Put(WriteOptions(), db_->DefaultColumnFamily(), Key(i),
-                       std::string(1024, 'a')));
-  }
-  ASSERT_OK(db_->Flush(FlushOptions()));
-  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(0),
-                             Key(kNumKeysPerFile)));
-
-  // the above range tombstone can be dropped, so that one alone won't cause a
-  // dedicated file to be opened. We can make one protected by snapshot that
-  // must be considered. Make its range outside the first subcompaction's range
-  // to exercise the tricky part of the code.
-  const Snapshot* snapshot = db_->GetSnapshot();
-  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(),
-                             Key(kNumKeysPerFile + 1),
-                             Key(kNumKeysPerFile + 2)));
-  ASSERT_OK(db_->Flush(FlushOptions()));
-
-  ASSERT_EQ(kNumFiles, NumTableFilesAtLevel(0));
-  ASSERT_EQ(1, NumTableFilesAtLevel(1));
-
-  db_->EnableAutoCompaction({db_->DefaultColumnFamily()});
-  dbfull()->TEST_WaitForCompact();
-  db_->ReleaseSnapshot(snapshot);
-}
-
-TEST_F(DBRangeDelTest, MemtableBloomFilter) {
-  // regression test for #2743. the range delete tombstones in memtable should
-  // be added even when Get() skips searching due to its prefix bloom filter
-  const int kMemtableSize = 1 << 20;              // 1MB
-  const int kMemtablePrefixFilterSize = 1 << 13;  // 8KB
-  const int kNumKeys = 1000;
-  const int kPrefixLen = 8;
-  Options options = CurrentOptions();
-  options.memtable_prefix_bloom_size_ratio =
-      static_cast<double>(kMemtablePrefixFilterSize) / kMemtableSize;
-  options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(kPrefixLen));
-  options.write_buffer_size = kMemtableSize;
-  Reopen(options);
-
-  for (int i = 0; i < kNumKeys; ++i) {
-    ASSERT_OK(Put(Key(i), "val"));
-  }
-  Flush();
-  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(0),
-                             Key(kNumKeys)));
-  for (int i = 0; i < kNumKeys; ++i) {
-    std::string value;
-    ASSERT_TRUE(db_->Get(ReadOptions(), Key(i), &value).IsNotFound());
-  }
-}
-
-TEST_F(DBRangeDelTest, CompactionTreatsSplitInputLevelDeletionAtomically) {
-  // make sure compaction treats files containing a split range deletion in the
-  // input level as an atomic unit. I.e., compacting any input-level file(s)
-  // containing a portion of the range deletion causes all other input-level
-  // files containing portions of that same range deletion to be included in the
-  // compaction.
-  const int kNumFilesPerLevel = 4, kValueBytes = 4 << 10;
-  Options options = CurrentOptions();
-  options.compression = kNoCompression;
-  options.level0_file_num_compaction_trigger = kNumFilesPerLevel;
-  options.memtable_factory.reset(
-      new SpecialSkipListFactory(2 /* num_entries_flush */));
-  options.target_file_size_base = kValueBytes;
-  // i == 0: CompactFiles
-  // i == 1: CompactRange
-  // i == 2: automatic compaction
-  for (int i = 0; i < 3; ++i) {
-    DestroyAndReopen(options);
-
-    ASSERT_OK(Put(Key(0), ""));
-    ASSERT_OK(db_->Flush(FlushOptions()));
-    MoveFilesToLevel(2);
-    ASSERT_EQ(1, NumTableFilesAtLevel(2));
-
-    // snapshot protects range tombstone from dropping due to becoming obsolete.
-    const Snapshot* snapshot = db_->GetSnapshot();
-    db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(0),
-                     Key(2 * kNumFilesPerLevel));
-
-    Random rnd(301);
-    std::string value = RandomString(&rnd, kValueBytes);
-    for (int j = 0; j < kNumFilesPerLevel; ++j) {
-      // give files overlapping key-ranges to prevent trivial move
-      ASSERT_OK(Put(Key(j), value));
-      ASSERT_OK(Put(Key(2 * kNumFilesPerLevel - 1 - j), value));
-      if (j > 0) {
-        dbfull()->TEST_WaitForFlushMemTable();
-        ASSERT_EQ(j, NumTableFilesAtLevel(0));
-      }
-    }
-    // put extra key to trigger final flush
-    ASSERT_OK(Put("", ""));
-    dbfull()->TEST_WaitForFlushMemTable();
-    dbfull()->TEST_WaitForCompact();
-    ASSERT_EQ(0, NumTableFilesAtLevel(0));
-    ASSERT_EQ(kNumFilesPerLevel, NumTableFilesAtLevel(1));
-
-    ColumnFamilyMetaData meta;
-    db_->GetColumnFamilyMetaData(&meta);
-    if (i == 0) {
-      ASSERT_OK(db_->CompactFiles(
-          CompactionOptions(), {meta.levels[1].files[0].name}, 2 /* level */));
-    } else if (i == 1) {
-      auto begin_str = Key(0), end_str = Key(1);
-      Slice begin = begin_str, end = end_str;
-      ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &begin, &end));
-    } else if (i == 2) {
-      ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(),
-                                {{"max_bytes_for_level_base", "10000"}}));
-      dbfull()->TEST_WaitForCompact();
-    }
-    ASSERT_EQ(0, NumTableFilesAtLevel(1));
-    ASSERT_GT(NumTableFilesAtLevel(2), 0);
-
-    db_->ReleaseSnapshot(snapshot);
-  }
-}
-
-TEST_F(DBRangeDelTest, UnorderedTombstones) {
-  // Regression test for #2752. Range delete tombstones between
-  // different snapshot stripes are not stored in order, so the first
-  // tombstone of each snapshot stripe should be checked as a smallest
-  // candidate.
-  Options options = CurrentOptions();
-  DestroyAndReopen(options);
-
-  auto cf = db_->DefaultColumnFamily();
-
-  ASSERT_OK(db_->Put(WriteOptions(), cf, "a", "a"));
-  ASSERT_OK(db_->Flush(FlushOptions(), cf));
-  ASSERT_EQ(1, NumTableFilesAtLevel(0));
-  ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr));
-  ASSERT_EQ(1, NumTableFilesAtLevel(1));
-
-  ASSERT_OK(db_->DeleteRange(WriteOptions(), cf, "b", "c"));
-  // Hold a snapshot to separate these two delete ranges.
-  auto snapshot = db_->GetSnapshot();
-  ASSERT_OK(db_->DeleteRange(WriteOptions(), cf, "a", "b"));
-  ASSERT_OK(db_->Flush(FlushOptions(), cf));
-  db_->ReleaseSnapshot(snapshot);
-
-  std::vector<std::vector<FileMetaData>> files;
-  dbfull()->TEST_GetFilesMetaData(cf, &files);
-  ASSERT_EQ(1, files[0].size());
-  ASSERT_EQ("a", files[0][0].smallest.user_key());
-  ASSERT_EQ("c", files[0][0].largest.user_key());
-
-  std::string v;
-  auto s = db_->Get(ReadOptions(), "a", &v);
-  ASSERT_TRUE(s.IsNotFound());
-}
-
 #endif  // ROCKSDB_LITE
 
 }  // namespace rocksdb

db/db_sst_test.cc

@@ -335,7 +335,6 @@ TEST_F(DBSSTTest, RateLimitedDelete) {
   ASSERT_OK(s);
   options.sst_file_manager->SetDeleteRateBytesPerSecond(rate_bytes_per_sec);
   auto sfm = static_cast<SstFileManagerImpl*>(options.sst_file_manager.get());
-  sfm->delete_scheduler()->TEST_SetMaxTrashDBRatio(1.1);
 
   ASSERT_OK(TryReopen(options));
   // Create 4 files in L0
@@ -401,7 +400,6 @@ TEST_F(DBSSTTest, DeleteSchedulerMultipleDBPaths) {
       env_, nullptr, trash_dir, rate_bytes_per_sec, false, &s));
   ASSERT_OK(s);
   auto sfm = static_cast<SstFileManagerImpl*>(options.sst_file_manager.get());
-  sfm->delete_scheduler()->TEST_SetMaxTrashDBRatio(1.1);
 
   DestroyAndReopen(options);
 
@@ -456,14 +454,9 @@ TEST_F(DBSSTTest, DestroyDBWithRateLimitedDelete) {
       [&](void* arg) { bg_delete_file++; });
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 
-  Status s;
   Options options = CurrentOptions();
   options.disable_auto_compactions = true;
   options.env = env_;
-  std::string trash_dir = test::TmpDir(env_) + "/trash";
-  options.sst_file_manager.reset(
-      NewSstFileManager(env_, nullptr, trash_dir, 0, false, &s));
-  ASSERT_OK(s);
   DestroyAndReopen(options);
 
   // Create 4 files in L0
@@ -476,12 +469,15 @@ TEST_F(DBSSTTest, DestroyDBWithRateLimitedDelete) {
 
   // Close DB and destroy it using DeleteScheduler
   Close();
+  std::string trash_dir = test::TmpDir(env_) + "/trash";
+  int64_t rate_bytes_per_sec = 1024 * 1024;  // 1 Mb / Sec
+  Status s;
+  options.sst_file_manager.reset(NewSstFileManager(
+      env_, nullptr, trash_dir, rate_bytes_per_sec, false, &s));
+  ASSERT_OK(s);
+  ASSERT_OK(DestroyDB(dbname_, options));
 
   auto sfm = static_cast<SstFileManagerImpl*>(options.sst_file_manager.get());
-
-  sfm->SetDeleteRateBytesPerSecond(1024 * 1024);
-  sfm->delete_scheduler()->TEST_SetMaxTrashDBRatio(1.1);
-  ASSERT_OK(DestroyDB(dbname_, options));
   sfm->WaitForEmptyTrash();
   // We have deleted the 4 sst files in the delete_scheduler
   ASSERT_EQ(bg_delete_file, 4);
@@ -650,18 +646,9 @@ TEST_F(DBSSTTest, OpenDBWithInfiniteMaxOpenFiles) {
 }
 
 TEST_F(DBSSTTest, GetTotalSstFilesSize) {
-  // We don't propagate oldest-key-time table property on compaction and
-  // just write 0 as default value. This affect the exact table size, since
-  // we encode table properties as varint64. Force time to be 0 to work around
-  // it. Should remove the workaround after we propagate the property on
-  // compaction.
-  std::unique_ptr<MockTimeEnv> mock_env(new MockTimeEnv(Env::Default()));
-  mock_env->set_current_time(0);
-
   Options options = CurrentOptions();
   options.disable_auto_compactions = true;
   options.compression = kNoCompression;
-  options.env = mock_env.get();
   DestroyAndReopen(options);
   // Generate 5 files in L0
   for (int i = 0; i < 5; i++) {
@@ -748,9 +735,6 @@ TEST_F(DBSSTTest, GetTotalSstFilesSize) {
   // Live SST files = 0
   // Total SST files = 0
   ASSERT_EQ(total_sst_files_size, 0);
-
-  // Close db before mock_env destruct.
-  Close();
 }
 
 TEST_F(DBSSTTest, GetTotalSstFilesSizeVersionsFilesShared) {

db/db_test.cc

@@ -23,7 +23,6 @@
 #include <alloca.h>
 #endif
 
-#include "cache/lru_cache.h"
 #include "db/db_impl.h"
 #include "db/db_test_util.h"
 #include "db/dbformat.h"
@@ -2235,10 +2234,6 @@ class ModelDB : public DB {
     return Status::NotSupported("Not implemented.");
   }
 
-  virtual Status VerifyChecksum() override {
-    return Status::NotSupported("Not implemented.");
-  }
-
   using DB::GetPropertiesOfAllTables;
   virtual Status GetPropertiesOfAllTables(
       ColumnFamilyHandle* column_family,
@@ -2795,7 +2790,7 @@ TEST_F(DBTest, FIFOCompactionTestWithCompaction) {
   ASSERT_EQ(NumTableFilesAtLevel(0), 10);
 
   for (int i = 0; i < 60; i++) {
-    // Generate and flush a file about 20KB.
+    // Generate and flush a file about 10KB.
     for (int j = 0; j < 20; j++) {
       ASSERT_OK(Put(ToString(i * 20 + j + 2000), RandomString(&rnd, 980)));
     }
@@ -2810,295 +2805,6 @@ TEST_F(DBTest, FIFOCompactionTestWithCompaction) {
   ASSERT_LE(SizeAtLevel(0),
             options.compaction_options_fifo.max_table_files_size);
 }
-
-TEST_F(DBTest, FIFOCompactionStyleWithCompactionAndDelete) {
-  Options options;
-  options.compaction_style = kCompactionStyleFIFO;
-  options.write_buffer_size = 20 << 10;  // 20K
-  options.arena_block_size = 4096;
-  options.compaction_options_fifo.max_table_files_size = 1500 << 10;  // 1MB
-  options.compaction_options_fifo.allow_compaction = true;
-  options.level0_file_num_compaction_trigger = 3;
-  options.compression = kNoCompression;
-  options.create_if_missing = true;
-  options = CurrentOptions(options);
-  DestroyAndReopen(options);
-
-  Random rnd(301);
-  for (int i = 0; i < 3; i++) {
-    // Each file contains a different key which will be dropped later.
-    ASSERT_OK(Put("a" + ToString(i), RandomString(&rnd, 500)));
-    ASSERT_OK(Put("key" + ToString(i), ""));
-    ASSERT_OK(Put("z" + ToString(i), RandomString(&rnd, 500)));
-    Flush();
-    ASSERT_OK(dbfull()->TEST_WaitForCompact());
-  }
-  ASSERT_EQ(NumTableFilesAtLevel(0), 1);
-  for (int i = 0; i < 3; i++) {
-    ASSERT_EQ("", Get("key" + ToString(i)));
-  }
-  for (int i = 0; i < 3; i++) {
-    // Each file contains a different key which will be dropped later.
-    ASSERT_OK(Put("a" + ToString(i), RandomString(&rnd, 500)));
-    ASSERT_OK(Delete("key" + ToString(i)));
-    ASSERT_OK(Put("z" + ToString(i), RandomString(&rnd, 500)));
-    Flush();
-    ASSERT_OK(dbfull()->TEST_WaitForCompact());
-  }
-  ASSERT_EQ(NumTableFilesAtLevel(0), 2);
-  for (int i = 0; i < 3; i++) {
-    ASSERT_EQ("NOT_FOUND", Get("key" + ToString(i)));
-  }
-}
-
-// Check that FIFO-with-TTL is not supported with max_open_files != -1.
-TEST_F(DBTest, FIFOCompactionWithTTLAndMaxOpenFilesTest) {
-  Options options;
-  options.compaction_style = kCompactionStyleFIFO;
-  options.create_if_missing = true;
-  options.compaction_options_fifo.ttl = 600;  // seconds
-
-  // Check that it is not supported with max_open_files != -1.
-  options.max_open_files = 100;
-  options = CurrentOptions(options);
-  ASSERT_TRUE(TryReopen(options).IsNotSupported());
-
-  options.max_open_files = -1;
-  ASSERT_OK(TryReopen(options));
-}
-
-// Check that FIFO-with-TTL is supported only with BlockBasedTableFactory.
-TEST_F(DBTest, FIFOCompactionWithTTLAndVariousTableFormatsTest) {
-  Options options;
-  options.compaction_style = kCompactionStyleFIFO;
-  options.create_if_missing = true;
-  options.compaction_options_fifo.ttl = 600;  // seconds
-
-  options = CurrentOptions(options);
-  options.table_factory.reset(NewBlockBasedTableFactory());
-  ASSERT_OK(TryReopen(options));
-
-  Destroy(options);
-  options.table_factory.reset(NewPlainTableFactory());
-  ASSERT_TRUE(TryReopen(options).IsNotSupported());
-
-  Destroy(options);
-  options.table_factory.reset(NewCuckooTableFactory());
-  ASSERT_TRUE(TryReopen(options).IsNotSupported());
-
-  Destroy(options);
-  options.table_factory.reset(NewAdaptiveTableFactory());
-  ASSERT_TRUE(TryReopen(options).IsNotSupported());
-}
-
-TEST_F(DBTest, FIFOCompactionWithTTLTest) {
-  Options options;
-  options.compaction_style = kCompactionStyleFIFO;
-  options.write_buffer_size = 10 << 10;  // 10KB
-  options.arena_block_size = 4096;
-  options.compression = kNoCompression;
-  options.create_if_missing = true;
-  env_->time_elapse_only_sleep_ = false;
-  options.env = env_;
-
-  // Test to make sure that all files with expired ttl are deleted on next
-  // manual compaction.
-  {
-    env_->addon_time_.store(0);
-    options.compaction_options_fifo.max_table_files_size = 150 << 10;  // 150KB
-    options.compaction_options_fifo.allow_compaction = false;
-    options.compaction_options_fifo.ttl = 1 * 60 * 60 ;  // 1 hour
-    options = CurrentOptions(options);
-    DestroyAndReopen(options);
-
-    Random rnd(301);
-    for (int i = 0; i < 10; i++) {
-      // Generate and flush a file about 10KB.
-      for (int j = 0; j < 10; j++) {
-        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
-      }
-      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
-    }
-    ASSERT_EQ(NumTableFilesAtLevel(0), 10);
-
-    // Sleep for 2 hours -- which is much greater than TTL.
-    // Note: Couldn't use SleepForMicroseconds because it takes an int instead
-    // of uint64_t. Hence used addon_time_ directly.
-    // env_->SleepForMicroseconds(2 * 60 * 60 * 1000 * 1000);
-    env_->addon_time_.fetch_add(2 * 60 * 60);
-
-    // Since no flushes and compactions have run, the db should still be in
-    // the same state even after considerable time has passed.
-    ASSERT_OK(dbfull()->TEST_WaitForCompact());
-    ASSERT_EQ(NumTableFilesAtLevel(0), 10);
-
-    dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
-    ASSERT_EQ(NumTableFilesAtLevel(0), 0);
-  }
-
-  // Test to make sure that all files with expired ttl are deleted on next
-  // automatic compaction.
-  {
-    options.compaction_options_fifo.max_table_files_size = 150 << 10;  // 150KB
-    options.compaction_options_fifo.allow_compaction = false;
-    options.compaction_options_fifo.ttl = 1 * 60 * 60;  // 1 hour
-    options = CurrentOptions(options);
-    DestroyAndReopen(options);
-
-    Random rnd(301);
-    for (int i = 0; i < 10; i++) {
-      // Generate and flush a file about 10KB.
-      for (int j = 0; j < 10; j++) {
-        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
-      }
-      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
-    }
-    ASSERT_EQ(NumTableFilesAtLevel(0), 10);
-
-    // Sleep for 2 hours -- which is much greater than TTL.
-    env_->addon_time_.fetch_add(2 * 60 * 60);
-    // Just to make sure that we are in the same state even after sleeping.
-    ASSERT_OK(dbfull()->TEST_WaitForCompact());
-    ASSERT_EQ(NumTableFilesAtLevel(0), 10);
-
-    // Create 1 more file to trigger TTL compaction. The old files are dropped.
-    for (int i = 0; i < 1; i++) {
-      for (int j = 0; j < 10; j++) {
-        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
-      }
-      Flush();
-    }
-
-    ASSERT_OK(dbfull()->TEST_WaitForCompact());
-    // Only the new 10 files remain.
-    ASSERT_EQ(NumTableFilesAtLevel(0), 1);
-    ASSERT_LE(SizeAtLevel(0),
-              options.compaction_options_fifo.max_table_files_size);
-  }
-
-  // Test that shows the fall back to size-based FIFO compaction if TTL-based
-  // deletion doesn't move the total size to be less than max_table_files_size.
-  {
-    options.write_buffer_size = 10 << 10;                              // 10KB
-    options.compaction_options_fifo.max_table_files_size = 150 << 10;  // 150KB
-    options.compaction_options_fifo.allow_compaction = false;
-    options.compaction_options_fifo.ttl =  1 * 60 * 60;  // 1 hour
-    options = CurrentOptions(options);
-    DestroyAndReopen(options);
-
-    Random rnd(301);
-    for (int i = 0; i < 3; i++) {
-      // Generate and flush a file about 10KB.
-      for (int j = 0; j < 10; j++) {
-        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
-      }
-      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
-    }
-    ASSERT_EQ(NumTableFilesAtLevel(0), 3);
-
-    // Sleep for 2 hours -- which is much greater than TTL.
-    env_->addon_time_.fetch_add(2 * 60 * 60);
-    // Just to make sure that we are in the same state even after sleeping.
-    ASSERT_OK(dbfull()->TEST_WaitForCompact());
-    ASSERT_EQ(NumTableFilesAtLevel(0), 3);
-
-    for (int i = 0; i < 5; i++) {
-      for (int j = 0; j < 140; j++) {
-        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
-      }
-      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
-    }
-    // Size limit is still guaranteed.
-    ASSERT_LE(SizeAtLevel(0),
-              options.compaction_options_fifo.max_table_files_size);
-  }
-
-  // Test with TTL + Intra-L0 compactions.
-  {
-    options.compaction_options_fifo.max_table_files_size = 150 << 10;  // 150KB
-    options.compaction_options_fifo.allow_compaction = true;
-    options.compaction_options_fifo.ttl = 1 * 60 * 60;  // 1 hour
-    options.level0_file_num_compaction_trigger = 6;
-    options = CurrentOptions(options);
-    DestroyAndReopen(options);
-
-    Random rnd(301);
-    for (int i = 0; i < 10; i++) {
-      // Generate and flush a file about 10KB.
-      for (int j = 0; j < 10; j++) {
-        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
-      }
-      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
-    }
-    // With Intra-L0 compaction, out of 10 files, 6 files will be compacted to 1
-    // (due to level0_file_num_compaction_trigger = 6).
-    // So total files = 1 + remaining 4 = 5.
-    ASSERT_EQ(NumTableFilesAtLevel(0), 5);
-
-    // Sleep for 2 hours -- which is much greater than TTL.
-    env_->addon_time_.fetch_add(2 * 60 * 60);
-    // Just to make sure that we are in the same state even after sleeping.
-    ASSERT_OK(dbfull()->TEST_WaitForCompact());
-    ASSERT_EQ(NumTableFilesAtLevel(0), 5);
-
-    // Create 10 more files. The old 5 files are dropped as their ttl expired.
-    for (int i = 0; i < 10; i++) {
-      for (int j = 0; j < 10; j++) {
-        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
-      }
-      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
-    }
-    ASSERT_EQ(NumTableFilesAtLevel(0), 5);
-    ASSERT_LE(SizeAtLevel(0),
-              options.compaction_options_fifo.max_table_files_size);
-  }
-
-  // Test with large TTL + Intra-L0 compactions.
-  // Files dropped based on size, as ttl doesn't kick in.
-  {
-    options.write_buffer_size = 20 << 10;                               // 20K
-    options.compaction_options_fifo.max_table_files_size = 1500 << 10;  // 1.5MB
-    options.compaction_options_fifo.allow_compaction = true;
-    options.compaction_options_fifo.ttl = 1 * 60 * 60;  // 1 hour
-    options.level0_file_num_compaction_trigger = 6;
-    options = CurrentOptions(options);
-    DestroyAndReopen(options);
-
-    Random rnd(301);
-    for (int i = 0; i < 60; i++) {
-      // Generate and flush a file about 20KB.
-      for (int j = 0; j < 20; j++) {
-        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
-      }
-      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
-    }
-    // It should be compacted to 10 files.
-    ASSERT_EQ(NumTableFilesAtLevel(0), 10);
-
-    for (int i = 0; i < 60; i++) {
-      // Generate and flush a file about 20KB.
-      for (int j = 0; j < 20; j++) {
-        ASSERT_OK(Put(ToString(i * 20 + j + 2000), RandomString(&rnd, 980)));
-      }
-      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
-    }
-
-    // It should be compacted to no more than 20 files.
-    ASSERT_GT(NumTableFilesAtLevel(0), 10);
-    ASSERT_LT(NumTableFilesAtLevel(0), 18);
-    // Size limit is still guaranteed.
-    ASSERT_LE(SizeAtLevel(0),
-              options.compaction_options_fifo.max_table_files_size);
-  }
-}
 #endif  // ROCKSDB_LITE
 
 #ifndef ROCKSDB_LITE
@@ -3347,23 +3053,11 @@ TEST_F(DBTest, DynamicMemtableOptions) {
       {"write_buffer_size", "131072"},
   }));
 
-  // The existing memtable inflated 64KB->128KB when we invoked SetOptions().
-  // Write 192KB, we should have a 128KB L0 file and a memtable with 64KB data.
-  gen_l0_kb(192);
-  ASSERT_EQ(NumTableFilesAtLevel(0), 1);  // (A)
-  ASSERT_LT(SizeAtLevel(0), k128KB + 2 * k5KB);
-  ASSERT_GT(SizeAtLevel(0), k128KB - 4 * k5KB);
-
-  // Decrease buffer size below current usage
-  ASSERT_OK(dbfull()->SetOptions({
-      {"write_buffer_size", "65536"},
-  }));
-  // The existing memtable became eligible for flush when we reduced its
-  // capacity to 64KB. Two keys need to be added to trigger flush: first causes
-  // memtable to be marked full, second schedules the flush. Then we should have
-  // a 128KB L0 file, a 64KB L0 file, and a memtable with just one key.
-  gen_l0_kb(2);
-  ASSERT_EQ(NumTableFilesAtLevel(0), 2);
+  // The existing memtable is still 64KB in size, after it becomes immutable,
+  // the next memtable will be 128KB in size. Write 256KB total, we should
+  // have a 64KB L0 file, a 128KB L0 file, and a memtable with 64KB data
+  gen_l0_kb(256);
+  ASSERT_EQ(NumTableFilesAtLevel(0), 2);  // (A)
   ASSERT_LT(SizeAtLevel(0), k128KB + k64KB + 2 * k5KB);
   ASSERT_GT(SizeAtLevel(0), k128KB + k64KB - 4 * k5KB);
 
@@ -5387,36 +5081,6 @@ TEST_F(DBTest, RowCache) {
   ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), 1);
   ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), 1);
 }
-
-TEST_F(DBTest, PinnableSliceAndRowCache) {
-  Options options = CurrentOptions();
-  options.statistics = rocksdb::CreateDBStatistics();
-  options.row_cache = NewLRUCache(8192);
-  DestroyAndReopen(options);
-
-  ASSERT_OK(Put("foo", "bar"));
-  ASSERT_OK(Flush());
-
-  ASSERT_EQ(Get("foo"), "bar");
-  ASSERT_EQ(
-      reinterpret_cast<LRUCache*>(options.row_cache.get())->TEST_GetLRUSize(),
-      1);
-
-  {
-    PinnableSlice pin_slice;
-    ASSERT_EQ(Get("foo", &pin_slice), Status::OK());
-    ASSERT_EQ(pin_slice.ToString(), "bar");
-    // Entry is already in cache, lookup will remove the element from lru
-    ASSERT_EQ(
-        reinterpret_cast<LRUCache*>(options.row_cache.get())->TEST_GetLRUSize(),
-        0);
-  }
-  // After PinnableSlice destruction element is added back in LRU
-  ASSERT_EQ(
-      reinterpret_cast<LRUCache*>(options.row_cache.get())->TEST_GetLRUSize(),
-      1);
-}
-
 #endif  // ROCKSDB_LITE
 
 TEST_F(DBTest, DeletingOldWalAfterDrop) {

db/db_test2.cc

@@ -1028,7 +1028,6 @@ TEST_F(DBTest2, PresetCompressionDict) {
   const int kNumL0Files = 5;
 
   Options options;
-  options.env = CurrentOptions().env; // Make sure to use any custom env that the test is configured with.
   options.allow_concurrent_memtable_write = false;
   options.arena_block_size = kBlockSizeBytes;
   options.compaction_style = kCompactionStyleUniversal;
@@ -2235,102 +2234,6 @@ TEST_F(DBTest2, LowPriWrite) {
   Put("", "", wo);
   ASSERT_EQ(1, rate_limit_count.load());
 }
-
-#ifndef ROCKSDB_LITE
-TEST_F(DBTest2, RateLimitedCompactionReads) {
-  // compaction input has 512KB data
-  const int kNumKeysPerFile = 128;
-  const int kBytesPerKey = 1024;
-  const int kNumL0Files = 4;
-
-  for (auto use_direct_io : {false, true}) {
-    if (use_direct_io && !IsDirectIOSupported()) {
-      continue;
-    }
-    Options options = CurrentOptions();
-    options.compression = kNoCompression;
-    options.level0_file_num_compaction_trigger = kNumL0Files;
-    options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
-    options.new_table_reader_for_compaction_inputs = true;
-    // takes roughly one second, split into 100 x 10ms intervals. Each interval
-    // permits 5.12KB, which is smaller than the block size, so this test
-    // exercises the code for chunking reads.
-    options.rate_limiter.reset(NewGenericRateLimiter(
-        static_cast<int64_t>(kNumL0Files * kNumKeysPerFile *
-                             kBytesPerKey) /* rate_bytes_per_sec */,
-        10 * 1000 /* refill_period_us */, 10 /* fairness */,
-        RateLimiter::Mode::kReadsOnly));
-    options.use_direct_io_for_flush_and_compaction = use_direct_io;
-    BlockBasedTableOptions bbto;
-    bbto.block_size = 16384;
-    bbto.no_block_cache = true;
-    options.table_factory.reset(new BlockBasedTableFactory(bbto));
-    DestroyAndReopen(options);
-
-    for (int i = 0; i < kNumL0Files; ++i) {
-      for (int j = 0; j <= kNumKeysPerFile; ++j) {
-        ASSERT_OK(Put(Key(j), DummyString(kBytesPerKey)));
-      }
-      dbfull()->TEST_WaitForFlushMemTable();
-      ASSERT_EQ(i + 1, NumTableFilesAtLevel(0));
-    }
-    dbfull()->TEST_WaitForCompact();
-    ASSERT_EQ(0, NumTableFilesAtLevel(0));
-
-    ASSERT_EQ(0, options.rate_limiter->GetTotalBytesThrough(Env::IO_HIGH));
-    // should be slightly above 512KB due to non-data blocks read. Arbitrarily
-    // chose 1MB as the upper bound on the total bytes read.
-    size_t rate_limited_bytes =
-        options.rate_limiter->GetTotalBytesThrough(Env::IO_LOW);
-    // Include the explict prefetch of the footer in direct I/O case.
-    size_t direct_io_extra = use_direct_io ? 512 * 1024 : 0;
-    ASSERT_GE(rate_limited_bytes,
-              static_cast<size_t>(kNumKeysPerFile * kBytesPerKey * kNumL0Files +
-                                  direct_io_extra));
-    ASSERT_LT(
-        rate_limited_bytes,
-        static_cast<size_t>(2 * kNumKeysPerFile * kBytesPerKey * kNumL0Files +
-                            direct_io_extra));
-
-    Iterator* iter = db_->NewIterator(ReadOptions());
-    for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
-      ASSERT_EQ(iter->value().ToString(), DummyString(kBytesPerKey));
-    }
-    delete iter;
-    // bytes read for user iterator shouldn't count against the rate limit.
-    ASSERT_EQ(rate_limited_bytes,
-              static_cast<size_t>(
-                  options.rate_limiter->GetTotalBytesThrough(Env::IO_LOW)));
-  }
-}
-#endif  // ROCKSDB_LITE
-
-// Make sure DB can be reopen with reduced number of levels, given no file
-// is on levels higher than the new num_levels.
-TEST_F(DBTest2, ReduceLevel) {
-  Options options;
-  options.disable_auto_compactions = true;
-  options.num_levels = 7;
-  Reopen(options);
-  Put("foo", "bar");
-  Flush();
-  MoveFilesToLevel(6);
-#ifndef ROCKSDB_LITE
-  ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel());
-#endif  // !ROCKSDB_LITE
-  CompactRangeOptions compact_options;
-  compact_options.change_level = true;
-  compact_options.target_level = 1;
-  dbfull()->CompactRange(compact_options, nullptr, nullptr);
-#ifndef ROCKSDB_LITE
-  ASSERT_EQ("0,1", FilesPerLevel());
-#endif  // !ROCKSDB_LITE
-  options.num_levels = 3;
-  Reopen(options);
-#ifndef ROCKSDB_LITE
-  ASSERT_EQ("0,1", FilesPerLevel());
-#endif  // !ROCKSDB_LITE
-}
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/db_test_util.cc

@@ -9,7 +9,6 @@
 
 #include "db/db_test_util.h"
 #include "db/forward_iterator.h"
-#include "rocksdb/env_encryption.h"
 
 namespace rocksdb {
 
@@ -42,24 +41,10 @@ SpecialEnv::SpecialEnv(Env* base)
   non_writable_count_ = 0;
   table_write_callback_ = nullptr;
 }
-#ifndef ROCKSDB_LITE
-ROT13BlockCipher rot13Cipher_(16);
-#endif  // ROCKSDB_LITE
 
 DBTestBase::DBTestBase(const std::string path)
     : mem_env_(!getenv("MEM_ENV") ? nullptr : new MockEnv(Env::Default())),
-#ifndef ROCKSDB_LITE
-      encrypted_env_(
-          !getenv("ENCRYPTED_ENV")
-              ? nullptr
-              : NewEncryptedEnv(mem_env_ ? mem_env_ : Env::Default(),
-                                new CTREncryptionProvider(rot13Cipher_))),
-#else
-      encrypted_env_(nullptr),
-#endif  // ROCKSDB_LITE
-      env_(new SpecialEnv(encrypted_env_
-                              ? encrypted_env_
-                              : (mem_env_ ? mem_env_ : Env::Default()))),
+      env_(new SpecialEnv(mem_env_ ? mem_env_ : Env::Default())),
       option_config_(kDefault) {
   env_->SetBackgroundThreads(1, Env::LOW);
   env_->SetBackgroundThreads(1, Env::HIGH);
@@ -296,7 +281,6 @@ Options DBTestBase::GetOptions(
       "NewWritableFile:O_DIRECT");
 #endif
 
-  bool can_allow_mmap = IsMemoryMappedAccessSupported();
   switch (option_config) {
 #ifndef ROCKSDB_LITE
     case kHashSkipList:
@@ -307,14 +291,14 @@ Options DBTestBase::GetOptions(
     case kPlainTableFirstBytePrefix:
       options.table_factory.reset(new PlainTableFactory());
       options.prefix_extractor.reset(NewFixedPrefixTransform(1));
-      options.allow_mmap_reads = can_allow_mmap;
+      options.allow_mmap_reads = true;
       options.max_sequential_skip_in_iterations = 999999;
       set_block_based_table_factory = false;
       break;
     case kPlainTableCappedPrefix:
       options.table_factory.reset(new PlainTableFactory());
       options.prefix_extractor.reset(NewCappedPrefixTransform(8));
-      options.allow_mmap_reads = can_allow_mmap;
+      options.allow_mmap_reads = true;
       options.max_sequential_skip_in_iterations = 999999;
       set_block_based_table_factory = false;
       break;
@@ -328,7 +312,7 @@ Options DBTestBase::GetOptions(
     case kPlainTableAllBytesPrefix:
       options.table_factory.reset(new PlainTableFactory());
       options.prefix_extractor.reset(NewNoopTransform());
-      options.allow_mmap_reads = can_allow_mmap;
+      options.allow_mmap_reads = true;
       options.max_sequential_skip_in_iterations = 999999;
       set_block_based_table_factory = false;
       break;
@@ -380,7 +364,7 @@ Options DBTestBase::GetOptions(
       options.wal_dir = alternative_wal_dir_;
       // mmap reads should be orthogonal to WalDir setting, so we piggyback to
       // this option config to test mmap reads as well
-      options.allow_mmap_reads = can_allow_mmap;
+      options.allow_mmap_reads = true;
       break;
     case kManifestFileSize:
       options.max_manifest_file_size = 50;  // 50 bytes
@@ -400,7 +384,7 @@ Options DBTestBase::GetOptions(
       options.num_levels = 8;
       break;
     case kCompressedBlockCache:
-      options.allow_mmap_writes = can_allow_mmap;
+      options.allow_mmap_writes = true;
       table_options.block_cache_compressed = NewLRUCache(8 * 1024 * 1024);
       break;
     case kInfiniteMaxOpenFiles:
@@ -485,12 +469,6 @@ Options DBTestBase::GetOptions(
       options.enable_pipelined_write = true;
       break;
     }
-    case kConcurrentWALWrites: {
-      // This options optimize 2PC commit path
-      options.concurrent_prepare = true;
-      options.manual_wal_flush = true;
-      break;
-    }
 
     default:
       break;
@@ -614,10 +592,6 @@ bool DBTestBase::IsDirectIOSupported() {
   return s.ok();
 }
 
-bool DBTestBase::IsMemoryMappedAccessSupported() const {
-  return (!encrypted_env_);
-}
-
 Status DBTestBase::Flush(int cf) {
   if (cf == 0) {
     return db_->Flush(FlushOptions());
@@ -697,13 +671,6 @@ std::string DBTestBase::Get(int cf, const std::string& k,
   return result;
 }
 
-Status DBTestBase::Get(const std::string& k, PinnableSlice* v) {
-  ReadOptions options;
-  options.verify_checksums = true;
-  Status s = dbfull()->Get(options, dbfull()->DefaultColumnFamily(), k, v);
-  return s;
-}
-
 uint64_t DBTestBase::GetNumSnapshots() {
   uint64_t int_num;
   EXPECT_TRUE(dbfull()->GetIntProperty("rocksdb.num-snapshots", &int_num));

db/db_test_util.h

@@ -247,13 +247,6 @@ class SpecialEnv : public EnvWrapper {
         }
       }
       Status Truncate(uint64_t size) override { return base_->Truncate(size); }
-      Status RangeSync(uint64_t offset, uint64_t nbytes) override {
-        Status s = base_->RangeSync(offset, nbytes);
-#if !(defined NDEBUG) || !defined(OS_WIN)
-        TEST_SYNC_POINT_CALLBACK("SpecialEnv::SStableFile::RangeSync", &s);
-#endif  // !(defined NDEBUG) || !defined(OS_WIN)
-        return s;
-      }
       Status Close() override {
 // SyncPoint is not supported in Released Windows Mode.
 #if !(defined NDEBUG) || !defined(OS_WIN)
@@ -263,11 +256,7 @@ class SpecialEnv : public EnvWrapper {
         TEST_SYNC_POINT_CALLBACK("DBTestWritableFile.GetPreallocationStatus",
                                  &preallocation_size);
 #endif  // !(defined NDEBUG) || !defined(OS_WIN)
-        Status s = base_->Close();
-#if !(defined NDEBUG) || !defined(OS_WIN)
-        TEST_SYNC_POINT_CALLBACK("SpecialEnv::SStableFile::Close", &s);
-#endif  // !(defined NDEBUG) || !defined(OS_WIN)
-        return s;
+        return base_->Close();
       }
       Status Flush() override { return base_->Flush(); }
       Status Sync() override {
@@ -275,11 +264,7 @@ class SpecialEnv : public EnvWrapper {
         while (env_->delay_sstable_sync_.load(std::memory_order_acquire)) {
           env_->SleepForMicroseconds(100000);
         }
-        Status s = base_->Sync();
-#if !(defined NDEBUG) || !defined(OS_WIN)
-        TEST_SYNC_POINT_CALLBACK("SpecialEnv::SStableFile::Sync", &s);
-#endif  // !(defined NDEBUG) || !defined(OS_WIN)
-        return s;
+        return base_->Sync();
       }
       void SetIOPriority(Env::IOPriority pri) override {
         base_->SetIOPriority(pri);
@@ -572,37 +557,6 @@ class SpecialEnv : public EnvWrapper {
   std::atomic<bool> is_wal_sync_thread_safe_{true};
 };
 
-class MockTimeEnv : public EnvWrapper {
- public:
-  explicit MockTimeEnv(Env* base) : EnvWrapper(base) {}
-
-  virtual Status GetCurrentTime(int64_t* time) override {
-    assert(time != nullptr);
-    assert(current_time_ <=
-           static_cast<uint64_t>(std::numeric_limits<int64_t>::max()));
-    *time = static_cast<int64_t>(current_time_);
-    return Status::OK();
-  }
-
-  virtual uint64_t NowMicros() override {
-    assert(current_time_ <= std::numeric_limits<uint64_t>::max() / 1000000);
-    return current_time_ * 1000000;
-  }
-
-  virtual uint64_t NowNanos() override {
-    assert(current_time_ <= std::numeric_limits<uint64_t>::max() / 1000000000);
-    return current_time_ * 1000000000;
-  }
-
-  void set_current_time(uint64_t time) {
-    assert(time >= current_time_);
-    current_time_ = time;
-  }
-
- private:
-  uint64_t current_time_ = 0;
-};
-
 #ifndef ROCKSDB_LITE
 class OnFileDeletionListener : public EventListener {
  public:
@@ -687,14 +641,13 @@ class DBTestBase : public testing::Test {
     kRecycleLogFiles = 28,
     kConcurrentSkipList = 29,
     kPipelinedWrite = 30,
-    kConcurrentWALWrites = 31,
-    kEnd = 32,
-    kDirectIO = 33,
-    kLevelSubcompactions = 34,
-    kUniversalSubcompactions = 35,
-    kBlockBasedTableWithIndexRestartInterval = 36,
-    kBlockBasedTableWithPartitionedIndex = 37,
-    kPartitionedFilterWithNewTableReaderForCompactions = 38,
+    kEnd = 31,
+    kDirectIO = 32,
+    kLevelSubcompactions = 33,
+    kUniversalSubcompactions = 34,
+    kBlockBasedTableWithIndexRestartInterval = 35,
+    kBlockBasedTableWithPartitionedIndex = 36,
+    kPartitionedFilterWithNewTableReaderForCompactions = 37,
   };
 
  public:
@@ -702,7 +655,6 @@ class DBTestBase : public testing::Test {
   std::string alternative_wal_dir_;
   std::string alternative_db_log_dir_;
   MockEnv* mem_env_;
-  Env* encrypted_env_;
   SpecialEnv* env_;
   DB* db_;
   std::vector<ColumnFamilyHandle*> handles_;
@@ -806,8 +758,6 @@ class DBTestBase : public testing::Test {
 
   bool IsDirectIOSupported();
 
-  bool IsMemoryMappedAccessSupported() const;
-
   Status Flush(int cf = 0);
 
   Status Put(const Slice& k, const Slice& v, WriteOptions wo = WriteOptions());
@@ -834,8 +784,6 @@ class DBTestBase : public testing::Test {
   std::string Get(int cf, const std::string& k,
                   const Snapshot* snapshot = nullptr);
 
-  Status Get(const std::string& k, PinnableSlice* v);
-
   uint64_t GetNumSnapshots();
 
   uint64_t GetTimeOldestSnapshots();

db/db_universal_compaction_test.cc

@@ -696,12 +696,17 @@ TEST_P(DBTestUniversalCompactionParallel, PickByFileNumberBug) {
     num_keys -= 100;
   }
 
-  // Hold the 1st compaction from finishing
+  // Wait for the 2nd background compaction process to start
+  TEST_SYNC_POINT("DBTestUniversalCompactionParallel::PickByFileNumberBug:0");
+  TEST_SYNC_POINT("DBTestUniversalCompactionParallel::PickByFileNumberBug:1");
+
+  // Hold the 1st and 2nd compaction from finishing
   TEST_SYNC_POINT("DBTestUniversalCompactionParallel::PickByFileNumberBug:2");
   dbfull()->TEST_WaitForCompact();
 
-  // There should only be one picked compaction as the score drops below one
-  // after the first one is picked.
+  // Although 2 compaction threads started, the second one did not compact
+  // anything because the number of files not being compacted is less than
+  // level0_file_num_compaction_trigger
   EXPECT_EQ(total_picked_compactions, 1);
   EXPECT_EQ(TotalTableFiles(), 4);
 
@@ -1365,140 +1370,6 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) {
   Destroy(options);
 }
 
-TEST_P(DBTestUniversalCompaction, FullCompactionInBottomPriThreadPool) {
-  const int kNumFilesTrigger = 3;
-  Env::Default()->SetBackgroundThreads(1, Env::Priority::BOTTOM);
-  for (bool allow_ingest_behind : {false, true}) {
-    Options options = CurrentOptions();
-    options.allow_ingest_behind = allow_ingest_behind;
-    options.compaction_style = kCompactionStyleUniversal;
-    options.num_levels = num_levels_;
-    options.write_buffer_size = 100 << 10;     // 100KB
-    options.target_file_size_base = 32 << 10;  // 32KB
-    options.level0_file_num_compaction_trigger = kNumFilesTrigger;
-    // Trigger compaction if size amplification exceeds 110%
-    options.compaction_options_universal.max_size_amplification_percent = 110;
-    DestroyAndReopen(options);
-
-    int num_bottom_pri_compactions = 0;
-    SyncPoint::GetInstance()->SetCallBack(
-        "DBImpl::BGWorkBottomCompaction",
-        [&](void* arg) { ++num_bottom_pri_compactions; });
-    SyncPoint::GetInstance()->EnableProcessing();
-
-    Random rnd(301);
-    for (int num = 0; num < kNumFilesTrigger; num++) {
-      ASSERT_EQ(NumSortedRuns(), num);
-      int key_idx = 0;
-      GenerateNewFile(&rnd, &key_idx);
-    }
-    dbfull()->TEST_WaitForCompact();
-
-    if (allow_ingest_behind || num_levels_ > 1) {
-      // allow_ingest_behind increases number of levels while sanitizing.
-      ASSERT_EQ(1, num_bottom_pri_compactions);
-    } else {
-      // for single-level universal, everything's bottom level so nothing should
-      // be executed in bottom-pri thread pool.
-      ASSERT_EQ(0, num_bottom_pri_compactions);
-    }
-    // Verify that size amplification did occur
-    ASSERT_EQ(NumSortedRuns(), 1);
-    rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-  }
-  Env::Default()->SetBackgroundThreads(0, Env::Priority::BOTTOM);
-}
-
-TEST_P(DBTestUniversalCompaction, ConcurrentBottomPriLowPriCompactions) {
-  if (num_levels_ == 1) {
-    // for single-level universal, everything's bottom level so nothing should
-    // be executed in bottom-pri thread pool.
-    return;
-  }
-  const int kNumFilesTrigger = 3;
-  Env::Default()->SetBackgroundThreads(1, Env::Priority::BOTTOM);
-  Options options = CurrentOptions();
-  options.compaction_style = kCompactionStyleUniversal;
-  options.num_levels = num_levels_;
-  options.write_buffer_size = 100 << 10;     // 100KB
-  options.target_file_size_base = 32 << 10;  // 32KB
-  options.level0_file_num_compaction_trigger = kNumFilesTrigger;
-  // Trigger compaction if size amplification exceeds 110%
-  options.compaction_options_universal.max_size_amplification_percent = 110;
-  DestroyAndReopen(options);
-
-  rocksdb::SyncPoint::GetInstance()->LoadDependency(
-      {// wait for the full compaction to be picked before adding files intended
-       // for the second one.
-       {"DBImpl::BackgroundCompaction:ForwardToBottomPriPool",
-        "DBTestUniversalCompaction:ConcurrentBottomPriLowPriCompactions:0"},
-       // the full (bottom-pri) compaction waits until a partial (low-pri)
-       // compaction has started to verify they can run in parallel.
-       {"DBImpl::BackgroundCompaction:NonTrivial",
-        "DBImpl::BGWorkBottomCompaction"}});
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  Random rnd(301);
-  for (int i = 0; i < 2; ++i) {
-    for (int num = 0; num < kNumFilesTrigger; num++) {
-      int key_idx = 0;
-      GenerateNewFile(&rnd, &key_idx, true /* no_wait */);
-      // use no_wait above because that one waits for flush and compaction. We
-      // don't want to wait for compaction because the full compaction is
-      // intentionally blocked while more files are flushed.
-      dbfull()->TEST_WaitForFlushMemTable();
-    }
-    if (i == 0) {
-      TEST_SYNC_POINT(
-          "DBTestUniversalCompaction:ConcurrentBottomPriLowPriCompactions:0");
-    }
-  }
-  dbfull()->TEST_WaitForCompact();
-
-  // First compaction should output to bottom level. Second should output to L0
-  // since older L0 files pending compaction prevent it from being placed lower.
-  ASSERT_EQ(NumSortedRuns(), 2);
-  ASSERT_GT(NumTableFilesAtLevel(0), 0);
-  ASSERT_GT(NumTableFilesAtLevel(num_levels_ - 1), 0);
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-  Env::Default()->SetBackgroundThreads(0, Env::Priority::BOTTOM);
-}
-
-TEST_P(DBTestUniversalCompaction, RecalculateScoreAfterPicking) {
-  // Regression test for extra compactions scheduled. Once enough compactions
-  // have been scheduled to bring the score below one, we should stop
-  // scheduling more; otherwise, other CFs/DBs may be delayed unnecessarily.
-  const int kNumFilesTrigger = 8;
-  Options options = CurrentOptions();
-  options.compaction_options_universal.max_merge_width = kNumFilesTrigger / 2;
-  options.compaction_options_universal.max_size_amplification_percent =
-      static_cast<unsigned int>(-1);
-  options.compaction_style = kCompactionStyleUniversal;
-  options.level0_file_num_compaction_trigger = kNumFilesTrigger;
-  options.num_levels = num_levels_;
-  options.write_buffer_size = 100 << 10;  // 100KB
-  Reopen(options);
-
-  std::atomic<int> num_compactions_attempted(0);
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::BackgroundCompaction:Start", [&](void* arg) {
-        ++num_compactions_attempted;
-      });
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-
-  Random rnd(301);
-  for (int num = 0; num < kNumFilesTrigger; num++) {
-    ASSERT_EQ(NumSortedRuns(), num);
-    int key_idx = 0;
-    GenerateNewFile(&rnd, &key_idx);
-  }
-  dbfull()->TEST_WaitForCompact();
-  // Compacting the first four files was enough to bring the score below one so
-  // there's no need to schedule any more compactions.
-  ASSERT_EQ(1, num_compactions_attempted);
-  ASSERT_EQ(NumSortedRuns(), 5);
-}
-
 INSTANTIATE_TEST_CASE_P(UniversalCompactionNumLevels, DBTestUniversalCompaction,
                         ::testing::Combine(::testing::Values(1, 3, 5),
                                            ::testing::Bool()));

db/db_wal_test.cc

@@ -120,11 +120,9 @@ TEST_F(DBWALTest, SyncWALNotWaitWrite) {
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 
   rocksdb::port::Thread thread([&]() { ASSERT_OK(Put("foo2", "bar2")); });
-  // Moving this to SyncWAL before the actual fsync
-  // TEST_SYNC_POINT("DBWALTest::SyncWALNotWaitWrite:1");
+  TEST_SYNC_POINT("DBWALTest::SyncWALNotWaitWrite:1");
   ASSERT_OK(db_->SyncWAL());
-  // Moving this to SyncWAL after actual fsync
-  // TEST_SYNC_POINT("DBWALTest::SyncWALNotWaitWrite:2");
+  TEST_SYNC_POINT("DBWALTest::SyncWALNotWaitWrite:2");
 
   thread.join();
 
@@ -660,7 +658,6 @@ TEST_F(DBWALTest, PartOfWritesWithWALDisabled) {
   ASSERT_OK(Flush(0));
   ASSERT_OK(Put(0, "key", "v5", wal_on));  // seq id 5
   ASSERT_EQ("v5", Get(0, "key"));
-  dbfull()->FlushWAL(false);
   // Simulate a crash.
   fault_env->SetFilesystemActive(false);
   Close();
@@ -730,7 +727,6 @@ class RecoveryTestHelper {
         batch.Put(key, value);
         WriteBatchInternal::SetSequence(&batch, seq);
         current_log_writer->AddRecord(WriteBatchInternal::Contents(&batch));
-        versions->SetLastToBeWrittenSequence(seq);
         versions->SetLastSequence(seq);
       }
     }
@@ -1115,7 +1111,6 @@ TEST_F(DBWALTest, RecoverWithoutFlushMultipleCF) {
   ASSERT_EQ(3, countWalFiles());
   Flush(1);
   ASSERT_OK(Put(2, "key7", kLargeValue));
-  dbfull()->FlushWAL(false);
   ASSERT_EQ(4, countWalFiles());
 
   // Reopen twice and validate.

db/db_write_test.cc

@@ -3,17 +3,12 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
 
-#include <atomic>
 #include <memory>
 #include <thread>
 #include <vector>
 #include "db/db_test_util.h"
 #include "db/write_batch_internal.h"
-#include "db/write_thread.h"
-#include "port/port.h"
 #include "port/stack_trace.h"
-#include "util/fault_injection_test_env.h"
-#include "util/string_util.h"
 #include "util/sync_point.h"
 
 namespace rocksdb {
@@ -23,9 +18,7 @@ class DBWriteTest : public DBTestBase, public testing::WithParamInterface<int> {
  public:
   DBWriteTest() : DBTestBase("/db_write_test") {}
 
-  Options GetOptions() { return DBTestBase::GetOptions(GetParam()); }
-
-  void Open() { DBTestBase::Reopen(GetOptions()); }
+  void Open() { DBTestBase::Reopen(GetOptions(GetParam())); }
 };
 
 // Sequence number should be return through input write batch.
@@ -74,50 +67,8 @@ TEST_P(DBWriteTest, ReturnSeuqneceNumberMultiThreaded) {
   }
 }
 
-TEST_P(DBWriteTest, IOErrorOnWALWritePropagateToWriteThreadFollower) {
-  constexpr int kNumThreads = 5;
-  std::unique_ptr<FaultInjectionTestEnv> mock_env(
-      new FaultInjectionTestEnv(Env::Default()));
-  Options options = GetOptions();
-  options.env = mock_env.get();
-  Reopen(options);
-  std::atomic<int> ready_count{0};
-  std::atomic<int> leader_count{0};
-  std::vector<port::Thread> threads;
-  mock_env->SetFilesystemActive(false);
-  // Wait until all threads linked to write threads, to make sure
-  // all threads join the same batch group.
-  SyncPoint::GetInstance()->SetCallBack(
-      "WriteThread::JoinBatchGroup:Wait", [&](void* arg) {
-        ready_count++;
-        auto* w = reinterpret_cast<WriteThread::Writer*>(arg);
-        if (w->state == WriteThread::STATE_GROUP_LEADER) {
-          leader_count++;
-          while (ready_count < kNumThreads) {
-            // busy waiting
-          }
-        }
-      });
-  SyncPoint::GetInstance()->EnableProcessing();
-  for (int i = 0; i < kNumThreads; i++) {
-    threads.push_back(port::Thread(
-        [&](int index) {
-          // All threads should fail.
-          ASSERT_FALSE(Put("key" + ToString(index), "value").ok());
-        },
-        i));
-  }
-  for (int i = 0; i < kNumThreads; i++) {
-    threads[i].join();
-  }
-  ASSERT_EQ(1, leader_count);
-  // Close before mock_env destruct.
-  Close();
-}
-
 INSTANTIATE_TEST_CASE_P(DBWriteTestInstance, DBWriteTest,
                         testing::Values(DBTestBase::kDefault,
-                                        DBTestBase::kConcurrentWALWrites,
                                         DBTestBase::kPipelinedWrite));
 
 }  // namespace rocksdb

db/dbformat.cc

@@ -27,7 +27,7 @@ namespace rocksdb {
 // and the value type is embedded as the low 8 bits in the sequence
 // number in internal keys, we need to use the highest-numbered
 // ValueType, not the lowest).
-const ValueType kValueTypeForSeek = kTypeBlobIndex;
+const ValueType kValueTypeForSeek = kTypeSingleDeletion;
 const ValueType kValueTypeForSeekForPrev = kTypeDeletion;
 
 uint64_t PackSequenceAndType(uint64_t seq, ValueType t) {

db/dbformat.h

@@ -47,8 +47,6 @@ enum ValueType : unsigned char {
   kTypeNoop = 0xD,                        // WAL only.
   kTypeColumnFamilyRangeDeletion = 0xE,   // WAL only.
   kTypeRangeDeletion = 0xF,               // meta block
-  kTypeColumnFamilyBlobIndex = 0x10,      // Blob DB only
-  kTypeBlobIndex = 0x11,                  // Blob DB only
   kMaxValue = 0x7F                        // Not used for storing records.
 };
 
@@ -59,7 +57,7 @@ extern const ValueType kValueTypeForSeekForPrev;
 // Checks whether a type is an inline value type
 // (i.e. a type used in memtable skiplist and sst file datablock).
 inline bool IsValueType(ValueType t) {
-  return t <= kTypeMerge || t == kTypeSingleDeletion || t == kTypeBlobIndex;
+  return t <= kTypeMerge || t == kTypeSingleDeletion;
 }
 
 // Checks whether a type is from user operation
@@ -80,18 +78,10 @@ struct ParsedInternalKey {
   SequenceNumber sequence;
   ValueType type;
 
-  ParsedInternalKey()
-      : sequence(kMaxSequenceNumber)  // Make code analyzer happy
-  {}  // Intentionally left uninitialized (for speed)
+  ParsedInternalKey() { }  // Intentionally left uninitialized (for speed)
   ParsedInternalKey(const Slice& u, const SequenceNumber& seq, ValueType t)
       : user_key(u), sequence(seq), type(t) { }
   std::string DebugString(bool hex = false) const;
-
-  void clear() {
-    user_key.clear();
-    sequence = 0;
-    type = kTypeDeletion;
-  }
 };
 
 // Return the length of the encoding of "key".
@@ -159,9 +149,6 @@ class InternalKeyComparator : public Comparator {
 
   int Compare(const InternalKey& a, const InternalKey& b) const;
   int Compare(const ParsedInternalKey& a, const ParsedInternalKey& b) const;
-  virtual const Comparator* GetRootComparator() const override {
-    return user_comparator_->GetRootComparator();
-  }
 };
 
 // Modules in this directory should keep internal keys wrapped inside

db/event_helpers.cc

@@ -37,24 +37,6 @@ void EventHelpers::NotifyTableFileCreationStarted(
 }
 #endif  // !ROCKSDB_LITE
 
-void EventHelpers::NotifyOnBackgroundError(
-    const std::vector<std::shared_ptr<EventListener>>& listeners,
-    BackgroundErrorReason reason, Status* bg_error,
-    InstrumentedMutex* db_mutex) {
-#ifndef ROCKSDB_LITE
-  if (listeners.size() == 0U) {
-    return;
-  }
-  db_mutex->AssertHeld();
-  // release lock while notifying events
-  db_mutex->Unlock();
-  for (auto& listener : listeners) {
-    listener->OnBackgroundError(reason, bg_error);
-  }
-  db_mutex->Lock();
-#endif  // ROCKSDB_LITE
-}
-
 void EventHelpers::LogAndNotifyTableFileCreationFinished(
     EventLogger* event_logger,
     const std::vector<std::shared_ptr<EventListener>>& listeners,

db/event_helpers.h

@@ -25,10 +25,6 @@ class EventHelpers {
       const std::string& db_name, const std::string& cf_name,
       const std::string& file_path, int job_id, TableFileCreationReason reason);
 #endif  // !ROCKSDB_LITE
-  static void NotifyOnBackgroundError(
-      const std::vector<std::shared_ptr<EventListener>>& listeners,
-      BackgroundErrorReason reason, Status* bg_error,
-      InstrumentedMutex* db_mutex);
   static void LogAndNotifyTableFileCreationFinished(
       EventLogger* event_logger,
       const std::vector<std::shared_ptr<EventListener>>& listeners,

db/experimental.cc

@@ -14,18 +14,20 @@ namespace experimental {
 
 Status SuggestCompactRange(DB* db, ColumnFamilyHandle* column_family,
                            const Slice* begin, const Slice* end) {
-  if (db == nullptr) {
-    return Status::InvalidArgument("DB is empty");
+  auto dbimpl = dynamic_cast<DBImpl*>(db);
+  if (dbimpl == nullptr) {
+    return Status::InvalidArgument("Didn't recognize DB object");
   }
 
-  return db->SuggestCompactRange(column_family, begin, end);
+  return dbimpl->SuggestCompactRange(column_family, begin, end);
 }
 
 Status PromoteL0(DB* db, ColumnFamilyHandle* column_family, int target_level) {
-  if (db == nullptr) {
+  auto dbimpl = dynamic_cast<DBImpl*>(db);
+  if (dbimpl == nullptr) {
     return Status::InvalidArgument("Didn't recognize DB object");
   }
-  return db->PromoteL0(column_family, target_level);
+  return dbimpl->PromoteL0(column_family, target_level);
 }
 
 #else  // ROCKSDB_LITE

db/external_sst_file_ingestion_job.cc

@@ -144,8 +144,6 @@ Status ExternalSstFileIngestionJob::NeedsFlush(bool* flush_needed) {
   return status;
 }
 
-// REQUIRES: we have become the only writer by entering both write_thread_ and
-// nonmem_write_thread_
 Status ExternalSstFileIngestionJob::Run() {
   Status status;
 #ifndef NDEBUG
@@ -164,8 +162,6 @@ Status ExternalSstFileIngestionJob::Run() {
     // if the dont overlap with any ranges since we have snapshots
     force_global_seqno = true;
   }
-  // It is safe to use this instead of LastToBeWrittenSequence since we are
-  // the only active writer, and hence they are equal
   const SequenceNumber last_seqno = versions_->LastSequence();
   SuperVersion* super_version = cfd_->GetSuperVersion();
   edit_.SetColumnFamily(cfd_->GetID());
@@ -199,7 +195,6 @@ Status ExternalSstFileIngestionJob::Run() {
   }
 
   if (consumed_seqno) {
-    versions_->SetLastToBeWrittenSequence(last_seqno + 1);
     versions_->SetLastSequence(last_seqno + 1);
   }
 
@@ -284,8 +279,7 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo(
   if (!status.ok()) {
     return status;
   }
-  sst_file_reader.reset(new RandomAccessFileReader(std::move(sst_file),
-                                                   external_file));
+  sst_file_reader.reset(new RandomAccessFileReader(std::move(sst_file)));
 
   status = cfd_->ioptions()->table_factory->NewTableReader(
       TableReaderOptions(*cfd_->ioptions(), env_options_,

db/fault_injection_test.cc

@@ -410,7 +410,6 @@ TEST_P(FaultInjectionTest, WriteOptionSyncTest) {
   write_options.sync = true;
   ASSERT_OK(
       db_->Put(write_options, Key(2, &key_space), Value(2, &value_space)));
-  db_->FlushWAL(false);
 
   env_->SetFilesystemActive(false);
   NoWriteTestReopenWithFault(kResetDropAndDeleteUnsynced);
@@ -495,7 +494,7 @@ TEST_P(FaultInjectionTest, ManualLogSyncTest) {
   ASSERT_OK(db_->Flush(flush_options));
   ASSERT_OK(
       db_->Put(write_options, Key(2, &key_space), Value(2, &value_space)));
-  ASSERT_OK(db_->FlushWAL(true));
+  ASSERT_OK(db_->SyncWAL());
 
   env_->SetFilesystemActive(false);
   NoWriteTestReopenWithFault(kResetDropAndDeleteUnsynced);

db/flush_job.cc

@@ -296,13 +296,6 @@ Status FlushJob::WriteLevel0Table() {
                                &output_compression_);
       EnvOptions optimized_env_options =
           db_options_.env->OptimizeForCompactionTableWrite(env_options_, db_options_);
-
-      int64_t _current_time = 0;
-      db_options_.env->GetCurrentTime(&_current_time);  // ignore error
-      const uint64_t current_time = static_cast<uint64_t>(_current_time);
-
-      uint64_t oldest_key_time = mems_.front()->ApproximateOldestKeyTime();
-
       s = BuildTable(
           dbname_, db_options_.env, *cfd_->ioptions(), mutable_cf_options_,
           optimized_env_options, cfd_->table_cache(), iter.get(),
@@ -313,8 +306,7 @@ Status FlushJob::WriteLevel0Table() {
           cfd_->ioptions()->compression_opts,
           mutable_cf_options_.paranoid_file_checks, cfd_->internal_stats(),
           TableFileCreationReason::kFlush, event_logger_, job_context_->job_id,
-          Env::IO_HIGH, &table_properties_, 0 /* level */, current_time,
-          oldest_key_time);
+          Env::IO_HIGH, &table_properties_, 0 /* level */);
       LogFlush(db_options_.info_log);
     }
     ROCKS_LOG_INFO(db_options_.info_log,

db/flush_job.h

@@ -68,7 +68,7 @@ class FlushJob {
   ~FlushJob();
 
   // Require db_mutex held.
-  // Once PickMemTable() is called, either Run() or Cancel() has to be called.
+  // Once PickMemTable() is called, either Run() or Cancel() has to be call.
   void PickMemTable();
   Status Run(FileMetaData* file_meta = nullptr);
   void Cancel();

db/internal_stats.cc

@@ -13,9 +13,8 @@
 #endif
 
 #include <inttypes.h>
-#include <algorithm>
-#include <limits>
 #include <string>
+#include <algorithm>
 #include <utility>
 #include <vector>
 #include "db/column_family.h"
@@ -244,7 +243,6 @@ static const std::string num_running_flushes = "num-running-flushes";
 static const std::string actual_delayed_write_rate =
     "actual-delayed-write-rate";
 static const std::string is_write_stopped = "is-write-stopped";
-static const std::string estimate_oldest_key_time = "estimate-oldest-key-time";
 
 const std::string DB::Properties::kNumFilesAtLevelPrefix =
                       rocksdb_prefix + num_files_at_level_prefix;
@@ -318,8 +316,6 @@ const std::string DB::Properties::kActualDelayedWriteRate =
     rocksdb_prefix + actual_delayed_write_rate;
 const std::string DB::Properties::kIsWriteStopped =
     rocksdb_prefix + is_write_stopped;
-const std::string DB::Properties::kEstimateOldestKeyTime =
-    rocksdb_prefix + estimate_oldest_key_time;
 
 const std::unordered_map<std::string, DBPropertyInfo>
     InternalStats::ppt_name_to_info = {
@@ -418,9 +414,6 @@ const std::unordered_map<std::string, DBPropertyInfo>
           nullptr}},
         {DB::Properties::kIsWriteStopped,
          {false, nullptr, &InternalStats::HandleIsWriteStopped, nullptr}},
-        {DB::Properties::kEstimateOldestKeyTime,
-         {false, nullptr, &InternalStats::HandleEstimateOldestKeyTime,
-          nullptr}},
 };
 
 const DBPropertyInfo* GetPropertyInfo(const Slice& property) {
@@ -549,7 +542,7 @@ bool InternalStats::HandleDBStats(std::string* value, Slice suffix) {
 
 bool InternalStats::HandleSsTables(std::string* value, Slice suffix) {
   auto* current = cfd_->current();
-  *value = current->DebugString(true, true);
+  *value = current->DebugString();
   return true;
 }
 
@@ -782,35 +775,6 @@ bool InternalStats::HandleIsWriteStopped(uint64_t* value, DBImpl* db,
   return true;
 }
 
-bool InternalStats::HandleEstimateOldestKeyTime(uint64_t* value, DBImpl* /*db*/,
-                                                Version* /*version*/) {
-  // TODO(yiwu): The property is currently available for fifo compaction
-  // with allow_compaction = false. This is because we don't propagate
-  // oldest_key_time on compaction.
-  if (cfd_->ioptions()->compaction_style != kCompactionStyleFIFO ||
-      cfd_->ioptions()->compaction_options_fifo.allow_compaction) {
-    return false;
-  }
-
-  TablePropertiesCollection collection;
-  auto s = cfd_->current()->GetPropertiesOfAllTables(&collection);
-  if (!s.ok()) {
-    return false;
-  }
-  *value = std::numeric_limits<uint64_t>::max();
-  for (auto& p : collection) {
-    *value = std::min(*value, p.second->oldest_key_time);
-    if (*value == 0) {
-      break;
-    }
-  }
-  if (*value > 0) {
-    *value = std::min({cfd_->mem()->ApproximateOldestKeyTime(),
-                       cfd_->imm()->ApproximateOldestKeyTime(), *value});
-  }
-  return *value > 0 && *value < std::numeric_limits<uint64_t>::max();
-}
-
 void InternalStats::DumpDBStats(std::string* value) {
   char buf[1000];
   // DB-level stats, only available from default column family

db/internal_stats.h

@@ -270,15 +270,10 @@ class InternalStats {
     ++cf_stats_count_[type];
   }
 
-  void AddDBStats(InternalDBStatsType type, uint64_t value,
-                  bool concurrent = false) {
+  void AddDBStats(InternalDBStatsType type, uint64_t value) {
     auto& v = db_stats_[type];
-    if (concurrent) {
-      v.fetch_add(value, std::memory_order_relaxed);
-    } else {
-      v.store(v.load(std::memory_order_relaxed) + value,
-              std::memory_order_relaxed);
-    }
+    v.store(v.load(std::memory_order_relaxed) + value,
+            std::memory_order_relaxed);
   }
 
   uint64_t GetDBStats(InternalDBStatsType type) {
@@ -475,8 +470,6 @@ class InternalStats {
   bool HandleActualDelayedWriteRate(uint64_t* value, DBImpl* db,
                                     Version* version);
   bool HandleIsWriteStopped(uint64_t* value, DBImpl* db, Version* version);
-  bool HandleEstimateOldestKeyTime(uint64_t* value, DBImpl* db,
-                                   Version* version);
 
   // Total number of background errors encountered. Every time a flush task
   // or compaction task fails, this counter is incremented. The failure can
@@ -555,8 +548,7 @@ class InternalStats {
 
   void AddCFStats(InternalCFStatsType type, uint64_t value) {}
 
-  void AddDBStats(InternalDBStatsType type, uint64_t value,
-                  bool concurrent = false) {}
+  void AddDBStats(InternalDBStatsType type, uint64_t value) {}
 
   HistogramImpl* GetFileReadHist(int level) { return nullptr; }
 

db/listener_test.cc

@@ -112,7 +112,6 @@ TEST_F(EventListenerTest, OnSingleDBCompactionTest) {
   const int kNumL0Files = 4;
 
   Options options;
-  options.env = CurrentOptions().env;
   options.create_if_missing = true;
   options.write_buffer_size = kEntrySize * kEntriesPerBuffer;
   options.compaction_style = kCompactionStyleLevel;
@@ -232,7 +231,6 @@ class TestFlushListener : public EventListener {
 
 TEST_F(EventListenerTest, OnSingleDBFlushTest) {
   Options options;
-  options.env = CurrentOptions().env;
   options.write_buffer_size = k110KB;
 #ifdef ROCKSDB_USING_THREAD_STATUS
   options.enable_thread_tracking = true;
@@ -269,7 +267,6 @@ TEST_F(EventListenerTest, OnSingleDBFlushTest) {
 
 TEST_F(EventListenerTest, MultiCF) {
   Options options;
-  options.env = CurrentOptions().env;
   options.write_buffer_size = k110KB;
 #ifdef ROCKSDB_USING_THREAD_STATUS
   options.enable_thread_tracking = true;
@@ -305,7 +302,6 @@ TEST_F(EventListenerTest, MultiCF) {
 
 TEST_F(EventListenerTest, MultiDBMultiListeners) {
   Options options;
-  options.env = CurrentOptions().env;
 #ifdef ROCKSDB_USING_THREAD_STATUS
   options.enable_thread_tracking = true;
 #endif  // ROCKSDB_USING_THREAD_STATUS
@@ -388,7 +384,6 @@ TEST_F(EventListenerTest, MultiDBMultiListeners) {
 
 TEST_F(EventListenerTest, DisableBGCompaction) {
   Options options;
-  options.env = CurrentOptions().env;
 #ifdef ROCKSDB_USING_THREAD_STATUS
   options.enable_thread_tracking = true;
 #endif  // ROCKSDB_USING_THREAD_STATUS
@@ -436,7 +431,6 @@ class TestCompactionReasonListener : public EventListener {
 
 TEST_F(EventListenerTest, CompactionReasonLevel) {
   Options options;
-  options.env = CurrentOptions().env;
   options.create_if_missing = true;
   options.memtable_factory.reset(
       new SpecialSkipListFactory(DBTestBase::kNumKeysByGenerateNewRandomFile));
@@ -502,7 +496,6 @@ TEST_F(EventListenerTest, CompactionReasonLevel) {
 
 TEST_F(EventListenerTest, CompactionReasonUniversal) {
   Options options;
-  options.env = CurrentOptions().env;
   options.create_if_missing = true;
   options.memtable_factory.reset(
       new SpecialSkipListFactory(DBTestBase::kNumKeysByGenerateNewRandomFile));
@@ -564,7 +557,6 @@ TEST_F(EventListenerTest, CompactionReasonUniversal) {
 
 TEST_F(EventListenerTest, CompactionReasonFIFO) {
   Options options;
-  options.env = CurrentOptions().env;
   options.create_if_missing = true;
   options.memtable_factory.reset(
       new SpecialSkipListFactory(DBTestBase::kNumKeysByGenerateNewRandomFile));
@@ -787,7 +779,6 @@ TEST_F(EventListenerTest, ColumnFamilyHandleDeletionStartedListenerTest) {
   auto listener =
       std::make_shared<ColumnFamilyHandleDeletionStartedListener>(cfs);
   Options options;
-  options.env = CurrentOptions().env;
   options.create_if_missing = true;
   options.listeners.push_back(listener);
   CreateAndReopenWithCF(cfs, options);
@@ -799,92 +790,6 @@ TEST_F(EventListenerTest, ColumnFamilyHandleDeletionStartedListenerTest) {
   ASSERT_EQ(listener->getCounter(), 3);
 }
 
-class BackgroundErrorListener : public EventListener {
- private:
-  SpecialEnv* env_;
-  int counter_;
-
- public:
-  BackgroundErrorListener(SpecialEnv* env) : env_(env), counter_(0) {}
-
-  void OnBackgroundError(BackgroundErrorReason reason, Status* bg_error) override {
-    if (counter_ == 0) {
-      // suppress the first error and disable write-dropping such that a retry
-      // can succeed.
-      *bg_error = Status::OK();
-      env_->drop_writes_.store(false, std::memory_order_release);
-      env_->no_slowdown_ = false;
-    }
-    ++counter_;
-  }
-
-  int counter() { return counter_; }
-};
-
-TEST_F(EventListenerTest, BackgroundErrorListenerFailedFlushTest) {
-  auto listener = std::make_shared<BackgroundErrorListener>(env_);
-  Options options;
-  options.create_if_missing = true;
-  options.env = env_;
-  options.listeners.push_back(listener);
-  options.memtable_factory.reset(new SpecialSkipListFactory(1));
-  options.paranoid_checks = true;
-  DestroyAndReopen(options);
-
-  // the usual TEST_WaitForFlushMemTable() doesn't work for failed flushes, so
-  // forge a custom one for the failed flush case.
-  rocksdb::SyncPoint::GetInstance()->LoadDependency(
-      {{"DBImpl::BGWorkFlush:done",
-        "EventListenerTest:BackgroundErrorListenerFailedFlushTest:1"}});
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-
-  env_->drop_writes_.store(true, std::memory_order_release);
-  env_->no_slowdown_ = true;
-
-  ASSERT_OK(Put("key0", "val"));
-  ASSERT_OK(Put("key1", "val"));
-  TEST_SYNC_POINT("EventListenerTest:BackgroundErrorListenerFailedFlushTest:1");
-  ASSERT_EQ(1, listener->counter());
-  ASSERT_OK(Put("key2", "val"));
-  ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
-  ASSERT_EQ(1, NumTableFilesAtLevel(0));
-}
-
-TEST_F(EventListenerTest, BackgroundErrorListenerFailedCompactionTest) {
-  auto listener = std::make_shared<BackgroundErrorListener>(env_);
-  Options options;
-  options.create_if_missing = true;
-  options.disable_auto_compactions = true;
-  options.env = env_;
-  options.level0_file_num_compaction_trigger = 2;
-  options.listeners.push_back(listener);
-  options.memtable_factory.reset(new SpecialSkipListFactory(2));
-  options.paranoid_checks = true;
-  DestroyAndReopen(options);
-
-  // third iteration triggers the second memtable's flush
-  for (int i = 0; i < 3; ++i) {
-    ASSERT_OK(Put("key0", "val"));
-    if (i > 0) {
-      ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
-    }
-    ASSERT_OK(Put("key1", "val"));
-  }
-  ASSERT_EQ(2, NumTableFilesAtLevel(0));
-
-  env_->drop_writes_.store(true, std::memory_order_release);
-  env_->no_slowdown_ = true;
-  ASSERT_OK(dbfull()->SetOptions({{"disable_auto_compactions", "false"}}));
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
-  ASSERT_EQ(1, listener->counter());
-
-  // trigger flush so compaction is triggered again; this time it succeeds
-  ASSERT_OK(Put("key0", "val"));
-  ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
-  ASSERT_EQ(0, NumTableFilesAtLevel(0));
-}
-
 }  // namespace rocksdb
 
 #endif  // ROCKSDB_LITE

db/log_writer.cc

@@ -18,22 +18,20 @@
 namespace rocksdb {
 namespace log {
 
-Writer::Writer(unique_ptr<WritableFileWriter>&& dest, uint64_t log_number,
-               bool recycle_log_files, bool manual_flush)
+Writer::Writer(unique_ptr<WritableFileWriter>&& dest,
+               uint64_t log_number, bool recycle_log_files)
     : dest_(std::move(dest)),
       block_offset_(0),
       log_number_(log_number),
-      recycle_log_files_(recycle_log_files),
-      manual_flush_(manual_flush) {
+      recycle_log_files_(recycle_log_files) {
   for (int i = 0; i <= kMaxRecordType; i++) {
     char t = static_cast<char>(i);
     type_crc_[i] = crc32c::Value(&t, 1);
   }
 }
 
-Writer::~Writer() { WriteBuffer(); }
-
-Status Writer::WriteBuffer() { return dest_->Flush(); }
+Writer::~Writer() {
+}
 
 Status Writer::AddRecord(const Slice& slice) {
   const char* ptr = slice.data();
@@ -129,9 +127,7 @@ Status Writer::EmitPhysicalRecord(RecordType t, const char* ptr, size_t n) {
   if (s.ok()) {
     s = dest_->Append(Slice(ptr, n));
     if (s.ok()) {
-      if (!manual_flush_) {
-        s = dest_->Flush();
-      }
+      s = dest_->Flush();
     }
   }
   block_offset_ += header_size + n;

db/log_writer.h

@@ -72,8 +72,8 @@ class Writer {
   // Create a writer that will append data to "*dest".
   // "*dest" must be initially empty.
   // "*dest" must remain live while this Writer is in use.
-  explicit Writer(unique_ptr<WritableFileWriter>&& dest, uint64_t log_number,
-                  bool recycle_log_files, bool manual_flush = false);
+  explicit Writer(unique_ptr<WritableFileWriter>&& dest,
+                  uint64_t log_number, bool recycle_log_files);
   ~Writer();
 
   Status AddRecord(const Slice& slice);
@@ -83,8 +83,6 @@ class Writer {
 
   uint64_t get_log_number() const { return log_number_; }
 
-  Status WriteBuffer();
-
  private:
   unique_ptr<WritableFileWriter> dest_;
   size_t block_offset_;       // Current offset in block
@@ -98,10 +96,6 @@ class Writer {
 
   Status EmitPhysicalRecord(RecordType type, const char* ptr, size_t length);
 
-  // If true, it does not flush after each write. Instead it relies on the upper
-  // layer to manually does the flush by calling ::WriteBuffer()
-  bool manual_flush_;
-
   // No copying allowed
   Writer(const Writer&);
   void operator=(const Writer&);

db/memtable.cc

@@ -9,9 +9,8 @@
 
 #include "db/memtable.h"
 
-#include <algorithm>
-#include <limits>
 #include <memory>
+#include <algorithm>
 
 #include "db/dbformat.h"
 #include "db/merge_context.h"
@@ -38,10 +37,10 @@
 
 namespace rocksdb {
 
-ImmutableMemTableOptions::ImmutableMemTableOptions(
-    const ImmutableCFOptions& ioptions,
-    const MutableCFOptions& mutable_cf_options)
-    : arena_block_size(mutable_cf_options.arena_block_size),
+MemTableOptions::MemTableOptions(const ImmutableCFOptions& ioptions,
+                                 const MutableCFOptions& mutable_cf_options)
+    : write_buffer_size(mutable_cf_options.write_buffer_size),
+      arena_block_size(mutable_cf_options.arena_block_size),
       memtable_prefix_bloom_bits(
           static_cast<uint32_t>(
               static_cast<double>(mutable_cf_options.write_buffer_size) *
@@ -82,7 +81,6 @@ MemTable::MemTable(const InternalKeyComparator& cmp,
       data_size_(0),
       num_entries_(0),
       num_deletes_(0),
-      write_buffer_size_(mutable_cf_options.write_buffer_size),
       flush_in_progress_(false),
       flush_completed_(false),
       file_number_(0),
@@ -98,8 +96,7 @@ MemTable::MemTable(const InternalKeyComparator& cmp,
       flush_state_(FLUSH_NOT_REQUESTED),
       env_(ioptions.env),
       insert_with_hint_prefix_extractor_(
-          ioptions.memtable_insert_with_hint_prefix_extractor),
-      oldest_key_time_(std::numeric_limits<uint64_t>::max()) {
+          ioptions.memtable_insert_with_hint_prefix_extractor) {
   UpdateFlushState();
   // something went wrong if we need to flush before inserting anything
   assert(!ShouldScheduleFlush());
@@ -136,7 +133,6 @@ size_t MemTable::ApproximateMemoryUsage() {
 }
 
 bool MemTable::ShouldFlushNow() const {
-  size_t write_buffer_size = write_buffer_size_.load(std::memory_order_relaxed);
   // In a lot of times, we cannot allocate arena blocks that exactly matches the
   // buffer size. Thus we have to decide if we should over-allocate or
   // under-allocate.
@@ -154,14 +150,16 @@ bool MemTable::ShouldFlushNow() const {
   // if we can still allocate one more block without exceeding the
   // over-allocation ratio, then we should not flush.
   if (allocated_memory + kArenaBlockSize <
-      write_buffer_size + kArenaBlockSize * kAllowOverAllocationRatio) {
+      moptions_.write_buffer_size +
+      kArenaBlockSize * kAllowOverAllocationRatio) {
     return false;
   }
 
-  // if user keeps adding entries that exceeds write_buffer_size, we need to
-  // flush earlier even though we still have much available memory left.
-  if (allocated_memory >
-      write_buffer_size + kArenaBlockSize * kAllowOverAllocationRatio) {
+  // if user keeps adding entries that exceeds moptions.write_buffer_size,
+  // we need to flush earlier even though we still have much available
+  // memory left.
+  if (allocated_memory > moptions_.write_buffer_size +
+      kArenaBlockSize * kAllowOverAllocationRatio) {
     return true;
   }
 
@@ -204,21 +202,6 @@ void MemTable::UpdateFlushState() {
   }
 }
 
-void MemTable::UpdateOldestKeyTime() {
-  uint64_t oldest_key_time = oldest_key_time_.load(std::memory_order_relaxed);
-  if (oldest_key_time == std::numeric_limits<uint64_t>::max()) {
-    int64_t current_time = 0;
-    auto s = env_->GetCurrentTime(&current_time);
-    if (s.ok()) {
-      assert(current_time >= 0);
-      // If fail, the timestamp is already set.
-      oldest_key_time_.compare_exchange_strong(
-          oldest_key_time, static_cast<uint64_t>(current_time),
-          std::memory_order_relaxed, std::memory_order_relaxed);
-    }
-  }
-}
-
 int MemTable::KeyComparator::operator()(const char* prefix_len_key1,
                                         const char* prefix_len_key2) const {
   // Internal keys are encoded as length-prefixed strings.
@@ -264,8 +247,7 @@ class MemTableIterator : public InternalIterator {
         comparator_(mem.comparator_),
         valid_(false),
         arena_mode_(arena != nullptr),
-        value_pinned_(
-            !mem.GetImmutableMemTableOptions()->inplace_update_support) {
+        value_pinned_(!mem.GetMemTableOptions()->inplace_update_support) {
     if (use_range_del_table) {
       iter_ = mem.range_del_table_->GetIterator(arena);
     } else if (prefix_extractor_ != nullptr && !read_options.total_order_seek) {
@@ -534,7 +516,6 @@ void MemTable::Add(SequenceNumber s, ValueType type,
   if (is_range_del_table_empty_ && type == kTypeRangeDeletion) {
     is_range_del_table_empty_ = false;
   }
-  UpdateOldestKeyTime();
 }
 
 // Callback from MemTable::Get()
@@ -556,7 +537,6 @@ struct Saver {
   Statistics* statistics;
   bool inplace_update_support;
   Env* env_;
-  bool* is_blob_index;
 };
 }  // namespace
 
@@ -586,26 +566,11 @@ static bool SaveValue(void* arg, const char* entry) {
     ValueType type;
     UnPackSequenceAndType(tag, &s->seq, &type);
 
-    if ((type == kTypeValue || type == kTypeMerge || type == kTypeBlobIndex) &&
+    if ((type == kTypeValue || type == kTypeMerge) &&
         range_del_agg->ShouldDelete(Slice(key_ptr, key_length))) {
       type = kTypeRangeDeletion;
     }
     switch (type) {
-      case kTypeBlobIndex:
-        if (s->is_blob_index == nullptr) {
-          ROCKS_LOG_ERROR(s->logger, "Encounter unexpected blob index.");
-          *(s->status) = Status::NotSupported(
-              "Encounter unsupported blob value. Please open DB with "
-              "rocksdb::blob_db::BlobDB instead.");
-        } else if (*(s->merge_in_progress)) {
-          *(s->status) =
-              Status::NotSupported("Blob DB does not support merge operator.");
-        }
-        if (!s->status->ok()) {
-          *(s->found_final_value) = true;
-          return false;
-        }
-      // intentional fallthrough
       case kTypeValue: {
         if (s->inplace_update_support) {
           s->mem->GetLock(s->key->user_key())->ReadLock();
@@ -624,9 +589,6 @@ static bool SaveValue(void* arg, const char* entry) {
           s->mem->GetLock(s->key->user_key())->ReadUnlock();
         }
         *(s->found_final_value) = true;
-        if (s->is_blob_index != nullptr) {
-          *(s->is_blob_index) = (type == kTypeBlobIndex);
-        }
         return false;
       }
       case kTypeDeletion:
@@ -673,7 +635,7 @@ static bool SaveValue(void* arg, const char* entry) {
 bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
                    MergeContext* merge_context,
                    RangeDelAggregator* range_del_agg, SequenceNumber* seq,
-                   const ReadOptions& read_opts, bool* is_blob_index) {
+                   const ReadOptions& read_opts) {
   // The sequence number is updated synchronously in version_set.h
   if (IsEmpty()) {
     // Avoiding recording stats for speed.
@@ -681,14 +643,6 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
   }
   PERF_TIMER_GUARD(get_from_memtable_time);
 
-  std::unique_ptr<InternalIterator> range_del_iter(
-      NewRangeTombstoneIterator(read_opts));
-  Status status = range_del_agg->AddTombstones(std::move(range_del_iter));
-  if (!status.ok()) {
-    *s = status;
-    return false;
-  }
-
   Slice user_key = key.user_key();
   bool found_final_value = false;
   bool merge_in_progress = s->IsMergeInProgress();
@@ -704,6 +658,13 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
     if (prefix_bloom_) {
       PERF_COUNTER_ADD(bloom_memtable_hit_count, 1);
     }
+    std::unique_ptr<InternalIterator> range_del_iter(
+        NewRangeTombstoneIterator(read_opts));
+    Status status = range_del_agg->AddTombstones(std::move(range_del_iter));
+    if (!status.ok()) {
+      *s = status;
+      return false;
+    }
     Saver saver;
     saver.status = s;
     saver.found_final_value = &found_final_value;
@@ -719,7 +680,6 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
     saver.inplace_update_support = moptions_.inplace_update_support;
     saver.statistics = moptions_.statistics;
     saver.env_ = env_;
-    saver.is_blob_index = is_blob_index;
     table_->Get(key, &saver, SaveValue);
 
     *seq = saver.seq;

db/memtable.h

@@ -35,9 +35,11 @@ class MemTableIterator;
 class MergeContext;
 class InternalIterator;
 
-struct ImmutableMemTableOptions {
-  explicit ImmutableMemTableOptions(const ImmutableCFOptions& ioptions,
-                                    const MutableCFOptions& mutable_cf_options);
+struct MemTableOptions {
+  explicit MemTableOptions(
+      const ImmutableCFOptions& ioptions,
+      const MutableCFOptions& mutable_cf_options);
+  size_t write_buffer_size;
   size_t arena_block_size;
   uint32_t memtable_prefix_bloom_bits;
   size_t memtable_huge_page_size;
@@ -185,15 +187,13 @@ class MemTable {
   // status returned indicates a corruption or other unexpected error.
   bool Get(const LookupKey& key, std::string* value, Status* s,
            MergeContext* merge_context, RangeDelAggregator* range_del_agg,
-           SequenceNumber* seq, const ReadOptions& read_opts,
-           bool* is_blob_index = nullptr);
+           SequenceNumber* seq, const ReadOptions& read_opts);
 
   bool Get(const LookupKey& key, std::string* value, Status* s,
            MergeContext* merge_context, RangeDelAggregator* range_del_agg,
-           const ReadOptions& read_opts, bool* is_blob_index = nullptr) {
+           const ReadOptions& read_opts) {
     SequenceNumber seq;
-    return Get(key, value, s, merge_context, range_del_agg, &seq, read_opts,
-               is_blob_index);
+    return Get(key, value, s, merge_context, range_del_agg, &seq, read_opts);
   }
 
   // Attempts to update the new_value inplace, else does normal Add
@@ -258,18 +258,6 @@ class MemTable {
     return num_deletes_.load(std::memory_order_relaxed);
   }
 
-  // Dynamically change the memtable's capacity. If set below the current usage,
-  // the next key added will trigger a flush. Can only increase size when
-  // memtable prefix bloom is disabled, since we can't easily allocate more
-  // space.
-  void UpdateWriteBufferSize(size_t new_write_buffer_size) {
-    if (prefix_bloom_ == nullptr ||
-        new_write_buffer_size < write_buffer_size_) {
-      write_buffer_size_.store(new_write_buffer_size,
-                               std::memory_order_relaxed);
-    }
-  }
-
   // Returns the edits area that is needed for flushing the memtable
   VersionEdit* GetEdits() { return &edit_; }
 
@@ -358,13 +346,7 @@ class MemTable {
     return comparator_.comparator;
   }
 
-  const ImmutableMemTableOptions* GetImmutableMemTableOptions() const {
-    return &moptions_;
-  }
-
-  uint64_t ApproximateOldestKeyTime() const {
-    return oldest_key_time_.load(std::memory_order_relaxed);
-  }
+  const MemTableOptions* GetMemTableOptions() const { return &moptions_; }
 
  private:
   enum FlushStateEnum { FLUSH_NOT_REQUESTED, FLUSH_REQUESTED, FLUSH_SCHEDULED };
@@ -374,7 +356,7 @@ class MemTable {
   friend class MemTableList;
 
   KeyComparator comparator_;
-  const ImmutableMemTableOptions moptions_;
+  const MemTableOptions moptions_;
   int refs_;
   const size_t kArenaBlockSize;
   AllocTracker mem_tracker_;
@@ -388,9 +370,6 @@ class MemTable {
   std::atomic<uint64_t> num_entries_;
   std::atomic<uint64_t> num_deletes_;
 
-  // Dynamically changeable memtable option
-  std::atomic<size_t> write_buffer_size_;
-
   // These are used to manage memtable flushes to storage
   bool flush_in_progress_; // started the flush
   bool flush_completed_;   // finished the flush
@@ -432,17 +411,12 @@ class MemTable {
   // Insert hints for each prefix.
   std::unordered_map<Slice, void*, SliceHasher> insert_hints_;
 
-  // Timestamp of oldest key
-  std::atomic<uint64_t> oldest_key_time_;
-
   // Returns a heuristic flush decision
   bool ShouldFlushNow() const;
 
   // Updates flush_state_ using ShouldFlushNow()
   void UpdateFlushState();
 
-  void UpdateOldestKeyTime();
-
   // No copying allowed
   MemTable(const MemTable&);
   MemTable& operator=(const MemTable&);

db/memtable_list.cc

@@ -10,7 +10,6 @@
 #endif
 
 #include <inttypes.h>
-#include <limits>
 #include <string>
 #include "db/memtable.h"
 #include "db/version_set.h"
@@ -104,31 +103,35 @@ int MemTableList::NumFlushed() const {
 bool MemTableListVersion::Get(const LookupKey& key, std::string* value,
                               Status* s, MergeContext* merge_context,
                               RangeDelAggregator* range_del_agg,
-                              SequenceNumber* seq, const ReadOptions& read_opts,
-                              bool* is_blob_index) {
+                              SequenceNumber* seq,
+                              const ReadOptions& read_opts) {
   return GetFromList(&memlist_, key, value, s, merge_context, range_del_agg,
-                     seq, read_opts, is_blob_index);
+                     seq, read_opts);
 }
 
-bool MemTableListVersion::GetFromHistory(
-    const LookupKey& key, std::string* value, Status* s,
-    MergeContext* merge_context, RangeDelAggregator* range_del_agg,
-    SequenceNumber* seq, const ReadOptions& read_opts, bool* is_blob_index) {
+bool MemTableListVersion::GetFromHistory(const LookupKey& key,
+                                         std::string* value, Status* s,
+                                         MergeContext* merge_context,
+                                         RangeDelAggregator* range_del_agg,
+                                         SequenceNumber* seq,
+                                         const ReadOptions& read_opts) {
   return GetFromList(&memlist_history_, key, value, s, merge_context,
-                     range_del_agg, seq, read_opts, is_blob_index);
+                     range_del_agg, seq, read_opts);
 }
 
-bool MemTableListVersion::GetFromList(
-    std::list<MemTable*>* list, const LookupKey& key, std::string* value,
-    Status* s, MergeContext* merge_context, RangeDelAggregator* range_del_agg,
-    SequenceNumber* seq, const ReadOptions& read_opts, bool* is_blob_index) {
+bool MemTableListVersion::GetFromList(std::list<MemTable*>* list,
+                                      const LookupKey& key, std::string* value,
+                                      Status* s, MergeContext* merge_context,
+                                      RangeDelAggregator* range_del_agg,
+                                      SequenceNumber* seq,
+                                      const ReadOptions& read_opts) {
   *seq = kMaxSequenceNumber;
 
   for (auto& memtable : *list) {
     SequenceNumber current_seq = kMaxSequenceNumber;
 
     bool done = memtable->Get(key, value, s, merge_context, range_del_agg,
-                              &current_seq, read_opts, is_blob_index);
+                              &current_seq, read_opts);
     if (*seq == kMaxSequenceNumber) {
       // Store the most recent sequence number of any operation on this key.
       // Since we only care about the most recent change, we only need to
@@ -444,13 +447,6 @@ size_t MemTableList::ApproximateUnflushedMemTablesMemoryUsage() {
 
 size_t MemTableList::ApproximateMemoryUsage() { return current_memory_usage_; }
 
-uint64_t MemTableList::ApproximateOldestKeyTime() const {
-  if (!current_->memlist_.empty()) {
-    return current_->memlist_.back()->ApproximateOldestKeyTime();
-  }
-  return std::numeric_limits<uint64_t>::max();
-}
-
 void MemTableList::InstallNewVersion() {
   if (current_->refs_ == 1) {
     // we're the only one using the version, just keep using it