Bump version to 5.7.2

Fix false removal of tombstone issue in FIFO and kCompactionStyleNone
Summary: Similar to the bug fixed by https://github.com/facebook/rocksdb/pull/2726, FIFO with compaction and kCompactionStyleNone during user customized CompactFiles() with output level to be 0 can suffer from the same problem. Fix it by leveraging the bottommost_level_ flag. Closes https://github.com/facebook/rocksdb/pull/2735 Differential Revision: D5626906 Pulled By: siying fbshipit-source-id: 2b148d0461c61dbd986d74655e384419ae442158
2026-11-07 03:22:34 -05:00 · 2017-08-15 14:19:55 -07:00 · 2017-08-15 14:12:19 -07:00 · 2017-08-15 14:08:44 -07:00 · 2017-08-14 10:27:56 -07:00 · 2017-08-11 19:09:12 -07:00
285 changed files with 9208 additions and 8579 deletions
@@ -0,0 +1,17 @@
+{
+  "project_id" : "rocksdb",
+  "conduit_uri" : "https://phabricator.fb.com/api/",
+  "copyright_holder" : "Facebook",
+  "load" : [
+    "arcanist_util"
+  ],
+  "lint.engine" : "FacebookFbcodeLintEngine",
+  "lint.engine.single.linter" : "FbcodeCppLinter",
+  "unit.engine" : "FacebookFbcodeUnitTestEngine",
+  "arcanist_configuration" : "FacebookArcanistConfiguration",
+  "base" : "git:HEAD^, hg:.^",
+  "git.default-relative-commit" : "HEAD^",
+  "git:arc.feature.start.default" : "origin/master",
+  "arc.feature.start.default" : "master",
+  "history.immutable" : false
+}
@@ -39,6 +39,7 @@ coverage/COVERAGE_REPORT
 .gdbhistory
 .gdb_history
 package/
+.phutil_module_cache
 unity.a
 tags
 etags
@@ -57,7 +57,7 @@ before_script:

 script:
  - ${CXX} --version
-  - if [ "${TEST_GROUP}" == 'platform_dependent' ]; then ccache -C && OPT=-DTRAVIS V=1 ROCKSDBTESTS_END=db_block_cache_test make -j4 all_but_some_tests check_some; fi
+  - if [ "${TEST_GROUP}" == 'platform_dependent' ]; then OPT=-DTRAVIS V=1 make -j4 all; OPT=-DTRAVIS V=1 ROCKSDBTESTS_END=db_block_cache_test make -j4 check_some; fi
  - if [ "${TEST_GROUP}" == '1' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=db_block_cache_test ROCKSDBTESTS_END=comparator_db_test make -j4 check_some; fi
  - if [ "${TEST_GROUP}" == '2' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=comparator_db_test make -j4 check_some; fi
  - if [ "${JOB_NAME}" == 'java_test' ]; then OPT=-DTRAVIS V=1 make clean jclean && make rocksdbjava jtest; fi
@@ -56,7 +56,6 @@ else()
      include_directories(${JEMALLOC_INCLUDE_DIR})
    endif()
  endif()
-
  option(WITH_SNAPPY "build with SNAPPY" OFF)
  if(WITH_SNAPPY)
    find_package(snappy REQUIRED)
@@ -64,38 +63,6 @@ else()
    include_directories(${SNAPPY_INCLUDE_DIR})
    list(APPEND THIRDPARTY_LIBS ${SNAPPY_LIBRARIES})
  endif()
-
-  option(WITH_ZLIB "build with zlib" OFF)
-  if(WITH_ZLIB)
-    find_package(zlib REQUIRED)
-    add_definitions(-DZLIB)
-    include_directories(${ZLIB_INCLUDE_DIR})
-    list(APPEND THIRDPARTY_LIBS ${ZLIB_LIBRARIES})
-  endif()
-
-  option(WITH_BZ2 "build with bzip2" OFF)
-  if(WITH_BZ2)
-    find_package(bzip2 REQUIRED)
-    add_definitions(-DBZIP2)
-    include_directories(${BZIP2_INCLUDE_DIR})
-    list(APPEND THIRDPARTY_LIBS ${BZIP2_LIBRARIES})
-  endif()
-
-  option(WITH_LZ4 "build with lz4" OFF)
-  if(WITH_LZ4)
-    find_package(lz4 REQUIRED)
-    add_definitions(-DLZ4)
-    include_directories(${LZ4_INCLUDE_DIR})
-    list(APPEND THIRDPARTY_LIBS ${LZ4_LIBRARIES})
-  endif()
-
-  option(WITH_ZSTD "build with zstd" OFF)
-  if(WITH_ZSTD)
-    find_package(zstd REQUIRED)
-    add_definitions(-DZSTD)
-    include_directories(${ZSTD_INCLUDE_DIR})
-    list(APPEND THIRDPARTY_LIBS ${ZSTD_LIBRARIES})
-  endif()
 endif()

 if(WIN32)
@@ -148,31 +115,6 @@ if(WIN32 AND MSVC)
  endif()
 endif()

-set(BUILD_VERSION_CC ${CMAKE_BINARY_DIR}/build_version.cc)
-configure_file(util/build_version.cc.in ${BUILD_VERSION_CC} @ONLY)
-add_library(build_version OBJECT ${BUILD_VERSION_CC})
-target_include_directories(build_version PRIVATE
-  ${CMAKE_CURRENT_SOURCE_DIR}/util)
-if(MSVC)
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W3 /wd4127 /wd4800 /wd4996 /wd4351")
-else()
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wshadow -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers")
-  if(MINGW)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format")
-  endif()
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
-  if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -fno-omit-frame-pointer")
-    include(CheckCXXCompilerFlag)
-    CHECK_CXX_COMPILER_FLAG("-momit-leaf-frame-pointer" HAVE_OMIT_LEAF_FRAME_POINTER)
-    if(HAVE_OMIT_LEAF_FRAME_POINTER)
-      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -momit-leaf-frame-pointer")
-    endif()
-  endif()
-endif()
-
 option(PORTABLE "build a portable binary" OFF)
 option(FORCE_SSE42 "force building with SSE4.2, even when PORTABLE=ON" OFF)
 if(PORTABLE)
@@ -216,6 +158,32 @@ if(HAVE_THREAD_LOCAL)
  add_definitions(-DROCKSDB_SUPPORT_THREAD_LOCAL)
 endif()

+set(BUILD_VERSION_CC ${CMAKE_BINARY_DIR}/build_version.cc)
+configure_file(util/build_version.cc.in ${BUILD_VERSION_CC} @ONLY)
+add_library(build_version OBJECT ${BUILD_VERSION_CC})
+target_include_directories(build_version PRIVATE
+  ${CMAKE_CURRENT_SOURCE_DIR}/util)
+if(MSVC)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W3 /wd4127 /wd4800 /wd4996 /wd4351")
+else()
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wshadow -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers")
+  if(MINGW)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format")
+  endif()
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+  if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
+    add_definitions(-DNDEBUG)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -fno-omit-frame-pointer")
+    include(CheckCXXCompilerFlag)
+    CHECK_CXX_COMPILER_FLAG("-momit-leaf-frame-pointer" HAVE_OMIT_LEAF_FRAME_POINTER)
+    if(HAVE_OMIT_LEAF_FRAME_POINTER)
+      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -momit-leaf-frame-pointer")
+    endif()
+  endif()
+endif()
+
 option(FAIL_ON_WARNINGS "Treat compile warnings as errors" ON)
 if(FAIL_ON_WARNINGS)
  if(MSVC)
@@ -325,7 +293,6 @@ endif()
 option(WITH_FALLOCATE "build with fallocate" ON)

 if(WITH_FALLOCATE)
-  set(CMAKE_REQUIRED_FLAGS ${CMAKE_C_FLAGS})
  include(CheckCSourceCompiles)
  CHECK_C_SOURCE_COMPILES("
 #include <fcntl.h>
@@ -510,15 +477,13 @@ set(SOURCES
        utilities/backupable/backupable_db.cc
        utilities/blob_db/blob_db.cc
        utilities/blob_db/blob_db_impl.cc
+        utilities/blob_db/blob_db_options_impl.cc
        utilities/blob_db/blob_dump_tool.cc
        utilities/blob_db/blob_file.cc
        utilities/blob_db/blob_log_reader.cc
        utilities/blob_db/blob_log_writer.cc
        utilities/blob_db/blob_log_format.cc
        utilities/blob_db/ttl_extractor.cc
-        utilities/cassandra/cassandra_compaction_filter.cc
-        utilities/cassandra/format.cc
-        utilities/cassandra/merge_operator.cc
        utilities/checkpoint/checkpoint_impl.cc
        utilities/col_buf_decoder.cc
        utilities/col_buf_encoder.cc
@@ -537,6 +502,8 @@ set(SOURCES
        utilities/memory/memory_util.cc
        utilities/merge_operators/max.cc
        utilities/merge_operators/put.cc
+        utilities/merge_operators/cassandra/format.cc
+        utilities/merge_operators/cassandra/merge_operator.cc
        utilities/merge_operators/string_append/stringappend.cc
        utilities/merge_operators/string_append/stringappend2.cc
        utilities/merge_operators/uint64add.cc
@@ -552,14 +519,13 @@ set(SOURCES
        utilities/spatialdb/spatial_db.cc
        utilities/table_properties_collectors/compact_on_deletion_collector.cc
        utilities/transactions/optimistic_transaction_db_impl.cc
-        utilities/transactions/optimistic_transaction.cc
+        utilities/transactions/optimistic_transaction_impl.cc
        utilities/transactions/transaction_base.cc
-        utilities/transactions/pessimistic_transaction_db.cc
+        utilities/transactions/transaction_db_impl.cc
        utilities/transactions/transaction_db_mutex_impl.cc
-        utilities/transactions/pessimistic_transaction.cc
+        utilities/transactions/transaction_impl.cc
        utilities/transactions/transaction_lock_mgr.cc
        utilities/transactions/transaction_util.cc
-        utilities/transactions/write_prepared_txn.cc
        utilities/ttl/db_ttl_impl.cc
        utilities/write_batch_with_index/write_batch_with_index.cc
        utilities/write_batch_with_index/write_batch_with_index_internal.cc
@@ -635,72 +601,7 @@ else()
  message(STATUS "JNI library is disabled")
 endif()

-# Installation and packaging
-if(WIN32)
-  option(ROCKSDB_INSTALL_ON_WINDOWS "Enable install target on Windows" OFF)
-endif()
-if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS)
-  if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
-    if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
-      # Change default installation prefix on Linux to /usr
-      set(CMAKE_INSTALL_PREFIX /usr CACHE PATH "Install path prefix, prepended onto install directories." FORCE)
-    endif()
-  endif()
-
-  include(GNUInstallDirs)
-  include(CMakePackageConfigHelpers)
-
-  set(package_config_destination ${CMAKE_INSTALL_LIBDIR}/cmake/rocksdb)
-
-  configure_package_config_file(
-    ${CMAKE_SOURCE_DIR}/cmake/RocksDBConfig.cmake.in RocksDBConfig.cmake
-    INSTALL_DESTINATION ${package_config_destination}
-  )
-
-  write_basic_package_version_file(
-    RocksDBConfigVersion.cmake
-    VERSION ${ROCKSDB_VERSION}
-    COMPATIBILITY SameMajorVersion
-  )
-
-  install(DIRECTORY include/rocksdb COMPONENT devel DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
-
-  install(
-    TARGETS ${ROCKSDB_STATIC_LIB}
-    EXPORT RocksDBTargets
-    COMPONENT devel
-    ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
-    INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
-  )
-
-  install(
-    TARGETS ${ROCKSDB_SHARED_LIB}
-    EXPORT RocksDBTargets
-    COMPONENT runtime
-    RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
-    LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
-    INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
-  )
-
-  install(
-    EXPORT RocksDBTargets
-    COMPONENT devel
-    DESTINATION ${package_config_destination}
-    NAMESPACE RocksDB::
-  )
-
-  install(
-    FILES
-    ${CMAKE_CURRENT_BINARY_DIR}/RocksDBConfig.cmake
-    ${CMAKE_CURRENT_BINARY_DIR}/RocksDBConfigVersion.cmake
-    COMPONENT devel
-    DESTINATION ${package_config_destination}
-  )
-endif()
-
-option(WITH_TESTS "build with tests" ON)
-if(WITH_TESTS)
-  set(TESTS
+set(TESTS
        cache/cache_test.cc
        cache/lru_cache_test.cc
        db/column_family_test.cc
@@ -806,10 +707,6 @@ if(WITH_TESTS)
        util/thread_local_test.cc
        utilities/backupable/backupable_db_test.cc
        utilities/blob_db/blob_db_test.cc
-        utilities/cassandra/cassandra_functional_test.cc
-        utilities/cassandra/cassandra_format_test.cc
-        utilities/cassandra/cassandra_row_merge_test.cc
-        utilities/cassandra/cassandra_serialize_test.cc
        utilities/checkpoint/checkpoint_test.cc
        utilities/column_aware_encoding_test.cc
        utilities/date_tiered/date_tiered_test.cc
@@ -818,6 +715,10 @@ if(WITH_TESTS)
        utilities/geodb/geodb_test.cc
        utilities/lua/rocks_lua_test.cc
        utilities/memory/memory_test.cc
+        utilities/merge_operators/cassandra/cassandra_merge_test.cc
+        utilities/merge_operators/cassandra/cassandra_format_test.cc
+        utilities/merge_operators/cassandra/cassandra_row_merge_test.cc
+        utilities/merge_operators/cassandra/cassandra_serialize_test.cc
        utilities/merge_operators/string_append/stringappend_test.cc
        utilities/object_registry_test.cc
        utilities/option_change_migration/option_change_migration_test.cc
@@ -826,94 +727,98 @@ if(WITH_TESTS)
        utilities/persistent_cache/persistent_cache_test.cc
        utilities/redis/redis_lists_test.cc
        utilities/spatialdb/spatial_db_test.cc
-        utilities/simulator_cache/sim_cache_test.cc
        utilities/table_properties_collectors/compact_on_deletion_collector_test.cc
        utilities/transactions/optimistic_transaction_test.cc
        utilities/transactions/transaction_test.cc
        utilities/ttl/ttl_test.cc
        utilities/write_batch_with_index/write_batch_with_index_test.cc
-  )
-  if(WITH_LIBRADOS)
-    list(APPEND TESTS utilities/env_librados_test.cc)
-  endif()
+)
+if(WITH_LIBRADOS)
+  list(APPEND TESTS utilities/env_librados_test.cc)
+endif()

-  set(BENCHMARKS
-    cache/cache_bench.cc
-    memtable/memtablerep_bench.cc
-    tools/db_bench.cc
-    table/table_reader_bench.cc
-    utilities/column_aware_encoding_exp.cc
-    utilities/persistent_cache/hash_table_bench.cc)
-  add_library(testharness OBJECT util/testharness.cc)
-  foreach(sourcefile ${BENCHMARKS})
+set(BENCHMARKS
+  cache/cache_bench.cc
+  memtable/memtablerep_bench.cc
+  tools/db_bench.cc
+  table/table_reader_bench.cc
+  utilities/column_aware_encoding_exp.cc
+  utilities/persistent_cache/hash_table_bench.cc)
+add_library(testharness OBJECT util/testharness.cc)
+foreach(sourcefile ${BENCHMARKS})
+  get_filename_component(exename ${sourcefile} NAME_WE)
+  add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile}
+    $<TARGET_OBJECTS:testharness>)
+  target_link_libraries(${exename}${ARTIFACT_SUFFIX} gtest ${LIBS})
+endforeach(sourcefile ${BENCHMARKS})
+
+# For test util library that is build only in DEBUG mode
+# and linked to tests. Add test only code that is not #ifdefed for Release here.
+set(TESTUTIL_SOURCE
+    db/db_test_util.cc
+    monitoring/thread_status_updater_debug.cc
+    table/mock_table.cc
+    util/fault_injection_test_env.cc
+    utilities/merge_operators/cassandra/test_utils.cc
+)
+# test utilities are only build in debug
+enable_testing()
+add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND})
+set(TESTUTILLIB testutillib${ARTIFACT_SUFFIX})
+add_library(${TESTUTILLIB} STATIC ${TESTUTIL_SOURCE})
+if(MSVC)
+  set_target_properties(${TESTUTILLIB} PROPERTIES COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/testutillib${ARTIFACT_SUFFIX}.pdb")
+endif()
+set_target_properties(${TESTUTILLIB}
+      PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
+      EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
+      EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
+      )
+
+# Tests are excluded from Release builds
+set(TEST_EXES ${TESTS})
+
+foreach(sourcefile ${TEST_EXES})
    get_filename_component(exename ${sourcefile} NAME_WE)
    add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile}
      $<TARGET_OBJECTS:testharness>)
-    target_link_libraries(${exename}${ARTIFACT_SUFFIX} gtest ${LIBS})
-  endforeach(sourcefile ${BENCHMARKS})
-
-  # For test util library that is build only in DEBUG mode
-  # and linked to tests. Add test only code that is not #ifdefed for Release here.
-  set(TESTUTIL_SOURCE
-      db/db_test_util.cc
-      monitoring/thread_status_updater_debug.cc
-      table/mock_table.cc
-      util/fault_injection_test_env.cc
-      utilities/cassandra/test_utils.cc
-  )
-  # test utilities are only build in debug
-  enable_testing()
-  add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND})
-  set(TESTUTILLIB testutillib${ARTIFACT_SUFFIX})
-  add_library(${TESTUTILLIB} STATIC ${TESTUTIL_SOURCE})
-  if(MSVC)
-    set_target_properties(${TESTUTILLIB} PROPERTIES COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/testutillib${ARTIFACT_SUFFIX}.pdb")
-  endif()
-  set_target_properties(${TESTUTILLIB}
-        PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
-        EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
-        EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
-        )
-
-  # Tests are excluded from Release builds
-  set(TEST_EXES ${TESTS})
-
-  foreach(sourcefile ${TEST_EXES})
-      get_filename_component(exename ${sourcefile} NAME_WE)
-      add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile}
-        $<TARGET_OBJECTS:testharness>)
-      set_target_properties(${exename}${ARTIFACT_SUFFIX}
-        PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
-        EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
-        EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
-        )
-      target_link_libraries(${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} gtest ${LIBS})
-      if(NOT "${exename}" MATCHES "db_sanity_test")
-        add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
-        add_dependencies(check ${exename}${ARTIFACT_SUFFIX})
-      endif()
-  endforeach(sourcefile ${TEST_EXES})
-
-  # C executables must link to a shared object
-  set(C_TESTS db/c_test.c)
-  set(C_TEST_EXES ${C_TESTS})
-
-  foreach(sourcefile ${C_TEST_EXES})
-      string(REPLACE ".c" "" exename ${sourcefile})
-      string(REGEX REPLACE "^((.+)/)+" "" exename ${exename})
-      add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile})
-      set_target_properties(${exename}${ARTIFACT_SUFFIX}
-        PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
-        EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
-        EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
-        )
-      target_link_libraries(${exename}${ARTIFACT_SUFFIX} ${ROCKSDB_IMPORT_LIB} testutillib${ARTIFACT_SUFFIX})
+    set_target_properties(${exename}${ARTIFACT_SUFFIX}
+      PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
+      EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
+      EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
+      )
+    target_link_libraries(${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} gtest ${LIBS})
+    if(NOT "${exename}" MATCHES "db_sanity_test")
      add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
      add_dependencies(check ${exename}${ARTIFACT_SUFFIX})
-  endforeach(sourcefile ${C_TEST_EXES})
-endif()
+    endif()
+endforeach(sourcefile ${TEST_EXES})

-option(WITH_TOOLS "build with tools" ON)
-if(WITH_TOOLS)
-  add_subdirectory(tools)
+# C executables must link to a shared object
+set(C_TESTS db/c_test.c)
+set(C_TEST_EXES ${C_TESTS})
+
+foreach(sourcefile ${C_TEST_EXES})
+    string(REPLACE ".c" "" exename ${sourcefile})
+    string(REGEX REPLACE "^((.+)/)+" "" exename ${exename})
+    add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile})
+    set_target_properties(${exename}${ARTIFACT_SUFFIX}
+      PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
+      EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
+      EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
+      )
+    target_link_libraries(${exename}${ARTIFACT_SUFFIX} ${ROCKSDB_IMPORT_LIB} testutillib${ARTIFACT_SUFFIX})
+    add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
+    add_dependencies(check ${exename}${ARTIFACT_SUFFIX})
+endforeach(sourcefile ${C_TEST_EXES})
+add_subdirectory(tools)
+
+# Installation and packaging for Linux
+if(NOT WIN32)
+install(TARGETS ${ROCKSDB_STATIC_LIB} COMPONENT devel ARCHIVE DESTINATION lib64)
+install(TARGETS ${ROCKSDB_SHARED_LIB} COMPONENT runtime DESTINATION lib64)
+install(DIRECTORY "${PROJECT_SOURCE_DIR}/include/rocksdb/"
+        COMPONENT devel
+        DESTINATION include/rocksdb)
+set(CMAKE_INSTALL_PREFIX /usr)
 endif()
@@ -1,28 +1,12 @@
 # Rocksdb Change Log
-## Unreleased
-### Public API Change
-### New Features
+## 5.7.2 (08/15/2017)
 ### Bug Fixes
+* Fix incorrect dropping of deletions issue with FIFO compaction.
+* Fix LITE build compiler error with missing abort().

-## 5.8.0 (08/30/2017)
-### Public API Change
-* Users of `Statistics::getHistogramString()` will see fewer histogram buckets and different bucket endpoints.
-* `Slice::compare` and BytewiseComparator `Compare` no longer accept `Slice`s containing nullptr.
-* `Transaction::Get` and `Transaction::GetForUpdate` variants with `PinnableSlice` added.
-
-### New Features
-* Add Iterator::Refresh(), which allows users to update the iterator state so that they can avoid some initialization costs of recreating iterators.
-* Replace dynamic_cast<> (except unit test) so people can choose to build with RTTI off. With make, release mode is by default built with -fno-rtti and debug mode is built without it. Users can override it by setting USE_RTTI=0 or 1.
-* Universal compactions including the bottom level can be executed in a dedicated thread pool. This alleviates head-of-line blocking in the compaction queue, which cause write stalling, particularly in multi-instance use cases. Users can enable this feature via `Env::SetBackgroundThreads(N, Env::Priority::BOTTOM)`, where `N > 0`.
-* Allow merge operator to be called even with a single merge operand during compactions, by appropriately overriding `MergeOperator::AllowSingleOperand`.
-* Add `DB::VerifyChecksum()`, which verifies the checksums in all SST files in a running DB.
-* Block-based table support for disabling checksums by setting `BlockBasedTableOptions::checksum = kNoChecksum`.
-
+## 5.7.1 (08/13/2017)
 ### Bug Fixes
-* Fix wrong latencies in `rocksdb.db.get.micros`, `rocksdb.db.write.micros`, and `rocksdb.sst.read.micros`.
 * Fix incorrect dropping of deletions during intra-L0 compaction.
-* Fix transient reappearance of keys covered by range deletions when memtable prefix bloom filter is enabled.
-* Fix potentially wrong file smallest key when range deletions separated by snapshot are written together.

 ## 5.7.0 (07/13/2017)
 ### Public API Change
@@ -113,8 +113,6 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
 * **Windows**:
  * For building with MS Visual Studio 13 you will need Update 4 installed.
  * Read and follow the instructions at CMakeLists.txt
-  * Or install via [vcpkg](https://github.com/microsoft/vcpkg) 
-       * run `vcpkg install rocksdb`

 * **AIX 6.1**
    * Install AIX Toolbox rpms with gcc
@@ -9,8 +9,5 @@ This is the list of all known third-party language bindings for RocksDB. If some
 * Haskell - https://hackage.haskell.org/package/rocksdb-haskell
 * PHP - https://github.com/Photonios/rocksdb-php
 * C# - https://github.com/warrenfalk/rocksdb-sharp
-* Rust
-    * https://github.com/spacejam/rust-rocksdb
-    * https://github.com/bh1xuw/rust-rocks
+* Rust - https://github.com/spacejam/rust-rocksdb
 * D programming language - https://github.com/b1naryth1ef/rocksdb
-* Erlang - https://gitlab.com/barrel-db/erlang-rocksdb
@@ -101,19 +101,7 @@ endif
 ifeq ($(DEBUG_LEVEL),0)
 OPT += -DNDEBUG
 DISABLE_WARNING_AS_ERROR=1
-
-ifneq ($(USE_RTTI), 1)
-	CXXFLAGS += -fno-rtti
 else
-	CXXFLAGS += -DROCKSDB_USE_RTTI
-endif
-else
-ifneq ($(USE_RTTI), 0)
-	CXXFLAGS += -DROCKSDB_USE_RTTI
-else
-	CXXFLAGS += -fno-rtti
-endif
-
 $(warning Warning: Compiling in debug mode. Don't use the resulting binary in production)
 endif

@@ -232,10 +220,6 @@ ifndef DISABLE_JEMALLOC
 		PLATFORM_CXXFLAGS += -DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE
 		PLATFORM_CCFLAGS  += -DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE
 	endif
-	ifdef WITH_JEMALLOC_FLAG
-		PLATFORM_LDFLAGS += -ljemalloc
-		JAVA_LDFLAGS += -ljemalloc
-	endif
 	EXEC_LDFLAGS := $(JEMALLOC_LIB) $(EXEC_LDFLAGS)
 	PLATFORM_CXXFLAGS += $(JEMALLOC_INCLUDE)
 	PLATFORM_CCFLAGS += $(JEMALLOC_INCLUDE)
@@ -421,7 +405,7 @@ TESTS = \
 	write_buffer_manager_test \
 	stringappend_test \
 	cassandra_format_test \
-	cassandra_functional_test \
+	cassandra_merge_test \
 	cassandra_row_merge_test \
 	cassandra_serialize_test \
 	ttl_test \
@@ -591,8 +575,6 @@ endif  # PLATFORM_SHARED_EXT

 all: $(LIBRARY) $(BENCHMARKS) tools tools_lib test_libs $(TESTS)

-all_but_some_tests: $(LIBRARY) $(BENCHMARKS) tools tools_lib test_libs $(SUBSET)
-
 static_lib: $(LIBRARY)

 shared_lib: $(SHARED)
@@ -796,8 +778,8 @@ ldb_tests: ldb
 crash_test: whitebox_crash_test blackbox_crash_test

 blackbox_crash_test: db_stress
-	python -u tools/db_crashtest.py --simple blackbox $(CRASH_TEST_EXT_ARGS)
-	python -u tools/db_crashtest.py blackbox $(CRASH_TEST_EXT_ARGS)
+	python -u tools/db_crashtest.py --simple blackbox
+	python -u tools/db_crashtest.py blackbox

 ifeq ($(CRASH_TEST_KILL_ODD),)
  CRASH_TEST_KILL_ODD=888887
@@ -805,9 +787,9 @@ endif

 whitebox_crash_test: db_stress
 	python -u tools/db_crashtest.py --simple whitebox --random_kill_odd \
-      $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
+      $(CRASH_TEST_KILL_ODD)
 	python -u tools/db_crashtest.py whitebox  --random_kill_odd \
-      $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
+      $(CRASH_TEST_KILL_ODD)

 asan_check:
 	$(MAKE) clean
@@ -1018,16 +1000,16 @@ option_change_migration_test: utilities/option_change_migration/option_change_mi
 stringappend_test: utilities/merge_operators/string_append/stringappend_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-cassandra_format_test: utilities/cassandra/cassandra_format_test.o utilities/cassandra/test_utils.o $(LIBOBJECTS) $(TESTHARNESS)
+cassandra_format_test: utilities/merge_operators/cassandra/cassandra_format_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-cassandra_functional_test: utilities/cassandra/cassandra_functional_test.o utilities/cassandra/test_utils.o $(LIBOBJECTS) $(TESTHARNESS)
+cassandra_merge_test: utilities/merge_operators/cassandra/cassandra_merge_test.o utilities/merge_operators/cassandra/test_utils.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-cassandra_row_merge_test: utilities/cassandra/cassandra_row_merge_test.o utilities/cassandra/test_utils.o $(LIBOBJECTS) $(TESTHARNESS)
+cassandra_row_merge_test: utilities/merge_operators/cassandra/cassandra_row_merge_test.o utilities/merge_operators/cassandra/test_utils.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-cassandra_serialize_test: utilities/cassandra/cassandra_serialize_test.o $(LIBOBJECTS) $(TESTHARNESS)
+cassandra_serialize_test: utilities/merge_operators/cassandra/cassandra_serialize_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

 redis_test: utilities/redis/redis_lists_test.o $(LIBOBJECTS) $(TESTHARNESS)
@@ -6,6 +6,8 @@ REPO_PATH = TARGETS_PATH[(TARGETS_PATH.find('fbcode/') + len('fbcode/')):] + "/"
 BUCK_BINS = "buck-out/gen/" + REPO_PATH
 TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh"
 rocksdb_compiler_flags = [
+  "-msse",
+  "-msse4.2",
  "-fno-builtin-memcmp",
  "-DROCKSDB_PLATFORM_POSIX",
  "-DROCKSDB_LIB_IO_POSIX",
@@ -14,8 +16,8 @@ rocksdb_compiler_flags = [
  "-DROCKSDB_RANGESYNC_PRESENT",
  "-DROCKSDB_SCHED_GETCPU_PRESENT",
  "-DROCKSDB_SUPPORT_THREAD_LOCAL",
+  "-DHAVE_SSE42",
  "-DOS_LINUX",
-  "-DROCKSDB_UBSAN_RUN",
  # Flags to enable libs we include
  "-DSNAPPY",
  "-DZLIB",
@@ -47,10 +49,6 @@ rocksdb_preprocessor_flags = [
  "-I" + REPO_PATH,
 ]

-rocksdb_arch_preprocessor_flags = {
-  "x86_64": ["-DHAVE_SSE42"],
-}
-
 cpp_library(
    name = "rocksdb_lib",
    headers = AutoHeaders.RECURSIVE_GLOB,
@@ -209,14 +207,12 @@ cpp_library(
      "utilities/backupable/backupable_db.cc",
      "utilities/blob_db/blob_db.cc",
      "utilities/blob_db/blob_db_impl.cc",
+      "utilities/blob_db/blob_db_options_impl.cc",
      "utilities/blob_db/blob_file.cc",
      "utilities/blob_db/blob_log_reader.cc",
      "utilities/blob_db/blob_log_writer.cc",
      "utilities/blob_db/blob_log_format.cc",
      "utilities/blob_db/ttl_extractor.cc",
-      "utilities/cassandra/cassandra_compaction_filter.cc",
-      "utilities/cassandra/format.cc",
-      "utilities/cassandra/merge_operator.cc",
      "utilities/checkpoint/checkpoint_impl.cc",
      "utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc",
      "utilities/convenience/info_log_finder.cc",
@@ -231,6 +227,8 @@ cpp_library(
      "utilities/leveldb_options/leveldb_options.cc",
      "utilities/lua/rocks_lua_compaction_filter.cc",
      "utilities/memory/memory_util.cc",
+      "utilities/merge_operators/cassandra/format.cc",
+      "utilities/merge_operators/cassandra/merge_operator.cc",
      "utilities/merge_operators/max.cc",
      "utilities/merge_operators/put.cc",
      "utilities/merge_operators/string_append/stringappend.cc",
@@ -248,14 +246,13 @@ cpp_library(
      "utilities/spatialdb/spatial_db.cc",
      "utilities/table_properties_collectors/compact_on_deletion_collector.cc",
      "utilities/transactions/optimistic_transaction_db_impl.cc",
-      "utilities/transactions/optimistic_transaction.cc",
+      "utilities/transactions/optimistic_transaction_impl.cc",
      "utilities/transactions/transaction_base.cc",
-      "utilities/transactions/pessimistic_transaction_db.cc",
+      "utilities/transactions/transaction_db_impl.cc",
      "utilities/transactions/transaction_db_mutex_impl.cc",
-      "utilities/transactions/pessimistic_transaction.cc",
+      "utilities/transactions/transaction_impl.cc",
      "utilities/transactions/transaction_lock_mgr.cc",
      "utilities/transactions/transaction_util.cc",
-      "utilities/transactions/write_prepared_txn.cc",
      "utilities/ttl/db_ttl_impl.cc",
      "utilities/write_batch_with_index/write_batch_with_index.cc",
      "utilities/write_batch_with_index/write_batch_with_index_internal.cc",
@@ -266,7 +263,6 @@ cpp_library(
    ],
    deps = [],
    preprocessor_flags = rocksdb_preprocessor_flags,
-    arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
    compiler_flags = rocksdb_compiler_flags,
    external_deps = rocksdb_external_deps,
 )
@@ -280,14 +276,13 @@ cpp_library(
      "util/testharness.cc",
      "util/testutil.cc",
      "db/db_test_util.cc",
-      "utilities/cassandra/test_utils.cc",
+      "utilities/merge_operators/cassandra/test_utils.cc",
      "utilities/col_buf_encoder.cc",
      "utilities/col_buf_decoder.cc",
      "utilities/column_aware_encoding_util.cc",
    ],
    deps = [":rocksdb_lib"],
    preprocessor_flags = rocksdb_preprocessor_flags,
-    arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
    compiler_flags = rocksdb_compiler_flags,
    external_deps = rocksdb_external_deps,
 )
@@ -301,7 +296,6 @@ cpp_library(
    ],
    deps = [":rocksdb_lib"],
    preprocessor_flags = rocksdb_preprocessor_flags,
-    arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
    compiler_flags = rocksdb_compiler_flags,
    external_deps = rocksdb_external_deps,
 )
@@ -312,7 +306,6 @@ cpp_library(
    srcs = ["env/env_basic_test.cc"],
    deps = [":rocksdb_test_lib"],
    preprocessor_flags = rocksdb_preprocessor_flags,
-    arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
    compiler_flags = rocksdb_compiler_flags,
    external_deps = rocksdb_external_deps,
 )
@@ -333,16 +326,16 @@ ROCKS_TESTS = [['arena_test', 'util/arena_test.cc', 'serial'],
 ['c_test', 'db/c_test.c', 'serial'],
 ['cache_test', 'cache/cache_test.cc', 'serial'],
 ['cassandra_format_test',
-  'utilities/cassandra/cassandra_format_test.cc',
+  'utilities/merge_operators/cassandra/cassandra_format_test.cc',
  'serial'],
- ['cassandra_functional_test',
-  'utilities/cassandra/cassandra_functional_test.cc',
+ ['cassandra_merge_test',
+  'utilities/merge_operators/cassandra/cassandra_merge_test.cc',
  'serial'],
 ['cassandra_row_merge_test',
-  'utilities/cassandra/cassandra_row_merge_test.cc',
+  'utilities/merge_operators/cassandra/cassandra_row_merge_test.cc',
  'serial'],
 ['cassandra_serialize_test',
-  'utilities/cassandra/cassandra_serialize_test.cc',
+  'utilities/merge_operators/cassandra/cassandra_serialize_test.cc',
  'serial'],
 ['checkpoint_test', 'utilities/checkpoint/checkpoint_test.cc', 'serial'],
 ['cleanable_test', 'table/cleanable_test.cc', 'serial'],
@@ -508,7 +501,6 @@ for test_cfg in ROCKS_TESTS:
      srcs = [test_cc],
      deps = [":rocksdb_test_lib"],
      preprocessor_flags = rocksdb_preprocessor_flags,
-      arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
      compiler_flags = rocksdb_compiler_flags,
      external_deps = rocksdb_external_deps,
    )
@@ -80,6 +80,3 @@ quasardb uses a heavily tuned RocksDB as its persistence layer.

 ## 360 Pika
 [360](http://www.360.cn/) [Pika](https://github.com/Qihoo360/pika) is a nosql compatible with redis. With the huge amount of data stored, redis may suffer for a capacity bottleneck, and pika was born for solving it. It has widely been widely used in many company
-
-## LzLabs
-LzLabs is using RocksDB as a storage engine in their multi-database distributed framework to store application configuration and user data.
@@ -0,0 +1,3 @@
+<?php
+
+phutil_register_library('arcanist_util', __FILE__);
@@ -0,0 +1,71 @@
+<?php
+
+/**
+ * This file is automatically generated. Use 'arc liberate' to rebuild it.
+ * @generated
+ * @phutil-library-version 2
+ */
+
+if (class_exists('ArcanistWorkflow')) {
+  phutil_register_library_map(array(
+    '__library_version__' => 2,
+    'class' =>
+    array(
+      'ArcanistCpplintLinter' => 'cpp_linter/ArcanistCpplintLinter.php',
+      'BaseDirectoryScopedFormatLinter' => 'cpp_linter/BaseDirectoryScopedFormatLinter.php',
+      'FacebookArcanistConfiguration' => 'config/FacebookArcanistConfiguration.php',
+      'FacebookFbcodeLintEngine' => 'lint_engine/FacebookFbcodeLintEngine.php',
+      'FacebookFbcodeUnitTestEngine' => 'unit_engine/FacebookFbcodeUnitTestEngine.php',
+      'FacebookHowtoevenLintEngine' => 'lint_engine/FacebookHowtoevenLintEngine.php',
+      'FacebookHowtoevenLinter' => 'cpp_linter/FacebookHowtoevenLinter.php',
+      'FbcodeClangFormatLinter' => 'cpp_linter/FbcodeClangFormatLinter.php',
+      'FbcodeCppLinter' => 'cpp_linter/FbcodeCppLinter.php',
+    ),
+    'function' =>
+    array(
+    ),
+    'xmap' =>
+    array(
+      'ArcanistCpplintLinter' => 'ArcanistLinter',
+      'BaseDirectoryScopedFormatLinter' => 'ArcanistLinter',
+      'FacebookArcanistConfiguration' => 'ArcanistConfiguration',
+      'FacebookFbcodeLintEngine' => 'ArcanistLintEngine',
+      'FacebookFbcodeUnitTestEngine' => 'ArcanistBaseUnitTestEngine',
+      'FacebookHowtoevenLintEngine' => 'ArcanistLintEngine',
+      'FacebookHowtoevenLinter' => 'ArcanistLinter',
+      'FbcodeClangFormatLinter' => 'BaseDirectoryScopedFormatLinter',
+      'FbcodeCppLinter' => 'ArcanistLinter',
+    ),
+  ));
+} else {
+  phutil_register_library_map(array(
+    '__library_version__' => 2,
+    'class' =>
+    array(
+      'ArcanistCpplintLinter' => 'cpp_linter/ArcanistCpplintLinter.php',
+      'BaseDirectoryScopedFormatLinter' => 'cpp_linter/BaseDirectoryScopedFormatLinter.php',
+      'FacebookArcanistConfiguration' => 'config/FacebookOldArcanistConfiguration.php',
+      'FacebookFbcodeLintEngine' => 'lint_engine/FacebookFbcodeLintEngine.php',
+      'FacebookFbcodeUnitTestEngine' => 'unit_engine/FacebookOldFbcodeUnitTestEngine.php',
+      'FacebookHowtoevenLintEngine' => 'lint_engine/FacebookHowtoevenLintEngine.php',
+      'FacebookHowtoevenLinter' => 'cpp_linter/FacebookHowtoevenLinter.php',
+      'FbcodeClangFormatLinter' => 'cpp_linter/FbcodeClangFormatLinter.php',
+      'FbcodeCppLinter' => 'cpp_linter/FbcodeCppLinter.php',
+    ),
+    'function' =>
+    array(
+    ),
+    'xmap' =>
+    array(
+      'ArcanistCpplintLinter' => 'ArcanistLinter',
+      'BaseDirectoryScopedFormatLinter' => 'ArcanistLinter',
+      'FacebookArcanistConfiguration' => 'ArcanistConfiguration',
+      'FacebookFbcodeLintEngine' => 'ArcanistLintEngine',
+      'FacebookFbcodeUnitTestEngine' => 'ArcanistBaseUnitTestEngine',
+      'FacebookHowtoevenLintEngine' => 'ArcanistLintEngine',
+      'FacebookHowtoevenLinter' => 'ArcanistLinter',
+      'FbcodeClangFormatLinter' => 'BaseDirectoryScopedFormatLinter',
+      'FbcodeCppLinter' => 'ArcanistLinter',
+    ),
+  ));
+}
@@ -0,0 +1,43 @@
+<?php
+// Copyright 2004-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+require('RocksDBCommonHelper.php');
+
+define("DIFF_COMMAND", "diff");
+
+class FacebookArcanistConfiguration extends ArcanistConfiguration {
+  public function getCustomArgumentsForCommand($command) {
+    if ($command == "land") {
+      return array(
+          'async' => array('help' => 'Just to make tools happy'));
+    }
+    return array();
+  }
+
+  public function didRunWorkflow($command,
+                                 ArcanistWorkflow $workflow,
+                                 $error_code) {
+    // Default options don't terminate on failure, but that's what we want. In
+    // the current case we use assertions intentionally as "terminate on failure
+    // invariants".
+    assert_options(ASSERT_BAIL, true);
+
+    assert($workflow);
+    assert(strlen($command) > 0);
+
+    if ($command == DIFF_COMMAND && !$workflow->isRawDiffSource()) {
+      $diffID = $workflow->getDiffId();
+
+      // When submitting a diff this code path gets executed multiple times in
+      // a row. We only care about the case when ID for the diff is provided
+      // because that's what we need to apply the diff and trigger the tests.
+      if (strlen($diffID) > 0) {
+        assert(is_numeric($diffID));
+        startTestsInSandcastle(true /* $applyDiff */, $workflow, $diffID);
+      }
+    }
+  }
+}
@@ -0,0 +1,43 @@
+<?php
+// Copyright 2004-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+require('RocksDBCommonHelper.php');
+
+define("DIFF_COMMAND", "diff");
+
+class FacebookArcanistConfiguration extends ArcanistConfiguration {
+  public function getCustomArgumentsForCommand($command) {
+    if ($command == "land") {
+      return array(
+          'async' => array('help' => 'Just to make tools happy'));
+    }
+    return array();
+  }
+
+  public function didRunWorkflow($command,
+                                 ArcanistBaseWorkflow $workflow,
+                                 $error_code) {
+    // Default options don't terminate on failure, but that's what we want. In
+    // the current case we use assertions intentionally as "terminate on failure
+    // invariants".
+    assert_options(ASSERT_BAIL, true);
+
+    assert($workflow);
+    assert(strlen($command) > 0);
+
+    if ($command == DIFF_COMMAND && !$workflow->isRawDiffSource()) {
+      $diffID = $workflow->getDiffId();
+
+      // When submitting a diff this code path gets executed multiple times in
+      // a row. We only care about the case when ID for the diff is provided
+      // because that's what we need to apply the diff and trigger the tests.
+      if (strlen($diffID) > 0) {
+        assert(is_numeric($diffID));
+        startTestsInSandcastle(true /* $applyDiff */, $workflow, $diffID);
+      }
+    }
+  }
+}
@@ -1,8 +1,8 @@
 <?php
 // Copyright 2004-present Facebook. All Rights Reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.

 // Name of the environment variables which need to be set by the entity which
 // triggers continuous runs so that code at the end of the file gets executed
@@ -21,17 +21,11 @@ function postURL($diffID, $url) {
  assert(is_numeric($diffID));
  assert(strlen($url) > 0);

-  $cmd_args = array(
-    'diff_id' => (int)$diffID,
-    'name' => sprintf(
-      'click here for sandcastle tests for D%d',
-      (int)$diffID
-    ),
-    'link' => $url
-  );
-  $cmd = 'echo ' . escapeshellarg(json_encode($cmd_args))
-         . ' | arc call-conduit differential.updateunitresults';
-
+  $cmd = 'echo \'{"diff_id": ' . $diffID . ', '
+         . '"name":"click here for sandcastle tests for D' . $diffID . '", '
+         . '"link":"' . $url . '"}\' | '
+         . 'arc call-conduit '
+         . 'differential.updateunitresults';
  shell_exec($cmd);
 }

@@ -41,15 +35,11 @@ function buildUpdateTestStatusCmd($diffID, $test, $status) {
  assert(strlen($test) > 0);
  assert(strlen($status) > 0);

-  $cmd_args = array(
-    'diff_id' => (int)$diffID,
-    'name' => $test,
-    'result' => $status
-  );
-
-  $cmd = 'echo ' . escapeshellarg(json_encode($cmd_args))
-         . ' | arc call-conduit differential.updateunitresults';
-
+  $cmd = 'echo \'{"diff_id": ' . $diffID . ', '
+         . '"name":"' . $test . '", '
+         . '"result":"' . $status . '"}\' | '
+         . 'arc call-conduit '
+         . 'differential.updateunitresults';
  return $cmd;
 }

@@ -78,7 +68,7 @@ function getSteps($applyDiff, $diffID, $username, $test) {
    // and authenticate using that in Sandcastle.
    $setup = array(
      "name" => "Setup arcrc",
-      "shell" => "echo " . escapeshellarg($arcrc_content) . " | base64 --decode"
+      "shell" => "echo " . $arcrc_content . " | base64 --decode"
                 . " | gzip -d > ~/.arcrc",
      "user" => "root"
    );
@@ -104,17 +94,7 @@ function getSteps($applyDiff, $diffID, $username, $test) {
    "user" => "root"
  );

-  // This fixes "FATAL: ThreadSanitizer can not mmap the shadow memory"
-  // Source:
-  // https://github.com/google/sanitizers/wiki/ThreadSanitizerCppManual#FAQ
-  $fix_kernel_issue = array(
-    "name" => "Fix kernel issue with tsan",
-    "shell" => "echo 2 >/proc/sys/kernel/randomize_va_space",
-    "user" => "root"
-  );
-
  $steps[] = $fix_git_ignore;
-  $steps[] = $fix_kernel_issue;

  // This will be the command used to execute particular type of tests.
  $cmd = "";
@@ -124,7 +104,7 @@ function getSteps($applyDiff, $diffID, $username, $test) {
    $patch = array(
      "name" => "Patch " . $diffID,
      "shell" => "arc --arcrc-file ~/.arcrc "
-                  . "patch --nocommit --diff " . escapeshellarg($diffID),
+                  . "patch --nocommit --diff " . $diffID,
      "user" => "root"
    );

@@ -135,8 +115,8 @@ function getSteps($applyDiff, $diffID, $username, $test) {
  }

  // Run the actual command.
-  $cmd = $cmd . "J=$(nproc) ./build_tools/precommit_checker.py " .
-           escapeshellarg($test) . "; exit_code=$?; ";
+  $cmd = $cmd . "J=$(nproc) ./build_tools/precommit_checker.py " . $test
+           . "; exit_code=$?; ";

  if ($applyDiff) {
    $cmd = $cmd . "([[ \$exit_code -eq 0 ]] &&"
@@ -169,7 +149,7 @@ function getSteps($applyDiff, $diffID, $username, $test) {
    "name" => "Run " . $test,
    "shell" => $cmd,
    "user" => "root",
-    "parser" => "python build_tools/error_filter.py " . escapeshellarg($test),
+    "parser" => "python build_tools/error_filter.py " . $test,
  );

  $steps[] = $run_test;
@@ -217,7 +197,7 @@ function getSandcastleConfig() {
    if (file_exists(PRIMARY_TOKEN_FILE)) {
      $cmd = 'cat ' . PRIMARY_TOKEN_FILE;
    } else {
-      $cmd = 'cat ' . escapeshellarg($cwd_token_file);
+      $cmd = 'cat ' . $cwd_token_file;
    }

    assert(strlen($cmd) > 0);
@@ -261,7 +241,7 @@ function getSandcastleConfig() {
  // List of tests we want to run in Sandcastle.
  $tests = array("unit", "unit_non_shm", "unit_481", "clang_unit", "tsan",
                 "asan", "lite_test", "valgrind", "release", "release_481",
-                 "clang_release", "clang_analyze", "code_cov",
+                 "clang_release", "punit", "clang_analyze", "code_cov",
                 "java_build", "no_compression", "unity", "ubsan");

  $send_email_template = array(
@@ -341,11 +321,9 @@ function getSandcastleConfig() {
  $app = $sandcastle_config[0];
  $token = $sandcastle_config[1];

-  $cmd = 'curl -s -k '
-          . ' -F app=' . escapeshellarg($app)
-          . ' -F token=' . escapeshellarg($token)
-          . ' -F job=' . escapeshellarg(json_encode($job))
-          .' ' . escapeshellarg($url);
+  $cmd = 'curl -s -k -F app=' . $app . ' '
+          . '-F token=' . $token . ' -F job=\'' . json_encode($job)
+          .'\' "' . $url . '"';

  $output = shell_exec($cmd);
  assert(strlen($output) > 0);
@@ -0,0 +1,88 @@
+<?php
+
+/**
+ * Uses google's cpplint.py to check code. RocksDB team forked this file from
+ * phabricator's /src/lint/linter/ArcanistCpplintLinter.php, and customized it
+ * for its own use.
+ *
+ * You can get it here:
+ * http://google-styleguide.googlecode.com/svn/trunk/cpplint/cpplint.py
+ * @group linter
+ */
+final class ArcanistCpplintLinter extends ArcanistLinter {
+
+  public function willLintPaths(array $paths) {
+    return;
+  }
+
+  public function getLinterName() {
+    return 'cpplint.py';
+  }
+
+  public function getLintPath() {
+    $bin = 'cpplint.py';
+    // Search under current dir
+    list($err) = exec_manual('which %s/%s', $this->linterDir(), $bin);
+    if (!$err) {
+      return $this->linterDir().'/'.$bin;
+    }
+
+    // Look for globally installed cpplint.py
+    list($err) = exec_manual('which %s', $bin);
+    if ($err) {
+      throw new ArcanistUsageException(
+        "cpplint.py does not appear to be installed on this system. Install ".
+        "it (e.g., with 'wget \"http://google-styleguide.googlecode.com/".
+        "svn/trunk/cpplint/cpplint.py\"') ".
+        "in your .arcconfig to point to the directory where it resides. ".
+        "Also don't forget to chmod a+x cpplint.py!");
+    }
+
+    return $bin;
+  }
+
+  public function lintPath($path) {
+    $bin = $this->getLintPath();
+    $path = $this->rocksdbDir().'/'.$path;
+
+    $f = new ExecFuture("%C $path", $bin);
+
+    list($err, $stdout, $stderr) = $f->resolve();
+
+    if ($err === 2) {
+      throw new Exception("cpplint failed to run correctly:\n".$stderr);
+    }
+
+    $lines = explode("\n", $stderr);
+    $messages = array();
+    foreach ($lines as $line) {
+      $line = trim($line);
+      $matches = null;
+      $regex = '/^[^:]+:(\d+):\s*(.*)\s*\[(.*)\] \[(\d+)\]$/';
+      if (!preg_match($regex, $line, $matches)) {
+        continue;
+      }
+      foreach ($matches as $key => $match) {
+        $matches[$key] = trim($match);
+      }
+      $message = new ArcanistLintMessage();
+      $message->setPath($path);
+      $message->setLine($matches[1]);
+      $message->setCode($matches[3]);
+      $message->setName($matches[3]);
+      $message->setDescription($matches[2]);
+      $message->setSeverity(ArcanistLintSeverity::SEVERITY_WARNING);
+      $this->addLintMessage($message);
+    }
+  }
+
+  // The path of this linter
+  private function linterDir() {
+    return dirname(__FILE__);
+  }
+
+  // TODO(kaili) a quick and dirty way to figure out rocksdb's root dir.
+  private function rocksdbDir() {
+    return $this->linterDir()."/../..";
+  }
+}
@@ -0,0 +1,74 @@
+<?php
+// Copyright 2004-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+abstract class BaseDirectoryScopedFormatLinter extends ArcanistLinter {
+
+  const LINT_FORMATTING = 1;
+
+  private $changedLines = array();
+  private $rawLintOutput = array();
+
+  abstract protected function getPathsToLint();
+
+  protected function shouldLintPath($path) {
+    foreach ($this->getPathsToLint() as $p) {
+      // check if $path starts with $p
+      if (strncmp($path, $p, strlen($p)) === 0) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  // API to tell this linter which lines were changed
+  final public function setPathChangedLines($path, $changed) {
+    $this->changedLines[$path] = $changed;
+  }
+
+  final public function willLintPaths(array $paths) {
+    $futures = array();
+    foreach ($paths as $path) {
+      if (!$this->shouldLintPath($path)) {
+        continue;
+      }
+
+      $changed = $this->changedLines[$path];
+      if (!isset($changed)) {
+        // do not run linter if there are no changes
+        continue;
+      }
+
+      $futures[$path] = $this->getFormatFuture($path, $changed);
+    }
+
+    foreach (id(new FutureIterator($futures))->limit(8) as $p => $f) {
+      $this->rawLintOutput[$p] = $f->resolvex();
+    }
+  }
+
+  abstract protected function getFormatFuture($path, array $changed);
+  abstract protected function getLintMessage($diff);
+
+  final public function lintPath($path) {
+    if (!isset($this->rawLintOutput[$path])) {
+      return;
+    }
+
+    list($new_content) = $this->rawLintOutput[$path];
+    $old_content = $this->getData($path);
+
+    if ($new_content != $old_content) {
+      $diff = ArcanistDiffUtils::renderDifferences($old_content, $new_content);
+      $this->raiseLintAtOffset(
+        0,
+        self::LINT_FORMATTING,
+        $this->getLintMessage($diff),
+        $old_content,
+        $new_content);
+    }
+  }
+
+}
@@ -0,0 +1,223 @@
+<?php
+// Copyright 2015-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+final class FacebookHowtoevenLinter extends ArcanistLinter {
+
+  const VERSION = 'fd9192f324c36d28136d14380f0b552a1385b59b';
+
+  private $parsedTargets = array();
+
+  public function getLinterName() {
+    return 'Howtoeven';
+  }
+
+  protected function getSeverity($code) {
+    $severities = array(
+      ArcanistLintSeverity::SEVERITY_DISABLED,
+      ArcanistLintSeverity::SEVERITY_ADVICE,
+      ArcanistLintSeverity::SEVERITY_WARNING,
+      ArcanistLintSeverity::SEVERITY_ERROR,
+    );
+    return idx($severities, $code, ArcanistLintSeverity::SEVERITY_WARNING);
+  }
+
+  public function willLintPaths(array $paths) {
+    // Cleanup previous runs.
+    $this->localExecx("rm -rf _build/_lint");
+
+    // Build compilation database.
+    $lintable_paths = $this->getLintablePaths($paths);
+    $interesting_paths = $this->getInterestingPaths($lintable_paths);
+
+    if (!$lintable_paths) {
+      return;
+    }
+
+    // Run lint.
+    try {
+      $this->localExecx(
+        "%C %C -p _build/dev/ %Ls",
+        $this->getBinaryPath(),
+        $this->getFilteredIssues(),
+        $lintable_paths);
+    } catch (CommandException $exception) {
+      PhutilConsole::getConsole()->writeErr($exception->getMessage());
+    }
+
+    // Load results.
+    $result = id(
+      new SQLite3(
+        $this->getProjectRoot().'/_build/_lint/lint.db',
+        SQLITE3_OPEN_READONLY))
+      ->query("SELECT * FROM raised_issues");
+
+    while ($issue = $result->fetchArray(SQLITE3_ASSOC)) {
+      // Skip issues not part of the linted file.
+      if (in_array($issue['file'], $interesting_paths)) {
+        $this->addLintMessage(id(new ArcanistLintMessage())
+          ->setPath($issue['file'])
+          ->setLine($issue['line'])
+          ->setChar($issue['column'])
+          ->setCode('Howtoeven')
+          ->setSeverity($this->getSeverity($issue['severity']))
+          ->setName('Hte-'.$issue['name'])
+          ->setDescription(
+            sprintf(
+              "%s\n\n%s",
+              ($issue['message']) ? $issue['message'] : $issue['description'],
+              $issue['explanation']))
+          ->setOriginalText(idx($issue, 'original', ''))
+          ->setReplacementText(idx($issue, 'replacement', '')));
+      }
+    }
+  }
+
+  public function lintPath($path) {
+  }
+
+  /**
+   * Get the paths that we know how to lint.
+   *
+   * The strategy is to first look whether there's an existing compilation
+   * database and use that if it's exhaustive. We generate our own only if
+   * necessary.
+   */
+  private function getLintablePaths($paths) {
+    // Replace headers with existing sources.
+    for ($i = 0; $i < count($paths); $i++) {
+      if (preg_match("/\.h$/", $paths[$i])) {
+        $header = preg_replace("/\.h$/", ".cpp", $paths[$i]);
+        if (file_exists($header)) {
+          $paths[$i] = $header;
+        }
+      }
+    }
+
+    // Check if database exists and is exhaustive.
+    $available_paths = $this->getAvailablePaths();
+    $lintable_paths = array_intersect($paths, $available_paths);
+    if ($paths === $lintable_paths) {
+      return $lintable_paths;
+    }
+
+    // Generate our own database.
+    $targets = $this->getTargetsFor($paths);
+    if (!$targets) {
+      PhutilConsole::getConsole()->writeErr(
+        "No build targets found for %s\n",
+        implode(', ', $paths));
+      return array();
+    }
+
+    $this->localExecx("./tools/build/bin/fbconfig.par -r %Ls", $targets);
+    $this->localExecx("./tools/build/bin/fbmake.par gen_cdb");
+
+    $available_paths = $this->getAvailablePaths();
+    $lintable_paths = array_intersect($paths, $available_paths);
+    if ($paths != $lintable_paths) {
+      PhutilConsole::getConsole()->writeErr(
+        "Can't lint %s\n",
+        implode(', ', array_diff($paths, $available_paths)));
+    }
+
+    // Return what we know how to lint.
+    return $lintable_paths;
+  }
+
+  /**
+   * Get the available paths in the current compilation database.
+   */
+  private function getAvailablePaths() {
+    $database_path = $this->getProjectRoot()
+      .'/_build/dev/compile_commands.json';
+    if (!file_exists($database_path)) {
+      return array();
+    }
+
+    $entries = json_decode(file_get_contents($database_path), true);
+    $paths = array();
+    foreach ($entries as $entry) {
+      $paths[] = $entry['file'];
+    }
+    return $paths;
+  }
+
+  /**
+   * Search for the targets directories for the given files.
+   */
+  private static function getTargetsFor($paths) {
+    $targets = array();
+    foreach ($paths as $path) {
+      while (($path = dirname($path)) !== '.') {
+        if (in_array('TARGETS', scandir($path))) {
+          $contents = file_get_contents($path.'/TARGETS');
+          if (strpos($contents, 'cpp_binary') !== false) {
+            $targets[] = $path;
+            break;
+          }
+        }
+      }
+    }
+    return array_unique($targets);
+  }
+
+  /**
+   * The paths that we actually want to report on.
+   */
+  private function getInterestingPaths($paths) {
+    $headers = array();
+    foreach ($paths as $path) {
+      $headers[] = preg_replace("/\.cpp$/", ".h", $path);
+    }
+    return array_merge($paths, $headers);
+  }
+
+  /**
+   * The path where the binary is located. Will return the current dewey binary
+   * unless the `HOWTOEVEN_BUILD` environment variable is set.
+   */
+  private function getBinaryPath() {
+    $path = sprintf(
+      "/mnt/dewey/fbcode/.commits/%s/builds/howtoeven/client",
+      self::VERSION);
+
+    $build = getenv('HOWTOEVEN_BUILD');
+    if ($build) {
+      $path = sprintf(
+        "./_build/%s/tools/howtoeven/client",
+        $build);
+      if (!file_exists($path)) {
+        PhutilConsole::getConsole()->writeErr(">> %s does not exist\n", $path);
+        exit(1);
+      }
+    }
+
+    return $path;
+  }
+
+  /**
+   * Execute the command in the root directory.
+   */
+  private function localExecx($command /* , ... */) {
+    $arguments = func_get_args();
+    return newv('ExecFuture', $arguments)
+      ->setCWD($this->getProjectRoot())
+      ->resolvex();
+  }
+
+  /**
+   * The root of the project.
+   */
+  private function getProjectRoot() {
+    return $this->getEngine()->getWorkingCopy()->getProjectRoot();
+  }
+
+  private function getFilteredIssues() {
+    $issues = getenv('HOWTOEVEN_ISSUES');
+    return ($issues) ? csprintf('-issues %s', $issues) : '';
+  }
+
+}
@@ -0,0 +1,58 @@
+<?php
+// Copyright 2004-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+final class FbcodeClangFormatLinter extends BaseDirectoryScopedFormatLinter {
+
+  const LINT_FORMATTING = 1;
+  const CLANG_FORMAT_BINARY = '/mnt/vol/engshare/admin/scripts/clang-format';
+
+  protected function getPathsToLint() {
+    return array('');
+  }
+
+  public function getLinterName() {
+    return 'CLANG_FORMAT';
+  }
+
+  public function getLintSeverityMap() {
+    return array(
+      self::LINT_FORMATTING => ArcanistLintSeverity::SEVERITY_ADVICE,
+    );
+  }
+
+  public function getLintNameMap() {
+    return array(
+      self::LINT_FORMATTING => pht('Changes are not clang-formatted'),
+    );
+  }
+
+  protected function getFormatFuture($path, array $changed) {
+    $args = "";
+    foreach ($changed as $key => $value) {
+      $args .= " --lines=$key:$key";
+    }
+
+    $binary = self::CLANG_FORMAT_BINARY;
+    if (!file_exists($binary)) {
+      // trust the $PATH
+      $binary = "clang-format";
+    }
+
+    return new ExecFuture(
+      "%s %s $args",
+      $binary,
+      $this->getEngine()->getFilePathOnDisk($path));
+  }
+
+  protected function getLintMessage($diff) {
+    $link_to_clang_format =
+      "[[ http://fburl.com/clang-format | clang-format ]]";
+    return <<<LINT_MSG
+Changes in this file were not formatted using $link_to_clang_format.
+Please run build_tools/format-diff.sh or `make format`
+LINT_MSG;
+  }
+}
@@ -0,0 +1,126 @@
+<?php
+// Copyright 2004-present Facebook.  All rights reserved.
+
+class FbcodeCppLinter extends ArcanistLinter {
+  const FLINT      = "/home/engshare/tools/flint";
+  const LINT_ERROR   = 1;
+  const LINT_WARNING = 2;
+  const LINT_ADVICE  = 3;
+  const C_FLAG = "--c_mode=true";
+
+  private $rawLintOutput = array();
+
+  public function willLintPaths(array $paths) {
+    if (!file_exists(self::FLINT)) {
+      return;
+    }
+    $futures = array();
+    foreach ($paths as $p) {
+      $lpath = $this->getEngine()->getFilePathOnDisk($p);
+      $lpath_file = file($lpath);
+      if (preg_match('/\.(c)$/', $lpath) ||
+          preg_match('/-\*-.*Mode: C[; ].*-\*-/', $lpath_file[0]) ||
+          preg_match('/vim(:.*)*:\s*(set\s+)?filetype=c\s*:/', $lpath_file[0])
+          ) {
+        $futures[$p] = new ExecFuture("%s %s %s 2>&1",
+                           self::FLINT, self::C_FLAG,
+                           $this->getEngine()->getFilePathOnDisk($p));
+      } else {
+        $futures[$p] = new ExecFuture("%s %s 2>&1",
+          self::FLINT, $this->getEngine()->getFilePathOnDisk($p));
+      }
+    }
+
+    foreach (Futures($futures)->limit(8) as $p => $f) {
+      $this->rawLintOutput[$p] = $f->resolvex();
+    }
+
+    return;
+  }
+
+  public function getLinterName() {
+    return "FBCPP";
+  }
+
+  public function lintPath($path) {
+    $this->runCppLint($path);
+  }
+
+  private function runCppLint($path) {
+    $msgs = $this->getCppLintOutput($path);
+    foreach ($msgs as $m) {
+      $this->raiseLintAtLine($m['line'], 0, $m['severity'], $m['msg']);
+    }
+  }
+
+  private function adviseOnEachPattern(
+    $path,
+    $regex,
+    $message,
+    $lint_type = self::LINT_ADVICE,
+    $match_idx = 0) {
+      $file_data = $this->getData($path);
+      $matches = array();
+      if (!preg_match_all($regex, $file_data, $matches, PREG_OFFSET_CAPTURE)) {
+        return;
+      }
+
+      foreach ($matches[$match_idx] as $match) {
+        list($match_str, $offset) = $match;
+        $this->raiseLintAtOffset($offset, $lint_type, $message, $match_str);
+      }
+  }
+
+  public function getLintSeverityMap() {
+    return array(
+      self::LINT_WARNING => ArcanistLintSeverity::SEVERITY_WARNING,
+      self::LINT_ADVICE  => ArcanistLintSeverity::SEVERITY_ADVICE,
+      self::LINT_ERROR   => ArcanistLintSeverity::SEVERITY_ERROR
+    );
+  }
+
+  public function getLintNameMap() {
+    return array(
+      self::LINT_ADVICE   => "CppLint Advice",
+      self::LINT_WARNING  => "CppLint Warning",
+      self::LINT_ERROR    => "CppLint Error"
+    );
+  }
+
+  private function getCppLintOutput($path) {
+     if (!array_key_exists($path, $this->rawLintOutput)) {
+       return array();
+     }
+    list($output) = $this->rawLintOutput[$path];
+
+    $msgs = array();
+    $current = null;
+    $matches = array();
+    foreach (explode("\n", $output) as $line) {
+      if (preg_match('/.*?:(\d+):(.*)/', $line, $matches)) {
+        if ($current) {
+          $msgs[] = $current;
+        }
+        $line = $matches[1];
+        $text = $matches[2];
+        if (preg_match('/.*Warning.*/', $text)) {
+          $sev = self::LINT_WARNING;
+        } else if (preg_match('/.*Advice.*/', $text)) {
+          $sev = self::LINT_ADVICE;
+        } else {
+          $sev = self::LINT_ERROR;
+        }
+        $current = array('line'     => $line,
+                         'msg'      => $text,
+                         'severity' => $sev);
+      } else if ($current) {
+        $current['msg'] .= ' ' . $line;
+      }
+    }
+    if ($current) {
+      $msgs[] = $current;
+    }
+
+    return $msgs;
+  }
+}
@@ -0,0 +1,138 @@
+<?php
+// Copyright 2004-present Facebook.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+class FacebookFbcodeLintEngine extends ArcanistLintEngine {
+
+  public function buildLinters() {
+    $linters = array();
+    $paths = $this->getPaths();
+
+    // Remove all deleted files, which are not checked by the
+    // following linters.
+    foreach ($paths as $key => $path) {
+      if (!Filesystem::pathExists($this->getFilePathOnDisk($path))) {
+        unset($paths[$key]);
+      }
+    }
+
+    $generated_linter = new ArcanistGeneratedLinter();
+    $linters[] = $generated_linter;
+
+    $nolint_linter = new ArcanistNoLintLinter();
+    $linters[] = $nolint_linter;
+
+    $text_linter = new ArcanistTextLinter();
+    $text_linter->setCustomSeverityMap(array(
+      ArcanistTextLinter::LINT_LINE_WRAP
+        => ArcanistLintSeverity::SEVERITY_ADVICE,
+    ));
+    $linters[] = $text_linter;
+
+    $java_text_linter = new ArcanistTextLinter();
+    $java_text_linter->setMaxLineLength(100);
+    $java_text_linter->setCustomSeverityMap(array(
+      ArcanistTextLinter::LINT_LINE_WRAP
+        => ArcanistLintSeverity::SEVERITY_ADVICE,
+    ));
+    $linters[] = $java_text_linter;
+
+    $python_linter = new ArcanistPEP8Linter();
+    $linters[] = $python_linter;
+
+    $cpp_linters = array();
+    $cpp_linters[] = $linters[] = new ArcanistCpplintLinter();
+    $cpp_linters[] = $linters[] = new FbcodeCppLinter();
+
+    $clang_format_linter = new FbcodeClangFormatLinter();
+    $linters[] = $clang_format_linter;
+
+    $spelling_linter = new ArcanistSpellingLinter();
+    $linters[] = $spelling_linter;
+
+    foreach ($paths as $path) {
+      $is_text = false;
+
+      $text_extensions = (
+        '/\.('.
+        'cpp|cxx|c|cc|h|hpp|hxx|tcc|'.
+        'py|rb|hs|pl|pm|tw|'.
+        'php|phpt|css|js|'.
+        'java|'.
+        'thrift|'.
+        'lua|'.
+        'siv|'.
+        'txt'.
+        ')$/'
+      );
+      if (preg_match($text_extensions, $path)) {
+        $is_text = true;
+      }
+      if ($is_text) {
+        $nolint_linter->addPath($path);
+
+        $generated_linter->addPath($path);
+        $generated_linter->addData($path, $this->loadData($path));
+
+        if (preg_match('/\.java$/', $path)) {
+          $java_text_linter->addPath($path);
+          $java_text_linter->addData($path, $this->loadData($path));
+        } else {
+          $text_linter->addPath($path);
+          $text_linter->addData($path, $this->loadData($path));
+        }
+
+        $spelling_linter->addPath($path);
+        $spelling_linter->addData($path, $this->loadData($path));
+      }
+      if (preg_match('/\.(cpp|c|cc|cxx|h|hh|hpp|hxx|tcc)$/', $path)
+          && !preg_match('/third-party/', $path)) {
+        foreach ($cpp_linters as &$linter) {
+          $linter->addPath($path);
+          $linter->addData($path, $this->loadData($path));
+        }
+
+        $clang_format_linter->addPath($path);
+        $clang_format_linter->addData($path, $this->loadData($path));
+        $clang_format_linter->setPathChangedLines(
+          $path, $this->getPathChangedLines($path));
+      }
+
+      // Match *.py and contbuild config files
+      if (preg_match('/(\.(py|tw|smcprops)|^contbuild\/configs\/[^\/]*)$/',
+                    $path)) {
+        $space_count = 4;
+        $real_path = $this->getFilePathOnDisk($path);
+        $dir = dirname($real_path);
+        do {
+          if (file_exists($dir.'/.python2space')) {
+            $space_count = 2;
+            break;
+          }
+          $dir = dirname($dir);
+        } while ($dir != '/' && $dir != '.');
+
+        $cur_path_linter = $python_linter;
+        $cur_path_linter->addPath($path);
+        $cur_path_linter->addData($path, $this->loadData($path));
+
+        if (preg_match('/\.tw$/', $path)) {
+          $cur_path_linter->setCustomSeverityMap(array(
+            'E251' => ArcanistLintSeverity::SEVERITY_DISABLED,
+          ));
+        }
+      }
+    }
+
+    $name_linter = new ArcanistFilenameLinter();
+    $linters[] = $name_linter;
+    foreach ($paths as $path) {
+      $name_linter->addPath($path);
+    }
+
+    return $linters;
+  }
+
+}
@@ -0,0 +1,27 @@
+<?php
+// Copyright 2015-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+final class FacebookHowtoevenLintEngine extends ArcanistLintEngine {
+
+  public function buildLinters() {
+    $paths = array();
+
+    foreach ($this->getPaths() as $path) {
+      // Don't try to lint deleted files or changed directories.
+      if (!Filesystem::pathExists($path) || is_dir($path)) {
+        continue;
+      }
+
+      if (preg_match('/\.(cpp|c|cc|cxx|h|hh|hpp|hxx|tcc)$/', $path)) {
+        $paths[] = $path;
+      }
+    }
+
+    $howtoeven = new FacebookHowtoevenLinter();
+    $howtoeven->setPaths($paths);
+    return array($howtoeven);
+  }
+}
@@ -0,0 +1,17 @@
+<?php
+// Copyright 2004-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+class FacebookFbcodeUnitTestEngine extends ArcanistUnitTestEngine {
+
+  public function run() {
+      // For a call to `arc call-conduit differential.updateunitresults` to
+      // succeed we need at least one entry here.
+      $result = new ArcanistUnitTestResult();
+      $result->setName("dummy_placeholder_entry");
+      $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
+      return array($result);
+  }
+}
@@ -0,0 +1,17 @@
+<?php
+// Copyright 2004-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+class FacebookFbcodeUnitTestEngine extends ArcanistBaseUnitTestEngine {
+
+  public function run() {
+      // For a call to `arc call-conduit differential.updateunitresults` to
+      // succeed we need at least one entry here.
+      $result = new ArcanistUnitTestResult();
+      $result->setName("dummy_placeholder_entry");
+      $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
+      return array($result);
+  }
+}
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 # Create a tmp directory for the test to use
 TEST_DIR=$(mktemp -d /dev/shm/fbcode_rocksdb_XXXXXXX)
 TEST_TMPDIR="$TEST_DIR" $@ && rm -rf "$TEST_DIR"
@@ -6,10 +6,12 @@ rocksdb_target_header = """
 import os

 TARGETS_PATH = os.path.dirname(__file__)
-REPO_PATH = "rocksdb/src/"
+REPO_PATH = TARGETS_PATH[(TARGETS_PATH.find('fbcode/') + len('fbcode/')):] + "/"
 BUCK_BINS = "buck-out/gen/" + REPO_PATH
 TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh"
 rocksdb_compiler_flags = [
+  "-msse",
+  "-msse4.2",
  "-fno-builtin-memcmp",
  "-DROCKSDB_PLATFORM_POSIX",
  "-DROCKSDB_LIB_IO_POSIX",
@@ -18,6 +20,7 @@ rocksdb_compiler_flags = [
  "-DROCKSDB_RANGESYNC_PRESENT",
  "-DROCKSDB_SCHED_GETCPU_PRESENT",
  "-DROCKSDB_SUPPORT_THREAD_LOCAL",
+  "-DHAVE_SSE42",
  "-DOS_LINUX",
  # Flags to enable libs we include
  "-DSNAPPY",
@@ -49,10 +52,6 @@ rocksdb_preprocessor_flags = [
  "-I" + REPO_PATH + "include/",
  "-I" + REPO_PATH,
 ]
-
-rocksdb_arch_preprocessor_flags = {
-  "x86_64": ["-DHAVE_SSE42"],
-}
 """


@@ -63,7 +62,6 @@ cpp_library(
    srcs = [%s],
    deps = [%s],
    preprocessor_flags = rocksdb_preprocessor_flags,
-    arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
    compiler_flags = rocksdb_compiler_flags,
    external_deps = rocksdb_external_deps,
 )
@@ -75,7 +73,6 @@ cpp_binary(
  srcs = [%s],
  deps = [%s],
  preprocessor_flags = rocksdb_preprocessor_flags,
-  arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
  compiler_flags = rocksdb_compiler_flags,
  external_deps = rocksdb_external_deps,
 )
@@ -98,7 +95,6 @@ for test_cfg in ROCKS_TESTS:
      srcs = [test_cc],
      deps = [":rocksdb_test_lib"],
      preprocessor_flags = rocksdb_preprocessor_flags,
-      arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
      compiler_flags = rocksdb_compiler_flags,
      external_deps = rocksdb_external_deps,
    )
@@ -317,11 +317,9 @@ EOF
    # Test whether jemalloc is available
    if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null -ljemalloc \
      2>/dev/null; then
-        # This will enable some preprocessor identifiers in the Makefile
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ljemalloc"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -ljemalloc"
        JEMALLOC=1
-        # JEMALLOC can be enabled either using the flag (like here) or by
-        # providing direct link to the jemalloc library
-        WITH_JEMALLOC_FLAG=1
    else
        # jemalloc is not available. Let's try tcmalloc
        if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null \
@@ -458,7 +456,7 @@ elif test -z "$PORTABLE"; then
  fi
 fi

-$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
+$CXX $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
  #include <cstdint>
  #include <nmmintrin.h>
  int main() {
@@ -519,14 +517,7 @@ echo "ROCKSDB_PATCH=$ROCKSDB_PATCH" >> "$OUTPUT"
 echo "CLANG_SCAN_BUILD=$CLANG_SCAN_BUILD" >> "$OUTPUT"
 echo "CLANG_ANALYZER=$CLANG_ANALYZER" >> "$OUTPUT"
 echo "PROFILING_FLAGS=$PROFILING_FLAGS" >> "$OUTPUT"
-# This will enable some related identifiers for the preprocessor
 if test -n "$JEMALLOC"; then
  echo "JEMALLOC=1" >> "$OUTPUT"
 fi
-# Indicates that jemalloc should be enabled using -ljemalloc flag
-# The alternative is to porvide a direct link to the library via JEMALLOC_LIB
-# and JEMALLOC_INCLUDE
-if test -n "$WITH_JEMALLOC_FLAG"; then
-  echo "WITH_JEMALLOC_FLAG=$WITH_JEMALLOC_FLAG" >> "$OUTPUT"
-fi
 echo "LUA_PATH=$LUA_PATH" >> "$OUTPUT"
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 #
 # Copyright (c) 2016, Facebook. All rights reserved.
 #
@@ -67,7 +67,7 @@ function update_repo_status {
 #

 # Path to the determinator from the root of the RocksDB repo.
-CONTRUN_DETERMINATOR=./build_tools/RocksDBCommonHelper.php
+CONTRUN_DETERMINATOR=./arcanist_util/config/RocksDBCommonHelper.php

 # Value of the previous commit.
 PREV_COMMIT=
@@ -1,2 +1,2 @@
-#!/usr/bin/env bash
+#!/bin/bash
 docker run -v $PWD:/rocks -w /rocks buildpack-deps make
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 # If clang_format_diff.py command is not specfied, we assume we are able to
 # access directly without any path.
 if [ -z $CLANG_FORMAT_DIFF ]
@@ -0,0 +1,53 @@
+#!/bin/bash
+#  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+#  This source code is licensed under the BSD-style license found in the
+#  LICENSE file in the root directory of this source tree. An additional grant
+#  of patent rights can be found in the PATENTS file in the same directory.
+
+set -e
+if [ -z "$GIT" ]
+then
+  GIT="git"
+fi
+
+# Print out the colored progress info so that it can be brainlessly
+# distinguished by users.
+function title() {
+  echo -e "\033[1;32m$*\033[0m"
+}
+
+usage="Create new RocksDB version and prepare it for the release process\n"
+usage+="USAGE: ./make_new_version.sh <version> [<remote>]\n"
+usage+="  version: specify a version without '.fb' suffix (e.g. 5.4).\n"
+usage+="  remote: name of the remote to push the branch to (default: origin)."
+
+# -- Pre-check
+if [[ $# < 1 ]]; then
+  echo -e $usage
+  exit 1
+fi
+
+ROCKSDB_VERSION=$1
+
+REMOTE="origin"
+if [[ $# > 1 ]]; then
+  REMOTE=$2
+fi
+
+GIT_BRANCH=`git rev-parse --abbrev-ref HEAD`
+echo $GIT_BRANCH
+
+if [ $GIT_BRANCH != "master" ]; then
+  echo "Error: Current branch is '$GIT_BRANCH', Please switch to master branch."
+  exit 1
+fi
+
+title "Adding new tag for this release ..."
+BRANCH="$ROCKSDB_VERSION.fb"
+$GIT checkout -b $BRANCH
+
+# Setting up the proxy for remote repo access
+title "Pushing new branch to remote repo ..."
+git push $REMOTE --set-upstream $BRANCH
+
+title "Branch $BRANCH is pushed to github;"
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash

 set -e

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 # This script is executed by Sandcastle
 # to determine next steps to run

@@ -564,7 +564,7 @@ TSAN_CRASH_TEST_COMMANDS="[
            {
                'name':'Compile and run',
                'timeout': 86400,
-                'shell':'set -o pipefail && $SHM $DEBUG $TSAN CRASH_TEST_KILL_ODD=1887 CRASH_TEST_EXT_ARGS=--log2_keys_per_lock=22  make J=1 crash_test || $CONTRUN_NAME=tsan_crash_test $TASK_CREATION_TOOL',
+                'shell':'set -o pipefail && $SHM $DEBUG $TSAN CRASH_TEST_KILL_ODD=1887 make J=1 crash_test || $CONTRUN_NAME=tsan_crash_test $TASK_CREATION_TOOL',
                'user':'root',
                $PARSER
            },
@@ -583,7 +583,39 @@ run_format_compatible()
  rm -rf /dev/shm/rocksdb
  mkdir /dev/shm/rocksdb

-  tools/check_format_compatible.sh
+  echo '
+  if [ -e "build_tools/build_detect_platform" ]
+  then
+    sed "s/tcmalloc/nothingnothingnothing/g" build_tools/build_detect_platform > $TEST_TMPDIR/temp_build_file
+    rm -rf build_tools/build_detect_platform
+    cp $TEST_TMPDIR/temp_build_file build_tools/build_detect_platform
+    chmod +x build_tools/build_detect_platform
+  fi
+
+  if [ -e "build_detect_platform" ]
+  then
+    sed "s/tcmalloc/nothingnothingnothing/g" build_detect_platform > $TEST_TMPDIR/temp_build_file
+    rm -rf build_detect_platform
+    cp $TEST_TMPDIR/temp_build_file build_detect_platform
+    chmod +x build_detect_platform
+  fi
+
+  make ldb -j32
+
+  if [ -e "build_detect_platform" ]
+  then
+    git checkout -- build_detect_platform
+  fi
+
+  if [ -e "build_tools/build_detect_platform" ]
+  then
+    git checkout -- build_tools/build_detect_platform
+  fi
+  ' > temp_build_ldb.sh
+
+  sed "s/make ldb -j32/source temp_build_ldb.sh/g" tools/check_format_compatible.sh > tools/temp_check_format_compatible.sh
+  chmod +x tools/temp_check_format_compatible.sh
+  tools/temp_check_format_compatible.sh
 }

 FORMAT_COMPATIBLE_COMMANDS="[
@@ -27,11 +27,6 @@ std::shared_ptr<Cache> NewClockCache(size_t capacity, int num_shard_bits,
 #include <atomic>
 #include <deque>

-// "tbb/concurrent_hash_map.h" requires RTTI if exception is enabled.
-// Disable it so users can chooose to disable RTTI.
-#ifndef ROCKSDB_USE_RTTI
-#define TBB_USE_EXCEPTIONS 0
-#endif
 #include "tbb/concurrent_hash_map.h"

 #include "cache/sharded_cache.h"
@@ -22,7 +22,7 @@

 namespace rocksdb {

-LRUHandleTable::LRUHandleTable() : list_(nullptr), length_(0), elems_(0) {
+LRUHandleTable::LRUHandleTable() : length_(0), elems_(0), list_(nullptr) {
  Resize();
 }

@@ -100,7 +100,7 @@ void LRUHandleTable::Resize() {
 }

 LRUCacheShard::LRUCacheShard()
-    : high_pri_pool_usage_(0), usage_(0), lru_usage_(0) {
+    : usage_(0), lru_usage_(0), high_pri_pool_usage_(0) {
  // Make empty circular linked list
  lru_.next = &lru_;
  lru_.prev = &lru_;
@@ -157,16 +157,6 @@ void LRUCacheShard::TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri) {
  *lru_low_pri = lru_low_pri_;
 }

-size_t LRUCacheShard::TEST_GetLRUSize() {
-  LRUHandle* lru_handle = lru_.next;
-  size_t lru_size = 0;
-  while (lru_handle != &lru_) {
-    lru_size++;
-    lru_handle = lru_handle->next;
-  }
-  return lru_size;
-}
-
 void LRUCacheShard::LRU_Remove(LRUHandle* e) {
  assert(e->next != nullptr);
  assert(e->prev != nullptr);
@@ -233,22 +223,6 @@ void LRUCacheShard::EvictFromLRU(size_t charge,
  }
 }

-void* LRUCacheShard::operator new(size_t size) {
-  return port::cacheline_aligned_alloc(size);
-}
-
-void* LRUCacheShard::operator new[](size_t size) {
-  return port::cacheline_aligned_alloc(size);
-}
-
-void LRUCacheShard::operator delete(void *memblock) {
-  port::cacheline_aligned_free(memblock);
-}
-
-void LRUCacheShard::operator delete[](void* memblock) {
-  port::cacheline_aligned_free(memblock);
-}
-
 void LRUCacheShard::SetCapacity(size_t capacity) {
  autovector<LRUHandle*> last_reference_list;
  {
@@ -464,18 +438,11 @@ std::string LRUCacheShard::GetPrintableOptions() const {
 LRUCache::LRUCache(size_t capacity, int num_shard_bits,
                   bool strict_capacity_limit, double high_pri_pool_ratio)
    : ShardedCache(capacity, num_shard_bits, strict_capacity_limit) {
-  num_shards_ = 1 << num_shard_bits;
-#if defined(_MSC_VER)
-#pragma warning(push)
-#pragma warning(disable: 4316) // We've validated the alignment with the new operators
-#endif
-  shards_ = new LRUCacheShard[num_shards_];
-#if defined(_MSC_VER)
-#pragma warning(pop)
-#endif
+  int num_shards = 1 << num_shard_bits;
+  shards_ = new LRUCacheShard[num_shards];
  SetCapacity(capacity);
  SetStrictCapacityLimit(strict_capacity_limit);
-  for (int i = 0; i < num_shards_; i++) {
+  for (int i = 0; i < num_shards; i++) {
    shards_[i].SetHighPriorityPoolRatio(high_pri_pool_ratio);
  }
 }
@@ -502,20 +469,7 @@ uint32_t LRUCache::GetHash(Handle* handle) const {
  return reinterpret_cast<const LRUHandle*>(handle)->hash;
 }

-void LRUCache::DisownData() {
-// Do not drop data if compile with ASAN to suppress leak warning.
-#ifndef __SANITIZE_ADDRESS__
-  shards_ = nullptr;
-#endif  // !__SANITIZE_ADDRESS__
-}
-
-size_t LRUCache::TEST_GetLRUSize() {
-  size_t lru_size_of_all_shards = 0;
-  for (int i = 0; i < num_shards_; i++) {
-    lru_size_of_all_shards += shards_[i].TEST_GetLRUSize();
-  }
-  return lru_size_of_all_shards;
-}
+void LRUCache::DisownData() { shards_ = nullptr; }

 std::shared_ptr<Cache> NewLRUCache(size_t capacity, int num_shard_bits,
                                   bool strict_capacity_limit,
@@ -148,13 +148,13 @@ class LRUHandleTable {

  // The table consists of an array of buckets where each bucket is
  // a linked list of cache entries that hash into the bucket.
-  LRUHandle** list_;
  uint32_t length_;
  uint32_t elems_;
+  LRUHandle** list_;
 };

 // A single shard of sharded cache.
-class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard : public CacheShard {
+class LRUCacheShard : public CacheShard {
 public:
  LRUCacheShard();
  virtual ~LRUCacheShard();
@@ -198,19 +198,6 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard : public CacheShard {

  void TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri);

-  //  Retrieves number of elements in LRU, for unit test purpose only
-  //  not threadsafe
-  size_t TEST_GetLRUSize();
-
-  // Overloading to aligned it to cache line size
-  void* operator new(size_t);
-
-  void* operator new[](size_t);
-
-  void operator delete(void *);
-
-  void operator delete[](void*);
-
 private:
  void LRU_Remove(LRUHandle* e);
  void LRU_Insert(LRUHandle* e);
@@ -232,6 +219,12 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard : public CacheShard {
  // Initialized before use.
  size_t capacity_;

+  // Memory size for entries residing in the cache
+  size_t usage_;
+
+  // Memory size for entries residing only in the LRU list
+  size_t lru_usage_;
+
  // Memory size for entries in high-pri pool.
  size_t high_pri_pool_usage_;

@@ -245,6 +238,11 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard : public CacheShard {
  // Remember the value to avoid recomputing each time.
  double high_pri_pool_capacity_;

+  // mutex_ protects the following state.
+  // We don't count mutex_ as the cache's internal state so semantically we
+  // don't mind mutex_ invoking the non-const actions.
+  mutable port::Mutex mutex_;
+
  // Dummy head of LRU list.
  // lru.prev is newest entry, lru.next is oldest entry.
  // LRU contains items which can be evicted, ie reference only by cache
@@ -253,29 +251,7 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard : public CacheShard {
  // Pointer to head of low-pri pool in LRU list.
  LRUHandle* lru_low_pri_;

-  // ------------^^^^^^^^^^^^^-----------
-  // Not frequently modified data members
-  // ------------------------------------
-  //
-  // We separate data members that are updated frequently from the ones that
-  // are not frequently updated so that they don't share the same cache line
-  // which will lead into false cache sharing
-  //
-  // ------------------------------------
-  // Frequently modified data members
-  // ------------vvvvvvvvvvvvv-----------
  LRUHandleTable table_;
-
-  // Memory size for entries residing in the cache
-  size_t usage_;
-
-  // Memory size for entries residing only in the LRU list
-  size_t lru_usage_;
-
-  // mutex_ protects the following state.
-  // We don't count mutex_ as the cache's internal state so semantically we
-  // don't mind mutex_ invoking the non-const actions.
-  mutable port::Mutex mutex_;
 };

 class LRUCache : public ShardedCache {
@@ -291,12 +267,8 @@ class LRUCache : public ShardedCache {
  virtual uint32_t GetHash(Handle* handle) const override;
  virtual void DisownData() override;

-  //  Retrieves number of elements in LRU, for unit test purpose only
-  size_t TEST_GetLRUSize();
-
 private:
  LRUCacheShard* shards_;
-  int num_shards_ = 0;
 };

 }  // namespace rocksdb
@@ -17,16 +17,7 @@ class LRUCacheTest : public testing::Test {
  ~LRUCacheTest() {}

  void NewCache(size_t capacity, double high_pri_pool_ratio = 0.0) {
-    cache_.reset(
-#if defined(_MSC_VER)
-#pragma warning(push)
-#pragma warning(disable: 4316) // We've validated the alignment with the new operators
-#endif
-      new LRUCacheShard()
-#if defined(_MSC_VER)
-#pragma warning(pop)
-#endif
-    );
+    cache_.reset(new LRUCacheShard());
    cache_->SetCapacity(capacity);
    cache_->SetStrictCapacityLimit(false);
    cache_->SetHighPriorityPoolRatio(high_pri_pool_ratio);
@@ -1,3 +0,0 @@
-@PACKAGE_INIT@
-include("${CMAKE_CURRENT_LIST_DIR}/RocksDBTargets.cmake")
-check_required_components(RocksDB)
@@ -1,21 +0,0 @@
-# - Find Bzip2
-# Find the bzip2 compression library and includes
-#
-# BZIP2_INCLUDE_DIR - where to find bzlib.h, etc.
-# BZIP2_LIBRARIES - List of libraries when using bzip2.
-# BZIP2_FOUND - True if bzip2 found.
-
-find_path(BZIP2_INCLUDE_DIR
-  NAMES bzlib.h
-  HINTS ${BZIP2_ROOT_DIR}/include)
-
-find_library(BZIP2_LIBRARIES
-  NAMES bz2
-  HINTS ${BZIP2_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(bzip2 DEFAULT_MSG BZIP2_LIBRARIES BZIP2_INCLUDE_DIR)
-
-mark_as_advanced(
-  BZIP2_LIBRARIES
-  BZIP2_INCLUDE_DIR)
@@ -1,21 +0,0 @@
-# - Find Lz4
-# Find the lz4 compression library and includes
-#
-# LZ4_INCLUDE_DIR - where to find lz4.h, etc.
-# LZ4_LIBRARIES - List of libraries when using lz4.
-# LZ4_FOUND - True if lz4 found.
-
-find_path(LZ4_INCLUDE_DIR
-  NAMES lz4.h
-  HINTS ${LZ4_ROOT_DIR}/include)
-
-find_library(LZ4_LIBRARIES
-  NAMES lz4
-  HINTS ${LZ4_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(lz4 DEFAULT_MSG LZ4_LIBRARIES LZ4_INCLUDE_DIR)
-
-mark_as_advanced(
-  LZ4_LIBRARIES
-  LZ4_INCLUDE_DIR)
@@ -1,21 +0,0 @@
-# - Find zlib
-# Find the zlib compression library and includes
-#
-# ZLIB_INCLUDE_DIR - where to find zlib.h, etc.
-# ZLIB_LIBRARIES - List of libraries when using zlib.
-# ZLIB_FOUND - True if zlib found.
-
-find_path(ZLIB_INCLUDE_DIR
-  NAMES zlib.h
-  HINTS ${ZLIB_ROOT_DIR}/include)
-
-find_library(ZLIB_LIBRARIES
-  NAMES z
-  HINTS ${ZLIB_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(zlib DEFAULT_MSG ZLIB_LIBRARIES ZLIB_INCLUDE_DIR)
-
-mark_as_advanced(
-  ZLIB_LIBRARIES
-  ZLIB_INCLUDE_DIR)
@@ -1,21 +0,0 @@
-# - Find zstd
-# Find the zstd compression library and includes
-#
-# ZSTD_INCLUDE_DIR - where to find zstd.h, etc.
-# ZSTD_LIBRARIES - List of libraries when using zstd.
-# ZSTD_FOUND - True if zstd found.
-
-find_path(ZSTD_INCLUDE_DIR
-  NAMES zstd.h
-  HINTS ${ZSTD_ROOT_DIR}/include)
-
-find_library(ZSTD_LIBRARIES
-  NAMES zstd
-  HINTS ${ZSTD_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(zstd DEFAULT_MSG ZSTD_LIBRARIES ZSTD_INCLUDE_DIR)
-
-mark_as_advanced(
-  ZSTD_LIBRARIES
-  ZSTD_INCLUDE_DIR)
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash

 # Exit on error.
 set -e
@@ -36,7 +36,6 @@
 #include "utilities/merge_operators.h"
 #include "rocksdb/utilities/transaction.h"
 #include "rocksdb/utilities/transaction_db.h"
-#include "rocksdb/utilities/optimistic_transaction_db.h"
 #include "rocksdb/utilities/checkpoint.h"

 using rocksdb::BytewiseComparator;
@@ -53,7 +52,6 @@ using rocksdb::CompressionType;
 using rocksdb::WALRecoveryMode;
 using rocksdb::DB;
 using rocksdb::DBOptions;
-using rocksdb::DbPath;
 using rocksdb::Env;
 using rocksdb::EnvOptions;
 using rocksdb::InfoLogLevel;
@@ -96,8 +94,6 @@ using rocksdb::PinnableSlice;
 using rocksdb::TransactionDBOptions;
 using rocksdb::TransactionDB;
 using rocksdb::TransactionOptions;
-using rocksdb::OptimisticTransactionDB;
-using rocksdb::OptimisticTransactionOptions;
 using rocksdb::Transaction;
 using rocksdb::Checkpoint;

@@ -156,12 +152,6 @@ struct rocksdb_transaction_t {
 struct rocksdb_checkpoint_t {
  Checkpoint* rep;
 };
-struct rocksdb_optimistictransactiondb_t {
-  OptimisticTransactionDB* rep;
-};
-struct rocksdb_optimistictransaction_options_t {
-  OptimisticTransactionOptions rep;
-};

 struct rocksdb_compactionfiltercontext_t {
  CompactionFilter::Context rep;
@@ -392,10 +382,6 @@ struct rocksdb_mergeoperator_t : public MergeOperator {
  }
 };

-struct rocksdb_dbpath_t {
-  DbPath rep;
-};
-
 struct rocksdb_env_t {
  Env* rep;
  bool is_default;
@@ -2023,16 +2009,6 @@ void rocksdb_options_set_paranoid_checks(
  opt->rep.paranoid_checks = v;
 }

-void rocksdb_options_set_db_paths(rocksdb_options_t* opt, 
-                                  const rocksdb_dbpath_t** dbpath_values, 
-                                  size_t num_paths) {
-  std::vector<DbPath> db_paths(num_paths);
-  for (size_t i = 0; i < num_paths; ++i) {
-    db_paths[i] = dbpath_values[i]->rep;
-  }
-  opt->rep.db_paths = db_paths;
-}
-
 void rocksdb_options_set_env(rocksdb_options_t* opt, rocksdb_env_t* env) {
  opt->rep.env = (env ? env->rep : nullptr);
 }
@@ -2111,10 +2087,6 @@ void rocksdb_options_enable_statistics(rocksdb_options_t* opt) {
  opt->rep.statistics = rocksdb::CreateDBStatistics();
 }

-void rocksdb_options_set_skip_stats_update_on_db_open(rocksdb_options_t* opt, unsigned char val) {
-  opt->rep.skip_stats_update_on_db_open = val;
-}
-
 void rocksdb_options_set_num_levels(rocksdb_options_t* opt, int n) {
  opt->rep.num_levels = n;
 }
@@ -2817,17 +2789,6 @@ size_t rocksdb_cache_get_pinned_usage(rocksdb_cache_t* cache) {
  return cache->rep->GetPinnedUsage();
 }

-rocksdb_dbpath_t* rocksdb_dbpath_create(const char* path, uint64_t target_size) {
-  rocksdb_dbpath_t* result = new rocksdb_dbpath_t;
-  result->rep.path = std::string(path);
-  result->rep.target_size = target_size;
-  return result;
-}
-
-void rocksdb_dbpath_destroy(rocksdb_dbpath_t* dbpath) {
-  delete dbpath;
-}
-
 rocksdb_env_t* rocksdb_create_default_env() {
  rocksdb_env_t* result = new rocksdb_env_t;
  result->rep = Env::Default();
@@ -3262,32 +3223,6 @@ void rocksdb_transaction_options_set_max_write_batch_size(
  opt->rep.max_write_batch_size = size;
 }

-rocksdb_optimistictransaction_options_t*
-rocksdb_optimistictransaction_options_create() {
-  return new rocksdb_optimistictransaction_options_t;
-}
-
-void rocksdb_optimistictransaction_options_destroy(
-    rocksdb_optimistictransaction_options_t* opt) {
-  delete opt;
-}
-
-void rocksdb_optimistictransaction_options_set_set_snapshot(
-    rocksdb_optimistictransaction_options_t* opt, unsigned char v) {
-  opt->rep.set_snapshot = v;
-}
-
-rocksdb_column_family_handle_t* rocksdb_transactiondb_create_column_family(
-    rocksdb_transactiondb_t* txn_db,
-    const rocksdb_options_t* column_family_options,
-    const char* column_family_name, char** errptr) {
-  rocksdb_column_family_handle_t* handle = new rocksdb_column_family_handle_t;
-  SaveError(errptr, txn_db->rep->CreateColumnFamily(
-                        ColumnFamilyOptions(column_family_options->rep),
-                        std::string(column_family_name), &(handle->rep)));
-  return handle;
-}
-
 rocksdb_transactiondb_t* rocksdb_transactiondb_open(
    const rocksdb_options_t* options,
    const rocksdb_transactiondb_options_t* txn_db_options, const char* name,
@@ -3344,14 +3279,7 @@ void rocksdb_transaction_destroy(rocksdb_transaction_t* txn) {
  delete txn;
 }

-const rocksdb_snapshot_t* rocksdb_transaction_get_snapshot(
-    rocksdb_transaction_t* txn) {
-  rocksdb_snapshot_t* result = new rocksdb_snapshot_t;
-  result->rep = txn->rep->GetSnapshot();
-  return result;
-}
-
-// Read a key inside a transaction
+//Read a key inside a transaction
 char* rocksdb_transaction_get(rocksdb_transaction_t* txn,
                              const rocksdb_readoptions_t* options,
                              const char* key, size_t klen, size_t* vlen,
@@ -3371,49 +3299,6 @@ char* rocksdb_transaction_get(rocksdb_transaction_t* txn,
  return result;
 }

-char* rocksdb_transaction_get_cf(rocksdb_transaction_t* txn,
-                                 const rocksdb_readoptions_t* options,
-                                 rocksdb_column_family_handle_t* column_family,
-                                 const char* key, size_t klen, size_t* vlen,
-                                 char** errptr) {
-  char* result = nullptr;
-  std::string tmp;
-  Status s =
-      txn->rep->Get(options->rep, column_family->rep, Slice(key, klen), &tmp);
-  if (s.ok()) {
-    *vlen = tmp.size();
-    result = CopyString(tmp);
-  } else {
-    *vlen = 0;
-    if (!s.IsNotFound()) {
-      SaveError(errptr, s);
-    }
-  }
-  return result;
-}
-
-// Read a key inside a transaction
-char* rocksdb_transaction_get_for_update(rocksdb_transaction_t* txn,
-                                         const rocksdb_readoptions_t* options,
-                                         const char* key, size_t klen,
-                                         size_t* vlen, unsigned char exclusive,
-                                         char** errptr) {
-  char* result = nullptr;
-  std::string tmp;
-  Status s =
-      txn->rep->GetForUpdate(options->rep, Slice(key, klen), &tmp, exclusive);
-  if (s.ok()) {
-    *vlen = tmp.size();
-    result = CopyString(tmp);
-  } else {
-    *vlen = 0;
-    if (!s.IsNotFound()) {
-      SaveError(errptr, s);
-    }
-  }
-  return result;
-}
-
 // Read a key outside a transaction
 char* rocksdb_transactiondb_get(
    rocksdb_transactiondb_t* txn_db,
@@ -3436,26 +3321,6 @@ char* rocksdb_transactiondb_get(
  return result;
 }

-char* rocksdb_transactiondb_get_cf(
-    rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options,
-    rocksdb_column_family_handle_t* column_family, const char* key,
-    size_t keylen, size_t* vallen, char** errptr) {
-  char* result = nullptr;
-  std::string tmp;
-  Status s = txn_db->rep->Get(options->rep, column_family->rep,
-                              Slice(key, keylen), &tmp);
-  if (s.ok()) {
-    *vallen = tmp.size();
-    result = CopyString(tmp);
-  } else {
-    *vallen = 0;
-    if (!s.IsNotFound()) {
-      SaveError(errptr, s);
-    }
-  }
-  return result;
-}
-
 // Put a key inside a transaction
 void rocksdb_transaction_put(rocksdb_transaction_t* txn, const char* key,
                             size_t klen, const char* val, size_t vlen,
@@ -3463,56 +3328,13 @@ void rocksdb_transaction_put(rocksdb_transaction_t* txn, const char* key,
  SaveError(errptr, txn->rep->Put(Slice(key, klen), Slice(val, vlen)));
 }

-void rocksdb_transaction_put_cf(rocksdb_transaction_t* txn,
-                                rocksdb_column_family_handle_t* column_family,
-                                const char* key, size_t klen, const char* val,
-                                size_t vlen, char** errptr) {
-  SaveError(errptr, txn->rep->Put(column_family->rep, Slice(key, klen),
-                                  Slice(val, vlen)));
-}
-
-// Put a key outside a transaction
+//Put a key outside a transaction
 void rocksdb_transactiondb_put(rocksdb_transactiondb_t* txn_db,
                               const rocksdb_writeoptions_t* options,
                               const char* key, size_t klen, const char* val,
                               size_t vlen, char** errptr) {
-  SaveError(errptr, txn_db->rep->Put(options->rep, Slice(key, klen), 
-                                     Slice(val, vlen)));
-}
-
-void rocksdb_transactiondb_put_cf(rocksdb_transactiondb_t* txn_db,
-                                  const rocksdb_writeoptions_t* options,
-                                  rocksdb_column_family_handle_t* column_family,
-                                  const char* key, size_t keylen,
-                                  const char* val, size_t vallen,
-                                  char** errptr) {
-  SaveError(errptr, txn_db->rep->Put(options->rep, column_family->rep,
-                                     Slice(key, keylen), Slice(val, vallen)));
-}
-
-// Write batch into transaction db
-void rocksdb_transactiondb_write(
-        rocksdb_transactiondb_t* db,
-        const rocksdb_writeoptions_t* options,
-        rocksdb_writebatch_t* batch,
-        char** errptr) {
-  SaveError(errptr, db->rep->Write(options->rep, &batch->rep));
-}
-
-// Merge a key inside a transaction
-void rocksdb_transaction_merge(rocksdb_transaction_t* txn, const char* key,
-                               size_t klen, const char* val, size_t vlen,
-                               char** errptr) {
-  SaveError(errptr, txn->rep->Merge(Slice(key, klen), Slice(val, vlen)));
-}
-
-// Merge a key outside a transaction
-void rocksdb_transactiondb_merge(rocksdb_transactiondb_t* txn_db,
-                                 const rocksdb_writeoptions_t* options,
-                                 const char* key, size_t klen, const char* val,
-                                 size_t vlen, char** errptr) {
  SaveError(errptr,
-    txn_db->rep->Merge(options->rep, Slice(key, klen), Slice(val, vlen)));
+            txn_db->rep->Put(options->rep, Slice(key, klen), Slice(val, vlen)));
 }

 // Delete a key inside a transaction
@@ -3521,12 +3343,6 @@ void rocksdb_transaction_delete(rocksdb_transaction_t* txn, const char* key,
  SaveError(errptr, txn->rep->Delete(Slice(key, klen)));
 }

-void rocksdb_transaction_delete_cf(
-    rocksdb_transaction_t* txn, rocksdb_column_family_handle_t* column_family,
-    const char* key, size_t klen, char** errptr) {
-  SaveError(errptr, txn->rep->Delete(column_family->rep, Slice(key, klen)));
-}
-
 // Delete a key outside a transaction
 void rocksdb_transactiondb_delete(rocksdb_transactiondb_t* txn_db,
                                  const rocksdb_writeoptions_t* options,
@@ -3534,14 +3350,6 @@ void rocksdb_transactiondb_delete(rocksdb_transactiondb_t* txn_db,
  SaveError(errptr, txn_db->rep->Delete(options->rep, Slice(key, klen)));
 }

-void rocksdb_transactiondb_delete_cf(
-    rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options,
-    rocksdb_column_family_handle_t* column_family, const char* key,
-    size_t keylen, char** errptr) {
-  SaveError(errptr, txn_db->rep->Delete(options->rep, column_family->rep,
-                                        Slice(key, keylen)));
-}
-
 // Create an iterator inside a transaction
 rocksdb_iterator_t* rocksdb_transaction_create_iterator(
    rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options) {
@@ -3550,14 +3358,6 @@ rocksdb_iterator_t* rocksdb_transaction_create_iterator(
  return result;
 }

-// Create an iterator outside a transaction
-rocksdb_iterator_t* rocksdb_transactiondb_create_iterator(
-    rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options) {
-  rocksdb_iterator_t* result = new rocksdb_iterator_t;
-  result->rep = txn_db->rep->NewIterator(options->rep);
-  return result;
-}
-
 void rocksdb_transactiondb_close(rocksdb_transactiondb_t* txn_db) {
  delete txn_db->rep;
  delete txn_db;
@@ -3574,42 +3374,6 @@ rocksdb_checkpoint_t* rocksdb_transactiondb_checkpoint_object_create(
  return result;
 }

-rocksdb_optimistictransactiondb_t* rocksdb_optimistictransactiondb_open(
-    const rocksdb_options_t* options, const char* name,
-    char** errptr) {
-  OptimisticTransactionDB* otxn_db;
-  if (SaveError(errptr, OptimisticTransactionDB::Open(
-                            options->rep, std::string(name), &otxn_db))) {
-    return nullptr;
-  }
-  rocksdb_optimistictransactiondb_t* result =
-      new rocksdb_optimistictransactiondb_t;
-  result->rep = otxn_db;
-  return result;
-}
-
-rocksdb_transaction_t* rocksdb_optimistictransaction_begin(
-    rocksdb_optimistictransactiondb_t* otxn_db,
-    const rocksdb_writeoptions_t* write_options,
-    const rocksdb_optimistictransaction_options_t* otxn_options,
-    rocksdb_transaction_t* old_txn) {
-  if (old_txn == nullptr) {
-    rocksdb_transaction_t* result = new rocksdb_transaction_t;
-    result->rep = otxn_db->rep->BeginTransaction(write_options->rep,
-                                                 otxn_options->rep, nullptr);
-    return result;
-  }
-  old_txn->rep = otxn_db->rep->BeginTransaction(
-      write_options->rep, otxn_options->rep, old_txn->rep);
-  return old_txn;
-}
-
-void rocksdb_optimistictransactiondb_close(
-    rocksdb_optimistictransactiondb_t* otxn_db) {
-  delete otxn_db->rep;
-  delete otxn_db;
-}
-
 void rocksdb_free(void* ptr) { free(ptr); }

 rocksdb_pinnableslice_t* rocksdb_get_pinned(
@@ -41,7 +41,6 @@ static char dbname[200];
 static char sstfilename[200];
 static char dbbackupname[200];
 static char dbcheckpointname[200];
-static char dbpathname[200];

 static void StartPhase(const char* name) {
  fprintf(stderr, "=== Test %s\n", name);
@@ -348,25 +347,10 @@ static void CheckTxnDBGet(
        Free(&val);
 }

-static void CheckTxnDBGetCF(rocksdb_transactiondb_t* txn_db,
-                            const rocksdb_readoptions_t* options,
-                            rocksdb_column_family_handle_t* column_family,
-                            const char* key, const char* expected) {
-  char* err = NULL;
-  size_t val_len;
-  char* val;
-  val = rocksdb_transactiondb_get_cf(txn_db, options, column_family, key,
-                                     strlen(key), &val_len, &err);
-  CheckNoError(err);
-  CheckEqual(expected, val, val_len);
-  Free(&val);
-}
-
 int main(int argc, char** argv) {
  rocksdb_t* db;
  rocksdb_comparator_t* cmp;
  rocksdb_cache_t* cache;
-  rocksdb_dbpath_t *dbpath;
  rocksdb_env_t* env;
  rocksdb_options_t* options;
  rocksdb_compactoptions_t* coptions;
@@ -401,14 +385,8 @@ int main(int argc, char** argv) {
           GetTempDir(),
           ((int)geteuid()));

-  snprintf(dbpathname, sizeof(dbpathname),
-           "%s/rocksdb_c_test-%d-dbpath",
-           GetTempDir(),
-           ((int) geteuid()));
-
  StartPhase("create_objects");
  cmp = rocksdb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
-  dbpath = rocksdb_dbpath_create(dbpathname, 1024 * 1024);
  env = rocksdb_create_default_env();
  cache = rocksdb_cache_create_lru(100000);

@@ -1379,18 +1357,6 @@ int main(int argc, char** argv) {
    CheckNoError(err);
    CheckTxnDBGet(txn_db, roptions, "foo", NULL);

-    // write batch into TransactionDB
-    rocksdb_writebatch_t* wb = rocksdb_writebatch_create();
-    rocksdb_writebatch_put(wb, "foo", 3, "a", 1);
-    rocksdb_writebatch_clear(wb);
-    rocksdb_writebatch_put(wb, "bar", 3, "b", 1);
-    rocksdb_writebatch_put(wb, "box", 3, "c", 1);
-    rocksdb_writebatch_delete(wb, "bar", 3);
-    rocksdb_transactiondb_write(txn_db, woptions, wb, &err);
-    rocksdb_writebatch_destroy(wb);
-    CheckTxnDBGet(txn_db, roptions, "box", "c");
-    CheckNoError(err);
-
    // begin a transaction
    txn = rocksdb_transaction_begin(txn_db, woptions, txn_options, NULL);
    // put
@@ -1447,23 +1413,6 @@ int main(int argc, char** argv) {
    CheckNoError(err);
    CheckTxnDBGet(txn_db, roptions, "bar", NULL);

-    // Column families.
-    rocksdb_column_family_handle_t* cfh;
-    cfh = rocksdb_transactiondb_create_column_family(txn_db, options,
-                                                     "txn_db_cf", &err);
-    CheckNoError(err);
-
-    rocksdb_transactiondb_put_cf(txn_db, woptions, cfh, "cf_foo", 6, "cf_hello",
-                                 8, &err);
-    CheckNoError(err);
-    CheckTxnDBGetCF(txn_db, roptions, cfh, "cf_foo", "cf_hello");
-
-    rocksdb_transactiondb_delete_cf(txn_db, woptions, cfh, "cf_foo", 6, &err);
-    CheckNoError(err);
-    CheckTxnDBGetCF(txn_db, roptions, cfh, "cf_foo", NULL);
-
-    rocksdb_column_family_handle_destroy(cfh);
-
    // close and destroy
    rocksdb_transaction_destroy(txn);
    rocksdb_transactiondb_close(txn_db);
@@ -1491,18 +1440,6 @@ int main(int argc, char** argv) {
    CheckNoError(err);
  }

-  // Simple sanity check that options setting db_paths work.
-  StartPhase("open_db_paths");
-  {
-    rocksdb_close(db);
-    rocksdb_destroy_db(options, dbname, &err);
-
-    const rocksdb_dbpath_t* paths[1] = {dbpath};
-    rocksdb_options_set_db_paths(options, paths, 1);
-    db = rocksdb_open(options, dbname, &err);
-    CheckNoError(err);
-  }
-  
  StartPhase("cleanup");
  rocksdb_close(db);
  rocksdb_options_destroy(options);
@@ -1512,7 +1449,6 @@ int main(int argc, char** argv) {
  rocksdb_compactoptions_destroy(coptions);
  rocksdb_cache_destroy(cache);
  rocksdb_comparator_destroy(cmp);
-  rocksdb_dbpath_destroy(dbpath);
  rocksdb_env_destroy(env);

  fprintf(stderr, "PASS\n");
@@ -521,7 +521,6 @@ TEST_F(ColumnFamilyTest, DontReuseColumnFamilyID) {
  }
 }

-#ifndef ROCKSDB_LITE
 TEST_F(ColumnFamilyTest, CreateCFRaceWithGetAggProperty) {
  Open();

@@ -543,7 +542,6 @@ TEST_F(ColumnFamilyTest, CreateCFRaceWithGetAggProperty) {

  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
 }
-#endif  // !ROCKSDB_LITE

 class FlushEmptyCFTestWithParam : public ColumnFamilyTest,
                                  public testing::WithParamInterface<bool> {
@@ -169,7 +169,6 @@ Compaction::Compaction(VersionStorageInfo* vstorage,
      bottommost_level_(IsBottommostLevel(output_level_, vstorage, inputs_)),
      is_full_compaction_(IsFullCompaction(vstorage, inputs_)),
      is_manual_compaction_(_manual_compaction),
-      is_trivial_move_(false),
      compaction_reason_(_compaction_reason) {
  MarkFilesBeingCompacted(true);
  if (is_manual_compaction_) {
@@ -16,8 +16,6 @@ struct CompactionIterationStats {
  int64_t num_record_drop_obsolete = 0;
  int64_t num_record_drop_range_del = 0;
  int64_t num_range_del_drop_obsolete = 0;
-  // Deletions obsoleted before bottom level due to file gap optimization.
-  int64_t num_optimized_del_drop_obsolete = 0;
  uint64_t total_filter_time = 0;

  // Input statistics
@@ -111,7 +111,6 @@ void CompactionIterator::ResetRecordCounts() {
  iter_stats_.num_record_drop_obsolete = 0;
  iter_stats_.num_record_drop_range_del = 0;
  iter_stats_.num_range_del_drop_obsolete = 0;
-  iter_stats_.num_optimized_del_drop_obsolete = 0;
 }

 void CompactionIterator::SeekToFirst() {
@@ -427,9 +426,6 @@ void CompactionIterator::NextFromInput() {
          // Can compact out this SingleDelete.
          ++iter_stats_.num_record_drop_obsolete;
          ++iter_stats_.num_single_del_fallthru;
-          if (!bottommost_level_) {
-            ++iter_stats_.num_optimized_del_drop_obsolete;
-          }
        } else {
          // Output SingleDelete
          valid_ = true;
@@ -471,9 +467,6 @@ void CompactionIterator::NextFromInput() {
      // Note:  Dropping this Delete will not affect TransactionDB
      // write-conflict checking since it is earlier than any snapshot.
      ++iter_stats_.num_record_drop_obsolete;
-      if (!bottommost_level_) {
-        ++iter_stats_.num_optimized_del_drop_obsolete;
-      }
      input_->Next();
    } else if (ikey_.type == kTypeMerge) {
      if (!merge_helper_->HasOperator()) {
@@ -455,111 +455,6 @@ TEST_F(CompactionIteratorTest, ShuttingDownInMerge) {
  EXPECT_EQ(2, filter.last_seen.load());
 }

-TEST_F(CompactionIteratorTest, SingleMergeOperand) {
-  class Filter : public CompactionFilter {
-    virtual Decision FilterV2(int level, const Slice& key, ValueType t,
-                              const Slice& existing_value,
-                              std::string* new_value,
-                              std::string* skip_until) const override {
-      std::string k = key.ToString();
-      std::string v = existing_value.ToString();
-
-      // See InitIterators() call below for the sequence of keys and their
-      // filtering decisions. Here we closely assert that compaction filter is
-      // called with the expected keys and only them, and with the right values.
-      if (k == "a") {
-        EXPECT_EQ(ValueType::kMergeOperand, t);
-        EXPECT_EQ("av1", v);
-        return Decision::kKeep;
-      } else if (k == "b") {
-        EXPECT_EQ(ValueType::kMergeOperand, t);
-        return Decision::kKeep;
-      } else if (k == "c") {
-        return Decision::kKeep;
-      }
-
-      ADD_FAILURE();
-      return Decision::kKeep;
-    }
-
-    const char* Name() const override {
-      return "CompactionIteratorTest.SingleMergeOperand::Filter";
-    }
-  };
-
-  class SingleMergeOp : public MergeOperator {
-   public:
-    bool FullMergeV2(const MergeOperationInput& merge_in,
-                     MergeOperationOutput* merge_out) const override {
-      // See InitIterators() call below for why "c" is the only key for which
-      // FullMergeV2 should be called.
-      EXPECT_EQ("c", merge_in.key.ToString());
-
-      std::string temp_value;
-      if (merge_in.existing_value != nullptr) {
-        temp_value = merge_in.existing_value->ToString();
-      }
-
-      for (auto& operand : merge_in.operand_list) {
-        temp_value.append(operand.ToString());
-      }
-      merge_out->new_value = temp_value;
-
-      return true;
-    }
-
-    bool PartialMergeMulti(const Slice& key,
-                           const std::deque<Slice>& operand_list,
-                           std::string* new_value,
-                           Logger* logger) const override {
-      std::string string_key = key.ToString();
-      EXPECT_TRUE(string_key == "a" || string_key == "b");
-
-      if (string_key == "a") {
-        EXPECT_EQ(1, operand_list.size());
-      } else if (string_key == "b") {
-        EXPECT_EQ(2, operand_list.size());
-      }
-
-      std::string temp_value;
-      for (auto& operand : operand_list) {
-        temp_value.append(operand.ToString());
-      }
-      swap(temp_value, *new_value);
-
-      return true;
-    }
-
-    const char* Name() const override {
-      return "CompactionIteratorTest SingleMergeOp";
-    }
-
-    bool AllowSingleOperand() const override { return true; }
-  };
-
-  SingleMergeOp merge_op;
-  Filter filter;
-  InitIterators(
-      // a should invoke PartialMergeMulti with a single merge operand.
-      {test::KeyStr("a", 50, kTypeMerge),
-       // b should invoke PartialMergeMulti with two operands.
-       test::KeyStr("b", 70, kTypeMerge), test::KeyStr("b", 60, kTypeMerge),
-       // c should invoke FullMerge due to kTypeValue at the beginning.
-       test::KeyStr("c", 90, kTypeMerge), test::KeyStr("c", 80, kTypeValue)},
-      {"av1", "bv2", "bv1", "cv2", "cv1"}, {}, {}, kMaxSequenceNumber,
-      &merge_op, &filter);
-
-  c_iter_->SeekToFirst();
-  ASSERT_TRUE(c_iter_->Valid());
-  ASSERT_EQ(test::KeyStr("a", 50, kTypeMerge), c_iter_->key().ToString());
-  ASSERT_EQ("av1", c_iter_->value().ToString());
-  c_iter_->Next();
-  ASSERT_TRUE(c_iter_->Valid());
-  ASSERT_EQ("bv1bv2", c_iter_->value().ToString());
-  c_iter_->Next();
-  ASSERT_EQ("cv1cv2", c_iter_->value().ToString());
-}
-
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -594,9 +594,6 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) {

  double read_write_amp = 0.0;
  double write_amp = 0.0;
-  double bytes_read_per_sec = 0;
-  double bytes_written_per_sec = 0;
-
  if (stats.bytes_read_non_output_levels > 0) {
    read_write_amp = (stats.bytes_written + stats.bytes_read_output_level +
                      stats.bytes_read_non_output_levels) /
@@ -604,22 +601,17 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) {
    write_amp = stats.bytes_written /
                static_cast<double>(stats.bytes_read_non_output_levels);
  }
-  if (stats.micros > 0) {
-    bytes_read_per_sec =
-        (stats.bytes_read_non_output_levels + stats.bytes_read_output_level) /
-        static_cast<double>(stats.micros);
-    bytes_written_per_sec =
-        stats.bytes_written / static_cast<double>(stats.micros);
-  }
-
  ROCKS_LOG_BUFFER(
      log_buffer_,
      "[%s] compacted to: %s, MB/sec: %.1f rd, %.1f wr, level %d, "
      "files in(%d, %d) out(%d) "
      "MB in(%.1f, %.1f) out(%.1f), read-write-amplify(%.1f) "
      "write-amplify(%.1f) %s, records in: %d, records dropped: %d\n",
-      cfd->GetName().c_str(), vstorage->LevelSummary(&tmp), bytes_read_per_sec,
-      bytes_written_per_sec, compact_->compaction->output_level(),
+      cfd->GetName().c_str(), vstorage->LevelSummary(&tmp),
+      (stats.bytes_read_non_output_levels + stats.bytes_read_output_level) /
+          static_cast<double>(stats.micros),
+      stats.bytes_written / static_cast<double>(stats.micros),
+      compact_->compaction->output_level(),
      stats.num_input_files_in_non_output_levels,
      stats.num_input_files_in_output_level, stats.num_output_files,
      stats.bytes_read_non_output_levels / 1048576.0,
@@ -1014,10 +1006,6 @@ void CompactionJob::RecordDroppedKeys(
    RecordTick(stats_, COMPACTION_RANGE_DEL_DROP_OBSOLETE,
               c_iter_stats.num_range_del_drop_obsolete);
  }
-  if (c_iter_stats.num_optimized_del_drop_obsolete > 0) {
-    RecordTick(stats_, COMPACTION_OPTIMIZED_DEL_DROP_OBSOLETE,
-               c_iter_stats.num_optimized_del_drop_obsolete);
-  }
 }

 Status CompactionJob::FinishCompactionOutputFile(
@@ -234,6 +234,11 @@ bool CompactionPicker::ExpandInputsToCleanCut(const std::string& cf_name,
  // If, after the expansion, there are files that are already under
  // compaction, then we must drop/cancel this compaction.
  if (AreFilesInCompaction(inputs->files)) {
+    ROCKS_LOG_WARN(
+        ioptions_.info_log,
+        "[%s] ExpandWhileOverlapping() failure because some of the necessary"
+        " compaction input files are currently being compacted.",
+        cf_name.c_str());
    return false;
  }
  return true;
@@ -1101,7 +1106,11 @@ void LevelCompactionBuilder::SetupInitialFiles() {
      }
      output_level_ =
          (start_level_ == 0) ? vstorage_->base_level() : start_level_ + 1;
-      if (PickFileToCompact()) {
+      if (PickFileToCompact() &&
+          compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_,
+                                                     &start_level_inputs_) &&
+          !compaction_picker_->FilesRangeOverlapWithCompaction(
+              {start_level_inputs_}, output_level_)) {
        // found the compaction!
        if (start_level_ == 0) {
          // L0 score = `num L0 files` / `level0_file_num_compaction_trigger`
@@ -1323,10 +1332,12 @@ bool LevelCompactionBuilder::PickFileToCompact() {
  const std::vector<FileMetaData*>& level_files =
      vstorage_->LevelFiles(start_level_);

-  unsigned int cmp_idx;
-  for (cmp_idx = vstorage_->NextCompactionIndex(start_level_);
-       cmp_idx < file_size.size(); cmp_idx++) {
-    int index = file_size[cmp_idx];
+  // record the first file that is not yet compacted
+  int nextIndex = -1;
+
+  for (unsigned int i = vstorage_->NextCompactionIndex(start_level_);
+       i < file_size.size(); i++) {
+    int index = file_size[i];
    auto* f = level_files[index];

    // do not pick a file to compact if it is being compacted
@@ -1335,42 +1346,27 @@ bool LevelCompactionBuilder::PickFileToCompact() {
      continue;
    }

-    start_level_inputs_.files.push_back(f);
-    start_level_inputs_.level = start_level_;
-    if (!compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_,
-                                                    &start_level_inputs_) ||
-        compaction_picker_->FilesRangeOverlapWithCompaction(
-            {start_level_inputs_}, output_level_)) {
-      // A locked (pending compaction) input-level file was pulled in due to
-      // user-key overlap.
-      start_level_inputs_.clear();
-      continue;
+    // remember the startIndex for the next call to PickCompaction
+    if (nextIndex == -1) {
+      nextIndex = i;
    }

-    // Now that input level is fully expanded, we check whether any output files
-    // are locked due to pending compaction.
-    //
-    // Note we rely on ExpandInputsToCleanCut() to tell us whether any output-
-    // level files are locked, not just the extra ones pulled in for user-key
-    // overlap.
-    InternalKey smallest, largest;
-    compaction_picker_->GetRange(start_level_inputs_, &smallest, &largest);
-    CompactionInputFiles output_level_inputs;
-    output_level_inputs.level = output_level_;
-    vstorage_->GetOverlappingInputs(output_level_, &smallest, &largest,
-                                    &output_level_inputs.files);
-    if (!output_level_inputs.empty() &&
-        !compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_,
-                                                    &output_level_inputs)) {
-      start_level_inputs_.clear();
+    // Do not pick this file if its parents at level+1 are being compacted.
+    // Maybe we can avoid redoing this work in SetupOtherInputs
+    parent_index_ = -1;
+    if (compaction_picker_->IsRangeInCompaction(vstorage_, &f->smallest,
+                                                &f->largest, output_level_,
+                                                &parent_index_)) {
      continue;
    }
+    start_level_inputs_.files.push_back(f);
+    start_level_inputs_.level = start_level_;
    base_index_ = index;
    break;
  }

  // store where to start the iteration in the next call to PickCompaction
-  vstorage_->SetNextCompactionIndex(start_level_, cmp_idx);
+  vstorage_->SetNextCompactionIndex(start_level_, nextIndex);

  return start_level_inputs_.size() > 0;
 }
@@ -512,7 +512,7 @@ TEST_F(CompactionPickerTest, NeedsCompactionFIFO) {
        kFileSize, 0, i * 100, i * 100 + 99);
    current_size += kFileSize;
    UpdateVersionStorageInfo();
-    ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()),
+    ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()),
              vstorage_->CompactionScore(0) >= 1);
  }
 }
@@ -852,80 +852,6 @@ TEST_F(CompactionPickerTest, OverlappingUserKeys9) {
  ASSERT_EQ(8U, compaction->input(1, 1)->fd.GetNumber());
 }

-TEST_F(CompactionPickerTest, OverlappingUserKeys10) {
-  // Locked file encountered when pulling in extra input-level files with same
-  // user keys. Verify we pick the next-best file from the same input level.
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.max_compaction_bytes = 100000000000u;
-
-  // file_number 2U is largest and thus first choice. But it overlaps with
-  // file_number 1U which is being compacted. So instead we pick the next-
-  // biggest file, 3U, which is eligible for compaction.
-  Add(1 /* level */, 1U /* file_number */, "100" /* smallest */,
-      "150" /* largest */, 1U /* file_size */);
-  file_map_[1U].first->being_compacted = true;
-  Add(1 /* level */, 2U /* file_number */, "150" /* smallest */,
-      "200" /* largest */, 1000000000U /* file_size */, 0 /* smallest_seq */,
-      0 /* largest_seq */);
-  Add(1 /* level */, 3U /* file_number */, "201" /* smallest */,
-      "250" /* largest */, 900000000U /* file_size */);
-  Add(2 /* level */, 4U /* file_number */, "100" /* smallest */,
-      "150" /* largest */, 1U /* file_size */);
-  Add(2 /* level */, 5U /* file_number */, "151" /* smallest */,
-      "200" /* largest */, 1U /* file_size */);
-  Add(2 /* level */, 6U /* file_number */, "201" /* smallest */,
-      "250" /* largest */, 1U /* file_size */);
-
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(2U, compaction->num_input_levels());
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(1U, compaction->num_input_files(1));
-  ASSERT_EQ(3U, compaction->input(0, 0)->fd.GetNumber());
-  ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber());
-}
-
-TEST_F(CompactionPickerTest, OverlappingUserKeys11) {
-  // Locked file encountered when pulling in extra output-level files with same
-  // user keys. Expected to skip that compaction and pick the next-best choice.
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.max_compaction_bytes = 100000000000u;
-
-  // score(L1) = 3.7
-  // score(L2) = 1.85
-  // There is no eligible file in L1 to compact since both candidates pull in
-  // file_number 5U, which overlaps with a file pending compaction (6U). The
-  // first eligible compaction is from L2->L3.
-  Add(1 /* level */, 2U /* file_number */, "151" /* smallest */,
-      "200" /* largest */, 1000000000U /* file_size */);
-  Add(1 /* level */, 3U /* file_number */, "201" /* smallest */,
-      "250" /* largest */, 1U /* file_size */);
-  Add(2 /* level */, 4U /* file_number */, "100" /* smallest */,
-      "149" /* largest */, 5000000000U /* file_size */);
-  Add(2 /* level */, 5U /* file_number */, "150" /* smallest */,
-      "201" /* largest */, 1U /* file_size */);
-  Add(2 /* level */, 6U /* file_number */, "201" /* smallest */,
-      "249" /* largest */, 1U /* file_size */, 0 /* smallest_seq */,
-      0 /* largest_seq */);
-  file_map_[6U].first->being_compacted = true;
-  Add(3 /* level */, 7U /* file_number */, "100" /* smallest */,
-      "149" /* largest */, 1U /* file_size */);
-
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(2U, compaction->num_input_levels());
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(1U, compaction->num_input_files(1));
-  ASSERT_EQ(4U, compaction->input(0, 0)->fd.GetNumber());
-  ASSERT_EQ(7U, compaction->input(1, 0)->fd.GetNumber());
-}
-
 TEST_F(CompactionPickerTest, NotScheduleL1IfL0WithHigherPri1) {
  NewVersionStorage(6, kCompactionStyleLevel);
  mutable_cf_options_.level0_file_num_compaction_trigger = 2;
@@ -1390,49 +1316,6 @@ TEST_F(CompactionPickerTest, IsTrivialMoveOff) {
  ASSERT_FALSE(compaction->IsTrivialMove());
 }

-TEST_F(CompactionPickerTest, CacheNextCompactionIndex) {
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.max_compaction_bytes = 100000000000u;
-
-  Add(1 /* level */, 1U /* file_number */, "100" /* smallest */,
-      "149" /* largest */, 1000000000U /* file_size */);
-  file_map_[1U].first->being_compacted = true;
-  Add(1 /* level */, 2U /* file_number */, "150" /* smallest */,
-      "199" /* largest */, 900000000U /* file_size */);
-  Add(1 /* level */, 3U /* file_number */, "200" /* smallest */,
-      "249" /* largest */, 800000000U /* file_size */);
-  Add(1 /* level */, 4U /* file_number */, "250" /* smallest */,
-      "299" /* largest */, 700000000U /* file_size */);
-  Add(2 /* level */, 5U /* file_number */, "150" /* smallest */,
-      "199" /* largest */, 1U /* file_size */);
-  file_map_[5U].first->being_compacted = true;
-
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(1U, compaction->num_input_levels());
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(0U, compaction->num_input_files(1));
-  ASSERT_EQ(3U, compaction->input(0, 0)->fd.GetNumber());
-  ASSERT_EQ(2, vstorage_->NextCompactionIndex(1 /* level */));
-
-  compaction.reset(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(1U, compaction->num_input_levels());
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(0U, compaction->num_input_files(1));
-  ASSERT_EQ(4U, compaction->input(0, 0)->fd.GetNumber());
-  ASSERT_EQ(3, vstorage_->NextCompactionIndex(1 /* level */));
-
-  compaction.reset(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() == nullptr);
-  ASSERT_EQ(4, vstorage_->NextCompactionIndex(1 /* level */));
-}
-
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -373,7 +373,6 @@ Compaction* UniversalCompactionPicker::PickCompaction(
              c->inputs(0)->size());

  RegisterCompaction(c);
-  vstorage->ComputeCompactionScore(ioptions_, mutable_cf_options);

  TEST_SYNC_POINT_CALLBACK("UniversalCompactionPicker::PickCompaction:Return",
                           c);
@@ -9,50 +9,19 @@
 #include "rocksdb/convenience.h"

 #include "db/db_impl.h"
-#include "util/cast_util.h"

 namespace rocksdb {

 void CancelAllBackgroundWork(DB* db, bool wait) {
-  (static_cast_with_check<DBImpl, DB>(db->GetRootDB()))
-      ->CancelAllBackgroundWork(wait);
+  (dynamic_cast<DBImpl*>(db->GetRootDB()))->CancelAllBackgroundWork(wait);
 }

 Status DeleteFilesInRange(DB* db, ColumnFamilyHandle* column_family,
                          const Slice* begin, const Slice* end) {
-  return (static_cast_with_check<DBImpl, DB>(db->GetRootDB()))
+  return (dynamic_cast<DBImpl*>(db->GetRootDB()))
      ->DeleteFilesInRange(column_family, begin, end);
 }

-Status VerifySstFileChecksum(const Options& options,
-                             const EnvOptions& env_options,
-                             const std::string& file_path) {
-  unique_ptr<RandomAccessFile> file;
-  uint64_t file_size;
-  InternalKeyComparator internal_comparator(options.comparator);
-  ImmutableCFOptions ioptions(options);
-
-  Status s = ioptions.env->NewRandomAccessFile(file_path, &file, env_options);
-  if (s.ok()) {
-    s = ioptions.env->GetFileSize(file_path, &file_size);
-  } else {
-    return s;
-  }
-  unique_ptr<TableReader> table_reader;
-  std::unique_ptr<RandomAccessFileReader> file_reader(
-      new RandomAccessFileReader(std::move(file), file_path));
-  s = ioptions.table_factory->NewTableReader(
-      TableReaderOptions(ioptions, env_options, internal_comparator,
-                         false /* skip_filters */, -1 /* level */),
-      std::move(file_reader), file_size, &table_reader,
-      false /* prefetch_index_and_filter_in_cache */);
-  if (!s.ok()) {
-    return s;
-  }
-  s = table_reader->VerifyChecksum();
-  return s;
-}
-
 }  // namespace rocksdb

 #endif  // ROCKSDB_LITE
@@ -20,7 +20,6 @@
 #include "db/log_format.h"
 #include "db/version_set.h"
 #include "rocksdb/cache.h"
-#include "rocksdb/convenience.h"
 #include "rocksdb/env.h"
 #include "rocksdb/table.h"
 #include "rocksdb/write_batch.h"
@@ -153,7 +152,7 @@ class CorruptionTest : public testing::Test {
    struct stat sbuf;
    if (stat(fname.c_str(), &sbuf) != 0) {
      const char* msg = strerror(errno);
-      FAIL() << fname << ": " << msg;
+      ASSERT_TRUE(false) << fname << ": " << msg;
    }

    if (offset < 0) {
@@ -180,9 +179,6 @@ class CorruptionTest : public testing::Test {
    }
    s = WriteStringToFile(Env::Default(), contents, fname);
    ASSERT_TRUE(s.ok()) << s.ToString();
-    Options options;
-    EnvOptions env_options;
-    ASSERT_NOK(VerifySstFileChecksum(options, env_options, fname));
  }

  void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) {
@@ -217,7 +213,7 @@ class CorruptionTest : public testing::Test {
        return;
      }
    }
-    FAIL() << "no file found at level";
+    ASSERT_TRUE(false) << "no file found at level";
  }


@@ -316,7 +312,6 @@ TEST_F(CorruptionTest, TableFile) {

  Corrupt(kTableFile, 100, 1);
  Check(99, 99);
-  ASSERT_NOK(dbi->VerifyChecksum());
 }

 TEST_F(CorruptionTest, TableFileIndexData) {
@@ -332,11 +327,9 @@ TEST_F(CorruptionTest, TableFileIndexData) {
  // corrupt an index block of an entire file
  Corrupt(kTableFile, -2000, 500);
  Reopen();
-  dbi = reinterpret_cast<DBImpl*>(db_);
  // one full file should be readable, since only one was corrupted
  // the other file should be fully non-readable, since index was corrupted
  Check(5000, 5000);
-  ASSERT_NOK(dbi->VerifyChecksum());
 }

 TEST_F(CorruptionTest, MissingDescriptor) {
@@ -396,12 +389,10 @@ TEST_F(CorruptionTest, CompactionInputError) {

  Corrupt(kTableFile, 100, 1);
  Check(9, 9);
-  ASSERT_NOK(dbi->VerifyChecksum());

  // Force compactions by writing lots of values
  Build(10000);
  Check(10000, 10000);
-  ASSERT_NOK(dbi->VerifyChecksum());
 }

 TEST_F(CorruptionTest, CompactionInputErrorParanoid) {
@@ -433,7 +424,6 @@ TEST_F(CorruptionTest, CompactionInputErrorParanoid) {

  CorruptTableFileAtLevel(0, 100, 1);
  Check(9, 9);
-  ASSERT_NOK(dbi->VerifyChecksum());

  // Write must eventually fail because of corrupted table
  Status s;
@@ -455,7 +445,6 @@ TEST_F(CorruptionTest, UnrelatedKeys) {
  DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
  dbi->TEST_FlushMemTable();
  Corrupt(kTableFile, 100, 1);
-  ASSERT_NOK(dbi->VerifyChecksum());

  std::string tmp1, tmp2;
  ASSERT_OK(db_->Put(WriteOptions(), Key(1000, &tmp1), Value(1000, &tmp2)));
@@ -175,8 +175,6 @@ TEST_F(DBBasicTest, LevelLimitReopen) {
  int i = 0;
  while (NumTableFilesAtLevel(2, 1) == 0) {
    ASSERT_OK(Put(1, Key(i++), value));
-    dbfull()->TEST_WaitForFlushMemTable();
-    dbfull()->TEST_WaitForCompact();
  }

  options.num_levels = 1;
@@ -360,6 +358,7 @@ TEST_F(DBBasicTest, FLUSH) {
    WriteOptions writeOpt = WriteOptions();
    writeOpt.disableWAL = true;
    SetPerfLevel(kEnableTime);
+    ;
    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v1"));
    // this will now also flush the last 2 writes
    ASSERT_OK(Flush(1));
@@ -368,7 +367,6 @@ TEST_F(DBBasicTest, FLUSH) {
    get_perf_context()->Reset();
    Get(1, "foo");
    ASSERT_TRUE((int)get_perf_context()->get_from_output_files_time > 0);
-    ASSERT_EQ(2, (int)get_perf_context()->get_read_bytes);

    ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
    ASSERT_EQ("v1", Get(1, "foo"));
@@ -725,7 +723,6 @@ TEST_F(DBBasicTest, FlushOneColumnFamily) {
 TEST_F(DBBasicTest, MultiGetSimple) {
  do {
    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
-    SetPerfLevel(kEnableCount);
    ASSERT_OK(Put(1, "k1", "v1"));
    ASSERT_OK(Put(1, "k2", "v2"));
    ASSERT_OK(Put(1, "k3", "v3"));
@@ -739,15 +736,12 @@ TEST_F(DBBasicTest, MultiGetSimple) {
    std::vector<std::string> values(20, "Temporary data to be overwritten");
    std::vector<ColumnFamilyHandle*> cfs(keys.size(), handles_[1]);

-    get_perf_context()->Reset();
    std::vector<Status> s = db_->MultiGet(ReadOptions(), cfs, keys, &values);
    ASSERT_EQ(values.size(), keys.size());
    ASSERT_EQ(values[0], "v1");
    ASSERT_EQ(values[1], "v2");
    ASSERT_EQ(values[2], "v3");
    ASSERT_EQ(values[4], "v5");
-    // four kv pairs * two bytes per value
-    ASSERT_EQ(8, (int)get_perf_context()->multiget_read_bytes);

    ASSERT_OK(s[0]);
    ASSERT_OK(s[1]);
@@ -755,7 +749,6 @@ TEST_F(DBBasicTest, MultiGetSimple) {
    ASSERT_TRUE(s[3].IsNotFound());
    ASSERT_OK(s[4]);
    ASSERT_TRUE(s[5].IsNotFound());
-    SetPerfLevel(kDisable);
  } while (ChangeCompactOptions());
 }

@@ -792,30 +785,36 @@ TEST_F(DBBasicTest, MultiGetEmpty) {
 TEST_F(DBBasicTest, ChecksumTest) {
  BlockBasedTableOptions table_options;
  Options options = CurrentOptions();
-  // change when new checksum type added
-  int max_checksum = static_cast<int>(kxxHash);
-  const int kNumPerFile = 2;

-  // generate one table with each type of checksum
-  for (int i = 0; i <= max_checksum; ++i) {
-    table_options.checksum = static_cast<ChecksumType>(i);
-    options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-    Reopen(options);
-    for (int j = 0; j < kNumPerFile; ++j) {
-      ASSERT_OK(Put(Key(i * kNumPerFile + j), Key(i * kNumPerFile + j)));
-    }
-    ASSERT_OK(Flush());
-  }
+  table_options.checksum = kCRC32c;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  Reopen(options);
+  ASSERT_OK(Put("a", "b"));
+  ASSERT_OK(Put("c", "d"));
+  ASSERT_OK(Flush());  // table with crc checksum

-  // verify data with each type of checksum
-  for (int i = 0; i <= kxxHash; ++i) {
-    table_options.checksum = static_cast<ChecksumType>(i);
-    options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-    Reopen(options);
-    for (int j = 0; j < (max_checksum + 1) * kNumPerFile; ++j) {
-      ASSERT_EQ(Key(j), Get(Key(j)));
-    }
-  }
+  table_options.checksum = kxxHash;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  Reopen(options);
+  ASSERT_OK(Put("e", "f"));
+  ASSERT_OK(Put("g", "h"));
+  ASSERT_OK(Flush());  // table with xxhash checksum
+
+  table_options.checksum = kCRC32c;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  Reopen(options);
+  ASSERT_EQ("b", Get("a"));
+  ASSERT_EQ("d", Get("c"));
+  ASSERT_EQ("f", Get("e"));
+  ASSERT_EQ("h", Get("g"));
+
+  table_options.checksum = kCRC32c;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  Reopen(options);
+  ASSERT_EQ("b", Get("a"));
+  ASSERT_EQ("d", Get("c"));
+  ASSERT_EQ("f", Get("e"));
+  ASSERT_EQ("h", Get("g"));
 }

 // On Windows you can have either memory mapped file or a file
@@ -497,7 +497,7 @@ TEST_F(DBBlockCacheTest, CompressedCache) {
        options.compression = kNoCompression;
        break;
      default:
-        FAIL();
+        ASSERT_TRUE(false);
    }
    CreateAndReopenWithCF({"pikachu"}, options);
    // default column family doesn't have block cache
@@ -560,7 +560,7 @@ TEST_F(DBBlockCacheTest, CompressedCache) {
        ASSERT_EQ(TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_HIT), 0);
        break;
      default:
-        FAIL();
+        ASSERT_TRUE(false);
    }

    options.create_if_missing = true;
@@ -2684,46 +2684,6 @@ TEST_P(DBCompactionTestWithParam, IntraL0CompactionDoesNotObsoleteDeletions) {
  ASSERT_TRUE(db_->Get(roptions, Key(0), &result).IsNotFound());
 }

-TEST_F(DBCompactionTest, OptimizedDeletionObsoleting) {
-  // Deletions can be dropped when compacted to non-last level if they fall
-  // outside the lower-level files' key-ranges.
-  const int kNumL0Files = 4;
-  Options options = CurrentOptions();
-  options.level0_file_num_compaction_trigger = kNumL0Files;
-  options.statistics = rocksdb::CreateDBStatistics();
-  DestroyAndReopen(options);
-
-  // put key 1 and 3 in separate L1, L2 files.
-  // So key 0, 2, and 4+ fall outside these levels' key-ranges.
-  for (int level = 2; level >= 1; --level) {
-    for (int i = 0; i < 2; ++i) {
-      Put(Key(2 * i + 1), "val");
-      Flush();
-    }
-    MoveFilesToLevel(level);
-    ASSERT_EQ(2, NumTableFilesAtLevel(level));
-  }
-
-  // Delete keys in range [1, 4]. These L0 files will be compacted with L1:
-  // - Tombstones for keys 2 and 4 can be dropped early.
-  // - Tombstones for keys 1 and 3 must be kept due to L2 files' key-ranges.
-  for (int i = 0; i < kNumL0Files; ++i) {
-    Put(Key(0), "val");  // sentinel to prevent trivial move
-    Delete(Key(i + 1));
-    Flush();
-  }
-  dbfull()->TEST_WaitForCompact();
-
-  for (int i = 0; i < kNumL0Files; ++i) {
-    std::string value;
-    ASSERT_TRUE(db_->Get(ReadOptions(), Key(i + 1), &value).IsNotFound());
-  }
-  ASSERT_EQ(2, options.statistics->getTickerCount(
-                   COMPACTION_OPTIMIZED_DEL_DROP_OBSOLETE));
-  ASSERT_EQ(2,
-            options.statistics->getTickerCount(COMPACTION_KEY_DROP_OBSOLETE));
-}
-
 INSTANTIATE_TEST_CASE_P(DBCompactionTestWithParam, DBCompactionTestWithParam,
                        ::testing::Values(std::make_tuple(1, true),
                                          std::make_tuple(1, false),
@@ -1,7 +1,9 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//  This source code is also licensed under the GPLv2 license found in the
+//  COPYING file in the root directory of this source tree.
 //
 #include "db/db_test_util.h"
 #include "port/stack_trace.h"
@@ -67,7 +67,6 @@
 #include "port/port.h"
 #include "rocksdb/cache.h"
 #include "rocksdb/compaction_filter.h"
-#include "rocksdb/convenience.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "rocksdb/merge_operator.h"
@@ -81,7 +80,6 @@
 #include "table/merging_iterator.h"
 #include "table/table_builder.h"
 #include "table/two_level_iterator.h"
-#include "tools/sst_dump_tool_imp.h"
 #include "util/auto_roll_logger.h"
 #include "util/autovector.h"
 #include "util/build_version.h"
@@ -170,7 +168,6 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
      last_batch_group_size_(0),
      unscheduled_flushes_(0),
      unscheduled_compactions_(0),
-      bg_bottom_compaction_scheduled_(0),
      bg_compaction_scheduled_(0),
      num_running_compactions_(0),
      bg_flush_scheduled_(0),
@@ -245,8 +242,7 @@ void DBImpl::CancelAllBackgroundWork(bool wait) {
    return;
  }
  // Wait for background work to finish
-  while (bg_bottom_compaction_scheduled_ || bg_compaction_scheduled_ ||
-         bg_flush_scheduled_) {
+  while (bg_compaction_scheduled_ || bg_flush_scheduled_) {
    bg_cv_.Wait();
  }
 }
@@ -256,18 +252,15 @@ DBImpl::~DBImpl() {
  // marker. After this we do a variant of the waiting and unschedule work
  // (to consider: moving all the waiting into CancelAllBackgroundWork(true))
  CancelAllBackgroundWork(false);
-  int bottom_compactions_unscheduled =
-      env_->UnSchedule(this, Env::Priority::BOTTOM);
  int compactions_unscheduled = env_->UnSchedule(this, Env::Priority::LOW);
  int flushes_unscheduled = env_->UnSchedule(this, Env::Priority::HIGH);
  mutex_.Lock();
-  bg_bottom_compaction_scheduled_ -= bottom_compactions_unscheduled;
  bg_compaction_scheduled_ -= compactions_unscheduled;
  bg_flush_scheduled_ -= flushes_unscheduled;

  // Wait for background work to finish
-  while (bg_bottom_compaction_scheduled_ || bg_compaction_scheduled_ ||
-         bg_flush_scheduled_ || bg_purge_scheduled_) {
+  while (bg_compaction_scheduled_ || bg_flush_scheduled_ ||
+         bg_purge_scheduled_) {
    TEST_SYNC_POINT("DBImpl::~DBImpl:WaitJob");
    bg_cv_.Wait();
  }
@@ -990,7 +983,6 @@ Status DBImpl::GetImpl(const ReadOptions& read_options,
    size_t size = pinnable_val->size();
    RecordTick(stats_, BYTES_READ, size);
    MeasureTime(stats_, BYTES_PER_READ, size);
-    PERF_COUNTER_ADD(get_read_bytes, size);
  }
  return s;
 }
@@ -1118,7 +1110,6 @@ std::vector<Status> DBImpl::MultiGet(
  RecordTick(stats_, NUMBER_MULTIGET_KEYS_READ, num_keys);
  RecordTick(stats_, NUMBER_MULTIGET_BYTES_READ, bytes_read);
  MeasureTime(stats_, BYTES_PER_MULTIGET, bytes_read);
-  PERF_COUNTER_ADD(multiget_read_bytes, bytes_read);
  PERF_TIMER_STOP(get_post_process_time);

  return stat_list;
@@ -1324,11 +1315,6 @@ Status DBImpl::DropColumnFamilyImpl(ColumnFamilyHandle* column_family) {
                                 &edit, &mutex_);
      write_thread_.ExitUnbatched(&w);
    }
-    if (s.ok()) {
-      auto* mutable_cf_options = cfd->GetLatestMutableCFOptions();
-      max_total_in_memory_state_ -= mutable_cf_options->write_buffer_size *
-                                    mutable_cf_options->max_write_buffer_number;
-    }

    if (!cf_support_snapshot) {
      // Dropped Column Family doesn't support snapshot. Need to recalculate
@@ -1350,6 +1336,9 @@ Status DBImpl::DropColumnFamilyImpl(ColumnFamilyHandle* column_family) {
    // later inside db_mutex.
    EraseThreadStatusCfInfo(cfd);
    assert(cfd->IsDropped());
+    auto* mutable_cf_options = cfd->GetLatestMutableCFOptions();
+    max_total_in_memory_state_ -= mutable_cf_options->write_buffer_size *
+                                  mutable_cf_options->max_write_buffer_number;
    ROCKS_LOG_INFO(immutable_db_options_.info_log,
                   "Dropped column family with id %u\n", cfd->GetID());
  } else {
@@ -1413,7 +1402,8 @@ Iterator* DBImpl::NewIterator(const ReadOptions& read_options,
    return NewDBIterator(
        env_, read_options, *cfd->ioptions(), cfd->user_comparator(), iter,
        kMaxSequenceNumber,
-        sv->mutable_cf_options.max_sequential_skip_in_iterations);
+        sv->mutable_cf_options.max_sequential_skip_in_iterations,
+        sv->version_number);
 #endif
  } else {
    SequenceNumber latest_snapshot = versions_->LastSequence();
@@ -1468,10 +1458,9 @@ Iterator* DBImpl::NewIterator(const ReadOptions& read_options,
    // likely that any iterator pointer is close to the iterator it points to so
    // that they are likely to be in the same cache line and/or page.
    ArenaWrappedDBIter* db_iter = NewArenaWrappedDbIterator(
-        env_, read_options, *cfd->ioptions(), snapshot,
+        env_, read_options, *cfd->ioptions(), cfd->user_comparator(), snapshot,
        sv->mutable_cf_options.max_sequential_skip_in_iterations,
-        sv->version_number,
-        ((read_options.snapshot != nullptr) ? nullptr : this), cfd);
+        sv->version_number);

    InternalIterator* internal_iter =
        NewInternalIterator(read_options, cfd, sv, db_iter->GetArena(),
@@ -1522,7 +1511,8 @@ Status DBImpl::NewIterators(
      iterators->push_back(NewDBIterator(
          env_, read_options, *cfd->ioptions(), cfd->user_comparator(), iter,
          kMaxSequenceNumber,
-          sv->mutable_cf_options.max_sequential_skip_in_iterations));
+          sv->mutable_cf_options.max_sequential_skip_in_iterations,
+          sv->version_number));
    }
 #endif
  } else {
@@ -1540,10 +1530,9 @@ Status DBImpl::NewIterators(
              : latest_snapshot;

      ArenaWrappedDBIter* db_iter = NewArenaWrappedDbIterator(
-          env_, read_options, *cfd->ioptions(), snapshot,
-          sv->mutable_cf_options.max_sequential_skip_in_iterations,
-          sv->version_number,
-          ((read_options.snapshot != nullptr) ? nullptr : this), cfd);
+          env_, read_options, *cfd->ioptions(), cfd->user_comparator(),
+          snapshot, sv->mutable_cf_options.max_sequential_skip_in_iterations,
+          sv->version_number);
      InternalIterator* internal_iter =
          NewInternalIterator(read_options, cfd, sv, db_iter->GetArena(),
                              db_iter->GetRangeDelAggregator());
@@ -2304,7 +2293,7 @@ Status DestroyDB(const std::string& dbname, const Options& options) {
    // Delete log files in the WAL dir
    for (const auto& file : walDirFiles) {
      if (ParseFileName(file, &number, &type) && type == kLogFile) {
-        Status del = env->DeleteFile(LogFileName(soptions.wal_dir, number));
+        Status del = env->DeleteFile(soptions.wal_dir + "/" + file);
        if (result.ok() && !del.ok()) {
          result = del;
        }
@@ -2746,54 +2735,6 @@ Status DBImpl::IngestExternalFile(
  return status;
 }

-Status DBImpl::VerifyChecksum() {
-  Status s;
-  Options options;
-  EnvOptions env_options;
-  std::vector<ColumnFamilyData*> cfd_list;
-  {
-    InstrumentedMutexLock l(&mutex_);
-    for (auto cfd : *versions_->GetColumnFamilySet()) {
-      if (!cfd->IsDropped() && cfd->initialized()) {
-        cfd->Ref();
-        cfd_list.push_back(cfd);
-      }
-    }
-  }
-  std::vector<SuperVersion*> sv_list;
-  for (auto cfd : cfd_list) {
-    sv_list.push_back(cfd->GetReferencedSuperVersion(&mutex_));
-  }
-  for (auto& sv : sv_list) {
-    VersionStorageInfo* vstorage = sv->current->storage_info();
-    for (int i = 0; i < vstorage->num_non_empty_levels() && s.ok(); i++) {
-      for (size_t j = 0; j < vstorage->LevelFilesBrief(i).num_files && s.ok();
-           j++) {
-        const auto& fd = vstorage->LevelFilesBrief(i).files[j].fd;
-        std::string fname = TableFileName(immutable_db_options_.db_paths,
-                                          fd.GetNumber(), fd.GetPathId());
-        s = rocksdb::VerifySstFileChecksum(options, env_options, fname);
-      }
-    }
-    if (!s.ok()) {
-      break;
-    }
-  }
-  {
-    InstrumentedMutexLock l(&mutex_);
-    for (auto sv : sv_list) {
-      if (sv && sv->Unref()) {
-        sv->Cleanup();
-        delete sv;
-      }
-    }
-    for (auto cfd : cfd_list) {
-        cfd->Unref();
-    }
-  }
-  return s;
-}
-
 void DBImpl::NotifyOnExternalFileIngested(
    ColumnFamilyData* cfd, const ExternalSstFileIngestionJob& ingestion_job) {
 #ifndef ROCKSDB_LITE
@@ -235,11 +235,11 @@ class DBImpl : public DB {
      ColumnFamilyHandle* column_family,
      ColumnFamilyMetaData* metadata) override;

+  // experimental API
  Status SuggestCompactRange(ColumnFamilyHandle* column_family,
-                             const Slice* begin, const Slice* end) override;
+                             const Slice* begin, const Slice* end);

-  Status PromoteL0(ColumnFamilyHandle* column_family,
-                   int target_level) override;
+  Status PromoteL0(ColumnFamilyHandle* column_family, int target_level);

  // Similar to Write() but will call the callback once on the single write
  // thread to determine whether it is safe to perform the write.
@@ -293,8 +293,6 @@ class DBImpl : public DB {
      const std::vector<std::string>& external_files,
      const IngestExternalFileOptions& ingestion_options) override;

-  virtual Status VerifyChecksum() override;
-
 #endif  // ROCKSDB_LITE

  // Similar to GetSnapshot(), but also lets the db know that this snapshot
@@ -496,12 +494,6 @@ class DBImpl : public DB {

  const WriteController& write_controller() { return write_controller_; }

-  InternalIterator* NewInternalIterator(const ReadOptions&,
-                                        ColumnFamilyData* cfd,
-                                        SuperVersion* super_version,
-                                        Arena* arena,
-                                        RangeDelAggregator* range_del_agg);
-
  // hollow transactions shell used for recovery.
  // these will then be passed to TransactionDB so that
  // locks can be reacquired before writing can resume.
@@ -560,7 +552,6 @@ class DBImpl : public DB {
  void AddToLogsToFreeQueue(log::Writer* log_writer) {
    logs_to_free_queue_.push_back(log_writer);
  }
-  InstrumentedMutex* mutex() { return &mutex_; }

  Status NewDB();

@@ -575,6 +566,12 @@ class DBImpl : public DB {
  std::unordered_map<std::string, RecoveredTransaction*>
      recovered_transactions_;

+  InternalIterator* NewInternalIterator(const ReadOptions&,
+                                        ColumnFamilyData* cfd,
+                                        SuperVersion* super_version,
+                                        Arena* arena,
+                                        RangeDelAggregator* range_del_agg);
+
  // Except in DB::Open(), WriteOptionsFile can only be called when:
  // Persist options to options file.
  // If need_mutex_lock = false, the method will lock DB mutex.
@@ -616,18 +613,16 @@ class DBImpl : public DB {
  Status WriteImpl(const WriteOptions& options, WriteBatch* updates,
                   WriteCallback* callback = nullptr,
                   uint64_t* log_used = nullptr, uint64_t log_ref = 0,
-                   bool disable_memtable = false, uint64_t* seq_used = nullptr);
+                   bool disable_memtable = false);

  Status PipelinedWriteImpl(const WriteOptions& options, WriteBatch* updates,
                            WriteCallback* callback = nullptr,
                            uint64_t* log_used = nullptr, uint64_t log_ref = 0,
-                            bool disable_memtable = false,
-                            uint64_t* seq_used = nullptr);
+                            bool disable_memtable = false);

  Status WriteImplWALOnly(const WriteOptions& options, WriteBatch* updates,
                          WriteCallback* callback = nullptr,
-                          uint64_t* log_used = nullptr, uint64_t log_ref = 0,
-                          uint64_t* seq_used = nullptr);
+                          uint64_t* log_used = nullptr, uint64_t log_ref = 0);

  uint64_t FindMinLogContainingOutstandingPrep();
  uint64_t FindMinPrepLogReferencedByMemTable();
@@ -635,9 +630,7 @@ class DBImpl : public DB {
 private:
  friend class DB;
  friend class InternalStats;
-  friend class PessimisticTransaction;
-  friend class WriteCommittedTxn;
-  friend class WritePreparedTxn;
+  friend class TransactionImpl;
 #ifndef ROCKSDB_LITE
  friend class ForwardIterator;
 #endif
@@ -662,7 +655,6 @@ class DBImpl : public DB {
    }
  };

-  struct PrepickedCompaction;
  struct PurgeFileInfo;

  // Recover the descriptor from persistent storage.  May do a significant
@@ -804,19 +796,14 @@ class DBImpl : public DB {
  void SchedulePendingPurge(std::string fname, FileType type, uint64_t number,
                            uint32_t path_id, int job_id);
  static void BGWorkCompaction(void* arg);
-  // Runs a pre-chosen universal compaction involving bottom level in a
-  // separate, bottom-pri thread pool.
-  static void BGWorkBottomCompaction(void* arg);
  static void BGWorkFlush(void* db);
  static void BGWorkPurge(void* arg);
  static void UnscheduleCallback(void* arg);
-  void BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction,
-                                Env::Priority bg_thread_pri);
+  void BackgroundCallCompaction(void* arg);
  void BackgroundCallFlush();
  void BackgroundCallPurge();
  Status BackgroundCompaction(bool* madeProgress, JobContext* job_context,
-                              LogBuffer* log_buffer,
-                              PrepickedCompaction* prepicked_compaction);
+                              LogBuffer* log_buffer, void* m = 0);
  Status BackgroundFlush(bool* madeProgress, JobContext* job_context,
                         LogBuffer* log_buffer);

@@ -1069,10 +1056,6 @@ class DBImpl : public DB {
  int unscheduled_flushes_;
  int unscheduled_compactions_;

-  // count how many background compactions are running or have been scheduled in
-  // the BOTTOM pool
-  int bg_bottom_compaction_scheduled_;
-
  // count how many background compactions are running or have been scheduled
  int bg_compaction_scheduled_;

@@ -1089,7 +1072,7 @@ class DBImpl : public DB {
  int bg_purge_scheduled_;

  // Information for a manual compaction
-  struct ManualCompactionState {
+  struct ManualCompaction {
    ColumnFamilyData* cfd;
    int input_level;
    int output_level;
@@ -1105,21 +1088,13 @@ class DBImpl : public DB {
    InternalKey* manual_end;      // how far we are compacting
    InternalKey tmp_storage;      // Used to keep track of compaction progress
    InternalKey tmp_storage1;     // Used to keep track of compaction progress
-  };
-  struct PrepickedCompaction {
-    // background compaction takes ownership of `compaction`.
    Compaction* compaction;
-    // caller retains ownership of `manual_compaction_state` as it is reused
-    // across background compactions.
-    ManualCompactionState* manual_compaction_state;  // nullptr if non-manual
  };
-  std::deque<ManualCompactionState*> manual_compaction_dequeue_;
+  std::deque<ManualCompaction*> manual_compaction_dequeue_;

  struct CompactionArg {
-    // caller retains ownership of `db`.
    DBImpl* db;
-    // background compaction takes ownership of `prepicked_compaction`.
-    PrepickedCompaction* prepicked_compaction;
+    ManualCompaction* m;
  };

  // Have we encountered a background error in paranoid mode?
@@ -1253,11 +1228,11 @@ class DBImpl : public DB {

  bool HasPendingManualCompaction();
  bool HasExclusiveManualCompaction();
-  void AddManualCompaction(ManualCompactionState* m);
-  void RemoveManualCompaction(ManualCompactionState* m);
-  bool ShouldntRunManualCompaction(ManualCompactionState* m);
+  void AddManualCompaction(ManualCompaction* m);
+  void RemoveManualCompaction(ManualCompaction* m);
+  bool ShouldntRunManualCompaction(ManualCompaction* m);
  bool HaveManualCompaction(ColumnFamilyData* cfd);
-  bool MCOverlap(ManualCompactionState* m, ManualCompactionState* m1);
+  bool MCOverlap(ManualCompaction* m, ManualCompaction* m1);

  size_t GetWalPreallocateBlockSize(uint64_t write_buffer_size) const;

@@ -612,8 +612,7 @@ Status DBImpl::CompactFilesImpl(
 Status DBImpl::PauseBackgroundWork() {
  InstrumentedMutexLock guard_lock(&mutex_);
  bg_compaction_paused_++;
-  while (bg_bottom_compaction_scheduled_ > 0 || bg_compaction_scheduled_ > 0 ||
-         bg_flush_scheduled_ > 0) {
+  while (bg_compaction_scheduled_ > 0 || bg_flush_scheduled_ > 0) {
    bg_cv_.Wait();
  }
  bg_work_paused_++;
@@ -809,7 +808,7 @@ Status DBImpl::RunManualCompaction(ColumnFamilyData* cfd, int input_level,

  bool scheduled = false;
  bool manual_conflict = false;
-  ManualCompactionState manual;
+  ManualCompaction manual;
  manual.cfd = cfd;
  manual.input_level = input_level;
  manual.output_level = output_level;
@@ -859,8 +858,7 @@ Status DBImpl::RunManualCompaction(ColumnFamilyData* cfd, int input_level,
  AddManualCompaction(&manual);
  TEST_SYNC_POINT_CALLBACK("DBImpl::RunManualCompaction:NotScheduled", &mutex_);
  if (exclusive) {
-    while (bg_bottom_compaction_scheduled_ > 0 ||
-           bg_compaction_scheduled_ > 0) {
+    while (bg_compaction_scheduled_ > 0) {
      TEST_SYNC_POINT("DBImpl::RunManualCompaction:WaitScheduled");
      ROCKS_LOG_INFO(
          immutable_db_options_.info_log,
@@ -880,14 +878,14 @@ Status DBImpl::RunManualCompaction(ColumnFamilyData* cfd, int input_level,
  while (!manual.done) {
    assert(HasPendingManualCompaction());
    manual_conflict = false;
-    Compaction* compaction;
    if (ShouldntRunManualCompaction(&manual) || (manual.in_progress == true) ||
        scheduled ||
-        ((manual.manual_end = &manual.tmp_storage1) &&
-         ((compaction = manual.cfd->CompactRange(
-               *manual.cfd->GetLatestMutableCFOptions(), manual.input_level,
-               manual.output_level, manual.output_path_id, manual.begin,
-               manual.end, &manual.manual_end, &manual_conflict)) == nullptr) &&
+        ((manual.manual_end = &manual.tmp_storage1)&&(
+             (manual.compaction = manual.cfd->CompactRange(
+                  *manual.cfd->GetLatestMutableCFOptions(), manual.input_level,
+                  manual.output_level, manual.output_path_id, manual.begin,
+                  manual.end, &manual.manual_end, &manual_conflict)) ==
+             nullptr) &&
         manual_conflict)) {
      // exclusive manual compactions should not see a conflict during
      // CompactRange
@@ -900,16 +898,14 @@ Status DBImpl::RunManualCompaction(ColumnFamilyData* cfd, int input_level,
        manual.incomplete = false;
      }
    } else if (!scheduled) {
-      if (compaction == nullptr) {
+      if (manual.compaction == nullptr) {
        manual.done = true;
        bg_cv_.SignalAll();
        continue;
      }
      ca = new CompactionArg;
      ca->db = this;
-      ca->prepicked_compaction = new PrepickedCompaction;
-      ca->prepicked_compaction->manual_compaction_state = &manual;
-      ca->prepicked_compaction->compaction = compaction;
+      ca->m = &manual;
      manual.incomplete = false;
      bg_compaction_scheduled_++;
      env_->Schedule(&DBImpl::BGWorkCompaction, ca, Env::Priority::LOW, this,
@@ -1051,7 +1047,7 @@ void DBImpl::MaybeScheduleFlushOrCompaction() {
         unscheduled_compactions_ > 0) {
    CompactionArg* ca = new CompactionArg;
    ca->db = this;
-    ca->prepicked_compaction = nullptr;
+    ca->m = nullptr;
    bg_compaction_scheduled_++;
    unscheduled_compactions_--;
    env_->Schedule(&DBImpl::BGWorkCompaction, ca, Env::Priority::LOW, this,
@@ -1156,23 +1152,7 @@ void DBImpl::BGWorkCompaction(void* arg) {
  delete reinterpret_cast<CompactionArg*>(arg);
  IOSTATS_SET_THREAD_POOL_ID(Env::Priority::LOW);
  TEST_SYNC_POINT("DBImpl::BGWorkCompaction");
-  auto prepicked_compaction =
-      static_cast<PrepickedCompaction*>(ca.prepicked_compaction);
-  reinterpret_cast<DBImpl*>(ca.db)->BackgroundCallCompaction(
-      prepicked_compaction, Env::Priority::LOW);
-  delete prepicked_compaction;
-}
-
-void DBImpl::BGWorkBottomCompaction(void* arg) {
-  CompactionArg ca = *(static_cast<CompactionArg*>(arg));
-  delete static_cast<CompactionArg*>(arg);
-  IOSTATS_SET_THREAD_POOL_ID(Env::Priority::BOTTOM);
-  TEST_SYNC_POINT("DBImpl::BGWorkBottomCompaction");
-  auto* prepicked_compaction = ca.prepicked_compaction;
-  assert(prepicked_compaction && prepicked_compaction->compaction &&
-         !prepicked_compaction->manual_compaction_state);
-  ca.db->BackgroundCallCompaction(prepicked_compaction, Env::Priority::BOTTOM);
-  delete prepicked_compaction;
+  reinterpret_cast<DBImpl*>(ca.db)->BackgroundCallCompaction(ca.m);
 }

 void DBImpl::BGWorkPurge(void* db) {
@@ -1185,11 +1165,8 @@ void DBImpl::BGWorkPurge(void* db) {
 void DBImpl::UnscheduleCallback(void* arg) {
  CompactionArg ca = *(reinterpret_cast<CompactionArg*>(arg));
  delete reinterpret_cast<CompactionArg*>(arg);
-  if (ca.prepicked_compaction != nullptr) {
-    if (ca.prepicked_compaction->compaction != nullptr) {
-      delete ca.prepicked_compaction->compaction;
-    }
-    delete ca.prepicked_compaction;
+  if ((ca.m != nullptr) && (ca.m->compaction != nullptr)) {
+    delete ca.m->compaction;
  }
  TEST_SYNC_POINT("DBImpl::UnscheduleCallback");
 }
@@ -1316,9 +1293,9 @@ void DBImpl::BackgroundCallFlush() {
  }
 }

-void DBImpl::BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction,
-                                      Env::Priority bg_thread_pri) {
+void DBImpl::BackgroundCallCompaction(void* arg) {
  bool made_progress = false;
+  ManualCompaction* m = reinterpret_cast<ManualCompaction*>(arg);
  JobContext job_context(next_job_id_.fetch_add(1), true);
  TEST_SYNC_POINT("BackgroundCallCompaction:0");
  MaybeDumpStats();
@@ -1336,11 +1313,9 @@ void DBImpl::BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction,
    auto pending_outputs_inserted_elem =
        CaptureCurrentFileNumberInPendingOutputs();

-    assert((bg_thread_pri == Env::Priority::BOTTOM &&
-            bg_bottom_compaction_scheduled_) ||
-           (bg_thread_pri == Env::Priority::LOW && bg_compaction_scheduled_));
-    Status s = BackgroundCompaction(&made_progress, &job_context, &log_buffer,
-                                    prepicked_compaction);
+    assert(bg_compaction_scheduled_);
+    Status s =
+        BackgroundCompaction(&made_progress, &job_context, &log_buffer, m);
    TEST_SYNC_POINT("BackgroundCallCompaction:1");
    if (!s.ok() && !s.IsShutdownInProgress()) {
      // Wait a little bit before retrying background compaction in
@@ -1386,24 +1361,17 @@ void DBImpl::BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction,

    assert(num_running_compactions_ > 0);
    num_running_compactions_--;
-    if (bg_thread_pri == Env::Priority::LOW) {
-      bg_compaction_scheduled_--;
-    } else {
-      assert(bg_thread_pri == Env::Priority::BOTTOM);
-      bg_bottom_compaction_scheduled_--;
-    }
+    bg_compaction_scheduled_--;

    versions_->GetColumnFamilySet()->FreeDeadColumnFamilies();

    // See if there's more work to be done
    MaybeScheduleFlushOrCompaction();
-    if (made_progress ||
-        (bg_compaction_scheduled_ == 0 &&
-         bg_bottom_compaction_scheduled_ == 0) ||
+    if (made_progress || bg_compaction_scheduled_ == 0 ||
        HasPendingManualCompaction()) {
      // signal if
      // * made_progress -- need to wakeup DelayWrite
-      // * bg_{bottom,}_compaction_scheduled_ == 0 -- need to wakeup ~DBImpl
+      // * bg_compaction_scheduled_ == 0 -- need to wakeup ~DBImpl
      // * HasPendingManualCompaction -- need to wakeup RunManualCompaction
      // If none of this is true, there is no need to signal since nobody is
      // waiting for it
@@ -1418,23 +1386,14 @@ void DBImpl::BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction,

 Status DBImpl::BackgroundCompaction(bool* made_progress,
                                    JobContext* job_context,
-                                    LogBuffer* log_buffer,
-                                    PrepickedCompaction* prepicked_compaction) {
-  ManualCompactionState* manual_compaction =
-      prepicked_compaction == nullptr
-          ? nullptr
-          : prepicked_compaction->manual_compaction_state;
+                                    LogBuffer* log_buffer, void* arg) {
+  ManualCompaction* manual_compaction =
+      reinterpret_cast<ManualCompaction*>(arg);
  *made_progress = false;
  mutex_.AssertHeld();
  TEST_SYNC_POINT("DBImpl::BackgroundCompaction:Start");

  bool is_manual = (manual_compaction != nullptr);
-  unique_ptr<Compaction> c;
-  if (prepicked_compaction != nullptr &&
-      prepicked_compaction->compaction != nullptr) {
-    c.reset(prepicked_compaction->compaction);
-  }
-  bool is_prepicked = is_manual || c;

  // (manual_compaction->in_progress == false);
  bool trivial_move_disallowed =
@@ -1451,6 +1410,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
      manual_compaction->status = status;
      manual_compaction->done = true;
      manual_compaction->in_progress = false;
+      delete manual_compaction->compaction;
      manual_compaction = nullptr;
    }
    return status;
@@ -1461,11 +1421,13 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
    manual_compaction->in_progress = true;
  }

+  unique_ptr<Compaction> c;
  // InternalKey manual_end_storage;
  // InternalKey* manual_end = &manual_end_storage;
  if (is_manual) {
-    ManualCompactionState* m = manual_compaction;
+    ManualCompaction* m = manual_compaction;
    assert(m->in_progress);
+    c.reset(std::move(m->compaction));
    if (!c) {
      m->done = true;
      m->manual_end = nullptr;
@@ -1487,7 +1449,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
               ? "(end)"
               : m->manual_end->DebugString().c_str()));
    }
-  } else if (!is_prepicked && !compaction_queue_.empty()) {
+  } else if (!compaction_queue_.empty()) {
    if (HaveManualCompaction(compaction_queue_.front())) {
      // Can't compact right now, but try again later
      TEST_SYNC_POINT("DBImpl::BackgroundCompaction()::Conflict");
@@ -1639,28 +1601,6 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,

    // Clear Instrument
    ThreadStatusUtil::ResetThreadStatus();
-  } else if (c->column_family_data()->ioptions()->compaction_style ==
-                 kCompactionStyleUniversal &&
-             !is_prepicked && c->output_level() > 0 &&
-             c->output_level() ==
-                 c->column_family_data()
-                     ->current()
-                     ->storage_info()
-                     ->MaxOutputLevel(
-                         immutable_db_options_.allow_ingest_behind) &&
-             env_->GetBackgroundThreads(Env::Priority::BOTTOM) > 0) {
-    // Forward universal compactions involving last level to the bottom pool
-    // if it exists, such that long-running compactions can't block short-
-    // lived ones, like L0->L0s.
-    TEST_SYNC_POINT("DBImpl::BackgroundCompaction:ForwardToBottomPriPool");
-    CompactionArg* ca = new CompactionArg;
-    ca->db = this;
-    ca->prepicked_compaction = new PrepickedCompaction;
-    ca->prepicked_compaction->compaction = c.release();
-    ca->prepicked_compaction->manual_compaction_state = nullptr;
-    ++bg_bottom_compaction_scheduled_;
-    env_->Schedule(&DBImpl::BGWorkBottomCompaction, ca, Env::Priority::BOTTOM,
-                   this, &DBImpl::UnscheduleCallback);
  } else {
    int output_level  __attribute__((unused)) = c->output_level();
    TEST_SYNC_POINT_CALLBACK("DBImpl::BackgroundCompaction:NonTrivial",
@@ -1724,7 +1664,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
  }

  if (is_manual) {
-    ManualCompactionState* m = manual_compaction;
+    ManualCompaction* m = manual_compaction;
    if (!status.ok()) {
      m->status = status;
      m->done = true;
@@ -1767,13 +1707,13 @@ bool DBImpl::HasPendingManualCompaction() {
  return (!manual_compaction_dequeue_.empty());
 }

-void DBImpl::AddManualCompaction(DBImpl::ManualCompactionState* m) {
+void DBImpl::AddManualCompaction(DBImpl::ManualCompaction* m) {
  manual_compaction_dequeue_.push_back(m);
 }

-void DBImpl::RemoveManualCompaction(DBImpl::ManualCompactionState* m) {
+void DBImpl::RemoveManualCompaction(DBImpl::ManualCompaction* m) {
  // Remove from queue
-  std::deque<ManualCompactionState*>::iterator it =
+  std::deque<ManualCompaction*>::iterator it =
      manual_compaction_dequeue_.begin();
  while (it != manual_compaction_dequeue_.end()) {
    if (m == (*it)) {
@@ -1786,17 +1726,16 @@ void DBImpl::RemoveManualCompaction(DBImpl::ManualCompactionState* m) {
  return;
 }

-bool DBImpl::ShouldntRunManualCompaction(ManualCompactionState* m) {
+bool DBImpl::ShouldntRunManualCompaction(ManualCompaction* m) {
  if (num_running_ingest_file_ > 0) {
    // We need to wait for other IngestExternalFile() calls to finish
    // before running a manual compaction.
    return true;
  }
  if (m->exclusive) {
-    return (bg_bottom_compaction_scheduled_ > 0 ||
-            bg_compaction_scheduled_ > 0);
+    return (bg_compaction_scheduled_ > 0);
  }
-  std::deque<ManualCompactionState*>::iterator it =
+  std::deque<ManualCompaction*>::iterator it =
      manual_compaction_dequeue_.begin();
  bool seen = false;
  while (it != manual_compaction_dequeue_.end()) {
@@ -1817,7 +1756,7 @@ bool DBImpl::ShouldntRunManualCompaction(ManualCompactionState* m) {

 bool DBImpl::HaveManualCompaction(ColumnFamilyData* cfd) {
  // Remove from priority queue
-  std::deque<ManualCompactionState*>::iterator it =
+  std::deque<ManualCompaction*>::iterator it =
      manual_compaction_dequeue_.begin();
  while (it != manual_compaction_dequeue_.end()) {
    if ((*it)->exclusive) {
@@ -1835,7 +1774,7 @@ bool DBImpl::HaveManualCompaction(ColumnFamilyData* cfd) {

 bool DBImpl::HasExclusiveManualCompaction() {
  // Remove from priority queue
-  std::deque<ManualCompactionState*>::iterator it =
+  std::deque<ManualCompaction*>::iterator it =
      manual_compaction_dequeue_.begin();
  while (it != manual_compaction_dequeue_.end()) {
    if ((*it)->exclusive) {
@@ -1846,7 +1785,7 @@ bool DBImpl::HasExclusiveManualCompaction() {
  return false;
 }

-bool DBImpl::MCOverlap(ManualCompactionState* m, ManualCompactionState* m1) {
+bool DBImpl::MCOverlap(ManualCompaction* m, ManualCompaction* m1) {
  if ((m->exclusive) || (m1->exclusive)) {
    return true;
  }
@@ -112,9 +112,7 @@ Status DBImpl::TEST_WaitForCompact() {
  // OR flush to finish.

  InstrumentedMutexLock l(&mutex_);
-  while ((bg_bottom_compaction_scheduled_ || bg_compaction_scheduled_ ||
-          bg_flush_scheduled_) &&
-         bg_error_.ok()) {
+  while ((bg_compaction_scheduled_ || bg_flush_scheduled_) && bg_error_.ok()) {
    bg_cv_.Wait();
  }
  return bg_error_;
@@ -368,9 +368,6 @@ void DBImpl::PurgeObsoleteFiles(const JobContext& state, bool schedule_only) {
    candidate_files.emplace_back(
        MakeTableFileName(kDumbDbName, file->fd.GetNumber()),
        file->fd.GetPathId());
-    if (file->table_reader_handle) {
-      table_cache_->Release(file->table_reader_handle);
-    }
    delete file;
  }

@@ -58,7 +58,7 @@ Iterator* DBImplReadOnly::NewIterator(const ReadOptions& read_options,
  SuperVersion* super_version = cfd->GetSuperVersion()->Ref();
  SequenceNumber latest_snapshot = versions_->LastSequence();
  auto db_iter = NewArenaWrappedDbIterator(
-      env_, read_options, *cfd->ioptions(),
+      env_, read_options, *cfd->ioptions(), cfd->user_comparator(),
      (read_options.snapshot != nullptr
           ? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
                 ->number_
@@ -87,7 +87,7 @@ Status DBImplReadOnly::NewIterators(
    auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(cfh)->cfd();
    auto* sv = cfd->GetSuperVersion()->Ref();
    auto* db_iter = NewArenaWrappedDbIterator(
-        env_, read_options, *cfd->ioptions(),
+        env_, read_options, *cfd->ioptions(), cfd->user_comparator(),
        (read_options.snapshot != nullptr
             ? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
                   ->number_
@@ -60,7 +60,7 @@ Status DBImpl::WriteWithCallback(const WriteOptions& write_options,
 Status DBImpl::WriteImpl(const WriteOptions& write_options,
                         WriteBatch* my_batch, WriteCallback* callback,
                         uint64_t* log_used, uint64_t log_ref,
-                         bool disable_memtable, uint64_t* seq_used) {
+                         bool disable_memtable) {
  if (my_batch == nullptr) {
    return Status::Corruption("Batch is nullptr!");
  }
@@ -79,12 +79,12 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,

  if (concurrent_prepare_ && disable_memtable) {
    return WriteImplWALOnly(write_options, my_batch, callback, log_used,
-                            log_ref, seq_used);
+                            log_ref);
  }

  if (immutable_db_options_.enable_pipelined_write) {
    return PipelinedWriteImpl(write_options, my_batch, callback, log_used,
-                              log_ref, disable_memtable, seq_used);
+                              log_ref, disable_memtable);
  }

  PERF_TIMER_GUARD(write_pre_and_post_process_time);
@@ -127,9 +127,6 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
    if (log_used != nullptr) {
      *log_used = w.log_used;
    }
-    if (seq_used != nullptr) {
-      *seq_used = w.sequence;
-    }
    // write is complete and leader has updated sequence
    return w.FinalStatus();
  }
@@ -281,9 +278,6 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
              write_options.ignore_missing_column_families, 0 /*log_number*/,
              this, true /*concurrent_memtable_writes*/);
        }
-        if (seq_used != nullptr) {
-          *seq_used = w.sequence;
-        }
      }
    }
  }
@@ -331,7 +325,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
 Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
                                  WriteBatch* my_batch, WriteCallback* callback,
                                  uint64_t* log_used, uint64_t log_ref,
-                                  bool disable_memtable, uint64_t* seq_used) {
+                                  bool disable_memtable) {
  PERF_TIMER_GUARD(write_pre_and_post_process_time);
  StopWatch write_sw(env_, immutable_db_options_.statistics.get(), DB_WRITE);

@@ -446,9 +440,6 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
      write_thread_.ExitAsMemTableWriter(&w, *w.write_group);
    }
  }
-  if (seq_used != nullptr) {
-    *seq_used = w.sequence;
-  }

  assert(w.state == WriteThread::STATE_COMPLETED);
  return w.FinalStatus();
@@ -456,8 +447,7 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,

 Status DBImpl::WriteImplWALOnly(const WriteOptions& write_options,
                                WriteBatch* my_batch, WriteCallback* callback,
-                                uint64_t* log_used, uint64_t log_ref,
-                                uint64_t* seq_used) {
+                                uint64_t* log_used, uint64_t log_ref) {
  Status status;
  PERF_TIMER_GUARD(write_pre_and_post_process_time);
  WriteThread::Writer w(write_options, my_batch, callback, log_ref,
@@ -474,9 +464,6 @@ Status DBImpl::WriteImplWALOnly(const WriteOptions& write_options,
    if (log_used != nullptr) {
      *log_used = w.log_used;
    }
-    if (seq_used != nullptr) {
-      *seq_used = w.sequence;
-    }
    return w.FinalStatus();
  }
  // else we are the leader of the write batch group
@@ -522,13 +509,6 @@ Status DBImpl::WriteImplWALOnly(const WriteOptions& write_options,
  // wal_write_mutex_ to ensure ordered events in WAL
  status = ConcurrentWriteToWAL(write_group, log_used, &last_sequence,
                                0 /*total_count*/);
-  auto curr_seq = last_sequence + 1;
-  for (auto* writer : write_group) {
-    if (writer->CheckCallback(this)) {
-      writer->sequence = curr_seq;
-      curr_seq += WriteBatchInternal::Count(writer->batch);
-    }
-  }
  if (status.ok() && write_options.sync) {
    // Requesting sync with concurrent_prepare_ is expected to be very rare. We
    // hance provide a simple implementation that is not necessarily efficient.
@@ -547,9 +527,6 @@ Status DBImpl::WriteImplWALOnly(const WriteOptions& write_options,
  if (status.ok()) {
    status = w.FinalStatus();
  }
-  if (seq_used != nullptr) {
-    *seq_used = w.sequence;
-  }
  return status;
 }

@@ -1081,7 +1058,7 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
  SuperVersion* new_superversion = nullptr;
  const MutableCFOptions mutable_cf_options = *cfd->GetLatestMutableCFOptions();

-  // Set memtable_info for memtable sealed callback
+  // Set current_memtble_info for memtable sealed callback
 #ifndef ROCKSDB_LITE
  MemTableInfo memtable_info;
  memtable_info.cf_name = cfd->GetName();
@@ -204,6 +204,7 @@ TEST_F(DBIOFailureTest, ManifestWriteError) {
    ASSERT_EQ("bar2", Get("foo2"));
  }
 }
+#endif  // ROCKSDB_LITE

 TEST_F(DBIOFailureTest, PutFailsParanoid) {
  // Test the following:
@@ -558,7 +559,6 @@ TEST_F(DBIOFailureTest, CompactionSstSyncError) {
  ASSERT_EQ("bar3", Get(1, "foo"));
 }
 #endif  // !(defined NDEBUG) || !defined(OS_WIN)
-#endif  // ROCKSDB_LITE
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -86,7 +86,6 @@ class DBIter: public Iterator {
      RecordTick(global_statistics, NUMBER_DB_PREV, prev_count_);
      RecordTick(global_statistics, NUMBER_DB_PREV_FOUND, prev_found_count_);
      RecordTick(global_statistics, ITER_BYTES_READ, bytes_read_);
-      PERF_COUNTER_ADD(iter_read_bytes, bytes_read_);
      ResetCounters();
    }

@@ -102,12 +101,12 @@ class DBIter: public Iterator {
    uint64_t bytes_read_;
  };

-  DBIter(Env* _env, const ReadOptions& read_options,
+  DBIter(Env* env, const ReadOptions& read_options,
         const ImmutableCFOptions& cf_options, const Comparator* cmp,
         InternalIterator* iter, SequenceNumber s, bool arena_mode,
-         uint64_t max_sequential_skip_in_iterations)
+         uint64_t max_sequential_skip_in_iterations, uint64_t version_number)
      : arena_mode_(arena_mode),
-        env_(_env),
+        env_(env),
        logger_(cf_options.info_log),
        user_comparator_(cmp),
        merge_operator_(cf_options.merge_operator),
@@ -117,6 +116,7 @@ class DBIter: public Iterator {
        valid_(false),
        current_entry_is_merged_(false),
        statistics_(cf_options.statistics),
+        version_number_(version_number),
        iterate_upper_bound_(read_options.iterate_upper_bound),
        prefix_same_as_start_(read_options.prefix_same_as_start),
        pin_thru_lifetime_(read_options.pin_data),
@@ -188,7 +188,10 @@ class DBIter: public Iterator {
    }
    if (prop_name == "rocksdb.iterator.super-version-number") {
      // First try to pass the value returned from inner iterator.
-      return iter_->GetProperty(prop_name, prop);
+      if (!iter_->GetProperty(prop_name, prop).ok()) {
+        *prop = ToString(version_number_);
+      }
+      return Status::OK();
    } else if (prop_name == "rocksdb.iterator.is-key-pinned") {
      if (valid_) {
        *prop = (pin_thru_lifetime_ && saved_key_.IsKeyPinned()) ? "1" : "0";
@@ -206,9 +209,6 @@ class DBIter: public Iterator {
  virtual void SeekForPrev(const Slice& target) override;
  virtual void SeekToFirst() override;
  virtual void SeekToLast() override;
-  Env* env() { return env_; }
-  void set_sequence(uint64_t s) { sequence_ = s; }
-  void set_valid(bool v) { valid_ = v; }

 private:
  void ReverseToForward();
@@ -260,7 +260,7 @@ class DBIter: public Iterator {
  const Comparator* const user_comparator_;
  const MergeOperator* const merge_operator_;
  InternalIterator* iter_;
-  SequenceNumber sequence_;
+  SequenceNumber const sequence_;

  Status status_;
  IterKey saved_key_;
@@ -274,6 +274,7 @@ class DBIter: public Iterator {
  uint64_t max_skip_;
  uint64_t max_skippable_internal_keys_;
  uint64_t num_internal_keys_skipped_;
+  uint64_t version_number_;
  const Slice* iterate_upper_bound_;
  IterKey prefix_start_buf_;
  Slice prefix_start_key_;
@@ -1015,7 +1016,6 @@ void DBIter::Seek(const Slice& target) {
      if (valid_) {
        RecordTick(statistics_, NUMBER_DB_SEEK_FOUND);
        RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
-        PERF_COUNTER_ADD(iter_read_bytes, key().size() + value().size());
      }
    }
  } else {
@@ -1058,7 +1058,6 @@ void DBIter::SeekForPrev(const Slice& target) {
      if (valid_) {
        RecordTick(statistics_, NUMBER_DB_SEEK_FOUND);
        RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
-        PERF_COUNTER_ADD(iter_read_bytes, key().size() + value().size());
      }
    }
  } else {
@@ -1097,7 +1096,6 @@ void DBIter::SeekToFirst() {
      if (valid_) {
        RecordTick(statistics_, NUMBER_DB_SEEK_FOUND);
        RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
-        PERF_COUNTER_ADD(iter_read_bytes, key().size() + value().size());
      }
    }
  } else {
@@ -1145,7 +1143,6 @@ void DBIter::SeekToLast() {
    if (valid_) {
      RecordTick(statistics_, NUMBER_DB_SEEK_FOUND);
      RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
-      PERF_COUNTER_ADD(iter_read_bytes, key().size() + value().size());
    }
  }
  if (valid_ && prefix_extractor_ && prefix_same_as_start_) {
@@ -1160,15 +1157,18 @@ Iterator* NewDBIterator(Env* env, const ReadOptions& read_options,
                        const Comparator* user_key_comparator,
                        InternalIterator* internal_iter,
                        const SequenceNumber& sequence,
-                        uint64_t max_sequential_skip_in_iterations) {
-  DBIter* db_iter = new DBIter(env, read_options, cf_options,
-                               user_key_comparator, internal_iter, sequence,
-                               false, max_sequential_skip_in_iterations);
+                        uint64_t max_sequential_skip_in_iterations,
+                        uint64_t version_number) {
+  DBIter* db_iter = new DBIter(
+      env, read_options, cf_options, user_key_comparator, internal_iter,
+      sequence, false, max_sequential_skip_in_iterations, version_number);
  return db_iter;
 }

 ArenaWrappedDBIter::~ArenaWrappedDBIter() { db_iter_->~DBIter(); }

+void ArenaWrappedDBIter::SetDBIter(DBIter* iter) { db_iter_ = iter; }
+
 RangeDelAggregator* ArenaWrappedDBIter::GetRangeDelAggregator() {
  return db_iter_->GetRangeDelAggregator();
 }
@@ -1193,67 +1193,26 @@ inline Slice ArenaWrappedDBIter::value() const { return db_iter_->value(); }
 inline Status ArenaWrappedDBIter::status() const { return db_iter_->status(); }
 inline Status ArenaWrappedDBIter::GetProperty(std::string prop_name,
                                              std::string* prop) {
-  if (prop_name == "rocksdb.iterator.super-version-number") {
-    // First try to pass the value returned from inner iterator.
-    if (!db_iter_->GetProperty(prop_name, prop).ok()) {
-      *prop = ToString(sv_number_);
-    }
-    return Status::OK();
-  }
  return db_iter_->GetProperty(prop_name, prop);
 }
-
-void ArenaWrappedDBIter::Init(Env* env, const ReadOptions& read_options,
-                              const ImmutableCFOptions& cf_options,
-                              const SequenceNumber& sequence,
-                              uint64_t max_sequential_skip_in_iteration,
-                              uint64_t version_number) {
-  auto mem = arena_.AllocateAligned(sizeof(DBIter));
-  db_iter_ = new (mem)
-      DBIter(env, read_options, cf_options, cf_options.user_comparator, nullptr,
-             sequence, true, max_sequential_skip_in_iteration);
-  sv_number_ = version_number;
-}
-
-Status ArenaWrappedDBIter::Refresh() {
-  if (cfd_ == nullptr || db_impl_ == nullptr) {
-    return Status::NotSupported("Creating renew iterator is not allowed.");
-  }
-  assert(db_iter_ != nullptr);
-  SequenceNumber latest_seq = db_impl_->GetLatestSequenceNumber();
-  uint64_t cur_sv_number = cfd_->GetSuperVersionNumber();
-  if (sv_number_ != cur_sv_number) {
-    Env* env = db_iter_->env();
-    db_iter_->~DBIter();
-    arena_.~Arena();
-    new (&arena_) Arena();
-
-    SuperVersion* sv = cfd_->GetReferencedSuperVersion(db_impl_->mutex());
-    Init(env, read_options_, *(cfd_->ioptions()), latest_seq,
-         sv->mutable_cf_options.max_sequential_skip_in_iterations,
-         cur_sv_number);
-
-    InternalIterator* internal_iter = db_impl_->NewInternalIterator(
-        read_options_, cfd_, sv, &arena_, db_iter_->GetRangeDelAggregator());
-    SetIterUnderDBIter(internal_iter);
-  } else {
-    db_iter_->set_sequence(latest_seq);
-    db_iter_->set_valid(false);
-  }
-  return Status::OK();
+void ArenaWrappedDBIter::RegisterCleanup(CleanupFunction function, void* arg1,
+                                         void* arg2) {
+  db_iter_->RegisterCleanup(function, arg1, arg2);
 }

 ArenaWrappedDBIter* NewArenaWrappedDbIterator(
    Env* env, const ReadOptions& read_options,
-    const ImmutableCFOptions& cf_options, const SequenceNumber& sequence,
-    uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
-    DBImpl* db_impl, ColumnFamilyData* cfd) {
+    const ImmutableCFOptions& cf_options, const Comparator* user_key_comparator,
+    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations,
+    uint64_t version_number) {
  ArenaWrappedDBIter* iter = new ArenaWrappedDBIter();
-  iter->Init(env, read_options, cf_options, sequence,
-             max_sequential_skip_in_iterations, version_number);
-  if (db_impl != nullptr && cfd != nullptr) {
-    iter->StoreRefreshInfo(read_options, db_impl, cfd);
-  }
+  Arena* arena = iter->GetArena();
+  auto mem = arena->AllocateAligned(sizeof(DBIter));
+  DBIter* db_iter = new (mem)
+      DBIter(env, read_options, cf_options, user_key_comparator, nullptr,
+             sequence, true, max_sequential_skip_in_iterations, version_number);
+
+  iter->SetDBIter(db_iter);

  return iter;
 }
@@ -10,7 +10,6 @@
 #pragma once
 #include <stdint.h>
 #include <string>
-#include "db/db_impl.h"
 #include "db/dbformat.h"
 #include "db/range_del_aggregator.h"
 #include "options/cf_options.h"
@@ -33,7 +32,8 @@ extern Iterator* NewDBIterator(Env* env, const ReadOptions& read_options,
                               const Comparator* user_key_comparator,
                               InternalIterator* internal_iter,
                               const SequenceNumber& sequence,
-                               uint64_t max_sequential_skip_in_iterations);
+                               uint64_t max_sequential_skip_in_iterations,
+                               uint64_t version_number);

 // A wrapper iterator which wraps DB Iterator and the arena, with which the DB
 // iterator is supposed be allocated. This class is used as an entry point of
@@ -49,6 +49,10 @@ class ArenaWrappedDBIter : public Iterator {
  virtual Arena* GetArena() { return &arena_; }
  virtual RangeDelAggregator* GetRangeDelAggregator();

+  // Set the DB Iterator to be wrapped
+
+  virtual void SetDBIter(DBIter* iter);
+
  // Set the internal iterator wrapped inside the DB Iterator. Usually it is
  // a merging iterator.
  virtual void SetIterUnderDBIter(InternalIterator* iter);
@@ -62,39 +66,20 @@ class ArenaWrappedDBIter : public Iterator {
  virtual Slice key() const override;
  virtual Slice value() const override;
  virtual Status status() const override;
-  virtual Status Refresh() override;

+  void RegisterCleanup(CleanupFunction function, void* arg1, void* arg2);
  virtual Status GetProperty(std::string prop_name, std::string* prop) override;

-  void Init(Env* env, const ReadOptions& read_options,
-            const ImmutableCFOptions& cf_options,
-            const SequenceNumber& sequence,
-            uint64_t max_sequential_skip_in_iterations,
-            uint64_t version_number);
-
-  void StoreRefreshInfo(const ReadOptions& read_options, DBImpl* db_impl,
-                        ColumnFamilyData* cfd) {
-    read_options_ = read_options;
-    db_impl_ = db_impl;
-    cfd_ = cfd;
-  }
-
 private:
  DBIter* db_iter_;
  Arena arena_;
-  uint64_t sv_number_;
-  ColumnFamilyData* cfd_ = nullptr;
-  DBImpl* db_impl_ = nullptr;
-  ReadOptions read_options_;
 };

 // Generate the arena wrapped iterator class.
-// `db_impl` and `cfd` are used for reneweal. If left null, renewal will not
-// be supported.
 extern ArenaWrappedDBIter* NewArenaWrappedDbIterator(
    Env* env, const ReadOptions& read_options,
-    const ImmutableCFOptions& cf_options, const SequenceNumber& sequence,
-    uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
-    DBImpl* db_impl = nullptr, ColumnFamilyData* cfd = nullptr);
+    const ImmutableCFOptions& cf_options, const Comparator* user_key_comparator,
+    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations,
+    uint64_t version_number);

 }  // namespace rocksdb
@@ -193,7 +193,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
    ReadOptions ro;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -225,7 +225,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
    ReadOptions ro;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -251,7 +251,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -283,7 +283,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -318,7 +318,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(!db_iter->Valid());
@@ -347,7 +347,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      7, options.max_sequential_skip_in_iterations));
+                      7, options.max_sequential_skip_in_iterations, 0));

    SetPerfLevel(kEnableCount);
    ASSERT_TRUE(GetPerfLevel() == kEnableCount);
@@ -384,7 +384,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      4, options.max_sequential_skip_in_iterations));
+                      4, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -409,7 +409,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(!db_iter->Valid());
@@ -431,7 +431,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -466,7 +466,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      7, options.max_sequential_skip_in_iterations));
+                      7, options.max_sequential_skip_in_iterations, 0));

    SetPerfLevel(kEnableCount);
    ASSERT_TRUE(GetPerfLevel() == kEnableCount);
@@ -495,7 +495,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
    ReadOptions ro;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToFirst();
    ASSERT_TRUE(db_iter->Valid());
@@ -537,7 +537,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
    ReadOptions ro;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      2, options.max_sequential_skip_in_iterations));
+                      2, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "b");
@@ -568,7 +568,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
    ReadOptions ro;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "c");
@@ -597,7 +597,7 @@ TEST_F(DBIteratorTest, DBIteratorEmpty) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      0, options.max_sequential_skip_in_iterations));
+                      0, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(!db_iter->Valid());
  }
@@ -608,7 +608,7 @@ TEST_F(DBIteratorTest, DBIteratorEmpty) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      0, options.max_sequential_skip_in_iterations));
+                      0, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToFirst();
    ASSERT_TRUE(!db_iter->Valid());
  }
@@ -630,7 +630,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkipCountSkips) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 2, options.max_sequential_skip_in_iterations));
+      internal_iter, 2, options.max_sequential_skip_in_iterations, 0));
  db_iter->SeekToLast();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "c");
@@ -673,7 +673,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
      options.statistics = rocksdb::CreateDBStatistics();
      std::unique_ptr<Iterator> db_iter(NewDBIterator(
          env_, ro, cf_options, BytewiseComparator(), internal_iter, i + 2,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));
      db_iter->SeekToLast();
      ASSERT_TRUE(db_iter->Valid());

@@ -708,7 +708,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {

      std::unique_ptr<Iterator> db_iter(NewDBIterator(
          env_, ro, cf_options, BytewiseComparator(), internal_iter, i + 2,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));
      db_iter->SeekToLast();
      ASSERT_TRUE(db_iter->Valid());

@@ -736,7 +736,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {

      std::unique_ptr<Iterator> db_iter(NewDBIterator(
          env_, ro, cf_options, BytewiseComparator(), internal_iter, 202,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));
      db_iter->SeekToLast();
      ASSERT_TRUE(db_iter->Valid());

@@ -768,7 +768,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
      internal_iter->Finish();
      std::unique_ptr<Iterator> db_iter(NewDBIterator(
          env_, ro, cf_options, BytewiseComparator(), internal_iter, i,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));
      db_iter->SeekToLast();
      ASSERT_TRUE(!db_iter->Valid());

@@ -784,7 +784,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
    internal_iter->Finish();
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      200, options.max_sequential_skip_in_iterations));
+                      200, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "c");
@@ -818,7 +818,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {

      std::unique_ptr<Iterator> db_iter(NewDBIterator(
          env_, ro, cf_options, BytewiseComparator(), internal_iter, i + 2,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));
      db_iter->SeekToLast();
      ASSERT_TRUE(db_iter->Valid());

@@ -852,7 +852,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {

      std::unique_ptr<Iterator> db_iter(NewDBIterator(
          env_, ro, cf_options, BytewiseComparator(), internal_iter, i + 2,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));
      db_iter->SeekToLast();
      ASSERT_TRUE(db_iter->Valid());

@@ -900,7 +900,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
    ro.max_skippable_internal_keys = 0;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToFirst();
    ASSERT_TRUE(db_iter->Valid());
@@ -946,7 +946,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
    ro.max_skippable_internal_keys = 2;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToFirst();
    ASSERT_TRUE(db_iter->Valid());
@@ -990,7 +990,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
    ro.max_skippable_internal_keys = 2;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToFirst();
    ASSERT_TRUE(db_iter->Valid());
@@ -1028,7 +1028,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
    ro.max_skippable_internal_keys = 2;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToFirst();
    ASSERT_TRUE(db_iter->Valid());
@@ -1063,7 +1063,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
    ro.max_skippable_internal_keys = 2;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -1093,7 +1093,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
    ro.max_skippable_internal_keys = 2;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToFirst();
    ASSERT_TRUE(db_iter->Valid());
@@ -1130,7 +1130,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
    ro.max_skippable_internal_keys = 2;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToFirst();
    ASSERT_TRUE(db_iter->Valid());
@@ -1167,7 +1167,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
      ro.max_skippable_internal_keys = i;
      std::unique_ptr<Iterator> db_iter(NewDBIterator(
          env_, ro, cf_options, BytewiseComparator(), internal_iter, 2 * i + 1,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));

      db_iter->SeekToFirst();
      ASSERT_TRUE(db_iter->Valid());
@@ -1220,7 +1220,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
      ro.max_skippable_internal_keys = i;
      std::unique_ptr<Iterator> db_iter(NewDBIterator(
          env_, ro, cf_options, BytewiseComparator(), internal_iter, 2 * i + 1,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));

      db_iter->SeekToFirst();
      ASSERT_TRUE(db_iter->Valid());
@@ -1258,7 +1258,7 @@ TEST_F(DBIteratorTest, DBIterator1) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 1, options.max_sequential_skip_in_iterations));
+      internal_iter, 1, options.max_sequential_skip_in_iterations, 0));
  db_iter->SeekToFirst();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1285,7 +1285,7 @@ TEST_F(DBIteratorTest, DBIterator2) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 0, options.max_sequential_skip_in_iterations));
+      internal_iter, 0, options.max_sequential_skip_in_iterations, 0));
  db_iter->SeekToFirst();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1309,7 +1309,7 @@ TEST_F(DBIteratorTest, DBIterator3) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 2, options.max_sequential_skip_in_iterations));
+      internal_iter, 2, options.max_sequential_skip_in_iterations, 0));
  db_iter->SeekToFirst();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1333,7 +1333,7 @@ TEST_F(DBIteratorTest, DBIterator4) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 4, options.max_sequential_skip_in_iterations));
+      internal_iter, 4, options.max_sequential_skip_in_iterations, 0));
  db_iter->SeekToFirst();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1365,7 +1365,7 @@ TEST_F(DBIteratorTest, DBIterator5) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      0, options.max_sequential_skip_in_iterations));
+                      0, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1387,7 +1387,7 @@ TEST_F(DBIteratorTest, DBIterator5) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      1, options.max_sequential_skip_in_iterations));
+                      1, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1409,7 +1409,7 @@ TEST_F(DBIteratorTest, DBIterator5) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      2, options.max_sequential_skip_in_iterations));
+                      2, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1431,7 +1431,7 @@ TEST_F(DBIteratorTest, DBIterator5) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      3, options.max_sequential_skip_in_iterations));
+                      3, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1453,7 +1453,7 @@ TEST_F(DBIteratorTest, DBIterator5) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      4, options.max_sequential_skip_in_iterations));
+                      4, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1475,7 +1475,7 @@ TEST_F(DBIteratorTest, DBIterator5) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      5, options.max_sequential_skip_in_iterations));
+                      5, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1497,7 +1497,7 @@ TEST_F(DBIteratorTest, DBIterator5) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      6, options.max_sequential_skip_in_iterations));
+                      6, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1517,7 +1517,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
    internal_iter->Finish();
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));
    db_iter->Seek("b");
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "b");
@@ -1546,7 +1546,7 @@ TEST_F(DBIteratorTest, DBIterator6) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      0, options.max_sequential_skip_in_iterations));
+                      0, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1568,7 +1568,7 @@ TEST_F(DBIteratorTest, DBIterator6) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      1, options.max_sequential_skip_in_iterations));
+                      1, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1590,7 +1590,7 @@ TEST_F(DBIteratorTest, DBIterator6) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      2, options.max_sequential_skip_in_iterations));
+                      2, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1612,7 +1612,7 @@ TEST_F(DBIteratorTest, DBIterator6) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      3, options.max_sequential_skip_in_iterations));
+                      3, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(!db_iter->Valid());
  }
@@ -1630,7 +1630,7 @@ TEST_F(DBIteratorTest, DBIterator6) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      4, options.max_sequential_skip_in_iterations));
+                      4, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1652,7 +1652,7 @@ TEST_F(DBIteratorTest, DBIterator6) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      5, options.max_sequential_skip_in_iterations));
+                      5, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1674,7 +1674,7 @@ TEST_F(DBIteratorTest, DBIterator6) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      6, options.max_sequential_skip_in_iterations));
+                      6, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1715,7 +1715,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      0, options.max_sequential_skip_in_iterations));
+                      0, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1749,7 +1749,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      2, options.max_sequential_skip_in_iterations));
+                      2, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -1789,7 +1789,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      4, options.max_sequential_skip_in_iterations));
+                      4, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -1829,7 +1829,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      5, options.max_sequential_skip_in_iterations));
+                      5, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -1874,7 +1874,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      6, options.max_sequential_skip_in_iterations));
+                      6, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -1920,7 +1920,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      7, options.max_sequential_skip_in_iterations));
+                      7, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -1960,7 +1960,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      9, options.max_sequential_skip_in_iterations));
+                      9, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -2006,7 +2006,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      13, options.max_sequential_skip_in_iterations));
+                      13, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -2053,7 +2053,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      14, options.max_sequential_skip_in_iterations));
+                      14, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -2083,7 +2083,7 @@ TEST_F(DBIteratorTest, DBIterator8) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 10, options.max_sequential_skip_in_iterations));
+      internal_iter, 10, options.max_sequential_skip_in_iterations, 0));
  db_iter->SeekToLast();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "b");
@@ -2113,7 +2113,7 @@ TEST_F(DBIteratorTest, DBIterator9) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-        internal_iter, 10, options.max_sequential_skip_in_iterations));
+        internal_iter, 10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -2179,7 +2179,7 @@ TEST_F(DBIteratorTest, DBIterator10) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 10, options.max_sequential_skip_in_iterations));
+      internal_iter, 10, options.max_sequential_skip_in_iterations, 0));

  db_iter->Seek("c");
  ASSERT_TRUE(db_iter->Valid());
@@ -2218,7 +2218,7 @@ TEST_F(DBIteratorTest, SeekToLastOccurrenceSeq0) {

  std::unique_ptr<Iterator> db_iter(
      NewDBIterator(env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-                    internal_iter, 10, 0 /* force seek */));
+                    internal_iter, 10, 0 /* force seek */, 0));
  db_iter->SeekToFirst();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -2246,7 +2246,7 @@ TEST_F(DBIteratorTest, DBIterator11) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 1, options.max_sequential_skip_in_iterations));
+      internal_iter, 1, options.max_sequential_skip_in_iterations, 0));
  db_iter->SeekToFirst();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -2272,7 +2272,7 @@ TEST_F(DBIteratorTest, DBIterator12) {

  std::unique_ptr<Iterator> db_iter(
      NewDBIterator(env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-                    internal_iter, 10, 0));
+                    internal_iter, 10, 0, 0));
  db_iter->SeekToLast();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "c");
@@ -2309,7 +2309,7 @@ TEST_F(DBIteratorTest, DBIterator13) {

  std::unique_ptr<Iterator> db_iter(
      NewDBIterator(env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-                    internal_iter, 2, 3));
+                    internal_iter, 2, 3, 0));
  db_iter->Seek("b");
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), key);
@@ -2337,7 +2337,7 @@ TEST_F(DBIteratorTest, DBIterator14) {

  std::unique_ptr<Iterator> db_iter(
      NewDBIterator(env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-                    internal_iter, 4, 1));
+                    internal_iter, 4, 1, 0));
  db_iter->Seek("b");
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "b");
@@ -2376,7 +2376,7 @@ class DBIterWithMergeIterTest : public testing::Test {
    db_iter_.reset(NewDBIterator(env_, ro_, ImmutableCFOptions(options_),
                                 BytewiseComparator(), merge_iter,
                                 8 /* read data earlier than seqId 8 */,
-                                 3 /* max iterators before reseek */));
+                                 3 /* max iterators before reseek */, 0));
  }

  Env* env_;
@@ -1719,15 +1719,12 @@ TEST_F(DBIteratorTest, IteratorWithLocalStatistics) {

  std::vector<port::Thread> threads;
  std::function<void()> reader_func_next = [&]() {
-    SetPerfLevel(kEnableCount);
-    get_perf_context()->Reset();
    Iterator* iter = db_->NewIterator(ReadOptions());

    iter->SeekToFirst();
    // Seek will bump ITER_BYTES_READ
-    uint64_t bytes = 0;
-    bytes += iter->key().size();
-    bytes += iter->value().size();
+    total_bytes += iter->key().size();
+    total_bytes += iter->value().size();
    while (true) {
      iter->Next();
      total_next++;
@@ -1736,25 +1733,20 @@ TEST_F(DBIteratorTest, IteratorWithLocalStatistics) {
        break;
      }
      total_next_found++;
-      bytes += iter->key().size();
-      bytes += iter->value().size();
+      total_bytes += iter->key().size();
+      total_bytes += iter->value().size();
    }

    delete iter;
-    ASSERT_EQ(bytes, get_perf_context()->iter_read_bytes);
-    SetPerfLevel(kDisable);
-    total_bytes += bytes;
  };

  std::function<void()> reader_func_prev = [&]() {
-    SetPerfLevel(kEnableCount);
    Iterator* iter = db_->NewIterator(ReadOptions());

    iter->SeekToLast();
    // Seek will bump ITER_BYTES_READ
-    uint64_t bytes = 0;
-    bytes += iter->key().size();
-    bytes += iter->value().size();
+    total_bytes += iter->key().size();
+    total_bytes += iter->value().size();
    while (true) {
      iter->Prev();
      total_prev++;
@@ -1763,14 +1755,11 @@ TEST_F(DBIteratorTest, IteratorWithLocalStatistics) {
        break;
      }
      total_prev_found++;
-      bytes += iter->key().size();
-      bytes += iter->value().size();
+      total_bytes += iter->key().size();
+      total_bytes += iter->value().size();
    }

    delete iter;
-    ASSERT_EQ(bytes, get_perf_context()->iter_read_bytes);
-    SetPerfLevel(kDisable);
-    total_bytes += bytes;
  };

  for (int i = 0; i < 10; i++) {
@@ -1920,65 +1909,6 @@ TEST_F(DBIteratorTest, DBIteratorSkipRecentDuplicatesTest) {
                 NUMBER_OF_RESEEKS_IN_ITERATION));
 }

-TEST_F(DBIteratorTest, Refresh) {
-  ASSERT_OK(Put("x", "y"));
-
-  std::unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions()));
-  iter->Seek(Slice("a"));
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("x")), 0);
-  iter->Next();
-  ASSERT_FALSE(iter->Valid());
-
-  ASSERT_OK(Put("c", "d"));
-
-  iter->Seek(Slice("a"));
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("x")), 0);
-  iter->Next();
-  ASSERT_FALSE(iter->Valid());
-
-  iter->Refresh();
-
-  iter->Seek(Slice("a"));
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("c")), 0);
-  iter->Next();
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("x")), 0);
-  iter->Next();
-  ASSERT_FALSE(iter->Valid());
-
-  dbfull()->Flush(FlushOptions());
-
-  ASSERT_OK(Put("m", "n"));
-
-  iter->Seek(Slice("a"));
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("c")), 0);
-  iter->Next();
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("x")), 0);
-  iter->Next();
-  ASSERT_FALSE(iter->Valid());
-
-  iter->Refresh();
-
-  iter->Seek(Slice("a"));
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("c")), 0);
-  iter->Next();
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("m")), 0);
-  iter->Next();
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("x")), 0);
-  iter->Next();
-  ASSERT_FALSE(iter->Valid());
-
-  iter.reset();
-}
-
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -868,134 +868,6 @@ TEST_F(DBRangeDelTest, SubcompactionHasEmptyDedicatedRangeDelFile) {
  db_->ReleaseSnapshot(snapshot);
 }

-TEST_F(DBRangeDelTest, MemtableBloomFilter) {
-  // regression test for #2743. the range delete tombstones in memtable should
-  // be added even when Get() skips searching due to its prefix bloom filter
-  const int kMemtableSize = 1 << 20;              // 1MB
-  const int kMemtablePrefixFilterSize = 1 << 13;  // 8KB
-  const int kNumKeys = 1000;
-  const int kPrefixLen = 8;
-  Options options = CurrentOptions();
-  options.memtable_prefix_bloom_size_ratio =
-      static_cast<double>(kMemtablePrefixFilterSize) / kMemtableSize;
-  options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(kPrefixLen));
-  options.write_buffer_size = kMemtableSize;
-  Reopen(options);
-
-  for (int i = 0; i < kNumKeys; ++i) {
-    ASSERT_OK(Put(Key(i), "val"));
-  }
-  Flush();
-  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(0),
-                             Key(kNumKeys)));
-  for (int i = 0; i < kNumKeys; ++i) {
-    std::string value;
-    ASSERT_TRUE(db_->Get(ReadOptions(), Key(i), &value).IsNotFound());
-  }
-}
-
-TEST_F(DBRangeDelTest, CompactionTreatsSplitInputLevelDeletionAtomically) {
-  // make sure compaction treats files containing a split range deletion in the
-  // input level as an atomic unit. I.e., compacting any input-level file(s)
-  // containing a portion of the range deletion causes all other input-level
-  // files containing portions of that same range deletion to be included in the
-  // compaction.
-  const int kNumFilesPerLevel = 4, kValueBytes = 4 << 10;
-  Options options = CurrentOptions();
-  options.compression = kNoCompression;
-  options.level0_file_num_compaction_trigger = kNumFilesPerLevel;
-  options.memtable_factory.reset(
-      new SpecialSkipListFactory(2 /* num_entries_flush */));
-  options.target_file_size_base = kValueBytes;
-  // i == 0: CompactFiles
-  // i == 1: CompactRange
-  // i == 2: automatic compaction
-  for (int i = 0; i < 3; ++i) {
-    DestroyAndReopen(options);
-
-    ASSERT_OK(Put(Key(0), ""));
-    ASSERT_OK(db_->Flush(FlushOptions()));
-    MoveFilesToLevel(2);
-    ASSERT_EQ(1, NumTableFilesAtLevel(2));
-
-    // snapshot protects range tombstone from dropping due to becoming obsolete.
-    const Snapshot* snapshot = db_->GetSnapshot();
-    db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(0),
-                     Key(2 * kNumFilesPerLevel));
-
-    Random rnd(301);
-    std::string value = RandomString(&rnd, kValueBytes);
-    for (int j = 0; j < kNumFilesPerLevel; ++j) {
-      // give files overlapping key-ranges to prevent trivial move
-      ASSERT_OK(Put(Key(j), value));
-      ASSERT_OK(Put(Key(2 * kNumFilesPerLevel - 1 - j), value));
-      if (j > 0) {
-        dbfull()->TEST_WaitForFlushMemTable();
-        ASSERT_EQ(j, NumTableFilesAtLevel(0));
-      }
-    }
-    // put extra key to trigger final flush
-    ASSERT_OK(Put("", ""));
-    dbfull()->TEST_WaitForFlushMemTable();
-    dbfull()->TEST_WaitForCompact();
-    ASSERT_EQ(0, NumTableFilesAtLevel(0));
-    ASSERT_EQ(kNumFilesPerLevel, NumTableFilesAtLevel(1));
-
-    ColumnFamilyMetaData meta;
-    db_->GetColumnFamilyMetaData(&meta);
-    if (i == 0) {
-      ASSERT_OK(db_->CompactFiles(
-          CompactionOptions(), {meta.levels[1].files[0].name}, 2 /* level */));
-    } else if (i == 1) {
-      auto begin_str = Key(0), end_str = Key(1);
-      Slice begin = begin_str, end = end_str;
-      ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &begin, &end));
-    } else if (i == 2) {
-      ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(),
-                                {{"max_bytes_for_level_base", "10000"}}));
-      dbfull()->TEST_WaitForCompact();
-    }
-    ASSERT_EQ(0, NumTableFilesAtLevel(1));
-    ASSERT_GT(NumTableFilesAtLevel(2), 0);
-
-    db_->ReleaseSnapshot(snapshot);
-  }
-}
-
-TEST_F(DBRangeDelTest, UnorderedTombstones) {
-  // Regression test for #2752. Range delete tombstones between
-  // different snapshot stripes are not stored in order, so the first
-  // tombstone of each snapshot stripe should be checked as a smallest
-  // candidate.
-  Options options = CurrentOptions();
-  DestroyAndReopen(options);
-
-  auto cf = db_->DefaultColumnFamily();
-
-  ASSERT_OK(db_->Put(WriteOptions(), cf, "a", "a"));
-  ASSERT_OK(db_->Flush(FlushOptions(), cf));
-  ASSERT_EQ(1, NumTableFilesAtLevel(0));
-  ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr));
-  ASSERT_EQ(1, NumTableFilesAtLevel(1));
-
-  ASSERT_OK(db_->DeleteRange(WriteOptions(), cf, "b", "c"));
-  // Hold a snapshot to separate these two delete ranges.
-  auto snapshot = db_->GetSnapshot();
-  ASSERT_OK(db_->DeleteRange(WriteOptions(), cf, "a", "b"));
-  ASSERT_OK(db_->Flush(FlushOptions(), cf));
-  db_->ReleaseSnapshot(snapshot);
-
-  std::vector<std::vector<FileMetaData>> files;
-  dbfull()->TEST_GetFilesMetaData(cf, &files);
-  ASSERT_EQ(1, files[0].size());
-  ASSERT_EQ("a", files[0][0].smallest.user_key());
-  ASSERT_EQ("c", files[0][0].largest.user_key());
-
-  std::string v;
-  auto s = db_->Get(ReadOptions(), "a", &v);
-  ASSERT_TRUE(s.IsNotFound());
-}
-
 #endif  // ROCKSDB_LITE

 }  // namespace rocksdb
@@ -23,7 +23,6 @@
 #include <alloca.h>
 #endif

-#include "cache/lru_cache.h"
 #include "db/db_impl.h"
 #include "db/db_test_util.h"
 #include "db/dbformat.h"
@@ -2235,10 +2234,6 @@ class ModelDB : public DB {
    return Status::NotSupported("Not implemented.");
  }

-  virtual Status VerifyChecksum() override {
-    return Status::NotSupported("Not implemented.");
-  }
-
  using DB::GetPropertiesOfAllTables;
  virtual Status GetPropertiesOfAllTables(
      ColumnFamilyHandle* column_family,
@@ -2891,23 +2886,20 @@ TEST_F(DBTest, FIFOCompactionWithTTLAndVariousTableFormatsTest) {
  ASSERT_TRUE(TryReopen(options).IsNotSupported());
 }

-TEST_F(DBTest, FIFOCompactionWithTTLTest) {
+TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
  Options options;
  options.compaction_style = kCompactionStyleFIFO;
  options.write_buffer_size = 10 << 10;  // 10KB
  options.arena_block_size = 4096;
  options.compression = kNoCompression;
  options.create_if_missing = true;
-  env_->time_elapse_only_sleep_ = false;
-  options.env = env_;

  // Test to make sure that all files with expired ttl are deleted on next
  // manual compaction.
  {
-    env_->addon_time_.store(0);
    options.compaction_options_fifo.max_table_files_size = 150 << 10;  // 150KB
    options.compaction_options_fifo.allow_compaction = false;
-    options.compaction_options_fifo.ttl = 1 * 60 * 60 ;  // 1 hour
+    options.compaction_options_fifo.ttl = 600;  // seconds
    options = CurrentOptions(options);
    DestroyAndReopen(options);

@@ -2918,21 +2910,19 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
      }
      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
    }
-    ASSERT_EQ(NumTableFilesAtLevel(0), 10);
-
-    // Sleep for 2 hours -- which is much greater than TTL.
-    // Note: Couldn't use SleepForMicroseconds because it takes an int instead
-    // of uint64_t. Hence used addon_time_ directly.
-    // env_->SleepForMicroseconds(2 * 60 * 60 * 1000 * 1000);
-    env_->addon_time_.fetch_add(2 * 60 * 60);
-
-    // Since no flushes and compactions have run, the db should still be in
-    // the same state even after considerable time has passed.
    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    ASSERT_EQ(NumTableFilesAtLevel(0), 10);

+    // sleep for 5 seconds
+    env_->SleepForMicroseconds(5 * 1000 * 1000);
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
+    ASSERT_EQ(NumTableFilesAtLevel(0), 10);
+
+    // change ttl to 1 sec. So all files should be deleted on next compaction.
+    options.compaction_options_fifo.ttl = 1;
+    Reopen(options);
+
    dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
    ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  }
@@ -2942,7 +2932,7 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
  {
    options.compaction_options_fifo.max_table_files_size = 150 << 10;  // 150KB
    options.compaction_options_fifo.allow_compaction = false;
-    options.compaction_options_fifo.ttl = 1 * 60 * 60;  // 1 hour
+    options.compaction_options_fifo.ttl = 5;  // seconds
    options = CurrentOptions(options);
    DestroyAndReopen(options);

@@ -2953,13 +2943,11 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
      }
      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
    }
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    ASSERT_EQ(NumTableFilesAtLevel(0), 10);

-    // Sleep for 2 hours -- which is much greater than TTL.
-    env_->addon_time_.fetch_add(2 * 60 * 60);
-    // Just to make sure that we are in the same state even after sleeping.
+    env_->SleepForMicroseconds(6 * 1000 * 1000);
    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    ASSERT_EQ(NumTableFilesAtLevel(0), 10);

@@ -2981,10 +2969,10 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
  // Test that shows the fall back to size-based FIFO compaction if TTL-based
  // deletion doesn't move the total size to be less than max_table_files_size.
  {
-    options.write_buffer_size = 10 << 10;                              // 10KB
+    options.write_buffer_size = 110 << 10;                             // 10KB
    options.compaction_options_fifo.max_table_files_size = 150 << 10;  // 150KB
    options.compaction_options_fifo.allow_compaction = false;
-    options.compaction_options_fifo.ttl =  1 * 60 * 60;  // 1 hour
+    options.compaction_options_fifo.ttl = 5;  // seconds
    options = CurrentOptions(options);
    DestroyAndReopen(options);

@@ -2995,13 +2983,11 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
      }
      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
    }
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    ASSERT_EQ(NumTableFilesAtLevel(0), 3);

-    // Sleep for 2 hours -- which is much greater than TTL.
-    env_->addon_time_.fetch_add(2 * 60 * 60);
-    // Just to make sure that we are in the same state even after sleeping.
+    env_->SleepForMicroseconds(6 * 1000 * 1000);
    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    ASSERT_EQ(NumTableFilesAtLevel(0), 3);

@@ -3010,8 +2996,8 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
      }
      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
    }
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    // Size limit is still guaranteed.
    ASSERT_LE(SizeAtLevel(0),
              options.compaction_options_fifo.max_table_files_size);
@@ -3021,7 +3007,7 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
  {
    options.compaction_options_fifo.max_table_files_size = 150 << 10;  // 150KB
    options.compaction_options_fifo.allow_compaction = true;
-    options.compaction_options_fifo.ttl = 1 * 60 * 60;  // 1 hour
+    options.compaction_options_fifo.ttl = 5;  // seconds
    options.level0_file_num_compaction_trigger = 6;
    options = CurrentOptions(options);
    DestroyAndReopen(options);
@@ -3033,16 +3019,15 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
      }
      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
    }
    // With Intra-L0 compaction, out of 10 files, 6 files will be compacted to 1
    // (due to level0_file_num_compaction_trigger = 6).
    // So total files = 1 + remaining 4 = 5.
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    ASSERT_EQ(NumTableFilesAtLevel(0), 5);

-    // Sleep for 2 hours -- which is much greater than TTL.
-    env_->addon_time_.fetch_add(2 * 60 * 60);
-    // Just to make sure that we are in the same state even after sleeping.
+    // Sleep for a little over ttl time.
+    env_->SleepForMicroseconds(6 * 1000 * 1000);
    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    ASSERT_EQ(NumTableFilesAtLevel(0), 5);

@@ -3052,8 +3037,8 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
      }
      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
    }
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    ASSERT_EQ(NumTableFilesAtLevel(0), 5);
    ASSERT_LE(SizeAtLevel(0),
              options.compaction_options_fifo.max_table_files_size);
@@ -3065,7 +3050,7 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
    options.write_buffer_size = 20 << 10;                               // 20K
    options.compaction_options_fifo.max_table_files_size = 1500 << 10;  // 1.5MB
    options.compaction_options_fifo.allow_compaction = true;
-    options.compaction_options_fifo.ttl = 1 * 60 * 60;  // 1 hour
+    options.compaction_options_fifo.ttl = 60 * 60;  // 1 hour
    options.level0_file_num_compaction_trigger = 6;
    options = CurrentOptions(options);
    DestroyAndReopen(options);
@@ -3077,8 +3062,8 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
      }
      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
    }
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    // It should be compacted to 10 files.
    ASSERT_EQ(NumTableFilesAtLevel(0), 10);

@@ -3088,8 +3073,8 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
        ASSERT_OK(Put(ToString(i * 20 + j + 2000), RandomString(&rnd, 980)));
      }
      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
    }
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());

    // It should be compacted to no more than 20 files.
    ASSERT_GT(NumTableFilesAtLevel(0), 10);
@@ -5375,36 +5360,6 @@ TEST_F(DBTest, RowCache) {
  ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), 1);
  ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), 1);
 }
-
-TEST_F(DBTest, PinnableSliceAndRowCache) {
-  Options options = CurrentOptions();
-  options.statistics = rocksdb::CreateDBStatistics();
-  options.row_cache = NewLRUCache(8192);
-  DestroyAndReopen(options);
-
-  ASSERT_OK(Put("foo", "bar"));
-  ASSERT_OK(Flush());
-
-  ASSERT_EQ(Get("foo"), "bar");
-  ASSERT_EQ(
-      reinterpret_cast<LRUCache*>(options.row_cache.get())->TEST_GetLRUSize(),
-      1);
-
-  {
-    PinnableSlice pin_slice;
-    ASSERT_EQ(Get("foo", &pin_slice), Status::OK());
-    ASSERT_EQ(pin_slice.ToString(), "bar");
-    // Entry is already in cache, lookup will remove the element from lru
-    ASSERT_EQ(
-        reinterpret_cast<LRUCache*>(options.row_cache.get())->TEST_GetLRUSize(),
-        0);
-  }
-  // After PinnableSlice destruction element is added back in LRU
-  ASSERT_EQ(
-      reinterpret_cast<LRUCache*>(options.row_cache.get())->TEST_GetLRUSize(),
-      1);
-}
-
 #endif  // ROCKSDB_LITE

 TEST_F(DBTest, DeletingOldWalAfterDrop) {
@@ -2236,7 +2236,6 @@ TEST_F(DBTest2, LowPriWrite) {
  ASSERT_EQ(1, rate_limit_count.load());
 }

-#ifndef ROCKSDB_LITE
 TEST_F(DBTest2, RateLimitedCompactionReads) {
  // compaction input has 512KB data
  const int kNumKeysPerFile = 128;
@@ -2282,15 +2281,12 @@ TEST_F(DBTest2, RateLimitedCompactionReads) {
    // chose 1MB as the upper bound on the total bytes read.
    size_t rate_limited_bytes =
        options.rate_limiter->GetTotalBytesThrough(Env::IO_LOW);
-    // Include the explict prefetch of the footer in direct I/O case.
-    size_t direct_io_extra = use_direct_io ? 512 * 1024 : 0;
-    ASSERT_GE(rate_limited_bytes,
-              static_cast<size_t>(kNumKeysPerFile * kBytesPerKey * kNumL0Files +
-                                  direct_io_extra));
+    ASSERT_GE(
+        rate_limited_bytes,
+        static_cast<size_t>(kNumKeysPerFile * kBytesPerKey * kNumL0Files));
    ASSERT_LT(
        rate_limited_bytes,
-        static_cast<size_t>(2 * kNumKeysPerFile * kBytesPerKey * kNumL0Files +
-                            direct_io_extra));
+        static_cast<size_t>(2 * kNumKeysPerFile * kBytesPerKey * kNumL0Files));

    Iterator* iter = db_->NewIterator(ReadOptions());
    for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
@@ -2303,28 +2299,7 @@ TEST_F(DBTest2, RateLimitedCompactionReads) {
                  options.rate_limiter->GetTotalBytesThrough(Env::IO_LOW)));
  }
 }
-#endif  // ROCKSDB_LITE

-// Make sure DB can be reopen with reduced number of levels, given no file
-// is on levels higher than the new num_levels.
-TEST_F(DBTest2, ReduceLevel) {
-  Options options;
-  options.disable_auto_compactions = true;
-  options.num_levels = 7;
-  Reopen(options);
-  Put("foo", "bar");
-  Flush();
-  MoveFilesToLevel(6);
-  ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel());
-  CompactRangeOptions compact_options;
-  compact_options.change_level = true;
-  compact_options.target_level = 1;
-  dbfull()->CompactRange(compact_options, nullptr, nullptr);
-  ASSERT_EQ("0,1", FilesPerLevel());
-  options.num_levels = 3;
-  Reopen(options);
-  ASSERT_EQ("0,1", FilesPerLevel());
-}
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -42,24 +42,13 @@ SpecialEnv::SpecialEnv(Env* base)
  non_writable_count_ = 0;
  table_write_callback_ = nullptr;
 }
-#ifndef ROCKSDB_LITE
+
 ROT13BlockCipher rot13Cipher_(16);
-#endif  // ROCKSDB_LITE

 DBTestBase::DBTestBase(const std::string path)
    : mem_env_(!getenv("MEM_ENV") ? nullptr : new MockEnv(Env::Default())),
-#ifndef ROCKSDB_LITE
-      encrypted_env_(
-          !getenv("ENCRYPTED_ENV")
-              ? nullptr
-              : NewEncryptedEnv(mem_env_ ? mem_env_ : Env::Default(),
-                                new CTREncryptionProvider(rot13Cipher_))),
-#else
-      encrypted_env_(nullptr),
-#endif  // ROCKSDB_LITE
-      env_(new SpecialEnv(encrypted_env_
-                              ? encrypted_env_
-                              : (mem_env_ ? mem_env_ : Env::Default()))),
+      encrypted_env_(!getenv("ENCRYPTED_ENV") ? nullptr : NewEncryptedEnv(mem_env_ ? mem_env_ : Env::Default(), new CTREncryptionProvider(rot13Cipher_))),
+      env_(new SpecialEnv(encrypted_env_ ? encrypted_env_ : (mem_env_ ? mem_env_ : Env::Default()))),
      option_config_(kDefault) {
  env_->SetBackgroundThreads(1, Env::LOW);
  env_->SetBackgroundThreads(1, Env::HIGH);
@@ -697,13 +686,6 @@ std::string DBTestBase::Get(int cf, const std::string& k,
  return result;
 }

-Status DBTestBase::Get(const std::string& k, PinnableSlice* v) {
-  ReadOptions options;
-  options.verify_checksums = true;
-  Status s = dbfull()->Get(options, dbfull()->DefaultColumnFamily(), k, v);
-  return s;
-}
-
 uint64_t DBTestBase::GetNumSnapshots() {
  uint64_t int_num;
  EXPECT_TRUE(dbfull()->GetIntProperty("rocksdb.num-snapshots", &int_num));
@@ -803,8 +803,6 @@ class DBTestBase : public testing::Test {
  std::string Get(int cf, const std::string& k,
                  const Snapshot* snapshot = nullptr);

-  Status Get(const std::string& k, PinnableSlice* v);
-
  uint64_t GetNumSnapshots();

  uint64_t GetTimeOldestSnapshots();
@@ -696,12 +696,17 @@ TEST_P(DBTestUniversalCompactionParallel, PickByFileNumberBug) {
    num_keys -= 100;
  }

-  // Hold the 1st compaction from finishing
+  // Wait for the 2nd background compaction process to start
+  TEST_SYNC_POINT("DBTestUniversalCompactionParallel::PickByFileNumberBug:0");
+  TEST_SYNC_POINT("DBTestUniversalCompactionParallel::PickByFileNumberBug:1");
+
+  // Hold the 1st and 2nd compaction from finishing
  TEST_SYNC_POINT("DBTestUniversalCompactionParallel::PickByFileNumberBug:2");
  dbfull()->TEST_WaitForCompact();

-  // There should only be one picked compaction as the score drops below one
-  // after the first one is picked.
+  // Although 2 compaction threads started, the second one did not compact
+  // anything because the number of files not being compacted is less than
+  // level0_file_num_compaction_trigger
  EXPECT_EQ(total_picked_compactions, 1);
  EXPECT_EQ(TotalTableFiles(), 4);

@@ -1365,140 +1370,6 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) {
  Destroy(options);
 }

-TEST_P(DBTestUniversalCompaction, FullCompactionInBottomPriThreadPool) {
-  const int kNumFilesTrigger = 3;
-  Env::Default()->SetBackgroundThreads(1, Env::Priority::BOTTOM);
-  for (bool allow_ingest_behind : {false, true}) {
-    Options options = CurrentOptions();
-    options.allow_ingest_behind = allow_ingest_behind;
-    options.compaction_style = kCompactionStyleUniversal;
-    options.num_levels = num_levels_;
-    options.write_buffer_size = 100 << 10;     // 100KB
-    options.target_file_size_base = 32 << 10;  // 32KB
-    options.level0_file_num_compaction_trigger = kNumFilesTrigger;
-    // Trigger compaction if size amplification exceeds 110%
-    options.compaction_options_universal.max_size_amplification_percent = 110;
-    DestroyAndReopen(options);
-
-    int num_bottom_pri_compactions = 0;
-    SyncPoint::GetInstance()->SetCallBack(
-        "DBImpl::BGWorkBottomCompaction",
-        [&](void* arg) { ++num_bottom_pri_compactions; });
-    SyncPoint::GetInstance()->EnableProcessing();
-
-    Random rnd(301);
-    for (int num = 0; num < kNumFilesTrigger; num++) {
-      ASSERT_EQ(NumSortedRuns(), num);
-      int key_idx = 0;
-      GenerateNewFile(&rnd, &key_idx);
-    }
-    dbfull()->TEST_WaitForCompact();
-
-    if (allow_ingest_behind || num_levels_ > 1) {
-      // allow_ingest_behind increases number of levels while sanitizing.
-      ASSERT_EQ(1, num_bottom_pri_compactions);
-    } else {
-      // for single-level universal, everything's bottom level so nothing should
-      // be executed in bottom-pri thread pool.
-      ASSERT_EQ(0, num_bottom_pri_compactions);
-    }
-    // Verify that size amplification did occur
-    ASSERT_EQ(NumSortedRuns(), 1);
-    rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-  }
-  Env::Default()->SetBackgroundThreads(0, Env::Priority::BOTTOM);
-}
-
-TEST_P(DBTestUniversalCompaction, ConcurrentBottomPriLowPriCompactions) {
-  if (num_levels_ == 1) {
-    // for single-level universal, everything's bottom level so nothing should
-    // be executed in bottom-pri thread pool.
-    return;
-  }
-  const int kNumFilesTrigger = 3;
-  Env::Default()->SetBackgroundThreads(1, Env::Priority::BOTTOM);
-  Options options = CurrentOptions();
-  options.compaction_style = kCompactionStyleUniversal;
-  options.num_levels = num_levels_;
-  options.write_buffer_size = 100 << 10;     // 100KB
-  options.target_file_size_base = 32 << 10;  // 32KB
-  options.level0_file_num_compaction_trigger = kNumFilesTrigger;
-  // Trigger compaction if size amplification exceeds 110%
-  options.compaction_options_universal.max_size_amplification_percent = 110;
-  DestroyAndReopen(options);
-
-  rocksdb::SyncPoint::GetInstance()->LoadDependency(
-      {// wait for the full compaction to be picked before adding files intended
-       // for the second one.
-       {"DBImpl::BackgroundCompaction:ForwardToBottomPriPool",
-        "DBTestUniversalCompaction:ConcurrentBottomPriLowPriCompactions:0"},
-       // the full (bottom-pri) compaction waits until a partial (low-pri)
-       // compaction has started to verify they can run in parallel.
-       {"DBImpl::BackgroundCompaction:NonTrivial",
-        "DBImpl::BGWorkBottomCompaction"}});
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  Random rnd(301);
-  for (int i = 0; i < 2; ++i) {
-    for (int num = 0; num < kNumFilesTrigger; num++) {
-      int key_idx = 0;
-      GenerateNewFile(&rnd, &key_idx, true /* no_wait */);
-      // use no_wait above because that one waits for flush and compaction. We
-      // don't want to wait for compaction because the full compaction is
-      // intentionally blocked while more files are flushed.
-      dbfull()->TEST_WaitForFlushMemTable();
-    }
-    if (i == 0) {
-      TEST_SYNC_POINT(
-          "DBTestUniversalCompaction:ConcurrentBottomPriLowPriCompactions:0");
-    }
-  }
-  dbfull()->TEST_WaitForCompact();
-
-  // First compaction should output to bottom level. Second should output to L0
-  // since older L0 files pending compaction prevent it from being placed lower.
-  ASSERT_EQ(NumSortedRuns(), 2);
-  ASSERT_GT(NumTableFilesAtLevel(0), 0);
-  ASSERT_GT(NumTableFilesAtLevel(num_levels_ - 1), 0);
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-  Env::Default()->SetBackgroundThreads(0, Env::Priority::BOTTOM);
-}
-
-TEST_P(DBTestUniversalCompaction, RecalculateScoreAfterPicking) {
-  // Regression test for extra compactions scheduled. Once enough compactions
-  // have been scheduled to bring the score below one, we should stop
-  // scheduling more; otherwise, other CFs/DBs may be delayed unnecessarily.
-  const int kNumFilesTrigger = 8;
-  Options options = CurrentOptions();
-  options.compaction_options_universal.max_merge_width = kNumFilesTrigger / 2;
-  options.compaction_options_universal.max_size_amplification_percent =
-      static_cast<unsigned int>(-1);
-  options.compaction_style = kCompactionStyleUniversal;
-  options.level0_file_num_compaction_trigger = kNumFilesTrigger;
-  options.num_levels = num_levels_;
-  options.write_buffer_size = 100 << 10;  // 100KB
-  Reopen(options);
-
-  std::atomic<int> num_compactions_attempted(0);
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::BackgroundCompaction:Start", [&](void* arg) {
-        ++num_compactions_attempted;
-      });
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-
-  Random rnd(301);
-  for (int num = 0; num < kNumFilesTrigger; num++) {
-    ASSERT_EQ(NumSortedRuns(), num);
-    int key_idx = 0;
-    GenerateNewFile(&rnd, &key_idx);
-  }
-  dbfull()->TEST_WaitForCompact();
-  // Compacting the first four files was enough to bring the score below one so
-  // there's no need to schedule any more compactions.
-  ASSERT_EQ(1, num_compactions_attempted);
-  ASSERT_EQ(NumSortedRuns(), 5);
-}
-
 INSTANTIATE_TEST_CASE_P(UniversalCompactionNumLevels, DBTestUniversalCompaction,
                        ::testing::Combine(::testing::Values(1, 3, 5),
                                           ::testing::Bool()));
@@ -84,12 +84,6 @@ struct ParsedInternalKey {
  ParsedInternalKey(const Slice& u, const SequenceNumber& seq, ValueType t)
      : user_key(u), sequence(seq), type(t) { }
  std::string DebugString(bool hex = false) const;
-
-  void clear() {
-    user_key.clear();
-    sequence = 0;
-    type = kTypeDeletion;
-  }
 };

 // Return the length of the encoding of "key".
@@ -157,9 +151,6 @@ class InternalKeyComparator : public Comparator {

  int Compare(const InternalKey& a, const InternalKey& b) const;
  int Compare(const ParsedInternalKey& a, const ParsedInternalKey& b) const;
-  virtual const Comparator* GetRootComparator() const override {
-    return user_comparator_->GetRootComparator();
-  }
 };

 // Modules in this directory should keep internal keys wrapped inside
@@ -14,18 +14,20 @@ namespace experimental {

 Status SuggestCompactRange(DB* db, ColumnFamilyHandle* column_family,
                           const Slice* begin, const Slice* end) {
-  if (db == nullptr) {
-    return Status::InvalidArgument("DB is empty");
+  auto dbimpl = dynamic_cast<DBImpl*>(db);
+  if (dbimpl == nullptr) {
+    return Status::InvalidArgument("Didn't recognize DB object");
  }

-  return db->SuggestCompactRange(column_family, begin, end);
+  return dbimpl->SuggestCompactRange(column_family, begin, end);
 }

 Status PromoteL0(DB* db, ColumnFamilyHandle* column_family, int target_level) {
-  if (db == nullptr) {
+  auto dbimpl = dynamic_cast<DBImpl*>(db);
+  if (dbimpl == nullptr) {
    return Status::InvalidArgument("Didn't recognize DB object");
  }
-  return db->PromoteL0(column_family, target_level);
+  return dbimpl->PromoteL0(column_family, target_level);
 }

 #else  // ROCKSDB_LITE
@@ -643,14 +643,6 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
  }
  PERF_TIMER_GUARD(get_from_memtable_time);

-  std::unique_ptr<InternalIterator> range_del_iter(
-      NewRangeTombstoneIterator(read_opts));
-  Status status = range_del_agg->AddTombstones(std::move(range_del_iter));
-  if (!status.ok()) {
-    *s = status;
-    return false;
-  }
-
  Slice user_key = key.user_key();
  bool found_final_value = false;
  bool merge_in_progress = s->IsMergeInProgress();
@@ -666,6 +658,13 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
    if (prefix_bloom_) {
      PERF_COUNTER_ADD(bloom_memtable_hit_count, 1);
    }
+    std::unique_ptr<InternalIterator> range_del_iter(
+        NewRangeTombstoneIterator(read_opts));
+    Status status = range_del_agg->AddTombstones(std::move(range_del_iter));
+    if (!status.ok()) {
+      *s = status;
+      return false;
+    }
    Saver saver;
    saver.status = s;
    saver.found_final_value = &found_final_value;
@@ -18,33 +18,6 @@

 namespace rocksdb {

-MergeHelper::MergeHelper(Env* env, const Comparator* user_comparator,
-                         const MergeOperator* user_merge_operator,
-                         const CompactionFilter* compaction_filter,
-                         Logger* logger, bool assert_valid_internal_key,
-                         SequenceNumber latest_snapshot, int level,
-                         Statistics* stats,
-                         const std::atomic<bool>* shutting_down)
-    : env_(env),
-      user_comparator_(user_comparator),
-      user_merge_operator_(user_merge_operator),
-      compaction_filter_(compaction_filter),
-      shutting_down_(shutting_down),
-      logger_(logger),
-      assert_valid_internal_key_(assert_valid_internal_key),
-      allow_single_operand_(false),
-      latest_snapshot_(latest_snapshot),
-      level_(level),
-      keys_(),
-      filter_timer_(env_),
-      total_filter_time_(0U),
-      stats_(stats) {
-  assert(user_comparator_ != nullptr);
-  if (user_merge_operator_) {
-    allow_single_operand_ = user_merge_operator_->AllowSingleOperand();
-  }
-}
-
 Status MergeHelper::TimedFullMerge(const MergeOperator* merge_operator,
                                   const Slice& key, const Slice* value,
                                   const std::vector<Slice>& operands,
@@ -228,11 +201,12 @@ Status MergeHelper::MergeUntil(InternalIterator* iter,
          ikey.sequence <= latest_snapshot_
              ? CompactionFilter::Decision::kKeep
              : FilterMerge(orig_ikey.user_key, value_slice);
-      if (filter != CompactionFilter::Decision::kRemoveAndSkipUntil &&
-          range_del_agg != nullptr &&
+      if (range_del_agg != nullptr &&
+
          range_del_agg->ShouldDelete(
              iter->key(),
-              RangeDelAggregator::RangePositioningMode::kForwardTraversal)) {
+              RangeDelAggregator::RangePositioningMode::kForwardTraversal) &&
+          filter != CompactionFilter::Decision::kRemoveAndSkipUntil) {
        filter = CompactionFilter::Decision::kRemove;
      }
      if (filter == CompactionFilter::Decision::kKeep ||
@@ -315,8 +289,7 @@ Status MergeHelper::MergeUntil(InternalIterator* iter,
    // Attempt to use the user's associative merge function to
    // merge the stacked merge operands into a single operand.
    s = Status::MergeInProgress();
-    if (merge_context_.GetNumOperands() >= 2 ||
-        (allow_single_operand_ && merge_context_.GetNumOperands() == 1)) {
+    if (merge_context_.GetNumOperands() >= 2) {
      bool merge_success = false;
      std::string merge_result;
      {
@@ -34,7 +34,22 @@ class MergeHelper {
              const CompactionFilter* compaction_filter, Logger* logger,
              bool assert_valid_internal_key, SequenceNumber latest_snapshot,
              int level = 0, Statistics* stats = nullptr,
-              const std::atomic<bool>* shutting_down = nullptr);
+              const std::atomic<bool>* shutting_down = nullptr)
+      : env_(env),
+        user_comparator_(user_comparator),
+        user_merge_operator_(user_merge_operator),
+        compaction_filter_(compaction_filter),
+        shutting_down_(shutting_down),
+        logger_(logger),
+        assert_valid_internal_key_(assert_valid_internal_key),
+        latest_snapshot_(latest_snapshot),
+        level_(level),
+        keys_(),
+        filter_timer_(env_),
+        total_filter_time_(0U),
+        stats_(stats) {
+    assert(user_comparator_ != nullptr);
+  }

  // Wrapper around MergeOperator::FullMergeV2() that records perf statistics.
  // Result of merge will be written to result if status returned is OK.
@@ -143,7 +158,6 @@ class MergeHelper {
  const std::atomic<bool>* shutting_down_;
  Logger* logger_;
  bool assert_valid_internal_key_; // enforce no internal key corruption?
-  bool allow_single_operand_;
  SequenceNumber latest_snapshot_;
  int level_;

@@ -357,8 +357,7 @@ Status RangeDelAggregator::AddTombstone(RangeTombstone tombstone) {
      ++new_range_dels_iter;
    }
  } else {
-    auto start_key = tombstone.start_key_;
-    tombstone_map.emplace(start_key, std::move(tombstone));
+    tombstone_map.emplace(tombstone.start_key_, std::move(tombstone));
  }
  return Status::OK();
 }
@@ -413,8 +412,8 @@ void RangeDelAggregator::AddToBuilder(

  // Note the order in which tombstones are stored is insignificant since we
  // insert them into a std::map on the read path.
+  bool first_added = false;
  while (stripe_map_iter != rep_->stripe_map_.end()) {
-    bool first_added = false;
    for (auto tombstone_map_iter = stripe_map_iter->second.raw_map.begin();
         tombstone_map_iter != stripe_map_iter->second.raw_map.end();
         ++tombstone_map_iter) {
@@ -453,7 +452,7 @@ void RangeDelAggregator::AddToBuilder(
      builder->Add(ikey_and_end_key.first.Encode(), ikey_and_end_key.second);
      if (!first_added) {
        first_added = true;
-        InternalKey smallest_candidate = std::move(ikey_and_end_key.first);
+        InternalKey smallest_candidate = std::move(ikey_and_end_key.first);;
        if (lower_bound != nullptr &&
            icmp_.user_comparator()->Compare(smallest_candidate.user_key(),
                                             *lower_bound) <= 0) {
@@ -176,7 +176,6 @@ class Repairer {
      status = db_impl->NewDB();
      delete db_impl;
    }
-
    if (status.ok()) {
      // Recover using the fresh manifest created by NewDB()
      status =
@@ -247,21 +246,9 @@ class Repairer {
  Status FindFiles() {
    std::vector<std::string> filenames;
    bool found_file = false;
-    std::vector<std::string> to_search_paths;
-
    for (size_t path_id = 0; path_id < db_options_.db_paths.size(); path_id++) {
-        to_search_paths.push_back(db_options_.db_paths[path_id].path);
-    }
-
-    // search wal_dir if user uses a customize wal_dir
-    if (!db_options_.wal_dir.empty() && 
-        db_options_.wal_dir != dbname_) {
-        to_search_paths.push_back(db_options_.wal_dir);
-    }
-
-    for (size_t path_id = 0; path_id < to_search_paths.size(); path_id++) {
      Status status =
-          env_->GetChildren(to_search_paths[path_id], &filenames);
+          env_->GetChildren(db_options_.db_paths[path_id].path, &filenames);
      if (!status.ok()) {
        return status;
      }
@@ -274,12 +261,14 @@ class Repairer {
      for (size_t i = 0; i < filenames.size(); i++) {
        if (ParseFileName(filenames[i], &number, &type)) {
          if (type == kDescriptorFile) {
+            assert(path_id == 0);
            manifests_.push_back(filenames[i]);
          } else {
            if (number + 1 > next_file_number_) {
              next_file_number_ = number + 1;
            }
            if (type == kLogFile) {
+              assert(path_id == 0);
              logs_.push_back(number);
            } else if (type == kTableFile) {
              table_fds_.emplace_back(number, static_cast<uint32_t>(path_id),
@@ -299,8 +288,7 @@ class Repairer {

  void ConvertLogFilesToTables() {
    for (size_t i = 0; i < logs_.size(); i++) {
-      // we should use LogFileName(wal_dir, logs_[i]) here. user might uses wal_dir option.
-      std::string logname = LogFileName(db_options_.wal_dir, logs_[i]);
+      std::string logname = LogFileName(dbname_, logs_[i]);
      Status status = ConvertLogToTable(logs_[i]);
      if (!status.ok()) {
        ROCKS_LOG_WARN(db_options_.info_log,
@@ -324,7 +312,7 @@ class Repairer {
    };

    // Open the log file
-    std::string logname = LogFileName(db_options_.wal_dir, log);
+    std::string logname = LogFileName(dbname_, log);
    unique_ptr<SequentialFile> lfile;
    Status status = env_->NewSequentialFile(
        logname, &lfile, env_->OptimizeForLogRead(env_options_));
@@ -174,40 +174,6 @@ TEST_F(RepairTest, UnflushedSst) {
  ASSERT_EQ(Get("key"), "val");
 }

-TEST_F(RepairTest, SeparateWalDir) {
-  do {
-    Options options = CurrentOptions();
-    DestroyAndReopen(options);
-    Put("key", "val");
-    Put("foo", "bar");
-    VectorLogPtr wal_files;
-    ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
-    ASSERT_EQ(wal_files.size(), 1);
-    uint64_t total_ssts_size;
-    GetAllSSTFiles(&total_ssts_size);
-    ASSERT_EQ(total_ssts_size, 0);
-    std::string manifest_path =
-      DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
-
-    Close();
-    ASSERT_OK(env_->FileExists(manifest_path));
-    ASSERT_OK(env_->DeleteFile(manifest_path));
-    ASSERT_OK(RepairDB(dbname_, options));
-
-    // make sure that all WALs are converted to SSTables.
-    options.wal_dir = "";
-
-    Reopen(options);
-    ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
-    ASSERT_EQ(wal_files.size(), 0);
-    GetAllSSTFiles(&total_ssts_size);
-    ASSERT_GT(total_ssts_size, 0);
-    ASSERT_EQ(Get("key"), "val");
-    ASSERT_EQ(Get("foo"), "bar");
-
- } while(ChangeWalOptions());
-}
-
 TEST_F(RepairTest, RepairMultipleColumnFamilies) {
  // Verify repair logic associates SST files with their original column
  // families.
@@ -74,11 +74,9 @@ class SnapshotList {
    count_--;
  }

-  // retrieve all snapshot numbers up until max_seq. They are sorted in
-  // ascending order.
+  // retrieve all snapshot numbers. They are sorted in ascending order.
  std::vector<SequenceNumber> GetAll(
-      SequenceNumber* oldest_write_conflict_snapshot = nullptr,
-      const SequenceNumber& max_seq = kMaxSequenceNumber) const {
+      SequenceNumber* oldest_write_conflict_snapshot = nullptr) {
    std::vector<SequenceNumber> ret;

    if (oldest_write_conflict_snapshot != nullptr) {
@@ -88,11 +86,8 @@ class SnapshotList {
    if (empty()) {
      return ret;
    }
-    const SnapshotImpl* s = &list_;
+    SnapshotImpl* s = &list_;
    while (s->next_ != &list_) {
-      if (s->next_->number_ > max_seq) {
-        break;
-      }
      ret.push_back(s->next_->number_);

      if (oldest_write_conflict_snapshot != nullptr &&
@@ -106,10 +106,10 @@ Status TableCache::GetTableReader(
    }
    StopWatch sw(ioptions_.env, ioptions_.statistics, TABLE_OPEN_IO_MICROS);
    std::unique_ptr<RandomAccessFileReader> file_reader(
-        new RandomAccessFileReader(
-            std::move(file), fname, ioptions_.env,
-            record_read_stats ? ioptions_.statistics : nullptr, SST_READ_MICROS,
-            file_read_hist, ioptions_.rate_limiter, for_compaction));
+        new RandomAccessFileReader(std::move(file), fname, ioptions_.env,
+                                   ioptions_.statistics, record_read_stats,
+                                   file_read_hist, ioptions_.rate_limiter,
+                                   for_compaction));
    s = ioptions_.table_factory->NewTableReader(
        TableReaderOptions(ioptions_, env_options, internal_comparator,
                           skip_filters, level),
@@ -225,7 +225,8 @@ InternalIterator* TableCache::NewIterator(
  }
  InternalIterator* result = nullptr;
  if (s.ok()) {
-    result = table_reader->NewIterator(options, arena, skip_filters);
+    result =
+      table_reader->NewIterator(options, arena, &icomparator, skip_filters);
    if (create_new_table_reader) {
      assert(handle == nullptr);
      result->RegisterCleanup(&DeleteTableReader, table_reader, nullptr);
@@ -310,7 +311,6 @@ Status TableCache::Get(const ReadOptions& options,
 #ifndef ROCKSDB_LITE
  IterKey row_cache_key;
  std::string row_cache_entry_buffer;
-
  // Check row cache if enabled. Since row cache does not currently store
  // sequence numbers, we cannot use it if we need to fetch the sequence.
  if (ioptions_.row_cache && !get_context->NeedToReadSequence()) {
@@ -334,26 +334,10 @@ Status TableCache::Get(const ReadOptions& options,

    if (auto row_handle =
            ioptions_.row_cache->Lookup(row_cache_key.GetUserKey())) {
-      // Cleanable routine to release the cache entry
-      Cleanable value_pinner;
-      auto release_cache_entry_func = [](void* cache_to_clean,
-                                         void* cache_handle) {
-        ((Cache*)cache_to_clean)->Release((Cache::Handle*)cache_handle);
-      };
      auto found_row_cache_entry = static_cast<const std::string*>(
          ioptions_.row_cache->Value(row_handle));
-      // If it comes here value is located on the cache.
-      // found_row_cache_entry points to the value on cache,
-      // and value_pinner has cleanup procedure for the cached entry.
-      // After replayGetContextLog() returns, get_context.pinnable_slice_
-      // will point to cache entry buffer (or a copy based on that) and
-      // cleanup routine under value_pinner will be delegated to
-      // get_context.pinnable_slice_. Cache entry is released when
-      // get_context.pinnable_slice_ is reset.
-      value_pinner.RegisterCleanup(release_cache_entry_func,
-                                   ioptions_.row_cache.get(), row_handle);
-      replayGetContextLog(*found_row_cache_entry, user_key, get_context,
-                          &value_pinner);
+      replayGetContextLog(*found_row_cache_entry, user_key, get_context);
+      ioptions_.row_cache->Release(row_handle);
      RecordTick(ioptions_.statistics, ROW_CACHE_HIT);
      done = true;
    } else {
@@ -17,7 +17,6 @@
 #include <algorithm>
 #include <atomic>
 #include <functional>
-#include <map>
 #include <set>
 #include <thread>
 #include <unordered_map>
@@ -88,16 +87,7 @@ class VersionBuilder::Rep {
  Logger* info_log_;
  TableCache* table_cache_;
  VersionStorageInfo* base_vstorage_;
-  int num_levels_;
  LevelState* levels_;
-  // Store states of levels larger than num_levels_. We do this instead of
-  // storing them in levels_ to avoid regression in case there are no files
-  // on invalid levels. The version is not consistent if in the end the files
-  // on invalid levels don't cancel out.
-  std::map<int, std::unordered_set<uint64_t>> invalid_levels_;
-  // Whether there are invalid new files or invalid deletion on levels larger
-  // than num_levels_.
-  bool has_invalid_levels_;
  FileComparator level_zero_cmp_;
  FileComparator level_nonzero_cmp_;

@@ -107,10 +97,8 @@ class VersionBuilder::Rep {
      : env_options_(env_options),
        info_log_(info_log),
        table_cache_(table_cache),
-        base_vstorage_(base_vstorage),
-        num_levels_(base_vstorage->num_levels()),
-        has_invalid_levels_(false) {
-    levels_ = new LevelState[num_levels_];
+        base_vstorage_(base_vstorage) {
+    levels_ = new LevelState[base_vstorage_->num_levels()];
    level_zero_cmp_.sort_method = FileComparator::kLevel0;
    level_nonzero_cmp_.sort_method = FileComparator::kLevelNon0;
    level_nonzero_cmp_.internal_comparator =
@@ -118,7 +106,7 @@ class VersionBuilder::Rep {
  }

  ~Rep() {
-    for (int level = 0; level < num_levels_; level++) {
+    for (int level = 0; level < base_vstorage_->num_levels(); level++) {
      const auto& added = levels_[level].added_files;
      for (auto& pair : added) {
        UnrefFile(pair.second);
@@ -149,7 +137,7 @@ class VersionBuilder::Rep {
    }
 #endif
    // make sure the files are sorted correctly
-    for (int level = 0; level < num_levels_; level++) {
+    for (int level = 0; level < vstorage->num_levels(); level++) {
      auto& level_files = vstorage->LevelFiles(level);
      for (size_t i = 1; i < level_files.size(); i++) {
        auto f1 = level_files[i - 1];
@@ -208,7 +196,7 @@ class VersionBuilder::Rep {
 #endif
    // a file to be deleted better exist in the previous version
    bool found = false;
-    for (int l = 0; !found && l < num_levels_; l++) {
+    for (int l = 0; !found && l < base_vstorage_->num_levels(); l++) {
      const std::vector<FileMetaData*>& base_files =
          base_vstorage_->LevelFiles(l);
      for (size_t i = 0; i < base_files.size(); i++) {
@@ -222,7 +210,7 @@ class VersionBuilder::Rep {
    // if the file did not exist in the previous version, then it
    // is possibly moved from lower level to higher level in current
    // version
-    for (int l = level + 1; !found && l < num_levels_; l++) {
+    for (int l = level + 1; !found && l < base_vstorage_->num_levels(); l++) {
      auto& level_added = levels_[l].added_files;
      auto got = level_added.find(number);
      if (got != level_added.end()) {
@@ -245,19 +233,6 @@ class VersionBuilder::Rep {
    }
  }

-  bool CheckConsistencyForNumLevels() {
-    // Make sure there are no files on or beyond num_levels().
-    if (has_invalid_levels_) {
-      return false;
-    }
-    for (auto& level : invalid_levels_) {
-      if (level.second.size() > 0) {
-        return false;
-      }
-    }
-    return true;
-  }
-
  // Apply all of the edits in *edit to the current state.
  void Apply(VersionEdit* edit) {
    CheckConsistency(base_vstorage_);
@@ -267,45 +242,26 @@ class VersionBuilder::Rep {
    for (const auto& del_file : del) {
      const auto level = del_file.first;
      const auto number = del_file.second;
-      if (level < num_levels_) {
-        levels_[level].deleted_files.insert(number);
-        CheckConsistencyForDeletes(edit, number, level);
+      levels_[level].deleted_files.insert(number);
+      CheckConsistencyForDeletes(edit, number, level);

-        auto exising = levels_[level].added_files.find(number);
-        if (exising != levels_[level].added_files.end()) {
-          UnrefFile(exising->second);
-          levels_[level].added_files.erase(number);
-        }
-      } else {
-        if (invalid_levels_[level].count(number) > 0) {
-          invalid_levels_[level].erase(number);
-        } else {
-          // Deleting an non-existing file on invalid level.
-          has_invalid_levels_ = true;
-        }
+      auto exising = levels_[level].added_files.find(number);
+      if (exising != levels_[level].added_files.end()) {
+        UnrefFile(exising->second);
+        levels_[level].added_files.erase(number);
      }
    }

    // Add new files
    for (const auto& new_file : edit->GetNewFiles()) {
      const int level = new_file.first;
-      if (level < num_levels_) {
-        FileMetaData* f = new FileMetaData(new_file.second);
-        f->refs = 1;
+      FileMetaData* f = new FileMetaData(new_file.second);
+      f->refs = 1;

-        assert(levels_[level].added_files.find(f->fd.GetNumber()) ==
-               levels_[level].added_files.end());
-        levels_[level].deleted_files.erase(f->fd.GetNumber());
-        levels_[level].added_files[f->fd.GetNumber()] = f;
-      } else {
-        uint64_t number = new_file.second.fd.GetNumber();
-        if (invalid_levels_[level].count(number) == 0) {
-          invalid_levels_[level].insert(number);
-        } else {
-          // Creating an already existing file on invalid level.
-          has_invalid_levels_ = true;
-        }
-      }
+      assert(levels_[level].added_files.find(f->fd.GetNumber()) ==
+             levels_[level].added_files.end());
+      levels_[level].deleted_files.erase(f->fd.GetNumber());
+      levels_[level].added_files[f->fd.GetNumber()] = f;
    }
  }

@@ -314,7 +270,7 @@ class VersionBuilder::Rep {
    CheckConsistency(base_vstorage_);
    CheckConsistency(vstorage);

-    for (int level = 0; level < num_levels_; level++) {
+    for (int level = 0; level < base_vstorage_->num_levels(); level++) {
      const auto& cmp = (level == 0) ? level_zero_cmp_ : level_nonzero_cmp_;
      // Merge the set of added files with the set of pre-existing files.
      // Drop any deleted files.  Store the result in *v.
@@ -369,7 +325,7 @@ class VersionBuilder::Rep {
    assert(table_cache_ != nullptr);
    // <file metadata, level>
    std::vector<std::pair<FileMetaData*, int>> files_meta;
-    for (int level = 0; level < num_levels_; level++) {
+    for (int level = 0; level < base_vstorage_->num_levels(); level++) {
      for (auto& file_meta_pair : levels_[level].added_files) {
        auto* file_meta = file_meta_pair.second;
        assert(!file_meta->table_reader_handle);
@@ -430,35 +386,24 @@ VersionBuilder::VersionBuilder(const EnvOptions& env_options,
                               VersionStorageInfo* base_vstorage,
                               Logger* info_log)
    : rep_(new Rep(env_options, info_log, table_cache, base_vstorage)) {}
-
 VersionBuilder::~VersionBuilder() { delete rep_; }
-
 void VersionBuilder::CheckConsistency(VersionStorageInfo* vstorage) {
  rep_->CheckConsistency(vstorage);
 }
-
 void VersionBuilder::CheckConsistencyForDeletes(VersionEdit* edit,
                                                uint64_t number, int level) {
  rep_->CheckConsistencyForDeletes(edit, number, level);
 }
-
-bool VersionBuilder::CheckConsistencyForNumLevels() {
-  return rep_->CheckConsistencyForNumLevels();
-}
-
 void VersionBuilder::Apply(VersionEdit* edit) { rep_->Apply(edit); }
-
 void VersionBuilder::SaveTo(VersionStorageInfo* vstorage) {
  rep_->SaveTo(vstorage);
 }
-
 void VersionBuilder::LoadTableHandlers(
    InternalStats* internal_stats, int max_threads,
    bool prefetch_index_and_filter_in_cache) {
  rep_->LoadTableHandlers(internal_stats, max_threads,
                          prefetch_index_and_filter_in_cache);
 }
-
 void VersionBuilder::MaybeAddFile(VersionStorageInfo* vstorage, int level,
                                  FileMetaData* f) {
  rep_->MaybeAddFile(vstorage, level, f);
@@ -29,7 +29,6 @@ class VersionBuilder {
  void CheckConsistency(VersionStorageInfo* vstorage);
  void CheckConsistencyForDeletes(VersionEdit* edit, uint64_t number,
                                  int level);
-  bool CheckConsistencyForNumLevels();
  void Apply(VersionEdit* edit);
  void SaveTo(VersionStorageInfo* vstorage);
  void LoadTableHandlers(InternalStats* internal_stats, int max_threads,
@@ -328,6 +328,10 @@ Version::~Version() {
      assert(f->refs > 0);
      f->refs--;
      if (f->refs <= 0) {
+        if (f->table_reader_handle) {
+          cfd_->table_cache()->EraseHandle(f->fd, f->table_reader_handle);
+          f->table_reader_handle = nullptr;
+        }
        vset_->obsolete_files_.push_back(f);
      }
    }
@@ -1227,14 +1231,6 @@ int VersionStorageInfo::MaxInputLevel() const {
  return 0;
 }

-int VersionStorageInfo::MaxOutputLevel(bool allow_ingest_behind) const {
-  if (allow_ingest_behind) {
-    assert(num_levels() > 1);
-    return num_levels() - 2;
-  }
-  return num_levels() - 1;
-}
-
 void VersionStorageInfo::EstimateCompactionBytesNeeded(
    const MutableCFOptions& mutable_cf_options) {
  // Only implemented for level-based compaction
@@ -2335,14 +2331,10 @@ void CloseTables(void* ptr, size_t) {
 VersionSet::~VersionSet() {
  // we need to delete column_family_set_ because its destructor depends on
  // VersionSet
-  Cache* table_cache = column_family_set_->get_table_cache();
-  table_cache->ApplyToAllCacheEntries(&CloseTables, false /* thread_safe */);
+  column_family_set_->get_table_cache()->ApplyToAllCacheEntries(&CloseTables,
+                                                                false);
  column_family_set_.reset();
  for (auto file : obsolete_files_) {
-    if (file->table_reader_handle) {
-      table_cache->Release(file->table_reader_handle);
-      TableCache::Evict(table_cache, file->fd.GetNumber());
-    }
    delete file;
  }
  obsolete_files_.clear();
@@ -2842,6 +2834,11 @@ Status VersionSet::Recover(
        cfd = column_family_set_->GetColumnFamily(edit.column_family_);
        // this should never happen since cf_in_builders is true
        assert(cfd != nullptr);
+        if (edit.max_level_ >= cfd->current()->storage_info()->num_levels()) {
+          s = Status::InvalidArgument(
+              "db has more levels than options.num_levels");
+          break;
+        }

        // if it is not column family add or column family drop,
        // then it's a file add/delete, which should be forwarded
@@ -2925,18 +2922,6 @@ Status VersionSet::Recover(
        list_of_not_found);
  }

-  if (s.ok()) {
-    for (auto cfd : *column_family_set_) {
-      assert(builders.count(cfd->GetID()) > 0);
-      auto* builder = builders[cfd->GetID()]->version_builder();
-      if (!builder->CheckConsistencyForNumLevels()) {
-        s = Status::InvalidArgument(
-            "db has more levels than options.num_levels");
-        break;
-      }
-    }
-  }
-
  if (s.ok()) {
    for (auto cfd : *column_family_set_) {
      if (cfd->IsDropped()) {
@@ -147,7 +147,6 @@ class VersionStorageInfo {
  }

  int MaxInputLevel() const;
-  int MaxOutputLevel(bool allow_ingest_behind) const;

  // Return level number that has idx'th highest score
  int CompactionScoreLevel(int idx) const { return compaction_level_[idx]; }
@@ -26,7 +26,6 @@
 #include "rocksdb/env.h"
 #include "rocksdb/options.h"
 #include "rocksdb/write_batch.h"
-#include "util/cast_util.h"
 #include "util/coding.h"
 #include "util/file_reader_writer.h"
 #include "util/filename.h"
@@ -274,8 +273,8 @@ namespace {
 struct CompareLogByPointer {
  bool operator()(const std::unique_ptr<LogFile>& a,
                  const std::unique_ptr<LogFile>& b) {
-    LogFileImpl* a_impl = static_cast_with_check<LogFileImpl, LogFile>(a.get());
-    LogFileImpl* b_impl = static_cast_with_check<LogFileImpl, LogFile>(b.get());
+    LogFileImpl* a_impl = dynamic_cast<LogFileImpl*>(a.get());
+    LogFileImpl* b_impl = dynamic_cast<LogFileImpl*>(b.get());
    return *a_impl < *b_impl;
  }
 };
@@ -1303,7 +1303,6 @@ Status WriteBatchInternal::InsertInto(WriteThread::WriteGroup& write_group,
      continue;
    }
    SetSequence(w->batch, inserter.sequence());
-    w->sequence = inserter.sequence();
    inserter.set_log_number_ref(w->log_ref);
    w->status = w->batch->Iterate(&inserter);
    if (!w->status.ok()) {
@@ -60,7 +60,7 @@ static std::string PrintContents(WriteBatch* b) {
    }
    for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
      ParsedInternalKey ikey;
-      ikey.clear();
+      memset((void*)&ikey, 0, sizeof(ikey));
      EXPECT_TRUE(ParseInternalKey(iter->key(), &ikey));
      switch (ikey.type) {
        case kTypeValue:
@@ -451,20 +451,20 @@ TEST_F(WriteBatchTest, DISABLED_ManyUpdates) {
    }
    virtual Status DeleteCF(uint32_t column_family_id,
                            const Slice& key) override {
-      ADD_FAILURE();
+      EXPECT_TRUE(false);
      return Status::OK();
    }
    virtual Status SingleDeleteCF(uint32_t column_family_id,
                                  const Slice& key) override {
-      ADD_FAILURE();
+      EXPECT_TRUE(false);
      return Status::OK();
    }
    virtual Status MergeCF(uint32_t column_family_id, const Slice& key,
                           const Slice& value) override {
-      ADD_FAILURE();
+      EXPECT_TRUE(false);
      return Status::OK();
    }
-    virtual void LogData(const Slice& blob) override { ADD_FAILURE(); }
+    virtual void LogData(const Slice& blob) override { EXPECT_TRUE(false); }
    virtual bool Continue() override { return num_seen < kNumUpdates; }
  } handler;

@@ -502,20 +502,20 @@ TEST_F(WriteBatchTest, DISABLED_LargeKeyValue) {
    }
    virtual Status DeleteCF(uint32_t column_family_id,
                            const Slice& key) override {
-      ADD_FAILURE();
+      EXPECT_TRUE(false);
      return Status::OK();
    }
    virtual Status SingleDeleteCF(uint32_t column_family_id,
                                  const Slice& key) override {
-      ADD_FAILURE();
+      EXPECT_TRUE(false);
      return Status::OK();
    }
    virtual Status MergeCF(uint32_t column_family_id, const Slice& key,
                           const Slice& value) override {
-      ADD_FAILURE();
+      EXPECT_TRUE(false);
      return Status::OK();
    }
-    virtual void LogData(const Slice& blob) override { ADD_FAILURE(); }
+    virtual void LogData(const Slice& blob) override { EXPECT_TRUE(false); }
    virtual bool Continue() override { return num_seen < 2; }
  } handler;

@@ -16,6 +16,7 @@
 #include "rocksdb/db.h"
 #include "rocksdb/write_batch.h"
 #include "port/port.h"
+#include "util/logging.h"
 #include "util/random.h"
 #include "util/sync_point.h"
 #include "util/testharness.h"
@@ -106,10 +107,6 @@ TEST_F(WriteCallbackTest, WriteWithCallbackTest) {
    std::vector<std::pair<string, string>> kvs_;
  };

-  // In each scenario we'll launch multiple threads to write.
-  // The size of each array equals to number of threads, and
-  // each boolean in it denote whether callback of corresponding
-  // thread should succeed or fail.
  std::vector<std::vector<WriteOP>> write_scenarios = {
      {true},
      {false},
@@ -148,37 +145,23 @@ TEST_F(WriteCallbackTest, WriteWithCallbackTest) {
              db_impl = dynamic_cast<DBImpl*>(db);
              ASSERT_TRUE(db_impl);

-              // Writers that have called JoinBatchGroup.
-              std::atomic<uint64_t> threads_joining(0);
-              // Writers that have linked to the queue
-              std::atomic<uint64_t> threads_linked(0);
-              // Writers that pass WriteThread::JoinBatchGroup:Wait sync-point.
-              std::atomic<uint64_t> threads_verified(0);
-
+              std::atomic<uint64_t> threads_waiting(0);
              std::atomic<uint64_t> seq(db_impl->GetLatestSequenceNumber());
              ASSERT_EQ(db_impl->GetLatestSequenceNumber(), 0);

-              rocksdb::SyncPoint::GetInstance()->SetCallBack(
-                  "WriteThread::JoinBatchGroup:Start", [&](void*) {
-                    uint64_t cur_threads_joining = threads_joining.fetch_add(1);
-                    // Wait for the last joined writer to link to the queue.
-                    // In this way the writers link to the queue one by one.
-                    // This allows us to confidently detect the first writer
-                    // who increases threads_linked as the leader.
-                    while (threads_linked.load() < cur_threads_joining) {
-                    }
-                  });
-
-              // Verification once writers call JoinBatchGroup.
              rocksdb::SyncPoint::GetInstance()->SetCallBack(
                  "WriteThread::JoinBatchGroup:Wait", [&](void* arg) {
-                    uint64_t cur_threads_linked = threads_linked.fetch_add(1);
+                    uint64_t cur_threads_waiting = 0;
                    bool is_leader = false;
                    bool is_last = false;

                    // who am i
-                    is_leader = (cur_threads_linked == 0);
-                    is_last = (cur_threads_linked == write_group.size() - 1);
+                    do {
+                      cur_threads_waiting = threads_waiting.load();
+                      is_leader = (cur_threads_waiting == 0);
+                      is_last = (cur_threads_waiting == write_group.size() - 1);
+                    } while (!threads_waiting.compare_exchange_strong(
+                        cur_threads_waiting, cur_threads_waiting + 1));

                    // check my state
                    auto* writer = reinterpret_cast<WriteThread::Writer*>(arg);
@@ -202,10 +185,8 @@ TEST_F(WriteCallbackTest, WriteWithCallbackTest) {
                                  !write_group.back().callback_.should_fail_);
                    }

-                    threads_verified.fetch_add(1);
-                    // Wait here until all verification in this sync-point
-                    // callback finish for all writers.
-                    while (threads_verified.load() < write_group.size()) {
+                    // wait for friends
+                    while (threads_waiting.load() < write_group.size()) {
                    }
                  });

@@ -230,20 +211,17 @@ TEST_F(WriteCallbackTest, WriteWithCallbackTest) {

              std::atomic<uint32_t> thread_num(0);
              std::atomic<char> dummy_key(0);
-
-              // Each write thread create a random write batch and write to DB
-              // with a write callback.
              std::function<void()> write_with_callback_func = [&]() {
                uint32_t i = thread_num.fetch_add(1);
                Random rnd(i);

                // leaders gotta lead
-                while (i > 0 && threads_verified.load() < 1) {
+                while (i > 0 && threads_waiting.load() < 1) {
                }

                // loser has to lose
                while (i == write_group.size() - 1 &&
-                       threads_verified.load() < write_group.size() - 1) {
+                       threads_waiting.load() < write_group.size() - 1) {
                }

                auto& write_op = write_group.at(i);
@@ -253,7 +231,11 @@ TEST_F(WriteCallbackTest, WriteWithCallbackTest) {
                // insert some keys
                for (uint32_t j = 0; j < rnd.Next() % 50; j++) {
                  // grab unique key
-                  char my_key = dummy_key.fetch_add(1);
+                  char my_key = 0;
+                  do {
+                    my_key = dummy_key.load();
+                  } while (
+                      !dummy_key.compare_exchange_strong(my_key, my_key + 1));

                  string skey(5, my_key);
                  string sval(10, my_key);
@@ -57,10 +57,6 @@ uint8_t WriteThread::AwaitState(Writer* w, uint8_t goal_mask,
                                AdaptationContext* ctx) {
  uint8_t state;

-  // 1. Busy loop using "pause" for 1 micro sec
-  // 2. Else SOMETIMES busy loop using "yield" for 100 micro sec (default)
-  // 3. Else blocking wait
-
  // On a modern Xeon each loop takes about 7 nanoseconds (most of which
  // is the effect of the pause instruction), so 200 iterations is a bit
  // more than a microsecond.  This is long enough that waits longer than
@@ -118,21 +114,13 @@ uint8_t WriteThread::AwaitState(Writer* w, uint8_t goal_mask,

  const size_t kMaxSlowYieldsWhileSpinning = 3;

-  // Whether the yield approach has any credit in this context. The credit is
-  // added by yield being succesfull before timing out, and decreased otherwise.
-  auto& yield_credit = ctx->value;
-  // Update the yield_credit based on sample runs or right after a hard failure
  bool update_ctx = false;
-  // Should we reinforce the yield credit
  bool would_spin_again = false;
-  // The samling base for updating the yeild credit. The sampling rate would be
-  // 1/sampling_base.
-  const int sampling_base = 256;

  if (max_yield_usec_ > 0) {
-    update_ctx = Random::GetTLSInstance()->OneIn(sampling_base);
+    update_ctx = Random::GetTLSInstance()->OneIn(256);

-    if (update_ctx || yield_credit.load(std::memory_order_relaxed) >= 0) {
+    if (update_ctx || ctx->value.load(std::memory_order_relaxed) >= 0) {
      // we're updating the adaptation statistics, or spinning has >
      // 50% chance of being shorter than max_yield_usec_ and causing no
      // involuntary context switches
@@ -161,7 +149,7 @@ uint8_t WriteThread::AwaitState(Writer* w, uint8_t goal_mask,
          // accurate enough to measure the yield duration
          ++slow_yield_count;
          if (slow_yield_count >= kMaxSlowYieldsWhileSpinning) {
-            // Not just one ivcsw, but several.  Immediately update yield_credit
+            // Not just one ivcsw, but several.  Immediately update ctx
            // and fall back to blocking
            update_ctx = true;
            break;
@@ -177,19 +165,11 @@ uint8_t WriteThread::AwaitState(Writer* w, uint8_t goal_mask,
  }

  if (update_ctx) {
-    // Since our update is sample based, it is ok if a thread overwrites the
-    // updates by other threads. Thus the update does not have to be atomic.
-    auto v = yield_credit.load(std::memory_order_relaxed);
+    auto v = ctx->value.load(std::memory_order_relaxed);
    // fixed point exponential decay with decay constant 1/1024, with +1
    // and -1 scaled to avoid overflow for int32_t
-    //
-    // On each update the positive credit is decayed by a facor of 1/1024 (i.e.,
-    // 0.1%). If the sampled yield was successful, the credit is also increased
-    // by X. Setting X=2^17 ensures that the credit never exceeds
-    // 2^17*2^10=2^27, which is lower than 2^31 the upperbound of int32_t. Same
-    // logic applies to negative credits.
-    v = v - (v / 1024) + (would_spin_again ? 1 : -1) * 131072;
-    yield_credit.store(v, std::memory_order_relaxed);
+    v = v + (v / 1024) + (would_spin_again ? 1 : -1) * 16384;
+    ctx->value.store(v, std::memory_order_relaxed);
  }

  assert((state & goal_mask) != 0);
@@ -287,11 +267,10 @@ void WriteThread::CompleteFollower(Writer* w, WriteGroup& write_group) {
  SetState(w, STATE_COMPLETED);
 }

-static WriteThread::AdaptationContext jbg_ctx("JoinBatchGroup");
 void WriteThread::JoinBatchGroup(Writer* w) {
-  TEST_SYNC_POINT_CALLBACK("WriteThread::JoinBatchGroup:Start", w);
-  assert(w->batch != nullptr);
+  static AdaptationContext ctx("JoinBatchGroup");

+  assert(w->batch != nullptr);
  bool linked_as_leader = LinkOne(w, &newest_writer_);
  if (linked_as_leader) {
    SetState(w, STATE_GROUP_LEADER);
@@ -315,7 +294,7 @@ void WriteThread::JoinBatchGroup(Writer* w) {
     */
    AwaitState(w, STATE_GROUP_LEADER | STATE_MEMTABLE_WRITER_LEADER |
                      STATE_PARALLEL_MEMTABLE_WRITER | STATE_COMPLETED,
-               &jbg_ctx);
+               &ctx);
    TEST_SYNC_POINT_CALLBACK("WriteThread::JoinBatchGroup:DoneWaiting", w);
  }
 }
@@ -494,9 +473,9 @@ void WriteThread::LaunchParallelMemTableWriters(WriteGroup* write_group) {
  }
 }

-static WriteThread::AdaptationContext cpmtw_ctx("CompleteParallelMemTableWriter");
 // This method is called by both the leader and parallel followers
 bool WriteThread::CompleteParallelMemTableWriter(Writer* w) {
+  static AdaptationContext ctx("CompleteParallelMemTableWriter");

  auto* write_group = w->write_group;
  if (!w->status.ok()) {
@@ -506,7 +485,7 @@ bool WriteThread::CompleteParallelMemTableWriter(Writer* w) {

  if (write_group->running-- > 1) {
    // we're not the last one
-    AwaitState(w, STATE_COMPLETED, &cpmtw_ctx);
+    AwaitState(w, STATE_COMPLETED, &ctx);
    return false;
  }
  // else we're the last parallel worker and should perform exit duties.
@@ -525,9 +504,9 @@ void WriteThread::ExitAsBatchGroupFollower(Writer* w) {
  SetState(write_group->leader, STATE_COMPLETED);
 }

-static WriteThread::AdaptationContext eabgl_ctx("ExitAsBatchGroupLeader");
 void WriteThread::ExitAsBatchGroupLeader(WriteGroup& write_group,
                                         Status status) {
+  static AdaptationContext ctx("ExitAsBatchGroupLeader");
  Writer* leader = write_group.leader;
  Writer* last_writer = write_group.last_writer;
  assert(leader->link_older == nullptr);
@@ -565,7 +544,7 @@ void WriteThread::ExitAsBatchGroupLeader(WriteGroup& write_group,
    }
    AwaitState(leader, STATE_MEMTABLE_WRITER_LEADER |
                           STATE_PARALLEL_MEMTABLE_WRITER | STATE_COMPLETED,
-               &eabgl_ctx);
+               &ctx);
  } else {
    Writer* head = newest_writer_.load(std::memory_order_acquire);
    if (head != last_writer ||
@@ -612,15 +591,15 @@ void WriteThread::ExitAsBatchGroupLeader(WriteGroup& write_group,
  }
 }

-static WriteThread::AdaptationContext eu_ctx("EnterUnbatched");
 void WriteThread::EnterUnbatched(Writer* w, InstrumentedMutex* mu) {
+  static AdaptationContext ctx("EnterUnbatched");
  assert(w != nullptr && w->batch == nullptr);
  mu->Unlock();
  bool linked_as_leader = LinkOne(w, &newest_writer_);
  if (!linked_as_leader) {
    TEST_SYNC_POINT("WriteThread::EnterUnbatched:Wait");
    // Last leader will not pick us as a follower since our batch is nullptr
-    AwaitState(w, STATE_GROUP_LEADER, &eu_ctx);
+    AwaitState(w, STATE_GROUP_LEADER, &ctx);
  }
  if (enable_pipelined_write_) {
    WaitForMemTableWriters();
@@ -640,15 +619,15 @@ void WriteThread::ExitUnbatched(Writer* w) {
  }
 }

-static WriteThread::AdaptationContext wfmw_ctx("WaitForMemTableWriters");
 void WriteThread::WaitForMemTableWriters() {
+  static AdaptationContext ctx("WaitForMemTableWriters");
  assert(enable_pipelined_write_);
  if (newest_memtable_writer_.load() == nullptr) {
    return;
  }
  Writer w;
  if (!LinkOne(&w, &newest_memtable_writer_)) {
-    AwaitState(&w, STATE_MEMTABLE_WRITER_LEADER, &wfmw_ctx);
+    AwaitState(&w, STATE_MEMTABLE_WRITER_LEADER, &ctx);
  }
  newest_memtable_writer_.store(nullptr);
 }
@@ -55,7 +55,7 @@ class WriteThread {

    // The state used to inform a waiting writer that it has become a
    // parallel memtable writer. It can be the group leader who launch the
-    // parallel writer group, or one of the followers. The writer should then
+    // praallel writer group, or one of the followers. The writer should then
    // apply its batch to the memtable concurrently and call
    // CompleteParallelMemTableWriter.
    STATE_PARALLEL_MEMTABLE_WRITER = 8,
@@ -13,7 +13,7 @@ RocksDB is an embeddable persistent key-value store for fast storage. RocksDB ca

 RocksDB builds on [LevelDB](https://code.google.com/p/leveldb/) to be scalable to run on servers with many CPU cores, to efficiently use fast storage, to support IO-bound, in-memory and write-once workloads, and to be flexible to allow for innovation.

-For the latest details, watch [Mark Callaghan’s and Igor Canadi’s talk at CMU on 10/2015](https://scs.hosted.panopto.com/Panopto/Pages/Viewer.aspx?id=f4e0eb37-ae18-468f-9248-cb73edad3e56). [Dhruba Borthakur’s introductory talk](https://github.com/facebook/rocksdb/blob/gh-pages-old/intro.pdf?raw=true) from the Data @ Scale 2013 conference provides some perspective about how RocksDB has evolved.
+For the latest details, watch [Mark Callaghan’s and Igor Canadi’s talk at CMU on 10/2015](https://scs.hosted.panopto.com/Panopto/Pages/Viewer.aspx?id=f4e0eb37-ae18-468f-9248-cb73edad3e56). [Dhruba Borthakur’s introductory talk](https://github.com/facebook/rocksdb/blob/gh-pages/intro.pdf?raw=true) from the Data @ Scale 2013 conference provides some perspective about how RocksDB has evolved.

 ## How does performance compare?
Author	SHA1	Message	Date
Yi Wu	cfaeb5846b	Bump version to 5.7.2	2017-08-15 14:19:55 -07:00
Siying Dong	62e9418a70	Fix false removal of tombstone issue in FIFO and kCompactionStyleNone Summary: Similar to the bug fixed by https://github.com/facebook/rocksdb/pull/2726, FIFO with compaction and kCompactionStyleNone during user customized CompactFiles() with output level to be 0 can suffer from the same problem. Fix it by leveraging the bottommost_level_ flag. Closes https://github.com/facebook/rocksdb/pull/2735 Differential Revision: D5626906 Pulled By: siying fbshipit-source-id: 2b148d0461c61dbd986d74655e384419ae442158	2017-08-15 14:12:19 -07:00
lxcode	03df818cff	Fix missing stdlib include required for abort() Summary: If ROCKSDB_LITE is defined, a call to abort() is introduced. This call requires stdlib.h. Build log of unpatched 5.7.1: http://beefy9.nyi.freebsd.org/data/110amd64-default/447974/logs/rocksdb-lite-5.7.1.log Closes https://github.com/facebook/rocksdb/pull/2744 Reviewed By: yiwu-arbug Differential Revision: D5632372 Pulled By: lxcode fbshipit-source-id: b2a8e692bf14ccf1f875f3a00463e87bba310a2b	2017-08-15 14:08:44 -07:00
Andrew Kryczka	96a092b1a3	empty commit	2017-08-14 10:27:56 -07:00
Andrew Kryczka	d59fdc3946	increment patch version for 5.7.1	2017-08-11 19:09:12 -07:00
Andrew Kryczka	e96d06c816	fix deletion dropping in intra-L0 Summary: `KeyNotExistsBeyondOutputLevel` didn't consider L0 files' key-ranges. So if a key only was covered by older L0 files' key-ranges, we would incorrectly drop deletions of that key. This PR just skips the deletion-dropping optimization when output level is L0. Closes https://github.com/facebook/rocksdb/pull/2726 Differential Revision: D5617286 Pulled By: ajkr fbshipit-source-id: 4bff1396b06d49a828ba4542f249191052915bce	2017-08-11 19:07:16 -07:00
Yi Wu	b86f89f25b	Move blob_db/ttl_extractor.h into blob_db/blob_db.h Summary: Move blob_db/ttl_extractor.h into blob_db/blob_db.h Also exclude TTLExtractor from LITE build. Closes https://github.com/facebook/rocksdb/pull/2665 Differential Revision: D5520009 Pulled By: yiwu-arbug fbshipit-source-id: 4813dcc272c7cc4bf2cdac285256d9a17d78c7b7	2017-07-28 14:39:11 -07:00
Yi Wu	48cf039064	Blob DB TTL extractor Summary: Introducing blob_db::TTLExtractor to replace extract_ttl_fn. The TTL extractor can be use to extract TTL from keys insert with Put or WriteBatch. Change over existing extract_ttl_fn are: * If value is changed, it will be return via std::string* (rather than Slice). With Slice the new value has to be part of the existing value. With std::string* the limitation is removed. * It can optionally return TTL or expiration. Other changes in this PR: * replace `std::chrono::system_clock` with `Env::NowMicros` so that I can mock time in tests. * add several TTL tests. * other minor naming change. Closes https://github.com/facebook/rocksdb/pull/2659 Differential Revision: D5512627 Pulled By: yiwu-arbug fbshipit-source-id: 0dfcb00d74d060b8534c6130c808e4d5d0a54440	2017-07-28 09:41:52 -07:00
Yi Wu	c39939d468	Reduce blob db noisy logging Summary: Remove some of the per-key logging by blob db to reduce noise. Closes https://github.com/facebook/rocksdb/pull/2587 Differential Revision: D5429115 Pulled By: yiwu-arbug fbshipit-source-id: b89328282fb8b3c64923ce48738c16017ce7feaf	2017-07-28 09:40:33 -07:00
Siying Dong	65afd8a1cc	Update java/rocksjni.pom Summary: Closes https://github.com/facebook/rocksdb/pull/2610 Differential Revision: D5445705 Pulled By: siying fbshipit-source-id: d5f97ffdf5bc18b853c3a106755aca96a650e428	2017-07-26 11:31:58 -07:00
Siying Dong	6df239cfee	Remove some left-over BSD headers Summary: Closes https://github.com/facebook/rocksdb/pull/2608 Differential Revision: D5444797 Pulled By: siying fbshipit-source-id: 690581d03f37822e059a16085088e8e2d8a45016	2017-07-26 11:31:42 -07:00
Siying Dong	54c4b3ab4f	Add back the LevelDB license file Summary: Closes https://github.com/facebook/rocksdb/pull/2591 Differential Revision: D5432696 Pulled By: siying fbshipit-source-id: a613230ab916de0b279a65ef429ede65460a8db2	2017-07-26 11:31:29 -07:00
Siying Dong	27768b0ec3	Remove the licensing description in CONTRIBUTING.md Summary: Closes https://github.com/facebook/rocksdb/pull/2590 Differential Revision: D5432539 Pulled By: siying fbshipit-source-id: 49902453bba3c95c1fb8354441b2198649e52bf4	2017-07-26 11:31:18 -07:00
Siying Dong	8673a8c567	Change RocksDB License Summary: Closes https://github.com/facebook/rocksdb/pull/2589 Differential Revision: D5431502 Pulled By: siying fbshipit-source-id: 8ebf8c87883daa9daa54b2303d11ce01ab1f6f75	2017-07-26 11:31:01 -07:00
Maysam Yabandeh	6124420d5d	Remove the orphan assert on !need_log_sync Summary: We initially had disabled support for write_options.sync when concurrent_prepare_ is set. We later added this support but the statement that asserts this combination is not used was left there. This patch cleans it up. Closes https://github.com/facebook/rocksdb/pull/2642 Differential Revision: D5496101 Pulled By: maysamyabandeh fbshipit-source-id: becbc503446f2a51bee24cc861958c090c724ec2	2017-07-26 07:31:36 -07:00
Maysam Yabandeh	258fd9ac46	Release note for partitioned index/filters Summary: Closes https://github.com/facebook/rocksdb/pull/2637 Differential Revision: D5489751 Pulled By: maysamyabandeh fbshipit-source-id: 0298f8960d4f86ce67959616615beee4d802c2e4	2017-07-25 11:20:15 -07:00