table/block.h: change memset

Summary: In gcc-7 the following is an error identified by -Werror=class-memaccess In file included from ./table/get_context.h:14:0, from db/version_set.cc:43: ./table/block.h: In constructor ‘rocksdb::BlockReadAmpBitmap::BlockReadAmpBitmap(size_t, size_t, rocksdb::Statistics*)’: ./table/block.h:73:53: error: ‘void* memset(void*, int, size_t)’ clearing an object of type ‘struct std::atomic<unsigned int>’ with no trivial copy-assignment; use value-initialization instead [-Werror=class-memaccess] memset(bitmap_, 0, bitmap_size * kBytesPersEntry); ^ In file included from ./db/version_set.h:23:0, from db/version_set.cc:12: /toolchain/include/c++/8.0.0/atomic:684:12: note: ‘struct std::atomic<unsigned int>’ declared here struct atomic<unsigned int> : __atomic_base<unsigned int> ^~~~~~~~~~~~~~~~~~~~ As a solution the default initializer can be applied in list context. Signed-off-by: Daniel Black <daniel.black@au.ibm.com> Closes https://github.com/facebook/rocksdb/pull/2561 Differential Revision: D5398714 Pulled By: siying fbshipit-source-id: d883fb88ec7535eee60d551038fe91f14488be36
Fix GitHub issue #3716 : gcc-8 warnings
2026-11-06 19:12:36 -05:00 · 2019-10-30 14:00:55 -07:00 · 2019-10-30 13:54:49 -07:00 · 2019-10-30 13:54:49 -07:00 · 2019-10-30 13:54:49 -07:00 · 2017-11-28 12:25:25 -08:00
276 changed files with 8929 additions and 7669 deletions
@@ -0,0 +1,17 @@
+{
+  "project_id" : "rocksdb",
+  "conduit_uri" : "https://phabricator.fb.com/api/",
+  "copyright_holder" : "Facebook",
+  "load" : [
+    "arcanist_util"
+  ],
+  "lint.engine" : "FacebookFbcodeLintEngine",
+  "lint.engine.single.linter" : "FbcodeCppLinter",
+  "unit.engine" : "FacebookFbcodeUnitTestEngine",
+  "arcanist_configuration" : "FacebookArcanistConfiguration",
+  "base" : "git:HEAD^, hg:.^",
+  "git.default-relative-commit" : "HEAD^",
+  "git:arc.feature.start.default" : "origin/master",
+  "arc.feature.start.default" : "master",
+  "history.immutable" : false
+}
@@ -39,6 +39,7 @@ coverage/COVERAGE_REPORT
 .gdbhistory
 .gdb_history
 package/
+.phutil_module_cache
 unity.a
 tags
 etags
@@ -57,7 +57,7 @@ before_script:

 script:
  - ${CXX} --version
-  - if [ "${TEST_GROUP}" == 'platform_dependent' ]; then ccache -C && OPT=-DTRAVIS V=1 ROCKSDBTESTS_END=db_block_cache_test make -j4 all_but_some_tests check_some; fi
+  - if [ "${TEST_GROUP}" == 'platform_dependent' ]; then OPT=-DTRAVIS V=1 make -j4 all; OPT=-DTRAVIS V=1 ROCKSDBTESTS_END=db_block_cache_test make -j4 check_some; fi
  - if [ "${TEST_GROUP}" == '1' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=db_block_cache_test ROCKSDBTESTS_END=comparator_db_test make -j4 check_some; fi
  - if [ "${TEST_GROUP}" == '2' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=comparator_db_test make -j4 check_some; fi
  - if [ "${JOB_NAME}" == 'java_test' ]; then OPT=-DTRAVIS V=1 make clean jclean && make rocksdbjava jtest; fi
@@ -56,7 +56,6 @@ else()
      include_directories(${JEMALLOC_INCLUDE_DIR})
    endif()
  endif()
-
  option(WITH_SNAPPY "build with SNAPPY" OFF)
  if(WITH_SNAPPY)
    find_package(snappy REQUIRED)
@@ -64,38 +63,6 @@ else()
    include_directories(${SNAPPY_INCLUDE_DIR})
    list(APPEND THIRDPARTY_LIBS ${SNAPPY_LIBRARIES})
  endif()
-
-  option(WITH_ZLIB "build with zlib" OFF)
-  if(WITH_ZLIB)
-    find_package(zlib REQUIRED)
-    add_definitions(-DZLIB)
-    include_directories(${ZLIB_INCLUDE_DIR})
-    list(APPEND THIRDPARTY_LIBS ${ZLIB_LIBRARIES})
-  endif()
-
-  option(WITH_BZ2 "build with bzip2" OFF)
-  if(WITH_BZ2)
-    find_package(bzip2 REQUIRED)
-    add_definitions(-DBZIP2)
-    include_directories(${BZIP2_INCLUDE_DIR})
-    list(APPEND THIRDPARTY_LIBS ${BZIP2_LIBRARIES})
-  endif()
-
-  option(WITH_LZ4 "build with lz4" OFF)
-  if(WITH_LZ4)
-    find_package(lz4 REQUIRED)
-    add_definitions(-DLZ4)
-    include_directories(${LZ4_INCLUDE_DIR})
-    list(APPEND THIRDPARTY_LIBS ${LZ4_LIBRARIES})
-  endif()
-
-  option(WITH_ZSTD "build with zstd" OFF)
-  if(WITH_ZSTD)
-    find_package(zstd REQUIRED)
-    add_definitions(-DZSTD)
-    include_directories(${ZSTD_INCLUDE_DIR})
-    list(APPEND THIRDPARTY_LIBS ${ZSTD_LIBRARIES})
-  endif()
 endif()

 if(WIN32)
@@ -148,31 +115,6 @@ if(WIN32 AND MSVC)
  endif()
 endif()

-set(BUILD_VERSION_CC ${CMAKE_BINARY_DIR}/build_version.cc)
-configure_file(util/build_version.cc.in ${BUILD_VERSION_CC} @ONLY)
-add_library(build_version OBJECT ${BUILD_VERSION_CC})
-target_include_directories(build_version PRIVATE
-  ${CMAKE_CURRENT_SOURCE_DIR}/util)
-if(MSVC)
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W3 /wd4127 /wd4800 /wd4996 /wd4351")
-else()
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wshadow -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers")
-  if(MINGW)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format")
-  endif()
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
-  if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -fno-omit-frame-pointer")
-    include(CheckCXXCompilerFlag)
-    CHECK_CXX_COMPILER_FLAG("-momit-leaf-frame-pointer" HAVE_OMIT_LEAF_FRAME_POINTER)
-    if(HAVE_OMIT_LEAF_FRAME_POINTER)
-      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -momit-leaf-frame-pointer")
-    endif()
-  endif()
-endif()
-
 option(PORTABLE "build a portable binary" OFF)
 option(FORCE_SSE42 "force building with SSE4.2, even when PORTABLE=ON" OFF)
 if(PORTABLE)
@@ -216,6 +158,32 @@ if(HAVE_THREAD_LOCAL)
  add_definitions(-DROCKSDB_SUPPORT_THREAD_LOCAL)
 endif()

+set(BUILD_VERSION_CC ${CMAKE_BINARY_DIR}/build_version.cc)
+configure_file(util/build_version.cc.in ${BUILD_VERSION_CC} @ONLY)
+add_library(build_version OBJECT ${BUILD_VERSION_CC})
+target_include_directories(build_version PRIVATE
+  ${CMAKE_CURRENT_SOURCE_DIR}/util)
+if(MSVC)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W3 /wd4127 /wd4800 /wd4996 /wd4351")
+else()
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wshadow -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers")
+  if(MINGW)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format")
+  endif()
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+  if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
+    add_definitions(-DNDEBUG)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -fno-omit-frame-pointer")
+    include(CheckCXXCompilerFlag)
+    CHECK_CXX_COMPILER_FLAG("-momit-leaf-frame-pointer" HAVE_OMIT_LEAF_FRAME_POINTER)
+    if(HAVE_OMIT_LEAF_FRAME_POINTER)
+      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -momit-leaf-frame-pointer")
+    endif()
+  endif()
+endif()
+
 option(FAIL_ON_WARNINGS "Treat compile warnings as errors" ON)
 if(FAIL_ON_WARNINGS)
  if(MSVC)
@@ -325,7 +293,6 @@ endif()
 option(WITH_FALLOCATE "build with fallocate" ON)

 if(WITH_FALLOCATE)
-  set(CMAKE_REQUIRED_FLAGS ${CMAKE_C_FLAGS})
  include(CheckCSourceCompiles)
  CHECK_C_SOURCE_COMPILES("
 #include <fcntl.h>
@@ -516,9 +483,6 @@ set(SOURCES
        utilities/blob_db/blob_log_writer.cc
        utilities/blob_db/blob_log_format.cc
        utilities/blob_db/ttl_extractor.cc
-        utilities/cassandra/cassandra_compaction_filter.cc
-        utilities/cassandra/format.cc
-        utilities/cassandra/merge_operator.cc
        utilities/checkpoint/checkpoint_impl.cc
        utilities/col_buf_decoder.cc
        utilities/col_buf_encoder.cc
@@ -537,6 +501,8 @@ set(SOURCES
        utilities/memory/memory_util.cc
        utilities/merge_operators/max.cc
        utilities/merge_operators/put.cc
+        utilities/merge_operators/cassandra/format.cc
+        utilities/merge_operators/cassandra/merge_operator.cc
        utilities/merge_operators/string_append/stringappend.cc
        utilities/merge_operators/string_append/stringappend2.cc
        utilities/merge_operators/uint64add.cc
@@ -552,14 +518,13 @@ set(SOURCES
        utilities/spatialdb/spatial_db.cc
        utilities/table_properties_collectors/compact_on_deletion_collector.cc
        utilities/transactions/optimistic_transaction_db_impl.cc
-        utilities/transactions/optimistic_transaction.cc
+        utilities/transactions/optimistic_transaction_impl.cc
        utilities/transactions/transaction_base.cc
-        utilities/transactions/pessimistic_transaction_db.cc
+        utilities/transactions/transaction_db_impl.cc
        utilities/transactions/transaction_db_mutex_impl.cc
-        utilities/transactions/pessimistic_transaction.cc
+        utilities/transactions/transaction_impl.cc
        utilities/transactions/transaction_lock_mgr.cc
        utilities/transactions/transaction_util.cc
-        utilities/transactions/write_prepared_txn.cc
        utilities/ttl/db_ttl_impl.cc
        utilities/write_batch_with_index/write_batch_with_index.cc
        utilities/write_batch_with_index/write_batch_with_index_internal.cc
@@ -635,72 +600,7 @@ else()
  message(STATUS "JNI library is disabled")
 endif()

-# Installation and packaging
-if(WIN32)
-  option(ROCKSDB_INSTALL_ON_WINDOWS "Enable install target on Windows" OFF)
-endif()
-if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS)
-  if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
-    if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
-      # Change default installation prefix on Linux to /usr
-      set(CMAKE_INSTALL_PREFIX /usr CACHE PATH "Install path prefix, prepended onto install directories." FORCE)
-    endif()
-  endif()
-
-  include(GNUInstallDirs)
-  include(CMakePackageConfigHelpers)
-
-  set(package_config_destination ${CMAKE_INSTALL_LIBDIR}/cmake/rocksdb)
-
-  configure_package_config_file(
-    ${CMAKE_SOURCE_DIR}/cmake/RocksDBConfig.cmake.in RocksDBConfig.cmake
-    INSTALL_DESTINATION ${package_config_destination}
-  )
-
-  write_basic_package_version_file(
-    RocksDBConfigVersion.cmake
-    VERSION ${ROCKSDB_VERSION}
-    COMPATIBILITY SameMajorVersion
-  )
-
-  install(DIRECTORY include/rocksdb COMPONENT devel DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
-
-  install(
-    TARGETS ${ROCKSDB_STATIC_LIB}
-    EXPORT RocksDBTargets
-    COMPONENT devel
-    ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
-    INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
-  )
-
-  install(
-    TARGETS ${ROCKSDB_SHARED_LIB}
-    EXPORT RocksDBTargets
-    COMPONENT runtime
-    RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
-    LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
-    INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
-  )
-
-  install(
-    EXPORT RocksDBTargets
-    COMPONENT devel
-    DESTINATION ${package_config_destination}
-    NAMESPACE RocksDB::
-  )
-
-  install(
-    FILES
-    ${CMAKE_CURRENT_BINARY_DIR}/RocksDBConfig.cmake
-    ${CMAKE_CURRENT_BINARY_DIR}/RocksDBConfigVersion.cmake
-    COMPONENT devel
-    DESTINATION ${package_config_destination}
-  )
-endif()
-
-option(WITH_TESTS "build with tests" ON)
-if(WITH_TESTS)
-  set(TESTS
+set(TESTS
        cache/cache_test.cc
        cache/lru_cache_test.cc
        db/column_family_test.cc
@@ -806,10 +706,6 @@ if(WITH_TESTS)
        util/thread_local_test.cc
        utilities/backupable/backupable_db_test.cc
        utilities/blob_db/blob_db_test.cc
-        utilities/cassandra/cassandra_functional_test.cc
-        utilities/cassandra/cassandra_format_test.cc
-        utilities/cassandra/cassandra_row_merge_test.cc
-        utilities/cassandra/cassandra_serialize_test.cc
        utilities/checkpoint/checkpoint_test.cc
        utilities/column_aware_encoding_test.cc
        utilities/date_tiered/date_tiered_test.cc
@@ -818,6 +714,10 @@ if(WITH_TESTS)
        utilities/geodb/geodb_test.cc
        utilities/lua/rocks_lua_test.cc
        utilities/memory/memory_test.cc
+        utilities/merge_operators/cassandra/cassandra_merge_test.cc
+        utilities/merge_operators/cassandra/cassandra_format_test.cc
+        utilities/merge_operators/cassandra/cassandra_row_merge_test.cc
+        utilities/merge_operators/cassandra/cassandra_serialize_test.cc
        utilities/merge_operators/string_append/stringappend_test.cc
        utilities/object_registry_test.cc
        utilities/option_change_migration/option_change_migration_test.cc
@@ -826,94 +726,98 @@ if(WITH_TESTS)
        utilities/persistent_cache/persistent_cache_test.cc
        utilities/redis/redis_lists_test.cc
        utilities/spatialdb/spatial_db_test.cc
-        utilities/simulator_cache/sim_cache_test.cc
        utilities/table_properties_collectors/compact_on_deletion_collector_test.cc
        utilities/transactions/optimistic_transaction_test.cc
        utilities/transactions/transaction_test.cc
        utilities/ttl/ttl_test.cc
        utilities/write_batch_with_index/write_batch_with_index_test.cc
-  )
-  if(WITH_LIBRADOS)
-    list(APPEND TESTS utilities/env_librados_test.cc)
-  endif()
+)
+if(WITH_LIBRADOS)
+  list(APPEND TESTS utilities/env_librados_test.cc)
+endif()

-  set(BENCHMARKS
-    cache/cache_bench.cc
-    memtable/memtablerep_bench.cc
-    tools/db_bench.cc
-    table/table_reader_bench.cc
-    utilities/column_aware_encoding_exp.cc
-    utilities/persistent_cache/hash_table_bench.cc)
-  add_library(testharness OBJECT util/testharness.cc)
-  foreach(sourcefile ${BENCHMARKS})
+set(BENCHMARKS
+  cache/cache_bench.cc
+  memtable/memtablerep_bench.cc
+  tools/db_bench.cc
+  table/table_reader_bench.cc
+  utilities/column_aware_encoding_exp.cc
+  utilities/persistent_cache/hash_table_bench.cc)
+add_library(testharness OBJECT util/testharness.cc)
+foreach(sourcefile ${BENCHMARKS})
+  get_filename_component(exename ${sourcefile} NAME_WE)
+  add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile}
+    $<TARGET_OBJECTS:testharness>)
+  target_link_libraries(${exename}${ARTIFACT_SUFFIX} gtest ${LIBS})
+endforeach(sourcefile ${BENCHMARKS})
+
+# For test util library that is build only in DEBUG mode
+# and linked to tests. Add test only code that is not #ifdefed for Release here.
+set(TESTUTIL_SOURCE
+    db/db_test_util.cc
+    monitoring/thread_status_updater_debug.cc
+    table/mock_table.cc
+    util/fault_injection_test_env.cc
+    utilities/merge_operators/cassandra/test_utils.cc
+)
+# test utilities are only build in debug
+enable_testing()
+add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND})
+set(TESTUTILLIB testutillib${ARTIFACT_SUFFIX})
+add_library(${TESTUTILLIB} STATIC ${TESTUTIL_SOURCE})
+if(MSVC)
+  set_target_properties(${TESTUTILLIB} PROPERTIES COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/testutillib${ARTIFACT_SUFFIX}.pdb")
+endif()
+set_target_properties(${TESTUTILLIB}
+      PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
+      EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
+      EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
+      )
+
+# Tests are excluded from Release builds
+set(TEST_EXES ${TESTS})
+
+foreach(sourcefile ${TEST_EXES})
    get_filename_component(exename ${sourcefile} NAME_WE)
    add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile}
      $<TARGET_OBJECTS:testharness>)
-    target_link_libraries(${exename}${ARTIFACT_SUFFIX} gtest ${LIBS})
-  endforeach(sourcefile ${BENCHMARKS})
-
-  # For test util library that is build only in DEBUG mode
-  # and linked to tests. Add test only code that is not #ifdefed for Release here.
-  set(TESTUTIL_SOURCE
-      db/db_test_util.cc
-      monitoring/thread_status_updater_debug.cc
-      table/mock_table.cc
-      util/fault_injection_test_env.cc
-      utilities/cassandra/test_utils.cc
-  )
-  # test utilities are only build in debug
-  enable_testing()
-  add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND})
-  set(TESTUTILLIB testutillib${ARTIFACT_SUFFIX})
-  add_library(${TESTUTILLIB} STATIC ${TESTUTIL_SOURCE})
-  if(MSVC)
-    set_target_properties(${TESTUTILLIB} PROPERTIES COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/testutillib${ARTIFACT_SUFFIX}.pdb")
-  endif()
-  set_target_properties(${TESTUTILLIB}
-        PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
-        EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
-        EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
-        )
-
-  # Tests are excluded from Release builds
-  set(TEST_EXES ${TESTS})
-
-  foreach(sourcefile ${TEST_EXES})
-      get_filename_component(exename ${sourcefile} NAME_WE)
-      add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile}
-        $<TARGET_OBJECTS:testharness>)
-      set_target_properties(${exename}${ARTIFACT_SUFFIX}
-        PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
-        EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
-        EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
-        )
-      target_link_libraries(${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} gtest ${LIBS})
-      if(NOT "${exename}" MATCHES "db_sanity_test")
-        add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
-        add_dependencies(check ${exename}${ARTIFACT_SUFFIX})
-      endif()
-  endforeach(sourcefile ${TEST_EXES})
-
-  # C executables must link to a shared object
-  set(C_TESTS db/c_test.c)
-  set(C_TEST_EXES ${C_TESTS})
-
-  foreach(sourcefile ${C_TEST_EXES})
-      string(REPLACE ".c" "" exename ${sourcefile})
-      string(REGEX REPLACE "^((.+)/)+" "" exename ${exename})
-      add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile})
-      set_target_properties(${exename}${ARTIFACT_SUFFIX}
-        PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
-        EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
-        EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
-        )
-      target_link_libraries(${exename}${ARTIFACT_SUFFIX} ${ROCKSDB_IMPORT_LIB} testutillib${ARTIFACT_SUFFIX})
+    set_target_properties(${exename}${ARTIFACT_SUFFIX}
+      PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
+      EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
+      EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
+      )
+    target_link_libraries(${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} gtest ${LIBS})
+    if(NOT "${exename}" MATCHES "db_sanity_test")
      add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
      add_dependencies(check ${exename}${ARTIFACT_SUFFIX})
-  endforeach(sourcefile ${C_TEST_EXES})
-endif()
+    endif()
+endforeach(sourcefile ${TEST_EXES})

-option(WITH_TOOLS "build with tools" ON)
-if(WITH_TOOLS)
-  add_subdirectory(tools)
+# C executables must link to a shared object
+set(C_TESTS db/c_test.c)
+set(C_TEST_EXES ${C_TESTS})
+
+foreach(sourcefile ${C_TEST_EXES})
+    string(REPLACE ".c" "" exename ${sourcefile})
+    string(REGEX REPLACE "^((.+)/)+" "" exename ${exename})
+    add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile})
+    set_target_properties(${exename}${ARTIFACT_SUFFIX}
+      PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
+      EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
+      EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
+      )
+    target_link_libraries(${exename}${ARTIFACT_SUFFIX} ${ROCKSDB_IMPORT_LIB} testutillib${ARTIFACT_SUFFIX})
+    add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
+    add_dependencies(check ${exename}${ARTIFACT_SUFFIX})
+endforeach(sourcefile ${C_TEST_EXES})
+add_subdirectory(tools)
+
+# Installation and packaging for Linux
+if(NOT WIN32)
+install(TARGETS ${ROCKSDB_STATIC_LIB} COMPONENT devel ARCHIVE DESTINATION lib64)
+install(TARGETS ${ROCKSDB_SHARED_LIB} COMPONENT runtime DESTINATION lib64)
+install(DIRECTORY "${PROJECT_SOURCE_DIR}/include/rocksdb/"
+        COMPONENT devel
+        DESTINATION include/rocksdb)
+set(CMAKE_INSTALL_PREFIX /usr)
 endif()
@@ -1,29 +1,25 @@
 # Rocksdb Change Log
-## Unreleased
-### Public API Change
-### New Features
+## 5.7.5 (11/28/2017)
 ### Bug Fixes
+* Fix IOError on WAL write doesn't propagate to write group follower

-## 5.8.0 (08/30/2017)
-### Public API Change
-* Users of `Statistics::getHistogramString()` will see fewer histogram buckets and different bucket endpoints.
-* `Slice::compare` and BytewiseComparator `Compare` no longer accept `Slice`s containing nullptr.
-* `Transaction::Get` and `Transaction::GetForUpdate` variants with `PinnableSlice` added.
-
-### New Features
-* Add Iterator::Refresh(), which allows users to update the iterator state so that they can avoid some initialization costs of recreating iterators.
-* Replace dynamic_cast<> (except unit test) so people can choose to build with RTTI off. With make, release mode is by default built with -fno-rtti and debug mode is built without it. Users can override it by setting USE_RTTI=0 or 1.
-* Universal compactions including the bottom level can be executed in a dedicated thread pool. This alleviates head-of-line blocking in the compaction queue, which cause write stalling, particularly in multi-instance use cases. Users can enable this feature via `Env::SetBackgroundThreads(N, Env::Priority::BOTTOM)`, where `N > 0`.
-* Allow merge operator to be called even with a single merge operand during compactions, by appropriately overriding `MergeOperator::AllowSingleOperand`.
-* Add `DB::VerifyChecksum()`, which verifies the checksums in all SST files in a running DB.
-* Block-based table support for disabling checksums by setting `BlockBasedTableOptions::checksum = kNoChecksum`.
+## 5.7.4 (08/31/2017)
+No significant changes.

+## 5.7.3 (08/29/2017)
 ### Bug Fixes
-* Fix wrong latencies in `rocksdb.db.get.micros`, `rocksdb.db.write.micros`, and `rocksdb.sst.read.micros`.
-* Fix incorrect dropping of deletions during intra-L0 compaction.
 * Fix transient reappearance of keys covered by range deletions when memtable prefix bloom filter is enabled.
 * Fix potentially wrong file smallest key when range deletions separated by snapshot are written together.

+## 5.7.2 (08/15/2017)
+### Bug Fixes
+* Fix incorrect dropping of deletions issue with FIFO compaction.
+* Fix LITE build compiler error with missing abort().
+
+## 5.7.1 (08/13/2017)
+### Bug Fixes
+* Fix incorrect dropping of deletions during intra-L0 compaction.
+
 ## 5.7.0 (07/13/2017)
 ### Public API Change
 * DB property "rocksdb.sstables" now prints keys in hex form.
@@ -113,8 +113,6 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
 * **Windows**:
  * For building with MS Visual Studio 13 you will need Update 4 installed.
  * Read and follow the instructions at CMakeLists.txt
-  * Or install via [vcpkg](https://github.com/microsoft/vcpkg) 
-       * run `vcpkg install rocksdb`

 * **AIX 6.1**
    * Install AIX Toolbox rpms with gcc
@@ -9,8 +9,5 @@ This is the list of all known third-party language bindings for RocksDB. If some
 * Haskell - https://hackage.haskell.org/package/rocksdb-haskell
 * PHP - https://github.com/Photonios/rocksdb-php
 * C# - https://github.com/warrenfalk/rocksdb-sharp
-* Rust
-    * https://github.com/spacejam/rust-rocksdb
-    * https://github.com/bh1xuw/rust-rocks
+* Rust - https://github.com/spacejam/rust-rocksdb
 * D programming language - https://github.com/b1naryth1ef/rocksdb
-* Erlang - https://gitlab.com/barrel-db/erlang-rocksdb
@@ -101,19 +101,7 @@ endif
 ifeq ($(DEBUG_LEVEL),0)
 OPT += -DNDEBUG
 DISABLE_WARNING_AS_ERROR=1
-
-ifneq ($(USE_RTTI), 1)
-	CXXFLAGS += -fno-rtti
 else
-	CXXFLAGS += -DROCKSDB_USE_RTTI
-endif
-else
-ifneq ($(USE_RTTI), 0)
-	CXXFLAGS += -DROCKSDB_USE_RTTI
-else
-	CXXFLAGS += -fno-rtti
-endif
-
 $(warning Warning: Compiling in debug mode. Don't use the resulting binary in production)
 endif

@@ -232,10 +220,6 @@ ifndef DISABLE_JEMALLOC
 		PLATFORM_CXXFLAGS += -DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE
 		PLATFORM_CCFLAGS  += -DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE
 	endif
-	ifdef WITH_JEMALLOC_FLAG
-		PLATFORM_LDFLAGS += -ljemalloc
-		JAVA_LDFLAGS += -ljemalloc
-	endif
 	EXEC_LDFLAGS := $(JEMALLOC_LIB) $(EXEC_LDFLAGS)
 	PLATFORM_CXXFLAGS += $(JEMALLOC_INCLUDE)
 	PLATFORM_CCFLAGS += $(JEMALLOC_INCLUDE)
@@ -421,7 +405,7 @@ TESTS = \
 	write_buffer_manager_test \
 	stringappend_test \
 	cassandra_format_test \
-	cassandra_functional_test \
+	cassandra_merge_test \
 	cassandra_row_merge_test \
 	cassandra_serialize_test \
 	ttl_test \
@@ -591,8 +575,6 @@ endif  # PLATFORM_SHARED_EXT

 all: $(LIBRARY) $(BENCHMARKS) tools tools_lib test_libs $(TESTS)

-all_but_some_tests: $(LIBRARY) $(BENCHMARKS) tools tools_lib test_libs $(SUBSET)
-
 static_lib: $(LIBRARY)

 shared_lib: $(SHARED)
@@ -796,8 +778,8 @@ ldb_tests: ldb
 crash_test: whitebox_crash_test blackbox_crash_test

 blackbox_crash_test: db_stress
-	python -u tools/db_crashtest.py --simple blackbox $(CRASH_TEST_EXT_ARGS)
-	python -u tools/db_crashtest.py blackbox $(CRASH_TEST_EXT_ARGS)
+	python -u tools/db_crashtest.py --simple blackbox
+	python -u tools/db_crashtest.py blackbox

 ifeq ($(CRASH_TEST_KILL_ODD),)
  CRASH_TEST_KILL_ODD=888887
@@ -805,9 +787,9 @@ endif

 whitebox_crash_test: db_stress
 	python -u tools/db_crashtest.py --simple whitebox --random_kill_odd \
-      $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
+      $(CRASH_TEST_KILL_ODD)
 	python -u tools/db_crashtest.py whitebox  --random_kill_odd \
-      $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
+      $(CRASH_TEST_KILL_ODD)

 asan_check:
 	$(MAKE) clean
@@ -1018,16 +1000,16 @@ option_change_migration_test: utilities/option_change_migration/option_change_mi
 stringappend_test: utilities/merge_operators/string_append/stringappend_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-cassandra_format_test: utilities/cassandra/cassandra_format_test.o utilities/cassandra/test_utils.o $(LIBOBJECTS) $(TESTHARNESS)
+cassandra_format_test: utilities/merge_operators/cassandra/cassandra_format_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-cassandra_functional_test: utilities/cassandra/cassandra_functional_test.o utilities/cassandra/test_utils.o $(LIBOBJECTS) $(TESTHARNESS)
+cassandra_merge_test: utilities/merge_operators/cassandra/cassandra_merge_test.o utilities/merge_operators/cassandra/test_utils.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-cassandra_row_merge_test: utilities/cassandra/cassandra_row_merge_test.o utilities/cassandra/test_utils.o $(LIBOBJECTS) $(TESTHARNESS)
+cassandra_row_merge_test: utilities/merge_operators/cassandra/cassandra_row_merge_test.o utilities/merge_operators/cassandra/test_utils.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-cassandra_serialize_test: utilities/cassandra/cassandra_serialize_test.o $(LIBOBJECTS) $(TESTHARNESS)
+cassandra_serialize_test: utilities/merge_operators/cassandra/cassandra_serialize_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

 redis_test: utilities/redis/redis_lists_test.o $(LIBOBJECTS) $(TESTHARNESS)
@@ -6,6 +6,8 @@ REPO_PATH = TARGETS_PATH[(TARGETS_PATH.find('fbcode/') + len('fbcode/')):] + "/"
 BUCK_BINS = "buck-out/gen/" + REPO_PATH
 TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh"
 rocksdb_compiler_flags = [
+  "-msse",
+  "-msse4.2",
  "-fno-builtin-memcmp",
  "-DROCKSDB_PLATFORM_POSIX",
  "-DROCKSDB_LIB_IO_POSIX",
@@ -14,8 +16,8 @@ rocksdb_compiler_flags = [
  "-DROCKSDB_RANGESYNC_PRESENT",
  "-DROCKSDB_SCHED_GETCPU_PRESENT",
  "-DROCKSDB_SUPPORT_THREAD_LOCAL",
+  "-DHAVE_SSE42",
  "-DOS_LINUX",
-  "-DROCKSDB_UBSAN_RUN",
  # Flags to enable libs we include
  "-DSNAPPY",
  "-DZLIB",
@@ -47,10 +49,6 @@ rocksdb_preprocessor_flags = [
  "-I" + REPO_PATH,
 ]

-rocksdb_arch_preprocessor_flags = {
-  "x86_64": ["-DHAVE_SSE42"],
-}
-
 cpp_library(
    name = "rocksdb_lib",
    headers = AutoHeaders.RECURSIVE_GLOB,
@@ -214,9 +212,6 @@ cpp_library(
      "utilities/blob_db/blob_log_writer.cc",
      "utilities/blob_db/blob_log_format.cc",
      "utilities/blob_db/ttl_extractor.cc",
-      "utilities/cassandra/cassandra_compaction_filter.cc",
-      "utilities/cassandra/format.cc",
-      "utilities/cassandra/merge_operator.cc",
      "utilities/checkpoint/checkpoint_impl.cc",
      "utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc",
      "utilities/convenience/info_log_finder.cc",
@@ -231,6 +226,8 @@ cpp_library(
      "utilities/leveldb_options/leveldb_options.cc",
      "utilities/lua/rocks_lua_compaction_filter.cc",
      "utilities/memory/memory_util.cc",
+      "utilities/merge_operators/cassandra/format.cc",
+      "utilities/merge_operators/cassandra/merge_operator.cc",
      "utilities/merge_operators/max.cc",
      "utilities/merge_operators/put.cc",
      "utilities/merge_operators/string_append/stringappend.cc",
@@ -248,14 +245,13 @@ cpp_library(
      "utilities/spatialdb/spatial_db.cc",
      "utilities/table_properties_collectors/compact_on_deletion_collector.cc",
      "utilities/transactions/optimistic_transaction_db_impl.cc",
-      "utilities/transactions/optimistic_transaction.cc",
+      "utilities/transactions/optimistic_transaction_impl.cc",
      "utilities/transactions/transaction_base.cc",
-      "utilities/transactions/pessimistic_transaction_db.cc",
+      "utilities/transactions/transaction_db_impl.cc",
      "utilities/transactions/transaction_db_mutex_impl.cc",
-      "utilities/transactions/pessimistic_transaction.cc",
+      "utilities/transactions/transaction_impl.cc",
      "utilities/transactions/transaction_lock_mgr.cc",
      "utilities/transactions/transaction_util.cc",
-      "utilities/transactions/write_prepared_txn.cc",
      "utilities/ttl/db_ttl_impl.cc",
      "utilities/write_batch_with_index/write_batch_with_index.cc",
      "utilities/write_batch_with_index/write_batch_with_index_internal.cc",
@@ -266,7 +262,6 @@ cpp_library(
    ],
    deps = [],
    preprocessor_flags = rocksdb_preprocessor_flags,
-    arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
    compiler_flags = rocksdb_compiler_flags,
    external_deps = rocksdb_external_deps,
 )
@@ -280,14 +275,13 @@ cpp_library(
      "util/testharness.cc",
      "util/testutil.cc",
      "db/db_test_util.cc",
-      "utilities/cassandra/test_utils.cc",
+      "utilities/merge_operators/cassandra/test_utils.cc",
      "utilities/col_buf_encoder.cc",
      "utilities/col_buf_decoder.cc",
      "utilities/column_aware_encoding_util.cc",
    ],
    deps = [":rocksdb_lib"],
    preprocessor_flags = rocksdb_preprocessor_flags,
-    arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
    compiler_flags = rocksdb_compiler_flags,
    external_deps = rocksdb_external_deps,
 )
@@ -301,7 +295,6 @@ cpp_library(
    ],
    deps = [":rocksdb_lib"],
    preprocessor_flags = rocksdb_preprocessor_flags,
-    arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
    compiler_flags = rocksdb_compiler_flags,
    external_deps = rocksdb_external_deps,
 )
@@ -312,7 +305,6 @@ cpp_library(
    srcs = ["env/env_basic_test.cc"],
    deps = [":rocksdb_test_lib"],
    preprocessor_flags = rocksdb_preprocessor_flags,
-    arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
    compiler_flags = rocksdb_compiler_flags,
    external_deps = rocksdb_external_deps,
 )
@@ -333,16 +325,16 @@ ROCKS_TESTS = [['arena_test', 'util/arena_test.cc', 'serial'],
 ['c_test', 'db/c_test.c', 'serial'],
 ['cache_test', 'cache/cache_test.cc', 'serial'],
 ['cassandra_format_test',
-  'utilities/cassandra/cassandra_format_test.cc',
+  'utilities/merge_operators/cassandra/cassandra_format_test.cc',
  'serial'],
- ['cassandra_functional_test',
-  'utilities/cassandra/cassandra_functional_test.cc',
+ ['cassandra_merge_test',
+  'utilities/merge_operators/cassandra/cassandra_merge_test.cc',
  'serial'],
 ['cassandra_row_merge_test',
-  'utilities/cassandra/cassandra_row_merge_test.cc',
+  'utilities/merge_operators/cassandra/cassandra_row_merge_test.cc',
  'serial'],
 ['cassandra_serialize_test',
-  'utilities/cassandra/cassandra_serialize_test.cc',
+  'utilities/merge_operators/cassandra/cassandra_serialize_test.cc',
  'serial'],
 ['checkpoint_test', 'utilities/checkpoint/checkpoint_test.cc', 'serial'],
 ['cleanable_test', 'table/cleanable_test.cc', 'serial'],
@@ -508,7 +500,6 @@ for test_cfg in ROCKS_TESTS:
      srcs = [test_cc],
      deps = [":rocksdb_test_lib"],
      preprocessor_flags = rocksdb_preprocessor_flags,
-      arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
      compiler_flags = rocksdb_compiler_flags,
      external_deps = rocksdb_external_deps,
    )
@@ -80,6 +80,3 @@ quasardb uses a heavily tuned RocksDB as its persistence layer.

 ## 360 Pika
 [360](http://www.360.cn/) [Pika](https://github.com/Qihoo360/pika) is a nosql compatible with redis. With the huge amount of data stored, redis may suffer for a capacity bottleneck, and pika was born for solving it. It has widely been widely used in many company
-
-## LzLabs
-LzLabs is using RocksDB as a storage engine in their multi-database distributed framework to store application configuration and user data.
@@ -0,0 +1,3 @@
+<?php
+
+phutil_register_library('arcanist_util', __FILE__);
@@ -0,0 +1,71 @@
+<?php
+
+/**
+ * This file is automatically generated. Use 'arc liberate' to rebuild it.
+ * @generated
+ * @phutil-library-version 2
+ */
+
+if (class_exists('ArcanistWorkflow')) {
+  phutil_register_library_map(array(
+    '__library_version__' => 2,
+    'class' =>
+    array(
+      'ArcanistCpplintLinter' => 'cpp_linter/ArcanistCpplintLinter.php',
+      'BaseDirectoryScopedFormatLinter' => 'cpp_linter/BaseDirectoryScopedFormatLinter.php',
+      'FacebookArcanistConfiguration' => 'config/FacebookArcanistConfiguration.php',
+      'FacebookFbcodeLintEngine' => 'lint_engine/FacebookFbcodeLintEngine.php',
+      'FacebookFbcodeUnitTestEngine' => 'unit_engine/FacebookFbcodeUnitTestEngine.php',
+      'FacebookHowtoevenLintEngine' => 'lint_engine/FacebookHowtoevenLintEngine.php',
+      'FacebookHowtoevenLinter' => 'cpp_linter/FacebookHowtoevenLinter.php',
+      'FbcodeClangFormatLinter' => 'cpp_linter/FbcodeClangFormatLinter.php',
+      'FbcodeCppLinter' => 'cpp_linter/FbcodeCppLinter.php',
+    ),
+    'function' =>
+    array(
+    ),
+    'xmap' =>
+    array(
+      'ArcanistCpplintLinter' => 'ArcanistLinter',
+      'BaseDirectoryScopedFormatLinter' => 'ArcanistLinter',
+      'FacebookArcanistConfiguration' => 'ArcanistConfiguration',
+      'FacebookFbcodeLintEngine' => 'ArcanistLintEngine',
+      'FacebookFbcodeUnitTestEngine' => 'ArcanistBaseUnitTestEngine',
+      'FacebookHowtoevenLintEngine' => 'ArcanistLintEngine',
+      'FacebookHowtoevenLinter' => 'ArcanistLinter',
+      'FbcodeClangFormatLinter' => 'BaseDirectoryScopedFormatLinter',
+      'FbcodeCppLinter' => 'ArcanistLinter',
+    ),
+  ));
+} else {
+  phutil_register_library_map(array(
+    '__library_version__' => 2,
+    'class' =>
+    array(
+      'ArcanistCpplintLinter' => 'cpp_linter/ArcanistCpplintLinter.php',
+      'BaseDirectoryScopedFormatLinter' => 'cpp_linter/BaseDirectoryScopedFormatLinter.php',
+      'FacebookArcanistConfiguration' => 'config/FacebookOldArcanistConfiguration.php',
+      'FacebookFbcodeLintEngine' => 'lint_engine/FacebookFbcodeLintEngine.php',
+      'FacebookFbcodeUnitTestEngine' => 'unit_engine/FacebookOldFbcodeUnitTestEngine.php',
+      'FacebookHowtoevenLintEngine' => 'lint_engine/FacebookHowtoevenLintEngine.php',
+      'FacebookHowtoevenLinter' => 'cpp_linter/FacebookHowtoevenLinter.php',
+      'FbcodeClangFormatLinter' => 'cpp_linter/FbcodeClangFormatLinter.php',
+      'FbcodeCppLinter' => 'cpp_linter/FbcodeCppLinter.php',
+    ),
+    'function' =>
+    array(
+    ),
+    'xmap' =>
+    array(
+      'ArcanistCpplintLinter' => 'ArcanistLinter',
+      'BaseDirectoryScopedFormatLinter' => 'ArcanistLinter',
+      'FacebookArcanistConfiguration' => 'ArcanistConfiguration',
+      'FacebookFbcodeLintEngine' => 'ArcanistLintEngine',
+      'FacebookFbcodeUnitTestEngine' => 'ArcanistBaseUnitTestEngine',
+      'FacebookHowtoevenLintEngine' => 'ArcanistLintEngine',
+      'FacebookHowtoevenLinter' => 'ArcanistLinter',
+      'FbcodeClangFormatLinter' => 'BaseDirectoryScopedFormatLinter',
+      'FbcodeCppLinter' => 'ArcanistLinter',
+    ),
+  ));
+}
@@ -0,0 +1,43 @@
+<?php
+// Copyright 2004-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+require('RocksDBCommonHelper.php');
+
+define("DIFF_COMMAND", "diff");
+
+class FacebookArcanistConfiguration extends ArcanistConfiguration {
+  public function getCustomArgumentsForCommand($command) {
+    if ($command == "land") {
+      return array(
+          'async' => array('help' => 'Just to make tools happy'));
+    }
+    return array();
+  }
+
+  public function didRunWorkflow($command,
+                                 ArcanistWorkflow $workflow,
+                                 $error_code) {
+    // Default options don't terminate on failure, but that's what we want. In
+    // the current case we use assertions intentionally as "terminate on failure
+    // invariants".
+    assert_options(ASSERT_BAIL, true);
+
+    assert($workflow);
+    assert(strlen($command) > 0);
+
+    if ($command == DIFF_COMMAND && !$workflow->isRawDiffSource()) {
+      $diffID = $workflow->getDiffId();
+
+      // When submitting a diff this code path gets executed multiple times in
+      // a row. We only care about the case when ID for the diff is provided
+      // because that's what we need to apply the diff and trigger the tests.
+      if (strlen($diffID) > 0) {
+        assert(is_numeric($diffID));
+        startTestsInSandcastle(true /* $applyDiff */, $workflow, $diffID);
+      }
+    }
+  }
+}
@@ -0,0 +1,43 @@
+<?php
+// Copyright 2004-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+require('RocksDBCommonHelper.php');
+
+define("DIFF_COMMAND", "diff");
+
+class FacebookArcanistConfiguration extends ArcanistConfiguration {
+  public function getCustomArgumentsForCommand($command) {
+    if ($command == "land") {
+      return array(
+          'async' => array('help' => 'Just to make tools happy'));
+    }
+    return array();
+  }
+
+  public function didRunWorkflow($command,
+                                 ArcanistBaseWorkflow $workflow,
+                                 $error_code) {
+    // Default options don't terminate on failure, but that's what we want. In
+    // the current case we use assertions intentionally as "terminate on failure
+    // invariants".
+    assert_options(ASSERT_BAIL, true);
+
+    assert($workflow);
+    assert(strlen($command) > 0);
+
+    if ($command == DIFF_COMMAND && !$workflow->isRawDiffSource()) {
+      $diffID = $workflow->getDiffId();
+
+      // When submitting a diff this code path gets executed multiple times in
+      // a row. We only care about the case when ID for the diff is provided
+      // because that's what we need to apply the diff and trigger the tests.
+      if (strlen($diffID) > 0) {
+        assert(is_numeric($diffID));
+        startTestsInSandcastle(true /* $applyDiff */, $workflow, $diffID);
+      }
+    }
+  }
+}
@@ -1,8 +1,8 @@
 <?php
 // Copyright 2004-present Facebook. All Rights Reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.

 // Name of the environment variables which need to be set by the entity which
 // triggers continuous runs so that code at the end of the file gets executed
@@ -21,17 +21,11 @@ function postURL($diffID, $url) {
  assert(is_numeric($diffID));
  assert(strlen($url) > 0);

-  $cmd_args = array(
-    'diff_id' => (int)$diffID,
-    'name' => sprintf(
-      'click here for sandcastle tests for D%d',
-      (int)$diffID
-    ),
-    'link' => $url
-  );
-  $cmd = 'echo ' . escapeshellarg(json_encode($cmd_args))
-         . ' | arc call-conduit differential.updateunitresults';
-
+  $cmd = 'echo \'{"diff_id": ' . $diffID . ', '
+         . '"name":"click here for sandcastle tests for D' . $diffID . '", '
+         . '"link":"' . $url . '"}\' | '
+         . 'arc call-conduit '
+         . 'differential.updateunitresults';
  shell_exec($cmd);
 }

@@ -41,15 +35,11 @@ function buildUpdateTestStatusCmd($diffID, $test, $status) {
  assert(strlen($test) > 0);
  assert(strlen($status) > 0);

-  $cmd_args = array(
-    'diff_id' => (int)$diffID,
-    'name' => $test,
-    'result' => $status
-  );
-
-  $cmd = 'echo ' . escapeshellarg(json_encode($cmd_args))
-         . ' | arc call-conduit differential.updateunitresults';
-
+  $cmd = 'echo \'{"diff_id": ' . $diffID . ', '
+         . '"name":"' . $test . '", '
+         . '"result":"' . $status . '"}\' | '
+         . 'arc call-conduit '
+         . 'differential.updateunitresults';
  return $cmd;
 }

@@ -78,7 +68,7 @@ function getSteps($applyDiff, $diffID, $username, $test) {
    // and authenticate using that in Sandcastle.
    $setup = array(
      "name" => "Setup arcrc",
-      "shell" => "echo " . escapeshellarg($arcrc_content) . " | base64 --decode"
+      "shell" => "echo " . $arcrc_content . " | base64 --decode"
                 . " | gzip -d > ~/.arcrc",
      "user" => "root"
    );
@@ -104,17 +94,7 @@ function getSteps($applyDiff, $diffID, $username, $test) {
    "user" => "root"
  );

-  // This fixes "FATAL: ThreadSanitizer can not mmap the shadow memory"
-  // Source:
-  // https://github.com/google/sanitizers/wiki/ThreadSanitizerCppManual#FAQ
-  $fix_kernel_issue = array(
-    "name" => "Fix kernel issue with tsan",
-    "shell" => "echo 2 >/proc/sys/kernel/randomize_va_space",
-    "user" => "root"
-  );
-
  $steps[] = $fix_git_ignore;
-  $steps[] = $fix_kernel_issue;

  // This will be the command used to execute particular type of tests.
  $cmd = "";
@@ -124,7 +104,7 @@ function getSteps($applyDiff, $diffID, $username, $test) {
    $patch = array(
      "name" => "Patch " . $diffID,
      "shell" => "arc --arcrc-file ~/.arcrc "
-                  . "patch --nocommit --diff " . escapeshellarg($diffID),
+                  . "patch --nocommit --diff " . $diffID,
      "user" => "root"
    );

@@ -135,8 +115,8 @@ function getSteps($applyDiff, $diffID, $username, $test) {
  }

  // Run the actual command.
-  $cmd = $cmd . "J=$(nproc) ./build_tools/precommit_checker.py " .
-           escapeshellarg($test) . "; exit_code=$?; ";
+  $cmd = $cmd . "J=$(nproc) ./build_tools/precommit_checker.py " . $test
+           . "; exit_code=$?; ";

  if ($applyDiff) {
    $cmd = $cmd . "([[ \$exit_code -eq 0 ]] &&"
@@ -169,7 +149,7 @@ function getSteps($applyDiff, $diffID, $username, $test) {
    "name" => "Run " . $test,
    "shell" => $cmd,
    "user" => "root",
-    "parser" => "python build_tools/error_filter.py " . escapeshellarg($test),
+    "parser" => "python build_tools/error_filter.py " . $test,
  );

  $steps[] = $run_test;
@@ -217,7 +197,7 @@ function getSandcastleConfig() {
    if (file_exists(PRIMARY_TOKEN_FILE)) {
      $cmd = 'cat ' . PRIMARY_TOKEN_FILE;
    } else {
-      $cmd = 'cat ' . escapeshellarg($cwd_token_file);
+      $cmd = 'cat ' . $cwd_token_file;
    }

    assert(strlen($cmd) > 0);
@@ -261,7 +241,7 @@ function getSandcastleConfig() {
  // List of tests we want to run in Sandcastle.
  $tests = array("unit", "unit_non_shm", "unit_481", "clang_unit", "tsan",
                 "asan", "lite_test", "valgrind", "release", "release_481",
-                 "clang_release", "clang_analyze", "code_cov",
+                 "clang_release", "punit", "clang_analyze", "code_cov",
                 "java_build", "no_compression", "unity", "ubsan");

  $send_email_template = array(
@@ -341,11 +321,9 @@ function getSandcastleConfig() {
  $app = $sandcastle_config[0];
  $token = $sandcastle_config[1];

-  $cmd = 'curl -s -k '
-          . ' -F app=' . escapeshellarg($app)
-          . ' -F token=' . escapeshellarg($token)
-          . ' -F job=' . escapeshellarg(json_encode($job))
-          .' ' . escapeshellarg($url);
+  $cmd = 'curl -s -k -F app=' . $app . ' '
+          . '-F token=' . $token . ' -F job=\'' . json_encode($job)
+          .'\' "' . $url . '"';

  $output = shell_exec($cmd);
  assert(strlen($output) > 0);
@@ -0,0 +1,88 @@
+<?php
+
+/**
+ * Uses google's cpplint.py to check code. RocksDB team forked this file from
+ * phabricator's /src/lint/linter/ArcanistCpplintLinter.php, and customized it
+ * for its own use.
+ *
+ * You can get it here:
+ * http://google-styleguide.googlecode.com/svn/trunk/cpplint/cpplint.py
+ * @group linter
+ */
+final class ArcanistCpplintLinter extends ArcanistLinter {
+
+  public function willLintPaths(array $paths) {
+    return;
+  }
+
+  public function getLinterName() {
+    return 'cpplint.py';
+  }
+
+  public function getLintPath() {
+    $bin = 'cpplint.py';
+    // Search under current dir
+    list($err) = exec_manual('which %s/%s', $this->linterDir(), $bin);
+    if (!$err) {
+      return $this->linterDir().'/'.$bin;
+    }
+
+    // Look for globally installed cpplint.py
+    list($err) = exec_manual('which %s', $bin);
+    if ($err) {
+      throw new ArcanistUsageException(
+        "cpplint.py does not appear to be installed on this system. Install ".
+        "it (e.g., with 'wget \"http://google-styleguide.googlecode.com/".
+        "svn/trunk/cpplint/cpplint.py\"') ".
+        "in your .arcconfig to point to the directory where it resides. ".
+        "Also don't forget to chmod a+x cpplint.py!");
+    }
+
+    return $bin;
+  }
+
+  public function lintPath($path) {
+    $bin = $this->getLintPath();
+    $path = $this->rocksdbDir().'/'.$path;
+
+    $f = new ExecFuture("%C $path", $bin);
+
+    list($err, $stdout, $stderr) = $f->resolve();
+
+    if ($err === 2) {
+      throw new Exception("cpplint failed to run correctly:\n".$stderr);
+    }
+
+    $lines = explode("\n", $stderr);
+    $messages = array();
+    foreach ($lines as $line) {
+      $line = trim($line);
+      $matches = null;
+      $regex = '/^[^:]+:(\d+):\s*(.*)\s*\[(.*)\] \[(\d+)\]$/';
+      if (!preg_match($regex, $line, $matches)) {
+        continue;
+      }
+      foreach ($matches as $key => $match) {
+        $matches[$key] = trim($match);
+      }
+      $message = new ArcanistLintMessage();
+      $message->setPath($path);
+      $message->setLine($matches[1]);
+      $message->setCode($matches[3]);
+      $message->setName($matches[3]);
+      $message->setDescription($matches[2]);
+      $message->setSeverity(ArcanistLintSeverity::SEVERITY_WARNING);
+      $this->addLintMessage($message);
+    }
+  }
+
+  // The path of this linter
+  private function linterDir() {
+    return dirname(__FILE__);
+  }
+
+  // TODO(kaili) a quick and dirty way to figure out rocksdb's root dir.
+  private function rocksdbDir() {
+    return $this->linterDir()."/../..";
+  }
+}
@@ -0,0 +1,74 @@
+<?php
+// Copyright 2004-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+abstract class BaseDirectoryScopedFormatLinter extends ArcanistLinter {
+
+  const LINT_FORMATTING = 1;
+
+  private $changedLines = array();
+  private $rawLintOutput = array();
+
+  abstract protected function getPathsToLint();
+
+  protected function shouldLintPath($path) {
+    foreach ($this->getPathsToLint() as $p) {
+      // check if $path starts with $p
+      if (strncmp($path, $p, strlen($p)) === 0) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  // API to tell this linter which lines were changed
+  final public function setPathChangedLines($path, $changed) {
+    $this->changedLines[$path] = $changed;
+  }
+
+  final public function willLintPaths(array $paths) {
+    $futures = array();
+    foreach ($paths as $path) {
+      if (!$this->shouldLintPath($path)) {
+        continue;
+      }
+
+      $changed = $this->changedLines[$path];
+      if (!isset($changed)) {
+        // do not run linter if there are no changes
+        continue;
+      }
+
+      $futures[$path] = $this->getFormatFuture($path, $changed);
+    }
+
+    foreach (id(new FutureIterator($futures))->limit(8) as $p => $f) {
+      $this->rawLintOutput[$p] = $f->resolvex();
+    }
+  }
+
+  abstract protected function getFormatFuture($path, array $changed);
+  abstract protected function getLintMessage($diff);
+
+  final public function lintPath($path) {
+    if (!isset($this->rawLintOutput[$path])) {
+      return;
+    }
+
+    list($new_content) = $this->rawLintOutput[$path];
+    $old_content = $this->getData($path);
+
+    if ($new_content != $old_content) {
+      $diff = ArcanistDiffUtils::renderDifferences($old_content, $new_content);
+      $this->raiseLintAtOffset(
+        0,
+        self::LINT_FORMATTING,
+        $this->getLintMessage($diff),
+        $old_content,
+        $new_content);
+    }
+  }
+
+}
@@ -0,0 +1,223 @@
+<?php
+// Copyright 2015-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+final class FacebookHowtoevenLinter extends ArcanistLinter {
+
+  const VERSION = 'fd9192f324c36d28136d14380f0b552a1385b59b';
+
+  private $parsedTargets = array();
+
+  public function getLinterName() {
+    return 'Howtoeven';
+  }
+
+  protected function getSeverity($code) {
+    $severities = array(
+      ArcanistLintSeverity::SEVERITY_DISABLED,
+      ArcanistLintSeverity::SEVERITY_ADVICE,
+      ArcanistLintSeverity::SEVERITY_WARNING,
+      ArcanistLintSeverity::SEVERITY_ERROR,
+    );
+    return idx($severities, $code, ArcanistLintSeverity::SEVERITY_WARNING);
+  }
+
+  public function willLintPaths(array $paths) {
+    // Cleanup previous runs.
+    $this->localExecx("rm -rf _build/_lint");
+
+    // Build compilation database.
+    $lintable_paths = $this->getLintablePaths($paths);
+    $interesting_paths = $this->getInterestingPaths($lintable_paths);
+
+    if (!$lintable_paths) {
+      return;
+    }
+
+    // Run lint.
+    try {
+      $this->localExecx(
+        "%C %C -p _build/dev/ %Ls",
+        $this->getBinaryPath(),
+        $this->getFilteredIssues(),
+        $lintable_paths);
+    } catch (CommandException $exception) {
+      PhutilConsole::getConsole()->writeErr($exception->getMessage());
+    }
+
+    // Load results.
+    $result = id(
+      new SQLite3(
+        $this->getProjectRoot().'/_build/_lint/lint.db',
+        SQLITE3_OPEN_READONLY))
+      ->query("SELECT * FROM raised_issues");
+
+    while ($issue = $result->fetchArray(SQLITE3_ASSOC)) {
+      // Skip issues not part of the linted file.
+      if (in_array($issue['file'], $interesting_paths)) {
+        $this->addLintMessage(id(new ArcanistLintMessage())
+          ->setPath($issue['file'])
+          ->setLine($issue['line'])
+          ->setChar($issue['column'])
+          ->setCode('Howtoeven')
+          ->setSeverity($this->getSeverity($issue['severity']))
+          ->setName('Hte-'.$issue['name'])
+          ->setDescription(
+            sprintf(
+              "%s\n\n%s",
+              ($issue['message']) ? $issue['message'] : $issue['description'],
+              $issue['explanation']))
+          ->setOriginalText(idx($issue, 'original', ''))
+          ->setReplacementText(idx($issue, 'replacement', '')));
+      }
+    }
+  }
+
+  public function lintPath($path) {
+  }
+
+  /**
+   * Get the paths that we know how to lint.
+   *
+   * The strategy is to first look whether there's an existing compilation
+   * database and use that if it's exhaustive. We generate our own only if
+   * necessary.
+   */
+  private function getLintablePaths($paths) {
+    // Replace headers with existing sources.
+    for ($i = 0; $i < count($paths); $i++) {
+      if (preg_match("/\.h$/", $paths[$i])) {
+        $header = preg_replace("/\.h$/", ".cpp", $paths[$i]);
+        if (file_exists($header)) {
+          $paths[$i] = $header;
+        }
+      }
+    }
+
+    // Check if database exists and is exhaustive.
+    $available_paths = $this->getAvailablePaths();
+    $lintable_paths = array_intersect($paths, $available_paths);
+    if ($paths === $lintable_paths) {
+      return $lintable_paths;
+    }
+
+    // Generate our own database.
+    $targets = $this->getTargetsFor($paths);
+    if (!$targets) {
+      PhutilConsole::getConsole()->writeErr(
+        "No build targets found for %s\n",
+        implode(', ', $paths));
+      return array();
+    }
+
+    $this->localExecx("./tools/build/bin/fbconfig.par -r %Ls", $targets);
+    $this->localExecx("./tools/build/bin/fbmake.par gen_cdb");
+
+    $available_paths = $this->getAvailablePaths();
+    $lintable_paths = array_intersect($paths, $available_paths);
+    if ($paths != $lintable_paths) {
+      PhutilConsole::getConsole()->writeErr(
+        "Can't lint %s\n",
+        implode(', ', array_diff($paths, $available_paths)));
+    }
+
+    // Return what we know how to lint.
+    return $lintable_paths;
+  }
+
+  /**
+   * Get the available paths in the current compilation database.
+   */
+  private function getAvailablePaths() {
+    $database_path = $this->getProjectRoot()
+      .'/_build/dev/compile_commands.json';
+    if (!file_exists($database_path)) {
+      return array();
+    }
+
+    $entries = json_decode(file_get_contents($database_path), true);
+    $paths = array();
+    foreach ($entries as $entry) {
+      $paths[] = $entry['file'];
+    }
+    return $paths;
+  }
+
+  /**
+   * Search for the targets directories for the given files.
+   */
+  private static function getTargetsFor($paths) {
+    $targets = array();
+    foreach ($paths as $path) {
+      while (($path = dirname($path)) !== '.') {
+        if (in_array('TARGETS', scandir($path))) {
+          $contents = file_get_contents($path.'/TARGETS');
+          if (strpos($contents, 'cpp_binary') !== false) {
+            $targets[] = $path;
+            break;
+          }
+        }
+      }
+    }
+    return array_unique($targets);
+  }
+
+  /**
+   * The paths that we actually want to report on.
+   */
+  private function getInterestingPaths($paths) {
+    $headers = array();
+    foreach ($paths as $path) {
+      $headers[] = preg_replace("/\.cpp$/", ".h", $path);
+    }
+    return array_merge($paths, $headers);
+  }
+
+  /**
+   * The path where the binary is located. Will return the current dewey binary
+   * unless the `HOWTOEVEN_BUILD` environment variable is set.
+   */
+  private function getBinaryPath() {
+    $path = sprintf(
+      "/mnt/dewey/fbcode/.commits/%s/builds/howtoeven/client",
+      self::VERSION);
+
+    $build = getenv('HOWTOEVEN_BUILD');
+    if ($build) {
+      $path = sprintf(
+        "./_build/%s/tools/howtoeven/client",
+        $build);
+      if (!file_exists($path)) {
+        PhutilConsole::getConsole()->writeErr(">> %s does not exist\n", $path);
+        exit(1);
+      }
+    }
+
+    return $path;
+  }
+
+  /**
+   * Execute the command in the root directory.
+   */
+  private function localExecx($command /* , ... */) {
+    $arguments = func_get_args();
+    return newv('ExecFuture', $arguments)
+      ->setCWD($this->getProjectRoot())
+      ->resolvex();
+  }
+
+  /**
+   * The root of the project.
+   */
+  private function getProjectRoot() {
+    return $this->getEngine()->getWorkingCopy()->getProjectRoot();
+  }
+
+  private function getFilteredIssues() {
+    $issues = getenv('HOWTOEVEN_ISSUES');
+    return ($issues) ? csprintf('-issues %s', $issues) : '';
+  }
+
+}
@@ -0,0 +1,58 @@
+<?php
+// Copyright 2004-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+final class FbcodeClangFormatLinter extends BaseDirectoryScopedFormatLinter {
+
+  const LINT_FORMATTING = 1;
+  const CLANG_FORMAT_BINARY = '/mnt/vol/engshare/admin/scripts/clang-format';
+
+  protected function getPathsToLint() {
+    return array('');
+  }
+
+  public function getLinterName() {
+    return 'CLANG_FORMAT';
+  }
+
+  public function getLintSeverityMap() {
+    return array(
+      self::LINT_FORMATTING => ArcanistLintSeverity::SEVERITY_ADVICE,
+    );
+  }
+
+  public function getLintNameMap() {
+    return array(
+      self::LINT_FORMATTING => pht('Changes are not clang-formatted'),
+    );
+  }
+
+  protected function getFormatFuture($path, array $changed) {
+    $args = "";
+    foreach ($changed as $key => $value) {
+      $args .= " --lines=$key:$key";
+    }
+
+    $binary = self::CLANG_FORMAT_BINARY;
+    if (!file_exists($binary)) {
+      // trust the $PATH
+      $binary = "clang-format";
+    }
+
+    return new ExecFuture(
+      "%s %s $args",
+      $binary,
+      $this->getEngine()->getFilePathOnDisk($path));
+  }
+
+  protected function getLintMessage($diff) {
+    $link_to_clang_format =
+      "[[ http://fburl.com/clang-format | clang-format ]]";
+    return <<<LINT_MSG
+Changes in this file were not formatted using $link_to_clang_format.
+Please run build_tools/format-diff.sh or `make format`
+LINT_MSG;
+  }
+}
@@ -0,0 +1,126 @@
+<?php
+// Copyright 2004-present Facebook.  All rights reserved.
+
+class FbcodeCppLinter extends ArcanistLinter {
+  const FLINT      = "/home/engshare/tools/flint";
+  const LINT_ERROR   = 1;
+  const LINT_WARNING = 2;
+  const LINT_ADVICE  = 3;
+  const C_FLAG = "--c_mode=true";
+
+  private $rawLintOutput = array();
+
+  public function willLintPaths(array $paths) {
+    if (!file_exists(self::FLINT)) {
+      return;
+    }
+    $futures = array();
+    foreach ($paths as $p) {
+      $lpath = $this->getEngine()->getFilePathOnDisk($p);
+      $lpath_file = file($lpath);
+      if (preg_match('/\.(c)$/', $lpath) ||
+          preg_match('/-\*-.*Mode: C[; ].*-\*-/', $lpath_file[0]) ||
+          preg_match('/vim(:.*)*:\s*(set\s+)?filetype=c\s*:/', $lpath_file[0])
+          ) {
+        $futures[$p] = new ExecFuture("%s %s %s 2>&1",
+                           self::FLINT, self::C_FLAG,
+                           $this->getEngine()->getFilePathOnDisk($p));
+      } else {
+        $futures[$p] = new ExecFuture("%s %s 2>&1",
+          self::FLINT, $this->getEngine()->getFilePathOnDisk($p));
+      }
+    }
+
+    foreach (Futures($futures)->limit(8) as $p => $f) {
+      $this->rawLintOutput[$p] = $f->resolvex();
+    }
+
+    return;
+  }
+
+  public function getLinterName() {
+    return "FBCPP";
+  }
+
+  public function lintPath($path) {
+    $this->runCppLint($path);
+  }
+
+  private function runCppLint($path) {
+    $msgs = $this->getCppLintOutput($path);
+    foreach ($msgs as $m) {
+      $this->raiseLintAtLine($m['line'], 0, $m['severity'], $m['msg']);
+    }
+  }
+
+  private function adviseOnEachPattern(
+    $path,
+    $regex,
+    $message,
+    $lint_type = self::LINT_ADVICE,
+    $match_idx = 0) {
+      $file_data = $this->getData($path);
+      $matches = array();
+      if (!preg_match_all($regex, $file_data, $matches, PREG_OFFSET_CAPTURE)) {
+        return;
+      }
+
+      foreach ($matches[$match_idx] as $match) {
+        list($match_str, $offset) = $match;
+        $this->raiseLintAtOffset($offset, $lint_type, $message, $match_str);
+      }
+  }
+
+  public function getLintSeverityMap() {
+    return array(
+      self::LINT_WARNING => ArcanistLintSeverity::SEVERITY_WARNING,
+      self::LINT_ADVICE  => ArcanistLintSeverity::SEVERITY_ADVICE,
+      self::LINT_ERROR   => ArcanistLintSeverity::SEVERITY_ERROR
+    );
+  }
+
+  public function getLintNameMap() {
+    return array(
+      self::LINT_ADVICE   => "CppLint Advice",
+      self::LINT_WARNING  => "CppLint Warning",
+      self::LINT_ERROR    => "CppLint Error"
+    );
+  }
+
+  private function getCppLintOutput($path) {
+     if (!array_key_exists($path, $this->rawLintOutput)) {
+       return array();
+     }
+    list($output) = $this->rawLintOutput[$path];
+
+    $msgs = array();
+    $current = null;
+    $matches = array();
+    foreach (explode("\n", $output) as $line) {
+      if (preg_match('/.*?:(\d+):(.*)/', $line, $matches)) {
+        if ($current) {
+          $msgs[] = $current;
+        }
+        $line = $matches[1];
+        $text = $matches[2];
+        if (preg_match('/.*Warning.*/', $text)) {
+          $sev = self::LINT_WARNING;
+        } else if (preg_match('/.*Advice.*/', $text)) {
+          $sev = self::LINT_ADVICE;
+        } else {
+          $sev = self::LINT_ERROR;
+        }
+        $current = array('line'     => $line,
+                         'msg'      => $text,
+                         'severity' => $sev);
+      } else if ($current) {
+        $current['msg'] .= ' ' . $line;
+      }
+    }
+    if ($current) {
+      $msgs[] = $current;
+    }
+
+    return $msgs;
+  }
+}
@@ -0,0 +1,138 @@
+<?php
+// Copyright 2004-present Facebook.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+class FacebookFbcodeLintEngine extends ArcanistLintEngine {
+
+  public function buildLinters() {
+    $linters = array();
+    $paths = $this->getPaths();
+
+    // Remove all deleted files, which are not checked by the
+    // following linters.
+    foreach ($paths as $key => $path) {
+      if (!Filesystem::pathExists($this->getFilePathOnDisk($path))) {
+        unset($paths[$key]);
+      }
+    }
+
+    $generated_linter = new ArcanistGeneratedLinter();
+    $linters[] = $generated_linter;
+
+    $nolint_linter = new ArcanistNoLintLinter();
+    $linters[] = $nolint_linter;
+
+    $text_linter = new ArcanistTextLinter();
+    $text_linter->setCustomSeverityMap(array(
+      ArcanistTextLinter::LINT_LINE_WRAP
+        => ArcanistLintSeverity::SEVERITY_ADVICE,
+    ));
+    $linters[] = $text_linter;
+
+    $java_text_linter = new ArcanistTextLinter();
+    $java_text_linter->setMaxLineLength(100);
+    $java_text_linter->setCustomSeverityMap(array(
+      ArcanistTextLinter::LINT_LINE_WRAP
+        => ArcanistLintSeverity::SEVERITY_ADVICE,
+    ));
+    $linters[] = $java_text_linter;
+
+    $python_linter = new ArcanistPEP8Linter();
+    $linters[] = $python_linter;
+
+    $cpp_linters = array();
+    $cpp_linters[] = $linters[] = new ArcanistCpplintLinter();
+    $cpp_linters[] = $linters[] = new FbcodeCppLinter();
+
+    $clang_format_linter = new FbcodeClangFormatLinter();
+    $linters[] = $clang_format_linter;
+
+    $spelling_linter = new ArcanistSpellingLinter();
+    $linters[] = $spelling_linter;
+
+    foreach ($paths as $path) {
+      $is_text = false;
+
+      $text_extensions = (
+        '/\.('.
+        'cpp|cxx|c|cc|h|hpp|hxx|tcc|'.
+        'py|rb|hs|pl|pm|tw|'.
+        'php|phpt|css|js|'.
+        'java|'.
+        'thrift|'.
+        'lua|'.
+        'siv|'.
+        'txt'.
+        ')$/'
+      );
+      if (preg_match($text_extensions, $path)) {
+        $is_text = true;
+      }
+      if ($is_text) {
+        $nolint_linter->addPath($path);
+
+        $generated_linter->addPath($path);
+        $generated_linter->addData($path, $this->loadData($path));
+
+        if (preg_match('/\.java$/', $path)) {
+          $java_text_linter->addPath($path);
+          $java_text_linter->addData($path, $this->loadData($path));
+        } else {
+          $text_linter->addPath($path);
+          $text_linter->addData($path, $this->loadData($path));
+        }
+
+        $spelling_linter->addPath($path);
+        $spelling_linter->addData($path, $this->loadData($path));
+      }
+      if (preg_match('/\.(cpp|c|cc|cxx|h|hh|hpp|hxx|tcc)$/', $path)
+          && !preg_match('/third-party/', $path)) {
+        foreach ($cpp_linters as &$linter) {
+          $linter->addPath($path);
+          $linter->addData($path, $this->loadData($path));
+        }
+
+        $clang_format_linter->addPath($path);
+        $clang_format_linter->addData($path, $this->loadData($path));
+        $clang_format_linter->setPathChangedLines(
+          $path, $this->getPathChangedLines($path));
+      }
+
+      // Match *.py and contbuild config files
+      if (preg_match('/(\.(py|tw|smcprops)|^contbuild\/configs\/[^\/]*)$/',
+                    $path)) {
+        $space_count = 4;
+        $real_path = $this->getFilePathOnDisk($path);
+        $dir = dirname($real_path);
+        do {
+          if (file_exists($dir.'/.python2space')) {
+            $space_count = 2;
+            break;
+          }
+          $dir = dirname($dir);
+        } while ($dir != '/' && $dir != '.');
+
+        $cur_path_linter = $python_linter;
+        $cur_path_linter->addPath($path);
+        $cur_path_linter->addData($path, $this->loadData($path));
+
+        if (preg_match('/\.tw$/', $path)) {
+          $cur_path_linter->setCustomSeverityMap(array(
+            'E251' => ArcanistLintSeverity::SEVERITY_DISABLED,
+          ));
+        }
+      }
+    }
+
+    $name_linter = new ArcanistFilenameLinter();
+    $linters[] = $name_linter;
+    foreach ($paths as $path) {
+      $name_linter->addPath($path);
+    }
+
+    return $linters;
+  }
+
+}
@@ -0,0 +1,27 @@
+<?php
+// Copyright 2015-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+final class FacebookHowtoevenLintEngine extends ArcanistLintEngine {
+
+  public function buildLinters() {
+    $paths = array();
+
+    foreach ($this->getPaths() as $path) {
+      // Don't try to lint deleted files or changed directories.
+      if (!Filesystem::pathExists($path) || is_dir($path)) {
+        continue;
+      }
+
+      if (preg_match('/\.(cpp|c|cc|cxx|h|hh|hpp|hxx|tcc)$/', $path)) {
+        $paths[] = $path;
+      }
+    }
+
+    $howtoeven = new FacebookHowtoevenLinter();
+    $howtoeven->setPaths($paths);
+    return array($howtoeven);
+  }
+}
@@ -0,0 +1,17 @@
+<?php
+// Copyright 2004-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+class FacebookFbcodeUnitTestEngine extends ArcanistUnitTestEngine {
+
+  public function run() {
+      // For a call to `arc call-conduit differential.updateunitresults` to
+      // succeed we need at least one entry here.
+      $result = new ArcanistUnitTestResult();
+      $result->setName("dummy_placeholder_entry");
+      $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
+      return array($result);
+  }
+}
@@ -0,0 +1,17 @@
+<?php
+// Copyright 2004-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+class FacebookFbcodeUnitTestEngine extends ArcanistBaseUnitTestEngine {
+
+  public function run() {
+      // For a call to `arc call-conduit differential.updateunitresults` to
+      // succeed we need at least one entry here.
+      $result = new ArcanistUnitTestResult();
+      $result->setName("dummy_placeholder_entry");
+      $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
+      return array($result);
+  }
+}
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 # Create a tmp directory for the test to use
 TEST_DIR=$(mktemp -d /dev/shm/fbcode_rocksdb_XXXXXXX)
 TEST_TMPDIR="$TEST_DIR" $@ && rm -rf "$TEST_DIR"
@@ -6,10 +6,12 @@ rocksdb_target_header = """
 import os

 TARGETS_PATH = os.path.dirname(__file__)
-REPO_PATH = "rocksdb/src/"
+REPO_PATH = TARGETS_PATH[(TARGETS_PATH.find('fbcode/') + len('fbcode/')):] + "/"
 BUCK_BINS = "buck-out/gen/" + REPO_PATH
 TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh"
 rocksdb_compiler_flags = [
+  "-msse",
+  "-msse4.2",
  "-fno-builtin-memcmp",
  "-DROCKSDB_PLATFORM_POSIX",
  "-DROCKSDB_LIB_IO_POSIX",
@@ -18,6 +20,7 @@ rocksdb_compiler_flags = [
  "-DROCKSDB_RANGESYNC_PRESENT",
  "-DROCKSDB_SCHED_GETCPU_PRESENT",
  "-DROCKSDB_SUPPORT_THREAD_LOCAL",
+  "-DHAVE_SSE42",
  "-DOS_LINUX",
  # Flags to enable libs we include
  "-DSNAPPY",
@@ -49,10 +52,6 @@ rocksdb_preprocessor_flags = [
  "-I" + REPO_PATH + "include/",
  "-I" + REPO_PATH,
 ]
-
-rocksdb_arch_preprocessor_flags = {
-  "x86_64": ["-DHAVE_SSE42"],
-}
 """


@@ -63,7 +62,6 @@ cpp_library(
    srcs = [%s],
    deps = [%s],
    preprocessor_flags = rocksdb_preprocessor_flags,
-    arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
    compiler_flags = rocksdb_compiler_flags,
    external_deps = rocksdb_external_deps,
 )
@@ -75,7 +73,6 @@ cpp_binary(
  srcs = [%s],
  deps = [%s],
  preprocessor_flags = rocksdb_preprocessor_flags,
-  arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
  compiler_flags = rocksdb_compiler_flags,
  external_deps = rocksdb_external_deps,
 )
@@ -98,7 +95,6 @@ for test_cfg in ROCKS_TESTS:
      srcs = [test_cc],
      deps = [":rocksdb_test_lib"],
      preprocessor_flags = rocksdb_preprocessor_flags,
-      arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
      compiler_flags = rocksdb_compiler_flags,
      external_deps = rocksdb_external_deps,
    )
@@ -51,11 +51,13 @@ if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
    FBCODE_BUILD="true"
    # If we're compiling with TSAN we need pic build
    PIC_BUILD=$COMPILE_WITH_TSAN
-    if [ -z "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
-      source "$PWD/build_tools/fbcode_config.sh"
-    else
+    if [ -n "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
      # we need this to build with MySQL. Don't use for other purposes.
      source "$PWD/build_tools/fbcode_config4.8.1.sh"
+    elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_5xx" ]; then
+      source "$PWD/build_tools/fbcode_config.sh"
+    else
+      source "$PWD/build_tools/fbcode_config_platform007.sh"
    fi
 fi

@@ -317,11 +319,9 @@ EOF
    # Test whether jemalloc is available
    if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null -ljemalloc \
      2>/dev/null; then
-        # This will enable some preprocessor identifiers in the Makefile
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ljemalloc"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -ljemalloc"
        JEMALLOC=1
-        # JEMALLOC can be enabled either using the flag (like here) or by
-        # providing direct link to the jemalloc library
-        WITH_JEMALLOC_FLAG=1
    else
        # jemalloc is not available. Let's try tcmalloc
        if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null \
@@ -458,7 +458,7 @@ elif test -z "$PORTABLE"; then
  fi
 fi

-$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
+$CXX $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
  #include <cstdint>
  #include <nmmintrin.h>
  int main() {
@@ -519,14 +519,7 @@ echo "ROCKSDB_PATCH=$ROCKSDB_PATCH" >> "$OUTPUT"
 echo "CLANG_SCAN_BUILD=$CLANG_SCAN_BUILD" >> "$OUTPUT"
 echo "CLANG_ANALYZER=$CLANG_ANALYZER" >> "$OUTPUT"
 echo "PROFILING_FLAGS=$PROFILING_FLAGS" >> "$OUTPUT"
-# This will enable some related identifiers for the preprocessor
 if test -n "$JEMALLOC"; then
  echo "JEMALLOC=1" >> "$OUTPUT"
 fi
-# Indicates that jemalloc should be enabled using -ljemalloc flag
-# The alternative is to porvide a direct link to the library via JEMALLOC_LIB
-# and JEMALLOC_INCLUDE
-if test -n "$WITH_JEMALLOC_FLAG"; then
-  echo "WITH_JEMALLOC_FLAG=$WITH_JEMALLOC_FLAG" >> "$OUTPUT"
-fi
 echo "LUA_PATH=$LUA_PATH" >> "$OUTPUT"
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 #
 # Copyright (c) 2016, Facebook. All rights reserved.
 #
@@ -67,7 +67,7 @@ function update_repo_status {
 #

 # Path to the determinator from the root of the RocksDB repo.
-CONTRUN_DETERMINATOR=./build_tools/RocksDBCommonHelper.php
+CONTRUN_DETERMINATOR=./arcanist_util/config/RocksDBCommonHelper.php

 # Value of the previous commit.
 PREV_COMMIT=
@@ -0,0 +1,18 @@
+GCC_BASE=/mnt/gvfs/third-party2/gcc/6e8e715624fd15256a7970073387793dfcf79b46/7.x/centos7-native/b2ef2b6
+CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/ef37e1faa1c29782abfac1ae65a291b9b7966f6d/stable/centos7-native/c9f9104
+LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/c67031f0f739ac61575a061518d6ef5038f99f90/7.x/platform007/5620abc
+GLIBC_BASE=/mnt/gvfs/third-party2/glibc/60d6f124a78798b73944f5ba87c2306ae3460153/2.26/platform007/f259413
+SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/platform007/ca4da3d
+ZLIB_BASE=/mnt/gvfs/third-party2/zlib/22c2d65676fb7c23cfa797c4f6937f38b026f3cf/1.2.8/platform007/ca4da3d
+BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/platform007/ca4da3d
+LZ4_BASE=/mnt/gvfs/third-party2/lz4/907b498203d297947f3bb70b9466f47e100f1873/r131/platform007/ca4da3d
+ZSTD_BASE=/mnt/gvfs/third-party2/zstd/3ee276cbacfad3074e3f07bf826ac47f06970f4e/1.3.5/platform007/15a3614
+GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/platform007/ca4da3d
+JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/9c910d36d6235cc40e8ff559358f1833452300ca/master/platform007/5b0f53e
+NUMA_BASE=/mnt/gvfs/third-party2/numa/9cbf2460284c669ed19c3ccb200a71f7dd7e53c7/2.0.11/platform007/ca4da3d
+LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/bf3d7497fe4e6d007354f0adffa16ce3003f8338/1.3/platform007/6f3e0a9
+TBB_BASE=/mnt/gvfs/third-party2/tbb/ff4e0b093534704d8abab678a4fd7f5ea7b094c7/2018_U5/platform007/ca4da3d
+KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/b5c4a61a5c483ba24722005ae07895971a2ac707/fb/platform007/da39a3e
+BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/92ff90349e2f43ea0a8246d8b1cf17b6869013e3/2.29.1/centos7-native/da39a3e
+VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/f3f697a28122e6bcd513273dd9c1ff23852fc59f/3.13.0/platform007/ca4da3d
+LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.3.4/platform007/5007832
@@ -1,2 +1,2 @@
-#!/usr/bin/env bash
+#!/bin/bash
 docker run -v $PWD:/rocks -w /rocks buildpack-deps make
@@ -0,0 +1,157 @@
+#!/bin/sh
+#
+# Set environment variables so that we can compile rocksdb using
+# fbcode settings.  It uses the latest g++ and clang compilers and also
+# uses jemalloc
+# Environment variables that change the behavior of this script:
+# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included
+
+
+BASEDIR=`dirname $BASH_SOURCE`
+source "$BASEDIR/dependencies_platform007.sh"
+
+CFLAGS=""
+
+# libgcc
+LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/7.3.0"
+LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
+
+# glibc
+GLIBC_INCLUDE="$GLIBC_BASE/include"
+GLIBC_LIBS=" -L $GLIBC_BASE/lib"
+
+# snappy
+SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
+if test -z $PIC_BUILD; then
+  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
+else
+  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a"
+fi
+CFLAGS+=" -DSNAPPY"
+
+if test -z $PIC_BUILD; then
+  # location of zlib headers and libraries
+  ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
+  ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
+  CFLAGS+=" -DZLIB"
+
+  # location of bzip headers and libraries
+  BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
+  BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a"
+  CFLAGS+=" -DBZIP2"
+
+  LZ4_INCLUDE=" -I $LZ4_BASE/include/"
+  LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
+  CFLAGS+=" -DLZ4"
+fi
+
+ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
+if test -z $PIC_BUILD; then
+  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
+else
+  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a"
+fi
+CFLAGS+=" -DZSTD"
+
+# location of gflags headers and libraries
+GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
+if test -z $PIC_BUILD; then
+  GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
+else
+  GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
+fi
+CFLAGS+=" -DGFLAGS=gflags"
+
+# location of jemalloc
+JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
+JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a"
+
+if test -z $PIC_BUILD; then
+  # location of numa
+  NUMA_INCLUDE=" -I $NUMA_BASE/include/"
+  NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
+  CFLAGS+=" -DNUMA"
+
+  # location of libunwind
+  LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
+fi
+
+# location of TBB
+TBB_INCLUDE=" -isystem $TBB_BASE/include/"
+if test -z $PIC_BUILD; then
+  TBB_LIBS="$TBB_BASE/lib/libtbb.a"
+else
+  TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a"
+fi
+CFLAGS+=" -DTBB"
+
+# use Intel SSE support for checksum calculations
+export USE_SSE=1
+export PORTABLE=1
+
+BINUTILS="$BINUTILS_BASE/bin"
+AR="$BINUTILS/ar"
+
+DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE"
+
+STDLIBS="-L $GCC_BASE/lib64"
+
+CLANG_BIN="$CLANG_BASE/bin"
+CLANG_LIB="$CLANG_BASE/lib"
+CLANG_SRC="$CLANG_BASE/../../src"
+
+CLANG_ANALYZER="$CLANG_BIN/clang++"
+CLANG_SCAN_BUILD="$CLANG_SRC/llvm/tools/clang/tools/scan-build/bin/scan-build"
+
+if [ -z "$USE_CLANG" ]; then
+  # gcc
+  CC="$GCC_BASE/bin/gcc"
+  CXX="$GCC_BASE/bin/g++"
+
+  CFLAGS+=" -B$BINUTILS/gold"
+  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
+  CFLAGS+=" -isystem $GLIBC_INCLUDE"
+  JEMALLOC=1
+else
+  # clang
+  CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
+  CC="$CLANG_BIN/clang"
+  CXX="$CLANG_BIN/clang++"
+
+  KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
+
+  CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
+  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x "
+  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x/x86_64-facebook-linux "
+  CFLAGS+=" -isystem $GLIBC_INCLUDE"
+  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
+  CFLAGS+=" -isystem $CLANG_INCLUDE"
+  CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
+  CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
+  CFLAGS+=" -Wno-expansion-to-defined "
+  CXXFLAGS="-nostdinc++"
+fi
+
+CFLAGS+=" $DEPS_INCLUDE"
+CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
+CXXFLAGS+=" $CFLAGS"
+
+EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
+EXEC_LDFLAGS+=" -B$BINUTILS/gold"
+EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform007/lib/ld.so"
+EXEC_LDFLAGS+=" $LIBUNWIND"
+EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform007/lib"
+# required by libtbb
+EXEC_LDFLAGS+=" -ldl"
+
+PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
+
+EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS"
+
+VALGRIND_VER="$VALGRIND_BASE/bin/"
+
+# lua not supported because it's on track for deprecation, I think
+LUA_PATH=
+LUA_LIB=
+
+export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 # If clang_format_diff.py command is not specfied, we assume we are able to
 # access directly without any path.
 if [ -z $CLANG_FORMAT_DIFF ]
@@ -0,0 +1,53 @@
+#!/bin/bash
+#  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+#  This source code is licensed under the BSD-style license found in the
+#  LICENSE file in the root directory of this source tree. An additional grant
+#  of patent rights can be found in the PATENTS file in the same directory.
+
+set -e
+if [ -z "$GIT" ]
+then
+  GIT="git"
+fi
+
+# Print out the colored progress info so that it can be brainlessly
+# distinguished by users.
+function title() {
+  echo -e "\033[1;32m$*\033[0m"
+}
+
+usage="Create new RocksDB version and prepare it for the release process\n"
+usage+="USAGE: ./make_new_version.sh <version> [<remote>]\n"
+usage+="  version: specify a version without '.fb' suffix (e.g. 5.4).\n"
+usage+="  remote: name of the remote to push the branch to (default: origin)."
+
+# -- Pre-check
+if [[ $# < 1 ]]; then
+  echo -e $usage
+  exit 1
+fi
+
+ROCKSDB_VERSION=$1
+
+REMOTE="origin"
+if [[ $# > 1 ]]; then
+  REMOTE=$2
+fi
+
+GIT_BRANCH=`git rev-parse --abbrev-ref HEAD`
+echo $GIT_BRANCH
+
+if [ $GIT_BRANCH != "master" ]; then
+  echo "Error: Current branch is '$GIT_BRANCH', Please switch to master branch."
+  exit 1
+fi
+
+title "Adding new tag for this release ..."
+BRANCH="$ROCKSDB_VERSION.fb"
+$GIT checkout -b $BRANCH
+
+# Setting up the proxy for remote repo access
+title "Pushing new branch to remote repo ..."
+git push $REMOTE --set-upstream $BRANCH
+
+title "Branch $BRANCH is pushed to github;"
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash

 set -e

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 # This script is executed by Sandcastle
 # to determine next steps to run

@@ -85,8 +85,9 @@ NON_SHM="TMPD=/tmp/rocksdb_test_tmp"
 GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1"
 ASAN="COMPILE_WITH_ASAN=1"
 CLANG="USE_CLANG=1"
-LITE="OPT=\"-DROCKSDB_LITE -g\""
-TSAN="COMPILE_WITH_TSAN=1"
+# in gcc-5 there are known problems with TSAN like https://gcc.gnu.org/bugzilla/show_bug.cgi?id=71090.
+# using platform007 gives us gcc-8 or higher which has that bug fixed.
+TSAN="ROCKSDB_FBCODE_BUILD_WITH_PLATFORM007=1 COMPILE_WITH_TSAN=1"
 UBSAN="COMPILE_WITH_UBSAN=1"
 DISABLE_JEMALLOC="DISABLE_JEMALLOC=1"
 HTTP_PROXY="https_proxy=http://fwdproxy.29.prn1:8080 http_proxy=http://fwdproxy.29.prn1:8080 ftp_proxy=http://fwdproxy.29.prn1:8080"
@@ -564,7 +565,7 @@ TSAN_CRASH_TEST_COMMANDS="[
            {
                'name':'Compile and run',
                'timeout': 86400,
-                'shell':'set -o pipefail && $SHM $DEBUG $TSAN CRASH_TEST_KILL_ODD=1887 CRASH_TEST_EXT_ARGS=--log2_keys_per_lock=22  make J=1 crash_test || $CONTRUN_NAME=tsan_crash_test $TASK_CREATION_TOOL',
+                'shell':'set -o pipefail && $SHM $DEBUG $TSAN CRASH_TEST_KILL_ODD=1887 make J=1 crash_test || $CONTRUN_NAME=tsan_crash_test $TASK_CREATION_TOOL',
                'user':'root',
                $PARSER
            },
@@ -583,7 +584,39 @@ run_format_compatible()
  rm -rf /dev/shm/rocksdb
  mkdir /dev/shm/rocksdb

-  tools/check_format_compatible.sh
+  echo '
+  if [ -e "build_tools/build_detect_platform" ]
+  then
+    sed "s/tcmalloc/nothingnothingnothing/g" build_tools/build_detect_platform > $TEST_TMPDIR/temp_build_file
+    rm -rf build_tools/build_detect_platform
+    cp $TEST_TMPDIR/temp_build_file build_tools/build_detect_platform
+    chmod +x build_tools/build_detect_platform
+  fi
+
+  if [ -e "build_detect_platform" ]
+  then
+    sed "s/tcmalloc/nothingnothingnothing/g" build_detect_platform > $TEST_TMPDIR/temp_build_file
+    rm -rf build_detect_platform
+    cp $TEST_TMPDIR/temp_build_file build_detect_platform
+    chmod +x build_detect_platform
+  fi
+
+  make ldb -j32
+
+  if [ -e "build_detect_platform" ]
+  then
+    git checkout -- build_detect_platform
+  fi
+
+  if [ -e "build_tools/build_detect_platform" ]
+  then
+    git checkout -- build_tools/build_detect_platform
+  fi
+  ' > temp_build_ldb.sh
+
+  sed "s/make ldb -j32/source temp_build_ldb.sh/g" tools/check_format_compatible.sh > tools/temp_check_format_compatible.sh
+  chmod +x tools/temp_check_format_compatible.sh
+  tools/temp_check_format_compatible.sh
 }

 FORMAT_COMPATIBLE_COMMANDS="[
@@ -52,6 +52,45 @@ function get_lib_base()
  log_variable $__res_var
 }

+###########################################################
+#                platform007 dependencies                 #
+###########################################################
+
+OUTPUT="$BASEDIR/dependencies_platform007.sh"
+
+rm -f "$OUTPUT"
+touch "$OUTPUT"
+
+echo "Writing dependencies to $OUTPUT"
+
+# Compilers locations
+GCC_BASE=`readlink -f $TP2_LATEST/gcc/7.x/centos7-native/*/`
+CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos7-native/*/`
+
+log_variable GCC_BASE
+log_variable CLANG_BASE
+
+# Libraries locations
+get_lib_base libgcc     7.x     platform007
+get_lib_base glibc      2.26    platform007
+get_lib_base snappy     LATEST  platform007
+get_lib_base zlib       LATEST  platform007
+get_lib_base bzip2      LATEST  platform007
+get_lib_base lz4        LATEST  platform007
+get_lib_base zstd       LATEST  platform007
+get_lib_base gflags     LATEST  platform007
+get_lib_base jemalloc   LATEST  platform007
+get_lib_base numa       LATEST  platform007
+get_lib_base libunwind  LATEST  platform007
+get_lib_base tbb        LATEST  platform007
+
+get_lib_base kernel-headers fb platform007
+get_lib_base binutils   LATEST centos7-native
+get_lib_base valgrind   LATEST platform007
+get_lib_base lua        5.3.4  platform007
+
+git diff $OUTPUT
+
 ###########################################################
 #                   5.x dependencies                      #
 ###########################################################
@@ -27,11 +27,6 @@ std::shared_ptr<Cache> NewClockCache(size_t capacity, int num_shard_bits,
 #include <atomic>
 #include <deque>

-// "tbb/concurrent_hash_map.h" requires RTTI if exception is enabled.
-// Disable it so users can chooose to disable RTTI.
-#ifndef ROCKSDB_USE_RTTI
-#define TBB_USE_EXCEPTIONS 0
-#endif
 #include "tbb/concurrent_hash_map.h"

 #include "cache/sharded_cache.h"
@@ -22,7 +22,7 @@

 namespace rocksdb {

-LRUHandleTable::LRUHandleTable() : list_(nullptr), length_(0), elems_(0) {
+LRUHandleTable::LRUHandleTable() : length_(0), elems_(0), list_(nullptr) {
  Resize();
 }

@@ -100,7 +100,7 @@ void LRUHandleTable::Resize() {
 }

 LRUCacheShard::LRUCacheShard()
-    : high_pri_pool_usage_(0), usage_(0), lru_usage_(0) {
+    : usage_(0), lru_usage_(0), high_pri_pool_usage_(0) {
  // Make empty circular linked list
  lru_.next = &lru_;
  lru_.prev = &lru_;
@@ -157,16 +157,6 @@ void LRUCacheShard::TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri) {
  *lru_low_pri = lru_low_pri_;
 }

-size_t LRUCacheShard::TEST_GetLRUSize() {
-  LRUHandle* lru_handle = lru_.next;
-  size_t lru_size = 0;
-  while (lru_handle != &lru_) {
-    lru_size++;
-    lru_handle = lru_handle->next;
-  }
-  return lru_size;
-}
-
 void LRUCacheShard::LRU_Remove(LRUHandle* e) {
  assert(e->next != nullptr);
  assert(e->prev != nullptr);
@@ -233,22 +223,6 @@ void LRUCacheShard::EvictFromLRU(size_t charge,
  }
 }

-void* LRUCacheShard::operator new(size_t size) {
-  return port::cacheline_aligned_alloc(size);
-}
-
-void* LRUCacheShard::operator new[](size_t size) {
-  return port::cacheline_aligned_alloc(size);
-}
-
-void LRUCacheShard::operator delete(void *memblock) {
-  port::cacheline_aligned_free(memblock);
-}
-
-void LRUCacheShard::operator delete[](void* memblock) {
-  port::cacheline_aligned_free(memblock);
-}
-
 void LRUCacheShard::SetCapacity(size_t capacity) {
  autovector<LRUHandle*> last_reference_list;
  {
@@ -464,18 +438,11 @@ std::string LRUCacheShard::GetPrintableOptions() const {
 LRUCache::LRUCache(size_t capacity, int num_shard_bits,
                   bool strict_capacity_limit, double high_pri_pool_ratio)
    : ShardedCache(capacity, num_shard_bits, strict_capacity_limit) {
-  num_shards_ = 1 << num_shard_bits;
-#if defined(_MSC_VER)
-#pragma warning(push)
-#pragma warning(disable: 4316) // We've validated the alignment with the new operators
-#endif
-  shards_ = new LRUCacheShard[num_shards_];
-#if defined(_MSC_VER)
-#pragma warning(pop)
-#endif
+  int num_shards = 1 << num_shard_bits;
+  shards_ = new LRUCacheShard[num_shards];
  SetCapacity(capacity);
  SetStrictCapacityLimit(strict_capacity_limit);
-  for (int i = 0; i < num_shards_; i++) {
+  for (int i = 0; i < num_shards; i++) {
    shards_[i].SetHighPriorityPoolRatio(high_pri_pool_ratio);
  }
 }
@@ -502,20 +469,7 @@ uint32_t LRUCache::GetHash(Handle* handle) const {
  return reinterpret_cast<const LRUHandle*>(handle)->hash;
 }

-void LRUCache::DisownData() {
-// Do not drop data if compile with ASAN to suppress leak warning.
-#ifndef __SANITIZE_ADDRESS__
-  shards_ = nullptr;
-#endif  // !__SANITIZE_ADDRESS__
-}
-
-size_t LRUCache::TEST_GetLRUSize() {
-  size_t lru_size_of_all_shards = 0;
-  for (int i = 0; i < num_shards_; i++) {
-    lru_size_of_all_shards += shards_[i].TEST_GetLRUSize();
-  }
-  return lru_size_of_all_shards;
-}
+void LRUCache::DisownData() { shards_ = nullptr; }

 std::shared_ptr<Cache> NewLRUCache(size_t capacity, int num_shard_bits,
                                   bool strict_capacity_limit,
@@ -148,13 +148,13 @@ class LRUHandleTable {

  // The table consists of an array of buckets where each bucket is
  // a linked list of cache entries that hash into the bucket.
-  LRUHandle** list_;
  uint32_t length_;
  uint32_t elems_;
+  LRUHandle** list_;
 };

 // A single shard of sharded cache.
-class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard : public CacheShard {
+class LRUCacheShard : public CacheShard {
 public:
  LRUCacheShard();
  virtual ~LRUCacheShard();
@@ -198,19 +198,6 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard : public CacheShard {

  void TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri);

-  //  Retrieves number of elements in LRU, for unit test purpose only
-  //  not threadsafe
-  size_t TEST_GetLRUSize();
-
-  // Overloading to aligned it to cache line size
-  void* operator new(size_t);
-
-  void* operator new[](size_t);
-
-  void operator delete(void *);
-
-  void operator delete[](void*);
-
 private:
  void LRU_Remove(LRUHandle* e);
  void LRU_Insert(LRUHandle* e);
@@ -232,6 +219,12 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard : public CacheShard {
  // Initialized before use.
  size_t capacity_;

+  // Memory size for entries residing in the cache
+  size_t usage_;
+
+  // Memory size for entries residing only in the LRU list
+  size_t lru_usage_;
+
  // Memory size for entries in high-pri pool.
  size_t high_pri_pool_usage_;

@@ -245,6 +238,11 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard : public CacheShard {
  // Remember the value to avoid recomputing each time.
  double high_pri_pool_capacity_;

+  // mutex_ protects the following state.
+  // We don't count mutex_ as the cache's internal state so semantically we
+  // don't mind mutex_ invoking the non-const actions.
+  mutable port::Mutex mutex_;
+
  // Dummy head of LRU list.
  // lru.prev is newest entry, lru.next is oldest entry.
  // LRU contains items which can be evicted, ie reference only by cache
@@ -253,29 +251,7 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard : public CacheShard {
  // Pointer to head of low-pri pool in LRU list.
  LRUHandle* lru_low_pri_;

-  // ------------^^^^^^^^^^^^^-----------
-  // Not frequently modified data members
-  // ------------------------------------
-  //
-  // We separate data members that are updated frequently from the ones that
-  // are not frequently updated so that they don't share the same cache line
-  // which will lead into false cache sharing
-  //
-  // ------------------------------------
-  // Frequently modified data members
-  // ------------vvvvvvvvvvvvv-----------
  LRUHandleTable table_;
-
-  // Memory size for entries residing in the cache
-  size_t usage_;
-
-  // Memory size for entries residing only in the LRU list
-  size_t lru_usage_;
-
-  // mutex_ protects the following state.
-  // We don't count mutex_ as the cache's internal state so semantically we
-  // don't mind mutex_ invoking the non-const actions.
-  mutable port::Mutex mutex_;
 };

 class LRUCache : public ShardedCache {
@@ -291,12 +267,8 @@ class LRUCache : public ShardedCache {
  virtual uint32_t GetHash(Handle* handle) const override;
  virtual void DisownData() override;

-  //  Retrieves number of elements in LRU, for unit test purpose only
-  size_t TEST_GetLRUSize();
-
 private:
  LRUCacheShard* shards_;
-  int num_shards_ = 0;
 };

 }  // namespace rocksdb
@@ -17,16 +17,7 @@ class LRUCacheTest : public testing::Test {
  ~LRUCacheTest() {}

  void NewCache(size_t capacity, double high_pri_pool_ratio = 0.0) {
-    cache_.reset(
-#if defined(_MSC_VER)
-#pragma warning(push)
-#pragma warning(disable: 4316) // We've validated the alignment with the new operators
-#endif
-      new LRUCacheShard()
-#if defined(_MSC_VER)
-#pragma warning(pop)
-#endif
-    );
+    cache_.reset(new LRUCacheShard());
    cache_->SetCapacity(capacity);
    cache_->SetStrictCapacityLimit(false);
    cache_->SetHighPriorityPoolRatio(high_pri_pool_ratio);
@@ -1,3 +0,0 @@
-@PACKAGE_INIT@
-include("${CMAKE_CURRENT_LIST_DIR}/RocksDBTargets.cmake")
-check_required_components(RocksDB)
@@ -1,21 +0,0 @@
-# - Find Bzip2
-# Find the bzip2 compression library and includes
-#
-# BZIP2_INCLUDE_DIR - where to find bzlib.h, etc.
-# BZIP2_LIBRARIES - List of libraries when using bzip2.
-# BZIP2_FOUND - True if bzip2 found.
-
-find_path(BZIP2_INCLUDE_DIR
-  NAMES bzlib.h
-  HINTS ${BZIP2_ROOT_DIR}/include)
-
-find_library(BZIP2_LIBRARIES
-  NAMES bz2
-  HINTS ${BZIP2_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(bzip2 DEFAULT_MSG BZIP2_LIBRARIES BZIP2_INCLUDE_DIR)
-
-mark_as_advanced(
-  BZIP2_LIBRARIES
-  BZIP2_INCLUDE_DIR)
@@ -1,21 +0,0 @@
-# - Find Lz4
-# Find the lz4 compression library and includes
-#
-# LZ4_INCLUDE_DIR - where to find lz4.h, etc.
-# LZ4_LIBRARIES - List of libraries when using lz4.
-# LZ4_FOUND - True if lz4 found.
-
-find_path(LZ4_INCLUDE_DIR
-  NAMES lz4.h
-  HINTS ${LZ4_ROOT_DIR}/include)
-
-find_library(LZ4_LIBRARIES
-  NAMES lz4
-  HINTS ${LZ4_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(lz4 DEFAULT_MSG LZ4_LIBRARIES LZ4_INCLUDE_DIR)
-
-mark_as_advanced(
-  LZ4_LIBRARIES
-  LZ4_INCLUDE_DIR)
@@ -1,21 +0,0 @@
-# - Find zlib
-# Find the zlib compression library and includes
-#
-# ZLIB_INCLUDE_DIR - where to find zlib.h, etc.
-# ZLIB_LIBRARIES - List of libraries when using zlib.
-# ZLIB_FOUND - True if zlib found.
-
-find_path(ZLIB_INCLUDE_DIR
-  NAMES zlib.h
-  HINTS ${ZLIB_ROOT_DIR}/include)
-
-find_library(ZLIB_LIBRARIES
-  NAMES z
-  HINTS ${ZLIB_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(zlib DEFAULT_MSG ZLIB_LIBRARIES ZLIB_INCLUDE_DIR)
-
-mark_as_advanced(
-  ZLIB_LIBRARIES
-  ZLIB_INCLUDE_DIR)
@@ -1,21 +0,0 @@
-# - Find zstd
-# Find the zstd compression library and includes
-#
-# ZSTD_INCLUDE_DIR - where to find zstd.h, etc.
-# ZSTD_LIBRARIES - List of libraries when using zstd.
-# ZSTD_FOUND - True if zstd found.
-
-find_path(ZSTD_INCLUDE_DIR
-  NAMES zstd.h
-  HINTS ${ZSTD_ROOT_DIR}/include)
-
-find_library(ZSTD_LIBRARIES
-  NAMES zstd
-  HINTS ${ZSTD_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(zstd DEFAULT_MSG ZSTD_LIBRARIES ZSTD_INCLUDE_DIR)
-
-mark_as_advanced(
-  ZSTD_LIBRARIES
-  ZSTD_INCLUDE_DIR)
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash

 # Exit on error.
 set -e
@@ -36,7 +36,6 @@
 #include "utilities/merge_operators.h"
 #include "rocksdb/utilities/transaction.h"
 #include "rocksdb/utilities/transaction_db.h"
-#include "rocksdb/utilities/optimistic_transaction_db.h"
 #include "rocksdb/utilities/checkpoint.h"

 using rocksdb::BytewiseComparator;
@@ -53,7 +52,6 @@ using rocksdb::CompressionType;
 using rocksdb::WALRecoveryMode;
 using rocksdb::DB;
 using rocksdb::DBOptions;
-using rocksdb::DbPath;
 using rocksdb::Env;
 using rocksdb::EnvOptions;
 using rocksdb::InfoLogLevel;
@@ -96,8 +94,6 @@ using rocksdb::PinnableSlice;
 using rocksdb::TransactionDBOptions;
 using rocksdb::TransactionDB;
 using rocksdb::TransactionOptions;
-using rocksdb::OptimisticTransactionDB;
-using rocksdb::OptimisticTransactionOptions;
 using rocksdb::Transaction;
 using rocksdb::Checkpoint;

@@ -156,12 +152,6 @@ struct rocksdb_transaction_t {
 struct rocksdb_checkpoint_t {
  Checkpoint* rep;
 };
-struct rocksdb_optimistictransactiondb_t {
-  OptimisticTransactionDB* rep;
-};
-struct rocksdb_optimistictransaction_options_t {
-  OptimisticTransactionOptions rep;
-};

 struct rocksdb_compactionfiltercontext_t {
  CompactionFilter::Context rep;
@@ -392,10 +382,6 @@ struct rocksdb_mergeoperator_t : public MergeOperator {
  }
 };

-struct rocksdb_dbpath_t {
-  DbPath rep;
-};
-
 struct rocksdb_env_t {
  Env* rep;
  bool is_default;
@@ -1386,23 +1372,24 @@ void rocksdb_writebatch_put_log_data(
  b->rep.PutLogData(Slice(blob, len));
 }

+class H : public WriteBatch::Handler {
+ public:
+  void* state_;
+  void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
+  void (*deleted_)(void*, const char* k, size_t klen);
+  virtual void Put(const Slice& key, const Slice& value) override {
+    (*put_)(state_, key.data(), key.size(), value.data(), value.size());
+  }
+  virtual void Delete(const Slice& key) override {
+    (*deleted_)(state_, key.data(), key.size());
+  }
+};
+
 void rocksdb_writebatch_iterate(
    rocksdb_writebatch_t* b,
    void* state,
    void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen),
    void (*deleted)(void*, const char* k, size_t klen)) {
-  class H : public WriteBatch::Handler {
-   public:
-    void* state_;
-    void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
-    void (*deleted_)(void*, const char* k, size_t klen);
-    virtual void Put(const Slice& key, const Slice& value) override {
-      (*put_)(state_, key.data(), key.size(), value.data(), value.size());
-    }
-    virtual void Delete(const Slice& key) override {
-      (*deleted_)(state_, key.data(), key.size());
-    }
-  };
  H handler;
  handler.state_ = state;
  handler.put_ = put;
@@ -1647,18 +1634,6 @@ void rocksdb_writebatch_wi_iterate(
    void* state,
    void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen),
    void (*deleted)(void*, const char* k, size_t klen)) {
-  class H : public WriteBatch::Handler {
-   public:
-    void* state_;
-    void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
-    void (*deleted_)(void*, const char* k, size_t klen);
-    virtual void Put(const Slice& key, const Slice& value) override {
-      (*put_)(state_, key.data(), key.size(), value.data(), value.size());
-    }
-    virtual void Delete(const Slice& key) override {
-      (*deleted_)(state_, key.data(), key.size());
-    }
-  };
  H handler;
  handler.state_ = state;
  handler.put_ = put;
@@ -2023,16 +1998,6 @@ void rocksdb_options_set_paranoid_checks(
  opt->rep.paranoid_checks = v;
 }

-void rocksdb_options_set_db_paths(rocksdb_options_t* opt, 
-                                  const rocksdb_dbpath_t** dbpath_values, 
-                                  size_t num_paths) {
-  std::vector<DbPath> db_paths(num_paths);
-  for (size_t i = 0; i < num_paths; ++i) {
-    db_paths[i] = dbpath_values[i]->rep;
-  }
-  opt->rep.db_paths = db_paths;
-}
-
 void rocksdb_options_set_env(rocksdb_options_t* opt, rocksdb_env_t* env) {
  opt->rep.env = (env ? env->rep : nullptr);
 }
@@ -2111,10 +2076,6 @@ void rocksdb_options_enable_statistics(rocksdb_options_t* opt) {
  opt->rep.statistics = rocksdb::CreateDBStatistics();
 }

-void rocksdb_options_set_skip_stats_update_on_db_open(rocksdb_options_t* opt, unsigned char val) {
-  opt->rep.skip_stats_update_on_db_open = val;
-}
-
 void rocksdb_options_set_num_levels(rocksdb_options_t* opt, int n) {
  opt->rep.num_levels = n;
 }
@@ -2817,17 +2778,6 @@ size_t rocksdb_cache_get_pinned_usage(rocksdb_cache_t* cache) {
  return cache->rep->GetPinnedUsage();
 }

-rocksdb_dbpath_t* rocksdb_dbpath_create(const char* path, uint64_t target_size) {
-  rocksdb_dbpath_t* result = new rocksdb_dbpath_t;
-  result->rep.path = std::string(path);
-  result->rep.target_size = target_size;
-  return result;
-}
-
-void rocksdb_dbpath_destroy(rocksdb_dbpath_t* dbpath) {
-  delete dbpath;
-}
-
 rocksdb_env_t* rocksdb_create_default_env() {
  rocksdb_env_t* result = new rocksdb_env_t;
  result->rep = Env::Default();
@@ -3005,20 +2955,21 @@ void rocksdb_slicetransform_destroy(rocksdb_slicetransform_t* st) {
  delete st;
 }

+struct Wrapper : public rocksdb_slicetransform_t {
+  const SliceTransform* rep_;
+  ~Wrapper() { delete rep_; }
+  const char* Name() const override { return rep_->Name(); }
+  Slice Transform(const Slice& src) const override {
+    return rep_->Transform(src);
+  }
+  bool InDomain(const Slice& src) const override {
+    return rep_->InDomain(src);
+  }
+  bool InRange(const Slice& src) const override { return rep_->InRange(src); }
+  static void DoNothing(void*) { }
+};
+
 rocksdb_slicetransform_t* rocksdb_slicetransform_create_fixed_prefix(size_t prefixLen) {
-  struct Wrapper : public rocksdb_slicetransform_t {
-    const SliceTransform* rep_;
-    ~Wrapper() { delete rep_; }
-    const char* Name() const override { return rep_->Name(); }
-    Slice Transform(const Slice& src) const override {
-      return rep_->Transform(src);
-    }
-    bool InDomain(const Slice& src) const override {
-      return rep_->InDomain(src);
-    }
-    bool InRange(const Slice& src) const override { return rep_->InRange(src); }
-    static void DoNothing(void*) { }
-  };
  Wrapper* wrapper = new Wrapper;
  wrapper->rep_ = rocksdb::NewFixedPrefixTransform(prefixLen);
  wrapper->state_ = nullptr;
@@ -3027,19 +2978,6 @@ rocksdb_slicetransform_t* rocksdb_slicetransform_create_fixed_prefix(size_t pref
 }

 rocksdb_slicetransform_t* rocksdb_slicetransform_create_noop() {
-  struct Wrapper : public rocksdb_slicetransform_t {
-    const SliceTransform* rep_;
-    ~Wrapper() { delete rep_; }
-    const char* Name() const override { return rep_->Name(); }
-    Slice Transform(const Slice& src) const override {
-      return rep_->Transform(src);
-    }
-    bool InDomain(const Slice& src) const override {
-      return rep_->InDomain(src);
-    }
-    bool InRange(const Slice& src) const override { return rep_->InRange(src); }
-    static void DoNothing(void*) { }
-  };
  Wrapper* wrapper = new Wrapper;
  wrapper->rep_ = rocksdb::NewNoopTransform();
  wrapper->state_ = nullptr;
@@ -3262,32 +3200,6 @@ void rocksdb_transaction_options_set_max_write_batch_size(
  opt->rep.max_write_batch_size = size;
 }

-rocksdb_optimistictransaction_options_t*
-rocksdb_optimistictransaction_options_create() {
-  return new rocksdb_optimistictransaction_options_t;
-}
-
-void rocksdb_optimistictransaction_options_destroy(
-    rocksdb_optimistictransaction_options_t* opt) {
-  delete opt;
-}
-
-void rocksdb_optimistictransaction_options_set_set_snapshot(
-    rocksdb_optimistictransaction_options_t* opt, unsigned char v) {
-  opt->rep.set_snapshot = v;
-}
-
-rocksdb_column_family_handle_t* rocksdb_transactiondb_create_column_family(
-    rocksdb_transactiondb_t* txn_db,
-    const rocksdb_options_t* column_family_options,
-    const char* column_family_name, char** errptr) {
-  rocksdb_column_family_handle_t* handle = new rocksdb_column_family_handle_t;
-  SaveError(errptr, txn_db->rep->CreateColumnFamily(
-                        ColumnFamilyOptions(column_family_options->rep),
-                        std::string(column_family_name), &(handle->rep)));
-  return handle;
-}
-
 rocksdb_transactiondb_t* rocksdb_transactiondb_open(
    const rocksdb_options_t* options,
    const rocksdb_transactiondb_options_t* txn_db_options, const char* name,
@@ -3344,14 +3256,7 @@ void rocksdb_transaction_destroy(rocksdb_transaction_t* txn) {
  delete txn;
 }

-const rocksdb_snapshot_t* rocksdb_transaction_get_snapshot(
-    rocksdb_transaction_t* txn) {
-  rocksdb_snapshot_t* result = new rocksdb_snapshot_t;
-  result->rep = txn->rep->GetSnapshot();
-  return result;
-}
-
-// Read a key inside a transaction
+//Read a key inside a transaction
 char* rocksdb_transaction_get(rocksdb_transaction_t* txn,
                              const rocksdb_readoptions_t* options,
                              const char* key, size_t klen, size_t* vlen,
@@ -3371,49 +3276,6 @@ char* rocksdb_transaction_get(rocksdb_transaction_t* txn,
  return result;
 }

-char* rocksdb_transaction_get_cf(rocksdb_transaction_t* txn,
-                                 const rocksdb_readoptions_t* options,
-                                 rocksdb_column_family_handle_t* column_family,
-                                 const char* key, size_t klen, size_t* vlen,
-                                 char** errptr) {
-  char* result = nullptr;
-  std::string tmp;
-  Status s =
-      txn->rep->Get(options->rep, column_family->rep, Slice(key, klen), &tmp);
-  if (s.ok()) {
-    *vlen = tmp.size();
-    result = CopyString(tmp);
-  } else {
-    *vlen = 0;
-    if (!s.IsNotFound()) {
-      SaveError(errptr, s);
-    }
-  }
-  return result;
-}
-
-// Read a key inside a transaction
-char* rocksdb_transaction_get_for_update(rocksdb_transaction_t* txn,
-                                         const rocksdb_readoptions_t* options,
-                                         const char* key, size_t klen,
-                                         size_t* vlen, unsigned char exclusive,
-                                         char** errptr) {
-  char* result = nullptr;
-  std::string tmp;
-  Status s =
-      txn->rep->GetForUpdate(options->rep, Slice(key, klen), &tmp, exclusive);
-  if (s.ok()) {
-    *vlen = tmp.size();
-    result = CopyString(tmp);
-  } else {
-    *vlen = 0;
-    if (!s.IsNotFound()) {
-      SaveError(errptr, s);
-    }
-  }
-  return result;
-}
-
 // Read a key outside a transaction
 char* rocksdb_transactiondb_get(
    rocksdb_transactiondb_t* txn_db,
@@ -3436,26 +3298,6 @@ char* rocksdb_transactiondb_get(
  return result;
 }

-char* rocksdb_transactiondb_get_cf(
-    rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options,
-    rocksdb_column_family_handle_t* column_family, const char* key,
-    size_t keylen, size_t* vallen, char** errptr) {
-  char* result = nullptr;
-  std::string tmp;
-  Status s = txn_db->rep->Get(options->rep, column_family->rep,
-                              Slice(key, keylen), &tmp);
-  if (s.ok()) {
-    *vallen = tmp.size();
-    result = CopyString(tmp);
-  } else {
-    *vallen = 0;
-    if (!s.IsNotFound()) {
-      SaveError(errptr, s);
-    }
-  }
-  return result;
-}
-
 // Put a key inside a transaction
 void rocksdb_transaction_put(rocksdb_transaction_t* txn, const char* key,
                             size_t klen, const char* val, size_t vlen,
@@ -3463,56 +3305,13 @@ void rocksdb_transaction_put(rocksdb_transaction_t* txn, const char* key,
  SaveError(errptr, txn->rep->Put(Slice(key, klen), Slice(val, vlen)));
 }

-void rocksdb_transaction_put_cf(rocksdb_transaction_t* txn,
-                                rocksdb_column_family_handle_t* column_family,
-                                const char* key, size_t klen, const char* val,
-                                size_t vlen, char** errptr) {
-  SaveError(errptr, txn->rep->Put(column_family->rep, Slice(key, klen),
-                                  Slice(val, vlen)));
-}
-
-// Put a key outside a transaction
+//Put a key outside a transaction
 void rocksdb_transactiondb_put(rocksdb_transactiondb_t* txn_db,
                               const rocksdb_writeoptions_t* options,
                               const char* key, size_t klen, const char* val,
                               size_t vlen, char** errptr) {
-  SaveError(errptr, txn_db->rep->Put(options->rep, Slice(key, klen), 
-                                     Slice(val, vlen)));
-}
-
-void rocksdb_transactiondb_put_cf(rocksdb_transactiondb_t* txn_db,
-                                  const rocksdb_writeoptions_t* options,
-                                  rocksdb_column_family_handle_t* column_family,
-                                  const char* key, size_t keylen,
-                                  const char* val, size_t vallen,
-                                  char** errptr) {
-  SaveError(errptr, txn_db->rep->Put(options->rep, column_family->rep,
-                                     Slice(key, keylen), Slice(val, vallen)));
-}
-
-// Write batch into transaction db
-void rocksdb_transactiondb_write(
-        rocksdb_transactiondb_t* db,
-        const rocksdb_writeoptions_t* options,
-        rocksdb_writebatch_t* batch,
-        char** errptr) {
-  SaveError(errptr, db->rep->Write(options->rep, &batch->rep));
-}
-
-// Merge a key inside a transaction
-void rocksdb_transaction_merge(rocksdb_transaction_t* txn, const char* key,
-                               size_t klen, const char* val, size_t vlen,
-                               char** errptr) {
-  SaveError(errptr, txn->rep->Merge(Slice(key, klen), Slice(val, vlen)));
-}
-
-// Merge a key outside a transaction
-void rocksdb_transactiondb_merge(rocksdb_transactiondb_t* txn_db,
-                                 const rocksdb_writeoptions_t* options,
-                                 const char* key, size_t klen, const char* val,
-                                 size_t vlen, char** errptr) {
  SaveError(errptr,
-    txn_db->rep->Merge(options->rep, Slice(key, klen), Slice(val, vlen)));
+            txn_db->rep->Put(options->rep, Slice(key, klen), Slice(val, vlen)));
 }

 // Delete a key inside a transaction
@@ -3521,12 +3320,6 @@ void rocksdb_transaction_delete(rocksdb_transaction_t* txn, const char* key,
  SaveError(errptr, txn->rep->Delete(Slice(key, klen)));
 }

-void rocksdb_transaction_delete_cf(
-    rocksdb_transaction_t* txn, rocksdb_column_family_handle_t* column_family,
-    const char* key, size_t klen, char** errptr) {
-  SaveError(errptr, txn->rep->Delete(column_family->rep, Slice(key, klen)));
-}
-
 // Delete a key outside a transaction
 void rocksdb_transactiondb_delete(rocksdb_transactiondb_t* txn_db,
                                  const rocksdb_writeoptions_t* options,
@@ -3534,14 +3327,6 @@ void rocksdb_transactiondb_delete(rocksdb_transactiondb_t* txn_db,
  SaveError(errptr, txn_db->rep->Delete(options->rep, Slice(key, klen)));
 }

-void rocksdb_transactiondb_delete_cf(
-    rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options,
-    rocksdb_column_family_handle_t* column_family, const char* key,
-    size_t keylen, char** errptr) {
-  SaveError(errptr, txn_db->rep->Delete(options->rep, column_family->rep,
-                                        Slice(key, keylen)));
-}
-
 // Create an iterator inside a transaction
 rocksdb_iterator_t* rocksdb_transaction_create_iterator(
    rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options) {
@@ -3550,14 +3335,6 @@ rocksdb_iterator_t* rocksdb_transaction_create_iterator(
  return result;
 }

-// Create an iterator outside a transaction
-rocksdb_iterator_t* rocksdb_transactiondb_create_iterator(
-    rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options) {
-  rocksdb_iterator_t* result = new rocksdb_iterator_t;
-  result->rep = txn_db->rep->NewIterator(options->rep);
-  return result;
-}
-
 void rocksdb_transactiondb_close(rocksdb_transactiondb_t* txn_db) {
  delete txn_db->rep;
  delete txn_db;
@@ -3574,42 +3351,6 @@ rocksdb_checkpoint_t* rocksdb_transactiondb_checkpoint_object_create(
  return result;
 }

-rocksdb_optimistictransactiondb_t* rocksdb_optimistictransactiondb_open(
-    const rocksdb_options_t* options, const char* name,
-    char** errptr) {
-  OptimisticTransactionDB* otxn_db;
-  if (SaveError(errptr, OptimisticTransactionDB::Open(
-                            options->rep, std::string(name), &otxn_db))) {
-    return nullptr;
-  }
-  rocksdb_optimistictransactiondb_t* result =
-      new rocksdb_optimistictransactiondb_t;
-  result->rep = otxn_db;
-  return result;
-}
-
-rocksdb_transaction_t* rocksdb_optimistictransaction_begin(
-    rocksdb_optimistictransactiondb_t* otxn_db,
-    const rocksdb_writeoptions_t* write_options,
-    const rocksdb_optimistictransaction_options_t* otxn_options,
-    rocksdb_transaction_t* old_txn) {
-  if (old_txn == nullptr) {
-    rocksdb_transaction_t* result = new rocksdb_transaction_t;
-    result->rep = otxn_db->rep->BeginTransaction(write_options->rep,
-                                                 otxn_options->rep, nullptr);
-    return result;
-  }
-  old_txn->rep = otxn_db->rep->BeginTransaction(
-      write_options->rep, otxn_options->rep, old_txn->rep);
-  return old_txn;
-}
-
-void rocksdb_optimistictransactiondb_close(
-    rocksdb_optimistictransactiondb_t* otxn_db) {
-  delete otxn_db->rep;
-  delete otxn_db;
-}
-
 void rocksdb_free(void* ptr) { free(ptr); }

 rocksdb_pinnableslice_t* rocksdb_get_pinned(
@@ -41,7 +41,6 @@ static char dbname[200];
 static char sstfilename[200];
 static char dbbackupname[200];
 static char dbcheckpointname[200];
-static char dbpathname[200];

 static void StartPhase(const char* name) {
  fprintf(stderr, "=== Test %s\n", name);
@@ -348,25 +347,10 @@ static void CheckTxnDBGet(
        Free(&val);
 }

-static void CheckTxnDBGetCF(rocksdb_transactiondb_t* txn_db,
-                            const rocksdb_readoptions_t* options,
-                            rocksdb_column_family_handle_t* column_family,
-                            const char* key, const char* expected) {
-  char* err = NULL;
-  size_t val_len;
-  char* val;
-  val = rocksdb_transactiondb_get_cf(txn_db, options, column_family, key,
-                                     strlen(key), &val_len, &err);
-  CheckNoError(err);
-  CheckEqual(expected, val, val_len);
-  Free(&val);
-}
-
 int main(int argc, char** argv) {
  rocksdb_t* db;
  rocksdb_comparator_t* cmp;
  rocksdb_cache_t* cache;
-  rocksdb_dbpath_t *dbpath;
  rocksdb_env_t* env;
  rocksdb_options_t* options;
  rocksdb_compactoptions_t* coptions;
@@ -401,14 +385,8 @@ int main(int argc, char** argv) {
           GetTempDir(),
           ((int)geteuid()));

-  snprintf(dbpathname, sizeof(dbpathname),
-           "%s/rocksdb_c_test-%d-dbpath",
-           GetTempDir(),
-           ((int) geteuid()));
-
  StartPhase("create_objects");
  cmp = rocksdb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
-  dbpath = rocksdb_dbpath_create(dbpathname, 1024 * 1024);
  env = rocksdb_create_default_env();
  cache = rocksdb_cache_create_lru(100000);

@@ -1379,18 +1357,6 @@ int main(int argc, char** argv) {
    CheckNoError(err);
    CheckTxnDBGet(txn_db, roptions, "foo", NULL);

-    // write batch into TransactionDB
-    rocksdb_writebatch_t* wb = rocksdb_writebatch_create();
-    rocksdb_writebatch_put(wb, "foo", 3, "a", 1);
-    rocksdb_writebatch_clear(wb);
-    rocksdb_writebatch_put(wb, "bar", 3, "b", 1);
-    rocksdb_writebatch_put(wb, "box", 3, "c", 1);
-    rocksdb_writebatch_delete(wb, "bar", 3);
-    rocksdb_transactiondb_write(txn_db, woptions, wb, &err);
-    rocksdb_writebatch_destroy(wb);
-    CheckTxnDBGet(txn_db, roptions, "box", "c");
-    CheckNoError(err);
-
    // begin a transaction
    txn = rocksdb_transaction_begin(txn_db, woptions, txn_options, NULL);
    // put
@@ -1447,23 +1413,6 @@ int main(int argc, char** argv) {
    CheckNoError(err);
    CheckTxnDBGet(txn_db, roptions, "bar", NULL);

-    // Column families.
-    rocksdb_column_family_handle_t* cfh;
-    cfh = rocksdb_transactiondb_create_column_family(txn_db, options,
-                                                     "txn_db_cf", &err);
-    CheckNoError(err);
-
-    rocksdb_transactiondb_put_cf(txn_db, woptions, cfh, "cf_foo", 6, "cf_hello",
-                                 8, &err);
-    CheckNoError(err);
-    CheckTxnDBGetCF(txn_db, roptions, cfh, "cf_foo", "cf_hello");
-
-    rocksdb_transactiondb_delete_cf(txn_db, woptions, cfh, "cf_foo", 6, &err);
-    CheckNoError(err);
-    CheckTxnDBGetCF(txn_db, roptions, cfh, "cf_foo", NULL);
-
-    rocksdb_column_family_handle_destroy(cfh);
-
    // close and destroy
    rocksdb_transaction_destroy(txn);
    rocksdb_transactiondb_close(txn_db);
@@ -1491,18 +1440,6 @@ int main(int argc, char** argv) {
    CheckNoError(err);
  }

-  // Simple sanity check that options setting db_paths work.
-  StartPhase("open_db_paths");
-  {
-    rocksdb_close(db);
-    rocksdb_destroy_db(options, dbname, &err);
-
-    const rocksdb_dbpath_t* paths[1] = {dbpath};
-    rocksdb_options_set_db_paths(options, paths, 1);
-    db = rocksdb_open(options, dbname, &err);
-    CheckNoError(err);
-  }
-  
  StartPhase("cleanup");
  rocksdb_close(db);
  rocksdb_options_destroy(options);
@@ -1512,7 +1449,6 @@ int main(int argc, char** argv) {
  rocksdb_compactoptions_destroy(coptions);
  rocksdb_cache_destroy(cache);
  rocksdb_comparator_destroy(cmp);
-  rocksdb_dbpath_destroy(dbpath);
  rocksdb_env_destroy(env);

  fprintf(stderr, "PASS\n");
@@ -521,7 +521,6 @@ TEST_F(ColumnFamilyTest, DontReuseColumnFamilyID) {
  }
 }

-#ifndef ROCKSDB_LITE
 TEST_F(ColumnFamilyTest, CreateCFRaceWithGetAggProperty) {
  Open();

@@ -543,7 +542,6 @@ TEST_F(ColumnFamilyTest, CreateCFRaceWithGetAggProperty) {

  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
 }
-#endif  // !ROCKSDB_LITE

 class FlushEmptyCFTestWithParam : public ColumnFamilyTest,
                                  public testing::WithParamInterface<bool> {
@@ -169,7 +169,6 @@ Compaction::Compaction(VersionStorageInfo* vstorage,
      bottommost_level_(IsBottommostLevel(output_level_, vstorage, inputs_)),
      is_full_compaction_(IsFullCompaction(vstorage, inputs_)),
      is_manual_compaction_(_manual_compaction),
-      is_trivial_move_(false),
      compaction_reason_(_compaction_reason) {
  MarkFilesBeingCompacted(true);
  if (is_manual_compaction_) {
@@ -16,8 +16,6 @@ struct CompactionIterationStats {
  int64_t num_record_drop_obsolete = 0;
  int64_t num_record_drop_range_del = 0;
  int64_t num_range_del_drop_obsolete = 0;
-  // Deletions obsoleted before bottom level due to file gap optimization.
-  int64_t num_optimized_del_drop_obsolete = 0;
  uint64_t total_filter_time = 0;

  // Input statistics
@@ -111,7 +111,6 @@ void CompactionIterator::ResetRecordCounts() {
  iter_stats_.num_record_drop_obsolete = 0;
  iter_stats_.num_record_drop_range_del = 0;
  iter_stats_.num_range_del_drop_obsolete = 0;
-  iter_stats_.num_optimized_del_drop_obsolete = 0;
 }

 void CompactionIterator::SeekToFirst() {
@@ -427,9 +426,6 @@ void CompactionIterator::NextFromInput() {
          // Can compact out this SingleDelete.
          ++iter_stats_.num_record_drop_obsolete;
          ++iter_stats_.num_single_del_fallthru;
-          if (!bottommost_level_) {
-            ++iter_stats_.num_optimized_del_drop_obsolete;
-          }
        } else {
          // Output SingleDelete
          valid_ = true;
@@ -471,9 +467,6 @@ void CompactionIterator::NextFromInput() {
      // Note:  Dropping this Delete will not affect TransactionDB
      // write-conflict checking since it is earlier than any snapshot.
      ++iter_stats_.num_record_drop_obsolete;
-      if (!bottommost_level_) {
-        ++iter_stats_.num_optimized_del_drop_obsolete;
-      }
      input_->Next();
    } else if (ikey_.type == kTypeMerge) {
      if (!merge_helper_->HasOperator()) {
@@ -455,111 +455,6 @@ TEST_F(CompactionIteratorTest, ShuttingDownInMerge) {
  EXPECT_EQ(2, filter.last_seen.load());
 }

-TEST_F(CompactionIteratorTest, SingleMergeOperand) {
-  class Filter : public CompactionFilter {
-    virtual Decision FilterV2(int level, const Slice& key, ValueType t,
-                              const Slice& existing_value,
-                              std::string* new_value,
-                              std::string* skip_until) const override {
-      std::string k = key.ToString();
-      std::string v = existing_value.ToString();
-
-      // See InitIterators() call below for the sequence of keys and their
-      // filtering decisions. Here we closely assert that compaction filter is
-      // called with the expected keys and only them, and with the right values.
-      if (k == "a") {
-        EXPECT_EQ(ValueType::kMergeOperand, t);
-        EXPECT_EQ("av1", v);
-        return Decision::kKeep;
-      } else if (k == "b") {
-        EXPECT_EQ(ValueType::kMergeOperand, t);
-        return Decision::kKeep;
-      } else if (k == "c") {
-        return Decision::kKeep;
-      }
-
-      ADD_FAILURE();
-      return Decision::kKeep;
-    }
-
-    const char* Name() const override {
-      return "CompactionIteratorTest.SingleMergeOperand::Filter";
-    }
-  };
-
-  class SingleMergeOp : public MergeOperator {
-   public:
-    bool FullMergeV2(const MergeOperationInput& merge_in,
-                     MergeOperationOutput* merge_out) const override {
-      // See InitIterators() call below for why "c" is the only key for which
-      // FullMergeV2 should be called.
-      EXPECT_EQ("c", merge_in.key.ToString());
-
-      std::string temp_value;
-      if (merge_in.existing_value != nullptr) {
-        temp_value = merge_in.existing_value->ToString();
-      }
-
-      for (auto& operand : merge_in.operand_list) {
-        temp_value.append(operand.ToString());
-      }
-      merge_out->new_value = temp_value;
-
-      return true;
-    }
-
-    bool PartialMergeMulti(const Slice& key,
-                           const std::deque<Slice>& operand_list,
-                           std::string* new_value,
-                           Logger* logger) const override {
-      std::string string_key = key.ToString();
-      EXPECT_TRUE(string_key == "a" || string_key == "b");
-
-      if (string_key == "a") {
-        EXPECT_EQ(1, operand_list.size());
-      } else if (string_key == "b") {
-        EXPECT_EQ(2, operand_list.size());
-      }
-
-      std::string temp_value;
-      for (auto& operand : operand_list) {
-        temp_value.append(operand.ToString());
-      }
-      swap(temp_value, *new_value);
-
-      return true;
-    }
-
-    const char* Name() const override {
-      return "CompactionIteratorTest SingleMergeOp";
-    }
-
-    bool AllowSingleOperand() const override { return true; }
-  };
-
-  SingleMergeOp merge_op;
-  Filter filter;
-  InitIterators(
-      // a should invoke PartialMergeMulti with a single merge operand.
-      {test::KeyStr("a", 50, kTypeMerge),
-       // b should invoke PartialMergeMulti with two operands.
-       test::KeyStr("b", 70, kTypeMerge), test::KeyStr("b", 60, kTypeMerge),
-       // c should invoke FullMerge due to kTypeValue at the beginning.
-       test::KeyStr("c", 90, kTypeMerge), test::KeyStr("c", 80, kTypeValue)},
-      {"av1", "bv2", "bv1", "cv2", "cv1"}, {}, {}, kMaxSequenceNumber,
-      &merge_op, &filter);
-
-  c_iter_->SeekToFirst();
-  ASSERT_TRUE(c_iter_->Valid());
-  ASSERT_EQ(test::KeyStr("a", 50, kTypeMerge), c_iter_->key().ToString());
-  ASSERT_EQ("av1", c_iter_->value().ToString());
-  c_iter_->Next();
-  ASSERT_TRUE(c_iter_->Valid());
-  ASSERT_EQ("bv1bv2", c_iter_->value().ToString());
-  c_iter_->Next();
-  ASSERT_EQ("cv1cv2", c_iter_->value().ToString());
-}
-
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -594,9 +594,6 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) {

  double read_write_amp = 0.0;
  double write_amp = 0.0;
-  double bytes_read_per_sec = 0;
-  double bytes_written_per_sec = 0;
-
  if (stats.bytes_read_non_output_levels > 0) {
    read_write_amp = (stats.bytes_written + stats.bytes_read_output_level +
                      stats.bytes_read_non_output_levels) /
@@ -604,22 +601,17 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) {
    write_amp = stats.bytes_written /
                static_cast<double>(stats.bytes_read_non_output_levels);
  }
-  if (stats.micros > 0) {
-    bytes_read_per_sec =
-        (stats.bytes_read_non_output_levels + stats.bytes_read_output_level) /
-        static_cast<double>(stats.micros);
-    bytes_written_per_sec =
-        stats.bytes_written / static_cast<double>(stats.micros);
-  }
-
  ROCKS_LOG_BUFFER(
      log_buffer_,
      "[%s] compacted to: %s, MB/sec: %.1f rd, %.1f wr, level %d, "
      "files in(%d, %d) out(%d) "
      "MB in(%.1f, %.1f) out(%.1f), read-write-amplify(%.1f) "
      "write-amplify(%.1f) %s, records in: %d, records dropped: %d\n",
-      cfd->GetName().c_str(), vstorage->LevelSummary(&tmp), bytes_read_per_sec,
-      bytes_written_per_sec, compact_->compaction->output_level(),
+      cfd->GetName().c_str(), vstorage->LevelSummary(&tmp),
+      (stats.bytes_read_non_output_levels + stats.bytes_read_output_level) /
+          static_cast<double>(stats.micros),
+      stats.bytes_written / static_cast<double>(stats.micros),
+      compact_->compaction->output_level(),
      stats.num_input_files_in_non_output_levels,
      stats.num_input_files_in_output_level, stats.num_output_files,
      stats.bytes_read_non_output_levels / 1048576.0,
@@ -1014,10 +1006,6 @@ void CompactionJob::RecordDroppedKeys(
    RecordTick(stats_, COMPACTION_RANGE_DEL_DROP_OBSOLETE,
               c_iter_stats.num_range_del_drop_obsolete);
  }
-  if (c_iter_stats.num_optimized_del_drop_obsolete > 0) {
-    RecordTick(stats_, COMPACTION_OPTIMIZED_DEL_DROP_OBSOLETE,
-               c_iter_stats.num_optimized_del_drop_obsolete);
-  }
 }

 Status CompactionJob::FinishCompactionOutputFile(
@@ -234,6 +234,11 @@ bool CompactionPicker::ExpandInputsToCleanCut(const std::string& cf_name,
  // If, after the expansion, there are files that are already under
  // compaction, then we must drop/cancel this compaction.
  if (AreFilesInCompaction(inputs->files)) {
+    ROCKS_LOG_WARN(
+        ioptions_.info_log,
+        "[%s] ExpandWhileOverlapping() failure because some of the necessary"
+        " compaction input files are currently being compacted.",
+        cf_name.c_str());
    return false;
  }
  return true;
@@ -1101,7 +1106,11 @@ void LevelCompactionBuilder::SetupInitialFiles() {
      }
      output_level_ =
          (start_level_ == 0) ? vstorage_->base_level() : start_level_ + 1;
-      if (PickFileToCompact()) {
+      if (PickFileToCompact() &&
+          compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_,
+                                                     &start_level_inputs_) &&
+          !compaction_picker_->FilesRangeOverlapWithCompaction(
+              {start_level_inputs_}, output_level_)) {
        // found the compaction!
        if (start_level_ == 0) {
          // L0 score = `num L0 files` / `level0_file_num_compaction_trigger`
@@ -1323,10 +1332,12 @@ bool LevelCompactionBuilder::PickFileToCompact() {
  const std::vector<FileMetaData*>& level_files =
      vstorage_->LevelFiles(start_level_);

-  unsigned int cmp_idx;
-  for (cmp_idx = vstorage_->NextCompactionIndex(start_level_);
-       cmp_idx < file_size.size(); cmp_idx++) {
-    int index = file_size[cmp_idx];
+  // record the first file that is not yet compacted
+  int nextIndex = -1;
+
+  for (unsigned int i = vstorage_->NextCompactionIndex(start_level_);
+       i < file_size.size(); i++) {
+    int index = file_size[i];
    auto* f = level_files[index];

    // do not pick a file to compact if it is being compacted
@@ -1335,42 +1346,27 @@ bool LevelCompactionBuilder::PickFileToCompact() {
      continue;
    }

-    start_level_inputs_.files.push_back(f);
-    start_level_inputs_.level = start_level_;
-    if (!compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_,
-                                                    &start_level_inputs_) ||
-        compaction_picker_->FilesRangeOverlapWithCompaction(
-            {start_level_inputs_}, output_level_)) {
-      // A locked (pending compaction) input-level file was pulled in due to
-      // user-key overlap.
-      start_level_inputs_.clear();
-      continue;
+    // remember the startIndex for the next call to PickCompaction
+    if (nextIndex == -1) {
+      nextIndex = i;
    }

-    // Now that input level is fully expanded, we check whether any output files
-    // are locked due to pending compaction.
-    //
-    // Note we rely on ExpandInputsToCleanCut() to tell us whether any output-
-    // level files are locked, not just the extra ones pulled in for user-key
-    // overlap.
-    InternalKey smallest, largest;
-    compaction_picker_->GetRange(start_level_inputs_, &smallest, &largest);
-    CompactionInputFiles output_level_inputs;
-    output_level_inputs.level = output_level_;
-    vstorage_->GetOverlappingInputs(output_level_, &smallest, &largest,
-                                    &output_level_inputs.files);
-    if (!output_level_inputs.empty() &&
-        !compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_,
-                                                    &output_level_inputs)) {
-      start_level_inputs_.clear();
+    // Do not pick this file if its parents at level+1 are being compacted.
+    // Maybe we can avoid redoing this work in SetupOtherInputs
+    parent_index_ = -1;
+    if (compaction_picker_->IsRangeInCompaction(vstorage_, &f->smallest,
+                                                &f->largest, output_level_,
+                                                &parent_index_)) {
      continue;
    }
+    start_level_inputs_.files.push_back(f);
+    start_level_inputs_.level = start_level_;
    base_index_ = index;
    break;
  }

  // store where to start the iteration in the next call to PickCompaction
-  vstorage_->SetNextCompactionIndex(start_level_, cmp_idx);
+  vstorage_->SetNextCompactionIndex(start_level_, nextIndex);

  return start_level_inputs_.size() > 0;
 }
@@ -512,7 +512,7 @@ TEST_F(CompactionPickerTest, NeedsCompactionFIFO) {
        kFileSize, 0, i * 100, i * 100 + 99);
    current_size += kFileSize;
    UpdateVersionStorageInfo();
-    ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()),
+    ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()),
              vstorage_->CompactionScore(0) >= 1);
  }
 }
@@ -852,80 +852,6 @@ TEST_F(CompactionPickerTest, OverlappingUserKeys9) {
  ASSERT_EQ(8U, compaction->input(1, 1)->fd.GetNumber());
 }

-TEST_F(CompactionPickerTest, OverlappingUserKeys10) {
-  // Locked file encountered when pulling in extra input-level files with same
-  // user keys. Verify we pick the next-best file from the same input level.
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.max_compaction_bytes = 100000000000u;
-
-  // file_number 2U is largest and thus first choice. But it overlaps with
-  // file_number 1U which is being compacted. So instead we pick the next-
-  // biggest file, 3U, which is eligible for compaction.
-  Add(1 /* level */, 1U /* file_number */, "100" /* smallest */,
-      "150" /* largest */, 1U /* file_size */);
-  file_map_[1U].first->being_compacted = true;
-  Add(1 /* level */, 2U /* file_number */, "150" /* smallest */,
-      "200" /* largest */, 1000000000U /* file_size */, 0 /* smallest_seq */,
-      0 /* largest_seq */);
-  Add(1 /* level */, 3U /* file_number */, "201" /* smallest */,
-      "250" /* largest */, 900000000U /* file_size */);
-  Add(2 /* level */, 4U /* file_number */, "100" /* smallest */,
-      "150" /* largest */, 1U /* file_size */);
-  Add(2 /* level */, 5U /* file_number */, "151" /* smallest */,
-      "200" /* largest */, 1U /* file_size */);
-  Add(2 /* level */, 6U /* file_number */, "201" /* smallest */,
-      "250" /* largest */, 1U /* file_size */);
-
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(2U, compaction->num_input_levels());
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(1U, compaction->num_input_files(1));
-  ASSERT_EQ(3U, compaction->input(0, 0)->fd.GetNumber());
-  ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber());
-}
-
-TEST_F(CompactionPickerTest, OverlappingUserKeys11) {
-  // Locked file encountered when pulling in extra output-level files with same
-  // user keys. Expected to skip that compaction and pick the next-best choice.
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.max_compaction_bytes = 100000000000u;
-
-  // score(L1) = 3.7
-  // score(L2) = 1.85
-  // There is no eligible file in L1 to compact since both candidates pull in
-  // file_number 5U, which overlaps with a file pending compaction (6U). The
-  // first eligible compaction is from L2->L3.
-  Add(1 /* level */, 2U /* file_number */, "151" /* smallest */,
-      "200" /* largest */, 1000000000U /* file_size */);
-  Add(1 /* level */, 3U /* file_number */, "201" /* smallest */,
-      "250" /* largest */, 1U /* file_size */);
-  Add(2 /* level */, 4U /* file_number */, "100" /* smallest */,
-      "149" /* largest */, 5000000000U /* file_size */);
-  Add(2 /* level */, 5U /* file_number */, "150" /* smallest */,
-      "201" /* largest */, 1U /* file_size */);
-  Add(2 /* level */, 6U /* file_number */, "201" /* smallest */,
-      "249" /* largest */, 1U /* file_size */, 0 /* smallest_seq */,
-      0 /* largest_seq */);
-  file_map_[6U].first->being_compacted = true;
-  Add(3 /* level */, 7U /* file_number */, "100" /* smallest */,
-      "149" /* largest */, 1U /* file_size */);
-
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(2U, compaction->num_input_levels());
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(1U, compaction->num_input_files(1));
-  ASSERT_EQ(4U, compaction->input(0, 0)->fd.GetNumber());
-  ASSERT_EQ(7U, compaction->input(1, 0)->fd.GetNumber());
-}
-
 TEST_F(CompactionPickerTest, NotScheduleL1IfL0WithHigherPri1) {
  NewVersionStorage(6, kCompactionStyleLevel);
  mutable_cf_options_.level0_file_num_compaction_trigger = 2;
@@ -1390,49 +1316,6 @@ TEST_F(CompactionPickerTest, IsTrivialMoveOff) {
  ASSERT_FALSE(compaction->IsTrivialMove());
 }

-TEST_F(CompactionPickerTest, CacheNextCompactionIndex) {
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.max_compaction_bytes = 100000000000u;
-
-  Add(1 /* level */, 1U /* file_number */, "100" /* smallest */,
-      "149" /* largest */, 1000000000U /* file_size */);
-  file_map_[1U].first->being_compacted = true;
-  Add(1 /* level */, 2U /* file_number */, "150" /* smallest */,
-      "199" /* largest */, 900000000U /* file_size */);
-  Add(1 /* level */, 3U /* file_number */, "200" /* smallest */,
-      "249" /* largest */, 800000000U /* file_size */);
-  Add(1 /* level */, 4U /* file_number */, "250" /* smallest */,
-      "299" /* largest */, 700000000U /* file_size */);
-  Add(2 /* level */, 5U /* file_number */, "150" /* smallest */,
-      "199" /* largest */, 1U /* file_size */);
-  file_map_[5U].first->being_compacted = true;
-
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(1U, compaction->num_input_levels());
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(0U, compaction->num_input_files(1));
-  ASSERT_EQ(3U, compaction->input(0, 0)->fd.GetNumber());
-  ASSERT_EQ(2, vstorage_->NextCompactionIndex(1 /* level */));
-
-  compaction.reset(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(1U, compaction->num_input_levels());
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(0U, compaction->num_input_files(1));
-  ASSERT_EQ(4U, compaction->input(0, 0)->fd.GetNumber());
-  ASSERT_EQ(3, vstorage_->NextCompactionIndex(1 /* level */));
-
-  compaction.reset(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() == nullptr);
-  ASSERT_EQ(4, vstorage_->NextCompactionIndex(1 /* level */));
-}
-
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -373,7 +373,6 @@ Compaction* UniversalCompactionPicker::PickCompaction(
              c->inputs(0)->size());

  RegisterCompaction(c);
-  vstorage->ComputeCompactionScore(ioptions_, mutable_cf_options);

  TEST_SYNC_POINT_CALLBACK("UniversalCompactionPicker::PickCompaction:Return",
                           c);
@@ -9,50 +9,19 @@
 #include "rocksdb/convenience.h"

 #include "db/db_impl.h"
-#include "util/cast_util.h"

 namespace rocksdb {

 void CancelAllBackgroundWork(DB* db, bool wait) {
-  (static_cast_with_check<DBImpl, DB>(db->GetRootDB()))
-      ->CancelAllBackgroundWork(wait);
+  (dynamic_cast<DBImpl*>(db->GetRootDB()))->CancelAllBackgroundWork(wait);
 }

 Status DeleteFilesInRange(DB* db, ColumnFamilyHandle* column_family,
                          const Slice* begin, const Slice* end) {
-  return (static_cast_with_check<DBImpl, DB>(db->GetRootDB()))
+  return (dynamic_cast<DBImpl*>(db->GetRootDB()))
      ->DeleteFilesInRange(column_family, begin, end);
 }

-Status VerifySstFileChecksum(const Options& options,
-                             const EnvOptions& env_options,
-                             const std::string& file_path) {
-  unique_ptr<RandomAccessFile> file;
-  uint64_t file_size;
-  InternalKeyComparator internal_comparator(options.comparator);
-  ImmutableCFOptions ioptions(options);
-
-  Status s = ioptions.env->NewRandomAccessFile(file_path, &file, env_options);
-  if (s.ok()) {
-    s = ioptions.env->GetFileSize(file_path, &file_size);
-  } else {
-    return s;
-  }
-  unique_ptr<TableReader> table_reader;
-  std::unique_ptr<RandomAccessFileReader> file_reader(
-      new RandomAccessFileReader(std::move(file), file_path));
-  s = ioptions.table_factory->NewTableReader(
-      TableReaderOptions(ioptions, env_options, internal_comparator,
-                         false /* skip_filters */, -1 /* level */),
-      std::move(file_reader), file_size, &table_reader,
-      false /* prefetch_index_and_filter_in_cache */);
-  if (!s.ok()) {
-    return s;
-  }
-  s = table_reader->VerifyChecksum();
-  return s;
-}
-
 }  // namespace rocksdb

 #endif  // ROCKSDB_LITE
@@ -20,7 +20,6 @@
 #include "db/log_format.h"
 #include "db/version_set.h"
 #include "rocksdb/cache.h"
-#include "rocksdb/convenience.h"
 #include "rocksdb/env.h"
 #include "rocksdb/table.h"
 #include "rocksdb/write_batch.h"
@@ -153,7 +152,7 @@ class CorruptionTest : public testing::Test {
    struct stat sbuf;
    if (stat(fname.c_str(), &sbuf) != 0) {
      const char* msg = strerror(errno);
-      FAIL() << fname << ": " << msg;
+      ASSERT_TRUE(false) << fname << ": " << msg;
    }

    if (offset < 0) {
@@ -180,9 +179,6 @@ class CorruptionTest : public testing::Test {
    }
    s = WriteStringToFile(Env::Default(), contents, fname);
    ASSERT_TRUE(s.ok()) << s.ToString();
-    Options options;
-    EnvOptions env_options;
-    ASSERT_NOK(VerifySstFileChecksum(options, env_options, fname));
  }

  void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) {
@@ -217,7 +213,7 @@ class CorruptionTest : public testing::Test {
        return;
      }
    }
-    FAIL() << "no file found at level";
+    ASSERT_TRUE(false) << "no file found at level";
  }


@@ -316,7 +312,6 @@ TEST_F(CorruptionTest, TableFile) {

  Corrupt(kTableFile, 100, 1);
  Check(99, 99);
-  ASSERT_NOK(dbi->VerifyChecksum());
 }

 TEST_F(CorruptionTest, TableFileIndexData) {
@@ -332,11 +327,9 @@ TEST_F(CorruptionTest, TableFileIndexData) {
  // corrupt an index block of an entire file
  Corrupt(kTableFile, -2000, 500);
  Reopen();
-  dbi = reinterpret_cast<DBImpl*>(db_);
  // one full file should be readable, since only one was corrupted
  // the other file should be fully non-readable, since index was corrupted
  Check(5000, 5000);
-  ASSERT_NOK(dbi->VerifyChecksum());
 }

 TEST_F(CorruptionTest, MissingDescriptor) {
@@ -396,12 +389,10 @@ TEST_F(CorruptionTest, CompactionInputError) {

  Corrupt(kTableFile, 100, 1);
  Check(9, 9);
-  ASSERT_NOK(dbi->VerifyChecksum());

  // Force compactions by writing lots of values
  Build(10000);
  Check(10000, 10000);
-  ASSERT_NOK(dbi->VerifyChecksum());
 }

 TEST_F(CorruptionTest, CompactionInputErrorParanoid) {
@@ -433,7 +424,6 @@ TEST_F(CorruptionTest, CompactionInputErrorParanoid) {

  CorruptTableFileAtLevel(0, 100, 1);
  Check(9, 9);
-  ASSERT_NOK(dbi->VerifyChecksum());

  // Write must eventually fail because of corrupted table
  Status s;
@@ -455,7 +445,6 @@ TEST_F(CorruptionTest, UnrelatedKeys) {
  DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
  dbi->TEST_FlushMemTable();
  Corrupt(kTableFile, 100, 1);
-  ASSERT_NOK(dbi->VerifyChecksum());

  std::string tmp1, tmp2;
  ASSERT_OK(db_->Put(WriteOptions(), Key(1000, &tmp1), Value(1000, &tmp2)));
@@ -175,8 +175,6 @@ TEST_F(DBBasicTest, LevelLimitReopen) {
  int i = 0;
  while (NumTableFilesAtLevel(2, 1) == 0) {
    ASSERT_OK(Put(1, Key(i++), value));
-    dbfull()->TEST_WaitForFlushMemTable();
-    dbfull()->TEST_WaitForCompact();
  }

  options.num_levels = 1;
@@ -360,6 +358,7 @@ TEST_F(DBBasicTest, FLUSH) {
    WriteOptions writeOpt = WriteOptions();
    writeOpt.disableWAL = true;
    SetPerfLevel(kEnableTime);
+    ;
    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v1"));
    // this will now also flush the last 2 writes
    ASSERT_OK(Flush(1));
@@ -368,7 +367,6 @@ TEST_F(DBBasicTest, FLUSH) {
    get_perf_context()->Reset();
    Get(1, "foo");
    ASSERT_TRUE((int)get_perf_context()->get_from_output_files_time > 0);
-    ASSERT_EQ(2, (int)get_perf_context()->get_read_bytes);

    ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
    ASSERT_EQ("v1", Get(1, "foo"));
@@ -725,7 +723,6 @@ TEST_F(DBBasicTest, FlushOneColumnFamily) {
 TEST_F(DBBasicTest, MultiGetSimple) {
  do {
    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
-    SetPerfLevel(kEnableCount);
    ASSERT_OK(Put(1, "k1", "v1"));
    ASSERT_OK(Put(1, "k2", "v2"));
    ASSERT_OK(Put(1, "k3", "v3"));
@@ -739,15 +736,12 @@ TEST_F(DBBasicTest, MultiGetSimple) {
    std::vector<std::string> values(20, "Temporary data to be overwritten");
    std::vector<ColumnFamilyHandle*> cfs(keys.size(), handles_[1]);

-    get_perf_context()->Reset();
    std::vector<Status> s = db_->MultiGet(ReadOptions(), cfs, keys, &values);
    ASSERT_EQ(values.size(), keys.size());
    ASSERT_EQ(values[0], "v1");
    ASSERT_EQ(values[1], "v2");
    ASSERT_EQ(values[2], "v3");
    ASSERT_EQ(values[4], "v5");
-    // four kv pairs * two bytes per value
-    ASSERT_EQ(8, (int)get_perf_context()->multiget_read_bytes);

    ASSERT_OK(s[0]);
    ASSERT_OK(s[1]);
@@ -755,7 +749,6 @@ TEST_F(DBBasicTest, MultiGetSimple) {
    ASSERT_TRUE(s[3].IsNotFound());
    ASSERT_OK(s[4]);
    ASSERT_TRUE(s[5].IsNotFound());
-    SetPerfLevel(kDisable);
  } while (ChangeCompactOptions());
 }

@@ -792,30 +785,36 @@ TEST_F(DBBasicTest, MultiGetEmpty) {
 TEST_F(DBBasicTest, ChecksumTest) {
  BlockBasedTableOptions table_options;
  Options options = CurrentOptions();
-  // change when new checksum type added
-  int max_checksum = static_cast<int>(kxxHash);
-  const int kNumPerFile = 2;

-  // generate one table with each type of checksum
-  for (int i = 0; i <= max_checksum; ++i) {
-    table_options.checksum = static_cast<ChecksumType>(i);
-    options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-    Reopen(options);
-    for (int j = 0; j < kNumPerFile; ++j) {
-      ASSERT_OK(Put(Key(i * kNumPerFile + j), Key(i * kNumPerFile + j)));
-    }
-    ASSERT_OK(Flush());
-  }
+  table_options.checksum = kCRC32c;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  Reopen(options);
+  ASSERT_OK(Put("a", "b"));
+  ASSERT_OK(Put("c", "d"));
+  ASSERT_OK(Flush());  // table with crc checksum

-  // verify data with each type of checksum
-  for (int i = 0; i <= kxxHash; ++i) {
-    table_options.checksum = static_cast<ChecksumType>(i);
-    options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-    Reopen(options);
-    for (int j = 0; j < (max_checksum + 1) * kNumPerFile; ++j) {
-      ASSERT_EQ(Key(j), Get(Key(j)));
-    }
-  }
+  table_options.checksum = kxxHash;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  Reopen(options);
+  ASSERT_OK(Put("e", "f"));
+  ASSERT_OK(Put("g", "h"));
+  ASSERT_OK(Flush());  // table with xxhash checksum
+
+  table_options.checksum = kCRC32c;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  Reopen(options);
+  ASSERT_EQ("b", Get("a"));
+  ASSERT_EQ("d", Get("c"));
+  ASSERT_EQ("f", Get("e"));
+  ASSERT_EQ("h", Get("g"));
+
+  table_options.checksum = kCRC32c;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  Reopen(options);
+  ASSERT_EQ("b", Get("a"));
+  ASSERT_EQ("d", Get("c"));
+  ASSERT_EQ("f", Get("e"));
+  ASSERT_EQ("h", Get("g"));
 }

 // On Windows you can have either memory mapped file or a file
@@ -497,7 +497,7 @@ TEST_F(DBBlockCacheTest, CompressedCache) {
        options.compression = kNoCompression;
        break;
      default:
-        FAIL();
+        ASSERT_TRUE(false);
    }
    CreateAndReopenWithCF({"pikachu"}, options);
    // default column family doesn't have block cache
@@ -560,7 +560,7 @@ TEST_F(DBBlockCacheTest, CompressedCache) {
        ASSERT_EQ(TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_HIT), 0);
        break;
      default:
-        FAIL();
+        ASSERT_TRUE(false);
    }

    options.create_if_missing = true;
@@ -2684,46 +2684,6 @@ TEST_P(DBCompactionTestWithParam, IntraL0CompactionDoesNotObsoleteDeletions) {
  ASSERT_TRUE(db_->Get(roptions, Key(0), &result).IsNotFound());
 }

-TEST_F(DBCompactionTest, OptimizedDeletionObsoleting) {
-  // Deletions can be dropped when compacted to non-last level if they fall
-  // outside the lower-level files' key-ranges.
-  const int kNumL0Files = 4;
-  Options options = CurrentOptions();
-  options.level0_file_num_compaction_trigger = kNumL0Files;
-  options.statistics = rocksdb::CreateDBStatistics();
-  DestroyAndReopen(options);
-
-  // put key 1 and 3 in separate L1, L2 files.
-  // So key 0, 2, and 4+ fall outside these levels' key-ranges.
-  for (int level = 2; level >= 1; --level) {
-    for (int i = 0; i < 2; ++i) {
-      Put(Key(2 * i + 1), "val");
-      Flush();
-    }
-    MoveFilesToLevel(level);
-    ASSERT_EQ(2, NumTableFilesAtLevel(level));
-  }
-
-  // Delete keys in range [1, 4]. These L0 files will be compacted with L1:
-  // - Tombstones for keys 2 and 4 can be dropped early.
-  // - Tombstones for keys 1 and 3 must be kept due to L2 files' key-ranges.
-  for (int i = 0; i < kNumL0Files; ++i) {
-    Put(Key(0), "val");  // sentinel to prevent trivial move
-    Delete(Key(i + 1));
-    Flush();
-  }
-  dbfull()->TEST_WaitForCompact();
-
-  for (int i = 0; i < kNumL0Files; ++i) {
-    std::string value;
-    ASSERT_TRUE(db_->Get(ReadOptions(), Key(i + 1), &value).IsNotFound());
-  }
-  ASSERT_EQ(2, options.statistics->getTickerCount(
-                   COMPACTION_OPTIMIZED_DEL_DROP_OBSOLETE));
-  ASSERT_EQ(2,
-            options.statistics->getTickerCount(COMPACTION_KEY_DROP_OBSOLETE));
-}
-
 INSTANTIATE_TEST_CASE_P(DBCompactionTestWithParam, DBCompactionTestWithParam,
                        ::testing::Values(std::make_tuple(1, true),
                                          std::make_tuple(1, false),
@@ -1,7 +1,9 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//  This source code is also licensed under the GPLv2 license found in the
+//  COPYING file in the root directory of this source tree.
 //
 #include "db/db_test_util.h"
 #include "port/stack_trace.h"
@@ -67,7 +67,6 @@
 #include "port/port.h"
 #include "rocksdb/cache.h"
 #include "rocksdb/compaction_filter.h"
-#include "rocksdb/convenience.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "rocksdb/merge_operator.h"
@@ -81,7 +80,6 @@
 #include "table/merging_iterator.h"
 #include "table/table_builder.h"
 #include "table/two_level_iterator.h"
-#include "tools/sst_dump_tool_imp.h"
 #include "util/auto_roll_logger.h"
 #include "util/autovector.h"
 #include "util/build_version.h"
@@ -170,7 +168,6 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
      last_batch_group_size_(0),
      unscheduled_flushes_(0),
      unscheduled_compactions_(0),
-      bg_bottom_compaction_scheduled_(0),
      bg_compaction_scheduled_(0),
      num_running_compactions_(0),
      bg_flush_scheduled_(0),
@@ -245,8 +242,7 @@ void DBImpl::CancelAllBackgroundWork(bool wait) {
    return;
  }
  // Wait for background work to finish
-  while (bg_bottom_compaction_scheduled_ || bg_compaction_scheduled_ ||
-         bg_flush_scheduled_) {
+  while (bg_compaction_scheduled_ || bg_flush_scheduled_) {
    bg_cv_.Wait();
  }
 }
@@ -256,18 +252,15 @@ DBImpl::~DBImpl() {
  // marker. After this we do a variant of the waiting and unschedule work
  // (to consider: moving all the waiting into CancelAllBackgroundWork(true))
  CancelAllBackgroundWork(false);
-  int bottom_compactions_unscheduled =
-      env_->UnSchedule(this, Env::Priority::BOTTOM);
  int compactions_unscheduled = env_->UnSchedule(this, Env::Priority::LOW);
  int flushes_unscheduled = env_->UnSchedule(this, Env::Priority::HIGH);
  mutex_.Lock();
-  bg_bottom_compaction_scheduled_ -= bottom_compactions_unscheduled;
  bg_compaction_scheduled_ -= compactions_unscheduled;
  bg_flush_scheduled_ -= flushes_unscheduled;

  // Wait for background work to finish
-  while (bg_bottom_compaction_scheduled_ || bg_compaction_scheduled_ ||
-         bg_flush_scheduled_ || bg_purge_scheduled_) {
+  while (bg_compaction_scheduled_ || bg_flush_scheduled_ ||
+         bg_purge_scheduled_) {
    TEST_SYNC_POINT("DBImpl::~DBImpl:WaitJob");
    bg_cv_.Wait();
  }
@@ -990,7 +983,6 @@ Status DBImpl::GetImpl(const ReadOptions& read_options,
    size_t size = pinnable_val->size();
    RecordTick(stats_, BYTES_READ, size);
    MeasureTime(stats_, BYTES_PER_READ, size);
-    PERF_COUNTER_ADD(get_read_bytes, size);
  }
  return s;
 }
@@ -1118,7 +1110,6 @@ std::vector<Status> DBImpl::MultiGet(
  RecordTick(stats_, NUMBER_MULTIGET_KEYS_READ, num_keys);
  RecordTick(stats_, NUMBER_MULTIGET_BYTES_READ, bytes_read);
  MeasureTime(stats_, BYTES_PER_MULTIGET, bytes_read);
-  PERF_COUNTER_ADD(multiget_read_bytes, bytes_read);
  PERF_TIMER_STOP(get_post_process_time);

  return stat_list;
@@ -1324,11 +1315,6 @@ Status DBImpl::DropColumnFamilyImpl(ColumnFamilyHandle* column_family) {
                                 &edit, &mutex_);
      write_thread_.ExitUnbatched(&w);
    }
-    if (s.ok()) {
-      auto* mutable_cf_options = cfd->GetLatestMutableCFOptions();
-      max_total_in_memory_state_ -= mutable_cf_options->write_buffer_size *
-                                    mutable_cf_options->max_write_buffer_number;
-    }

    if (!cf_support_snapshot) {
      // Dropped Column Family doesn't support snapshot. Need to recalculate
@@ -1350,6 +1336,9 @@ Status DBImpl::DropColumnFamilyImpl(ColumnFamilyHandle* column_family) {
    // later inside db_mutex.
    EraseThreadStatusCfInfo(cfd);
    assert(cfd->IsDropped());
+    auto* mutable_cf_options = cfd->GetLatestMutableCFOptions();
+    max_total_in_memory_state_ -= mutable_cf_options->write_buffer_size *
+                                  mutable_cf_options->max_write_buffer_number;
    ROCKS_LOG_INFO(immutable_db_options_.info_log,
                   "Dropped column family with id %u\n", cfd->GetID());
  } else {
@@ -1413,7 +1402,8 @@ Iterator* DBImpl::NewIterator(const ReadOptions& read_options,
    return NewDBIterator(
        env_, read_options, *cfd->ioptions(), cfd->user_comparator(), iter,
        kMaxSequenceNumber,
-        sv->mutable_cf_options.max_sequential_skip_in_iterations);
+        sv->mutable_cf_options.max_sequential_skip_in_iterations,
+        sv->version_number);
 #endif
  } else {
    SequenceNumber latest_snapshot = versions_->LastSequence();
@@ -1468,10 +1458,9 @@ Iterator* DBImpl::NewIterator(const ReadOptions& read_options,
    // likely that any iterator pointer is close to the iterator it points to so
    // that they are likely to be in the same cache line and/or page.
    ArenaWrappedDBIter* db_iter = NewArenaWrappedDbIterator(
-        env_, read_options, *cfd->ioptions(), snapshot,
+        env_, read_options, *cfd->ioptions(), cfd->user_comparator(), snapshot,
        sv->mutable_cf_options.max_sequential_skip_in_iterations,
-        sv->version_number,
-        ((read_options.snapshot != nullptr) ? nullptr : this), cfd);
+        sv->version_number);

    InternalIterator* internal_iter =
        NewInternalIterator(read_options, cfd, sv, db_iter->GetArena(),
@@ -1522,7 +1511,8 @@ Status DBImpl::NewIterators(
      iterators->push_back(NewDBIterator(
          env_, read_options, *cfd->ioptions(), cfd->user_comparator(), iter,
          kMaxSequenceNumber,
-          sv->mutable_cf_options.max_sequential_skip_in_iterations));
+          sv->mutable_cf_options.max_sequential_skip_in_iterations,
+          sv->version_number));
    }
 #endif
  } else {
@@ -1540,10 +1530,9 @@ Status DBImpl::NewIterators(
              : latest_snapshot;

      ArenaWrappedDBIter* db_iter = NewArenaWrappedDbIterator(
-          env_, read_options, *cfd->ioptions(), snapshot,
-          sv->mutable_cf_options.max_sequential_skip_in_iterations,
-          sv->version_number,
-          ((read_options.snapshot != nullptr) ? nullptr : this), cfd);
+          env_, read_options, *cfd->ioptions(), cfd->user_comparator(),
+          snapshot, sv->mutable_cf_options.max_sequential_skip_in_iterations,
+          sv->version_number);
      InternalIterator* internal_iter =
          NewInternalIterator(read_options, cfd, sv, db_iter->GetArena(),
                              db_iter->GetRangeDelAggregator());
@@ -2304,7 +2293,7 @@ Status DestroyDB(const std::string& dbname, const Options& options) {
    // Delete log files in the WAL dir
    for (const auto& file : walDirFiles) {
      if (ParseFileName(file, &number, &type) && type == kLogFile) {
-        Status del = env->DeleteFile(LogFileName(soptions.wal_dir, number));
+        Status del = env->DeleteFile(soptions.wal_dir + "/" + file);
        if (result.ok() && !del.ok()) {
          result = del;
        }
@@ -2746,54 +2735,6 @@ Status DBImpl::IngestExternalFile(
  return status;
 }

-Status DBImpl::VerifyChecksum() {
-  Status s;
-  Options options;
-  EnvOptions env_options;
-  std::vector<ColumnFamilyData*> cfd_list;
-  {
-    InstrumentedMutexLock l(&mutex_);
-    for (auto cfd : *versions_->GetColumnFamilySet()) {
-      if (!cfd->IsDropped() && cfd->initialized()) {
-        cfd->Ref();
-        cfd_list.push_back(cfd);
-      }
-    }
-  }
-  std::vector<SuperVersion*> sv_list;
-  for (auto cfd : cfd_list) {
-    sv_list.push_back(cfd->GetReferencedSuperVersion(&mutex_));
-  }
-  for (auto& sv : sv_list) {
-    VersionStorageInfo* vstorage = sv->current->storage_info();
-    for (int i = 0; i < vstorage->num_non_empty_levels() && s.ok(); i++) {
-      for (size_t j = 0; j < vstorage->LevelFilesBrief(i).num_files && s.ok();
-           j++) {
-        const auto& fd = vstorage->LevelFilesBrief(i).files[j].fd;
-        std::string fname = TableFileName(immutable_db_options_.db_paths,
-                                          fd.GetNumber(), fd.GetPathId());
-        s = rocksdb::VerifySstFileChecksum(options, env_options, fname);
-      }
-    }
-    if (!s.ok()) {
-      break;
-    }
-  }
-  {
-    InstrumentedMutexLock l(&mutex_);
-    for (auto sv : sv_list) {
-      if (sv && sv->Unref()) {
-        sv->Cleanup();
-        delete sv;
-      }
-    }
-    for (auto cfd : cfd_list) {
-        cfd->Unref();
-    }
-  }
-  return s;
-}
-
 void DBImpl::NotifyOnExternalFileIngested(
    ColumnFamilyData* cfd, const ExternalSstFileIngestionJob& ingestion_job) {
 #ifndef ROCKSDB_LITE
@@ -235,11 +235,11 @@ class DBImpl : public DB {
      ColumnFamilyHandle* column_family,
      ColumnFamilyMetaData* metadata) override;

+  // experimental API
  Status SuggestCompactRange(ColumnFamilyHandle* column_family,
-                             const Slice* begin, const Slice* end) override;
+                             const Slice* begin, const Slice* end);

-  Status PromoteL0(ColumnFamilyHandle* column_family,
-                   int target_level) override;
+  Status PromoteL0(ColumnFamilyHandle* column_family, int target_level);

  // Similar to Write() but will call the callback once on the single write
  // thread to determine whether it is safe to perform the write.
@@ -293,8 +293,6 @@ class DBImpl : public DB {
      const std::vector<std::string>& external_files,
      const IngestExternalFileOptions& ingestion_options) override;

-  virtual Status VerifyChecksum() override;
-
 #endif  // ROCKSDB_LITE

  // Similar to GetSnapshot(), but also lets the db know that this snapshot
@@ -496,12 +494,6 @@ class DBImpl : public DB {

  const WriteController& write_controller() { return write_controller_; }

-  InternalIterator* NewInternalIterator(const ReadOptions&,
-                                        ColumnFamilyData* cfd,
-                                        SuperVersion* super_version,
-                                        Arena* arena,
-                                        RangeDelAggregator* range_del_agg);
-
  // hollow transactions shell used for recovery.
  // these will then be passed to TransactionDB so that
  // locks can be reacquired before writing can resume.
@@ -560,7 +552,6 @@ class DBImpl : public DB {
  void AddToLogsToFreeQueue(log::Writer* log_writer) {
    logs_to_free_queue_.push_back(log_writer);
  }
-  InstrumentedMutex* mutex() { return &mutex_; }

  Status NewDB();

@@ -575,6 +566,12 @@ class DBImpl : public DB {
  std::unordered_map<std::string, RecoveredTransaction*>
      recovered_transactions_;

+  InternalIterator* NewInternalIterator(const ReadOptions&,
+                                        ColumnFamilyData* cfd,
+                                        SuperVersion* super_version,
+                                        Arena* arena,
+                                        RangeDelAggregator* range_del_agg);
+
  // Except in DB::Open(), WriteOptionsFile can only be called when:
  // Persist options to options file.
  // If need_mutex_lock = false, the method will lock DB mutex.
@@ -616,18 +613,16 @@ class DBImpl : public DB {
  Status WriteImpl(const WriteOptions& options, WriteBatch* updates,
                   WriteCallback* callback = nullptr,
                   uint64_t* log_used = nullptr, uint64_t log_ref = 0,
-                   bool disable_memtable = false, uint64_t* seq_used = nullptr);
+                   bool disable_memtable = false);

  Status PipelinedWriteImpl(const WriteOptions& options, WriteBatch* updates,
                            WriteCallback* callback = nullptr,
                            uint64_t* log_used = nullptr, uint64_t log_ref = 0,
-                            bool disable_memtable = false,
-                            uint64_t* seq_used = nullptr);
+                            bool disable_memtable = false);

  Status WriteImplWALOnly(const WriteOptions& options, WriteBatch* updates,
                          WriteCallback* callback = nullptr,
-                          uint64_t* log_used = nullptr, uint64_t log_ref = 0,
-                          uint64_t* seq_used = nullptr);
+                          uint64_t* log_used = nullptr, uint64_t log_ref = 0);

  uint64_t FindMinLogContainingOutstandingPrep();
  uint64_t FindMinPrepLogReferencedByMemTable();
@@ -635,9 +630,7 @@ class DBImpl : public DB {
 private:
  friend class DB;
  friend class InternalStats;
-  friend class PessimisticTransaction;
-  friend class WriteCommittedTxn;
-  friend class WritePreparedTxn;
+  friend class TransactionImpl;
 #ifndef ROCKSDB_LITE
  friend class ForwardIterator;
 #endif
@@ -662,7 +655,6 @@ class DBImpl : public DB {
    }
  };

-  struct PrepickedCompaction;
  struct PurgeFileInfo;

  // Recover the descriptor from persistent storage.  May do a significant
@@ -804,19 +796,14 @@ class DBImpl : public DB {
  void SchedulePendingPurge(std::string fname, FileType type, uint64_t number,
                            uint32_t path_id, int job_id);
  static void BGWorkCompaction(void* arg);
-  // Runs a pre-chosen universal compaction involving bottom level in a
-  // separate, bottom-pri thread pool.
-  static void BGWorkBottomCompaction(void* arg);
  static void BGWorkFlush(void* db);
  static void BGWorkPurge(void* arg);
  static void UnscheduleCallback(void* arg);
-  void BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction,
-                                Env::Priority bg_thread_pri);
+  void BackgroundCallCompaction(void* arg);
  void BackgroundCallFlush();
  void BackgroundCallPurge();
  Status BackgroundCompaction(bool* madeProgress, JobContext* job_context,
-                              LogBuffer* log_buffer,
-                              PrepickedCompaction* prepicked_compaction);
+                              LogBuffer* log_buffer, void* m = 0);
  Status BackgroundFlush(bool* madeProgress, JobContext* job_context,
                         LogBuffer* log_buffer);

@@ -1069,10 +1056,6 @@ class DBImpl : public DB {
  int unscheduled_flushes_;
  int unscheduled_compactions_;

-  // count how many background compactions are running or have been scheduled in
-  // the BOTTOM pool
-  int bg_bottom_compaction_scheduled_;
-
  // count how many background compactions are running or have been scheduled
  int bg_compaction_scheduled_;

@@ -1089,7 +1072,7 @@ class DBImpl : public DB {
  int bg_purge_scheduled_;

  // Information for a manual compaction
-  struct ManualCompactionState {
+  struct ManualCompaction {
    ColumnFamilyData* cfd;
    int input_level;
    int output_level;
@@ -1105,21 +1088,13 @@ class DBImpl : public DB {
    InternalKey* manual_end;      // how far we are compacting
    InternalKey tmp_storage;      // Used to keep track of compaction progress
    InternalKey tmp_storage1;     // Used to keep track of compaction progress
-  };
-  struct PrepickedCompaction {
-    // background compaction takes ownership of `compaction`.
    Compaction* compaction;
-    // caller retains ownership of `manual_compaction_state` as it is reused
-    // across background compactions.
-    ManualCompactionState* manual_compaction_state;  // nullptr if non-manual
  };
-  std::deque<ManualCompactionState*> manual_compaction_dequeue_;
+  std::deque<ManualCompaction*> manual_compaction_dequeue_;

  struct CompactionArg {
-    // caller retains ownership of `db`.
    DBImpl* db;
-    // background compaction takes ownership of `prepicked_compaction`.
-    PrepickedCompaction* prepicked_compaction;
+    ManualCompaction* m;
  };

  // Have we encountered a background error in paranoid mode?
@@ -1253,11 +1228,11 @@ class DBImpl : public DB {

  bool HasPendingManualCompaction();
  bool HasExclusiveManualCompaction();
-  void AddManualCompaction(ManualCompactionState* m);
-  void RemoveManualCompaction(ManualCompactionState* m);
-  bool ShouldntRunManualCompaction(ManualCompactionState* m);
+  void AddManualCompaction(ManualCompaction* m);
+  void RemoveManualCompaction(ManualCompaction* m);
+  bool ShouldntRunManualCompaction(ManualCompaction* m);
  bool HaveManualCompaction(ColumnFamilyData* cfd);
-  bool MCOverlap(ManualCompactionState* m, ManualCompactionState* m1);
+  bool MCOverlap(ManualCompaction* m, ManualCompaction* m1);

  size_t GetWalPreallocateBlockSize(uint64_t write_buffer_size) const;

@@ -612,8 +612,7 @@ Status DBImpl::CompactFilesImpl(
 Status DBImpl::PauseBackgroundWork() {
  InstrumentedMutexLock guard_lock(&mutex_);
  bg_compaction_paused_++;
-  while (bg_bottom_compaction_scheduled_ > 0 || bg_compaction_scheduled_ > 0 ||
-         bg_flush_scheduled_ > 0) {
+  while (bg_compaction_scheduled_ > 0 || bg_flush_scheduled_ > 0) {
    bg_cv_.Wait();
  }
  bg_work_paused_++;
@@ -809,7 +808,7 @@ Status DBImpl::RunManualCompaction(ColumnFamilyData* cfd, int input_level,

  bool scheduled = false;
  bool manual_conflict = false;
-  ManualCompactionState manual;
+  ManualCompaction manual;
  manual.cfd = cfd;
  manual.input_level = input_level;
  manual.output_level = output_level;
@@ -859,8 +858,7 @@ Status DBImpl::RunManualCompaction(ColumnFamilyData* cfd, int input_level,
  AddManualCompaction(&manual);
  TEST_SYNC_POINT_CALLBACK("DBImpl::RunManualCompaction:NotScheduled", &mutex_);
  if (exclusive) {
-    while (bg_bottom_compaction_scheduled_ > 0 ||
-           bg_compaction_scheduled_ > 0) {
+    while (bg_compaction_scheduled_ > 0) {
      TEST_SYNC_POINT("DBImpl::RunManualCompaction:WaitScheduled");
      ROCKS_LOG_INFO(
          immutable_db_options_.info_log,
@@ -880,14 +878,14 @@ Status DBImpl::RunManualCompaction(ColumnFamilyData* cfd, int input_level,
  while (!manual.done) {
    assert(HasPendingManualCompaction());
    manual_conflict = false;
-    Compaction* compaction;
    if (ShouldntRunManualCompaction(&manual) || (manual.in_progress == true) ||
        scheduled ||
-        ((manual.manual_end = &manual.tmp_storage1) &&
-         ((compaction = manual.cfd->CompactRange(
-               *manual.cfd->GetLatestMutableCFOptions(), manual.input_level,
-               manual.output_level, manual.output_path_id, manual.begin,
-               manual.end, &manual.manual_end, &manual_conflict)) == nullptr) &&
+        ((manual.manual_end = &manual.tmp_storage1)&&(
+             (manual.compaction = manual.cfd->CompactRange(
+                  *manual.cfd->GetLatestMutableCFOptions(), manual.input_level,
+                  manual.output_level, manual.output_path_id, manual.begin,
+                  manual.end, &manual.manual_end, &manual_conflict)) ==
+             nullptr) &&
         manual_conflict)) {
      // exclusive manual compactions should not see a conflict during
      // CompactRange
@@ -900,16 +898,14 @@ Status DBImpl::RunManualCompaction(ColumnFamilyData* cfd, int input_level,
        manual.incomplete = false;
      }
    } else if (!scheduled) {
-      if (compaction == nullptr) {
+      if (manual.compaction == nullptr) {
        manual.done = true;
        bg_cv_.SignalAll();
        continue;
      }
      ca = new CompactionArg;
      ca->db = this;
-      ca->prepicked_compaction = new PrepickedCompaction;
-      ca->prepicked_compaction->manual_compaction_state = &manual;
-      ca->prepicked_compaction->compaction = compaction;
+      ca->m = &manual;
      manual.incomplete = false;
      bg_compaction_scheduled_++;
      env_->Schedule(&DBImpl::BGWorkCompaction, ca, Env::Priority::LOW, this,
@@ -1051,7 +1047,7 @@ void DBImpl::MaybeScheduleFlushOrCompaction() {
         unscheduled_compactions_ > 0) {
    CompactionArg* ca = new CompactionArg;
    ca->db = this;
-    ca->prepicked_compaction = nullptr;
+    ca->m = nullptr;
    bg_compaction_scheduled_++;
    unscheduled_compactions_--;
    env_->Schedule(&DBImpl::BGWorkCompaction, ca, Env::Priority::LOW, this,
@@ -1156,23 +1152,7 @@ void DBImpl::BGWorkCompaction(void* arg) {
  delete reinterpret_cast<CompactionArg*>(arg);
  IOSTATS_SET_THREAD_POOL_ID(Env::Priority::LOW);
  TEST_SYNC_POINT("DBImpl::BGWorkCompaction");
-  auto prepicked_compaction =
-      static_cast<PrepickedCompaction*>(ca.prepicked_compaction);
-  reinterpret_cast<DBImpl*>(ca.db)->BackgroundCallCompaction(
-      prepicked_compaction, Env::Priority::LOW);
-  delete prepicked_compaction;
-}
-
-void DBImpl::BGWorkBottomCompaction(void* arg) {
-  CompactionArg ca = *(static_cast<CompactionArg*>(arg));
-  delete static_cast<CompactionArg*>(arg);
-  IOSTATS_SET_THREAD_POOL_ID(Env::Priority::BOTTOM);
-  TEST_SYNC_POINT("DBImpl::BGWorkBottomCompaction");
-  auto* prepicked_compaction = ca.prepicked_compaction;
-  assert(prepicked_compaction && prepicked_compaction->compaction &&
-         !prepicked_compaction->manual_compaction_state);
-  ca.db->BackgroundCallCompaction(prepicked_compaction, Env::Priority::BOTTOM);
-  delete prepicked_compaction;
+  reinterpret_cast<DBImpl*>(ca.db)->BackgroundCallCompaction(ca.m);
 }

 void DBImpl::BGWorkPurge(void* db) {
@@ -1185,11 +1165,8 @@ void DBImpl::BGWorkPurge(void* db) {
 void DBImpl::UnscheduleCallback(void* arg) {
  CompactionArg ca = *(reinterpret_cast<CompactionArg*>(arg));
  delete reinterpret_cast<CompactionArg*>(arg);
-  if (ca.prepicked_compaction != nullptr) {
-    if (ca.prepicked_compaction->compaction != nullptr) {
-      delete ca.prepicked_compaction->compaction;
-    }
-    delete ca.prepicked_compaction;
+  if ((ca.m != nullptr) && (ca.m->compaction != nullptr)) {
+    delete ca.m->compaction;
  }
  TEST_SYNC_POINT("DBImpl::UnscheduleCallback");
 }
@@ -1316,9 +1293,9 @@ void DBImpl::BackgroundCallFlush() {
  }
 }

-void DBImpl::BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction,
-                                      Env::Priority bg_thread_pri) {
+void DBImpl::BackgroundCallCompaction(void* arg) {
  bool made_progress = false;
+  ManualCompaction* m = reinterpret_cast<ManualCompaction*>(arg);
  JobContext job_context(next_job_id_.fetch_add(1), true);
  TEST_SYNC_POINT("BackgroundCallCompaction:0");
  MaybeDumpStats();
@@ -1336,11 +1313,9 @@ void DBImpl::BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction,
    auto pending_outputs_inserted_elem =
        CaptureCurrentFileNumberInPendingOutputs();

-    assert((bg_thread_pri == Env::Priority::BOTTOM &&
-            bg_bottom_compaction_scheduled_) ||
-           (bg_thread_pri == Env::Priority::LOW && bg_compaction_scheduled_));
-    Status s = BackgroundCompaction(&made_progress, &job_context, &log_buffer,
-                                    prepicked_compaction);
+    assert(bg_compaction_scheduled_);
+    Status s =
+        BackgroundCompaction(&made_progress, &job_context, &log_buffer, m);
    TEST_SYNC_POINT("BackgroundCallCompaction:1");
    if (!s.ok() && !s.IsShutdownInProgress()) {
      // Wait a little bit before retrying background compaction in
@@ -1386,24 +1361,17 @@ void DBImpl::BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction,

    assert(num_running_compactions_ > 0);
    num_running_compactions_--;
-    if (bg_thread_pri == Env::Priority::LOW) {
-      bg_compaction_scheduled_--;
-    } else {
-      assert(bg_thread_pri == Env::Priority::BOTTOM);
-      bg_bottom_compaction_scheduled_--;
-    }
+    bg_compaction_scheduled_--;

    versions_->GetColumnFamilySet()->FreeDeadColumnFamilies();

    // See if there's more work to be done
    MaybeScheduleFlushOrCompaction();
-    if (made_progress ||
-        (bg_compaction_scheduled_ == 0 &&
-         bg_bottom_compaction_scheduled_ == 0) ||
+    if (made_progress || bg_compaction_scheduled_ == 0 ||
        HasPendingManualCompaction()) {
      // signal if
      // * made_progress -- need to wakeup DelayWrite
-      // * bg_{bottom,}_compaction_scheduled_ == 0 -- need to wakeup ~DBImpl
+      // * bg_compaction_scheduled_ == 0 -- need to wakeup ~DBImpl
      // * HasPendingManualCompaction -- need to wakeup RunManualCompaction
      // If none of this is true, there is no need to signal since nobody is
      // waiting for it
@@ -1418,23 +1386,14 @@ void DBImpl::BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction,

 Status DBImpl::BackgroundCompaction(bool* made_progress,
                                    JobContext* job_context,
-                                    LogBuffer* log_buffer,
-                                    PrepickedCompaction* prepicked_compaction) {
-  ManualCompactionState* manual_compaction =
-      prepicked_compaction == nullptr
-          ? nullptr
-          : prepicked_compaction->manual_compaction_state;
+                                    LogBuffer* log_buffer, void* arg) {
+  ManualCompaction* manual_compaction =
+      reinterpret_cast<ManualCompaction*>(arg);
  *made_progress = false;
  mutex_.AssertHeld();
  TEST_SYNC_POINT("DBImpl::BackgroundCompaction:Start");

  bool is_manual = (manual_compaction != nullptr);
-  unique_ptr<Compaction> c;
-  if (prepicked_compaction != nullptr &&
-      prepicked_compaction->compaction != nullptr) {
-    c.reset(prepicked_compaction->compaction);
-  }
-  bool is_prepicked = is_manual || c;

  // (manual_compaction->in_progress == false);
  bool trivial_move_disallowed =
@@ -1451,6 +1410,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
      manual_compaction->status = status;
      manual_compaction->done = true;
      manual_compaction->in_progress = false;
+      delete manual_compaction->compaction;
      manual_compaction = nullptr;
    }
    return status;
@@ -1461,11 +1421,13 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
    manual_compaction->in_progress = true;
  }

+  unique_ptr<Compaction> c;
  // InternalKey manual_end_storage;
  // InternalKey* manual_end = &manual_end_storage;
  if (is_manual) {
-    ManualCompactionState* m = manual_compaction;
+    ManualCompaction* m = manual_compaction;
    assert(m->in_progress);
+    c.reset(std::move(m->compaction));
    if (!c) {
      m->done = true;
      m->manual_end = nullptr;
@@ -1487,7 +1449,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
               ? "(end)"
               : m->manual_end->DebugString().c_str()));
    }
-  } else if (!is_prepicked && !compaction_queue_.empty()) {
+  } else if (!compaction_queue_.empty()) {
    if (HaveManualCompaction(compaction_queue_.front())) {
      // Can't compact right now, but try again later
      TEST_SYNC_POINT("DBImpl::BackgroundCompaction()::Conflict");
@@ -1639,28 +1601,6 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,

    // Clear Instrument
    ThreadStatusUtil::ResetThreadStatus();
-  } else if (c->column_family_data()->ioptions()->compaction_style ==
-                 kCompactionStyleUniversal &&
-             !is_prepicked && c->output_level() > 0 &&
-             c->output_level() ==
-                 c->column_family_data()
-                     ->current()
-                     ->storage_info()
-                     ->MaxOutputLevel(
-                         immutable_db_options_.allow_ingest_behind) &&
-             env_->GetBackgroundThreads(Env::Priority::BOTTOM) > 0) {
-    // Forward universal compactions involving last level to the bottom pool
-    // if it exists, such that long-running compactions can't block short-
-    // lived ones, like L0->L0s.
-    TEST_SYNC_POINT("DBImpl::BackgroundCompaction:ForwardToBottomPriPool");
-    CompactionArg* ca = new CompactionArg;
-    ca->db = this;
-    ca->prepicked_compaction = new PrepickedCompaction;
-    ca->prepicked_compaction->compaction = c.release();
-    ca->prepicked_compaction->manual_compaction_state = nullptr;
-    ++bg_bottom_compaction_scheduled_;
-    env_->Schedule(&DBImpl::BGWorkBottomCompaction, ca, Env::Priority::BOTTOM,
-                   this, &DBImpl::UnscheduleCallback);
  } else {
    int output_level  __attribute__((unused)) = c->output_level();
    TEST_SYNC_POINT_CALLBACK("DBImpl::BackgroundCompaction:NonTrivial",
@@ -1724,7 +1664,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
  }

  if (is_manual) {
-    ManualCompactionState* m = manual_compaction;
+    ManualCompaction* m = manual_compaction;
    if (!status.ok()) {
      m->status = status;
      m->done = true;
@@ -1767,13 +1707,13 @@ bool DBImpl::HasPendingManualCompaction() {
  return (!manual_compaction_dequeue_.empty());
 }

-void DBImpl::AddManualCompaction(DBImpl::ManualCompactionState* m) {
+void DBImpl::AddManualCompaction(DBImpl::ManualCompaction* m) {
  manual_compaction_dequeue_.push_back(m);
 }

-void DBImpl::RemoveManualCompaction(DBImpl::ManualCompactionState* m) {
+void DBImpl::RemoveManualCompaction(DBImpl::ManualCompaction* m) {
  // Remove from queue
-  std::deque<ManualCompactionState*>::iterator it =
+  std::deque<ManualCompaction*>::iterator it =
      manual_compaction_dequeue_.begin();
  while (it != manual_compaction_dequeue_.end()) {
    if (m == (*it)) {
@@ -1786,17 +1726,16 @@ void DBImpl::RemoveManualCompaction(DBImpl::ManualCompactionState* m) {
  return;
 }

-bool DBImpl::ShouldntRunManualCompaction(ManualCompactionState* m) {
+bool DBImpl::ShouldntRunManualCompaction(ManualCompaction* m) {
  if (num_running_ingest_file_ > 0) {
    // We need to wait for other IngestExternalFile() calls to finish
    // before running a manual compaction.
    return true;
  }
  if (m->exclusive) {
-    return (bg_bottom_compaction_scheduled_ > 0 ||
-            bg_compaction_scheduled_ > 0);
+    return (bg_compaction_scheduled_ > 0);
  }
-  std::deque<ManualCompactionState*>::iterator it =
+  std::deque<ManualCompaction*>::iterator it =
      manual_compaction_dequeue_.begin();
  bool seen = false;
  while (it != manual_compaction_dequeue_.end()) {
@@ -1817,7 +1756,7 @@ bool DBImpl::ShouldntRunManualCompaction(ManualCompactionState* m) {

 bool DBImpl::HaveManualCompaction(ColumnFamilyData* cfd) {
  // Remove from priority queue
-  std::deque<ManualCompactionState*>::iterator it =
+  std::deque<ManualCompaction*>::iterator it =
      manual_compaction_dequeue_.begin();
  while (it != manual_compaction_dequeue_.end()) {
    if ((*it)->exclusive) {
@@ -1835,7 +1774,7 @@ bool DBImpl::HaveManualCompaction(ColumnFamilyData* cfd) {

 bool DBImpl::HasExclusiveManualCompaction() {
  // Remove from priority queue
-  std::deque<ManualCompactionState*>::iterator it =
+  std::deque<ManualCompaction*>::iterator it =
      manual_compaction_dequeue_.begin();
  while (it != manual_compaction_dequeue_.end()) {
    if ((*it)->exclusive) {
@@ -1846,7 +1785,7 @@ bool DBImpl::HasExclusiveManualCompaction() {
  return false;
 }

-bool DBImpl::MCOverlap(ManualCompactionState* m, ManualCompactionState* m1) {
+bool DBImpl::MCOverlap(ManualCompaction* m, ManualCompaction* m1) {
  if ((m->exclusive) || (m1->exclusive)) {
    return true;
  }
@@ -112,9 +112,7 @@ Status DBImpl::TEST_WaitForCompact() {
  // OR flush to finish.

  InstrumentedMutexLock l(&mutex_);
-  while ((bg_bottom_compaction_scheduled_ || bg_compaction_scheduled_ ||
-          bg_flush_scheduled_) &&
-         bg_error_.ok()) {
+  while ((bg_compaction_scheduled_ || bg_flush_scheduled_) && bg_error_.ok()) {
    bg_cv_.Wait();
  }
  return bg_error_;
@@ -368,9 +368,6 @@ void DBImpl::PurgeObsoleteFiles(const JobContext& state, bool schedule_only) {
    candidate_files.emplace_back(
        MakeTableFileName(kDumbDbName, file->fd.GetNumber()),
        file->fd.GetPathId());
-    if (file->table_reader_handle) {
-      table_cache_->Release(file->table_reader_handle);
-    }
    delete file;
  }

@@ -58,7 +58,7 @@ Iterator* DBImplReadOnly::NewIterator(const ReadOptions& read_options,
  SuperVersion* super_version = cfd->GetSuperVersion()->Ref();
  SequenceNumber latest_snapshot = versions_->LastSequence();
  auto db_iter = NewArenaWrappedDbIterator(
-      env_, read_options, *cfd->ioptions(),
+      env_, read_options, *cfd->ioptions(), cfd->user_comparator(),
      (read_options.snapshot != nullptr
           ? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
                 ->number_
@@ -87,7 +87,7 @@ Status DBImplReadOnly::NewIterators(
    auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(cfh)->cfd();
    auto* sv = cfd->GetSuperVersion()->Ref();
    auto* db_iter = NewArenaWrappedDbIterator(
-        env_, read_options, *cfd->ioptions(),
+        env_, read_options, *cfd->ioptions(), cfd->user_comparator(),
        (read_options.snapshot != nullptr
             ? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
                   ->number_
@@ -60,7 +60,7 @@ Status DBImpl::WriteWithCallback(const WriteOptions& write_options,
 Status DBImpl::WriteImpl(const WriteOptions& write_options,
                         WriteBatch* my_batch, WriteCallback* callback,
                         uint64_t* log_used, uint64_t log_ref,
-                         bool disable_memtable, uint64_t* seq_used) {
+                         bool disable_memtable) {
  if (my_batch == nullptr) {
    return Status::Corruption("Batch is nullptr!");
  }
@@ -79,12 +79,12 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,

  if (concurrent_prepare_ && disable_memtable) {
    return WriteImplWALOnly(write_options, my_batch, callback, log_used,
-                            log_ref, seq_used);
+                            log_ref);
  }

  if (immutable_db_options_.enable_pipelined_write) {
    return PipelinedWriteImpl(write_options, my_batch, callback, log_used,
-                              log_ref, disable_memtable, seq_used);
+                              log_ref, disable_memtable);
  }

  PERF_TIMER_GUARD(write_pre_and_post_process_time);
@@ -127,9 +127,6 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
    if (log_used != nullptr) {
      *log_used = w.log_used;
    }
-    if (seq_used != nullptr) {
-      *seq_used = w.sequence;
-    }
    // write is complete and leader has updated sequence
    return w.FinalStatus();
  }
@@ -281,9 +278,6 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
              write_options.ignore_missing_column_families, 0 /*log_number*/,
              this, true /*concurrent_memtable_writes*/);
        }
-        if (seq_used != nullptr) {
-          *seq_used = w.sequence;
-        }
      }
    }
  }
@@ -319,7 +313,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
      versions_->SetLastSequence(last_sequence);
    }
    MemTableInsertStatusCheck(w.status);
-    write_thread_.ExitAsBatchGroupLeader(write_group, w.status);
+    write_thread_.ExitAsBatchGroupLeader(write_group, status);
  }

  if (status.ok()) {
@@ -331,7 +325,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
 Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
                                  WriteBatch* my_batch, WriteCallback* callback,
                                  uint64_t* log_used, uint64_t log_ref,
-                                  bool disable_memtable, uint64_t* seq_used) {
+                                  bool disable_memtable) {
  PERF_TIMER_GUARD(write_pre_and_post_process_time);
  StopWatch write_sw(env_, immutable_db_options_.statistics.get(), DB_WRITE);

@@ -446,9 +440,6 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
      write_thread_.ExitAsMemTableWriter(&w, *w.write_group);
    }
  }
-  if (seq_used != nullptr) {
-    *seq_used = w.sequence;
-  }

  assert(w.state == WriteThread::STATE_COMPLETED);
  return w.FinalStatus();
@@ -456,8 +447,7 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,

 Status DBImpl::WriteImplWALOnly(const WriteOptions& write_options,
                                WriteBatch* my_batch, WriteCallback* callback,
-                                uint64_t* log_used, uint64_t log_ref,
-                                uint64_t* seq_used) {
+                                uint64_t* log_used, uint64_t log_ref) {
  Status status;
  PERF_TIMER_GUARD(write_pre_and_post_process_time);
  WriteThread::Writer w(write_options, my_batch, callback, log_ref,
@@ -474,9 +464,6 @@ Status DBImpl::WriteImplWALOnly(const WriteOptions& write_options,
    if (log_used != nullptr) {
      *log_used = w.log_used;
    }
-    if (seq_used != nullptr) {
-      *seq_used = w.sequence;
-    }
    return w.FinalStatus();
  }
  // else we are the leader of the write batch group
@@ -522,13 +509,6 @@ Status DBImpl::WriteImplWALOnly(const WriteOptions& write_options,
  // wal_write_mutex_ to ensure ordered events in WAL
  status = ConcurrentWriteToWAL(write_group, log_used, &last_sequence,
                                0 /*total_count*/);
-  auto curr_seq = last_sequence + 1;
-  for (auto* writer : write_group) {
-    if (writer->CheckCallback(this)) {
-      writer->sequence = curr_seq;
-      curr_seq += WriteBatchInternal::Count(writer->batch);
-    }
-  }
  if (status.ok() && write_options.sync) {
    // Requesting sync with concurrent_prepare_ is expected to be very rare. We
    // hance provide a simple implementation that is not necessarily efficient.
@@ -543,13 +523,10 @@ Status DBImpl::WriteImplWALOnly(const WriteOptions& write_options,
  if (!w.CallbackFailed()) {
    WriteCallbackStatusCheck(status);
  }
-  nonmem_write_thread_.ExitAsBatchGroupLeader(write_group, w.status);
+  nonmem_write_thread_.ExitAsBatchGroupLeader(write_group, status);
  if (status.ok()) {
    status = w.FinalStatus();
  }
-  if (seq_used != nullptr) {
-    *seq_used = w.sequence;
-  }
  return status;
 }

@@ -1081,7 +1058,7 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
  SuperVersion* new_superversion = nullptr;
  const MutableCFOptions mutable_cf_options = *cfd->GetLatestMutableCFOptions();

-  // Set memtable_info for memtable sealed callback
+  // Set current_memtble_info for memtable sealed callback
 #ifndef ROCKSDB_LITE
  MemTableInfo memtable_info;
  memtable_info.cf_name = cfd->GetName();
@@ -204,6 +204,7 @@ TEST_F(DBIOFailureTest, ManifestWriteError) {
    ASSERT_EQ("bar2", Get("foo2"));
  }
 }
+#endif  // ROCKSDB_LITE

 TEST_F(DBIOFailureTest, PutFailsParanoid) {
  // Test the following:
@@ -558,7 +559,6 @@ TEST_F(DBIOFailureTest, CompactionSstSyncError) {
  ASSERT_EQ("bar3", Get(1, "foo"));
 }
 #endif  // !(defined NDEBUG) || !defined(OS_WIN)
-#endif  // ROCKSDB_LITE
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -86,7 +86,6 @@ class DBIter: public Iterator {
      RecordTick(global_statistics, NUMBER_DB_PREV, prev_count_);
      RecordTick(global_statistics, NUMBER_DB_PREV_FOUND, prev_found_count_);
      RecordTick(global_statistics, ITER_BYTES_READ, bytes_read_);
-      PERF_COUNTER_ADD(iter_read_bytes, bytes_read_);
      ResetCounters();
    }

@@ -102,12 +101,12 @@ class DBIter: public Iterator {
    uint64_t bytes_read_;
  };

-  DBIter(Env* _env, const ReadOptions& read_options,
+  DBIter(Env* env, const ReadOptions& read_options,
         const ImmutableCFOptions& cf_options, const Comparator* cmp,
         InternalIterator* iter, SequenceNumber s, bool arena_mode,
-         uint64_t max_sequential_skip_in_iterations)
+         uint64_t max_sequential_skip_in_iterations, uint64_t version_number)
      : arena_mode_(arena_mode),
-        env_(_env),
+        env_(env),
        logger_(cf_options.info_log),
        user_comparator_(cmp),
        merge_operator_(cf_options.merge_operator),
@@ -117,6 +116,7 @@ class DBIter: public Iterator {
        valid_(false),
        current_entry_is_merged_(false),
        statistics_(cf_options.statistics),
+        version_number_(version_number),
        iterate_upper_bound_(read_options.iterate_upper_bound),
        prefix_same_as_start_(read_options.prefix_same_as_start),
        pin_thru_lifetime_(read_options.pin_data),
@@ -188,7 +188,10 @@ class DBIter: public Iterator {
    }
    if (prop_name == "rocksdb.iterator.super-version-number") {
      // First try to pass the value returned from inner iterator.
-      return iter_->GetProperty(prop_name, prop);
+      if (!iter_->GetProperty(prop_name, prop).ok()) {
+        *prop = ToString(version_number_);
+      }
+      return Status::OK();
    } else if (prop_name == "rocksdb.iterator.is-key-pinned") {
      if (valid_) {
        *prop = (pin_thru_lifetime_ && saved_key_.IsKeyPinned()) ? "1" : "0";
@@ -206,9 +209,6 @@ class DBIter: public Iterator {
  virtual void SeekForPrev(const Slice& target) override;
  virtual void SeekToFirst() override;
  virtual void SeekToLast() override;
-  Env* env() { return env_; }
-  void set_sequence(uint64_t s) { sequence_ = s; }
-  void set_valid(bool v) { valid_ = v; }

 private:
  void ReverseToForward();
@@ -260,7 +260,7 @@ class DBIter: public Iterator {
  const Comparator* const user_comparator_;
  const MergeOperator* const merge_operator_;
  InternalIterator* iter_;
-  SequenceNumber sequence_;
+  SequenceNumber const sequence_;

  Status status_;
  IterKey saved_key_;
@@ -274,6 +274,7 @@ class DBIter: public Iterator {
  uint64_t max_skip_;
  uint64_t max_skippable_internal_keys_;
  uint64_t num_internal_keys_skipped_;
+  uint64_t version_number_;
  const Slice* iterate_upper_bound_;
  IterKey prefix_start_buf_;
  Slice prefix_start_key_;
@@ -1015,7 +1016,6 @@ void DBIter::Seek(const Slice& target) {
      if (valid_) {
        RecordTick(statistics_, NUMBER_DB_SEEK_FOUND);
        RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
-        PERF_COUNTER_ADD(iter_read_bytes, key().size() + value().size());
      }
    }
  } else {
@@ -1058,7 +1058,6 @@ void DBIter::SeekForPrev(const Slice& target) {
      if (valid_) {
        RecordTick(statistics_, NUMBER_DB_SEEK_FOUND);
        RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
-        PERF_COUNTER_ADD(iter_read_bytes, key().size() + value().size());
      }
    }
  } else {
@@ -1097,7 +1096,6 @@ void DBIter::SeekToFirst() {
      if (valid_) {
        RecordTick(statistics_, NUMBER_DB_SEEK_FOUND);
        RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
-        PERF_COUNTER_ADD(iter_read_bytes, key().size() + value().size());
      }
    }
  } else {
@@ -1145,7 +1143,6 @@ void DBIter::SeekToLast() {
    if (valid_) {
      RecordTick(statistics_, NUMBER_DB_SEEK_FOUND);
      RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
-      PERF_COUNTER_ADD(iter_read_bytes, key().size() + value().size());
    }
  }
  if (valid_ && prefix_extractor_ && prefix_same_as_start_) {
@@ -1160,15 +1157,18 @@ Iterator* NewDBIterator(Env* env, const ReadOptions& read_options,
                        const Comparator* user_key_comparator,
                        InternalIterator* internal_iter,
                        const SequenceNumber& sequence,
-                        uint64_t max_sequential_skip_in_iterations) {
-  DBIter* db_iter = new DBIter(env, read_options, cf_options,
-                               user_key_comparator, internal_iter, sequence,
-                               false, max_sequential_skip_in_iterations);
+                        uint64_t max_sequential_skip_in_iterations,
+                        uint64_t version_number) {
+  DBIter* db_iter = new DBIter(
+      env, read_options, cf_options, user_key_comparator, internal_iter,
+      sequence, false, max_sequential_skip_in_iterations, version_number);
  return db_iter;
 }

 ArenaWrappedDBIter::~ArenaWrappedDBIter() { db_iter_->~DBIter(); }

+void ArenaWrappedDBIter::SetDBIter(DBIter* iter) { db_iter_ = iter; }
+
 RangeDelAggregator* ArenaWrappedDBIter::GetRangeDelAggregator() {
  return db_iter_->GetRangeDelAggregator();
 }
@@ -1193,67 +1193,26 @@ inline Slice ArenaWrappedDBIter::value() const { return db_iter_->value(); }
 inline Status ArenaWrappedDBIter::status() const { return db_iter_->status(); }
 inline Status ArenaWrappedDBIter::GetProperty(std::string prop_name,
                                              std::string* prop) {
-  if (prop_name == "rocksdb.iterator.super-version-number") {
-    // First try to pass the value returned from inner iterator.
-    if (!db_iter_->GetProperty(prop_name, prop).ok()) {
-      *prop = ToString(sv_number_);
-    }
-    return Status::OK();
-  }
  return db_iter_->GetProperty(prop_name, prop);
 }
-
-void ArenaWrappedDBIter::Init(Env* env, const ReadOptions& read_options,
-                              const ImmutableCFOptions& cf_options,
-                              const SequenceNumber& sequence,
-                              uint64_t max_sequential_skip_in_iteration,
-                              uint64_t version_number) {
-  auto mem = arena_.AllocateAligned(sizeof(DBIter));
-  db_iter_ = new (mem)
-      DBIter(env, read_options, cf_options, cf_options.user_comparator, nullptr,
-             sequence, true, max_sequential_skip_in_iteration);
-  sv_number_ = version_number;
-}
-
-Status ArenaWrappedDBIter::Refresh() {
-  if (cfd_ == nullptr || db_impl_ == nullptr) {
-    return Status::NotSupported("Creating renew iterator is not allowed.");
-  }
-  assert(db_iter_ != nullptr);
-  SequenceNumber latest_seq = db_impl_->GetLatestSequenceNumber();
-  uint64_t cur_sv_number = cfd_->GetSuperVersionNumber();
-  if (sv_number_ != cur_sv_number) {
-    Env* env = db_iter_->env();
-    db_iter_->~DBIter();
-    arena_.~Arena();
-    new (&arena_) Arena();
-
-    SuperVersion* sv = cfd_->GetReferencedSuperVersion(db_impl_->mutex());
-    Init(env, read_options_, *(cfd_->ioptions()), latest_seq,
-         sv->mutable_cf_options.max_sequential_skip_in_iterations,
-         cur_sv_number);
-
-    InternalIterator* internal_iter = db_impl_->NewInternalIterator(
-        read_options_, cfd_, sv, &arena_, db_iter_->GetRangeDelAggregator());
-    SetIterUnderDBIter(internal_iter);
-  } else {
-    db_iter_->set_sequence(latest_seq);
-    db_iter_->set_valid(false);
-  }
-  return Status::OK();
+void ArenaWrappedDBIter::RegisterCleanup(CleanupFunction function, void* arg1,
+                                         void* arg2) {
+  db_iter_->RegisterCleanup(function, arg1, arg2);
 }

 ArenaWrappedDBIter* NewArenaWrappedDbIterator(
    Env* env, const ReadOptions& read_options,
-    const ImmutableCFOptions& cf_options, const SequenceNumber& sequence,
-    uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
-    DBImpl* db_impl, ColumnFamilyData* cfd) {
+    const ImmutableCFOptions& cf_options, const Comparator* user_key_comparator,
+    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations,
+    uint64_t version_number) {
  ArenaWrappedDBIter* iter = new ArenaWrappedDBIter();
-  iter->Init(env, read_options, cf_options, sequence,
-             max_sequential_skip_in_iterations, version_number);
-  if (db_impl != nullptr && cfd != nullptr) {
-    iter->StoreRefreshInfo(read_options, db_impl, cfd);
-  }
+  Arena* arena = iter->GetArena();
+  auto mem = arena->AllocateAligned(sizeof(DBIter));
+  DBIter* db_iter = new (mem)
+      DBIter(env, read_options, cf_options, user_key_comparator, nullptr,
+             sequence, true, max_sequential_skip_in_iterations, version_number);
+
+  iter->SetDBIter(db_iter);

  return iter;
 }
@@ -10,7 +10,6 @@
 #pragma once
 #include <stdint.h>
 #include <string>
-#include "db/db_impl.h"
 #include "db/dbformat.h"
 #include "db/range_del_aggregator.h"
 #include "options/cf_options.h"
@@ -33,7 +32,8 @@ extern Iterator* NewDBIterator(Env* env, const ReadOptions& read_options,
                               const Comparator* user_key_comparator,
                               InternalIterator* internal_iter,
                               const SequenceNumber& sequence,
-                               uint64_t max_sequential_skip_in_iterations);
+                               uint64_t max_sequential_skip_in_iterations,
+                               uint64_t version_number);

 // A wrapper iterator which wraps DB Iterator and the arena, with which the DB
 // iterator is supposed be allocated. This class is used as an entry point of
@@ -49,6 +49,10 @@ class ArenaWrappedDBIter : public Iterator {
  virtual Arena* GetArena() { return &arena_; }
  virtual RangeDelAggregator* GetRangeDelAggregator();

+  // Set the DB Iterator to be wrapped
+
+  virtual void SetDBIter(DBIter* iter);
+
  // Set the internal iterator wrapped inside the DB Iterator. Usually it is
  // a merging iterator.
  virtual void SetIterUnderDBIter(InternalIterator* iter);
@@ -62,39 +66,20 @@ class ArenaWrappedDBIter : public Iterator {
  virtual Slice key() const override;
  virtual Slice value() const override;
  virtual Status status() const override;
-  virtual Status Refresh() override;

+  void RegisterCleanup(CleanupFunction function, void* arg1, void* arg2);
  virtual Status GetProperty(std::string prop_name, std::string* prop) override;

-  void Init(Env* env, const ReadOptions& read_options,
-            const ImmutableCFOptions& cf_options,
-            const SequenceNumber& sequence,
-            uint64_t max_sequential_skip_in_iterations,
-            uint64_t version_number);
-
-  void StoreRefreshInfo(const ReadOptions& read_options, DBImpl* db_impl,
-                        ColumnFamilyData* cfd) {
-    read_options_ = read_options;
-    db_impl_ = db_impl;
-    cfd_ = cfd;
-  }
-
 private:
  DBIter* db_iter_;
  Arena arena_;
-  uint64_t sv_number_;
-  ColumnFamilyData* cfd_ = nullptr;
-  DBImpl* db_impl_ = nullptr;
-  ReadOptions read_options_;
 };

 // Generate the arena wrapped iterator class.
-// `db_impl` and `cfd` are used for reneweal. If left null, renewal will not
-// be supported.
 extern ArenaWrappedDBIter* NewArenaWrappedDbIterator(
    Env* env, const ReadOptions& read_options,
-    const ImmutableCFOptions& cf_options, const SequenceNumber& sequence,
-    uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
-    DBImpl* db_impl = nullptr, ColumnFamilyData* cfd = nullptr);
+    const ImmutableCFOptions& cf_options, const Comparator* user_key_comparator,
+    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations,
+    uint64_t version_number);

 }  // namespace rocksdb
@@ -193,7 +193,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
    ReadOptions ro;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -225,7 +225,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
    ReadOptions ro;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -251,7 +251,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -283,7 +283,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -318,7 +318,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(!db_iter->Valid());
@@ -347,7 +347,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      7, options.max_sequential_skip_in_iterations));
+                      7, options.max_sequential_skip_in_iterations, 0));

    SetPerfLevel(kEnableCount);
    ASSERT_TRUE(GetPerfLevel() == kEnableCount);
@@ -384,7 +384,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      4, options.max_sequential_skip_in_iterations));
+                      4, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -409,7 +409,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(!db_iter->Valid());
@@ -431,7 +431,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -466,7 +466,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      7, options.max_sequential_skip_in_iterations));
+                      7, options.max_sequential_skip_in_iterations, 0));

    SetPerfLevel(kEnableCount);
    ASSERT_TRUE(GetPerfLevel() == kEnableCount);
@@ -495,7 +495,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
    ReadOptions ro;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToFirst();
    ASSERT_TRUE(db_iter->Valid());
@@ -537,7 +537,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
    ReadOptions ro;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      2, options.max_sequential_skip_in_iterations));
+                      2, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "b");
@@ -568,7 +568,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
    ReadOptions ro;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "c");
@@ -597,7 +597,7 @@ TEST_F(DBIteratorTest, DBIteratorEmpty) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      0, options.max_sequential_skip_in_iterations));
+                      0, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(!db_iter->Valid());
  }
@@ -608,7 +608,7 @@ TEST_F(DBIteratorTest, DBIteratorEmpty) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      0, options.max_sequential_skip_in_iterations));
+                      0, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToFirst();
    ASSERT_TRUE(!db_iter->Valid());
  }
@@ -630,7 +630,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkipCountSkips) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 2, options.max_sequential_skip_in_iterations));
+      internal_iter, 2, options.max_sequential_skip_in_iterations, 0));
  db_iter->SeekToLast();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "c");
@@ -673,7 +673,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
      options.statistics = rocksdb::CreateDBStatistics();
      std::unique_ptr<Iterator> db_iter(NewDBIterator(
          env_, ro, cf_options, BytewiseComparator(), internal_iter, i + 2,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));
      db_iter->SeekToLast();
      ASSERT_TRUE(db_iter->Valid());

@@ -708,7 +708,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {

      std::unique_ptr<Iterator> db_iter(NewDBIterator(
          env_, ro, cf_options, BytewiseComparator(), internal_iter, i + 2,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));
      db_iter->SeekToLast();
      ASSERT_TRUE(db_iter->Valid());

@@ -736,7 +736,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {

      std::unique_ptr<Iterator> db_iter(NewDBIterator(
          env_, ro, cf_options, BytewiseComparator(), internal_iter, 202,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));
      db_iter->SeekToLast();
      ASSERT_TRUE(db_iter->Valid());

@@ -768,7 +768,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
      internal_iter->Finish();
      std::unique_ptr<Iterator> db_iter(NewDBIterator(
          env_, ro, cf_options, BytewiseComparator(), internal_iter, i,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));
      db_iter->SeekToLast();
      ASSERT_TRUE(!db_iter->Valid());

@@ -784,7 +784,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
    internal_iter->Finish();
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      200, options.max_sequential_skip_in_iterations));
+                      200, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "c");
@@ -818,7 +818,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {

      std::unique_ptr<Iterator> db_iter(NewDBIterator(
          env_, ro, cf_options, BytewiseComparator(), internal_iter, i + 2,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));
      db_iter->SeekToLast();
      ASSERT_TRUE(db_iter->Valid());

@@ -852,7 +852,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {

      std::unique_ptr<Iterator> db_iter(NewDBIterator(
          env_, ro, cf_options, BytewiseComparator(), internal_iter, i + 2,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));
      db_iter->SeekToLast();
      ASSERT_TRUE(db_iter->Valid());

@@ -900,7 +900,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
    ro.max_skippable_internal_keys = 0;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToFirst();
    ASSERT_TRUE(db_iter->Valid());
@@ -946,7 +946,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
    ro.max_skippable_internal_keys = 2;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToFirst();
    ASSERT_TRUE(db_iter->Valid());
@@ -990,7 +990,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
    ro.max_skippable_internal_keys = 2;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToFirst();
    ASSERT_TRUE(db_iter->Valid());
@@ -1028,7 +1028,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
    ro.max_skippable_internal_keys = 2;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToFirst();
    ASSERT_TRUE(db_iter->Valid());
@@ -1063,7 +1063,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
    ro.max_skippable_internal_keys = 2;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -1093,7 +1093,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
    ro.max_skippable_internal_keys = 2;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToFirst();
    ASSERT_TRUE(db_iter->Valid());
@@ -1130,7 +1130,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
    ro.max_skippable_internal_keys = 2;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToFirst();
    ASSERT_TRUE(db_iter->Valid());
@@ -1167,7 +1167,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
      ro.max_skippable_internal_keys = i;
      std::unique_ptr<Iterator> db_iter(NewDBIterator(
          env_, ro, cf_options, BytewiseComparator(), internal_iter, 2 * i + 1,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));

      db_iter->SeekToFirst();
      ASSERT_TRUE(db_iter->Valid());
@@ -1220,7 +1220,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
      ro.max_skippable_internal_keys = i;
      std::unique_ptr<Iterator> db_iter(NewDBIterator(
          env_, ro, cf_options, BytewiseComparator(), internal_iter, 2 * i + 1,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));

      db_iter->SeekToFirst();
      ASSERT_TRUE(db_iter->Valid());
@@ -1258,7 +1258,7 @@ TEST_F(DBIteratorTest, DBIterator1) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 1, options.max_sequential_skip_in_iterations));
+      internal_iter, 1, options.max_sequential_skip_in_iterations, 0));
  db_iter->SeekToFirst();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1285,7 +1285,7 @@ TEST_F(DBIteratorTest, DBIterator2) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 0, options.max_sequential_skip_in_iterations));
+      internal_iter, 0, options.max_sequential_skip_in_iterations, 0));
  db_iter->SeekToFirst();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1309,7 +1309,7 @@ TEST_F(DBIteratorTest, DBIterator3) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 2, options.max_sequential_skip_in_iterations));
+      internal_iter, 2, options.max_sequential_skip_in_iterations, 0));
  db_iter->SeekToFirst();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1333,7 +1333,7 @@ TEST_F(DBIteratorTest, DBIterator4) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 4, options.max_sequential_skip_in_iterations));
+      internal_iter, 4, options.max_sequential_skip_in_iterations, 0));
  db_iter->SeekToFirst();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1365,7 +1365,7 @@ TEST_F(DBIteratorTest, DBIterator5) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      0, options.max_sequential_skip_in_iterations));
+                      0, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1387,7 +1387,7 @@ TEST_F(DBIteratorTest, DBIterator5) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      1, options.max_sequential_skip_in_iterations));
+                      1, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1409,7 +1409,7 @@ TEST_F(DBIteratorTest, DBIterator5) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      2, options.max_sequential_skip_in_iterations));
+                      2, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1431,7 +1431,7 @@ TEST_F(DBIteratorTest, DBIterator5) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      3, options.max_sequential_skip_in_iterations));
+                      3, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1453,7 +1453,7 @@ TEST_F(DBIteratorTest, DBIterator5) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      4, options.max_sequential_skip_in_iterations));
+                      4, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1475,7 +1475,7 @@ TEST_F(DBIteratorTest, DBIterator5) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      5, options.max_sequential_skip_in_iterations));
+                      5, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1497,7 +1497,7 @@ TEST_F(DBIteratorTest, DBIterator5) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      6, options.max_sequential_skip_in_iterations));
+                      6, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1517,7 +1517,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
    internal_iter->Finish();
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));
    db_iter->Seek("b");
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "b");
@@ -1546,7 +1546,7 @@ TEST_F(DBIteratorTest, DBIterator6) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      0, options.max_sequential_skip_in_iterations));
+                      0, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1568,7 +1568,7 @@ TEST_F(DBIteratorTest, DBIterator6) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      1, options.max_sequential_skip_in_iterations));
+                      1, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1590,7 +1590,7 @@ TEST_F(DBIteratorTest, DBIterator6) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      2, options.max_sequential_skip_in_iterations));
+                      2, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1612,7 +1612,7 @@ TEST_F(DBIteratorTest, DBIterator6) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      3, options.max_sequential_skip_in_iterations));
+                      3, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(!db_iter->Valid());
  }
@@ -1630,7 +1630,7 @@ TEST_F(DBIteratorTest, DBIterator6) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      4, options.max_sequential_skip_in_iterations));
+                      4, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1652,7 +1652,7 @@ TEST_F(DBIteratorTest, DBIterator6) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      5, options.max_sequential_skip_in_iterations));
+                      5, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1674,7 +1674,7 @@ TEST_F(DBIteratorTest, DBIterator6) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      6, options.max_sequential_skip_in_iterations));
+                      6, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1715,7 +1715,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      0, options.max_sequential_skip_in_iterations));
+                      0, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1749,7 +1749,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      2, options.max_sequential_skip_in_iterations));
+                      2, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -1789,7 +1789,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      4, options.max_sequential_skip_in_iterations));
+                      4, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -1829,7 +1829,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      5, options.max_sequential_skip_in_iterations));
+                      5, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -1874,7 +1874,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      6, options.max_sequential_skip_in_iterations));
+                      6, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -1920,7 +1920,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      7, options.max_sequential_skip_in_iterations));
+                      7, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -1960,7 +1960,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      9, options.max_sequential_skip_in_iterations));
+                      9, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -2006,7 +2006,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      13, options.max_sequential_skip_in_iterations));
+                      13, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -2053,7 +2053,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      14, options.max_sequential_skip_in_iterations));
+                      14, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -2083,7 +2083,7 @@ TEST_F(DBIteratorTest, DBIterator8) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 10, options.max_sequential_skip_in_iterations));
+      internal_iter, 10, options.max_sequential_skip_in_iterations, 0));
  db_iter->SeekToLast();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "b");
@@ -2113,7 +2113,7 @@ TEST_F(DBIteratorTest, DBIterator9) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-        internal_iter, 10, options.max_sequential_skip_in_iterations));
+        internal_iter, 10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -2179,7 +2179,7 @@ TEST_F(DBIteratorTest, DBIterator10) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 10, options.max_sequential_skip_in_iterations));
+      internal_iter, 10, options.max_sequential_skip_in_iterations, 0));

  db_iter->Seek("c");
  ASSERT_TRUE(db_iter->Valid());
@@ -2218,7 +2218,7 @@ TEST_F(DBIteratorTest, SeekToLastOccurrenceSeq0) {

  std::unique_ptr<Iterator> db_iter(
      NewDBIterator(env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-                    internal_iter, 10, 0 /* force seek */));
+                    internal_iter, 10, 0 /* force seek */, 0));
  db_iter->SeekToFirst();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -2246,7 +2246,7 @@ TEST_F(DBIteratorTest, DBIterator11) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 1, options.max_sequential_skip_in_iterations));
+      internal_iter, 1, options.max_sequential_skip_in_iterations, 0));
  db_iter->SeekToFirst();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -2272,7 +2272,7 @@ TEST_F(DBIteratorTest, DBIterator12) {

  std::unique_ptr<Iterator> db_iter(
      NewDBIterator(env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-                    internal_iter, 10, 0));
+                    internal_iter, 10, 0, 0));
  db_iter->SeekToLast();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "c");
@@ -2309,7 +2309,7 @@ TEST_F(DBIteratorTest, DBIterator13) {

  std::unique_ptr<Iterator> db_iter(
      NewDBIterator(env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-                    internal_iter, 2, 3));
+                    internal_iter, 2, 3, 0));
  db_iter->Seek("b");
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), key);
@@ -2337,7 +2337,7 @@ TEST_F(DBIteratorTest, DBIterator14) {

  std::unique_ptr<Iterator> db_iter(
      NewDBIterator(env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-                    internal_iter, 4, 1));
+                    internal_iter, 4, 1, 0));
  db_iter->Seek("b");
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "b");
@@ -2376,7 +2376,7 @@ class DBIterWithMergeIterTest : public testing::Test {
    db_iter_.reset(NewDBIterator(env_, ro_, ImmutableCFOptions(options_),
                                 BytewiseComparator(), merge_iter,
                                 8 /* read data earlier than seqId 8 */,
-                                 3 /* max iterators before reseek */));
+                                 3 /* max iterators before reseek */, 0));
  }

  Env* env_;
@@ -1719,15 +1719,12 @@ TEST_F(DBIteratorTest, IteratorWithLocalStatistics) {

  std::vector<port::Thread> threads;
  std::function<void()> reader_func_next = [&]() {
-    SetPerfLevel(kEnableCount);
-    get_perf_context()->Reset();
    Iterator* iter = db_->NewIterator(ReadOptions());

    iter->SeekToFirst();
    // Seek will bump ITER_BYTES_READ
-    uint64_t bytes = 0;
-    bytes += iter->key().size();
-    bytes += iter->value().size();
+    total_bytes += iter->key().size();
+    total_bytes += iter->value().size();
    while (true) {
      iter->Next();
      total_next++;
@@ -1736,25 +1733,20 @@ TEST_F(DBIteratorTest, IteratorWithLocalStatistics) {
        break;
      }
      total_next_found++;
-      bytes += iter->key().size();
-      bytes += iter->value().size();
+      total_bytes += iter->key().size();
+      total_bytes += iter->value().size();
    }

    delete iter;
-    ASSERT_EQ(bytes, get_perf_context()->iter_read_bytes);
-    SetPerfLevel(kDisable);
-    total_bytes += bytes;
  };

  std::function<void()> reader_func_prev = [&]() {
-    SetPerfLevel(kEnableCount);
    Iterator* iter = db_->NewIterator(ReadOptions());

    iter->SeekToLast();
    // Seek will bump ITER_BYTES_READ
-    uint64_t bytes = 0;
-    bytes += iter->key().size();
-    bytes += iter->value().size();
+    total_bytes += iter->key().size();
+    total_bytes += iter->value().size();
    while (true) {
      iter->Prev();
      total_prev++;
@@ -1763,14 +1755,11 @@ TEST_F(DBIteratorTest, IteratorWithLocalStatistics) {
        break;
      }
      total_prev_found++;
-      bytes += iter->key().size();
-      bytes += iter->value().size();
+      total_bytes += iter->key().size();
+      total_bytes += iter->value().size();
    }

    delete iter;
-    ASSERT_EQ(bytes, get_perf_context()->iter_read_bytes);
-    SetPerfLevel(kDisable);
-    total_bytes += bytes;
  };

  for (int i = 0; i < 10; i++) {
@@ -1920,65 +1909,6 @@ TEST_F(DBIteratorTest, DBIteratorSkipRecentDuplicatesTest) {
                 NUMBER_OF_RESEEKS_IN_ITERATION));
 }

-TEST_F(DBIteratorTest, Refresh) {
-  ASSERT_OK(Put("x", "y"));
-
-  std::unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions()));
-  iter->Seek(Slice("a"));
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("x")), 0);
-  iter->Next();
-  ASSERT_FALSE(iter->Valid());
-
-  ASSERT_OK(Put("c", "d"));
-
-  iter->Seek(Slice("a"));
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("x")), 0);
-  iter->Next();
-  ASSERT_FALSE(iter->Valid());
-
-  iter->Refresh();
-
-  iter->Seek(Slice("a"));
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("c")), 0);
-  iter->Next();
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("x")), 0);
-  iter->Next();
-  ASSERT_FALSE(iter->Valid());
-
-  dbfull()->Flush(FlushOptions());
-
-  ASSERT_OK(Put("m", "n"));
-
-  iter->Seek(Slice("a"));
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("c")), 0);
-  iter->Next();
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("x")), 0);
-  iter->Next();
-  ASSERT_FALSE(iter->Valid());
-
-  iter->Refresh();
-
-  iter->Seek(Slice("a"));
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("c")), 0);
-  iter->Next();
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("m")), 0);
-  iter->Next();
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("x")), 0);
-  iter->Next();
-  ASSERT_FALSE(iter->Valid());
-
-  iter.reset();
-}
-
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -894,74 +894,6 @@ TEST_F(DBRangeDelTest, MemtableBloomFilter) {
  }
 }

-TEST_F(DBRangeDelTest, CompactionTreatsSplitInputLevelDeletionAtomically) {
-  // make sure compaction treats files containing a split range deletion in the
-  // input level as an atomic unit. I.e., compacting any input-level file(s)
-  // containing a portion of the range deletion causes all other input-level
-  // files containing portions of that same range deletion to be included in the
-  // compaction.
-  const int kNumFilesPerLevel = 4, kValueBytes = 4 << 10;
-  Options options = CurrentOptions();
-  options.compression = kNoCompression;
-  options.level0_file_num_compaction_trigger = kNumFilesPerLevel;
-  options.memtable_factory.reset(
-      new SpecialSkipListFactory(2 /* num_entries_flush */));
-  options.target_file_size_base = kValueBytes;
-  // i == 0: CompactFiles
-  // i == 1: CompactRange
-  // i == 2: automatic compaction
-  for (int i = 0; i < 3; ++i) {
-    DestroyAndReopen(options);
-
-    ASSERT_OK(Put(Key(0), ""));
-    ASSERT_OK(db_->Flush(FlushOptions()));
-    MoveFilesToLevel(2);
-    ASSERT_EQ(1, NumTableFilesAtLevel(2));
-
-    // snapshot protects range tombstone from dropping due to becoming obsolete.
-    const Snapshot* snapshot = db_->GetSnapshot();
-    db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(0),
-                     Key(2 * kNumFilesPerLevel));
-
-    Random rnd(301);
-    std::string value = RandomString(&rnd, kValueBytes);
-    for (int j = 0; j < kNumFilesPerLevel; ++j) {
-      // give files overlapping key-ranges to prevent trivial move
-      ASSERT_OK(Put(Key(j), value));
-      ASSERT_OK(Put(Key(2 * kNumFilesPerLevel - 1 - j), value));
-      if (j > 0) {
-        dbfull()->TEST_WaitForFlushMemTable();
-        ASSERT_EQ(j, NumTableFilesAtLevel(0));
-      }
-    }
-    // put extra key to trigger final flush
-    ASSERT_OK(Put("", ""));
-    dbfull()->TEST_WaitForFlushMemTable();
-    dbfull()->TEST_WaitForCompact();
-    ASSERT_EQ(0, NumTableFilesAtLevel(0));
-    ASSERT_EQ(kNumFilesPerLevel, NumTableFilesAtLevel(1));
-
-    ColumnFamilyMetaData meta;
-    db_->GetColumnFamilyMetaData(&meta);
-    if (i == 0) {
-      ASSERT_OK(db_->CompactFiles(
-          CompactionOptions(), {meta.levels[1].files[0].name}, 2 /* level */));
-    } else if (i == 1) {
-      auto begin_str = Key(0), end_str = Key(1);
-      Slice begin = begin_str, end = end_str;
-      ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &begin, &end));
-    } else if (i == 2) {
-      ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(),
-                                {{"max_bytes_for_level_base", "10000"}}));
-      dbfull()->TEST_WaitForCompact();
-    }
-    ASSERT_EQ(0, NumTableFilesAtLevel(1));
-    ASSERT_GT(NumTableFilesAtLevel(2), 0);
-
-    db_->ReleaseSnapshot(snapshot);
-  }
-}
-
 TEST_F(DBRangeDelTest, UnorderedTombstones) {
  // Regression test for #2752. Range delete tombstones between
  // different snapshot stripes are not stored in order, so the first
@@ -23,7 +23,6 @@
 #include <alloca.h>
 #endif

-#include "cache/lru_cache.h"
 #include "db/db_impl.h"
 #include "db/db_test_util.h"
 #include "db/dbformat.h"
@@ -2235,10 +2234,6 @@ class ModelDB : public DB {
    return Status::NotSupported("Not implemented.");
  }

-  virtual Status VerifyChecksum() override {
-    return Status::NotSupported("Not implemented.");
-  }
-
  using DB::GetPropertiesOfAllTables;
  virtual Status GetPropertiesOfAllTables(
      ColumnFamilyHandle* column_family,
@@ -2891,23 +2886,20 @@ TEST_F(DBTest, FIFOCompactionWithTTLAndVariousTableFormatsTest) {
  ASSERT_TRUE(TryReopen(options).IsNotSupported());
 }

-TEST_F(DBTest, FIFOCompactionWithTTLTest) {
+TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
  Options options;
  options.compaction_style = kCompactionStyleFIFO;
  options.write_buffer_size = 10 << 10;  // 10KB
  options.arena_block_size = 4096;
  options.compression = kNoCompression;
  options.create_if_missing = true;
-  env_->time_elapse_only_sleep_ = false;
-  options.env = env_;

  // Test to make sure that all files with expired ttl are deleted on next
  // manual compaction.
  {
-    env_->addon_time_.store(0);
    options.compaction_options_fifo.max_table_files_size = 150 << 10;  // 150KB
    options.compaction_options_fifo.allow_compaction = false;
-    options.compaction_options_fifo.ttl = 1 * 60 * 60 ;  // 1 hour
+    options.compaction_options_fifo.ttl = 600;  // seconds
    options = CurrentOptions(options);
    DestroyAndReopen(options);

@@ -2918,21 +2910,19 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
      }
      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
    }
-    ASSERT_EQ(NumTableFilesAtLevel(0), 10);
-
-    // Sleep for 2 hours -- which is much greater than TTL.
-    // Note: Couldn't use SleepForMicroseconds because it takes an int instead
-    // of uint64_t. Hence used addon_time_ directly.
-    // env_->SleepForMicroseconds(2 * 60 * 60 * 1000 * 1000);
-    env_->addon_time_.fetch_add(2 * 60 * 60);
-
-    // Since no flushes and compactions have run, the db should still be in
-    // the same state even after considerable time has passed.
    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    ASSERT_EQ(NumTableFilesAtLevel(0), 10);

+    // sleep for 5 seconds
+    env_->SleepForMicroseconds(5 * 1000 * 1000);
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
+    ASSERT_EQ(NumTableFilesAtLevel(0), 10);
+
+    // change ttl to 1 sec. So all files should be deleted on next compaction.
+    options.compaction_options_fifo.ttl = 1;
+    Reopen(options);
+
    dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
    ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  }
@@ -2942,7 +2932,7 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
  {
    options.compaction_options_fifo.max_table_files_size = 150 << 10;  // 150KB
    options.compaction_options_fifo.allow_compaction = false;
-    options.compaction_options_fifo.ttl = 1 * 60 * 60;  // 1 hour
+    options.compaction_options_fifo.ttl = 5;  // seconds
    options = CurrentOptions(options);
    DestroyAndReopen(options);

@@ -2953,13 +2943,11 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
      }
      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
    }
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    ASSERT_EQ(NumTableFilesAtLevel(0), 10);

-    // Sleep for 2 hours -- which is much greater than TTL.
-    env_->addon_time_.fetch_add(2 * 60 * 60);
-    // Just to make sure that we are in the same state even after sleeping.
+    env_->SleepForMicroseconds(6 * 1000 * 1000);
    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    ASSERT_EQ(NumTableFilesAtLevel(0), 10);

@@ -2981,10 +2969,10 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
  // Test that shows the fall back to size-based FIFO compaction if TTL-based
  // deletion doesn't move the total size to be less than max_table_files_size.
  {
-    options.write_buffer_size = 10 << 10;                              // 10KB
+    options.write_buffer_size = 110 << 10;                             // 10KB
    options.compaction_options_fifo.max_table_files_size = 150 << 10;  // 150KB
    options.compaction_options_fifo.allow_compaction = false;
-    options.compaction_options_fifo.ttl =  1 * 60 * 60;  // 1 hour
+    options.compaction_options_fifo.ttl = 5;  // seconds
    options = CurrentOptions(options);
    DestroyAndReopen(options);

@@ -2995,13 +2983,11 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
      }
      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
    }
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    ASSERT_EQ(NumTableFilesAtLevel(0), 3);

-    // Sleep for 2 hours -- which is much greater than TTL.
-    env_->addon_time_.fetch_add(2 * 60 * 60);
-    // Just to make sure that we are in the same state even after sleeping.
+    env_->SleepForMicroseconds(6 * 1000 * 1000);
    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    ASSERT_EQ(NumTableFilesAtLevel(0), 3);

@@ -3010,8 +2996,8 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
      }
      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
    }
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    // Size limit is still guaranteed.
    ASSERT_LE(SizeAtLevel(0),
              options.compaction_options_fifo.max_table_files_size);
@@ -3021,7 +3007,7 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
  {
    options.compaction_options_fifo.max_table_files_size = 150 << 10;  // 150KB
    options.compaction_options_fifo.allow_compaction = true;
-    options.compaction_options_fifo.ttl = 1 * 60 * 60;  // 1 hour
+    options.compaction_options_fifo.ttl = 5;  // seconds
    options.level0_file_num_compaction_trigger = 6;
    options = CurrentOptions(options);
    DestroyAndReopen(options);
@@ -3033,16 +3019,15 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
      }
      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
    }
    // With Intra-L0 compaction, out of 10 files, 6 files will be compacted to 1
    // (due to level0_file_num_compaction_trigger = 6).
    // So total files = 1 + remaining 4 = 5.
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    ASSERT_EQ(NumTableFilesAtLevel(0), 5);

-    // Sleep for 2 hours -- which is much greater than TTL.
-    env_->addon_time_.fetch_add(2 * 60 * 60);
-    // Just to make sure that we are in the same state even after sleeping.
+    // Sleep for a little over ttl time.
+    env_->SleepForMicroseconds(6 * 1000 * 1000);
    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    ASSERT_EQ(NumTableFilesAtLevel(0), 5);

@@ -3052,8 +3037,8 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
      }
      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
    }
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    ASSERT_EQ(NumTableFilesAtLevel(0), 5);
    ASSERT_LE(SizeAtLevel(0),
              options.compaction_options_fifo.max_table_files_size);
@@ -3065,7 +3050,7 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
    options.write_buffer_size = 20 << 10;                               // 20K
    options.compaction_options_fifo.max_table_files_size = 1500 << 10;  // 1.5MB
    options.compaction_options_fifo.allow_compaction = true;
-    options.compaction_options_fifo.ttl = 1 * 60 * 60;  // 1 hour
+    options.compaction_options_fifo.ttl = 60 * 60;  // 1 hour
    options.level0_file_num_compaction_trigger = 6;
    options = CurrentOptions(options);
    DestroyAndReopen(options);
@@ -3077,8 +3062,8 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
      }
      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
    }
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    // It should be compacted to 10 files.
    ASSERT_EQ(NumTableFilesAtLevel(0), 10);

@@ -3088,8 +3073,8 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
        ASSERT_OK(Put(ToString(i * 20 + j + 2000), RandomString(&rnd, 980)));
      }
      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
    }
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());

    // It should be compacted to no more than 20 files.
    ASSERT_GT(NumTableFilesAtLevel(0), 10);
@@ -5375,36 +5360,6 @@ TEST_F(DBTest, RowCache) {
  ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), 1);
  ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), 1);
 }
-
-TEST_F(DBTest, PinnableSliceAndRowCache) {
-  Options options = CurrentOptions();
-  options.statistics = rocksdb::CreateDBStatistics();
-  options.row_cache = NewLRUCache(8192);
-  DestroyAndReopen(options);
-
-  ASSERT_OK(Put("foo", "bar"));
-  ASSERT_OK(Flush());
-
-  ASSERT_EQ(Get("foo"), "bar");
-  ASSERT_EQ(
-      reinterpret_cast<LRUCache*>(options.row_cache.get())->TEST_GetLRUSize(),
-      1);
-
-  {
-    PinnableSlice pin_slice;
-    ASSERT_EQ(Get("foo", &pin_slice), Status::OK());
-    ASSERT_EQ(pin_slice.ToString(), "bar");
-    // Entry is already in cache, lookup will remove the element from lru
-    ASSERT_EQ(
-        reinterpret_cast<LRUCache*>(options.row_cache.get())->TEST_GetLRUSize(),
-        0);
-  }
-  // After PinnableSlice destruction element is added back in LRU
-  ASSERT_EQ(
-      reinterpret_cast<LRUCache*>(options.row_cache.get())->TEST_GetLRUSize(),
-      1);
-}
-
 #endif  // ROCKSDB_LITE

 TEST_F(DBTest, DeletingOldWalAfterDrop) {
@@ -2236,7 +2236,6 @@ TEST_F(DBTest2, LowPriWrite) {
  ASSERT_EQ(1, rate_limit_count.load());
 }

-#ifndef ROCKSDB_LITE
 TEST_F(DBTest2, RateLimitedCompactionReads) {
  // compaction input has 512KB data
  const int kNumKeysPerFile = 128;
@@ -2282,15 +2281,12 @@ TEST_F(DBTest2, RateLimitedCompactionReads) {
    // chose 1MB as the upper bound on the total bytes read.
    size_t rate_limited_bytes =
        options.rate_limiter->GetTotalBytesThrough(Env::IO_LOW);
-    // Include the explict prefetch of the footer in direct I/O case.
-    size_t direct_io_extra = use_direct_io ? 512 * 1024 : 0;
-    ASSERT_GE(rate_limited_bytes,
-              static_cast<size_t>(kNumKeysPerFile * kBytesPerKey * kNumL0Files +
-                                  direct_io_extra));
+    ASSERT_GE(
+        rate_limited_bytes,
+        static_cast<size_t>(kNumKeysPerFile * kBytesPerKey * kNumL0Files));
    ASSERT_LT(
        rate_limited_bytes,
-        static_cast<size_t>(2 * kNumKeysPerFile * kBytesPerKey * kNumL0Files +
-                            direct_io_extra));
+        static_cast<size_t>(2 * kNumKeysPerFile * kBytesPerKey * kNumL0Files));

    Iterator* iter = db_->NewIterator(ReadOptions());
    for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
@@ -2303,28 +2299,7 @@ TEST_F(DBTest2, RateLimitedCompactionReads) {
                  options.rate_limiter->GetTotalBytesThrough(Env::IO_LOW)));
  }
 }
-#endif  // ROCKSDB_LITE

-// Make sure DB can be reopen with reduced number of levels, given no file
-// is on levels higher than the new num_levels.
-TEST_F(DBTest2, ReduceLevel) {
-  Options options;
-  options.disable_auto_compactions = true;
-  options.num_levels = 7;
-  Reopen(options);
-  Put("foo", "bar");
-  Flush();
-  MoveFilesToLevel(6);
-  ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel());
-  CompactRangeOptions compact_options;
-  compact_options.change_level = true;
-  compact_options.target_level = 1;
-  dbfull()->CompactRange(compact_options, nullptr, nullptr);
-  ASSERT_EQ("0,1", FilesPerLevel());
-  options.num_levels = 3;
-  Reopen(options);
-  ASSERT_EQ("0,1", FilesPerLevel());
-}
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -42,24 +42,13 @@ SpecialEnv::SpecialEnv(Env* base)
  non_writable_count_ = 0;
  table_write_callback_ = nullptr;
 }
-#ifndef ROCKSDB_LITE
+
 ROT13BlockCipher rot13Cipher_(16);
-#endif  // ROCKSDB_LITE

 DBTestBase::DBTestBase(const std::string path)
    : mem_env_(!getenv("MEM_ENV") ? nullptr : new MockEnv(Env::Default())),
-#ifndef ROCKSDB_LITE
-      encrypted_env_(
-          !getenv("ENCRYPTED_ENV")
-              ? nullptr
-              : NewEncryptedEnv(mem_env_ ? mem_env_ : Env::Default(),
-                                new CTREncryptionProvider(rot13Cipher_))),
-#else
-      encrypted_env_(nullptr),
-#endif  // ROCKSDB_LITE
-      env_(new SpecialEnv(encrypted_env_
-                              ? encrypted_env_
-                              : (mem_env_ ? mem_env_ : Env::Default()))),
+      encrypted_env_(!getenv("ENCRYPTED_ENV") ? nullptr : NewEncryptedEnv(mem_env_ ? mem_env_ : Env::Default(), new CTREncryptionProvider(rot13Cipher_))),
+      env_(new SpecialEnv(encrypted_env_ ? encrypted_env_ : (mem_env_ ? mem_env_ : Env::Default()))),
      option_config_(kDefault) {
  env_->SetBackgroundThreads(1, Env::LOW);
  env_->SetBackgroundThreads(1, Env::HIGH);
@@ -697,13 +686,6 @@ std::string DBTestBase::Get(int cf, const std::string& k,
  return result;
 }

-Status DBTestBase::Get(const std::string& k, PinnableSlice* v) {
-  ReadOptions options;
-  options.verify_checksums = true;
-  Status s = dbfull()->Get(options, dbfull()->DefaultColumnFamily(), k, v);
-  return s;
-}
-
 uint64_t DBTestBase::GetNumSnapshots() {
  uint64_t int_num;
  EXPECT_TRUE(dbfull()->GetIntProperty("rocksdb.num-snapshots", &int_num));
@@ -803,8 +803,6 @@ class DBTestBase : public testing::Test {
  std::string Get(int cf, const std::string& k,
                  const Snapshot* snapshot = nullptr);

-  Status Get(const std::string& k, PinnableSlice* v);
-
  uint64_t GetNumSnapshots();

  uint64_t GetTimeOldestSnapshots();
@@ -696,12 +696,17 @@ TEST_P(DBTestUniversalCompactionParallel, PickByFileNumberBug) {
    num_keys -= 100;
  }

-  // Hold the 1st compaction from finishing
+  // Wait for the 2nd background compaction process to start
+  TEST_SYNC_POINT("DBTestUniversalCompactionParallel::PickByFileNumberBug:0");
+  TEST_SYNC_POINT("DBTestUniversalCompactionParallel::PickByFileNumberBug:1");
+
+  // Hold the 1st and 2nd compaction from finishing
  TEST_SYNC_POINT("DBTestUniversalCompactionParallel::PickByFileNumberBug:2");
  dbfull()->TEST_WaitForCompact();

-  // There should only be one picked compaction as the score drops below one
-  // after the first one is picked.
+  // Although 2 compaction threads started, the second one did not compact
+  // anything because the number of files not being compacted is less than
+  // level0_file_num_compaction_trigger
  EXPECT_EQ(total_picked_compactions, 1);
  EXPECT_EQ(TotalTableFiles(), 4);

@@ -1365,140 +1370,6 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) {
  Destroy(options);
 }

-TEST_P(DBTestUniversalCompaction, FullCompactionInBottomPriThreadPool) {
-  const int kNumFilesTrigger = 3;
-  Env::Default()->SetBackgroundThreads(1, Env::Priority::BOTTOM);
-  for (bool allow_ingest_behind : {false, true}) {
-    Options options = CurrentOptions();
-    options.allow_ingest_behind = allow_ingest_behind;
-    options.compaction_style = kCompactionStyleUniversal;
-    options.num_levels = num_levels_;
-    options.write_buffer_size = 100 << 10;     // 100KB
-    options.target_file_size_base = 32 << 10;  // 32KB
-    options.level0_file_num_compaction_trigger = kNumFilesTrigger;
-    // Trigger compaction if size amplification exceeds 110%
-    options.compaction_options_universal.max_size_amplification_percent = 110;
-    DestroyAndReopen(options);
-
-    int num_bottom_pri_compactions = 0;
-    SyncPoint::GetInstance()->SetCallBack(
-        "DBImpl::BGWorkBottomCompaction",
-        [&](void* arg) { ++num_bottom_pri_compactions; });
-    SyncPoint::GetInstance()->EnableProcessing();
-
-    Random rnd(301);
-    for (int num = 0; num < kNumFilesTrigger; num++) {
-      ASSERT_EQ(NumSortedRuns(), num);
-      int key_idx = 0;
-      GenerateNewFile(&rnd, &key_idx);
-    }
-    dbfull()->TEST_WaitForCompact();
-
-    if (allow_ingest_behind || num_levels_ > 1) {
-      // allow_ingest_behind increases number of levels while sanitizing.
-      ASSERT_EQ(1, num_bottom_pri_compactions);
-    } else {
-      // for single-level universal, everything's bottom level so nothing should
-      // be executed in bottom-pri thread pool.
-      ASSERT_EQ(0, num_bottom_pri_compactions);
-    }
-    // Verify that size amplification did occur
-    ASSERT_EQ(NumSortedRuns(), 1);
-    rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-  }
-  Env::Default()->SetBackgroundThreads(0, Env::Priority::BOTTOM);
-}
-
-TEST_P(DBTestUniversalCompaction, ConcurrentBottomPriLowPriCompactions) {
-  if (num_levels_ == 1) {
-    // for single-level universal, everything's bottom level so nothing should
-    // be executed in bottom-pri thread pool.
-    return;
-  }
-  const int kNumFilesTrigger = 3;
-  Env::Default()->SetBackgroundThreads(1, Env::Priority::BOTTOM);
-  Options options = CurrentOptions();
-  options.compaction_style = kCompactionStyleUniversal;
-  options.num_levels = num_levels_;
-  options.write_buffer_size = 100 << 10;     // 100KB
-  options.target_file_size_base = 32 << 10;  // 32KB
-  options.level0_file_num_compaction_trigger = kNumFilesTrigger;
-  // Trigger compaction if size amplification exceeds 110%
-  options.compaction_options_universal.max_size_amplification_percent = 110;
-  DestroyAndReopen(options);
-
-  rocksdb::SyncPoint::GetInstance()->LoadDependency(
-      {// wait for the full compaction to be picked before adding files intended
-       // for the second one.
-       {"DBImpl::BackgroundCompaction:ForwardToBottomPriPool",
-        "DBTestUniversalCompaction:ConcurrentBottomPriLowPriCompactions:0"},
-       // the full (bottom-pri) compaction waits until a partial (low-pri)
-       // compaction has started to verify they can run in parallel.
-       {"DBImpl::BackgroundCompaction:NonTrivial",
-        "DBImpl::BGWorkBottomCompaction"}});
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  Random rnd(301);
-  for (int i = 0; i < 2; ++i) {
-    for (int num = 0; num < kNumFilesTrigger; num++) {
-      int key_idx = 0;
-      GenerateNewFile(&rnd, &key_idx, true /* no_wait */);
-      // use no_wait above because that one waits for flush and compaction. We
-      // don't want to wait for compaction because the full compaction is
-      // intentionally blocked while more files are flushed.
-      dbfull()->TEST_WaitForFlushMemTable();
-    }
-    if (i == 0) {
-      TEST_SYNC_POINT(
-          "DBTestUniversalCompaction:ConcurrentBottomPriLowPriCompactions:0");
-    }
-  }
-  dbfull()->TEST_WaitForCompact();
-
-  // First compaction should output to bottom level. Second should output to L0
-  // since older L0 files pending compaction prevent it from being placed lower.
-  ASSERT_EQ(NumSortedRuns(), 2);
-  ASSERT_GT(NumTableFilesAtLevel(0), 0);
-  ASSERT_GT(NumTableFilesAtLevel(num_levels_ - 1), 0);
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-  Env::Default()->SetBackgroundThreads(0, Env::Priority::BOTTOM);
-}
-
-TEST_P(DBTestUniversalCompaction, RecalculateScoreAfterPicking) {
-  // Regression test for extra compactions scheduled. Once enough compactions
-  // have been scheduled to bring the score below one, we should stop
-  // scheduling more; otherwise, other CFs/DBs may be delayed unnecessarily.
-  const int kNumFilesTrigger = 8;
-  Options options = CurrentOptions();
-  options.compaction_options_universal.max_merge_width = kNumFilesTrigger / 2;
-  options.compaction_options_universal.max_size_amplification_percent =
-      static_cast<unsigned int>(-1);
-  options.compaction_style = kCompactionStyleUniversal;
-  options.level0_file_num_compaction_trigger = kNumFilesTrigger;
-  options.num_levels = num_levels_;
-  options.write_buffer_size = 100 << 10;  // 100KB
-  Reopen(options);
-
-  std::atomic<int> num_compactions_attempted(0);
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::BackgroundCompaction:Start", [&](void* arg) {
-        ++num_compactions_attempted;
-      });
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-
-  Random rnd(301);
-  for (int num = 0; num < kNumFilesTrigger; num++) {
-    ASSERT_EQ(NumSortedRuns(), num);
-    int key_idx = 0;
-    GenerateNewFile(&rnd, &key_idx);
-  }
-  dbfull()->TEST_WaitForCompact();
-  // Compacting the first four files was enough to bring the score below one so
-  // there's no need to schedule any more compactions.
-  ASSERT_EQ(1, num_compactions_attempted);
-  ASSERT_EQ(NumSortedRuns(), 5);
-}
-
 INSTANTIATE_TEST_CASE_P(UniversalCompactionNumLevels, DBTestUniversalCompaction,
                        ::testing::Combine(::testing::Values(1, 3, 5),
                                           ::testing::Bool()));
@@ -3,12 +3,17 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).

+#include <atomic>
 #include <memory>
 #include <thread>
 #include <vector>
 #include "db/db_test_util.h"
 #include "db/write_batch_internal.h"
+#include "db/write_thread.h"
+#include "port/port.h"
 #include "port/stack_trace.h"
+#include "util/fault_injection_test_env.h"
+#include "util/string_util.h"
 #include "util/sync_point.h"

 namespace rocksdb {
@@ -18,7 +23,9 @@ class DBWriteTest : public DBTestBase, public testing::WithParamInterface<int> {
 public:
  DBWriteTest() : DBTestBase("/db_write_test") {}

-  void Open() { DBTestBase::Reopen(GetOptions(GetParam())); }
+  Options GetOptions() { return DBTestBase::GetOptions(GetParam()); }
+
+  void Open() { DBTestBase::Reopen(GetOptions()); }
 };

 // Sequence number should be return through input write batch.
@@ -67,6 +74,47 @@ TEST_P(DBWriteTest, ReturnSeuqneceNumberMultiThreaded) {
  }
 }

+TEST_P(DBWriteTest, IOErrorOnWALWritePropagateToWriteThreadFollower) {
+  constexpr int kNumThreads = 5;
+  std::unique_ptr<FaultInjectionTestEnv> mock_env(
+      new FaultInjectionTestEnv(Env::Default()));
+  Options options = GetOptions();
+  options.env = mock_env.get();
+  Reopen(options);
+  std::atomic<int> ready_count{0};
+  std::atomic<int> leader_count{0};
+  std::vector<port::Thread> threads;
+  mock_env->SetFilesystemActive(false);
+  // Wait until all threads linked to write threads, to make sure
+  // all threads join the same batch group.
+  SyncPoint::GetInstance()->SetCallBack(
+      "WriteThread::JoinBatchGroup:Wait", [&](void* arg) {
+        ready_count++;
+        auto* w = reinterpret_cast<WriteThread::Writer*>(arg);
+        if (w->state == WriteThread::STATE_GROUP_LEADER) {
+          leader_count++;
+          while (ready_count < kNumThreads) {
+            // busy waiting
+          }
+        }
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+  for (int i = 0; i < kNumThreads; i++) {
+    threads.push_back(port::Thread(
+        [&](int index) {
+          // All threads should fail.
+          ASSERT_FALSE(Put("key" + ToString(index), "value").ok());
+        },
+        i));
+  }
+  for (int i = 0; i < kNumThreads; i++) {
+    threads[i].join();
+  }
+  ASSERT_EQ(1, leader_count);
+  // Close before mock_env destruct.
+  Close();
+}
+
 INSTANTIATE_TEST_CASE_P(DBWriteTestInstance, DBWriteTest,
                        testing::Values(DBTestBase::kDefault,
                                        DBTestBase::kConcurrentWALWrites,
@@ -84,12 +84,6 @@ struct ParsedInternalKey {
  ParsedInternalKey(const Slice& u, const SequenceNumber& seq, ValueType t)
      : user_key(u), sequence(seq), type(t) { }
  std::string DebugString(bool hex = false) const;
-
-  void clear() {
-    user_key.clear();
-    sequence = 0;
-    type = kTypeDeletion;
-  }
 };

 // Return the length of the encoding of "key".
@@ -157,9 +151,6 @@ class InternalKeyComparator : public Comparator {

  int Compare(const InternalKey& a, const InternalKey& b) const;
  int Compare(const ParsedInternalKey& a, const ParsedInternalKey& b) const;
-  virtual const Comparator* GetRootComparator() const override {
-    return user_comparator_->GetRootComparator();
-  }
 };

 // Modules in this directory should keep internal keys wrapped inside
@@ -14,18 +14,20 @@ namespace experimental {

 Status SuggestCompactRange(DB* db, ColumnFamilyHandle* column_family,
                           const Slice* begin, const Slice* end) {
-  if (db == nullptr) {
-    return Status::InvalidArgument("DB is empty");
+  auto dbimpl = dynamic_cast<DBImpl*>(db);
+  if (dbimpl == nullptr) {
+    return Status::InvalidArgument("Didn't recognize DB object");
  }

-  return db->SuggestCompactRange(column_family, begin, end);
+  return dbimpl->SuggestCompactRange(column_family, begin, end);
 }

 Status PromoteL0(DB* db, ColumnFamilyHandle* column_family, int target_level) {
-  if (db == nullptr) {
+  auto dbimpl = dynamic_cast<DBImpl*>(db);
+  if (dbimpl == nullptr) {
    return Status::InvalidArgument("Didn't recognize DB object");
  }
-  return db->PromoteL0(column_family, target_level);
+  return dbimpl->PromoteL0(column_family, target_level);
 }

 #else  // ROCKSDB_LITE
@@ -18,33 +18,6 @@

 namespace rocksdb {

-MergeHelper::MergeHelper(Env* env, const Comparator* user_comparator,
-                         const MergeOperator* user_merge_operator,
-                         const CompactionFilter* compaction_filter,
-                         Logger* logger, bool assert_valid_internal_key,
-                         SequenceNumber latest_snapshot, int level,
-                         Statistics* stats,
-                         const std::atomic<bool>* shutting_down)
-    : env_(env),
-      user_comparator_(user_comparator),
-      user_merge_operator_(user_merge_operator),
-      compaction_filter_(compaction_filter),
-      shutting_down_(shutting_down),
-      logger_(logger),
-      assert_valid_internal_key_(assert_valid_internal_key),
-      allow_single_operand_(false),
-      latest_snapshot_(latest_snapshot),
-      level_(level),
-      keys_(),
-      filter_timer_(env_),
-      total_filter_time_(0U),
-      stats_(stats) {
-  assert(user_comparator_ != nullptr);
-  if (user_merge_operator_) {
-    allow_single_operand_ = user_merge_operator_->AllowSingleOperand();
-  }
-}
-
 Status MergeHelper::TimedFullMerge(const MergeOperator* merge_operator,
                                   const Slice& key, const Slice* value,
                                   const std::vector<Slice>& operands,
@@ -228,11 +201,12 @@ Status MergeHelper::MergeUntil(InternalIterator* iter,
          ikey.sequence <= latest_snapshot_
              ? CompactionFilter::Decision::kKeep
              : FilterMerge(orig_ikey.user_key, value_slice);
-      if (filter != CompactionFilter::Decision::kRemoveAndSkipUntil &&
-          range_del_agg != nullptr &&
+      if (range_del_agg != nullptr &&
+
          range_del_agg->ShouldDelete(
              iter->key(),
-              RangeDelAggregator::RangePositioningMode::kForwardTraversal)) {
+              RangeDelAggregator::RangePositioningMode::kForwardTraversal) &&
+          filter != CompactionFilter::Decision::kRemoveAndSkipUntil) {
        filter = CompactionFilter::Decision::kRemove;
      }
      if (filter == CompactionFilter::Decision::kKeep ||
@@ -315,8 +289,7 @@ Status MergeHelper::MergeUntil(InternalIterator* iter,
    // Attempt to use the user's associative merge function to
    // merge the stacked merge operands into a single operand.
    s = Status::MergeInProgress();
-    if (merge_context_.GetNumOperands() >= 2 ||
-        (allow_single_operand_ && merge_context_.GetNumOperands() == 1)) {
+    if (merge_context_.GetNumOperands() >= 2) {
      bool merge_success = false;
      std::string merge_result;
      {
@@ -34,7 +34,22 @@ class MergeHelper {
              const CompactionFilter* compaction_filter, Logger* logger,
              bool assert_valid_internal_key, SequenceNumber latest_snapshot,
              int level = 0, Statistics* stats = nullptr,
-              const std::atomic<bool>* shutting_down = nullptr);
+              const std::atomic<bool>* shutting_down = nullptr)
+      : env_(env),
+        user_comparator_(user_comparator),
+        user_merge_operator_(user_merge_operator),
+        compaction_filter_(compaction_filter),
+        shutting_down_(shutting_down),
+        logger_(logger),
+        assert_valid_internal_key_(assert_valid_internal_key),
+        latest_snapshot_(latest_snapshot),
+        level_(level),
+        keys_(),
+        filter_timer_(env_),
+        total_filter_time_(0U),
+        stats_(stats) {
+    assert(user_comparator_ != nullptr);
+  }

  // Wrapper around MergeOperator::FullMergeV2() that records perf statistics.
  // Result of merge will be written to result if status returned is OK.
@@ -143,7 +158,6 @@ class MergeHelper {
  const std::atomic<bool>* shutting_down_;
  Logger* logger_;
  bool assert_valid_internal_key_; // enforce no internal key corruption?
-  bool allow_single_operand_;
  SequenceNumber latest_snapshot_;
  int level_;

@@ -357,8 +357,7 @@ Status RangeDelAggregator::AddTombstone(RangeTombstone tombstone) {
      ++new_range_dels_iter;
    }
  } else {
-    auto start_key = tombstone.start_key_;
-    tombstone_map.emplace(start_key, std::move(tombstone));
+    tombstone_map.emplace(tombstone.start_key_, std::move(tombstone));
  }
  return Status::OK();
 }
@@ -176,7 +176,6 @@ class Repairer {
      status = db_impl->NewDB();
      delete db_impl;
    }
-
    if (status.ok()) {
      // Recover using the fresh manifest created by NewDB()
      status =
@@ -247,21 +246,9 @@ class Repairer {
  Status FindFiles() {
    std::vector<std::string> filenames;
    bool found_file = false;
-    std::vector<std::string> to_search_paths;
-
    for (size_t path_id = 0; path_id < db_options_.db_paths.size(); path_id++) {
-        to_search_paths.push_back(db_options_.db_paths[path_id].path);
-    }
-
-    // search wal_dir if user uses a customize wal_dir
-    if (!db_options_.wal_dir.empty() && 
-        db_options_.wal_dir != dbname_) {
-        to_search_paths.push_back(db_options_.wal_dir);
-    }
-
-    for (size_t path_id = 0; path_id < to_search_paths.size(); path_id++) {
      Status status =
-          env_->GetChildren(to_search_paths[path_id], &filenames);
+          env_->GetChildren(db_options_.db_paths[path_id].path, &filenames);
      if (!status.ok()) {
        return status;
      }
@@ -274,12 +261,14 @@ class Repairer {
      for (size_t i = 0; i < filenames.size(); i++) {
        if (ParseFileName(filenames[i], &number, &type)) {
          if (type == kDescriptorFile) {
+            assert(path_id == 0);
            manifests_.push_back(filenames[i]);
          } else {
            if (number + 1 > next_file_number_) {
              next_file_number_ = number + 1;
            }
            if (type == kLogFile) {
+              assert(path_id == 0);
              logs_.push_back(number);
            } else if (type == kTableFile) {
              table_fds_.emplace_back(number, static_cast<uint32_t>(path_id),
@@ -299,8 +288,7 @@ class Repairer {

  void ConvertLogFilesToTables() {
    for (size_t i = 0; i < logs_.size(); i++) {
-      // we should use LogFileName(wal_dir, logs_[i]) here. user might uses wal_dir option.
-      std::string logname = LogFileName(db_options_.wal_dir, logs_[i]);
+      std::string logname = LogFileName(dbname_, logs_[i]);
      Status status = ConvertLogToTable(logs_[i]);
      if (!status.ok()) {
        ROCKS_LOG_WARN(db_options_.info_log,
@@ -324,7 +312,7 @@ class Repairer {
    };

    // Open the log file
-    std::string logname = LogFileName(db_options_.wal_dir, log);
+    std::string logname = LogFileName(dbname_, log);
    unique_ptr<SequentialFile> lfile;
    Status status = env_->NewSequentialFile(
        logname, &lfile, env_->OptimizeForLogRead(env_options_));
@@ -174,40 +174,6 @@ TEST_F(RepairTest, UnflushedSst) {
  ASSERT_EQ(Get("key"), "val");
 }

-TEST_F(RepairTest, SeparateWalDir) {
-  do {
-    Options options = CurrentOptions();
-    DestroyAndReopen(options);
-    Put("key", "val");
-    Put("foo", "bar");
-    VectorLogPtr wal_files;
-    ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
-    ASSERT_EQ(wal_files.size(), 1);
-    uint64_t total_ssts_size;
-    GetAllSSTFiles(&total_ssts_size);
-    ASSERT_EQ(total_ssts_size, 0);
-    std::string manifest_path =
-      DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
-
-    Close();
-    ASSERT_OK(env_->FileExists(manifest_path));
-    ASSERT_OK(env_->DeleteFile(manifest_path));
-    ASSERT_OK(RepairDB(dbname_, options));
-
-    // make sure that all WALs are converted to SSTables.
-    options.wal_dir = "";
-
-    Reopen(options);
-    ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
-    ASSERT_EQ(wal_files.size(), 0);
-    GetAllSSTFiles(&total_ssts_size);
-    ASSERT_GT(total_ssts_size, 0);
-    ASSERT_EQ(Get("key"), "val");
-    ASSERT_EQ(Get("foo"), "bar");
-
- } while(ChangeWalOptions());
-}
-
 TEST_F(RepairTest, RepairMultipleColumnFamilies) {
  // Verify repair logic associates SST files with their original column
  // families.
@@ -74,11 +74,9 @@ class SnapshotList {
    count_--;
  }

-  // retrieve all snapshot numbers up until max_seq. They are sorted in
-  // ascending order.
+  // retrieve all snapshot numbers. They are sorted in ascending order.
  std::vector<SequenceNumber> GetAll(
-      SequenceNumber* oldest_write_conflict_snapshot = nullptr,
-      const SequenceNumber& max_seq = kMaxSequenceNumber) const {
+      SequenceNumber* oldest_write_conflict_snapshot = nullptr) {
    std::vector<SequenceNumber> ret;

    if (oldest_write_conflict_snapshot != nullptr) {
@@ -88,11 +86,8 @@ class SnapshotList {
    if (empty()) {
      return ret;
    }
-    const SnapshotImpl* s = &list_;
+    SnapshotImpl* s = &list_;
    while (s->next_ != &list_) {
-      if (s->next_->number_ > max_seq) {
-        break;
-      }
      ret.push_back(s->next_->number_);

      if (oldest_write_conflict_snapshot != nullptr &&
@@ -106,10 +106,10 @@ Status TableCache::GetTableReader(
    }
    StopWatch sw(ioptions_.env, ioptions_.statistics, TABLE_OPEN_IO_MICROS);
    std::unique_ptr<RandomAccessFileReader> file_reader(
-        new RandomAccessFileReader(
-            std::move(file), fname, ioptions_.env,
-            record_read_stats ? ioptions_.statistics : nullptr, SST_READ_MICROS,
-            file_read_hist, ioptions_.rate_limiter, for_compaction));
+        new RandomAccessFileReader(std::move(file), fname, ioptions_.env,
+                                   ioptions_.statistics, record_read_stats,
+                                   file_read_hist, ioptions_.rate_limiter,
+                                   for_compaction));
    s = ioptions_.table_factory->NewTableReader(
        TableReaderOptions(ioptions_, env_options, internal_comparator,
                           skip_filters, level),
@@ -225,7 +225,8 @@ InternalIterator* TableCache::NewIterator(
  }
  InternalIterator* result = nullptr;
  if (s.ok()) {
-    result = table_reader->NewIterator(options, arena, skip_filters);
+    result =
+      table_reader->NewIterator(options, arena, &icomparator, skip_filters);
    if (create_new_table_reader) {
      assert(handle == nullptr);
      result->RegisterCleanup(&DeleteTableReader, table_reader, nullptr);
@@ -310,7 +311,6 @@ Status TableCache::Get(const ReadOptions& options,
 #ifndef ROCKSDB_LITE
  IterKey row_cache_key;
  std::string row_cache_entry_buffer;
-
  // Check row cache if enabled. Since row cache does not currently store
  // sequence numbers, we cannot use it if we need to fetch the sequence.
  if (ioptions_.row_cache && !get_context->NeedToReadSequence()) {
@@ -334,26 +334,10 @@ Status TableCache::Get(const ReadOptions& options,

    if (auto row_handle =
            ioptions_.row_cache->Lookup(row_cache_key.GetUserKey())) {
-      // Cleanable routine to release the cache entry
-      Cleanable value_pinner;
-      auto release_cache_entry_func = [](void* cache_to_clean,
-                                         void* cache_handle) {
-        ((Cache*)cache_to_clean)->Release((Cache::Handle*)cache_handle);
-      };
      auto found_row_cache_entry = static_cast<const std::string*>(
          ioptions_.row_cache->Value(row_handle));
-      // If it comes here value is located on the cache.
-      // found_row_cache_entry points to the value on cache,
-      // and value_pinner has cleanup procedure for the cached entry.
-      // After replayGetContextLog() returns, get_context.pinnable_slice_
-      // will point to cache entry buffer (or a copy based on that) and
-      // cleanup routine under value_pinner will be delegated to
-      // get_context.pinnable_slice_. Cache entry is released when
-      // get_context.pinnable_slice_ is reset.
-      value_pinner.RegisterCleanup(release_cache_entry_func,
-                                   ioptions_.row_cache.get(), row_handle);
-      replayGetContextLog(*found_row_cache_entry, user_key, get_context,
-                          &value_pinner);
+      replayGetContextLog(*found_row_cache_entry, user_key, get_context);
+      ioptions_.row_cache->Release(row_handle);
      RecordTick(ioptions_.statistics, ROW_CACHE_HIT);
      done = true;
    } else {
@@ -17,7 +17,6 @@
 #include <algorithm>
 #include <atomic>
 #include <functional>
-#include <map>
 #include <set>
 #include <thread>
 #include <unordered_map>
@@ -88,16 +87,7 @@ class VersionBuilder::Rep {
  Logger* info_log_;
  TableCache* table_cache_;
  VersionStorageInfo* base_vstorage_;
-  int num_levels_;
  LevelState* levels_;
-  // Store states of levels larger than num_levels_. We do this instead of
-  // storing them in levels_ to avoid regression in case there are no files
-  // on invalid levels. The version is not consistent if in the end the files
-  // on invalid levels don't cancel out.
-  std::map<int, std::unordered_set<uint64_t>> invalid_levels_;
-  // Whether there are invalid new files or invalid deletion on levels larger
-  // than num_levels_.
-  bool has_invalid_levels_;
  FileComparator level_zero_cmp_;
  FileComparator level_nonzero_cmp_;

@@ -107,10 +97,8 @@ class VersionBuilder::Rep {
      : env_options_(env_options),
        info_log_(info_log),
        table_cache_(table_cache),
-        base_vstorage_(base_vstorage),
-        num_levels_(base_vstorage->num_levels()),
-        has_invalid_levels_(false) {
-    levels_ = new LevelState[num_levels_];
+        base_vstorage_(base_vstorage) {
+    levels_ = new LevelState[base_vstorage_->num_levels()];
    level_zero_cmp_.sort_method = FileComparator::kLevel0;
    level_nonzero_cmp_.sort_method = FileComparator::kLevelNon0;
    level_nonzero_cmp_.internal_comparator =
@@ -118,7 +106,7 @@ class VersionBuilder::Rep {
  }

  ~Rep() {
-    for (int level = 0; level < num_levels_; level++) {
+    for (int level = 0; level < base_vstorage_->num_levels(); level++) {
      const auto& added = levels_[level].added_files;
      for (auto& pair : added) {
        UnrefFile(pair.second);
@@ -149,7 +137,7 @@ class VersionBuilder::Rep {
    }
 #endif
    // make sure the files are sorted correctly
-    for (int level = 0; level < num_levels_; level++) {
+    for (int level = 0; level < vstorage->num_levels(); level++) {
      auto& level_files = vstorage->LevelFiles(level);
      for (size_t i = 1; i < level_files.size(); i++) {
        auto f1 = level_files[i - 1];
@@ -208,7 +196,7 @@ class VersionBuilder::Rep {
 #endif
    // a file to be deleted better exist in the previous version
    bool found = false;
-    for (int l = 0; !found && l < num_levels_; l++) {
+    for (int l = 0; !found && l < base_vstorage_->num_levels(); l++) {
      const std::vector<FileMetaData*>& base_files =
          base_vstorage_->LevelFiles(l);
      for (size_t i = 0; i < base_files.size(); i++) {
@@ -222,7 +210,7 @@ class VersionBuilder::Rep {
    // if the file did not exist in the previous version, then it
    // is possibly moved from lower level to higher level in current
    // version
-    for (int l = level + 1; !found && l < num_levels_; l++) {
+    for (int l = level + 1; !found && l < base_vstorage_->num_levels(); l++) {
      auto& level_added = levels_[l].added_files;
      auto got = level_added.find(number);
      if (got != level_added.end()) {
@@ -245,19 +233,6 @@ class VersionBuilder::Rep {
    }
  }

-  bool CheckConsistencyForNumLevels() {
-    // Make sure there are no files on or beyond num_levels().
-    if (has_invalid_levels_) {
-      return false;
-    }
-    for (auto& level : invalid_levels_) {
-      if (level.second.size() > 0) {
-        return false;
-      }
-    }
-    return true;
-  }
-
  // Apply all of the edits in *edit to the current state.
  void Apply(VersionEdit* edit) {
    CheckConsistency(base_vstorage_);
@@ -267,45 +242,26 @@ class VersionBuilder::Rep {
    for (const auto& del_file : del) {
      const auto level = del_file.first;
      const auto number = del_file.second;
-      if (level < num_levels_) {
-        levels_[level].deleted_files.insert(number);
-        CheckConsistencyForDeletes(edit, number, level);
+      levels_[level].deleted_files.insert(number);
+      CheckConsistencyForDeletes(edit, number, level);

-        auto exising = levels_[level].added_files.find(number);
-        if (exising != levels_[level].added_files.end()) {
-          UnrefFile(exising->second);
-          levels_[level].added_files.erase(number);
-        }
-      } else {
-        if (invalid_levels_[level].count(number) > 0) {
-          invalid_levels_[level].erase(number);
-        } else {
-          // Deleting an non-existing file on invalid level.
-          has_invalid_levels_ = true;
-        }
+      auto exising = levels_[level].added_files.find(number);
+      if (exising != levels_[level].added_files.end()) {
+        UnrefFile(exising->second);
+        levels_[level].added_files.erase(number);
      }
    }

    // Add new files
    for (const auto& new_file : edit->GetNewFiles()) {
      const int level = new_file.first;
-      if (level < num_levels_) {
-        FileMetaData* f = new FileMetaData(new_file.second);
-        f->refs = 1;
+      FileMetaData* f = new FileMetaData(new_file.second);
+      f->refs = 1;

-        assert(levels_[level].added_files.find(f->fd.GetNumber()) ==
-               levels_[level].added_files.end());
-        levels_[level].deleted_files.erase(f->fd.GetNumber());
-        levels_[level].added_files[f->fd.GetNumber()] = f;
-      } else {
-        uint64_t number = new_file.second.fd.GetNumber();
-        if (invalid_levels_[level].count(number) == 0) {
-          invalid_levels_[level].insert(number);
-        } else {
-          // Creating an already existing file on invalid level.
-          has_invalid_levels_ = true;
-        }
-      }
+      assert(levels_[level].added_files.find(f->fd.GetNumber()) ==
+             levels_[level].added_files.end());
+      levels_[level].deleted_files.erase(f->fd.GetNumber());
+      levels_[level].added_files[f->fd.GetNumber()] = f;
    }
  }

@@ -314,7 +270,7 @@ class VersionBuilder::Rep {
    CheckConsistency(base_vstorage_);
    CheckConsistency(vstorage);

-    for (int level = 0; level < num_levels_; level++) {
+    for (int level = 0; level < base_vstorage_->num_levels(); level++) {
      const auto& cmp = (level == 0) ? level_zero_cmp_ : level_nonzero_cmp_;
      // Merge the set of added files with the set of pre-existing files.
      // Drop any deleted files.  Store the result in *v.
@@ -369,7 +325,7 @@ class VersionBuilder::Rep {
    assert(table_cache_ != nullptr);
    // <file metadata, level>
    std::vector<std::pair<FileMetaData*, int>> files_meta;
-    for (int level = 0; level < num_levels_; level++) {
+    for (int level = 0; level < base_vstorage_->num_levels(); level++) {
      for (auto& file_meta_pair : levels_[level].added_files) {
        auto* file_meta = file_meta_pair.second;
        assert(!file_meta->table_reader_handle);
@@ -430,35 +386,24 @@ VersionBuilder::VersionBuilder(const EnvOptions& env_options,
                               VersionStorageInfo* base_vstorage,
                               Logger* info_log)
    : rep_(new Rep(env_options, info_log, table_cache, base_vstorage)) {}
-
 VersionBuilder::~VersionBuilder() { delete rep_; }
-
 void VersionBuilder::CheckConsistency(VersionStorageInfo* vstorage) {
  rep_->CheckConsistency(vstorage);
 }
-
 void VersionBuilder::CheckConsistencyForDeletes(VersionEdit* edit,
                                                uint64_t number, int level) {
  rep_->CheckConsistencyForDeletes(edit, number, level);
 }
-
-bool VersionBuilder::CheckConsistencyForNumLevels() {
-  return rep_->CheckConsistencyForNumLevels();
-}
-
 void VersionBuilder::Apply(VersionEdit* edit) { rep_->Apply(edit); }
-
 void VersionBuilder::SaveTo(VersionStorageInfo* vstorage) {
  rep_->SaveTo(vstorage);
 }
-
 void VersionBuilder::LoadTableHandlers(
    InternalStats* internal_stats, int max_threads,
    bool prefetch_index_and_filter_in_cache) {
  rep_->LoadTableHandlers(internal_stats, max_threads,
                          prefetch_index_and_filter_in_cache);
 }
-
 void VersionBuilder::MaybeAddFile(VersionStorageInfo* vstorage, int level,
                                  FileMetaData* f) {
  rep_->MaybeAddFile(vstorage, level, f);
@@ -29,7 +29,6 @@ class VersionBuilder {
  void CheckConsistency(VersionStorageInfo* vstorage);
  void CheckConsistencyForDeletes(VersionEdit* edit, uint64_t number,
                                  int level);
-  bool CheckConsistencyForNumLevels();
  void Apply(VersionEdit* edit);
  void SaveTo(VersionStorageInfo* vstorage);
  void LoadTableHandlers(InternalStats* internal_stats, int max_threads,
@@ -328,6 +328,10 @@ Version::~Version() {
      assert(f->refs > 0);
      f->refs--;
      if (f->refs <= 0) {
+        if (f->table_reader_handle) {
+          cfd_->table_cache()->EraseHandle(f->fd, f->table_reader_handle);
+          f->table_reader_handle = nullptr;
+        }
        vset_->obsolete_files_.push_back(f);
      }
    }
@@ -1227,14 +1231,6 @@ int VersionStorageInfo::MaxInputLevel() const {
  return 0;
 }

-int VersionStorageInfo::MaxOutputLevel(bool allow_ingest_behind) const {
-  if (allow_ingest_behind) {
-    assert(num_levels() > 1);
-    return num_levels() - 2;
-  }
-  return num_levels() - 1;
-}
-
 void VersionStorageInfo::EstimateCompactionBytesNeeded(
    const MutableCFOptions& mutable_cf_options) {
  // Only implemented for level-based compaction
@@ -2335,14 +2331,10 @@ void CloseTables(void* ptr, size_t) {
 VersionSet::~VersionSet() {
  // we need to delete column_family_set_ because its destructor depends on
  // VersionSet
-  Cache* table_cache = column_family_set_->get_table_cache();
-  table_cache->ApplyToAllCacheEntries(&CloseTables, false /* thread_safe */);
+  column_family_set_->get_table_cache()->ApplyToAllCacheEntries(&CloseTables,
+                                                                false);
  column_family_set_.reset();
  for (auto file : obsolete_files_) {
-    if (file->table_reader_handle) {
-      table_cache->Release(file->table_reader_handle);
-      TableCache::Evict(table_cache, file->fd.GetNumber());
-    }
    delete file;
  }
  obsolete_files_.clear();
@@ -2842,6 +2834,11 @@ Status VersionSet::Recover(
        cfd = column_family_set_->GetColumnFamily(edit.column_family_);
        // this should never happen since cf_in_builders is true
        assert(cfd != nullptr);
+        if (edit.max_level_ >= cfd->current()->storage_info()->num_levels()) {
+          s = Status::InvalidArgument(
+              "db has more levels than options.num_levels");
+          break;
+        }

        // if it is not column family add or column family drop,
        // then it's a file add/delete, which should be forwarded
@@ -2925,18 +2922,6 @@ Status VersionSet::Recover(
        list_of_not_found);
  }

-  if (s.ok()) {
-    for (auto cfd : *column_family_set_) {
-      assert(builders.count(cfd->GetID()) > 0);
-      auto* builder = builders[cfd->GetID()]->version_builder();
-      if (!builder->CheckConsistencyForNumLevels()) {
-        s = Status::InvalidArgument(
-            "db has more levels than options.num_levels");
-        break;
-      }
-    }
-  }
-
  if (s.ok()) {
    for (auto cfd : *column_family_set_) {
      if (cfd->IsDropped()) {
@@ -147,7 +147,6 @@ class VersionStorageInfo {
  }

  int MaxInputLevel() const;
-  int MaxOutputLevel(bool allow_ingest_behind) const;

  // Return level number that has idx'th highest score
  int CompactionScoreLevel(int idx) const { return compaction_level_[idx]; }
@@ -26,7 +26,6 @@
 #include "rocksdb/env.h"
 #include "rocksdb/options.h"
 #include "rocksdb/write_batch.h"
-#include "util/cast_util.h"
 #include "util/coding.h"
 #include "util/file_reader_writer.h"
 #include "util/filename.h"
@@ -274,8 +273,8 @@ namespace {
 struct CompareLogByPointer {
  bool operator()(const std::unique_ptr<LogFile>& a,
                  const std::unique_ptr<LogFile>& b) {
-    LogFileImpl* a_impl = static_cast_with_check<LogFileImpl, LogFile>(a.get());
-    LogFileImpl* b_impl = static_cast_with_check<LogFileImpl, LogFile>(b.get());
+    LogFileImpl* a_impl = dynamic_cast<LogFileImpl*>(a.get());
+    LogFileImpl* b_impl = dynamic_cast<LogFileImpl*>(b.get());
    return *a_impl < *b_impl;
  }
 };
@@ -1303,7 +1303,6 @@ Status WriteBatchInternal::InsertInto(WriteThread::WriteGroup& write_group,
      continue;
    }
    SetSequence(w->batch, inserter.sequence());
-    w->sequence = inserter.sequence();
    inserter.set_log_number_ref(w->log_ref);
    w->status = w->batch->Iterate(&inserter);
    if (!w->status.ok()) {
@@ -60,7 +60,7 @@ static std::string PrintContents(WriteBatch* b) {
    }
    for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
      ParsedInternalKey ikey;
-      ikey.clear();
+      memset((void*)&ikey, 0, sizeof(ikey));
      EXPECT_TRUE(ParseInternalKey(iter->key(), &ikey));
      switch (ikey.type) {
        case kTypeValue:
@@ -451,20 +451,20 @@ TEST_F(WriteBatchTest, DISABLED_ManyUpdates) {
    }
    virtual Status DeleteCF(uint32_t column_family_id,
                            const Slice& key) override {
-      ADD_FAILURE();
+      EXPECT_TRUE(false);
      return Status::OK();
    }
    virtual Status SingleDeleteCF(uint32_t column_family_id,
                                  const Slice& key) override {
-      ADD_FAILURE();
+      EXPECT_TRUE(false);
      return Status::OK();
    }
    virtual Status MergeCF(uint32_t column_family_id, const Slice& key,
                           const Slice& value) override {
-      ADD_FAILURE();
+      EXPECT_TRUE(false);
      return Status::OK();
    }
-    virtual void LogData(const Slice& blob) override { ADD_FAILURE(); }
+    virtual void LogData(const Slice& blob) override { EXPECT_TRUE(false); }
    virtual bool Continue() override { return num_seen < kNumUpdates; }
  } handler;

@@ -502,20 +502,20 @@ TEST_F(WriteBatchTest, DISABLED_LargeKeyValue) {
    }
    virtual Status DeleteCF(uint32_t column_family_id,
                            const Slice& key) override {
-      ADD_FAILURE();
+      EXPECT_TRUE(false);
      return Status::OK();
    }
    virtual Status SingleDeleteCF(uint32_t column_family_id,
                                  const Slice& key) override {
-      ADD_FAILURE();
+      EXPECT_TRUE(false);
      return Status::OK();
    }
    virtual Status MergeCF(uint32_t column_family_id, const Slice& key,
                           const Slice& value) override {
-      ADD_FAILURE();
+      EXPECT_TRUE(false);
      return Status::OK();
    }
-    virtual void LogData(const Slice& blob) override { ADD_FAILURE(); }
+    virtual void LogData(const Slice& blob) override { EXPECT_TRUE(false); }
    virtual bool Continue() override { return num_seen < 2; }
  } handler;

@@ -16,6 +16,7 @@
 #include "rocksdb/db.h"
 #include "rocksdb/write_batch.h"
 #include "port/port.h"
+#include "util/logging.h"
 #include "util/random.h"
 #include "util/sync_point.h"
 #include "util/testharness.h"
@@ -106,10 +107,6 @@ TEST_F(WriteCallbackTest, WriteWithCallbackTest) {
    std::vector<std::pair<string, string>> kvs_;
  };

-  // In each scenario we'll launch multiple threads to write.
-  // The size of each array equals to number of threads, and
-  // each boolean in it denote whether callback of corresponding
-  // thread should succeed or fail.
  std::vector<std::vector<WriteOP>> write_scenarios = {
      {true},
      {false},
@@ -137,6 +134,11 @@ TEST_F(WriteCallbackTest, WriteWithCallbackTest) {
              options.allow_concurrent_memtable_write = allow_parallel;
              options.enable_pipelined_write = enable_pipelined_write;
              options.concurrent_prepare = two_queues;
+              if (options.enable_pipelined_write &&
+                  options.concurrent_prepare) {
+                // This combination is not supported
+                continue;
+              }

              ReadOptions read_options;
              DB* db;
@@ -148,37 +150,23 @@ TEST_F(WriteCallbackTest, WriteWithCallbackTest) {
              db_impl = dynamic_cast<DBImpl*>(db);
              ASSERT_TRUE(db_impl);

-              // Writers that have called JoinBatchGroup.
-              std::atomic<uint64_t> threads_joining(0);
-              // Writers that have linked to the queue
-              std::atomic<uint64_t> threads_linked(0);
-              // Writers that pass WriteThread::JoinBatchGroup:Wait sync-point.
-              std::atomic<uint64_t> threads_verified(0);
-
+              std::atomic<uint64_t> threads_waiting(0);
              std::atomic<uint64_t> seq(db_impl->GetLatestSequenceNumber());
              ASSERT_EQ(db_impl->GetLatestSequenceNumber(), 0);

-              rocksdb::SyncPoint::GetInstance()->SetCallBack(
-                  "WriteThread::JoinBatchGroup:Start", [&](void*) {
-                    uint64_t cur_threads_joining = threads_joining.fetch_add(1);
-                    // Wait for the last joined writer to link to the queue.
-                    // In this way the writers link to the queue one by one.
-                    // This allows us to confidently detect the first writer
-                    // who increases threads_linked as the leader.
-                    while (threads_linked.load() < cur_threads_joining) {
-                    }
-                  });
-
-              // Verification once writers call JoinBatchGroup.
              rocksdb::SyncPoint::GetInstance()->SetCallBack(
                  "WriteThread::JoinBatchGroup:Wait", [&](void* arg) {
-                    uint64_t cur_threads_linked = threads_linked.fetch_add(1);
+                    uint64_t cur_threads_waiting = 0;
                    bool is_leader = false;
                    bool is_last = false;

                    // who am i
-                    is_leader = (cur_threads_linked == 0);
-                    is_last = (cur_threads_linked == write_group.size() - 1);
+                    do {
+                      cur_threads_waiting = threads_waiting.load();
+                      is_leader = (cur_threads_waiting == 0);
+                      is_last = (cur_threads_waiting == write_group.size() - 1);
+                    } while (!threads_waiting.compare_exchange_strong(
+                        cur_threads_waiting, cur_threads_waiting + 1));

                    // check my state
                    auto* writer = reinterpret_cast<WriteThread::Writer*>(arg);
@@ -202,10 +190,8 @@ TEST_F(WriteCallbackTest, WriteWithCallbackTest) {
                                  !write_group.back().callback_.should_fail_);
                    }

-                    threads_verified.fetch_add(1);
-                    // Wait here until all verification in this sync-point
-                    // callback finish for all writers.
-                    while (threads_verified.load() < write_group.size()) {
+                    // wait for friends
+                    while (threads_waiting.load() < write_group.size()) {
                    }
                  });

@@ -230,20 +216,17 @@ TEST_F(WriteCallbackTest, WriteWithCallbackTest) {

              std::atomic<uint32_t> thread_num(0);
              std::atomic<char> dummy_key(0);
-
-              // Each write thread create a random write batch and write to DB
-              // with a write callback.
              std::function<void()> write_with_callback_func = [&]() {
                uint32_t i = thread_num.fetch_add(1);
                Random rnd(i);

                // leaders gotta lead
-                while (i > 0 && threads_verified.load() < 1) {
+                while (i > 0 && threads_waiting.load() < 1) {
                }

                // loser has to lose
                while (i == write_group.size() - 1 &&
-                       threads_verified.load() < write_group.size() - 1) {
+                       threads_waiting.load() < write_group.size() - 1) {
                }

                auto& write_op = write_group.at(i);
@@ -253,7 +236,11 @@ TEST_F(WriteCallbackTest, WriteWithCallbackTest) {
                // insert some keys
                for (uint32_t j = 0; j < rnd.Next() % 50; j++) {
                  // grab unique key
-                  char my_key = dummy_key.fetch_add(1);
+                  char my_key = 0;
+                  do {
+                    my_key = dummy_key.load();
+                  } while (
+                      !dummy_key.compare_exchange_strong(my_key, my_key + 1));

                  string skey(5, my_key);
                  string sval(10, my_key);
Author	SHA1	Message	Date
Daniel Black	ede73d650c	table/block.h: change memset Summary: In gcc-7 the following is an error identified by -Werror=class-memaccess In file included from ./table/get_context.h:14:0, from db/version_set.cc:43: ./table/block.h: In constructor ‘rocksdb::BlockReadAmpBitmap::BlockReadAmpBitmap(size_t, size_t, rocksdb::Statistics)’: ./table/block.h:73:53: error: ‘void memset(void, int, size_t)’ clearing an object of type ‘struct std::atomic<unsigned int>’ with no trivial copy-assignment; use value-initialization instead [-Werror=class-memaccess] memset(bitmap_, 0, bitmap_size kBytesPersEntry); ^ In file included from ./db/version_set.h:23:0, from db/version_set.cc:12: /toolchain/include/c++/8.0.0/atomic:684:12: note: ‘struct std::atomic<unsigned int>’ declared here struct atomic<unsigned int> : __atomic_base<unsigned int> ^~~~~~~~~~~~~~~~~~~~ As a solution the default initializer can be applied in list context. Signed-off-by: Daniel Black <daniel.black@au.ibm.com> Closes https://github.com/facebook/rocksdb/pull/2561 Differential Revision: D5398714 Pulled By: siying fbshipit-source-id: d883fb88ec7535eee60d551038fe91f14488be36	2019-10-30 14:00:55 -07:00
przemyslaw.skibinski@percona.com	531d1186c4	Fix GitHub issue #3716 : gcc-8 warnings Summary: Fix the following gcc-8 warnings: - conflicting C language linkage declaration [-Werror] - writing to an object with no trivial copy-assignment [-Werror=class-memaccess] - array subscript -1 is below array bounds [-Werror=array-bounds] Solves https://github.com/facebook/rocksdb/issues/3716 Closes https://github.com/facebook/rocksdb/pull/3736 Differential Revision: D7684161 Pulled By: yiwu-arbug fbshipit-source-id: 47c0423d26b74add251f1d3595211eee1e41e54a	2019-10-30 13:54:49 -07:00
Vijay Nadimpalli	c8da4be8f5	Making platform 007 (gcc 7) default in build_detect_platform.sh (#5947 ) Summary: Making platform 007 (gcc 7) default in build_detect_platform.sh. Pull Request resolved: https://github.com/facebook/rocksdb/pull/5947 Differential Revision: D18038837 Pulled By: vjnadimpalli fbshipit-source-id: 9ac2ddaa93bf328a416faec028970e039886378e	2019-10-30 13:54:49 -07:00
Andrew Kryczka	90a87bbf61	Add latest toolchain (gcc-8, etc.) build support for fbcode users (#4923 ) Summary: - When building with internal dependencies, specify this toolchain by setting `ROCKSDB_FBCODE_BUILD_WITH_PLATFORM007=1` - It is not enabled by default. However, it is enabled for TSAN builds in CI since there is a known problem with TSAN in gcc-5: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=71090 - I did not add support for Lua since (1) we agreed to deprecate it, and (2) we only have an internal build for v5.3 with this toolchain while that has breaking changes compared to our current version (v5.2). Pull Request resolved: https://github.com/facebook/rocksdb/pull/4923 Differential Revision: D13827226 Pulled By: ajkr fbshipit-source-id: 9aa3388ed3679777cfb15ef8cbcb83c07f62f947	2019-10-30 13:54:49 -07:00
Yi Wu	359e48333d	Bump version to 5.7.5	2017-11-28 12:25:25 -08:00
Yi Wu	30da984d07	Fix IOError on WAL write doesn't propagate to write group follower Summary: This is a simpler version of #3097 by removing all unrelated changes. Fixing the bug where concurrent writes may get Status::OK while it actually gets IOError on WAL write. This happens when multiple writes form a write batch group, and the leader get an IOError while writing to WAL. The leader failed to pass the error to followers in the group, and the followers end up returning Status::OK() while actually writing nothing. The bug only affect writes in a batch group. Future writes after the batch group will correctly return immediately with the IOError. Closes https://github.com/facebook/rocksdb/pull/3201 Differential Revision: D6421644 Pulled By: yiwu-arbug fbshipit-source-id: 1c2a455c5b73f6842423785eb8a9dbfbb191dc0e	2017-11-28 12:24:28 -08:00
Maysam Yabandeh	e3171c89c4	Exclude incompatible options in test Summary: options.enable_pipelined_write and options.concurrent_prepare are incompatible and should not be set together. Closes https://github.com/facebook/rocksdb/pull/2875 Differential Revision: D5818358 Pulled By: maysamyabandeh fbshipit-source-id: dad862508f00817ab302f8b61729accf38315fb8	2017-10-26 16:58:13 -07:00
gladiator	dcdf073654	fix missing manual_wal_flush for DBOptions ctor Summary: currently `ImmutableDBOptions::Dump` use default value for `concurrent_prepare` and `manual_wal_flush`, because DBOptions ctor does not init those member variables. so in LOG file, it will be ``` Options.concurrent_prepare: 0 Options.manual_wal_flush: 0 ``` Closes https://github.com/facebook/rocksdb/pull/2864 Differential Revision: D5816240 Pulled By: ajkr fbshipit-source-id: 82335e8bcae3dceedc6a99224e7998de5fad1e50	2017-09-14 15:37:51 -07:00
Yi Wu	096593bc60	Bump version to 5.7.4	2017-08-31 14:27:13 -07:00
Yi Wu	a03ae04493	make blob file close synchronous Summary: Fixing flaky blob_db_test. To close a blob file, blob db used to add a CloseSeqWrite job to the background thread to close it. Changing file close to be synchronous in order to simplify logic, and fix flaky blob_db_test. Closes https://github.com/facebook/rocksdb/pull/2787 Differential Revision: D5699387 Pulled By: yiwu-arbug fbshipit-source-id: dd07a945cd435cd3808fce7ee4ea57817409474a	2017-08-31 14:22:10 -07:00
yiwu-arbug	766fb69ec1	Blob db create a snapshot before every read Summary: If GC kicks in between * A Get() reads index entry from base db. * The Get() read from a blob file The GC can delete the corresponding blob file, making the key not found. Fortunately we have existing logic to avoid deleting a blob file if it is referenced by a snapshot. So the fix is to explicitly create a snapshot before reading index entry from base db. Closes https://github.com/facebook/rocksdb/pull/2754 Differential Revision: D5655956 Pulled By: yiwu-arbug fbshipit-source-id: e4ccbc51331362542e7343175bbcbdea5830f544	2017-08-31 14:22:05 -07:00
yiwu-arbug	e7de994745	GC the oldest file when out of space Summary: When out of space, blob db should GC the oldest file. The current implementation GC the newest one instead. Fixing it. Closes https://github.com/facebook/rocksdb/pull/2757 Differential Revision: D5657611 Pulled By: yiwu-arbug fbshipit-source-id: 56c30a4c52e6ab04551dda8c5c46006d4070b28d	2017-08-31 14:21:31 -07:00
yiwu-arbug	a4c3406242	Fix blob db crash during calculating write amp Summary: On initial call to BlobDBImpl::WaStats() `all_periods_write_` would be empty, so it will crash when we call pop_front() at line 1627. Apparently it is mean to pop only when `all_periods_write_.size() > kWriteAmplificationStatsPeriods`. The whole write amp calculation doesn't seems to be correct and it is not being exposed. Will work on it later. Test Plan Change kWriteAmplificationStatsPeriodMillisecs to 1000 (1 second) and run db_bench --use_blob_db for 5 minutes. Closes https://github.com/facebook/rocksdb/pull/2751 Differential Revision: D5648269 Pulled By: yiwu-arbug fbshipit-source-id: b843d9a09bb5f9e1b713d101ec7b87e54b5115a4	2017-08-31 14:21:01 -07:00
yiwu-arbug	672efad99a	Fix blob DB transaction usage while GC Summary: While GC, blob DB use optimistic transaction to delete or replace the index entry in LSM, to guarantee correctness if there's a normal write writing to the same key. However, the previous implementation doesn't call SetSnapshot() nor use GetForUpdate() of transaction API, instead it do its own sequence number checking before beginning the transaction. A normal write can sneak in after the sequence number check and overwrite the key, and the GC will delete or relocate the old version of the key by mistake. Update the code to property use GetForUpdate() to check the existing index entry. After the patch the sequence number store with each blob record is useless, So I'm considering remove the sequence number from blob record, in another patch. Closes https://github.com/facebook/rocksdb/pull/2703 Differential Revision: D5589178 Pulled By: yiwu-arbug fbshipit-source-id: 8dc960cd5f4e61b36024ba7c32d05584ce149c24	2017-08-31 14:20:55 -07:00
Yi Wu	b024e77f49	Avoid blob db call Sync() while writing Summary: The FsyncFiles background job call Fsync() periodically for blob files. However it can access WritableFileWriter concurrently with a Put() or Write(). And WritableFileWriter does not support concurrent access. It will lead to WritableFileWriter buffer being flush with same content twice, and blob file end up corrupted. Fixing by simply let FsyncFiles hold write_mutex_. Closes https://github.com/facebook/rocksdb/pull/2685 Differential Revision: D5561908 Pulled By: yiwu-arbug fbshipit-source-id: f0bb5bcab0e05694e053b8c49eab43640721e872	2017-08-31 14:17:33 -07:00
Yi Wu	6cc403ab71	Update all blob db TTL and timestamps to uint64_t Summary: The current blob db implementation use mix of int32_t, uint32_t and uint64_t for TTL and expiration. Update all timestamps to uint64_t for consistency. Closes https://github.com/facebook/rocksdb/pull/2683 Differential Revision: D5557103 Pulled By: yiwu-arbug fbshipit-source-id: e4eab2691629a755e614e8cf1eed9c3a681d0c42	2017-08-31 14:17:33 -07:00
Yi Wu	1339b299bc	Allow concurrent writes to blob db Summary: I'm going with brute-force solution, just letting Put() and Write() holding a mutex before writing. May improve concurrent writing with finer granularity locking later. Closes https://github.com/facebook/rocksdb/pull/2682 Differential Revision: D5552690 Pulled By: yiwu-arbug fbshipit-source-id: 039abd675b5d274a7af6428198d1733cafecef4c	2017-08-31 14:17:33 -07:00
Yi Wu	1b33ee8e7b	Blob DB garbage collection should keep keys with newer version Summary: Fix the bug where if blob db garbage collection revmoe keys with newer version. It shouldn't delete the key from base db when sequence number in base db is not equal to the one in blob log. Closes https://github.com/facebook/rocksdb/pull/2678 Differential Revision: D5549752 Pulled By: yiwu-arbug fbshipit-source-id: abb8649260963b5c389748023970fd746279d227	2017-08-31 14:16:43 -07:00
Yi Wu	b14207c0c5	Dump Blob DB options to info log Summary: * Dump blob db options to info log * Remove BlobDBOptionsImpl to disallow dynamic cast BlobDBOptions into BlobDBOptionsImpl. Move options there to be constants or into BlobDBOptions. The dynamic cast is broken after #2645 * Change some of the default options * Remove blob_db_options.min_blob_size, which is unimplemented. Will implement it soon. Closes https://github.com/facebook/rocksdb/pull/2671 Differential Revision: D5529912 Pulled By: yiwu-arbug fbshipit-source-id: dcd58ca981db5bcc7f123b65a0d6f6ae0dc703c7	2017-08-31 14:03:03 -07:00
Andrew Kryczka	acef93ed65	Bump version to 5.7.3	2017-08-29 23:08:32 -07:00
Andrew Kryczka	cf38a47a64	update HISTORY.md for DeleteRange bug fix Summary: fixed in #2799 Closes https://github.com/facebook/rocksdb/pull/2805 Differential Revision: D5734324 Pulled By: ajkr fbshipit-source-id: a285d4e84bf1018dc2257fd6c3e7c075a7243263	2017-08-29 23:05:48 -07:00
Huachao Huang	b722a18f67	Fix wrong smallest key of delete range tombstones Summary: Since tombstones are not stored in order, we may get a wrong smallest key if we only consider the first added tombstone. Check https://github.com/facebook/rocksdb/issues/2752 for more details. Closes https://github.com/facebook/rocksdb/pull/2799 Differential Revision: D5728217 Pulled By: ajkr fbshipit-source-id: 4a53edb0ca80d2a9fcf10749e52d47d57d6417d3	2017-08-29 23:04:42 -07:00
Andrew Kryczka	2b81d372eb	fix deleterange with memtable prefix bloom Summary: the range delete tombstones in memtable should be added to the aggregator even when the memtable's prefix bloom filter tells us the lookup key's not there. This bug could cause data to temporarily reappear until the memtable containing range deletions is flushed. Reported in #2743. Closes https://github.com/facebook/rocksdb/pull/2745 Differential Revision: D5639007 Pulled By: ajkr fbshipit-source-id: 04fc6facb6f978340a3f639536f4ca7c0d73dfc9	2017-08-29 23:02:34 -07:00
Yi Wu	cfaeb5846b	Bump version to 5.7.2	2017-08-15 14:19:55 -07:00
Siying Dong	62e9418a70	Fix false removal of tombstone issue in FIFO and kCompactionStyleNone Summary: Similar to the bug fixed by https://github.com/facebook/rocksdb/pull/2726, FIFO with compaction and kCompactionStyleNone during user customized CompactFiles() with output level to be 0 can suffer from the same problem. Fix it by leveraging the bottommost_level_ flag. Closes https://github.com/facebook/rocksdb/pull/2735 Differential Revision: D5626906 Pulled By: siying fbshipit-source-id: 2b148d0461c61dbd986d74655e384419ae442158	2017-08-15 14:12:19 -07:00
lxcode	03df818cff	Fix missing stdlib include required for abort() Summary: If ROCKSDB_LITE is defined, a call to abort() is introduced. This call requires stdlib.h. Build log of unpatched 5.7.1: http://beefy9.nyi.freebsd.org/data/110amd64-default/447974/logs/rocksdb-lite-5.7.1.log Closes https://github.com/facebook/rocksdb/pull/2744 Reviewed By: yiwu-arbug Differential Revision: D5632372 Pulled By: lxcode fbshipit-source-id: b2a8e692bf14ccf1f875f3a00463e87bba310a2b	2017-08-15 14:08:44 -07:00
Andrew Kryczka	96a092b1a3	empty commit	2017-08-14 10:27:56 -07:00
Andrew Kryczka	d59fdc3946	increment patch version for 5.7.1	2017-08-11 19:09:12 -07:00
Andrew Kryczka	e96d06c816	fix deletion dropping in intra-L0 Summary: `KeyNotExistsBeyondOutputLevel` didn't consider L0 files' key-ranges. So if a key only was covered by older L0 files' key-ranges, we would incorrectly drop deletions of that key. This PR just skips the deletion-dropping optimization when output level is L0. Closes https://github.com/facebook/rocksdb/pull/2726 Differential Revision: D5617286 Pulled By: ajkr fbshipit-source-id: 4bff1396b06d49a828ba4542f249191052915bce	2017-08-11 19:07:16 -07:00
Yi Wu	b86f89f25b	Move blob_db/ttl_extractor.h into blob_db/blob_db.h Summary: Move blob_db/ttl_extractor.h into blob_db/blob_db.h Also exclude TTLExtractor from LITE build. Closes https://github.com/facebook/rocksdb/pull/2665 Differential Revision: D5520009 Pulled By: yiwu-arbug fbshipit-source-id: 4813dcc272c7cc4bf2cdac285256d9a17d78c7b7	2017-07-28 14:39:11 -07:00
Yi Wu	48cf039064	Blob DB TTL extractor Summary: Introducing blob_db::TTLExtractor to replace extract_ttl_fn. The TTL extractor can be use to extract TTL from keys insert with Put or WriteBatch. Change over existing extract_ttl_fn are: * If value is changed, it will be return via std::string* (rather than Slice). With Slice the new value has to be part of the existing value. With std::string* the limitation is removed. * It can optionally return TTL or expiration. Other changes in this PR: * replace `std::chrono::system_clock` with `Env::NowMicros` so that I can mock time in tests. * add several TTL tests. * other minor naming change. Closes https://github.com/facebook/rocksdb/pull/2659 Differential Revision: D5512627 Pulled By: yiwu-arbug fbshipit-source-id: 0dfcb00d74d060b8534c6130c808e4d5d0a54440	2017-07-28 09:41:52 -07:00
Yi Wu	c39939d468	Reduce blob db noisy logging Summary: Remove some of the per-key logging by blob db to reduce noise. Closes https://github.com/facebook/rocksdb/pull/2587 Differential Revision: D5429115 Pulled By: yiwu-arbug fbshipit-source-id: b89328282fb8b3c64923ce48738c16017ce7feaf	2017-07-28 09:40:33 -07:00
Siying Dong	65afd8a1cc	Update java/rocksjni.pom Summary: Closes https://github.com/facebook/rocksdb/pull/2610 Differential Revision: D5445705 Pulled By: siying fbshipit-source-id: d5f97ffdf5bc18b853c3a106755aca96a650e428	2017-07-26 11:31:58 -07:00
Siying Dong	6df239cfee	Remove some left-over BSD headers Summary: Closes https://github.com/facebook/rocksdb/pull/2608 Differential Revision: D5444797 Pulled By: siying fbshipit-source-id: 690581d03f37822e059a16085088e8e2d8a45016	2017-07-26 11:31:42 -07:00
Siying Dong	54c4b3ab4f	Add back the LevelDB license file Summary: Closes https://github.com/facebook/rocksdb/pull/2591 Differential Revision: D5432696 Pulled By: siying fbshipit-source-id: a613230ab916de0b279a65ef429ede65460a8db2	2017-07-26 11:31:29 -07:00
Siying Dong	27768b0ec3	Remove the licensing description in CONTRIBUTING.md Summary: Closes https://github.com/facebook/rocksdb/pull/2590 Differential Revision: D5432539 Pulled By: siying fbshipit-source-id: 49902453bba3c95c1fb8354441b2198649e52bf4	2017-07-26 11:31:18 -07:00
Siying Dong	8673a8c567	Change RocksDB License Summary: Closes https://github.com/facebook/rocksdb/pull/2589 Differential Revision: D5431502 Pulled By: siying fbshipit-source-id: 8ebf8c87883daa9daa54b2303d11ce01ab1f6f75	2017-07-26 11:31:01 -07:00
Maysam Yabandeh	6124420d5d	Remove the orphan assert on !need_log_sync Summary: We initially had disabled support for write_options.sync when concurrent_prepare_ is set. We later added this support but the statement that asserts this combination is not used was left there. This patch cleans it up. Closes https://github.com/facebook/rocksdb/pull/2642 Differential Revision: D5496101 Pulled By: maysamyabandeh fbshipit-source-id: becbc503446f2a51bee24cc861958c090c724ec2	2017-07-26 07:31:36 -07:00
Maysam Yabandeh	258fd9ac46	Release note for partitioned index/filters Summary: Closes https://github.com/facebook/rocksdb/pull/2637 Differential Revision: D5489751 Pulled By: maysamyabandeh fbshipit-source-id: 0298f8960d4f86ce67959616615beee4d802c2e4	2017-07-25 11:20:15 -07:00