Disable error as warning

Add one more #include<functional>
Add some include<functional>
2026-11-07 11:32:34 -05:00 · 2019-11-05 11:04:14 -08:00 · 2019-11-05 11:04:14 -08:00 · 2019-10-31 14:28:16 -07:00 · 2019-10-31 14:28:08 -07:00 · 2017-07-17 22:22:16 -07:00
925 changed files with 31977 additions and 115561 deletions
@@ -1,6 +1,6 @@
 {
  "project_id" : "rocksdb",
-  "conduit_uri" : "https://phabricator.fb.com/api/",
+  "conduit_uri" : "https://reviews.facebook.net/",
  "copyright_holder" : "Facebook",
  "load" : [
    "arcanist_util"
@@ -1,3 +1,4 @@
+TARGETS
 make_config.mk

 *.a
@@ -31,12 +32,10 @@ build/
 ldb
 manifest_dump
 sst_dump
-column_aware_encoding_exp
 util/build_version.cc
 build_tools/VALGRIND_LOGS/
 coverage/COVERAGE_REPORT
 .gdbhistory
-.gdb_history
 package/
 .phutil_module_cache
 unity.a
@@ -65,9 +64,3 @@ java/javadoc
 scan_build_report/
 t
 LOG
-
-db_logs/
-tp2/
-fbcode/
-fbcode
-buckifier/*.pyc
@@ -1,61 +1,45 @@
 sudo: false
-dist: trusty
 language: cpp
 os:
  - linux
  - osx
 compiler:
  - clang
-  - gcc
-jdk:
-  - oraclejdk7
-cache:
-  - ccache
-  - apt

 addons:
   apt:
      sources: ['ubuntu-toolchain-r-test', 'llvm-toolchain-precise-3.6']
-      packages: ['clang-3.6' , 'g++-6', 'zlib1g-dev', 'libbz2-dev', 'libsnappy-dev', 'curl', 'libgflags-dev']
+      packages: ['clang-3.6' , 'zlib1g-dev', 'libbz2-dev', 'libsnappy-dev', 'curl']
 env:
-  - TEST_GROUP=platform_dependent1
-  - TEST_GROUP=platform_dependent2
-  - TEST_GROUP=platform_dependent3
-  - TEST_GROUP=1
-  - TEST_GROUP=2
+  # Run all tests before db_block_cache_test (db_test, db_test2)
+  - JOB_NAME=unittests ROCKSDBTESTS_END=db_block_cache_test
+  # Run all tests starting from db_block_cache_test (db_block_cache_test, db_iter_test, ...)
+  - JOB_NAME=unittests ROCKSDBTESTS_START=db_block_cache_test
  # Run java tests
  - JOB_NAME=java_test
  # Build ROCKSDB_LITE
  - JOB_NAME=lite_build
-  # Build examples
-  - JOB_NAME=examples

-matrix:
-  exclude:
-  - os: osx
-    compiler: gcc
-  - os: osx
-    env: TEST_GROUP=1
-  - os: osx
-    env: TEST_GROUP=2
+install:
+  # Build gflags
+  # TODO(noetzli): Remove when gflags available through Travis
+  - pushd /tmp/ && curl -L https://github.com/gflags/gflags/archive/v2.1.2.tar.gz -o gflags.tar.gz && tar xfz gflags.tar.gz && cd gflags-2.1.2 && cmake . && make && popd

 before_script:
-  - if [[ "${TRAVIS_OS_NAME}" == 'linux' && "${CXX}" == 'clang++' ]]; then CXX=clang++-3.6; fi
-  # test one linux g++ build with g++-6
-  - if [[ "${TRAVIS_OS_NAME}" == 'linux' && "${CXX}" == 'g++' && "${JOB_NAME}" == 'unittests' ]]; then CXX=g++-6; fi
-  # Limit the maximum number of open file descriptors to 8192
-  - ulimit -n 8192 || true
+  # Add gflags to include/library paths
+  # TODO(noetzli): Remove when gflags available through Travis
+  - export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/tmp/gflags-2.1.2/lib"
+  - export LIBRARY_PATH="$LIBRARY_PATH:/tmp/gflags-2.1.2/lib"
+  - export CPLUS_INCLUDE_PATH="$CPLUS_INCLUDE_PATH:/tmp/gflags-2.1.2/include"
+  - if [[ "${TRAVIS_OS_NAME}" == 'linux' ]]; then CXX=clang++-3.6; fi
+  - if [[ "${TRAVIS_OS_NAME}" == 'osx' ]]; then brew install gflags snappy; fi
+  # Limit the maximum number of open file descriptors to 2000
+  - ulimit -n 2000 || true

 script:
-  - ${CXX} --version
-  - if [[ "${TEST_GROUP}" == 'platform_dependent1' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_END=external_sst_file_basic_test make -j4 check_some; fi
-  - if [[ "${TEST_GROUP}" == 'platform_dependent2' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=external_sst_file_basic_test ROCKSDBTESTS_END=checkpoint_test make -j4 check_some; fi
-  - if [[ "${TEST_GROUP}" == 'platform_dependent3' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=checkpoint_test ROCKSDBTESTS_END=db_block_cache_test make -j4 check_some; fi
-  - if [[ "${TEST_GROUP}" == '1' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=db_block_cache_test ROCKSDBTESTS_END=comparator_db_test make -j4 check_some; fi
-  - if [[ "${TEST_GROUP}" == '2' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=comparator_db_test make -j4 check_some; fi
+  - if [[ "${JOB_NAME}" == 'unittests' ]]; then OPT=-DTRAVIS V=1 make -j4 check_some; fi
  - if [[ "${JOB_NAME}" == 'java_test' ]]; then OPT=-DTRAVIS V=1 make clean jclean rocksdbjava jtest; fi
  - if [[ "${JOB_NAME}" == 'lite_build' ]]; then OPT="-DTRAVIS -DROCKSDB_LITE" V=1 make -j4 static_lib; fi
-  - if [[ "${JOB_NAME}" == 'examples' ]]; then OPT=-DTRAVIS V=1 make -j4 static_lib; cd examples; make -j4; fi
 notifications:
    email:
      - leveldb@fb.com
@@ -1,8 +1,7 @@
-# Prerequisites for Windows:
-#     This cmake build is for Windows 64-bit only.
+# This cmake build is for Windows 64-bit only.
 #
 # Prerequisites:
-#     You must have at least Visual Studio 2015 Update 3. Start the Developer Command Prompt window that is a part of Visual Studio installation.
+#     You must have Visual Studio 2013 Update 4 installed. Start the Developer Command Prompt window that is a part of Visual Studio installation.
 #     Run the build commands from within the Developer Command Prompt window to have paths to the compiler and runtime libraries set.
 #     You must have git.exe in your %PATH% environment variable.
 #
@@ -14,7 +13,8 @@
 #        cd build
 # 3. Run cmake to generate project files for Windows, add more options to enable required third-party libraries.
 #    See thirdparty.inc for more information.
-#        sample command: cmake -G "Visual Studio 14 Win64" -DGFLAGS=1 -DSNAPPY=1 -DJEMALLOC=1 -DJNI=1 ..
+#        sample command: cmake -G "Visual Studio 12 Win64" -DGFLAGS=1 -DSNAPPY=1 -DJEMALLOC=1 -DJNI=1 ..
+#        OR for VS Studio 15 cmake -G "Visual Studio 14 Win64" -DGFLAGS=1 -DSNAPPY=1 -DJEMALLOC=1 -DJNI=1 ..
 # 4. Then build the project in debug mode (you may want to add /m[:<N>] flag to run msbuild in <N> parallel threads
 #                                          or simply /m ot use all avail cores)
 #        msbuild rocksdb.sln
@@ -25,163 +25,41 @@
 # 5. And release mode (/m[:<N>] is also supported)
 #        msbuild rocksdb.sln /p:Configuration=Release
 #
-# Linux:
-#
-# 1. Install a recent toolchain such as devtoolset-3 if you're on a older distro. C++11 required.
-# 2. mkdir build; cd build
-# 3. cmake ..
-# 4. make -j

 cmake_minimum_required(VERSION 2.6)
 project(rocksdb)

-if(POLICY CMP0042)
-  cmake_policy(SET CMP0042 NEW)
-endif()
-
-list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/modules/")
-
-if(MSVC)
-  include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc)
-else()
-  option(WITH_JEMALLOC "build with JeMalloc" OFF)
-  if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
-    # FreeBSD has jemaloc as default malloc
-    # but it does not have all the jemalloc files in include/...
-    set(WITH_JEMALLOC ON)
-  else()
-    if(WITH_JEMALLOC)
-      find_package(JeMalloc REQUIRED)
-      add_definitions(-DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE)
-      include_directories(${JEMALLOC_INCLUDE_DIR})
-    endif()
-  endif() 
-  option(WITH_SNAPPY "build with SNAPPY" OFF)
-  if(WITH_SNAPPY)
-    find_package(snappy REQUIRED)
-    add_definitions(-DSNAPPY)
-    include_directories(${SNAPPY_INCLUDE_DIR})
-    list(APPEND THIRDPARTY_LIBS ${SNAPPY_LIBRARIES})
-  endif()
-endif()
-
-if(WIN32)
-  execute_process(COMMAND powershell -Command "Get-Date -format MM_dd_yyyy" OUTPUT_VARIABLE DATE)
-  execute_process(COMMAND powershell -Command "Get-Date -format HH:mm:ss" OUTPUT_VARIABLE TIME)
-  string(REGEX REPLACE "(..)_(..)_..(..).*" "\\1/\\2/\\3" DATE "${DATE}")
-  string(REGEX REPLACE "(..):(.....).*" " \\1:\\2" TIME "${TIME}")
-  set(GIT_DATE_TIME "${DATE} ${TIME}")
-else()
-  execute_process(COMMAND date "+%Y/%m/%d %H:%M:%S" OUTPUT_VARIABLE DATETIME)
-  string(REGEX REPLACE "\n" "" DATETIME ${DATETIME})
-  set(GIT_DATE_TIME "${DATETIME}")
-endif()
+include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc)
+execute_process(COMMAND powershell -Command "Get-Date -format MM_dd_yyyy" OUTPUT_VARIABLE DATE)
+execute_process(COMMAND powershell -Command "Get-Date -format HH:mm:ss" OUTPUT_VARIABLE TIME)
+string(REGEX REPLACE "(..)_(..)_..(..).*" "\\1/\\2/\\3" DATE ${DATE})
+string(REGEX REPLACE "(..):(.....).*" " \\1:\\2" TIME ${TIME})
+string(CONCAT GIT_DATE_TIME ${DATE} ${TIME})

 find_package(Git)

-if(GIT_FOUND AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git")
-  if(WIN32)
+if (GIT_FOUND AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git")
    execute_process(COMMAND $ENV{COMSPEC} /C ${GIT_EXECUTABLE} -C ${CMAKE_CURRENT_SOURCE_DIR} rev-parse HEAD OUTPUT_VARIABLE GIT_SHA)
-  else()
-    execute_process(COMMAND ${GIT_EXECUTABLE} -C ${CMAKE_CURRENT_SOURCE_DIR} rev-parse HEAD OUTPUT_VARIABLE GIT_SHA)
-  endif()
 else()
-  set(GIT_SHA 0)
+    set(GIT_SHA 0)
 endif()

-string(REGEX REPLACE "[^0-9a-f]+" "" GIT_SHA "${GIT_SHA}")
+string(REGEX REPLACE "[^0-9a-f]+" "" GIT_SHA ${GIT_SHA})

-if(NOT WIN32)
-  execute_process(COMMAND
-      "./build_tools/version.sh" "full"
-      WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
-      OUTPUT_VARIABLE ROCKSDB_VERSION
-  )
-  string(STRIP "${ROCKSDB_VERSION}" ROCKSDB_VERSION)
-  execute_process(COMMAND
-      "./build_tools/version.sh" "major"
-      WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
-      OUTPUT_VARIABLE ROCKSDB_VERSION_MAJOR
-  )
-  string(STRIP "${ROCKSDB_VERSION_MAJOR}" ROCKSDB_VERSION_MAJOR)
-endif()
+set(BUILD_VERSION_CC ${CMAKE_CURRENT_SOURCE_DIR}/util/build_version.cc)

-if(WIN32)
-  option(WITH_AVX2 "build with AVX2" ON)
-  if(WITH_AVX2)
-    if(MSVC)
-      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2")
-    else()
-      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2")
-    endif()
-  endif()
-endif()
+add_custom_command(OUTPUT ${BUILD_VERSION_CC}
+    COMMAND echo "#include \"build_version.h\"" > ${BUILD_VERSION_CC}
+    COMMAND echo "const char* rocksdb_build_git_sha = \"rocksdb_build_git_sha:${GIT_SHA}\";" >> ${BUILD_VERSION_CC}
+    COMMAND echo "const char* rocksdb_build_git_datetime = \"rocksdb_build_git_datetime:${GIT_DATE_TIME}\";" >> ${BUILD_VERSION_CC}
+    COMMAND echo const char* rocksdb_build_compile_date = __DATE__\; >> ${BUILD_VERSION_CC}
+)

-set(BUILD_VERSION_CC ${CMAKE_BINARY_DIR}/build_version.cc)
-configure_file(util/build_version.cc.in ${BUILD_VERSION_CC} @ONLY)
-add_library(build_version OBJECT ${BUILD_VERSION_CC})
-target_include_directories(build_version PRIVATE
-  ${CMAKE_CURRENT_SOURCE_DIR}/util)
-if(MSVC)
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W3 /wd4127 /wd4800 /wd4996 /wd4351")
-else()
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wshadow -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers")
-  if(MINGW)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format")
-  endif()
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
-  if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
-    include(CheckCXXCompilerFlag)
-    CHECK_CXX_COMPILER_FLAG("-momit-leaf-frame-pointer" HAVE_OMIT_LEAF_FRAME_POINTER)
-    if(HAVE_OMIT_LEAF_FRAME_POINTER)
-      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -momit-leaf-frame-pointer")
-    endif()
-  endif()
-endif()
+add_custom_target(GenerateBuildVersion DEPENDS ${BUILD_VERSION_CC})

-option(FAIL_ON_WARNINGS "Treat compile warnings as errors" ON)
-if(FAIL_ON_WARNINGS)
-  if(MSVC)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /WX")
-  else() # assume GCC
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")
-  endif()
-endif()
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo  /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")

-option(WITH_ASAN "build with ASAN" OFF)
-if(WITH_ASAN)
-  add_definitions(-DROCKSDB_TSAN_RUN)
-  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address")
-  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address")
-  if(WITH_JEMALLOC)
-    message(FATAL "ASAN does not work well with JeMalloc")
-  endif()
-endif()
-
-option(WITH_TSAN "build with TSAN" OFF)
-if(WITH_TSAN)
-  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread -pie")
-  add_definitions(-DROCKSDB_TSAN_RUN)
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread -fPIC")
-  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=thread -fPIC")
-  if(WITH_JEMALLOC)
-    message(FATAL "TSAN does not work well with JeMalloc")
-  endif()
-endif()
-
-option(WITH_UBSAN "build with UBSAN" OFF)
-if(WITH_UBSAN)
-  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined")
-  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=undefined")
-  if(WITH_JEMALLOC)
-    message(FATAL "UBSAN does not work well with JeMalloc")
-  endif()
-endif()
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W3 /WX /wd4127 /wd4800 /wd4996 /wd4351")

 # Used to run CI build and tests so we can run faster
 set(OPTIMIZE_DEBUG_DEFAULT 0)        # Debug build is unoptimized by default use -DOPTDBG=1 to optimize
@@ -192,99 +70,35 @@ else()
   set(OPTIMIZE_DEBUG ${OPTIMIZE_DEBUG_DEFAULT})
 endif()

-if(MSVC)
-  if((${OPTIMIZE_DEBUG} EQUAL 1))
-    message(STATUS "Debug optimization is enabled")
-    set(CMAKE_CXX_FLAGS_DEBUG "/Oxt /MDd")
-  else()
-    set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1 /Gm /MDd")
-  endif()
-  set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Oxt /Zp8 /Gm- /Gy /MD")
-
-  set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /DEBUG")
-  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG")
+if((${OPTIMIZE_DEBUG} EQUAL 1))
+   message(STATUS "Debug optimization is enabled")
+   set(CMAKE_CXX_FLAGS_DEBUG "/Oxt /MDd")
+else()
+   set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1 /Gm /MDd")
 endif()

-if(CMAKE_COMPILER_IS_GNUCXX)
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-builtin-memcmp")
-endif()
+set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Oxt /Zp8 /Gm- /Gy /MD")

-option(ROCKSDB_LITE "Build RocksDBLite version" OFF)
-if(ROCKSDB_LITE)
-  add_definitions(-DROCKSDB_LITE)
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-exceptions")
-endif()
+set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /DEBUG")
+set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG")

-if(CMAKE_SYSTEM_NAME MATCHES "Cygwin")
-  add_definitions(-fno-builtin-memcmp -DCYGWIN)
-elseif(CMAKE_SYSTEM_NAME MATCHES "Darwin")
-  add_definitions(-DOS_MACOSX)
-  if(CMAKE_SYSTEM_PROCESSOR MATCHES arm)
-    add_definitions(-DIOS_CROSS_COMPILE -DROCKSDB_LITE)
-    # no debug info for IOS, that will make our library big
-    add_definitions(-DNDEBUG)
-  endif()
-elseif(CMAKE_SYSTEM_NAME MATCHES "Linux")
-  add_definitions(-DOS_LINUX)
-elseif(CMAKE_SYSTEM_NAME MATCHES "SunOS")
-  add_definitions(-DOS_SOLARIS)
-elseif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
-  add_definitions(-DOS_FREEBSD)
-elseif(CMAKE_SYSTEM_NAME MATCHES "NetBSD")
-  add_definitions(-DOS_NETBSD)
-elseif(CMAKE_SYSTEM_NAME MATCHES "OpenBSD")
-  add_definitions(-DOS_OPENBSD)
-elseif(CMAKE_SYSTEM_NAME MATCHES "DragonFly")
-  add_definitions(-DOS_DRAGONFLYBSD)
-elseif(CMAKE_SYSTEM_NAME MATCHES "Android")
-  add_definitions(-DOS_ANDROID)
-elseif(CMAKE_SYSTEM_NAME MATCHES "Windows")
-  add_definitions(-DWIN32 -DOS_WIN -D_MBCS -DWIN64 -DNOMINMAX)
-  if(MINGW)
-    add_definitions(-D_WIN32_WINNT=_WIN32_WINNT_VISTA)
-  endif()
-endif()
-
-if(NOT WIN32)
-  add_definitions(-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX)
-endif()
-
-option(WITH_FALLOCATE "build with fallocate" ON)
-
-if(WITH_FALLOCATE)
-  include(CheckCSourceCompiles)
-  CHECK_C_SOURCE_COMPILES("
-#include <fcntl.h>
-#include <linux/falloc.h>
-int main() {
- int fd = open(\"/dev/null\", 0);
- fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, 1024);
-}
-" HAVE_FALLOCATE)
-  if(HAVE_FALLOCATE)
-    add_definitions(-DROCKSDB_FALLOCATE_PRESENT)
-  endif()
-endif()
-
-include(CheckFunctionExists)
-CHECK_FUNCTION_EXISTS(malloc_usable_size HAVE_MALLOC_USABLE_SIZE)
-if(HAVE_MALLOC_USABLE_SIZE)
-  add_definitions(-DROCKSDB_MALLOC_USABLE_SIZE)
-endif()
+add_definitions(-DWIN32 -DOS_WIN -D_MBCS -DWIN64 -DNOMINMAX)

 include_directories(${PROJECT_SOURCE_DIR})
 include_directories(${PROJECT_SOURCE_DIR}/include)
-include_directories(SYSTEM ${PROJECT_SOURCE_DIR}/third-party/gtest-1.7.0/fused-src)
-find_package(Threads REQUIRED)
+include_directories(${PROJECT_SOURCE_DIR}/third-party/gtest-1.7.0/fused-src)
+
+set(ROCKSDB_LIBS rocksdblib${ARTIFACT_SUFFIX})
+set(THIRDPARTY_LIBS ${THIRDPARTY_LIBS} gtest)
+set(SYSTEM_LIBS ${SYSTEM_LIBS} Shlwapi.lib Rpcrt4.lib)
+
+set(LIBS ${ROCKSDB_LIBS} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})

 add_subdirectory(third-party/gtest-1.7.0/fused-src/gtest)

 # Main library source code
-
 set(SOURCES
-        cache/clock_cache.cc
-        cache/lru_cache.cc
-        cache/sharded_cache.cc
+        db/auto_roll_logger.cc
        db/builder.cc
        db/c.cc
        db/column_family.cc
@@ -293,23 +107,18 @@ set(SOURCES
        db/compaction_iterator.cc
        db/compaction_job.cc
        db/compaction_picker.cc
-        db/compaction_picker_universal.cc
        db/convenience.cc
+        db/dbformat.cc
        db/db_filesnapshot.cc
        db/db_impl.cc
-        db/db_impl_write.cc
-        db/db_impl_compaction_flush.cc
-        db/db_impl_files.cc
-        db/db_impl_open.cc
        db/db_impl_debug.cc
        db/db_impl_experimental.cc
        db/db_impl_readonly.cc
        db/db_info_dumper.cc
        db/db_iter.cc
-        db/dbformat.cc
        db/event_helpers.cc
        db/experimental.cc
-        db/external_sst_file_ingestion_job.cc
+        db/filename.cc
        db/file_indexer.cc
        db/flush_job.cc
        db/flush_scheduler.cc
@@ -319,11 +128,12 @@ set(SOURCES
        db/log_writer.cc
        db/managed_iterator.cc
        db/memtable.cc
+        db/memtable_allocator.cc
        db/memtable_list.cc
        db/merge_helper.cc
        db/merge_operator.cc
-        db/range_del_aggregator.cc
        db/repair.cc
+        db/slice.cc
        db/snapshot_impl.cc
        db/table_cache.cc
        db/table_properties_collector.cc
@@ -336,34 +146,17 @@ set(SOURCES
        db/write_batch_base.cc
        db/write_controller.cc
        db/write_thread.cc
-        env/env.cc
-        env/env_chroot.cc
-        env/env_hdfs.cc
-        env/memenv.cc
+        db/xfunc_test_points.cc
        memtable/hash_cuckoo_rep.cc
        memtable/hash_linklist_rep.cc
        memtable/hash_skiplist_rep.cc
-        memtable/memtable_allocator.cc
        memtable/skiplistrep.cc
        memtable/vectorrep.cc
-        monitoring/histogram.cc
-        monitoring/histogram_windowing.cc
-        monitoring/instrumented_mutex.cc
-        monitoring/iostats_context.cc
-        monitoring/perf_context.cc
-        monitoring/perf_level.cc
-        monitoring/statistics.cc
-        monitoring/thread_status_impl.cc
-        monitoring/thread_status_updater.cc
-        monitoring/thread_status_util.cc
-        monitoring/thread_status_util_debug.cc
-        options/cf_options.cc
-        options/db_options.cc
-        options/options.cc
-        options/options_helper.cc
-        options/options_parser.cc
-        options/options_sanity_check.cc
        port/stack_trace.cc
+        port/win/env_win.cc
+        port/win/port_win.cc
+        port/win/win_logger.cc
+        port/win/xpress_win.cc
        table/adaptive_table_factory.cc
        table/block.cc
        table/block_based_filter_block.cc
@@ -371,6 +164,7 @@ set(SOURCES
        table/block_based_table_factory.cc
        table/block_based_table_reader.cc
        table/block_builder.cc
+        table/block_hash_index.cc
        table/block_prefix_index.cc
        table/bloom_block.cc
        table/cuckoo_table_builder.cc
@@ -380,28 +174,24 @@ set(SOURCES
        table/format.cc
        table/full_filter_block.cc
        table/get_context.cc
-        table/index_builder.cc
        table/iterator.cc
-        table/merging_iterator.cc
+        table/merger.cc
+        table/sst_file_writer.cc
        table/meta_blocks.cc
-        table/partitioned_filter_block.cc
-        table/persistent_cache_helper.cc
        table/plain_table_builder.cc
        table/plain_table_factory.cc
        table/plain_table_index.cc
        table/plain_table_key_coding.cc
        table/plain_table_reader.cc
-        table/sst_file_writer.cc
        table/table_properties.cc
        table/two_level_iterator.cc
+        tools/sst_dump_tool.cc
        tools/db_bench_tool.cc
        tools/dump/db_dump_tool.cc
-        tools/ldb_cmd.cc
-        tools/ldb_tool.cc
-        tools/sst_dump_tool.cc
        util/arena.cc
-        util/auto_roll_logger.cc
        util/bloom.cc
+        util/build_version.cc
+        util/cache.cc
        util/coding.cc
        util/compaction_job_stats_impl.cc
        util/comparator.cc
@@ -409,206 +199,174 @@ set(SOURCES
        util/crc32c.cc
        util/delete_scheduler.cc
        util/dynamic_bloom.cc
+        util/env.cc
+        util/env_hdfs.cc
        util/event_logger.cc
-        util/file_reader_writer.cc
        util/file_util.cc
-        util/filename.cc
+        util/file_reader_writer.cc
+        util/sst_file_manager_impl.cc
        util/filter_policy.cc
        util/hash.cc
+        util/histogram.cc
+        util/histogram_windowing.cc
+        util/instrumented_mutex.cc
+        util/iostats_context.cc
+        tools/ldb_cmd.cc
+        tools/ldb_tool.cc
+        util/logging.cc
        util/log_buffer.cc
+        util/memenv.cc
        util/murmurhash.cc
+        util/mutable_cf_options.cc
+        util/options.cc
+        util/options_builder.cc
+        util/options_helper.cc
+        util/options_parser.cc
+        util/options_sanity_check.cc
+        util/perf_context.cc
+        util/perf_level.cc
        util/random.cc
        util/rate_limiter.cc
        util/slice.cc
-        util/sst_file_manager_impl.cc
+        util/statistics.cc
        util/status.cc
        util/status_message.cc
        util/string_util.cc
        util/sync_point.cc
+        util/testharness.cc
        util/testutil.cc
        util/thread_local.cc
-        util/threadpool_imp.cc
+        util/thread_status_impl.cc
+        util/thread_status_updater.cc
+        util/thread_status_util.cc
+        util/thread_status_util_debug.cc
        util/transaction_test_util.cc
+        util/xfunc.cc
        util/xxhash.cc
        utilities/backupable/backupable_db.cc
-        utilities/blob_db/blob_db.cc
-        utilities/blob_db/blob_db_impl.cc
-        utilities/blob_db/blob_db_options_impl.cc
-        utilities/blob_db/blob_file.cc
-        utilities/blob_db/blob_log_reader.cc
-        utilities/blob_db/blob_log_writer.cc
-        utilities/blob_db/blob_log_format.cc
        utilities/checkpoint/checkpoint.cc
-        utilities/col_buf_decoder.cc
-        utilities/col_buf_encoder.cc
-        utilities/column_aware_encoding_util.cc
        utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc
-        utilities/date_tiered/date_tiered_db_impl.cc
        utilities/document/document_db.cc
        utilities/document/json_document.cc
        utilities/document/json_document_builder.cc
        utilities/env_mirror.cc
-        utilities/env_timed.cc
+        utilities/flashcache/flashcache.cc
        utilities/geodb/geodb_impl.cc
        utilities/leveldb_options/leveldb_options.cc
-        utilities/lua/rocks_lua_compaction_filter.cc
        utilities/memory/memory_util.cc
-        utilities/merge_operators/max.cc
-        utilities/merge_operators/put.cc
        utilities/merge_operators/string_append/stringappend.cc
        utilities/merge_operators/string_append/stringappend2.cc
+        utilities/merge_operators/put.cc
        utilities/merge_operators/uint64add.cc
-        utilities/option_change_migration/option_change_migration.cc
        utilities/options/options_util.cc
-        utilities/persistent_cache/block_cache_tier.cc
-        utilities/persistent_cache/block_cache_tier_file.cc
-        utilities/persistent_cache/block_cache_tier_metadata.cc
-        utilities/persistent_cache/persistent_cache_tier.cc
-        utilities/persistent_cache/volatile_tier_impl.cc
        utilities/redis/redis_lists.cc
-        utilities/simulator_cache/sim_cache.cc
        utilities/spatialdb/spatial_db.cc
        utilities/table_properties_collectors/compact_on_deletion_collector.cc
-        utilities/transactions/optimistic_transaction_db_impl.cc
        utilities/transactions/optimistic_transaction_impl.cc
+        utilities/transactions/optimistic_transaction_db_impl.cc
        utilities/transactions/transaction_base.cc
+        utilities/transactions/transaction_impl.cc
        utilities/transactions/transaction_db_impl.cc
        utilities/transactions/transaction_db_mutex_impl.cc
-        utilities/transactions/transaction_impl.cc
        utilities/transactions/transaction_lock_mgr.cc
        utilities/transactions/transaction_util.cc
        utilities/ttl/db_ttl_impl.cc
        utilities/write_batch_with_index/write_batch_with_index.cc
        utilities/write_batch_with_index/write_batch_with_index_internal.cc
-        $<TARGET_OBJECTS:build_version>)
+)

-if(WIN32)
-  list(APPEND SOURCES
-    port/win/io_win.cc
-    port/win/env_win.cc
-    port/win/env_default.cc
-    port/win/port_win.cc
-    port/win/win_logger.cc
-    port/win/win_thread.cc
-    port/win/xpress_win.cc)
-else()
-  list(APPEND SOURCES
-    port/port_posix.cc
-    env/env_posix.cc
-    env/io_posix.cc)
-endif()
+# For test util library that is build only in DEBUG mode
+# and linked to tests. Add test only code that is not #ifdefed for Release here.
+set(TESTUTIL_SOURCE
+    db/db_test_util.cc
+    table/mock_table.cc
+    util/mock_env.cc
+    util/thread_status_updater_debug.cc
+)

-if(WIN32)
-  set(SYSTEM_LIBS ${SYSTEM_LIBS} Shlwapi.lib Rpcrt4.lib)
-  set(ROCKSDB_STATIC_LIB rocksdblib${ARTIFACT_SUFFIX})
-  set(ROCKSDB_IMPORT_LIB rocksdb${ARTIFACT_SUFFIX})
-  set(LIBS ${ROCKSDB_STATIC_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
-else()
-  set(SYSTEM_LIBS ${CMAKE_THREAD_LIBS_INIT} rt)
-  set(ROCKSDB_STATIC_LIB rocksdb${ARTIFACT_SUFFIX})
-  set(ROCKSDB_SHARED_LIB rocksdb-shared)
-  set(ROCKSDB_IMPORT_LIB ${ROCKSDB_SHARED_LIB})
-  set(LIBS ${ROCKSDB_SHARED_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
+add_library(rocksdblib${ARTIFACT_SUFFIX} ${SOURCES})
+set_target_properties(rocksdblib${ARTIFACT_SUFFIX} PROPERTIES COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/rocksdblib${ARTIFACT_SUFFIX}.pdb")
+add_dependencies(rocksdblib${ARTIFACT_SUFFIX} GenerateBuildVersion)

-  add_library(${ROCKSDB_SHARED_LIB} SHARED ${SOURCES})
-  target_link_libraries(${ROCKSDB_SHARED_LIB}
-    ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
-  set_target_properties(${ROCKSDB_SHARED_LIB} PROPERTIES
-                        LINKER_LANGUAGE CXX
-                        VERSION ${ROCKSDB_VERSION}
-                        SOVERSION ${ROCKSDB_VERSION_MAJOR}
-                        CXX_STANDARD 11
-                        OUTPUT_NAME "rocksdb")
-endif()
+add_library(rocksdb${ARTIFACT_SUFFIX} SHARED ${SOURCES})
+set_target_properties(rocksdb${ARTIFACT_SUFFIX} PROPERTIES COMPILE_FLAGS "-DROCKSDB_DLL -DROCKSDB_LIBRARY_EXPORTS /Fd${CMAKE_CFG_INTDIR}/rocksdb${ARTIFACT_SUFFIX}.pdb")
+add_dependencies(rocksdb${ARTIFACT_SUFFIX} GenerateBuildVersion)
+target_link_libraries(rocksdb${ARTIFACT_SUFFIX} ${LIBS})

-option(WITH_LIBRADOS "Build with librados" OFF)
-if(WITH_LIBRADOS)
-  list(APPEND SOURCES
-    utilities/env_librados.cc)
-  list(APPEND THIRDPARTY_LIBS rados)
-endif()
-
-add_library(${ROCKSDB_STATIC_LIB} STATIC ${SOURCES})
-target_link_libraries(${ROCKSDB_STATIC_LIB}
-  ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
-
-if(WIN32)
-  add_library(${ROCKSDB_IMPORT_LIB} SHARED ${SOURCES})
-  target_link_libraries(${ROCKSDB_IMPORT_LIB}
-    ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
-  set_target_properties(${ROCKSDB_IMPORT_LIB} PROPERTIES
-    COMPILE_DEFINITIONS "ROCKSDB_DLL;ROCKSDB_LIBRARY_EXPORTS")
-  if(MSVC)
-    set_target_properties(${ROCKSDB_STATIC_LIB} PROPERTIES
-      COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/${ROCKSDB_STATIC_LIB}.pdb")
-    set_target_properties(${ROCKSDB_IMPORT_LIB} PROPERTIES
-      COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/${ROCKSDB_IMPORT_LIB}.pdb")
+if (DEFINED JNI)
+  if (${JNI} EQUAL 1)
+    message(STATUS "JNI library is enabled")
+    add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/java)
+  else()
+    message(STATUS "JNI library is disabled")
  endif()
-endif()
-
-option(WITH_JNI "build with JNI" OFF)
-if(WITH_JNI OR JNI)
-  message(STATUS "JNI library is enabled")
-  add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/java)
 else()
  message(STATUS "JNI library is disabled")
 endif()

+set(APPS
+        tools/db_bench.cc
+        db/memtablerep_bench.cc
+        table/table_reader_bench.cc
+        tools/db_stress.cc
+        tools/write_stress.cc
+        tools/db_repl_stress.cc
+        tools/ldb.cc
+        tools/sst_dump.cc
+        tools/dump/rocksdb_dump.cc
+        tools/dump/rocksdb_undump.cc
+        util/cache_bench.cc
+)
+
+set(C_TESTS db/c_test.c)
+
 set(TESTS
-        cache/cache_test.cc
-        cache/lru_cache_test.cc
+        db/auto_roll_logger_test.cc
        db/column_family_test.cc
        db/compact_files_test.cc
        db/compaction_iterator_test.cc
-        db/compaction_job_stats_test.cc
        db/compaction_job_test.cc
+        db/compaction_job_stats_test.cc
        db/compaction_picker_test.cc
        db/comparator_db_test.cc
        db/corruption_test.cc
        db/cuckoo_table_db_test.cc
-        db/db_basic_test.cc
-        db/db_block_cache_test.cc
-        db/db_bloom_filter_test.cc
        db/db_compaction_filter_test.cc
        db/db_compaction_test.cc
        db/db_dynamic_level_test.cc
-        db/db_flush_test.cc
        db/db_inplace_update_test.cc
-        db/db_io_failure_test.cc
        db/db_iter_test.cc
-        db/db_iterator_test.cc
        db/db_log_iter_test.cc
-        db/db_memtable_test.cc
-        db/db_merge_operator_test.cc
-        db/db_options_test.cc
        db/db_properties_test.cc
-        db/db_range_del_test.cc
-        db/db_sst_test.cc
        db/db_table_properties_test.cc
        db/db_tailing_iter_test.cc
        db/db_test.cc
        db/db_test2.cc
+        db/db_block_cache_test.cc
+        db/db_bloom_filter_test.cc
+        db/db_iterator_test.cc
+        db/db_sst_test.cc
        db/db_universal_compaction_test.cc
        db/db_wal_test.cc
        db/dbformat_test.cc
        db/deletefile_test.cc
-        db/external_sst_file_basic_test.cc
-        db/external_sst_file_test.cc
        db/fault_injection_test.cc
        db/file_indexer_test.cc
        db/filename_test.cc
        db/flush_job_test.cc
+        db/inlineskiplist_test.cc
        db/listener_test.cc
        db/log_test.cc
        db/manual_compaction_test.cc
        db/memtable_list_test.cc
-        db/merge_helper_test.cc
        db/merge_test.cc
+        db/merge_helper_test.cc
        db/options_file_test.cc
        db/perf_context_test.cc
        db/plain_table_db_test.cc
        db/prefix_test.cc
        db/repair_test.cc
+        db/skiplist_test.cc
        db/table_properties_collector_test.cc
        db/version_builder_test.cc
        db/version_edit_test.cc
@@ -617,58 +375,49 @@ set(TESTS
        db/write_batch_test.cc
        db/write_callback_test.cc
        db/write_controller_test.cc
-        env/env_basic_test.cc
-        env/env_test.cc
-        env/mock_env_test.cc
-        memtable/inlineskiplist_test.cc
-        memtable/skiplist_test.cc
-        monitoring/histogram_test.cc
-        monitoring/iostats_context_test.cc
-        monitoring/statistics_test.cc
-        options/options_settable_test.cc
-        options/options_test.cc
        table/block_based_filter_block_test.cc
+        table/block_hash_index_test.cc
        table/block_test.cc
        table/cuckoo_table_builder_test.cc
        table/cuckoo_table_reader_test.cc
        table/full_filter_block_test.cc
        table/merger_test.cc
        table/table_test.cc
+        tools/db_sanity_test.cc
        tools/ldb_cmd_test.cc
        tools/reduce_levels_test.cc
        tools/sst_dump_test.cc
        util/arena_test.cc
-        util/auto_roll_logger_test.cc
        util/autovector_test.cc
        util/bloom_test.cc
+        util/cache_test.cc
        util/coding_test.cc
        util/crc32c_test.cc
        util/delete_scheduler_test.cc
        util/dynamic_bloom_test.cc
+        util/env_test.cc
        util/event_logger_test.cc
-        util/file_reader_writer_test.cc
        util/filelock_test.cc
+        util/file_reader_writer_test.cc
        util/heap_test.cc
+        util/histogram_test.cc
+        util/iostats_context_test.cc
+        util/memenv_test.cc
+        util/mock_env_test.cc
+        util/options_settable_test.cc
+        util/options_test.cc
        util/rate_limiter_test.cc
        util/slice_transform_test.cc
-        util/timer_queue_test.cc
        util/thread_list_test.cc
        util/thread_local_test.cc
        utilities/backupable/backupable_db_test.cc
        utilities/checkpoint/checkpoint_test.cc
-        utilities/column_aware_encoding_test.cc
-        utilities/date_tiered/date_tiered_test.cc
        utilities/document/document_db_test.cc
        utilities/document/json_document_test.cc
        utilities/geodb/geodb_test.cc
-        utilities/lua/rocks_lua_test.cc
        utilities/memory/memory_test.cc
        utilities/merge_operators/string_append/stringappend_test.cc
-        utilities/object_registry_test.cc
-        utilities/option_change_migration/option_change_migration_test.cc
        utilities/options/options_util_test.cc
-        utilities/persistent_cache/hash_table_test.cc
-        utilities/persistent_cache/persistent_cache_test.cc
        utilities/redis/redis_lists_test.cc
        utilities/spatialdb/spatial_db_test.cc
        utilities/table_properties_collectors/compact_on_deletion_collector_test.cc
@@ -677,42 +426,20 @@ set(TESTS
        utilities/ttl/ttl_test.cc
        utilities/write_batch_with_index/write_batch_with_index_test.cc
 )
-if(WITH_LIBRADOS)
-  list(APPEND TESTS utilities/env_librados_test.cc)
-endif()

-set(BENCHMARKS
-  cache/cache_bench.cc
-  memtable/memtablerep_bench.cc
-  tools/db_bench.cc
-  table/table_reader_bench.cc
-  utilities/column_aware_encoding_exp.cc
-  utilities/persistent_cache/hash_table_bench.cc)
-add_library(testharness OBJECT util/testharness.cc)
-foreach(sourcefile ${BENCHMARKS})
-  get_filename_component(exename ${sourcefile} NAME_WE)
-  add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile}
-    $<TARGET_OBJECTS:testharness>)
-  target_link_libraries(${exename}${ARTIFACT_SUFFIX} gtest ${LIBS})
-endforeach(sourcefile ${BENCHMARKS})
+set(EXES ${APPS})
+
+foreach(sourcefile ${EXES})
+    string(REPLACE ".cc" "" exename ${sourcefile})
+    string(REGEX REPLACE "^((.+)/)+" "" exename ${exename})
+    add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile})
+    target_link_libraries(${exename}${ARTIFACT_SUFFIX} ${LIBS})
+endforeach(sourcefile ${EXES})

-# For test util library that is build only in DEBUG mode
-# and linked to tests. Add test only code that is not #ifdefed for Release here.
-set(TESTUTIL_SOURCE
-    db/db_test_util.cc
-    env/mock_env.cc
-    monitoring/thread_status_updater_debug.cc
-    table/mock_table.cc
-    util/fault_injection_test_env.cc
-)
 # test utilities are only build in debug
-enable_testing()
-add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND})
 set(TESTUTILLIB testutillib${ARTIFACT_SUFFIX})
 add_library(${TESTUTILLIB} STATIC ${TESTUTIL_SOURCE})
-if(MSVC)
-  set_target_properties(${TESTUTILLIB} PROPERTIES COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/testutillib${ARTIFACT_SUFFIX}.pdb")
-endif()
+set_target_properties(${TESTUTILLIB} PROPERTIES COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/testutillib${ARTIFACT_SUFFIX}.pdb")
 set_target_properties(${TESTUTILLIB}
      PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
      EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
@@ -723,23 +450,18 @@ set_target_properties(${TESTUTILLIB}
 set(TEST_EXES ${TESTS})

 foreach(sourcefile ${TEST_EXES})
-    get_filename_component(exename ${sourcefile} NAME_WE)
-    add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile}
-      $<TARGET_OBJECTS:testharness>)
+    string(REPLACE ".cc" "" exename ${sourcefile})
+    string(REGEX REPLACE "^((.+)/)+" "" exename ${exename})
+    add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile})
    set_target_properties(${exename}${ARTIFACT_SUFFIX}
      PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
      EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
      EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
      )
-    target_link_libraries(${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} gtest ${LIBS})
-    if(NOT "${exename}" MATCHES "db_sanity_test")
-      add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
-      add_dependencies(check ${exename}${ARTIFACT_SUFFIX})
-    endif()
+    target_link_libraries(${exename}${ARTIFACT_SUFFIX} ${LIBS} testutillib${ARTIFACT_SUFFIX})
 endforeach(sourcefile ${TEST_EXES})

 # C executables must link to a shared object
-set(C_TESTS db/c_test.c)
 set(C_TEST_EXES ${C_TESTS})

 foreach(sourcefile ${C_TEST_EXES})
@@ -751,18 +473,5 @@ foreach(sourcefile ${C_TEST_EXES})
      EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
      EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
      )
-    target_link_libraries(${exename}${ARTIFACT_SUFFIX} ${ROCKSDB_IMPORT_LIB} testutillib${ARTIFACT_SUFFIX})
-    add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
-    add_dependencies(check ${exename}${ARTIFACT_SUFFIX})
+    target_link_libraries(${exename}${ARTIFACT_SUFFIX} rocksdb${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX})
 endforeach(sourcefile ${C_TEST_EXES})
-add_subdirectory(tools)
-
-# Installation and packaging for Linux
-if(NOT WIN32)
-install(TARGETS ${ROCKSDB_STATIC_LIB} COMPONENT devel ARCHIVE DESTINATION lib64)
-install(TARGETS ${ROCKSDB_SHARED_LIB} COMPONENT runtime DESTINATION lib64)
-install(DIRECTORY "${PROJECT_SOURCE_DIR}/include/rocksdb/"
-        COMPONENT devel
-        DESTINATION include/rocksdb)
-set(CMAKE_INSTALL_PREFIX /usr)
-endif()
@@ -1,11 +1,4 @@
 # RocksDB default options change log
-## Unreleased
-* Change the default of delayed slowdown value to 16MB/s and further increase the L0 stop condition to 36 files.
-
-## 5.0 (11/17/2016)
-* Options::allow_concurrent_memtable_write and Options::enable_write_thread_adaptive_yield are now true by default
-* Options.level0_stop_writes_trigger default value changes from 24 to 32.
-
 ## 4.8.0 (5/2/2016)
 * options.max_open_files changes from 5000 to -1. It improves performance, but users need to set file descriptor limit to be large enough and watch memory usage for index and bloom filters.
 * options.base_background_compactions changes from max_background_compactions to 1. When users set higher max_background_compactions but the write throughput is not high, the writes are less spiky to disks.
@@ -1,175 +1,10 @@
 # Rocksdb Change Log
-## 5.4.7 (06/28/2017)
-### Bug Fixes
-* Shouldn't ignore return value of fsync() in flush.
-
-## 5.4.6 (05/30/2017)
-### Bug Fixes
-* Fix db_bench with lite build.
-
-## 5.4.5 (05/19/2017)
-### Bug Fixes
-* Facility for cross-building RocksJava using Docker.
-* Build and link with ZStd when creating the static RocksJava build.
-* Fix snprintf's usage to be cross-platform.
-* Fix build error with blob DB.
-* Fix readamp test type inconsistency.
-
-## 5.4.4 (05/11/2017)
-### New Features
-* Added DB::CreateColumnFamilie() and DB::DropColumnFamilies() to bulk create/drop column families.
-
-### Bug Fixes
-* Fix a manual / auto compaction data race.
-* Fix CentOS 5 cross-building of RocksJava.
-
-## 5.4.3 (05/10/2017)
-### Bug Fixes
-* Fix memory alignment with logical sector size.
-* Fix alignment in ReadaheadRandomAccessFile.
-* Fix bias with read amplification stats (READ_AMP_ESTIMATE_USEFUL_BYTES and READ_AMP_TOTAL_READ_BYTES).
-
-## 5.4.2 (05/08/2017)
-### Bug Fixes
-* Fix huge fallocate calls fail and make XFS unhappy.
-
-## 5.4.1 (05/02/2017)
-### Bug Fxies
-* Fix WriteBatchWithIndex address use after scope error.
-* Fix WritableFile buffer size in direct IO.
-* Add prefetch to PosixRandomAccessFile in buffered io.
-* Fix PinnableSlice access invalid address when row cache is enabled.
-
-## 5.4.0 (04/11/2017)
-### Public API Change
-* Support dynamically change `stats_dump_period_sec` option via SetDBOptions().
-* Added ReadOptions::max_skippable_internal_keys to set a threshold to fail a request as incomplete when too many keys are being skipped when using iterators.
-* DB::Get in place of std::string accepts PinnableSlice, which avoids the extra memcpy of value to std::string in most of cases.
-    * PinnableSlice releases the pinned resources that contain the value when it is destructed or when ::Reset() is called on it.
-    * The old API that accepts std::string, although discouraged, is still supported.
-* Replace Options::use_direct_writes with Options::use_direct_io_for_flush_and_compaction. Read Direct IO wiki for details.
-* Added CompactionEventListener and EventListener::OnFlushBegin interfaces.
-
-### New Features
-* Memtable flush can be avoided during checkpoint creation if total log file size is smaller than a threshold specified by the user.
-* Introduce level-based L0->L0 compactions to reduce file count, so write delays are incurred less often.
-* (Experimental) Partitioning filters which creates an index on the partitions. The feature can be enabled by setting partition_filters when using kFullFilter. Currently the feature also requires two-level indexing to be enabled. Number of partitions is the same as the number of partitions for indexes, which is controlled by metadata_block_size.
-* DB::ResetStats() to reset internal stats.
-
-## 5.3.0 (03/08/2017)
-### Public API Change
-* Remove disableDataSync option.
-* Remove timeout_hint_us option from WriteOptions. The option has been deprecated and has no effect since 3.13.0.
-* Remove option min_partial_merge_operands. Partial merge operands will always be merged in flush or compaction if there are more than one.
-* Remove option verify_checksums_in_compaction. Compaction will always verify checksum.
-
-### Bug Fixes
-* Fix the bug that iterator may skip keys
-
-## 5.2.0 (02/08/2017)
-### Public API Change
-* NewLRUCache() will determine number of shard bits automatically based on capacity, if the user doesn't pass one. This also impacts the default block cache when the user doesn't explict provide one.
-* Change the default of delayed slowdown value to 16MB/s and further increase the L0 stop condition to 36 files.
-* Options::use_direct_writes and Options::use_direct_reads are now ready to use.
-* (Experimental) Two-level indexing that partition the index and creates a 2nd level index on the partitions. The feature can be enabled by setting kTwoLevelIndexSearch as IndexType and configuring index_per_partition.
-
-### New Features
-* Added new overloaded function GetApproximateSizes that allows to specify if memtable stats should be computed only without computing SST files' stats approximations.
-* Added new function GetApproximateMemTableStats that approximates both number of records and size of memtables.
-* Add Direct I/O mode for SST file I/O
-
-### Bug Fixes
-* RangeSync() should work if ROCKSDB_FALLOCATE_PRESENT is not set
-* Fix wrong results in a data race case in Get()
-* Some fixes related to 2PC.
-* Fix bugs of data corruption in direct I/O
-
-## 5.1.0 (01/13/2017)
-* Support dynamically change `delete_obsolete_files_period_micros` option via SetDBOptions().
-* Added EventListener::OnExternalFileIngested which will be called when IngestExternalFile() add a file successfully.
-* BackupEngine::Open and BackupEngineReadOnly::Open now always return error statuses matching those of the backup Env.
-
-### Bug Fixes
-* Fix the bug that if 2PC is enabled, checkpoints may loss some recent transactions.
-* When file copying is needed when creating checkpoints or bulk loading files, fsync the file after the file copying.
-
-## 5.0.0 (11/17/2016)
-### Public API Change
-* Options::max_bytes_for_level_multiplier is now a double along with all getters and setters.
-* Support dynamically change `delayed_write_rate` and `max_total_wal_size` options via SetDBOptions().
-* Introduce DB::DeleteRange for optimized deletion of large ranges of contiguous keys.
-* Support dynamically change `delayed_write_rate` option via SetDBOptions().
-* Options::allow_concurrent_memtable_write and Options::enable_write_thread_adaptive_yield are now true by default.
-* Remove Tickers::SEQUENCE_NUMBER to avoid confusion if statistics object is shared among RocksDB instance. Alternatively DB::GetLatestSequenceNumber() can be used to get the same value.
-* Options.level0_stop_writes_trigger default value changes from 24 to 32.
-* New compaction filter API: CompactionFilter::FilterV2(). Allows to drop ranges of keys.
-* Removed flashcache support.
-* DB::AddFile() is deprecated and is replaced with DB::IngestExternalFile(). DB::IngestExternalFile() remove all the restrictions that existed for DB::AddFile.
-
-### New Features
-* Add avoid_flush_during_shutdown option, which speeds up DB shutdown by not flushing unpersisted data (i.e. with disableWAL = true). Unpersisted data will be lost. The options is dynamically changeable via SetDBOptions().
-* Add memtable_insert_with_hint_prefix_extractor option. The option is mean to reduce CPU usage for inserting keys into memtable, if keys can be group by prefix and insert for each prefix are sequential or almost sequential. See include/rocksdb/options.h for more details.
-* Add LuaCompactionFilter in utilities.  This allows developers to write compaction filters in Lua.  To use this feature, LUA_PATH needs to be set to the root directory of Lua.
-* No longer populate "LATEST_BACKUP" file in backup directory, which formerly contained the number of the latest backup. The latest backup can be determined by finding the highest numbered file in the "meta/" subdirectory.
-
-## 4.13.0 (10/18/2016)
-### Public API Change
-* DB::GetOptions() reflect dynamic changed options (i.e. through DB::SetOptions()) and return copy of options instead of reference.
-* Added Statistics::getAndResetTickerCount().
-
-### New Features
-* Add DB::SetDBOptions() to dynamic change base_background_compactions and max_background_compactions.
-* Added Iterator::SeekForPrev(). This new API will seek to the last key that less than or equal to the target key.
-
-## 4.12.0 (9/12/2016)
-### Public API Change
-* CancelAllBackgroundWork() flushes all memtables for databases containing writes that have bypassed the WAL (writes issued with WriteOptions::disableWAL=true) before shutting down background threads.
-* Merge options source_compaction_factor, max_grandparent_overlap_bytes and expanded_compaction_factor into max_compaction_bytes.
-* Remove ImmutableCFOptions.
-* Add a compression type ZSTD, which can work with ZSTD 0.8.0 or up. Still keep ZSTDNotFinal for compatibility reasons.
-
-### New Features
-* Introduce NewClockCache, which is based on CLOCK algorithm with better concurrent performance in some cases. It can be used to replace the default LRU-based block cache and table cache. To use it, RocksDB need to be linked with TBB lib.
-* Change ticker/histogram statistics implementations to accumulate data in thread-local storage, which improves CPU performance by reducing cache coherency costs. Callers of CreateDBStatistics do not need to change anything to use this feature.
-* Block cache mid-point insertion, where index and filter block are inserted into LRU block cache with higher priority. The feature can be enabled by setting BlockBasedTableOptions::cache_index_and_filter_blocks_with_high_priority to true and high_pri_pool_ratio > 0 when creating NewLRUCache.
-
-## 4.11.0 (8/1/2016)
-### Public API Change
-* options.memtable_prefix_bloom_huge_page_tlb_size => memtable_huge_page_size. When it is set, RocksDB will try to allocate memory from huge page for memtable too, rather than just memtable bloom filter.
-
-### New Features
-* A tool to migrate DB after options change. See include/rocksdb/utilities/option_change_migration.h.
-* Add ReadOptions.background_purge_on_iterator_cleanup. If true, we avoid file deletion when destorying iterators.
-
-## 4.10.0 (7/5/2016)
-### Public API Change
-* options.memtable_prefix_bloom_bits changes to options.memtable_prefix_bloom_bits_ratio and deprecate options.memtable_prefix_bloom_probes
-* enum type CompressionType and PerfLevel changes from char to unsigned char. Value of all PerfLevel shift by one.
-* Deprecate options.filter_deletes.
-
-### New Features
-* Add avoid_flush_during_recovery option.
-* Add a read option background_purge_on_iterator_cleanup to avoid deleting files in foreground when destroying iterators. Instead, a job is scheduled in high priority queue and would be executed in a separate background thread.
-* RepairDB support for column families. RepairDB now associates data with non-default column families using information embedded in the SST/WAL files (4.7 or later). For data written by 4.6 or earlier, RepairDB associates it with the default column family.
-* Add options.write_buffer_manager which allows users to control total memtable sizes across multiple DB instances.
-
-## 4.9.0 (6/9/2016)
-### Public API changes
-* Add bottommost_compression option, This option can be used to set a specific compression algorithm for the bottommost level (Last level containing files in the DB).
-* Introduce CompactionJobInfo::compression, This field state the compression algorithm used to generate the output files of the compaction.
-* Deprecate BlockBaseTableOptions.hash_index_allow_collision=false
-* Deprecate options builder (GetOptions()).
-
-### New Features
-* Introduce NewSimCache() in rocksdb/utilities/sim_cache.h. This function creates a block cache that is able to give simulation results (mainly hit rate) of simulating block behavior with a configurable cache size.
-
 ## 4.8.0 (5/2/2016)
 ### Public API Change
 * Allow preset compression dictionary for improved compression of block-based tables. This is supported for zlib, zstd, and lz4. The compression dictionary's size is configurable via CompressionOptions::max_dict_bytes.
 * Delete deprecated classes for creating backups (BackupableDB) and restoring from backups (RestoreBackupableDB). Now, BackupEngine should be used for creating backups, and BackupEngineReadOnly should be used for restorations. For more details, see https://github.com/facebook/rocksdb/wiki/How-to-backup-RocksDB%3F
 * Expose estimate of per-level compression ratio via DB property: "rocksdb.compression-ratio-at-levelN".
 * Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
-
 ### New Features
 * Add ReadOptions::readahead_size. If non-zero, NewIterator will create a new table reader which performs reads of the given size.

@@ -181,10 +16,9 @@
 ## 4.6.0 (3/10/2016)
 ### Public API Changes
 * Change default of BlockBasedTableOptions.format_version to 2. It means default DB created by 4.6 or up cannot be opened by RocksDB version 3.9 or earlier.
-* Added strict_capacity_limit option to NewLRUCache. If the flag is set to true, insert to cache will fail if no enough capacity can be free. Signature of Cache::Insert() is updated accordingly.
-* Tickers [NUMBER_DB_NEXT, NUMBER_DB_PREV, NUMBER_DB_NEXT_FOUND, NUMBER_DB_PREV_FOUND, ITER_BYTES_READ] are not updated immediately. The are updated when the Iterator is deleted.
+* Added strict_capacity_limit option to NewLRUCache. If the flag is set to true, insert to cache will fail if no enough capacity can be free. Signiture of Cache::Insert() is updated accordingly.
+* Tickers [NUMBER_DB_NEXT, NUMBER_DB_PREV, NUMBER_DB_NEXT_FOUND, NUMBER_DB_PREV_FOUND, ITER_BYTES_READ] are not updated immediately. The are updated when the Iterator is deleted.  
 * Add monotonically increasing counter (DB property "rocksdb.current-super-version-number") that increments upon any change to the LSM tree.
-
 ### New Features
 * Add CompactionPri::kMinOverlappingRatio, a compaction picking mode friendly to write amplification.
 * Deprecate Iterator::IsKeyPinned() and replace it with Iterator::GetProperty() with prop_name="rocksdb.iterator.is.key.pinned"
@@ -198,7 +32,7 @@
 ### New Features
 * ldb tool now supports operations to non-default column families.
 * Add kPersistedTier to ReadTier.  This option allows Get and MultiGet to read only the persited data and skip mem-tables if writes were done with disableWAL = true.
-* Add DBOptions::sst_file_manager. Use NewSstFileManager() in include/rocksdb/sst_file_manager.h to create a SstFileManager that can be used to track the total size of SST files and control the SST files deletion rate.
+* Add DBOptions::sst_file_manager. Use NewSstFileManager() in include/rocksdb/sst_file_manager.h to create a SstFileManager that can be used to track the total size of SST files and control the SST files deletion rate.  

 ## 4.4.0 (1/14/2016)
 ### Public API Changes
@@ -297,8 +131,8 @@
 * options.hard_rate_limit is deprecated.
 * When options.soft_rate_limit or options.level0_slowdown_writes_trigger is triggered, the way to slow down writes is changed to: write rate to DB is limited to to options.delayed_write_rate.
 * DB::GetApproximateSizes() adds a parameter to allow the estimation to include data in mem table, with default to be not to include. It is now only supported in skip list mem table.
-* DB::CompactRange() now accept CompactRangeOptions instead of multiple parameters. CompactRangeOptions is defined in include/rocksdb/options.h.
-* CompactRange() will now skip bottommost level compaction for level based compaction if there is no compaction filter, bottommost_level_compaction is introduced in CompactRangeOptions to control when it's possible to skip bottommost level compaction. This mean that if you want the compaction to produce a single file you need to set bottommost_level_compaction to BottommostLevelCompaction::kForce.
+* DB::CompactRange() now accept CompactRangeOptions instead of multiple paramters. CompactRangeOptions is defined in include/rocksdb/options.h.
+* CompactRange() will now skip bottommost level compaction for level based compaction if there is no compaction filter, bottommost_level_compaction is introduced in CompactRangeOptions to control when it's possbile to skip bottommost level compaction. This mean that if you want the compaction to produce a single file you need to set bottommost_level_compaction to BottommostLevelCompaction::kForce.
 * Add Cache.GetPinnedUsage() to get the size of memory occupied by entries that are in use by the system.
 * DB:Open() will fail if the compression specified in Options is not linked with the binary. If you see this failure, recompile RocksDB with compression libraries present on your system. Also, previously our default compression was snappy. This behavior is now changed. Now, the default compression is snappy only if it's available on the system. If it isn't we change the default to kNoCompression.
 * We changed how we account for memory used in block cache. Previously, we only counted the sum of block sizes currently present in block cache. Now, we count the actual memory usage of the blocks. For example, a block of size 4.5KB will use 8KB memory with jemalloc. This might decrease your memory usage and possibly decrease performance. Increase block cache size if you see this happening after an upgrade.
@@ -330,7 +164,7 @@
  Lower numbered levels will be placed earlier in the db_paths and higher
  numbered levels will be placed later in the db_paths vector.
 * Potentially big performance improvements if you're using RocksDB with lots of column families (100-1000)
-* Added BlockBasedTableOptions.format_version option, which allows user to specify which version of block based table he wants. As a general guideline, newer versions have more features, but might not be readable by older versions of RocksDB.
+* Added BlockBasedTableOptions.format_version option, which allows user to specify which version of block based table he wants. As a general guidline, newer versions have more features, but might not be readable by older versions of RocksDB.
 * Added new block based table format (version 2), which you can enable by setting BlockBasedTableOptions.format_version = 2. This format changes how we encode size information in compressed blocks and should help with memory allocations if you're using Zlib or BZip2 compressions.
 * MemEnv (env that stores data in memory) is now available in default library build. You can create it by calling NewMemEnv().
 * Add SliceTransform.SameResultWhenAppended() to help users determine it is safe to apply prefix bloom/hash.
@@ -406,7 +240,7 @@

 ## 3.5.0 (9/3/2014)
 ### New Features
-* Add include/utilities/write_batch_with_index.h, providing a utility class to query data out of WriteBatch when building it.
+* Add include/utilities/write_batch_with_index.h, providing a utilitiy class to query data out of WriteBatch when building it.
 * Move BlockBasedTable related options to BlockBasedTableOptions from Options. Change corresponding JNI interface. Options affected include:
  no_block_cache, block_cache, block_cache_compressed, block_size, block_size_deviation, block_restart_interval, filter_policy, whole_key_filtering. filter_policy is changed to shared_ptr from a raw pointer.
 * Remove deprecated options: disable_seek_compaction and db_stats_log_interval
@@ -435,7 +269,7 @@
 ### New Features
 * Added JSON API prototype.
 * HashLinklist reduces performance outlier caused by skewed bucket by switching data in the bucket from linked list to skip list. Add parameter threshold_use_skiplist in NewHashLinkListRepFactory().
-* RocksDB is now able to reclaim storage space more effectively during the compaction process.  This is done by compensating the size of each deletion entry by the 2X average value size, which makes compaction to be triggered by deletion entries more easily.
+* RocksDB is now able to reclaim storage space more effectively during the compaction process.  This is done by compensating the size of each deletion entry by the 2X average value size, which makes compaction to be triggerred by deletion entries more easily.
 * Add TimeOut API to write.  Now WriteOptions have a variable called timeout_hint_us.  With timeout_hint_us set to non-zero, any write associated with this timeout_hint_us may be aborted when it runs longer than the specified timeout_hint_us, and it is guaranteed that any write completes earlier than the specified time-out will not be aborted due to the time-out condition.
 * Add a rate_limiter option, which controls total throughput of flush and compaction. The throughput is specified in bytes/sec. Flush always has precedence over compaction when available bandwidth is constrained.

@@ -450,11 +284,11 @@
 2) It added some complexity to the important code-paths,
 3) None of our internal customers were really using it.
 Because of that, Options::disable_seek_compaction is now obsolete. It is still a parameter in Options, so it does not break the build, but it does not have any effect. We plan to completely remove it at some point, so we ask users to please remove this option from your code base.
-* Add two parameters to NewHashLinkListRepFactory() for logging on too many entries in a hash bucket when flushing.
+* Add two paramters to NewHashLinkListRepFactory() for logging on too many entries in a hash bucket when flushing.
 * Added new option BlockBasedTableOptions::hash_index_allow_collision. When enabled, prefix hash index for block-based table will not store prefix and allow hash collision, reducing memory consumption.

 ### New Features
-* PlainTable now supports a new key encoding: for keys of the same prefix, the prefix is only written once. It can be enabled through encoding_type parameter of NewPlainTableFactory()
+* PlainTable now supports a new key encoding: for keys of the same prefix, the prefix is only written once. It can be enabled through encoding_type paramter of NewPlainTableFactory()
 * Add AdaptiveTableFactory, which is used to convert from a DB of PlainTable to BlockBasedTabe, or vise versa. It can be created using NewAdaptiveTableFactory()

 ### Performance Improvements
@@ -1,6 +1,6 @@
 ## Compilation

-**Important**: If you plan to run RocksDB in production, don't compile using default
+**Important**: If you plan to run RocksDB in production, don't compile using default 
 `make` or `make all`. That will compile RocksDB in debug mode, which is much slower
 than release mode.

@@ -22,18 +22,15 @@ use binaries compiled by `make all` in production.

 * By default the binary we produce is optimized for the platform you're compiling on
 (-march=native or the equivalent). If you want to build a portable binary, add 'PORTABLE=1' before
-your make commands, like this: `PORTABLE=1 make static_lib`. If you want to build a binary that
-makes use of SSE4, add 'USE_SSE=1' before your make commands, like this: `USE_SSE=1 make static_lib`.
+your make commands, like this: `PORTABLE=1 make static_lib`

 ## Dependencies

 * You can link RocksDB with following compression libraries:
  - [zlib](http://www.zlib.net/) - a library for data compression.
  - [bzip2](http://www.bzip.org/) - a library for data compression.
-  - [snappy](http://google.github.io/snappy/) - a library for fast
+  - [snappy](https://code.google.com/p/snappy/) - a library for fast
      data compression.
-  - [zstandard](http://www.zstd.net) - Fast real-time compression
-      algorithm.

 * All our tools depend on:
  - [gflags](https://gflags.github.io/gflags/) - a library that handles
@@ -43,7 +40,7 @@ makes use of SSE4, add 'USE_SSE=1' before your make commands, like this: `USE_SS
 ## Supported platforms

 * **Linux - Ubuntu**
-    * Upgrade your gcc to version at least 4.8 to get C++11 support.
+    * Upgrade your gcc to version at least 4.7 to get C++11 support.
    * Install gflags. First, try: `sudo apt-get install libgflags-dev`
      If this doesn't work and you're using Ubuntu, here's a nice tutorial:
      (http://askubuntu.com/questions/312173/installing-gflags-12-04)
@@ -51,27 +48,21 @@ makes use of SSE4, add 'USE_SSE=1' before your make commands, like this: `USE_SS
      `sudo apt-get install libsnappy-dev`.
    * Install zlib. Try: `sudo apt-get install zlib1g-dev`.
    * Install bzip2: `sudo apt-get install libbz2-dev`.
-    * Install zstandard: `sudo apt-get install libzstd-dev`.
-
-* **Linux - CentOS / RHEL**
-    * Upgrade your gcc to version at least 4.8 to get C++11 support:
-      `yum install gcc48-c++`
+* **Linux - CentOS**
+    * Upgrade your gcc to version at least 4.7 to get C++11 support:
+      `yum install gcc47-c++`
    * Install gflags:

-              git clone https://github.com/gflags/gflags.git
-              cd gflags
-              git checkout v2.0
+              wget https://gflags.googlecode.com/files/gflags-2.0-no-svn-files.tar.gz
+              tar -xzvf gflags-2.0-no-svn-files.tar.gz
+              cd gflags-2.0
              ./configure && make && sudo make install

-      **Notice**: Once installed, please add the include path for gflags to your CPATH env var and the
-      lib path to LIBRARY_PATH. If installed with default settings, the lib will be /usr/local/lib
-      and the include path will be /usr/local/include.
-
    * Install snappy:

-              wget https://github.com/google/snappy/releases/download/1.1.4/snappy-1.1.4.tar.gz
-              tar -xzvf snappy-1.1.4.tar.gz
-              cd snappy-1.1.4
+              wget https://snappy.googlecode.com/files/snappy-1.1.1.tar.gz
+              tar -xzvf snappy-1.1.1.tar.gz
+              cd snappy-1.1.1
              ./configure && make && sudo make install

    * Install zlib:
@@ -84,20 +75,12 @@ makes use of SSE4, add 'USE_SSE=1' before your make commands, like this: `USE_SS
              sudo yum install bzip2
              sudo yum install bzip2-devel

-    * Install zstandard:
-
-             wget https://github.com/facebook/zstd/archive/v1.1.3.tar.gz
-             mv v1.1.3.tar.gz zstd-1.1.3.tar.gz
-             tar zxvf zstd-1.1.3.tar.gz
-             cd zstd-1.1.3
-             make && sudo make install
-
 * **OS X**:
    * Install latest C++ compiler that supports C++ 11:
        * Update XCode:  run `xcode-select --install` (or install it from XCode App's settting).
        * Install via [homebrew](http://brew.sh/).
            * If you're first time developer in MacOS, you still need to run: `xcode-select --install` in your command line.
-            * run `brew tap homebrew/versions; brew install gcc48 --use-llvm` to install gcc 4.8 (or higher).
+            * run `brew tap homebrew/versions; brew install gcc47 --use-llvm` to install gcc 4.7 (or higher).
    * run `brew install rocksdb`

 * **iOS**:
@@ -9,5 +9,3 @@ This is the list of all known third-party language bindings for RocksDB. If some
 * Haskell - https://hackage.haskell.org/package/rocksdb-haskell
 * PHP - https://github.com/Photonios/rocksdb-php
 * C# - https://github.com/warrenfalk/rocksdb-sharp
-* Rust - https://github.com/spacejam/rust-rocksdb
-* D programming language - https://github.com/b1naryth1ef/rocksdb
@@ -6,9 +6,6 @@

 #-----------------------------------------------

-BASH_EXISTS := $(shell which bash)
-SHELL := $(shell which bash)
-
 CLEAN_FILES = # deliberately empty, so we can append below.
 CFLAGS += ${EXTRA_CFLAGS}
 CXXFLAGS += ${EXTRA_CXXFLAGS}
@@ -133,27 +130,14 @@ am__v_AR_ = $(am__v_AR_$(AM_DEFAULT_VERBOSITY))
 am__v_AR_0 = @echo "  AR      " $@;
 am__v_AR_1 =

-ifdef ROCKSDB_USE_LIBRADOS
-LIB_SOURCES += utilities/env_librados.cc
-LDFLAGS += -lrados
-endif
-
 AM_LINK = $(AM_V_CCLD)$(CXX) $^ $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
+
 # detect what platform we're building on
-dummy := $(shell (export ROCKSDB_ROOT="$(CURDIR)"; export PORTABLE="$(PORTABLE)"; "$(CURDIR)/build_tools/build_detect_platform" "$(CURDIR)/make_config.mk"))
+dummy := $(shell (export ROCKSDB_ROOT="$(CURDIR)"; "$(CURDIR)/build_tools/build_detect_platform" "$(CURDIR)/make_config.mk"))
 # this file is generated by the previous line to set build flags and sources
 include make_config.mk
 CLEAN_FILES += make_config.mk

-missing_make_config_paths := $(shell				\
-	grep "\/\S*" -o $(CURDIR)/make_config.mk | 		\
-	while read path;					\
-		do [ -e $$path ] || echo $$path; 		\
-	done | sort | uniq)
-
-$(foreach path, $(missing_make_config_paths), \
-	$(warning Warning: $(path) dont exist))
-
 ifneq ($(PLATFORM), IOS)
 CFLAGS += -g
 CXXFLAGS += -g
@@ -169,8 +153,6 @@ ifneq ($(filter -DROCKSDB_LITE,$(OPT)),)
 	# found
 	CFLAGS += -fno-exceptions
 	CXXFLAGS += -fno-exceptions
-	# LUA is not supported under ROCKSDB_LITE
-	LUA_PATH =
 endif

 # ASAN doesn't work well with jemalloc. If we're compiling with ASAN, we should use regular malloc.
@@ -190,8 +172,6 @@ ifdef COMPILE_WITH_TSAN
        # Turn off -pg when enabling TSAN testing, because that induces
        # a link failure.  TODO: find the root cause
 	PROFILING_FLAGS =
-	# LUA is not supported under TSAN
-	LUA_PATH =
 endif

 # USAN doesn't work well with jemalloc. If we're compiling with USAN, we should use regular malloc.
@@ -204,16 +184,15 @@ endif

 ifndef DISABLE_JEMALLOC
 	ifdef JEMALLOC
-		PLATFORM_CXXFLAGS += -DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE
-		PLATFORM_CCFLAGS  += -DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE
+		PLATFORM_CXXFLAGS += "-DROCKSDB_JEMALLOC"
+		PLATFORM_CCFLAGS +=  "-DROCKSDB_JEMALLOC"
 	endif
 	EXEC_LDFLAGS := $(JEMALLOC_LIB) $(EXEC_LDFLAGS)
 	PLATFORM_CXXFLAGS += $(JEMALLOC_INCLUDE)
 	PLATFORM_CCFLAGS += $(JEMALLOC_INCLUDE)
 endif

-export GTEST_THROW_ON_FAILURE=1
-export GTEST_HAS_EXCEPTIONS=1
+export GTEST_THROW_ON_FAILURE=1 GTEST_HAS_EXCEPTIONS=1
 GTEST_DIR = ./third-party/gtest-1.7.0/fused-src
 PLATFORM_CCFLAGS += -isystem $(GTEST_DIR)
 PLATFORM_CXXFLAGS += -isystem $(GTEST_DIR)
@@ -224,38 +203,6 @@ default: all
 WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare -Wshadow \
  -Wno-unused-parameter

-ifndef DISABLE_WARNING_AS_ERROR
-	WARNING_FLAGS += -Werror
-endif
-
-
-ifdef LUA_PATH
-
-ifndef LUA_INCLUDE
-LUA_INCLUDE=$(LUA_PATH)/include
-endif
-
-LUA_INCLUDE_FILE=$(LUA_INCLUDE)/lualib.h
-
-ifeq ("$(wildcard $(LUA_INCLUDE_FILE))", "")
-# LUA_INCLUDE_FILE does not exist
-$(error Cannot find lualib.h under $(LUA_INCLUDE).  Try to specify both LUA_PATH and LUA_INCLUDE manually)
-endif
-LUA_FLAGS = -I$(LUA_INCLUDE) -DLUA -DLUA_COMPAT_ALL
-CFLAGS += $(LUA_FLAGS)
-CXXFLAGS += $(LUA_FLAGS)
-
-ifndef LUA_LIB
-LUA_LIB = $(LUA_PATH)/lib/liblua.a
-endif
-ifeq ("$(wildcard $(LUA_LIB))", "") # LUA_LIB does not exist
-$(error $(LUA_LIB) does not exist.  Try to specify both LUA_PATH and LUA_LIB manually)
-endif
-LDFLAGS += $(LUA_LIB)
-
-endif
-
-
 CFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CCFLAGS) $(OPT)
 CXXFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers

@@ -267,7 +214,14 @@ ifdef FORCE_GIT_SHA
 else
 	git_sha := $(shell git rev-parse HEAD 2>/dev/null)
 endif
-gen_build_version = sed -e s/@@GIT_SHA@@/$(git_sha)/ -e s/@@GIT_DATE_TIME@@/$(date)/ util/build_version.cc.in
+gen_build_version =							\
+  printf '%s\n'								\
+    '\#include "build_version.h"'					\
+    'const char* rocksdb_build_git_sha =				\
+      "rocksdb_build_git_sha:$(git_sha)";'			\
+    'const char* rocksdb_build_git_date =				\
+      "rocksdb_build_git_date:$(date)";'				\
+    'const char* rocksdb_build_compile_date = __DATE__;'

 # Record the version of the source that we are compiling.
 # We keep a record of the git revision in this file.  It is then built
@@ -284,8 +238,8 @@ util/build_version.cc: FORCE
 	else mv -f $@-t $@; fi

 LIBOBJECTS = $(LIB_SOURCES:.cc=.o)
-LIBOBJECTS += $(TOOL_LIB_SOURCES:.cc=.o)
-MOCKOBJECTS = $(MOCK_LIB_SOURCES:.cc=.o)
+LIBOBJECTS += $(TOOL_SOURCES:.cc=.o)
+MOCKOBJECTS = $(MOCK_SOURCES:.cc=.o)

 GTEST = $(GTEST_DIR)/gtest/gtest-all.o
 TESTUTIL = ./util/testutil.o
@@ -295,93 +249,73 @@ VALGRIND_VER := $(join $(VALGRIND_VER),valgrind)

 VALGRIND_OPTS = --error-exitcode=$(VALGRIND_ERROR) --leak-check=full

-BENCHTOOLOBJECTS = $(BENCH_LIB_SOURCES:.cc=.o) $(LIBOBJECTS) $(TESTUTIL)
-
-EXPOBJECTS = $(EXP_LIB_SOURCES:.cc=.o) $(LIBOBJECTS) $(TESTUTIL)
+BENCHTOOLOBJECTS = $(BENCH_SOURCES:.cc=.o) $(LIBOBJECTS) $(TESTUTIL)

 TESTS = \
-	db_basic_test \
-	db_test2 \
-	external_sst_file_basic_test \
-	auto_roll_logger_test \
-	bloom_test \
-	dynamic_bloom_test \
-	c_test \
-	checkpoint_test \
-	crc32c_test \
-	coding_test \
-	inlineskiplist_test \
-	env_basic_test \
-	env_test \
-	thread_local_test \
-	rate_limiter_test \
-	perf_context_test \
-	iostats_context_test \
-	db_wal_test \
-	db_block_cache_test \
 	db_test \
+	db_test2 \
+	db_block_cache_test \
 	db_bloom_filter_test \
 	db_iter_test \
 	db_log_iter_test \
 	db_compaction_filter_test \
 	db_compaction_test \
 	db_dynamic_level_test \
-	db_flush_test \
 	db_inplace_update_test \
 	db_iterator_test \
-	db_memtable_test \
-	db_merge_operator_test \
-	db_options_test \
-	db_range_del_test \
 	db_sst_test \
 	db_tailing_iter_test \
 	db_universal_compaction_test \
-	db_io_failure_test \
+	db_wal_test \
 	db_properties_test \
 	db_table_properties_test \
+	block_hash_index_test \
 	autovector_test \
-	cleanable_test \
 	column_family_test \
 	table_properties_collector_test \
 	arena_test \
+	auto_roll_logger_test \
 	block_test \
+	bloom_test \
+	dynamic_bloom_test \
+	c_test \
 	cache_test \
+	checkpoint_test \
+	coding_test \
 	corruption_test \
+	crc32c_test \
 	slice_transform_test \
 	dbformat_test \
+	env_test \
 	fault_injection_test \
 	filelock_test \
 	filename_test \
 	file_reader_writer_test \
 	block_based_filter_block_test \
 	full_filter_block_test \
-	partitioned_filter_block_test \
-	hash_table_test \
 	histogram_test \
+	inlineskiplist_test \
 	log_test \
 	manual_compaction_test \
+	memenv_test \
 	mock_env_test \
 	memtable_list_test \
 	merge_helper_test \
 	memory_test \
 	merge_test \
 	merger_test \
-	util_merge_operators_test \
 	options_file_test \
 	redis_test \
 	reduce_levels_test \
 	plain_table_db_test \
 	comparator_db_test \
-	external_sst_file_test \
 	prefix_test \
 	skiplist_test \
 	stringappend_test \
 	ttl_test \
-	date_tiered_test \
 	backupable_db_test \
 	document_db_test \
 	json_document_test \
-	sim_cache_test \
 	spatial_db_test \
 	version_edit_test \
 	version_set_test \
@@ -393,13 +327,14 @@ TESTS = \
 	write_controller_test\
 	deletefile_test \
 	table_test \
+	thread_local_test \
 	geodb_test \
+	rate_limiter_test \
 	delete_scheduler_test \
 	options_test \
 	options_settable_test \
 	options_util_test \
 	event_logger_test \
-	timer_queue_test \
 	cuckoo_table_builder_test \
 	cuckoo_table_reader_test \
 	cuckoo_table_db_test \
@@ -410,40 +345,28 @@ TESTS = \
 	compaction_job_test \
 	thread_list_test \
 	sst_dump_test \
-	column_aware_encoding_test \
 	compact_files_test \
+	perf_context_test \
 	optimistic_transaction_test \
 	write_callback_test \
 	heap_test \
 	compact_on_deletion_collector_test \
 	compaction_job_stats_test \
-	option_change_migration_test \
 	transaction_test \
 	ldb_cmd_test \
-	persistent_cache_test \
-	statistics_test \
-	lua_test \
-	range_del_aggregator_test \
-	lru_cache_test \
-	object_registry_test \
-	repair_test \
-	env_timed_test \
+	iostats_context_test

 PARALLEL_TEST = \
 	backupable_db_test \
+	compact_on_deletion_collector_test \
 	db_compaction_filter_test \
 	db_compaction_test \
-	db_sst_test \
 	db_test \
 	db_universal_compaction_test \
-	db_wal_test \
-	external_sst_file_test \
 	fault_injection_test \
 	inlineskiplist_test \
 	manual_compaction_test \
-	persistent_cache_test \
-	table_test \
-	transaction_test
+	table_test

 SUBSET := $(TESTS)
 ifdef ROCKSDBTESTS_START
@@ -464,11 +387,8 @@ TOOLS = \
 	rocksdb_dump \
 	rocksdb_undump

-TEST_LIBS = \
-	librocksdb_env_basic_test.a
-
 # TODO: add back forward_iterator_bench, after making it build in all environemnts.
-BENCHMARKS = db_bench table_reader_bench cache_bench memtablerep_bench column_aware_encoding_exp persistent_cache_bench
+BENCHMARKS = db_bench table_reader_bench cache_bench memtablerep_bench

 # if user didn't config LIBNAME, set the default
 ifeq ($(LIBNAME),)
@@ -480,7 +400,6 @@ else
 endif
 endif
 LIBRARY = ${LIBNAME}.a
-TOOLS_LIBRARY = ${LIBNAME}_tools.a

 ROCKSDB_MAJOR = $(shell egrep "ROCKSDB_MAJOR.[0-9]" include/rocksdb/version.h | cut -d ' ' -f 3)
 ROCKSDB_MINOR = $(shell egrep "ROCKSDB_MINOR.[0-9]" include/rocksdb/version.h | cut -d ' ' -f 3)
@@ -524,7 +443,7 @@ $(SHARED3): $(SHARED4)
 endif

 $(SHARED4):
-	$(CXX) $(PLATFORM_SHARED_LDFLAGS)$(SHARED3) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(LIB_SOURCES) $(TOOL_LIB_SOURCES) \
+	$(CXX) $(PLATFORM_SHARED_LDFLAGS)$(SHARED3) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(LIB_SOURCES) $(TOOL_SOURCES) \
 		$(LDFLAGS) -o $@

 endif  # PLATFORM_SHARED_EXT
@@ -532,10 +451,10 @@ endif  # PLATFORM_SHARED_EXT
 .PHONY: blackbox_crash_test check clean coverage crash_test ldb_tests package \
 	release tags valgrind_check whitebox_crash_test format static_lib shared_lib all \
 	dbg rocksdbjavastatic rocksdbjava install install-static install-shared uninstall \
-	analyze tools tools_lib
+	analyze tools


-all: $(LIBRARY) $(BENCHMARKS) tools tools_lib test_libs $(TESTS)
+all: $(LIBRARY) $(BENCHMARKS) tools $(TESTS)

 static_lib: $(LIBRARY)

@@ -543,10 +462,6 @@ shared_lib: $(SHARED)

 tools: $(TOOLS)

-tools_lib: $(TOOLS_LIBRARY)
-
-test_libs: $(TEST_LIBS)
-
 dbg: $(LIBRARY) $(BENCHMARKS) tools $(TESTS)

 # creates static library and programs
@@ -616,7 +531,7 @@ $(parallel_tests): $(PARALLEL_TEST)
      '#!/bin/sh' \
      "d=\$(TMPD)$$TEST_SCRIPT" \
      'mkdir -p $$d' \
-      "TEST_TMPDIR=\$$d $(DRIVER) ./$$TEST_BINARY --gtest_filter=$$TEST_NAME" \
+			"TEST_TMPDIR=\$$d ./$$TEST_BINARY --gtest_filter=$$TEST_NAME" \
 		> $$TEST_SCRIPT; \
 		chmod a=rx $$TEST_SCRIPT; \
 	done
@@ -643,7 +558,7 @@ gen_parallel_tests:
 # 107.816 PASS t/DBTest.EncodeDecompressedBlockSizeTest
 #
 slow_test_regexp = \
-	^t/run-table_test-HarnessTest.Randomized$$|^t/run-db_test-.*(?:FileCreationRandomFailure|EncodeDecompressedBlockSizeTest)$$|^.*RecoverFromCorruptedWALWithoutFlush$$
+	^t/run-table_test-HarnessTest.Randomized$$|^t/run-db_test-.*(?:FileCreationRandomFailure|EncodeDecompressedBlockSizeTest)$$
 prioritize_long_running_tests =						\
  perl -pe 's,($(slow_test_regexp)),100 $$1,'				\
    | sort -k1,1gr							\
@@ -673,9 +588,7 @@ check_0:
 	} \
 	  | $(prioritize_long_running_tests)				\
 	  | grep -E '$(tests-regexp)'					\
-	  | build_tools/gnu_parallel -j$(J) --plain --joblog=LOG $$eta --gnu '{} >& t/log-{/}'
-
-valgrind-blacklist-regexp = InlineSkipTest.ConcurrentInsert|TransactionTest.DeadlockStress|DBCompactionTest.SuggestCompactRangeNoTwoLevel0Compactions|BackupableDBTest.RateLimiting|DBTest.CloseSpeedup|DBTest.ThreadStatusFlush|DBTest.RateLimitingTest|DBTest.EncodeDecompressedBlockSizeTest|FaultInjectionTest.UninstalledCompaction|HarnessTest.Randomized|ExternalSSTFileTest.CompactDuringAddFileRandom|ExternalSSTFileTest.IngestFileWithGlobalSeqnoRandomized
+	  | parallel -j$(J) --joblog=LOG $$eta --gnu '{} >& t/log-{/}'

 .PHONY: valgrind_check_0
 valgrind_check_0:
@@ -690,10 +603,9 @@ valgrind_check_0:
 	}								\
 	  | $(prioritize_long_running_tests)				\
 	  | grep -E '$(tests-regexp)'					\
-	  | grep -E -v '$(valgrind-blacklist-regexp)'					\
-	  | build_tools/gnu_parallel -j$(J) --plain --joblog=LOG $$eta --gnu \
-	  '(if [[ "{}" == "./"* ]] ; then $(DRIVER) {}; else {}; fi) ' \
-	  '>& t/valgrind_log-{/}'
+	  | parallel -j$(J) --joblog=LOG $$eta --gnu \
+      'if [[ "{}" == "./"* ]] ; then $(DRIVER) {} >& t/valgrind_log-{/}; ' \
+      'else {} >& t/valgrind_log-{/}; fi'

 CLEAN_FILES += t LOG $(TMPD)

@@ -712,7 +624,7 @@ watch-log:
 check: all
 	$(MAKE) gen_parallel_tests
 	$(AM_V_GEN)if test "$(J)" != 1                                  \
-	    && (build_tools/gnu_parallel --gnu --help 2>/dev/null) |                    \
+	    && (parallel --gnu --help 2>/dev/null) |                    \
 	        grep -q 'GNU Parallel';                                 \
 	then                                                            \
 	    $(MAKE) T="$$t" TMPD=$(TMPD) check_0;                       \
@@ -726,8 +638,7 @@ ifeq ($(filter -DROCKSDB_LITE,$(OPT)),)
 	sh tools/rocksdb_dump_test.sh
 endif

-# TODO add ldb_tests
-check_some: $(SUBSET)
+check_some: $(SUBSET) ldb_tests
 	for t in $(SUBSET); do echo "===== Running $$t"; ./$$t || exit 1; done

 .PHONY: ldb_tests
@@ -740,15 +651,9 @@ blackbox_crash_test: db_stress
 	python -u tools/db_crashtest.py --simple blackbox
 	python -u tools/db_crashtest.py blackbox

-ifeq ($(CRASH_TEST_KILL_ODD),)
-  CRASH_TEST_KILL_ODD=888887
-endif
-
 whitebox_crash_test: db_stress
-	python -u tools/db_crashtest.py --simple whitebox --random_kill_odd \
-      $(CRASH_TEST_KILL_ODD)
-	python -u tools/db_crashtest.py whitebox  --random_kill_odd \
-      $(CRASH_TEST_KILL_ODD)
+	python -u tools/db_crashtest.py --simple whitebox
+	python -u tools/db_crashtest.py whitebox

 asan_check:
 	$(MAKE) clean
@@ -770,13 +675,10 @@ ubsan_crash_test:
 	COMPILE_WITH_UBSAN=1 $(MAKE) crash_test
 	$(MAKE) clean

-valgrind_test:
-	DISABLE_JEMALLOC=1 $(MAKE) valgrind_check
-
 valgrind_check: $(TESTS)
-	$(MAKE) DRIVER="$(VALGRIND_VER) $(VALGRIND_OPTS)" gen_parallel_tests
+	$(MAKE) gen_parallel_tests
 	$(AM_V_GEN)if test "$(J)" != 1                                  \
-	    && (build_tools/gnu_parallel --gnu --help 2>/dev/null) |                    \
+	    && (parallel --gnu --help 2>/dev/null) |                    \
 	        grep -q 'GNU Parallel';                                 \
 	then                                                            \
      $(MAKE) TMPD=$(TMPD)                                        \
@@ -798,10 +700,10 @@ parloop:
 	for t in $(PAR_TEST); do		\
 		echo "===== Running $$t in parallel $(NUM_PAR)";\
 		if [ $(db_test) -eq 1 ]; then \
-			seq $(J) | v="$$t" build_tools/gnu_parallel --gnu --plain 's=$(TMPD)/rdb-{};  export TEST_TMPDIR=$$s;' \
+			seq $(J) | v="$$t" parallel --gnu 's=$(TMPD)/rdb-{};  export TEST_TMPDIR=$$s;' \
 				'timeout 2m ./db_test --gtest_filter=$$v >> $$s/log-{} 2>1'; \
 		else\
-			seq $(J) | v="./$$t" build_tools/gnu_parallel --gnu --plain 's=$(TMPD)/rdb-{};' \
+			seq $(J) | v="./$$t" parallel --gnu 's=$(TMPD)/rdb-{};' \
 			     'export TEST_TMPDIR=$$s; timeout 10m $$v >> $$s/log-{} 2>1'; \
 		fi; \
 		ret_code=$$?; \
@@ -822,7 +724,7 @@ test_names = \

 parallel_check: $(TESTS)
 	$(AM_V_GEN)if test "$(J)" > 1                                  \
-	    && (build_tools/gnu_parallel --gnu --help 2>/dev/null) |                    \
+	    && (parallel --gnu --help 2>/dev/null) |                    \
 	        grep -q 'GNU Parallel';                                 \
 	then                                                            \
 	    echo Running in parallel $(J);			\
@@ -832,7 +734,7 @@ parallel_check: $(TESTS)
 	ret_bad=0;							\
 	echo $(J);\
 	echo Test Dir: $(TMPD); \
-        seq $(J) | build_tools/gnu_parallel --gnu --plain 's=$(TMPD)/rdb-{}; rm -rf $$s; mkdir $$s'; \
+        seq $(J) | parallel --gnu 's=$(TMPD)/rdb-{}; rm -rf $$s; mkdir $$s'; \
 	$(MAKE)  PAR_TEST="$(shell $(test_names))" TMPD=$(TMPD) \
 		J=$(J) db_test=1 parloop; \
 	$(MAKE) PAR_TEST="$(filter-out db_test, $(TESTS))" \
@@ -871,7 +773,6 @@ clean:
 	find . -name "*.[oda]" -exec rm -f {} \;
 	find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm {} \;
 	rm -rf bzip2* snappy* zlib* lz4*
-	cd java; $(MAKE) clean

 tags:
 	ctags * -R
@@ -890,24 +791,16 @@ $(LIBRARY): $(LIBOBJECTS)
 	$(AM_V_AR)rm -f $@
 	$(AM_V_at)$(AR) $(ARFLAGS) $@ $(LIBOBJECTS)

-$(TOOLS_LIBRARY): $(BENCH_LIB_SOURCES:.cc=.o) $(TOOL_LIB_SOURCES:.cc=.o) $(LIB_SOURCES:.cc=.o) $(TESTUTIL)
-	$(AM_V_AR)rm -f $@
-	$(AM_V_at)$(AR) $(ARFLAGS) $@ $^
-
-librocksdb_env_basic_test.a: env/env_basic_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_V_AR)rm -f $@
-	$(AM_V_at)$(AR) $(ARFLAGS) $@ $^
-
 db_bench: tools/db_bench.o $(BENCHTOOLOBJECTS)
 	$(AM_LINK)

-cache_bench: cache/cache_bench.o $(LIBOBJECTS) $(TESTUTIL)
+cache_bench: util/cache_bench.o $(LIBOBJECTS) $(TESTUTIL)
 	$(AM_LINK)

-persistent_cache_bench: utilities/persistent_cache/persistent_cache_bench.o $(LIBOBJECTS) $(TESTUTIL)
+memtablerep_bench: db/memtablerep_bench.o $(LIBOBJECTS) $(TESTUTIL)
 	$(AM_LINK)

-memtablerep_bench: memtable/memtablerep_bench.o $(LIBOBJECTS) $(TESTUTIL)
+block_hash_index_test: table/block_hash_index_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

 db_stress: tools/db_stress.o $(LIBOBJECTS) $(TESTUTIL)
@@ -928,7 +821,7 @@ arena_test: util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS)
 autovector_test: util/autovector_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-column_family_test: db/column_family_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
+column_family_test: db/column_family_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

 table_properties_collector_test: db/table_properties_collector_test.o $(LIBOBJECTS) $(TESTHARNESS)
@@ -943,25 +836,19 @@ dynamic_bloom_test: util/dynamic_bloom_test.o $(LIBOBJECTS) $(TESTHARNESS)
 c_test: db/c_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-cache_test: cache/cache_test.o $(LIBOBJECTS) $(TESTHARNESS)
+cache_test: util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

 coding_test: util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-option_change_migration_test: utilities/option_change_migration/option_change_migration_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 stringappend_test: utilities/merge_operators/string_append/stringappend_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

 redis_test: utilities/redis/redis_lists_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-hash_table_test: utilities/persistent_cache/hash_table_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
-histogram_test: monitoring/histogram_test.o $(LIBOBJECTS) $(TESTHARNESS)
+histogram_test: util/histogram_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

 thread_local_test: util/thread_local_test.o $(LIBOBJECTS) $(TESTHARNESS)
@@ -976,9 +863,6 @@ crc32c_test: util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS)
 slice_transform_test: util/slice_transform_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-db_basic_test: db/db_basic_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 db_test: db/db_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

@@ -1003,36 +887,15 @@ db_compaction_test: db/db_compaction_test.o db/db_test_util.o $(LIBOBJECTS) $(TE
 db_dynamic_level_test: db/db_dynamic_level_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-db_flush_test: db/db_flush_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 db_inplace_update_test: db/db_inplace_update_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

 db_iterator_test: db/db_iterator_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-db_memtable_test: db/db_memtable_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
-db_merge_operator_test: db/db_merge_operator_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
-db_options_test: db/db_options_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
-db_range_del_test: db/db_range_del_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 db_sst_test: db/db_sst_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-external_sst_file_basic_test: db/external_sst_file_basic_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
-external_sst_file_test: db/external_sst_file_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 db_tailing_iter_test: db/db_tailing_iter_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

@@ -1045,9 +908,6 @@ db_universal_compaction_test: db/db_universal_compaction_test.o db/db_test_util.
 db_wal_test: db/db_wal_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-db_io_failure_test: db/db_io_failure_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 db_properties_test: db/db_properties_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

@@ -1084,32 +944,15 @@ document_db_test: utilities/document/document_db_test.o $(LIBOBJECTS) $(TESTHARN
 json_document_test: utilities/document/json_document_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-sim_cache_test: utilities/simulator_cache/sim_cache_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 spatial_db_test: utilities/spatialdb/spatial_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

 env_mirror_test: utilities/env_mirror_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-env_timed_test: utilities/env_timed_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
-ifdef ROCKSDB_USE_LIBRADOS
-env_librados_test: utilities/env_librados_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_V_CCLD)$(CXX) $^ $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
-endif
-
-object_registry_test: utilities/object_registry_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 ttl_test: utilities/ttl/ttl_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-date_tiered_test: utilities/date_tiered/date_tiered_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 write_batch_with_index_test: utilities/write_batch_with_index/write_batch_with_index_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

@@ -1134,10 +977,7 @@ wal_manager_test: db/wal_manager_test.o $(LIBOBJECTS) $(TESTHARNESS)
 dbformat_test: db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-env_basic_test: env/env_basic_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
-env_test: env/env_test.o $(LIBOBJECTS) $(TESTHARNESS)
+env_test: util/env_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

 fault_injection_test: db/fault_injection_test.o $(LIBOBJECTS) $(TESTHARNESS)
@@ -1161,25 +1001,19 @@ block_based_filter_block_test: table/block_based_filter_block_test.o $(LIBOBJECT
 full_filter_block_test: table/full_filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-partitioned_filter_block_test: table/partitioned_filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 log_test: db/log_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-cleanable_test: table/cleanable_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 table_test: table/table_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

 block_test: table/block_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-inlineskiplist_test: memtable/inlineskiplist_test.o $(LIBOBJECTS) $(TESTHARNESS)
+inlineskiplist_test: db/inlineskiplist_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-skiplist_test: memtable/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS)
+skiplist_test: db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

 version_edit_test: db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS)
@@ -1218,9 +1052,6 @@ merge_test: db/merge_test.o $(LIBOBJECTS) $(TESTHARNESS)
 merger_test: table/merger_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-util_merge_operators_test: utilities/util_merge_operators_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 options_file_test: db/options_file_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

@@ -1254,34 +1085,28 @@ thread_list_test: util/thread_list_test.o $(LIBOBJECTS) $(TESTHARNESS)
 compact_files_test: db/compact_files_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-options_test: options/options_test.o $(LIBOBJECTS) $(TESTHARNESS)
+options_test: util/options_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-options_settable_test: options/options_settable_test.o $(LIBOBJECTS) $(TESTHARNESS)
+options_settable_test: util/options_settable_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

 options_util_test: utilities/options/options_util_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-db_bench_tool_test: tools/db_bench_tool_test.o $(BENCHTOOLOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 event_logger_test: util/event_logger_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-timer_queue_test: util/timer_queue_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 sst_dump_test: tools/sst_dump_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-column_aware_encoding_test: utilities/column_aware_encoding_test.o $(TESTHARNESS) $(EXPOBJECTS)
+memenv_test : util/memenv_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

 optimistic_transaction_test: utilities/transactions/optimistic_transaction_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-mock_env_test : env/mock_env_test.o $(LIBOBJECTS) $(TESTHARNESS)
+mock_env_test : util/mock_env_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

 manual_compaction_test: db/manual_compaction_test.o $(LIBOBJECTS) $(TESTHARNESS)
@@ -1290,7 +1115,7 @@ manual_compaction_test: db/manual_compaction_test.o $(LIBOBJECTS) $(TESTHARNESS)
 filelock_test: util/filelock_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-auto_roll_logger_test: util/auto_roll_logger_test.o $(LIBOBJECTS) $(TESTHARNESS)
+auto_roll_logger_test: db/auto_roll_logger_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

 memtable_list_test: db/memtable_list_test.o $(LIBOBJECTS) $(TESTHARNESS)
@@ -1308,9 +1133,6 @@ transaction_test: utilities/transactions/transaction_test.o $(LIBOBJECTS) $(TEST
 sst_dump: tools/sst_dump.o $(LIBOBJECTS)
 	$(AM_LINK)

-column_aware_encoding_exp: utilities/column_aware_encoding_exp.o $(EXPOBJECTS)
-	$(AM_LINK)
-
 repair_test: db/repair_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

@@ -1320,24 +1142,9 @@ ldb_cmd_test: tools/ldb_cmd_test.o $(LIBOBJECTS) $(TESTHARNESS)
 ldb: tools/ldb.o $(LIBOBJECTS)
 	$(AM_LINK)

-iostats_context_test: monitoring/iostats_context_test.o $(LIBOBJECTS) $(TESTHARNESS)
+iostats_context_test: util/iostats_context_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_V_CCLD)$(CXX) $^ $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)

-persistent_cache_test: utilities/persistent_cache/persistent_cache_test.o  db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
-statistics_test: monitoring/statistics_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
-lru_cache_test: cache/lru_cache_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
-lua_test: utilities/lua/rocks_lua_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
-range_del_aggregator_test: db/range_del_aggregator_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 #-------------------------------------------------
 # make install related stuff
 INSTALL_PATH ?= /usr/local
@@ -1385,127 +1192,65 @@ ifeq ($(PLATFORM), OS_SOLARIS)
 else
 	ARCH := $(shell getconf LONG_BIT)
 endif
-
-ifeq (,$(findstring ppc,$(MACHINE)))
-        ROCKSDBJNILIB = librocksdbjni-linux$(ARCH).so
-else
-        ROCKSDBJNILIB = librocksdbjni-linux-$(MACHINE).so
-endif
+ROCKSDBJNILIB = librocksdbjni-linux$(ARCH).so
 ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-linux$(ARCH).jar
 ROCKSDB_JAR_ALL = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH).jar
 ROCKSDB_JAVADOCS_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-javadoc.jar
 ROCKSDB_SOURCES_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-sources.jar
-SHA256_CMD = sha256sum
-
-ZLIB_VER = 1.2.11
-ZLIB_SHA256 = c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1
-BZIP2_VER = 1.0.6
-BZIP2_SHA256 = a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd
-SNAPPY_VER = 1.1.4
-SNAPPY_SHA256 = 134bfe122fd25599bb807bb8130e7ba6d9bdb851e0b16efcb83ac4f5d0b70057
-LZ4_VER = 1.7.5
-LZ4_SHA256 = 0190cacd63022ccb86f44fa5041dc6c3804407ad61550ca21c382827319e7e7e
-ZSTD_VER ?= 1.2.0
-ZSTD_SHA256 ?= 4a7e4593a3638276ca7f2a09dc4f38e674d8317bbea51626393ca73fc047cbfb
-ZSTD_DOWNLOAD_BASE ?= https://github.com/facebook/zstd/archive

 ifeq ($(PLATFORM), OS_MACOSX)
 	ROCKSDBJNILIB = librocksdbjni-osx.jnilib
 	ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-osx.jar
-	SHA256_CMD = openssl sha256 -r
 ifneq ("$(wildcard $(JAVA_HOME)/include/darwin)","")
 	JAVA_INCLUDE = -I$(JAVA_HOME)/include -I $(JAVA_HOME)/include/darwin
 else
 	JAVA_INCLUDE = -I/System/Library/Frameworks/JavaVM.framework/Headers/
 endif
 endif
-ifeq ($(PLATFORM), OS_FREEBSD)
-	JAVA_INCLUDE += -I$(JAVA_HOME)/include/freebsd
-	ROCKSDBJNILIB = librocksdbjni-freebsd$(ARCH).so
-	ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-freebsd$(ARCH).jar
-endif
 ifeq ($(PLATFORM), OS_SOLARIS)
 	ROCKSDBJNILIB = librocksdbjni-solaris$(ARCH).so
 	ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-solaris$(ARCH).jar
 	JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/solaris
-	SHA256_CMD = digest -a sha256
 endif

 libz.a:
-	-rm -rf zlib-$(ZLIB_VER)
-	curl -O -L http://zlib.net/zlib-$(ZLIB_VER).tar.gz
-	ZLIB_SHA256_ACTUAL=`$(SHA256_CMD) zlib-$(ZLIB_VER).tar.gz | cut -d ' ' -f 1`; \
-	if [ "$(ZLIB_SHA256)" != "$$ZLIB_SHA256_ACTUAL" ]; then \
-		echo zlib-$(ZLIB_VER).tar.gz checksum mismatch, expected=\"$(ZLIB_SHA256)\" actual=\"$$ZLIB_SHA256_ACTUAL\"; \
-		exit 1; \
-	fi
-	tar xvzf zlib-$(ZLIB_VER).tar.gz
-	cd zlib-$(ZLIB_VER) && CFLAGS='-fPIC' ./configure --static && make
-	cp zlib-$(ZLIB_VER)/libz.a .
+	-rm -rf zlib-1.2.8
+	curl -O http://zlib.net/zlib-1.2.8.tar.gz
+	tar xvzf zlib-1.2.8.tar.gz
+	cd zlib-1.2.8 && CFLAGS='-fPIC' ./configure --static && make
+	cp zlib-1.2.8/libz.a .

 libbz2.a:
-	-rm -rf bzip2-$(BZIP2_VER)
-	curl -O -L http://www.bzip.org/$(BZIP2_VER)/bzip2-$(BZIP2_VER).tar.gz
-	BZIP2_SHA256_ACTUAL=`$(SHA256_CMD) bzip2-$(BZIP2_VER).tar.gz | cut -d ' ' -f 1`; \
-	if [ "$(BZIP2_SHA256)" != "$$BZIP2_SHA256_ACTUAL" ]; then \
-		echo bzip2-$(BZIP2_VER).tar.gz checksum mismatch, expected=\"$(BZIP2_SHA256)\" actual=\"$$BZIP2_SHA256_ACTUAL\"; \
-		exit 1; \
-	fi
-	tar xvzf bzip2-$(BZIP2_VER).tar.gz
-	cd bzip2-$(BZIP2_VER) && make CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64'
-	cp bzip2-$(BZIP2_VER)/libbz2.a .
+	-rm -rf bzip2-1.0.6
+	curl -O  http://www.bzip.org/1.0.6/bzip2-1.0.6.tar.gz
+	tar xvzf bzip2-1.0.6.tar.gz
+	cd bzip2-1.0.6 && make CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64'
+	cp bzip2-1.0.6/libbz2.a .

 libsnappy.a:
-	-rm -rf snappy-$(SNAPPY_VER)
-	curl -O -L https://github.com/google/snappy/releases/download/$(SNAPPY_VER)/snappy-$(SNAPPY_VER).tar.gz
-	SNAPPY_SHA256_ACTUAL=`$(SHA256_CMD) snappy-$(SNAPPY_VER).tar.gz | cut -d ' ' -f 1`; \
-	if [ "$(SNAPPY_SHA256)" != "$$SNAPPY_SHA256_ACTUAL" ]; then \
-		echo snappy-$(SNAPPY_VER).tar.gz checksum mismatch, expected=\"$(SNAPPY_SHA256)\" actual=\"$$SNAPPY_SHA256_ACTUAL\"; \
-		exit 1; \
-	fi
-	tar xvzf snappy-$(SNAPPY_VER).tar.gz
-	cd snappy-$(SNAPPY_VER) && ./configure --with-pic --enable-static
-	cd snappy-$(SNAPPY_VER) && make
-	cp snappy-$(SNAPPY_VER)/.libs/libsnappy.a .
+	-rm -rf snappy-1.1.1
+	curl -O https://snappy.googlecode.com/files/snappy-1.1.1.tar.gz
+	tar xvzf snappy-1.1.1.tar.gz
+	cd snappy-1.1.1 && ./configure --with-pic --enable-static
+	cd snappy-1.1.1 && make
+	cp snappy-1.1.1/.libs/libsnappy.a .

 liblz4.a:
-	-rm -rf lz4-$(LZ4_VER)
-	curl -O -L https://github.com/lz4/lz4/archive/v$(LZ4_VER).tar.gz
-	mv v$(LZ4_VER).tar.gz lz4-$(LZ4_VER).tar.gz
-	LZ4_SHA256_ACTUAL=`$(SHA256_CMD) lz4-$(LZ4_VER).tar.gz | cut -d ' ' -f 1`; \
-	if [ "$(LZ4_SHA256)" != "$$LZ4_SHA256_ACTUAL" ]; then \
-		echo lz4-$(LZ4_VER).tar.gz checksum mismatch, expected=\"$(LZ4_SHA256)\" actual=\"$$LZ4_SHA256_ACTUAL\"; \
-		exit 1; \
-	fi
-	tar xvzf lz4-$(LZ4_VER).tar.gz
-	cd lz4-$(LZ4_VER)/lib && make CFLAGS='-fPIC' all
-	cp lz4-$(LZ4_VER)/lib/liblz4.a .
-
-libzstd.a:
-	-rm -rf zstd-$(ZSTD_VER)
-	curl -O -L ${ZSTD_DOWNLOAD_BASE}/v$(ZSTD_VER).tar.gz
-	mv v$(ZSTD_VER).tar.gz zstd-$(ZSTD_VER).tar.gz
-	ZSTD_SHA256_ACTUAL=`$(SHA256_CMD) zstd-$(ZSTD_VER).tar.gz | cut -d ' ' -f 1`; \
-	if [ "$(ZSTD_SHA256)" != "$$ZSTD_SHA256_ACTUAL" ]; then \
-		echo zstd-$(ZSTD_VER).tar.gz checksum mismatch, expected=\"$(ZSTD_SHA256)\" actual=\"$$ZSTD_SHA256_ACTUAL\"; \
-		exit 1; \
-	fi
-	tar xvzf zstd-$(ZSTD_VER).tar.gz
-	cd zstd-$(ZSTD_VER)/lib && make CFLAGS='-fPIC -O2 ${EXTRA_CFLAGS}' all
-	cp zstd-$(ZSTD_VER)/lib/libzstd.a .
+	   -rm -rf lz4-r127
+	   curl -O https://codeload.github.com/Cyan4973/lz4/tar.gz/r127
+	   mv r127 lz4-r127.tar.gz
+	   tar xvzf lz4-r127.tar.gz
+	   cd lz4-r127/lib && make CFLAGS='-fPIC' all
+	   cp lz4-r127/lib/liblz4.a .

 # A version of each $(LIBOBJECTS) compiled with -fPIC and a fixed set of static compression libraries
 java_static_libobjects = $(patsubst %,jls/%,$(LIBOBJECTS))
 CLEAN_FILES += jls

-ifneq ($(ROCKSDB_JAVA_NO_COMPRESSION), 1)
-JAVA_COMPRESSIONS = libz.a libbz2.a libsnappy.a liblz4.a libzstd.a
-endif
+JAVA_STATIC_FLAGS = -DZLIB -DBZIP2 -DSNAPPY -DLZ4
+JAVA_STATIC_INCLUDES = -I./zlib-1.2.8 -I./bzip2-1.0.6 -I./snappy-1.1.1 -I./lz4-r127/lib

-JAVA_STATIC_FLAGS = -DZLIB -DBZIP2 -DSNAPPY -DLZ4 -DZSTD
-JAVA_STATIC_INCLUDES = -I./zlib-$(ZLIB_VER) -I./bzip2-$(BZIP2_VER) -I./snappy-$(SNAPPY_VER) -I./lz4-$(LZ4_VER)/lib -I./zstd-$(ZSTD_VER)/lib
-
-$(java_static_libobjects): jls/%.o: %.cc $(JAVA_COMPRESSIONS)
+$(java_static_libobjects): jls/%.o: %.cc libz.a libbz2.a libsnappy.a liblz4.a
 	$(AM_V_CC)mkdir -p $(@D) && $(CXX) $(CXXFLAGS) $(JAVA_STATIC_FLAGS) $(JAVA_STATIC_INCLUDES) -fPIC -c $< -o $@ $(COVERAGEFLAGS)

 rocksdbjavastatic: $(java_static_libobjects)
@@ -1514,7 +1259,7 @@ rocksdbjavastatic: $(java_static_libobjects)
 	$(CXX) $(CXXFLAGS) -I./java/. $(JAVA_INCLUDE) -shared -fPIC \
 	  -o ./java/target/$(ROCKSDBJNILIB) $(JNI_NATIVE_SOURCES) \
 	  $(java_static_libobjects) $(COVERAGEFLAGS) \
-	  $(JAVA_COMPRESSIONS) $(JAVA_STATIC_LDFLAGS)
+	  libz.a libbz2.a libsnappy.a liblz4.a $(JAVA_STATIC_LDFLAGS)
 	cd java/target;strip -S -x $(ROCKSDBJNILIB)
 	cd java;jar -cf target/$(ROCKSDB_JAR) HISTORY*.md
 	cd java/target;jar -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB)
@@ -1528,32 +1273,12 @@ rocksdbjavastaticrelease: rocksdbjavastatic
 	cd java/target;jar -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib
 	cd java/target/classes;jar -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class

-rocksdbjavastaticreleasedocker: rocksdbjavastatic
-	DOCKER_LINUX_X64_CONTAINER=`docker ps -aqf name=rocksdb_linux_x64-be`; \
-	if [ -z "$$DOCKER_LINUX_X64_CONTAINER" ]; then \
-		docker container create --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host --name rocksdb_linux_x64-be evolvedbinary/rocksjava:centos5_x64-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh; \
-	fi
-	docker start -a rocksdb_linux_x64-be
-	DOCKER_LINUX_X86_CONTAINER=`docker ps -aqf name=rocksdb_linux_x86-be`; \
-	if [ -z "$$DOCKER_LINUX_X86_CONTAINER" ]; then \
-		docker container create --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host --name rocksdb_linux_x86-be evolvedbinary/rocksjava:centos5_x86-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh; \
-	fi
-	docker start -a rocksdb_linux_x86-be
-	cd java;jar -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md
-	cd java/target;jar -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib
-	cd java/target/classes;jar -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class
-
-rocksdbjavastaticpublish: rocksdbjavastaticrelease rocksdbjavastaticpublishcentral
-
-rocksdbjavastaticpublishdocker: rocksdbjavastaticreleasedocker rocksdbjavastaticpublishcentral
-
-rocksdbjavastaticpublishcentral:
+rocksdbjavastaticpublish: rocksdbjavastaticrelease
 	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/rocksjni.pom -Dfile=java/target/rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-javadoc.jar -Dclassifier=javadoc
 	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/rocksjni.pom -Dfile=java/target/rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-sources.jar -Dclassifier=sources
 	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/rocksjni.pom -Dfile=java/target/rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-linux64.jar -Dclassifier=linux64
 	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/rocksjni.pom -Dfile=java/target/rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-linux32.jar -Dclassifier=linux32
 	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/rocksjni.pom -Dfile=java/target/rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-osx.jar -Dclassifier=osx
-	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/rocksjni.pom -Dfile=java/target/rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-win64.jar -Dclassifier=win64
 	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/rocksjni.pom -Dfile=java/target/rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH).jar

 # A version of each $(LIBOBJECTS) compiled with -fPIC
@@ -1574,12 +1299,6 @@ rocksdbjava: $(java_libobjects)
 jclean:
 	cd java;$(MAKE) clean;

-jtest_compile: rocksdbjava
-	cd java;$(MAKE) java_test
-
-jtest_run:
-	cd java;$(MAKE) run_test
-
 jtest: rocksdbjava
 	cd java;$(MAKE) sample;$(MAKE) test;

@@ -1588,9 +1307,16 @@ jdb_bench:

 commit_prereq: build_tools/rocksdb-lego-determinator \
               build_tools/precommit_checker.py
-	J=$(J) build_tools/precommit_checker.py unit unit_481 clang_unit release release_481 clang_release tsan asan ubsan lite unit_non_shm
+	J=$(J) build_tools/precommit_checker.py unit unit_481 clang_unit tsan asan ubsan lite
 	$(MAKE) clean && $(MAKE) jclean && $(MAKE) rocksdbjava;

+xfunc:
+	for xftest in $(XFUNC_TESTS); do \
+		echo "===== Running xftest $$xftest"; \
+		make check ROCKSDB_XFUNC_TEST="$$xftest" tests-regexp="DBTest" ;\
+	done
+
+
 # ---------------------------------------------------------------------------
 #  	Platform-specific compilation
 # ---------------------------------------------------------------------------
@@ -1629,7 +1355,7 @@ endif
 #  	Source files dependencies detection
 # ---------------------------------------------------------------------------

-all_sources = $(LIB_SOURCES) $(MAIN_SOURCES) $(MOCK_LIB_SOURCES) $(TOOL_LIB_SOURCES) $(BENCH_LIB_SOURCES) $(TEST_LIB_SOURCES) $(EXP_LIB_SOURCES)
+all_sources = $(LIB_SOURCES) $(TEST_BENCH_SOURCES) $(MOCK_SOURCES)
 DEPFILES = $(all_sources:.cc=.d)

 # Add proper dependency support so changing a .h file forces a .cc file to
@@ -1,8 +1,6 @@
 ## RocksDB: A Persistent Key-Value Store for Flash and RAM Storage

 [![Build Status](https://travis-ci.org/facebook/rocksdb.svg?branch=master)](https://travis-ci.org/facebook/rocksdb)
-[![Build status](https://ci.appveyor.com/api/projects/status/fbgfu0so3afcno78/branch/master?svg=true)](https://ci.appveyor.com/project/Facebook/rocksdb/branch/master)
-

 RocksDB is developed and maintained by Facebook Database Engineering Team.
 It is built on earlier work on LevelDB by Sanjay Ghemawat (sanjay@google.com)
@@ -1,476 +0,0 @@
-REPO_PATH = "internal_repo_rocksdb/repo/"
-BUCK_BINS = "buck-out/gen/" + REPO_PATH
-TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh"
-rocksdb_compiler_flags = [
-  "-msse",
-  "-msse4.2",
-  "-fno-builtin-memcmp",
-  "-DROCKSDB_PLATFORM_POSIX",
-  "-DROCKSDB_LIB_IO_POSIX",
-  "-DROCKSDB_FALLOCATE_PRESENT",
-  "-DROCKSDB_MALLOC_USABLE_SIZE",
-  "-DOS_LINUX",
-  # Flags to enable libs we include
-  "-DSNAPPY",
-  "-DZLIB",
-  "-DBZIP2",
-  "-DLZ4",
-  "-DZSTD",
-  "-DGFLAGS=gflags",
-  "-DNUMA",
-  "-DTBB",
-  # Needed to compile in fbcode
-  "-Wno-expansion-to-defined",
-]
-
-rocksdb_external_deps = [
-  ('bzip2', None, 'bz2'),
-  ('snappy', None, "snappy"),
-  ('zlib', None, 'z'),
-  ('gflags', None, 'gflags'),
-  ('lz4', None, 'lz4'),
-  ('zstd', None),
-  ('tbb', None),
-  ("numa", "2.0.8", "numa"),
-  ("googletest", None, "gtest"),
-]
-
-rocksdb_preprocessor_flags = [
-  # Directories with files for #include
-  "-I" + REPO_PATH + "include/",
-  "-I" + REPO_PATH,
-]
-
-cpp_library(
-    name = "rocksdb_lib",
-    headers = AutoHeaders.RECURSIVE_GLOB,
-    srcs = [
-      "cache/clock_cache.cc",
-      "cache/lru_cache.cc",
-      "cache/sharded_cache.cc",
-      "db/builder.cc",
-      "db/c.cc",
-      "db/column_family.cc",
-      "db/compacted_db_impl.cc",
-      "db/compaction.cc",
-      "db/compaction_iterator.cc",
-      "db/compaction_job.cc",
-      "db/compaction_picker.cc",
-      "db/compaction_picker_universal.cc",
-      "db/convenience.cc",
-      "db/db_filesnapshot.cc",
-      "db/db_impl.cc",
-      "db/db_impl_write.cc",
-      "db/db_impl_compaction_flush.cc",
-      "db/db_impl_files.cc",
-      "db/db_impl_open.cc",
-      "db/db_impl_debug.cc",
-      "db/db_impl_experimental.cc",
-      "db/db_impl_readonly.cc",
-      "db/db_info_dumper.cc",
-      "db/db_iter.cc",
-      "db/dbformat.cc",
-      "db/event_helpers.cc",
-      "db/experimental.cc",
-      "db/external_sst_file_ingestion_job.cc",
-      "db/file_indexer.cc",
-      "db/flush_job.cc",
-      "db/flush_scheduler.cc",
-      "db/forward_iterator.cc",
-      "db/internal_stats.cc",
-      "db/log_reader.cc",
-      "db/log_writer.cc",
-      "db/managed_iterator.cc",
-      "db/memtable.cc",
-      "db/memtable_list.cc",
-      "db/merge_helper.cc",
-      "db/merge_operator.cc",
-      "db/range_del_aggregator.cc",
-      "db/repair.cc",
-      "db/snapshot_impl.cc",
-      "db/table_cache.cc",
-      "db/table_properties_collector.cc",
-      "db/transaction_log_impl.cc",
-      "db/version_builder.cc",
-      "db/version_edit.cc",
-      "db/version_set.cc",
-      "db/wal_manager.cc",
-      "db/write_batch.cc",
-      "db/write_batch_base.cc",
-      "db/write_controller.cc",
-      "db/write_thread.cc",
-      "env/env.cc",
-      "env/env_chroot.cc",
-      "env/env_hdfs.cc",
-      "env/env_posix.cc",
-      "env/io_posix.cc",
-      "env/memenv.cc",
-      "memtable/hash_cuckoo_rep.cc",
-      "memtable/hash_linklist_rep.cc",
-      "memtable/hash_skiplist_rep.cc",
-      "memtable/memtable_allocator.cc",
-      "memtable/skiplistrep.cc",
-      "memtable/vectorrep.cc",
-      "monitoring/histogram.cc",
-      "monitoring/histogram_windowing.cc",
-      "monitoring/instrumented_mutex.cc",
-      "monitoring/iostats_context.cc",
-      "monitoring/perf_context.cc",
-      "monitoring/perf_level.cc",
-      "monitoring/statistics.cc",
-      "monitoring/thread_status_impl.cc",
-      "monitoring/thread_status_updater.cc",
-      "monitoring/thread_status_updater_debug.cc",
-      "monitoring/thread_status_util.cc",
-      "monitoring/thread_status_util_debug.cc",
-      "options/cf_options.cc",
-      "options/db_options.cc",
-      "options/options.cc",
-      "options/options_helper.cc",
-      "options/options_parser.cc",
-      "options/options_sanity_check.cc",
-      "port/port_posix.cc",
-      "port/stack_trace.cc",
-      "table/adaptive_table_factory.cc",
-      "table/block.cc",
-      "table/block_based_filter_block.cc",
-      "table/block_based_table_builder.cc",
-      "table/block_based_table_factory.cc",
-      "table/block_based_table_reader.cc",
-      "table/block_builder.cc",
-      "table/block_prefix_index.cc",
-      "table/bloom_block.cc",
-      "table/cuckoo_table_builder.cc",
-      "table/cuckoo_table_factory.cc",
-      "table/cuckoo_table_reader.cc",
-      "table/flush_block_policy.cc",
-      "table/format.cc",
-      "table/full_filter_block.cc",
-      "table/get_context.cc",
-      "table/index_builder.cc",
-      "table/iterator.cc",
-      "table/merging_iterator.cc",
-      "table/meta_blocks.cc",
-      "table/partitioned_filter_block.cc",
-      "table/persistent_cache_helper.cc",
-      "table/plain_table_builder.cc",
-      "table/plain_table_factory.cc",
-      "table/plain_table_index.cc",
-      "table/plain_table_key_coding.cc",
-      "table/plain_table_reader.cc",
-      "table/sst_file_writer.cc",
-      "table/table_properties.cc",
-      "table/two_level_iterator.cc",
-      "tools/dump/db_dump_tool.cc",
-      "util/arena.cc",
-      "util/auto_roll_logger.cc",
-      "util/bloom.cc",
-      "util/build_version.cc",
-      "util/coding.cc",
-      "util/compaction_job_stats_impl.cc",
-      "util/comparator.cc",
-      "util/concurrent_arena.cc",
-      "util/crc32c.cc",
-      "util/delete_scheduler.cc",
-      "util/dynamic_bloom.cc",
-      "util/event_logger.cc",
-      "util/file_reader_writer.cc",
-      "util/file_util.cc",
-      "util/filename.cc",
-      "util/filter_policy.cc",
-      "util/hash.cc",
-      "util/log_buffer.cc",
-      "util/murmurhash.cc",
-      "util/random.cc",
-      "util/rate_limiter.cc",
-      "util/slice.cc",
-      "util/sst_file_manager_impl.cc",
-      "util/status.cc",
-      "util/status_message.cc",
-      "util/string_util.cc",
-      "util/sync_point.cc",
-      "util/thread_local.cc",
-      "util/threadpool_imp.cc",
-      "util/transaction_test_util.cc",
-      "util/xxhash.cc",
-      "utilities/backupable/backupable_db.cc",
-      "utilities/blob_db/blob_db.cc",
-      "utilities/blob_db/blob_db_impl.cc",
-      "utilities/blob_db/blob_db_options_impl.cc",
-      "utilities/blob_db/blob_file.cc",
-      "utilities/blob_db/blob_log_reader.cc",
-      "utilities/blob_db/blob_log_writer.cc",
-      "utilities/blob_db/blob_log_format.cc",
-      "utilities/checkpoint/checkpoint.cc",
-      "utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc",
-      "utilities/convenience/info_log_finder.cc",
-      "utilities/date_tiered/date_tiered_db_impl.cc",
-      "utilities/document/document_db.cc",
-      "utilities/document/json_document.cc",
-      "utilities/document/json_document_builder.cc",
-      "utilities/env_mirror.cc",
-      "utilities/env_timed.cc",
-      "utilities/geodb/geodb_impl.cc",
-      "utilities/leveldb_options/leveldb_options.cc",
-      "utilities/lua/rocks_lua_compaction_filter.cc",
-      "utilities/memory/memory_util.cc",
-      "utilities/merge_operators/max.cc",
-      "utilities/merge_operators/put.cc",
-      "utilities/merge_operators/string_append/stringappend.cc",
-      "utilities/merge_operators/string_append/stringappend2.cc",
-      "utilities/merge_operators/uint64add.cc",
-      "utilities/option_change_migration/option_change_migration.cc",
-      "utilities/options/options_util.cc",
-      "utilities/persistent_cache/block_cache_tier.cc",
-      "utilities/persistent_cache/block_cache_tier_file.cc",
-      "utilities/persistent_cache/block_cache_tier_metadata.cc",
-      "utilities/persistent_cache/persistent_cache_tier.cc",
-      "utilities/persistent_cache/volatile_tier_impl.cc",
-      "utilities/redis/redis_lists.cc",
-      "utilities/simulator_cache/sim_cache.cc",
-      "utilities/spatialdb/spatial_db.cc",
-      "utilities/table_properties_collectors/compact_on_deletion_collector.cc",
-      "utilities/transactions/optimistic_transaction_db_impl.cc",
-      "utilities/transactions/optimistic_transaction_impl.cc",
-      "utilities/transactions/transaction_base.cc",
-      "utilities/transactions/transaction_db_impl.cc",
-      "utilities/transactions/transaction_db_mutex_impl.cc",
-      "utilities/transactions/transaction_impl.cc",
-      "utilities/transactions/transaction_lock_mgr.cc",
-      "utilities/transactions/transaction_util.cc",
-      "utilities/ttl/db_ttl_impl.cc",
-      "utilities/write_batch_with_index/write_batch_with_index.cc",
-      "utilities/write_batch_with_index/write_batch_with_index_internal.cc",
-      "tools/ldb_cmd.cc",
-      "tools/ldb_tool.cc",
-      "tools/sst_dump_tool.cc",
-    ],
-    deps = [],
-    preprocessor_flags = rocksdb_preprocessor_flags,
-    compiler_flags = rocksdb_compiler_flags,
-    external_deps = rocksdb_external_deps,
-)
-
-cpp_library(
-    name = "rocksdb_test_lib",
-    headers = AutoHeaders.RECURSIVE_GLOB,
-    srcs = [
-      "env/mock_env.cc",
-      "table/mock_table.cc",
-      "util/fault_injection_test_env.cc",
-      "util/testharness.cc",
-      "util/testutil.cc",
-      "db/db_test_util.cc",
-      "utilities/col_buf_encoder.cc",
-      "utilities/col_buf_decoder.cc",
-      "utilities/column_aware_encoding_util.cc",
-    ],
-    deps = [":rocksdb_lib"],
-    preprocessor_flags = rocksdb_preprocessor_flags,
-    compiler_flags = rocksdb_compiler_flags,
-    external_deps = rocksdb_external_deps,
-)
-
-cpp_library(
-    name = "rocksdb_tools_lib",
-    headers = AutoHeaders.RECURSIVE_GLOB,
-    srcs = [
-      "tools/db_bench_tool.cc",
-      "util/testutil.cc",
-    ],
-    deps = [":rocksdb_lib"],
-    preprocessor_flags = rocksdb_preprocessor_flags,
-    compiler_flags = rocksdb_compiler_flags,
-    external_deps = rocksdb_external_deps,
-)
-
-cpp_library(
-    name = "env_basic_test_lib",
-    headers = AutoHeaders.RECURSIVE_GLOB,
-    srcs = ["env/env_basic_test.cc"],
-    deps = [":rocksdb_test_lib"],
-    preprocessor_flags = rocksdb_preprocessor_flags,
-    compiler_flags = rocksdb_compiler_flags,
-    external_deps = rocksdb_external_deps,
-)
-
-# [test_name, test_src, test_type]
-ROCKS_TESTS = [['merger_test', 'table/merger_test.cc', 'serial'],
- ['cache_test', 'cache/cache_test.cc', 'serial'],
- ['options_file_test', 'db/options_file_test.cc', 'serial'],
- ['compaction_picker_test', 'db/compaction_picker_test.cc', 'serial'],
- ['corruption_test', 'db/corruption_test.cc', 'serial'],
- ['thread_list_test', 'util/thread_list_test.cc', 'serial'],
- ['table_properties_collector_test',
-  'db/table_properties_collector_test.cc',
-  'serial'],
- ['document_db_test', 'utilities/document/document_db_test.cc', 'serial'],
- ['event_logger_test', 'util/event_logger_test.cc', 'serial'],
- ['coding_test', 'util/coding_test.cc', 'serial'],
- ['statistics_test', 'monitoring/statistics_test.cc', 'serial'],
- ['options_settable_test', 'options/options_settable_test.cc', 'serial'],
- ['sst_dump_test', 'tools/sst_dump_test.cc', 'serial'],
- ['column_aware_encoding_test',
-  'utilities/column_aware_encoding_test.cc',
-  'serial'],
- ['db_iterator_test', 'db/db_iterator_test.cc', 'serial'],
- ['db_sst_test', 'db/db_sst_test.cc', 'parallel'],
- ['geodb_test', 'utilities/geodb/geodb_test.cc', 'serial'],
- ['listener_test', 'db/listener_test.cc', 'serial'],
- ['write_callback_test', 'db/write_callback_test.cc', 'serial'],
- ['version_set_test', 'db/version_set_test.cc', 'serial'],
- ['full_filter_block_test', 'table/full_filter_block_test.cc', 'serial'],
- ['cleanable_test', 'table/cleanable_test.cc', 'serial'],
- ['checkpoint_test', 'utilities/checkpoint/checkpoint_test.cc', 'serial'],
- ['compact_files_test', 'db/compact_files_test.cc', 'serial'],
- ['db_options_test', 'db/db_options_test.cc', 'serial'],
- ['object_registry_test', 'utilities/object_registry_test.cc', 'serial'],
- ['auto_roll_logger_test', 'util/auto_roll_logger_test.cc', 'serial'],
- ['dbformat_test', 'db/dbformat_test.cc', 'serial'],
- ['write_batch_with_index_test',
-  'utilities/write_batch_with_index/write_batch_with_index_test.cc',
-  'serial'],
- ['json_document_test', 'utilities/document/json_document_test.cc', 'serial'],
- ['file_reader_writer_test', 'util/file_reader_writer_test.cc', 'serial'],
- ['repair_test', 'db/repair_test.cc', 'serial'],
- ['persistent_cache_test',
-  'utilities/persistent_cache/persistent_cache_test.cc',
-  'parallel'],
- ['db_bloom_filter_test', 'db/db_bloom_filter_test.cc', 'serial'],
- ['external_sst_file_basic_test',
-  'db/external_sst_file_basic_test.cc',
-  'serial'],
- ['options_test', 'options/options_test.cc', 'serial'],
- ['perf_context_test', 'db/perf_context_test.cc', 'serial'],
- ['db_block_cache_test', 'db/db_block_cache_test.cc', 'serial'],
- ['heap_test', 'util/heap_test.cc', 'serial'],
- ['db_test2', 'db/db_test2.cc', 'serial'],
- ['filelock_test', 'util/filelock_test.cc', 'serial'],
- ['write_controller_test', 'db/write_controller_test.cc', 'serial'],
- ['compaction_iterator_test', 'db/compaction_iterator_test.cc', 'serial'],
- ['spatial_db_test', 'utilities/spatialdb/spatial_db_test.cc', 'serial'],
- ['c_test', 'db/c_test.c', 'serial'],
- ['range_del_aggregator_test', 'db/range_del_aggregator_test.cc', 'serial'],
- ['date_tiered_test', 'utilities/date_tiered/date_tiered_test.cc', 'serial'],
- ['ldb_cmd_test', 'tools/ldb_cmd_test.cc', 'serial'],
- ['db_test', 'db/db_test.cc', 'parallel'],
- ['block_based_filter_block_test',
-  'table/block_based_filter_block_test.cc',
-  'serial'],
- ['merge_test', 'db/merge_test.cc', 'serial'],
- ['bloom_test', 'util/bloom_test.cc', 'serial'],
- ['block_test', 'table/block_test.cc', 'serial'],
- ['cuckoo_table_builder_test', 'table/cuckoo_table_builder_test.cc', 'serial'],
- ['backupable_db_test',
-  'utilities/backupable/backupable_db_test.cc',
-  'parallel'],
- ['db_flush_test', 'db/db_flush_test.cc', 'serial'],
- ['filename_test', 'db/filename_test.cc', 'serial'],
- ['cuckoo_table_reader_test', 'table/cuckoo_table_reader_test.cc', 'serial'],
- ['slice_transform_test', 'util/slice_transform_test.cc', 'serial'],
- ['cuckoo_table_db_test', 'db/cuckoo_table_db_test.cc', 'serial'],
- ['inlineskiplist_test', 'memtable/inlineskiplist_test.cc', 'parallel'],
- ['optimistic_transaction_test',
-  'utilities/transactions/optimistic_transaction_test.cc',
-  'serial'],
- ['hash_table_test',
-  'utilities/persistent_cache/hash_table_test.cc',
-  'serial'],
- ['db_dynamic_level_test', 'db/db_dynamic_level_test.cc', 'serial'],
- ['option_change_migration_test',
-  'utilities/option_change_migration/option_change_migration_test.cc',
-  'serial'],
- ['db_inplace_update_test', 'db/db_inplace_update_test.cc', 'serial'],
- ['autovector_test', 'util/autovector_test.cc', 'serial'],
- ['db_iter_test', 'db/db_iter_test.cc', 'serial'],
- ['flush_job_test', 'db/flush_job_test.cc', 'serial'],
- ['wal_manager_test', 'db/wal_manager_test.cc', 'serial'],
- ['write_batch_test', 'db/write_batch_test.cc', 'serial'],
- ['crc32c_test', 'util/crc32c_test.cc', 'serial'],
- ['rate_limiter_test', 'util/rate_limiter_test.cc', 'serial'],
- ['external_sst_file_test', 'db/external_sst_file_test.cc', 'parallel'],
- ['compaction_job_test', 'db/compaction_job_test.cc', 'serial'],
- ['mock_env_test', 'env/mock_env_test.cc', 'serial'],
- ['db_table_properties_test', 'db/db_table_properties_test.cc', 'serial'],
- ['db_compaction_test', 'db/db_compaction_test.cc', 'parallel'],
- ['arena_test', 'util/arena_test.cc', 'serial'],
- ['stringappend_test',
-  'utilities/merge_operators/string_append/stringappend_test.cc',
-  'serial'],
- ['reduce_levels_test', 'tools/reduce_levels_test.cc', 'serial'],
- ['prefix_test', 'db/prefix_test.cc', 'serial'],
- ['ttl_test', 'utilities/ttl/ttl_test.cc', 'serial'],
- ['merge_helper_test', 'db/merge_helper_test.cc', 'serial'],
- ['file_indexer_test', 'db/file_indexer_test.cc', 'serial'],
- ['memory_test', 'utilities/memory/memory_test.cc', 'serial'],
- ['log_test', 'db/log_test.cc', 'serial'],
- ['env_timed_test', 'utilities/env_timed_test.cc', 'serial'],
- ['deletefile_test', 'db/deletefile_test.cc', 'serial'],
- ['partitioned_filter_block_test',
-  'table/partitioned_filter_block_test.cc',
-  'serial'],
- ['comparator_db_test', 'db/comparator_db_test.cc', 'serial'],
- ['compaction_job_stats_test', 'db/compaction_job_stats_test.cc', 'serial'],
- ['thread_local_test', 'util/thread_local_test.cc', 'serial'],
- ['version_builder_test', 'db/version_builder_test.cc', 'serial'],
- ['db_range_del_test', 'db/db_range_del_test.cc', 'serial'],
- ['table_test', 'table/table_test.cc', 'parallel'],
- ['db_tailing_iter_test', 'db/db_tailing_iter_test.cc', 'serial'],
- ['db_compaction_filter_test', 'db/db_compaction_filter_test.cc', 'parallel'],
- ['options_util_test', 'utilities/options/options_util_test.cc', 'serial'],
- ['dynamic_bloom_test', 'util/dynamic_bloom_test.cc', 'serial'],
- ['db_basic_test', 'db/db_basic_test.cc', 'serial'],
- ['db_merge_operator_test', 'db/db_merge_operator_test.cc', 'serial'],
- ['manual_compaction_test', 'db/manual_compaction_test.cc', 'parallel'],
- ['delete_scheduler_test', 'util/delete_scheduler_test.cc', 'serial'],
- ['transaction_test', 'utilities/transactions/transaction_test.cc', 'serial'],
- ['db_io_failure_test', 'db/db_io_failure_test.cc', 'serial'],
- ['db_log_iter_test', 'db/db_log_iter_test.cc', 'serial'],
- ['compact_on_deletion_collector_test',
-  'utilities/table_properties_collectors/compact_on_deletion_collector_test.cc',
-  'serial'],
- ['env_test', 'env/env_test.cc', 'serial'],
- ['db_wal_test', 'db/db_wal_test.cc', 'parallel'],
- ['sim_cache_test', 'utilities/simulator_cache/sim_cache_test.cc', 'serial'],
- ['db_memtable_test', 'db/db_memtable_test.cc', 'serial'],
- ['db_universal_compaction_test',
-  'db/db_universal_compaction_test.cc',
-  'parallel'],
- ['histogram_test', 'monitoring/histogram_test.cc', 'serial'],
- ['util_merge_operators_test',
-  'utilities/util_merge_operators_test.cc',
-  'serial'],
- ['fault_injection_test', 'db/fault_injection_test.cc', 'parallel'],
- ['env_basic_test', 'env/env_basic_test.cc', 'serial'],
- ['iostats_context_test', 'monitoring/iostats_context_test.cc', 'serial'],
- ['memtable_list_test', 'db/memtable_list_test.cc', 'serial'],
- ['column_family_test', 'db/column_family_test.cc', 'serial'],
- ['db_properties_test', 'db/db_properties_test.cc', 'serial'],
- ['version_edit_test', 'db/version_edit_test.cc', 'serial'],
- ['skiplist_test', 'memtable/skiplist_test.cc', 'serial'],
- ['lru_cache_test', 'cache/lru_cache_test.cc', 'serial'],
- ['plain_table_db_test', 'db/plain_table_db_test.cc', 'serial']]
-
-
-# Generate a test rule for each entry in ROCKS_TESTS
-for test_cfg in ROCKS_TESTS:
-    test_name = test_cfg[0]
-    test_cc = test_cfg[1]
-    ttype = "gtest" if test_cfg[2] == "parallel" else "simple"
-    test_bin = test_name + "_bin"
-
-    cpp_binary (
-      name = test_bin,
-      srcs = [test_cc],
-      deps = [":rocksdb_test_lib"],
-      preprocessor_flags = rocksdb_preprocessor_flags,
-      compiler_flags = rocksdb_compiler_flags,
-      external_deps = rocksdb_external_deps,
-    )
-
-    custom_unittest(
-      name = test_name,
-      type = ttype,
-      deps = [":" + test_bin],
-      command = [TEST_RUNNER, BUCK_BINS + test_bin]
-    )
@@ -1,16 +1,10 @@
 This document lists users of RocksDB and their use cases. If you are using RocksDB, please open a pull request and add yourself to the list.

 ## Facebook
-At Facebook, we use RocksDB as storage engines in multiple data management services and a backend for many different stateful services, including:
+At Facebook, we use RocksDB as a backend for many different stateful services. We're also experimenting with running RocksDB as a storage engine for two databases:

 1. MyRocks -- https://github.com/MySQLOnRocksDB/mysql-5.6
 2. MongoRocks -- https://github.com/mongodb-partners/mongo-rocks
-3. ZippyDB --  Facebook's distributed key-value store with Paxos-style replication, built on top of RocksDB.[*] https://www.youtube.com/watch?v=DfiN7pG0D0khtt
-4. Laser -- Laser is a high query throughput, low (millisecond) latency, key-value storage service built on top of RocksDB.[*]
-4. Dragon -- a distributed graph query engine. https://code.facebook.com/posts/1737605303120405/dragon-a-distributed-graph-query-engine/
-5. Stylus -- a low-level stream processing framework writtenin C++.[*]
-
-[*] https://research.facebook.com/publications/realtime-data-processing-at-facebook/

 ## LinkedIn
 Two different use cases at Linkedin are using RocksDB as a storage engine:
@@ -62,21 +56,3 @@ Pinterest's Object Retrieval System uses RocksDB for storage: https://www.youtub
 ## quasardb
 [quasardb](https://www.quasardb.net) is a high-performance, distributed, transactional key-value database that integrates well with in-memory analytics engines such as Apache Spark. 
 quasardb uses a heavily tuned RocksDB as its persistence layer.
-
-## Netflix
-[Netflix](http://techblog.netflix.com/2016/05/application-data-caching-using-ssds.html) Netflix uses RocksDB on AWS EC2 instances with local SSD drives to cache application data.
-
-## TiKV
-[TiKV](https://github.com/pingcap/tikv) is a GEO-replicated, high-performance, distributed, transactional key-value database. TiKV is powered by Rust and Raft. TiKV uses RocksDB as its persistence layer.
-
-## Apache Flink
-[Apache Flink](https://flink.apache.org/news/2016/03/08/release-1.0.0.html) uses RocksDB to store state locally on a machine.
-
-## Dgraph
-[Dgraph](https://github.com/dgraph-io/dgraph) is an open-source, scalable, distributed, low latency, high throughput Graph database .They use RocksDB to store state locally on a machine.
-
-## Uber
-[Uber](http://eng.uber.com/cherami/) uses RocksDB as a durable and and scalable task queue.
-
-## 360 Pika
-[360](http://www.360.cn/) [Pika](https://github.com/Qihoo360/pika) is a nosql compatible with redis. With the huge amount of data stored, redis may suffer for a capacity bottleneck, and pika was born for solving it. It has widely been widely used in many company
@@ -1,4 +1,3 @@
-# Vagrant file
 Vagrant.configure("2") do |config|

  config.vm.provider "virtualbox" do |v|
@@ -1,9 +1,8 @@
 version: 1.0.{build}
-image: Visual Studio 2015
 before_build:
 - md %APPVEYOR_BUILD_FOLDER%\build
 - cd %APPVEYOR_BUILD_FOLDER%\build
- cmake -G "Visual Studio 14 2015 Win64" -DOPTDBG=1 -DXPRESS=1 ..
+- cmake -G "Visual Studio 12 Win64" -DOPTDBG=1 ..
 - cd ..
 build:
  project: build\rocksdb.sln
@@ -11,5 +10,6 @@ build:
  verbosity: minimal
 test:
 test_script:
- ps: build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test2,db_test,env_basic_test,env_test -Concurrency 8
+- ps: build_tools\run_ci_db_test.ps1 -EnableRerun -Run db_test -Exclude DBTest.Randomized,DBTest.FileCreationRandomFailure -Concurrency 18
+- ps: build_tools\run_ci_db_test.ps1 -Run env_test -Concurrency 1

@@ -6,66 +6,33 @@
 * @phutil-library-version 2
 */

-if (class_exists('ArcanistWorkflow')) {
-  phutil_register_library_map(array(
-    '__library_version__' => 2,
-    'class' =>
-    array(
-      'ArcanistCpplintLinter' => 'cpp_linter/ArcanistCpplintLinter.php',
-      'BaseDirectoryScopedFormatLinter' => 'cpp_linter/BaseDirectoryScopedFormatLinter.php',
-      'FacebookArcanistConfiguration' => 'config/FacebookArcanistConfiguration.php',
-      'FacebookFbcodeLintEngine' => 'lint_engine/FacebookFbcodeLintEngine.php',
-      'FacebookFbcodeUnitTestEngine' => 'unit_engine/FacebookFbcodeUnitTestEngine.php',
-      'FacebookHowtoevenLintEngine' => 'lint_engine/FacebookHowtoevenLintEngine.php',
-      'FacebookHowtoevenLinter' => 'cpp_linter/FacebookHowtoevenLinter.php',
-      'FbcodeClangFormatLinter' => 'cpp_linter/FbcodeClangFormatLinter.php',
-      'FbcodeCppLinter' => 'cpp_linter/FbcodeCppLinter.php',
-    ),
-    'function' =>
-    array(
-    ),
-    'xmap' =>
-    array(
-      'ArcanistCpplintLinter' => 'ArcanistLinter',
-      'BaseDirectoryScopedFormatLinter' => 'ArcanistLinter',
-      'FacebookArcanistConfiguration' => 'ArcanistConfiguration',
-      'FacebookFbcodeLintEngine' => 'ArcanistLintEngine',
-      'FacebookFbcodeUnitTestEngine' => 'ArcanistBaseUnitTestEngine',
-      'FacebookHowtoevenLintEngine' => 'ArcanistLintEngine',
-      'FacebookHowtoevenLinter' => 'ArcanistLinter',
-      'FbcodeClangFormatLinter' => 'BaseDirectoryScopedFormatLinter',
-      'FbcodeCppLinter' => 'ArcanistLinter',
-    ),
-  ));
-} else {
-  phutil_register_library_map(array(
-    '__library_version__' => 2,
-    'class' =>
-    array(
-      'ArcanistCpplintLinter' => 'cpp_linter/ArcanistCpplintLinter.php',
-      'BaseDirectoryScopedFormatLinter' => 'cpp_linter/BaseDirectoryScopedFormatLinter.php',
-      'FacebookArcanistConfiguration' => 'config/FacebookOldArcanistConfiguration.php',
-      'FacebookFbcodeLintEngine' => 'lint_engine/FacebookFbcodeLintEngine.php',
-      'FacebookFbcodeUnitTestEngine' => 'unit_engine/FacebookOldFbcodeUnitTestEngine.php',
-      'FacebookHowtoevenLintEngine' => 'lint_engine/FacebookHowtoevenLintEngine.php',
-      'FacebookHowtoevenLinter' => 'cpp_linter/FacebookHowtoevenLinter.php',
-      'FbcodeClangFormatLinter' => 'cpp_linter/FbcodeClangFormatLinter.php',
-      'FbcodeCppLinter' => 'cpp_linter/FbcodeCppLinter.php',
-    ),
-    'function' =>
-    array(
-    ),
-    'xmap' =>
-    array(
-      'ArcanistCpplintLinter' => 'ArcanistLinter',
-      'BaseDirectoryScopedFormatLinter' => 'ArcanistLinter',
-      'FacebookArcanistConfiguration' => 'ArcanistConfiguration',
-      'FacebookFbcodeLintEngine' => 'ArcanistLintEngine',
-      'FacebookFbcodeUnitTestEngine' => 'ArcanistBaseUnitTestEngine',
-      'FacebookHowtoevenLintEngine' => 'ArcanistLintEngine',
-      'FacebookHowtoevenLinter' => 'ArcanistLinter',
-      'FbcodeClangFormatLinter' => 'BaseDirectoryScopedFormatLinter',
-      'FbcodeCppLinter' => 'ArcanistLinter',
-    ),
-  ));
-}
+phutil_register_library_map(array(
+  '__library_version__' => 2,
+  'class' =>
+  array(
+    'ArcanistCpplintLinter' => 'cpp_linter/ArcanistCpplintLinter.php',
+    'BaseDirectoryScopedFormatLinter' => 'cpp_linter/BaseDirectoryScopedFormatLinter.php',
+    'FacebookArcanistConfiguration' => 'config/FacebookArcanistConfiguration.php',
+    'FacebookFbcodeLintEngine' => 'lint_engine/FacebookFbcodeLintEngine.php',
+    'FacebookFbcodeUnitTestEngine' => 'unit_engine/FacebookFbcodeUnitTestEngine.php',
+    'FacebookHowtoevenLintEngine' => 'lint_engine/FacebookHowtoevenLintEngine.php',
+    'FacebookHowtoevenLinter' => 'cpp_linter/FacebookHowtoevenLinter.php',
+    'FbcodeClangFormatLinter' => 'cpp_linter/FbcodeClangFormatLinter.php',
+    'FbcodeCppLinter' => 'cpp_linter/FbcodeCppLinter.php',
+  ),
+  'function' =>
+  array(
+  ),
+  'xmap' =>
+  array(
+    'ArcanistCpplintLinter' => 'ArcanistLinter',
+    'BaseDirectoryScopedFormatLinter' => 'ArcanistLinter',
+    'FacebookArcanistConfiguration' => 'ArcanistConfiguration',
+    'FacebookFbcodeLintEngine' => 'ArcanistLintEngine',
+    'FacebookFbcodeUnitTestEngine' => 'ArcanistBaseUnitTestEngine',
+    'FacebookHowtoevenLintEngine' => 'ArcanistLintEngine',
+    'FacebookHowtoevenLinter' => 'ArcanistLinter',
+    'FbcodeClangFormatLinter' => 'BaseDirectoryScopedFormatLinter',
+    'FbcodeCppLinter' => 'ArcanistLinter',
+  ),
+));
@@ -4,40 +4,203 @@
 // LICENSE file in the root directory of this source tree. An additional grant
 // of patent rights can be found in the PATENTS file in the same directory.

-require('RocksDBCommonHelper.php');
-
-define("DIFF_COMMAND", "diff");
-
 class FacebookArcanistConfiguration extends ArcanistConfiguration {
-  public function getCustomArgumentsForCommand($command) {
-    if ($command == "land") {
-      return array(
-          'async' => array('help' => 'Just to make tools happy'));
-    }
-    return array();
-  }

  public function didRunWorkflow($command,
-                                 ArcanistWorkflow $workflow,
+                                 ArcanistBaseWorkflow $workflow,
                                 $error_code) {
-    // Default options don't terminate on failure, but that's what we want. In
-    // the current case we use assertions intentionally as "terminate on failure
-    // invariants".
-    assert_options(ASSERT_BAIL, true);
-
-    assert($workflow);
-    assert(strlen($command) > 0);
-
-    if ($command == DIFF_COMMAND && !$workflow->isRawDiffSource()) {
-      $diffID = $workflow->getDiffId();
-
-      // When submitting a diff this code path gets executed multiple times in
-      // a row. We only care about the case when ID for the diff is provided
-      // because that's what we need to apply the diff and trigger the tests.
-      if (strlen($diffID) > 0) {
-        assert(is_numeric($diffID));
-        startTestsInSandcastle(true /* $applyDiff */, $workflow, $diffID);
-      }
+    if ($command == 'diff' && !$workflow->isRawDiffSource()) {
+      $this->startTestsInJenkins($workflow);
+      $this->startTestsInSandcastle($workflow);
    }
  }
+
+  //////////////////////////////////////////////////////////////////////
+  /*  Run tests in sandcastle */
+  function postURL($diffID, $url) {
+    $cmd = 'echo \'{"diff_id": "' . $diffID . '", '
+           . '"name":"click here for sandcastle tests for D' . $diffID . '", '
+           . '"link":"' . $url . '"}\' | '
+           . 'http_proxy=fwdproxy.any.facebook.com:8080 '
+           . 'https_proxy=fwdproxy.any.facebook.com:8080 arc call-conduit '
+           . 'differential.updateunitresults';
+    shell_exec($cmd);
+  }
+
+  function updateTestCommand($diffID, $test, $status) {
+    $cmd = 'echo \'{"diff_id": "' . $diffID . '", '
+           . '"name":"' . $test . '", '
+           . '"result":"' . $status . '"}\' | '
+           . 'http_proxy=fwdproxy.any.facebook.com:8080 '
+           . 'https_proxy=fwdproxy.any.facebook.com:8080 arc call-conduit '
+           . 'differential.updateunitresults';
+    return $cmd;
+  }
+
+  function updateTest($diffID, $test) {
+    shell_exec($this->updateTestCommand($diffID, $test, "waiting"));
+  }
+
+  function getSteps($diffID, $username, $test) {
+    $arcrc_content = exec("cat ~/.arcrc | gzip -f | base64 -w0");
+
+    // Sandcastle machines don't have arc setup. We copy the user certificate
+    // and authenticate using that in sandcastle
+    $setup = array(
+      "name" => "Setup arcrc",
+      "shell" => "echo " . $arcrc_content . " | base64 --decode"
+                 . " | gzip -d > ~/.arcrc",
+      "user" => "root"
+    );
+
+    // arc demands certain permission on its config
+    $fix_permission = array(
+      "name" => "Fix environment",
+      "shell" => "chmod 600 ~/.arcrc",
+      "user" => "root"
+    );
+
+    // fbcode is a sub-repo. We cannot patch until we add it to ignore otherwise
+    // git thinks it is uncommited change
+    $fix_git_ignore = array(
+      "name" => "Fix git ignore",
+      "shell" => "echo fbcode >> .git/info/exclude",
+      "user" => "root"
+    );
+
+    // Patch the code (keep your fingures crossed)
+    $patch = array(
+      "name" => "Patch " . $diffID,
+      "shell" => "HTTPS_PROXY=fwdproxy:8080 arc --arcrc-file ~/.arcrc "
+                  . "patch --diff " . $diffID,
+      "user" => "root"
+    );
+
+    // Clean up the user arc config we are using
+    $cleanup = array(
+      "name" => "Arc cleanup",
+      "shell" => "rm -f ~/.arcrc",
+      "user" => "root"
+    );
+
+    // Construct the steps in the order of execution
+    $steps[] = $setup;
+    $steps[] = $fix_permission;
+    $steps[] = $fix_git_ignore;
+    $steps[] = $patch;
+
+    // Run the actual command
+    $this->updateTest($diffID, $test);
+    $cmd = $this->updateTestCommand($diffID, $test, "running") . ";"
+           . "./build_tools/precommit_checker.py " . $test
+           . "; exit_code=$?; ([[ \$exit_code -eq 0 ]] &&"
+           . $this->updateTestCommand($diffID, $test, "pass") . ")"
+           . "||" . $this->updateTestCommand($diffID, $test, "fail")
+           . "; cat /tmp/precommit-check.log"
+           . "; for f in `ls t/log-*`; do echo \$f; cat \$f; done;"
+           . "[[ \$exit_code -eq 0 ]]";
+
+    $run_test = array(
+      "name" => "Run " . $test,
+      "shell" => $cmd,
+      "user" => "root",
+    );
+
+    $steps[] = $run_test;
+    $steps[] = $cleanup;
+
+    return $steps;
+  }
+
+  function startTestsInSandcastle($workflow) {
+    // extract information we need from workflow or CLI
+    $diffID = $workflow->getDiffId();
+    $username = exec("whoami");
+
+    if ($diffID == null || $username == null) {
+      // there is no diff and we can't extract username
+      // we cannot schedule sandcasstle job
+      return;
+    }
+
+    if (strcmp(getenv("ROCKSDB_CHECK_ALL"), 1) == 0) {
+      // extract all tests from the CI definition
+      $output = file_get_contents("build_tools/rocksdb-lego-determinator");
+      preg_match_all('/[ ]{2}([a-zA-Z0-9_]+)[\)]{1}/', $output, $matches);
+      $tests = $matches[1];
+    } else {
+      // manually list of tests we want to run in sandcastle
+      $tests = array(
+        "unit", "unit_481", "clang_unit", "tsan", "asan", "lite", "valgrind"
+      );
+    }
+
+    // construct a job definition for each test and add it to the master plan
+    foreach ($tests as $test) {
+      $arg[] = array(
+        "name" => "RocksDB diff " . $diffID . " test " . $test,
+        "steps" => $this->getSteps($diffID, $username, $test)
+      );
+    }
+
+    // we cannot submit the parallel execution master plan to sandcastle
+    // we need supply the job plan as a determinator
+    // so we construct a small job that will spit out the master job plan
+    // which sandcastle will parse and execute
+    // Why compress ? Otherwise we run over the max string size.
+    $cmd = "echo " . base64_encode(json_encode($arg))
+           . " | gzip -f | base64 -w0";
+    $arg_encoded = shell_exec($cmd);
+
+    $command = array(
+      "name" => "Run diff " . $diffID . "for user " . $username,
+      "steps" => array()
+    );
+
+    $command["steps"][] = array(
+      "name" => "Generate determinator",
+      "shell" => "echo " . $arg_encoded . " | base64 --decode | gzip -d"
+                 . " | base64 --decode",
+      "determinator" => true,
+      "user" => "root"
+    );
+
+    // submit to sandcastle
+    $url = 'https://interngraph.intern.facebook.com/sandcastle/generate?'
+            .'command=SandcastleUniversalCommand'
+            .'&vcs=rocksdb-git&revision=origin%2Fmaster&type=lego'
+            .'&user=krad&alias=rocksdb-precommit'
+            .'&command-args=' . urlencode(json_encode($command));
+
+    $cmd = 'https_proxy= HTTPS_PROXY= curl -s -k -F app=659387027470559 '
+            . '-F token=AeO_3f2Ya3TujjnxGD4 "' . $url . '"';
+
+    $output = shell_exec($cmd);
+
+    // extract sandcastle URL from the response
+    preg_match('/url": "(.+)"/', $output, $sandcastle_url);
+
+    echo "\nSandcastle URL: " . $sandcastle_url[1] . "\n";
+
+    // Ask phabricator to display it on the diff UI
+    $this->postURL($diffID, $sandcastle_url[1]);
+  }
+
+  //////////////////////////////////////////////////////////////////////
+  /* Send off builds to jenkins */
+  function startTestsInJenkins($workflow) {
+    $diffID = $workflow->getDiffID();
+    if ($diffID === null) {
+      return;
+    }
+
+    $results = $workflow->getTestResults();
+    if (!$results) {
+      return;
+    }
+
+    $url = "https://ci-builds.fb.com/view/rocksdb/job/rocksdb_diff_check/"
+               ."buildWithParameters?token=AUTH&DIFF_ID=$diffID";
+    system("curl --noproxy '*' \"$url\" > /dev/null 2>&1");
+  }
 }
@@ -1,43 +0,0 @@
-<?php
-// Copyright 2004-present Facebook. All Rights Reserved.
-// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree. An additional grant
-// of patent rights can be found in the PATENTS file in the same directory.
-
-require('RocksDBCommonHelper.php');
-
-define("DIFF_COMMAND", "diff");
-
-class FacebookArcanistConfiguration extends ArcanistConfiguration {
-  public function getCustomArgumentsForCommand($command) {
-    if ($command == "land") {
-      return array(
-          'async' => array('help' => 'Just to make tools happy'));
-    }
-    return array();
-  }
-
-  public function didRunWorkflow($command,
-                                 ArcanistBaseWorkflow $workflow,
-                                 $error_code) {
-    // Default options don't terminate on failure, but that's what we want. In
-    // the current case we use assertions intentionally as "terminate on failure
-    // invariants".
-    assert_options(ASSERT_BAIL, true);
-
-    assert($workflow);
-    assert(strlen($command) > 0);
-
-    if ($command == DIFF_COMMAND && !$workflow->isRawDiffSource()) {
-      $diffID = $workflow->getDiffId();
-
-      // When submitting a diff this code path gets executed multiple times in
-      // a row. We only care about the case when ID for the diff is provided
-      // because that's what we need to apply the diff and trigger the tests.
-      if (strlen($diffID) > 0) {
-        assert(is_numeric($diffID));
-        startTestsInSandcastle(true /* $applyDiff */, $workflow, $diffID);
-      }
-    }
-  }
-}
@@ -1,355 +0,0 @@
-<?php
-// Copyright 2004-present Facebook. All Rights Reserved.
-// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree. An additional grant
-// of patent rights can be found in the PATENTS file in the same directory.
-
-// Name of the environment variables which need to be set by the entity which
-// triggers continuous runs so that code at the end of the file gets executed
-// and Sandcastle run starts.
-define("ENV_POST_RECEIVE_HOOK", "POST_RECEIVE_HOOK");
-define("ENV_HTTPS_APP_VALUE", "HTTPS_APP_VALUE");
-define("ENV_HTTPS_TOKEN_VALUE", "HTTPS_TOKEN_VALUE");
-
-define("PRIMARY_TOKEN_FILE", '/home/krad/.sandcastle');
-define("CONT_RUN_ALIAS", "leveldb");
-
-//////////////////////////////////////////////////////////////////////
-/*  Run tests in sandcastle */
-function postURL($diffID, $url) {
-  assert(strlen($diffID) > 0);
-  assert(is_numeric($diffID));
-  assert(strlen($url) > 0);
-
-  $cmd = 'echo \'{"diff_id": ' . $diffID . ', '
-         . '"name":"click here for sandcastle tests for D' . $diffID . '", '
-         . '"link":"' . $url . '"}\' | '
-         . 'arc call-conduit '
-         . 'differential.updateunitresults';
-  shell_exec($cmd);
-}
-
-function buildUpdateTestStatusCmd($diffID, $test, $status) {
-  assert(strlen($diffID) > 0);
-  assert(is_numeric($diffID));
-  assert(strlen($test) > 0);
-  assert(strlen($status) > 0);
-
-  $cmd = 'echo \'{"diff_id": ' . $diffID . ', '
-         . '"name":"' . $test . '", '
-         . '"result":"' . $status . '"}\' | '
-         . 'arc call-conduit '
-         . 'differential.updateunitresults';
-  return $cmd;
-}
-
-function updateTestStatus($diffID, $test) {
-  assert(strlen($diffID) > 0);
-  assert(is_numeric($diffID));
-  assert(strlen($test) > 0);
-
-  shell_exec(buildUpdateTestStatusCmd($diffID, $test, "waiting"));
-}
-
-function getSteps($applyDiff, $diffID, $username, $test) {
-  assert(strlen($username) > 0);
-  assert(strlen($test) > 0);
-
-  if ($applyDiff) {
-    assert(strlen($diffID) > 0);
-    assert(is_numeric($diffID));
-
-    $arcrc_content = (PHP_OS == "Darwin" ?
-        exec("cat ~/.arcrc | gzip -f | base64") :
-            exec("cat ~/.arcrc | gzip -f | base64 -w0"));
-    assert(strlen($arcrc_content) > 0);
-
-    // Sandcastle machines don't have arc setup. We copy the user certificate
-    // and authenticate using that in Sandcastle.
-    $setup = array(
-      "name" => "Setup arcrc",
-      "shell" => "echo " . $arcrc_content . " | base64 --decode"
-                 . " | gzip -d > ~/.arcrc",
-      "user" => "root"
-    );
-
-    // arc demands certain permission on its config.
-    // also fix the sticky bit issue in sandcastle
-    $fix_permission = array(
-      "name" => "Fix environment",
-      "shell" => "chmod 600 ~/.arcrc && chmod +t /dev/shm",
-      "user" => "root"
-    );
-
-    // Construct the steps in the order of execution.
-    $steps[] = $setup;
-    $steps[] = $fix_permission;
-  }
-
-  // fbcode is a sub-repo. We cannot patch until we add it to ignore otherwise
-  // Git thinks it is an uncommited change.
-  $fix_git_ignore = array(
-    "name" => "Fix git ignore",
-    "shell" => "echo fbcode >> .git/info/exclude",
-    "user" => "root"
-  );
-
-  $steps[] = $fix_git_ignore;
-
-  // This will be the command used to execute particular type of tests.
-  $cmd = "";
-
-  if ($applyDiff) {
-    // Patch the code (keep your fingures crossed).
-    $patch = array(
-      "name" => "Patch " . $diffID,
-      "shell" => "arc --arcrc-file ~/.arcrc "
-                  . "patch --nocommit --diff " . $diffID,
-      "user" => "root"
-    );
-
-    $steps[] = $patch;
-
-    updateTestStatus($diffID, $test);
-    $cmd = buildUpdateTestStatusCmd($diffID, $test, "running") . "; ";
-  }
-
-  // Run the actual command.
-  $cmd = $cmd . "J=$(nproc) ./build_tools/precommit_checker.py " . $test
-           . "; exit_code=$?; ";
-
-  if ($applyDiff) {
-    $cmd = $cmd . "([[ \$exit_code -eq 0 ]] &&"
-                . buildUpdateTestStatusCmd($diffID, $test, "pass") . ")"
-                . "||" . buildUpdateTestStatusCmd($diffID, $test, "fail")
-                . "; ";
-  }
-
-  // shell command to sort the tests based on exit code and print
-  // the output of the log files.
-  $cat_sorted_logs = "
-    while read code log_file;
-      do echo \"################ cat \$log_file [exit_code : \$code] ################\";
-      cat \$log_file;
-    done < <(tail -n +2 LOG | sort -k7,7n -k4,4gr | awk '{print \$7,\$NF}')";
-
-  // Shell command to cat all log files
-  $cat_all_logs = "for f in `ls t/!(run-*)`; do echo \$f;cat \$f; done";
-
-  // If LOG file exist use it to cat log files sorted by exit code, otherwise
-  // cat everything
-  $logs_cmd = "if [ -f LOG ]; then {$cat_sorted_logs}; else {$cat_all_logs}; fi";
-
-  $cmd = $cmd . " cat /tmp/precommit-check.log"
-              . "; shopt -s extglob; {$logs_cmd}"
-              . "; shopt -u extglob; [[ \$exit_code -eq 0 ]]";
-  assert(strlen($cmd) > 0);
-
-  $run_test = array(
-    "name" => "Run " . $test,
-    "shell" => $cmd,
-    "user" => "root",
-    "parser" => "python build_tools/error_filter.py " . $test,
-  );
-
-  $steps[] = $run_test;
-
-  if ($applyDiff) {
-    // Clean up the user arc config we are using.
-    $cleanup = array(
-      "name" => "Arc cleanup",
-      "shell" => "rm -f ~/.arcrc",
-      "user" => "root"
-    );
-
-    $steps[] = $cleanup;
-  }
-
-  assert(count($steps) > 0);
-  return $steps;
-}
-
-function getSandcastleConfig() {
-  $sandcastle_config = array();
-
-  $cwd = getcwd();
-  $cwd_token_file = "{$cwd}/.sandcastle";
-  // This is a case when we're executed from a continuous run. Fetch the values
-  // from the environment.
-  if (getenv(ENV_POST_RECEIVE_HOOK)) {
-    $sandcastle_config[0] = getenv(ENV_HTTPS_APP_VALUE);
-    $sandcastle_config[1] = getenv(ENV_HTTPS_TOKEN_VALUE);
-  } else {
-    // This is a typical `[p]arc diff` case. Fetch the values from the specific
-    // configuration files.
-    for ($i = 0; $i < 50; $i++) {
-      if (file_exists(PRIMARY_TOKEN_FILE) ||
-          file_exists($cwd_token_file)) {
-        break;
-      }
-      // If we failed to fetch the tokens, sleep for 0.2 second and try again
-      usleep(200000);
-    }
-    assert(file_exists(PRIMARY_TOKEN_FILE) ||
-           file_exists($cwd_token_file));
-
-    // Try the primary location first, followed by a secondary.
-    if (file_exists(PRIMARY_TOKEN_FILE)) {
-      $cmd = 'cat ' . PRIMARY_TOKEN_FILE;
-    } else {
-      $cmd = 'cat ' . $cwd_token_file;
-    }
-
-    assert(strlen($cmd) > 0);
-    $sandcastle_config = explode(':', rtrim(shell_exec($cmd)));
-  }
-
-  // In this case be very explicit about the implications.
-  if (count($sandcastle_config) != 2) {
-    echo "Sandcastle configuration files don't contain valid information " .
-         "or the necessary environment variables aren't defined. Unable " .
-         "to validate the code changes.";
-    exit(1);
-  }
-
-  assert(strlen($sandcastle_config[0]) > 0);
-  assert(strlen($sandcastle_config[1]) > 0);
-  assert(count($sandcastle_config) > 0);
-
-  return $sandcastle_config;
-}
-
-// This function can be called either from `[p]arc diff` command or during
-// the Git post-receive hook.
- function startTestsInSandcastle($applyDiff, $workflow, $diffID) {
-  // Default options don't terminate on failure, but that's what we want. In
-  // the current case we use assertions intentionally as "terminate on failure
-  // invariants".
-  assert_options(ASSERT_BAIL, true);
-
-  // In case of a diff we'll send notificatios to the author. Else it'll go to
-  // the entire team because failures indicate that build quality has regressed.
-  $username = $applyDiff ? exec("whoami") : CONT_RUN_ALIAS;
-  assert(strlen($username) > 0);
-
-  if ($applyDiff) {
-    assert($workflow);
-    assert(strlen($diffID) > 0);
-    assert(is_numeric($diffID));
-  }
-
-  // List of tests we want to run in Sandcastle.
-  $tests = array("unit", "unit_non_shm", "unit_481", "clang_unit", "tsan",
-                 "asan", "lite_test", "valgrind", "release", "release_481",
-                 "clang_release", "punit", "clang_analyze", "code_cov",
-                 "java_build", "no_compression", "unity", "ubsan");
-
-  $send_email_template = array(
-    'type' => 'email',
-    'triggers' => array('fail'),
-    'emails' => array($username . '@fb.com'),
-  );
-
-  // Construct a job definition for each test and add it to the master plan.
-  foreach ($tests as $test) {
-    $stepName = "RocksDB diff " . $diffID . " test " . $test;
-
-    if (!$applyDiff) {
-      $stepName = "RocksDB continuous integration test " . $test;
-    }
-
-    $arg[] = array(
-      "name" => $stepName,
-      "report" => array($send_email_template),
-      "steps" => getSteps($applyDiff, $diffID, $username, $test)
-    );
-  }
-
-  // We cannot submit the parallel execution master plan to Sandcastle and
-  // need supply the job plan as a determinator. So we construct a small job
-  // that will spit out the master job plan which Sandcastle will parse and
-  // execute. Why compress the job definitions? Otherwise we run over the max
-  // string size.
-  $cmd = "echo " . base64_encode(json_encode($arg))
-         . (PHP_OS == "Darwin" ?
-             " | gzip -f | base64" :
-                 " | gzip -f | base64 -w0");
-  assert(strlen($cmd) > 0);
-
-  $arg_encoded = shell_exec($cmd);
-  assert(strlen($arg_encoded) > 0);
-
-  $runName = "Run diff " . $diffID . "for user " . $username;
-
-  if (!$applyDiff) {
-    $runName = "RocksDB continuous integration build and test run";
-  }
-
-  $command = array(
-    "name" => $runName,
-    "steps" => array()
-  );
-
-  $command["steps"][] = array(
-    "name" => "Generate determinator",
-    "shell" => "echo " . $arg_encoded . " | base64 --decode | gzip -d"
-               . " | base64 --decode",
-    "determinator" => true,
-    "user" => "root"
-  );
-
-  // Submit to Sandcastle.
-  $url = 'https://interngraph.intern.facebook.com/sandcastle/create';
-
-  $job = array(
-    'command' => 'SandcastleUniversalCommand',
-    'args' => $command,
-    'capabilities' => array(
-      'vcs' => 'rocksdb-int-git',
-      'type' => 'lego',
-    ),
-    'hash' => 'origin/master',
-    'user' => $username,
-    'alias' => 'rocksdb-precommit',
-    'tags' => array('rocksdb'),
-    'description' => 'Rocksdb precommit job',
-  );
-
-  // Fetch the configuration necessary to submit a successful HTTPS request.
-  $sandcastle_config = getSandcastleConfig();
-
-  $app = $sandcastle_config[0];
-  $token = $sandcastle_config[1];
-
-  $cmd = 'curl -s -k -F app=' . $app . ' '
-          . '-F token=' . $token . ' -F job=\'' . json_encode($job)
-          .'\' "' . $url . '"';
-
-  $output = shell_exec($cmd);
-  assert(strlen($output) > 0);
-
-  // Extract Sandcastle URL from the response.
-  preg_match('/url": "(.+)"/', $output, $sandcastle_url);
-
-  assert(count($sandcastle_url) > 0, "Unable to submit Sandcastle request.");
-  assert(strlen($sandcastle_url[1]) > 0, "Unable to extract Sandcastle URL.");
-
-  if ($applyDiff) {
-    echo "\nSandcastle URL: " . $sandcastle_url[1] . "\n";
-    // Ask Phabricator to display it on the diff UI.
-    postURL($diffID, $sandcastle_url[1]);
-  } else {
-    echo "Continuous integration started Sandcastle tests. You can look at ";
-    echo "the progress at:\n" . $sandcastle_url[1] . "\n";
-  }
-}
-
-// Continuous run cript will set the environment variable and based on that
-// we'll trigger the execution of tests in Sandcastle. In that case we don't
-// need to apply any diffs and there's no associated workflow either.
-if (getenv(ENV_POST_RECEIVE_HOOK)) {
-  startTestsInSandcastle(
-    false /* $applyDiff */,
-    NULL /* $workflow */,
-    NULL /* $diffID */);
-}
@@ -44,7 +44,7 @@ abstract class BaseDirectoryScopedFormatLinter extends ArcanistLinter {
      $futures[$path] = $this->getFormatFuture($path, $changed);
    }

-    foreach (id(new FutureIterator($futures))->limit(8) as $p => $f) {
+    foreach (Futures($futures)->limit(8) as $p => $f) {
      $this->rawLintOutput[$p] = $f->resolvex();
    }
  }
@@ -42,12 +42,14 @@ class FacebookFbcodeLintEngine extends ArcanistLintEngine {
    $python_linter = new ArcanistPEP8Linter();
    $linters[] = $python_linter;

-    $cpp_linters = array();
-    $cpp_linters[] = $linters[] = new ArcanistCpplintLinter();
-    $cpp_linters[] = $linters[] = new FbcodeCppLinter();
+    if (!$this->getCommitHookMode()) {
+      $cpp_linters = array();
+      $cpp_linters[] = $linters[] = new ArcanistCpplintLinter();
+      $cpp_linters[] = $linters[] = new FbcodeCppLinter();

-    $clang_format_linter = new FbcodeClangFormatLinter();
-    $linters[] = $clang_format_linter;
+      $clang_format_linter = new FbcodeClangFormatLinter();
+      $linters[] = $clang_format_linter;
+    }

    $spelling_linter = new ArcanistSpellingLinter();
    $linters[] = $spelling_linter;
@@ -4,14 +4,18 @@
 // LICENSE file in the root directory of this source tree. An additional grant
 // of patent rights can be found in the PATENTS file in the same directory.

-class FacebookFbcodeUnitTestEngine extends ArcanistUnitTestEngine {
+class FacebookFbcodeUnitTestEngine extends ArcanistBaseUnitTestEngine {

  public function run() {
-      // For a call to `arc call-conduit differential.updateunitresults` to
-      // succeed we need at least one entry here.
-      $result = new ArcanistUnitTestResult();
-      $result->setName("dummy_placeholder_entry");
-      $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
-      return array($result);
+    // Here we create a new unit test "jenkins_async_test" and promise we'll
+    // update the results later.
+    // Jenkins updates the results using `arc call-conduit
+    // differential.updateunitresults` call. If you change the name here, also
+    // make sure to change the name in Jenkins script that updates the test
+    // result -- they have to be the same.
+    $result = new ArcanistUnitTestResult();
+    $result->setName("jenkins_async_test");
+    $result->setResult(ArcanistUnitTestResult::RESULT_POSTPONED);
+    return array($result);
  }
 }
@@ -1,17 +0,0 @@
-<?php
-// Copyright 2004-present Facebook. All Rights Reserved.
-// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree. An additional grant
-// of patent rights can be found in the PATENTS file in the same directory.
-
-class FacebookFbcodeUnitTestEngine extends ArcanistBaseUnitTestEngine {
-
-  public function run() {
-      // For a call to `arc call-conduit differential.updateunitresults` to
-      // succeed we need at least one entry here.
-      $result = new ArcanistUnitTestResult();
-      $result->setName("dummy_placeholder_entry");
-      $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
-      return array($result);
-  }
-}
@@ -1,172 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-from targets_builder import TARGETSBuilder
-from optparse import OptionParser
-import os
-import fnmatch
-import sys
-import tempfile
-
-from util import ColorString
-import util
-
-# tests to export as libraries for inclusion in other projects
-_EXPORTED_TEST_LIBS = ["env_basic_test"]
-
-# Parse src.mk files as a Dictionary of
-# VAR_NAME => list of files
-def parse_src_mk(repo_path):
-    src_mk = repo_path + "/src.mk"
-    src_files = {}
-    for line in open(src_mk):
-        line = line.strip()
-        if len(line) == 0 or line[0] == '#':
-            continue
-        if '=' in line:
-            current_src = line.split('=')[0].strip()
-            src_files[current_src] = []
-        elif '.cc' in line:
-            src_path = line.split('.cc')[0].strip() + '.cc'
-            src_files[current_src].append(src_path)
-    return src_files
-
-
-# get all .cc / .c files
-def get_cc_files(repo_path):
-    cc_files = []
-    for root, dirnames, filenames in os.walk(repo_path):
-        root = root[(len(repo_path) + 1):]
-        if "java" in root:
-            # Skip java
-            continue
-        for filename in fnmatch.filter(filenames, '*.cc'):
-            cc_files.append(os.path.join(root, filename))
-        for filename in fnmatch.filter(filenames, '*.c'):
-            cc_files.append(os.path.join(root, filename))
-    return cc_files
-
-
-# Get tests from Makefile
-def get_tests(repo_path):
-    Makefile = repo_path + "/Makefile"
-
-    # Dictionary TEST_NAME => IS_PARALLEL
-    tests = {}
-
-    found_tests = False
-    for line in open(Makefile):
-        line = line.strip()
-        if line.startswith("TESTS ="):
-            found_tests = True
-        elif found_tests:
-            if line.endswith("\\"):
-                # remove the trailing \
-                line = line[:-1]
-                line = line.strip()
-                tests[line] = False
-            else:
-                # we consumed all the tests
-                break
-
-    found_parallel_tests = False
-    for line in open(Makefile):
-        line = line.strip()
-        if line.startswith("PARALLEL_TEST ="):
-            found_parallel_tests = True
-        elif found_parallel_tests:
-            if line.endswith("\\"):
-                # remove the trailing \
-                line = line[:-1]
-                line = line.strip()
-                tests[line] = True
-            else:
-                # we consumed all the parallel tests
-                break
-
-    return tests
-
-
-# Prepare TARGETS file for buck
-def generate_targets(repo_path):
-    print(ColorString.info("Generating TARGETS"))
-    # parsed src.mk file
-    src_mk = parse_src_mk(repo_path)
-    # get all .cc files
-    cc_files = get_cc_files(repo_path)
-    # get tests from Makefile
-    tests = get_tests(repo_path)
-
-    if src_mk is None or cc_files is None or tests is None:
-        return False
-
-    TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path)
-    # rocksdb_lib
-    TARGETS.add_library(
-        "rocksdb_lib",
-        src_mk["LIB_SOURCES"] +
-        src_mk["TOOL_LIB_SOURCES"])
-    # rocksdb_test_lib
-    TARGETS.add_library(
-        "rocksdb_test_lib",
-        src_mk.get("MOCK_LIB_SOURCES", []) +
-        src_mk.get("TEST_LIB_SOURCES", []) +
-        src_mk.get("EXP_LIB_SOURCES", []),
-        [":rocksdb_lib"])
-    # rocksdb_tools_lib
-    TARGETS.add_library(
-        "rocksdb_tools_lib",
-        src_mk.get("BENCH_LIB_SOURCES", []) +
-        ["util/testutil.cc"],
-        [":rocksdb_lib"])
-
-    # test for every test we found in the Makefile
-    for test in tests:
-        match_src = [src for src in cc_files if ("/%s.c" % test) in src]
-        if len(match_src) == 0:
-            print(ColorString.warning("Cannot find .cc file for %s" % test))
-            continue
-        elif len(match_src) > 1:
-            print(ColorString.warning("Found more than one .cc for %s" % test))
-            print(match_src)
-            continue
-
-        assert(len(match_src) == 1)
-        is_parallel = tests[test]
-        TARGETS.register_test(test, match_src[0], is_parallel)
-
-        if test in _EXPORTED_TEST_LIBS:
-            test_library = "%s_lib" % test
-            TARGETS.add_library(test_library, match_src, [":rocksdb_test_lib"])
-    TARGETS.flush_tests()
-
-    print(ColorString.info("Generated TARGETS Summary:"))
-    print(ColorString.info("- %d libs" % TARGETS.total_lib))
-    print(ColorString.info("- %d binarys" % TARGETS.total_bin))
-    print(ColorString.info("- %d tests" % TARGETS.total_test))
-    return True
-
-
-def get_rocksdb_path():
-    # rocksdb = {script_dir}/..
-    script_dir = os.path.dirname(sys.argv[0])
-    script_dir = os.path.abspath(script_dir)
-    rocksdb_path = os.path.abspath(
-        os.path.join(script_dir, "../"))
-
-    return rocksdb_path
-
-def exit_with_error(msg):
-    print(ColorString.error(msg))
-    sys.exit(1)
-
-
-def main():
-    # Generate TARGETS file for buck
-    ok = generate_targets(get_rocksdb_path())
-    if not ok:
-        exit_with_error("Failed to generate TARGETS files")
-
-if __name__ == "__main__":
-    main()
@@ -1,4 +0,0 @@
-#!/bin/bash
-# Create a tmp directory for the test to use
-TEST_DIR=$(mktemp -d /dev/shm/fbcode_rocksdb_XXXXXXX)
-TEST_TMPDIR="$TEST_DIR" $@ && rm -rf "$TEST_DIR"
@@ -1,65 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-import targets_cfg
-import pprint
-
-# TODO(tec): replace this with PrettyPrinter
-def pretty_list(lst, indent=6):
-    if lst is None or len(lst) == 0:
-        return ""
-
-    if len(lst) == 1:
-        return "\"%s\"" % lst[0]
-
-    separator = "\",\n%s\"" % (" " * indent)
-    res = separator.join(lst)
-    res = "\n" + (" " * indent) + "\"" + res + "\",\n" + (" " * (indent - 2))
-    return res
-
-
-class TARGETSBuilder:
-    def __init__(self, path):
-        self.path = path
-        self.targets_file = open(path, 'w')
-        self.targets_file.write(targets_cfg.rocksdb_target_header)
-        self.total_lib = 0
-        self.total_bin = 0
-        self.total_test = 0
-        self.tests_cfg = []
-
-    def __del__(self):
-        self.targets_file.close()
-
-    def add_library(self, name, srcs, deps=None, headers=None):
-        if headers is None:
-            headers = "AutoHeaders.RECURSIVE_GLOB"
-        self.targets_file.write(targets_cfg.library_template % (
-            name,
-            headers,
-            pretty_list(srcs),
-            pretty_list(deps)))
-        self.total_lib = self.total_lib + 1
-
-    def add_binary(self, name, srcs, deps=None):
-        self.targets_file.write(targets_cfg.binary_template % (
-            name,
-            pretty_list(srcs),
-            pretty_list(deps)))
-        self.total_bin = self.total_bin + 1
-
-    def register_test(self, test_name, src, is_parallel):
-        exec_mode = "serial"
-        if is_parallel:
-            exec_mode = "parallel"
-        self.tests_cfg.append([test_name, str(src), str(exec_mode)])
-
-        self.total_test = self.total_test + 1
-
-    def flush_tests(self):
-        self.targets_file.write(targets_cfg.unittests_template % (
-            pprint.PrettyPrinter().pformat(self.tests_cfg)
-        ))
-
-        self.tests_cfg = []
@@ -1,112 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-rocksdb_target_header = """REPO_PATH = "internal_repo_rocksdb/repo/"
-BUCK_BINS = "buck-out/gen/" + REPO_PATH
-TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh"
-rocksdb_compiler_flags = [
-  "-msse",
-  "-msse4.2",
-  "-fno-builtin-memcmp",
-  "-DROCKSDB_PLATFORM_POSIX",
-  "-DROCKSDB_LIB_IO_POSIX",
-  "-DROCKSDB_FALLOCATE_PRESENT",
-  "-DROCKSDB_MALLOC_USABLE_SIZE",
-  "-DOS_LINUX",
-  # Flags to enable libs we include
-  "-DSNAPPY",
-  "-DZLIB",
-  "-DBZIP2",
-  "-DLZ4",
-  "-DZSTD",
-  "-DGFLAGS=gflags",
-  "-DNUMA",
-  "-DTBB",
-  # Needed to compile in fbcode
-  "-Wno-expansion-to-defined",
-]
-
-rocksdb_external_deps = [
-  ('bzip2', None, 'bz2'),
-  ('snappy', None, "snappy"),
-  ('zlib', None, 'z'),
-  ('gflags', None, 'gflags'),
-  ('lz4', None, 'lz4'),
-  ('zstd', None),
-  ('tbb', None),
-  ("numa", "2.0.8", "numa"),
-  ("googletest", None, "gtest"),
-]
-
-rocksdb_preprocessor_flags = [
-  # Directories with files for #include
-  "-I" + REPO_PATH + "include/",
-  "-I" + REPO_PATH,
-]
-"""
-
-
-library_template = """
-cpp_library(
-    name = "%s",
-    headers = %s,
-    srcs = [%s],
-    deps = [%s],
-    preprocessor_flags = rocksdb_preprocessor_flags,
-    compiler_flags = rocksdb_compiler_flags,
-    external_deps = rocksdb_external_deps,
-)
-"""
-
-binary_template = """
-cpp_binary(
-  name = "%s",
-  srcs = [%s],
-  deps = [%s],
-  preprocessor_flags = rocksdb_preprocessor_flags,
-  compiler_flags = rocksdb_compiler_flags,
-  external_deps = rocksdb_external_deps,
-)
-"""
-
-unittests_template = """
-# [test_name, test_src, test_type]
-ROCKS_TESTS = %s
-
-
-# Generate a test rule for each entry in ROCKS_TESTS
-for test_cfg in ROCKS_TESTS:
-    test_name = test_cfg[0]
-    test_cc = test_cfg[1]
-    ttype = "gtest" if test_cfg[2] == "parallel" else "simple"
-    test_bin = test_name + "_bin"
-
-    cpp_binary (
-      name = test_bin,
-      srcs = [test_cc],
-      deps = [":rocksdb_test_lib"],
-      preprocessor_flags = rocksdb_preprocessor_flags,
-      compiler_flags = rocksdb_compiler_flags,
-      external_deps = rocksdb_external_deps,
-    )
-
-    custom_unittest(
-      name = test_name,
-      type = ttype,
-      deps = [":" + test_bin],
-      command = [TEST_RUNNER, BUCK_BINS + test_bin]
-    )
-
-custom_unittest(
-    name = "make_rocksdbjavastatic",
-    type = "simple",
-    command = ["internal_repo_rocksdb/make_rocksdbjavastatic.sh"],
-)
-
-custom_unittest(
-    name = "make_rocksdb_lite_release",
-    type = "simple",
-    command = ["internal_repo_rocksdb/make_rocksdb_lite_release.sh"],
-)
-"""
@@ -1,107 +0,0 @@
-"""
-This module keeps commonly used components.
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-import subprocess
-import os
-import time
-
-class ColorString:
-    """ Generate colorful strings on terminal """
-    HEADER = '\033[95m'
-    BLUE = '\033[94m'
-    GREEN = '\033[92m'
-    WARNING = '\033[93m'
-    FAIL = '\033[91m'
-    ENDC = '\033[0m'
-
-    @staticmethod
-    def _make_color_str(text, color):
-        return "".join([color, text.encode('utf-8'), ColorString.ENDC])
-
-    @staticmethod
-    def ok(text):
-        if ColorString.is_disabled:
-            return text
-        return ColorString._make_color_str(text, ColorString.GREEN)
-
-    @staticmethod
-    def info(text):
-        if ColorString.is_disabled:
-            return text
-        return ColorString._make_color_str(text, ColorString.BLUE)
-
-    @staticmethod
-    def header(text):
-        if ColorString.is_disabled:
-            return text
-        return ColorString._make_color_str(text, ColorString.HEADER)
-
-    @staticmethod
-    def error(text):
-        if ColorString.is_disabled:
-            return text
-        return ColorString._make_color_str(text, ColorString.FAIL)
-
-    @staticmethod
-    def warning(text):
-        if ColorString.is_disabled:
-            return text
-        return ColorString._make_color_str(text, ColorString.WARNING)
-
-    is_disabled = False
-
-
-def run_shell_command(shell_cmd, cmd_dir=None):
-    """ Run a single shell command.
-        @returns a tuple of shell command return code, stdout, stderr """
-
-    if cmd_dir is not None and not os.path.exists(cmd_dir):
-        run_shell_command("mkdir -p %s" % cmd_dir)
-
-    start = time.time()
-    print("\t>>> Running: " + shell_cmd)
-    p = subprocess.Popen(shell_cmd,
-                         shell=True,
-                         stdout=subprocess.PIPE,
-                         stderr=subprocess.PIPE,
-                         cwd=cmd_dir)
-    stdout, stderr = p.communicate()
-    end = time.time()
-
-    # Report time if we spent more than 5 minutes executing a command
-    execution_time = end - start
-    if execution_time > (60 * 5):
-        mins = (execution_time / 60)
-        secs = (execution_time % 60)
-        print("\t>time spent: %d minutes %d seconds" % (mins, secs))
-
-
-    return p.returncode, stdout, stderr
-
-
-def run_shell_commands(shell_cmds, cmd_dir=None, verbose=False):
-    """ Execute a sequence of shell commands, which is equivalent to
-        running `cmd1 && cmd2 && cmd3`
-        @returns boolean indication if all commands succeeds.
-    """
-
-    if cmd_dir:
-        print("\t=== Set current working directory => %s" % cmd_dir)
-
-    for shell_cmd in shell_cmds:
-        ret_code, stdout, stderr = run_shell_command(shell_cmd, cmd_dir)
-        if stdout:
-            if verbose or ret_code != 0:
-                print(ColorString.info("stdout: \n"), stdout)
-        if stderr:
-            # contents in stderr is not necessarily to be error messages.
-            if verbose or ret_code != 0:
-                print(ColorString.error("stderr: \n"), stderr)
-        if ret_code != 0:
-            return False
-
-    return True
@@ -19,21 +19,22 @@
 #
 # The PLATFORM_CCFLAGS and PLATFORM_CXXFLAGS might include the following:
 #
-#       -DROCKSDB_PLATFORM_POSIX    if posix-platform based
+#       -DLEVELDB_PLATFORM_POSIX if cstdatomic is present
+#       -DLEVELDB_PLATFORM_NOATOMIC if it is not
 #       -DSNAPPY                    if the Snappy library is present
 #       -DLZ4                       if the LZ4 library is present
 #       -DZSTD                      if the ZSTD library is present
 #       -DNUMA                      if the NUMA library is present
-#       -DTBB                       if the TBB library is present
 #
 # Using gflags in rocksdb:
 # Our project depends on gflags, which requires users to take some extra steps
 # before they can compile the whole repository:
 #   1. Install gflags. You may download it from here:
-#      https://gflags.github.io/gflags/ (Mac users can `brew install gflags`)
-#   2. Once installed, add the include path for gflags to your CPATH env var and
-#      the lib path to LIBRARY_PATH. If installed with default settings, the lib
-#      will be /usr/local/lib and the include path will be /usr/local/include
+#      https://code.google.com/p/gflags/
+#   2. Once install, add the include path/lib path for gflags to CPATH and
+#      LIBRARY_PATH respectively. If installed with default mode, the
+#      lib and include path will be /usr/local/lib and /usr/local/include
+# Mac user can do this by having brew installed and running brew install gflags

 OUTPUT=$1
 if test -z "$OUTPUT"; then
@@ -51,12 +52,7 @@ if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
    FBCODE_BUILD="true"
    # If we're compiling with TSAN we need pic build
    PIC_BUILD=$COMPILE_WITH_TSAN
-    if [ -z "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
-      source "$PWD/build_tools/fbcode_config.sh"
-    else
-      # we need this to build with MySQL. Don't use for other purposes.
-      source "$PWD/build_tools/fbcode_config4.8.1.sh"
-    fi
+    source "$PWD/build_tools/fbcode_config.sh"
 fi

 # Delete existing output, if it exists
@@ -169,7 +165,7 @@ case "$TARGET_OS" in
        ;;
    OS_ANDROID_CROSSCOMPILE)
        PLATFORM=OS_ANDROID
-	COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_ANDROID -DROCKSDB_PLATFORM_POSIX"
+	COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_ANDROID -DLEVELDB_PLATFORM_POSIX"
 	PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS "  # All pthread features are in the Android C library
        # PORT_FILES=port/android/android.cc
        CROSS_COMPILE=true
@@ -287,7 +283,7 @@ EOF
    # Test whether numa is available
    $CXX $CFLAGS -x c++ - -o /dev/null -lnuma 2>/dev/null  <<EOF
      #include <numa.h>
-      #include <numaif.h>
+      #inlcude <numaif.h>
      int main() {}
 EOF
    if [ "$?" = 0 ]; then
@@ -296,17 +292,6 @@ EOF
        JAVA_LDFLAGS="$JAVA_LDFLAGS -lnuma"
    fi

-    # Test whether tbb is available
-    $CXX $CFLAGS $LDFLAGS -x c++ - -o /dev/null -ltbb 2>/dev/null  <<EOF
-      #include <tbb/tbb.h>
-      int main() {}
-EOF
-    if [ "$?" = 0 ]; then
-        COMMON_FLAGS="$COMMON_FLAGS -DTBB"
-        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltbb"
-        JAVA_LDFLAGS="$JAVA_LDFLAGS -ltbb"
-    fi
-
    # Test whether jemalloc is available
    if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null -ljemalloc \
      2>/dev/null; then
@@ -466,4 +451,3 @@ echo "PROFILING_FLAGS=$PROFILING_FLAGS" >> "$OUTPUT"
 if test -n "$JEMALLOC"; then
  echo "JEMALLOC=1" >> "$OUTPUT"
 fi
-echo "LUA_PATH=$LUA_PATH" >> "$OUTPUT"
@@ -1,135 +0,0 @@
-#!/bin/bash
-#
-# Copyright (c) 2016, Facebook. All rights reserved.
-#
-# Overall wrapper script for RocksDB continuous builds. The implementation is a
-# trivial pulling scheme. We loop infinitely, check if any new changes have been
-# committed, if yes then trigger a Sandcastle run, and finally go to sleep again
-# for a certain interval.
-#
-
-SRC_GIT_REPO=/data/git/rocksdb-public
-error=0
-
-function log {
-  DATE=`date +%Y-%m-%d:%H:%M:%S`
-  echo $DATE $@
-}
-
-function log_err {
-  log "ERROR: $@ Error code: $error."
-}
-
-function update_repo_status {
-  # Update the parent first.
-  pushd $SRC_GIT_REPO
-
-  # This is a fatal error. Something in the environment isn't right and we will
-  # terminate the execution.
-  error=$?
-  if [ ! $error -eq 0 ]; then
-    log_err "Where is $SRC_GIT_REPO?"
-    exit $error
-  fi
-
-  HTTPS_PROXY=fwdproxy:8080 git fetch -f
-
-  error=$?
-  if [ ! $error -eq 0 ]; then
-    log_err "git fetch -f failed."
-    popd
-    return $error
-  fi
-
-  git update-ref refs/heads/master refs/remotes/origin/master
-
-  error=$?
-  if [ ! $error -eq 0 ]; then
-    log_err "git update-ref failed."
-    popd
-    return $error
-  fi
-
-  popd
-
-  # We're back in an instance-specific directory. Get the latest changes.
-  git pull --rebase
-
-  error=$?
-  if [ ! $error -eq 0 ]; then
-    log_err "git pull --rebase failed."
-    return $error
-  fi
-}
-
-#
-# Execution starts here.
-#
-
-# Path to the determinator from the root of the RocksDB repo.
-CONTRUN_DETERMINATOR=./arcanist_util/config/RocksDBCommonHelper.php
-
-# Value of the previous commit.
-PREV_COMMIT=
-
-log "Starting to monitor for new RocksDB changes ..."
-log "Running under `pwd` as `whoami`."
-
-# Paranoia. Make sure that we're using the right branch.
-git checkout master
-
-error=$?
-if [ ! $error -eq 0 ]; then
-  log_err "This is not good. Can't checkout master. Bye-bye!"
-  exit 1
-fi
-
-# We'll run forever and let the execution environment terminate us if we'll
-# exceed whatever timeout is set for the job.
-while true;
-do
-  # Get the latest changes committed.
-  update_repo_status
-
-  error=$?
-  if [  $error -eq 0 ]; then
-    LAST_COMMIT=`git log -1 | head -1 | grep commit | awk '{ print $2; }'`
-
-    log "Last commit is '$LAST_COMMIT', previous commit is '$PREV_COMMIT'."
-
-    if [ "$PREV_COMMIT" == "$LAST_COMMIT" ]; then
-      log "There were no changes since the last time I checked. Going to sleep."
-    else
-      if [ ! -z "$LAST_COMMIT" ]; then
-        log "New code has been committed or previous commit not known. " \
-            "Will trigger the tests."
-
-        PREV_COMMIT=$LAST_COMMIT
-        log "Updated previous commit to '$PREV_COMMIT'."
-
-        #
-        # This is where we'll trigger the Sandcastle run. The values for
-        # HTTPS_APP_VALUE and HTTPS_APP_VALUE will be set in the container we're
-        # running in.
-        #
-        POST_RECEIVE_HOOK=1 php $CONTRUN_DETERMINATOR
-
-        error=$?
-        if [ $error -eq 0 ]; then
-          log "Sandcastle run successfully triggered."
-        else
-          log_err "Failed to trigger Sandcastle run."
-        fi
-      else
-        log_err "Previous commit not updated. Don't know what the last one is."
-      fi
-    fi
-  else
-    log_err "Getting latest changes failed. Will skip running tests for now."
-  fi
-
-  # Always sleep, even if errors happens while trying to determine the latest
-  # commit. This will prevent us terminating in case of transient errors.
-  log "Will go to sleep for 5 minutes."
-  sleep 5m
-done
@@ -1,18 +1,19 @@
-GCC_BASE=/mnt/gvfs/third-party2/gcc/cf7d14c625ce30bae1a4661c2319c5a283e4dd22/4.9.x/centos6-native/108cf83
-CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/8598c375b0e94e1448182eb3df034704144a838d/stable/centos6-native/3f16ddd
-LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/d6e0a7da6faba45f5e5b1638f9edd7afc2f34e7d/4.9.x/gcc-4.9-glibc-2.20/024dbc3
-GLIBC_BASE=/mnt/gvfs/third-party2/glibc/d282e6e8f3d20f4e40a516834847bdc038e07973/2.20/gcc-4.9-glibc-2.20/500e281
-SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/8c38a4c1e52b4c2cc8a9cdc31b9c947ed7dbfcb4/1.1.3/gcc-4.9-glibc-2.20/e9936bf
-ZLIB_BASE=/mnt/gvfs/third-party2/zlib/0882df3713c7a84f15abe368dc004581f20b39d7/1.2.8/gcc-5-glibc-2.23/9bc6787
-BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/740325875f6729f42d28deaa2147b0854f3a347e/1.0.6/gcc-5-glibc-2.23/9bc6787
-LZ4_BASE=/mnt/gvfs/third-party2/lz4/0e790b441e2d9acd68d51e1d2e028f88c6a79ddf/r131/gcc-5-glibc-2.23/9bc6787
-ZSTD_BASE=/mnt/gvfs/third-party2/zstd/9455f75ff7f4831dc9fda02a6a0f8c68922fad8f/1.0.0/gcc-5-glibc-2.23/9bc6787
-GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/f001a51b2854957676d07306ef3abf67186b5c8b/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a
-JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/fc8a13ca1fffa4d0765c716c5a0b49f0c107518f/master/gcc-5-glibc-2.23/1c32b4b
-NUMA_BASE=/mnt/gvfs/third-party2/numa/17c514c4d102a25ca15f4558be564eeed76f4b6a/2.0.8/gcc-5-glibc-2.23/9bc6787
-LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/ad576de2a1ea560c4d3434304f0fc4e079bede42/trunk/gcc-5-glibc-2.23/b1847cb
-TBB_BASE=/mnt/gvfs/third-party2/tbb/9d9a554877d0c5bef330fe818ab7178806dd316a/4.0_update2/gcc-4.9-glibc-2.20/e9936bf
-KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/7c111ff27e0c466235163f00f280a9d617c3d2ec/4.0.9-36_fbk5_2933_gd092e3f/gcc-5-glibc-2.23/da39a3e
-BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/b7fd454c4b10c6a81015d4524ed06cdeab558490/2.26/centos6-native/da39a3e
-VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d7f4d4d86674a57668e3a96f76f0e17dd0eb8765/3.10.0/gcc-4.9-glibc-2.20/e9936bf
-LUA_BASE=/mnt/gvfs/third-party2/lua/61e4abf5813bbc39bc4f548757ccfcadde175a48/5.2.3/gcc-4.9-glibc-2.20/690f0d7
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+GCC_BASE=/mnt/gvfs/third-party2/gcc/7331085db891a2ef4a88a48a751d834e8d68f4cb/7.x/centos7-native/b2ef2b6
+CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/963d9aeda70cc4779885b1277484fe7544a04e3e/9.0.0/platform007/9e92d53/
+LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/6ace84e956873d53638c738b6f65f3f469cca74c/7.x/platform007/5620abc
+GLIBC_BASE=/mnt/gvfs/third-party2/glibc/192b0f42d63dcf6210d6ceae387b49af049e6e0c/2.26/platform007/f259413
+SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/platform007/ca4da3d
+ZLIB_BASE=/mnt/gvfs/third-party2/zlib/2d9f0b9a4274cc21f61272a9e89bdb859bce8f1f/1.2.8/platform007/ca4da3d
+BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/platform007/ca4da3d
+LZ4_BASE=/mnt/gvfs/third-party2/lz4/0f607f8fc442ea7d6b876931b1898bb573d5e5da/1.9.1/platform007/ca4da3d
+ZSTD_BASE=/mnt/gvfs/third-party2/zstd/ca22bc441a4eb709e9e0b1f9fec9750fed7b31c5/1.4.x/platform007/15a3614
+GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/platform007/ca4da3d
+JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/c26f08f47ac35fc31da2633b7da92d6b863246eb/master/platform007/c26c002
+NUMA_BASE=/mnt/gvfs/third-party2/numa/3f3fb57a5ccc5fd21c66416c0b83e0aa76a05376/2.0.11/platform007/ca4da3d
+LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/40c73d874898b386a71847f1b99115d93822d11f/1.4/platform007/6f3e0a9
+TBB_BASE=/mnt/gvfs/third-party2/tbb/4ce8e8dba77cdbd81b75d6f0c32fd7a1b76a11ec/2018_U5/platform007/ca4da3d
+KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/fb251ecd2f5ae16f8671f7014c246e52a748fe0b/fb/platform007/da39a3e
+BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/ab9f09bba370e7066cafd4eb59752db93f2e8312/2.29.1/platform007/15a3614
+VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d42d152a15636529b0861ec493927200ebebca8e/3.15.0/platform007/ca4da3d
+LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.3.4/platform007/5007832
@@ -1,18 +1,16 @@
-GCC_BASE=/mnt/gvfs/third-party2/gcc/cf7d14c625ce30bae1a4661c2319c5a283e4dd22/4.8.1/centos6-native/cc6c9dc
-CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/8598c375b0e94e1448182eb3df034704144a838d/stable/centos6-native/3f16ddd
-LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/d6e0a7da6faba45f5e5b1638f9edd7afc2f34e7d/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc
-GLIBC_BASE=/mnt/gvfs/third-party2/glibc/d282e6e8f3d20f4e40a516834847bdc038e07973/2.17/gcc-4.8.1-glibc-2.17/99df8fc
-SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/8c38a4c1e52b4c2cc8a9cdc31b9c947ed7dbfcb4/1.1.3/gcc-4.8.1-glibc-2.17/c3f970a
-ZLIB_BASE=/mnt/gvfs/third-party2/zlib/0882df3713c7a84f15abe368dc004581f20b39d7/1.2.8/gcc-4.8.1-glibc-2.17/c3f970a
-BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/740325875f6729f42d28deaa2147b0854f3a347e/1.0.6/gcc-4.8.1-glibc-2.17/c3f970a
-LZ4_BASE=/mnt/gvfs/third-party2/lz4/0e790b441e2d9acd68d51e1d2e028f88c6a79ddf/r131/gcc-4.8.1-glibc-2.17/c3f970a
-ZSTD_BASE=/mnt/gvfs/third-party2/zstd/9455f75ff7f4831dc9fda02a6a0f8c68922fad8f/1.0.0/gcc-4.8.1-glibc-2.17/c3f970a
-GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/f001a51b2854957676d07306ef3abf67186b5c8b/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a
-JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/fc8a13ca1fffa4d0765c716c5a0b49f0c107518f/master/gcc-4.8.1-glibc-2.17/8d31e51
-NUMA_BASE=/mnt/gvfs/third-party2/numa/17c514c4d102a25ca15f4558be564eeed76f4b6a/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a
-LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/ad576de2a1ea560c4d3434304f0fc4e079bede42/trunk/gcc-4.8.1-glibc-2.17/675d945
-TBB_BASE=/mnt/gvfs/third-party2/tbb/9d9a554877d0c5bef330fe818ab7178806dd316a/4.0_update2/gcc-4.8.1-glibc-2.17/c3f970a
-KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/7c111ff27e0c466235163f00f280a9d617c3d2ec/4.0.9-36_fbk5_2933_gd092e3f/gcc-4.8.1-glibc-2.17/da39a3e
-BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/b7fd454c4b10c6a81015d4524ed06cdeab558490/2.26/centos6-native/da39a3e
-VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d7f4d4d86674a57668e3a96f76f0e17dd0eb8765/3.8.1/gcc-4.8.1-glibc-2.17/c3f970a
-LUA_BASE=/mnt/gvfs/third-party2/lua/61e4abf5813bbc39bc4f548757ccfcadde175a48/5.2.3/centos6-native/730f94e
+GCC_BASE=/mnt/gvfs/third-party2/gcc/ebc96bc2fb751b5a0300b8d91a95bdf24ac1d88b/4.8.1/centos6-native/cc6c9dc
+CLANG_BASE=/mnt/gvfs/third-party2/clang/fc904e50a9266b9d7b98cae1993afa0c5aae1440/3.7.1/centos6-native/9d9ecb9/
+LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/f97108c12512b3b0789ac4515d836bdb1eae1142/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc
+GLIBC_BASE=/mnt/gvfs/third-party2/glibc/f5484f168c0e4d19823d41df052c5870c6e575a4/2.17/gcc-4.8.1-glibc-2.17/99df8fc
+SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/cbf6f1f209e5bd160bdc5d971744e039f36b1566/1.1.3/gcc-4.8.1-glibc-2.17/c3f970a
+ZLIB_BASE=/mnt/gvfs/third-party2/zlib/6d39cb54708049f527e713ad19f2aadb9d3667e8/1.2.8/gcc-4.8.1-glibc-2.17/c3f970a
+BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/2ddd45f0853bfc8bb1c27f0f447236a1a26c338a/1.0.6/gcc-4.8.1-glibc-2.17/c3f970a
+LZ4_BASE=/mnt/gvfs/third-party2/lz4/6858fac689e0f92e584224d91bdb0e39f6c8320d/r131/gcc-4.8.1-glibc-2.17/c3f970a
+ZSTD_BASE=/mnt/gvfs/third-party2/zstd/cb6c4880fcb4fee471574ba6af63a3882155a16a/0.5.1/gcc-4.8.1-glibc-2.17/c3f970a
+GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/c7275a4ceae0aca0929e56964a31dafc53c1ee96/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a
+JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/d9acac8a9a2c9378eb696e22ffa8bd0e97d9206b/master/gcc-4.8.1-glibc-2.17/8d31e51
+NUMA_BASE=/mnt/gvfs/third-party2/numa/ae54a5ed22cdabb1c6446dce4e8ffae5b4446d73/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a
+LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/41bfa3759ce52c071f5fd547ec9ecd2522929f0a/trunk/gcc-4.8.1-glibc-2.17/675d945
+KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/1a48835975c66d30e47770ec419758ed3b9ba010/3.10.62-62_fbk17_03959_ge29cc63/gcc-4.8.1-glibc-2.17/da39a3e
+BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/71454c53fffcb716a0beb9a90047aff7fb5c984a/2.26/centos6-native/da39a3e
+VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/af85c56f424cd5edfc2c97588299b44ecdec96bb/3.8.1/gcc-4.8.1-glibc-2.17/c3f970a
@@ -117,11 +117,8 @@ class TsanErrorParser(MatchErrorParser):
 _TEST_NAME_TO_PARSERS = {
    'punit': [CompilerErrorParser, GTestErrorParser],
    'unit': [CompilerErrorParser, GTestErrorParser],
-    'release': [CompilerErrorParser, GTestErrorParser],
    'unit_481': [CompilerErrorParser, GTestErrorParser],
-    'release_481': [CompilerErrorParser, GTestErrorParser],
    'clang_unit': [CompilerErrorParser, GTestErrorParser],
-    'clang_release': [CompilerErrorParser, GTestErrorParser],
    'clang_analyze': [CompilerErrorParser, ScanBuildErrorParser],
    'code_cov': [CompilerErrorParser, GTestErrorParser],
    'unity': [CompilerErrorParser, GTestErrorParser],
@@ -13,7 +13,7 @@ source "$BASEDIR/dependencies.sh"
 CFLAGS=""

 # libgcc
-LIBGCC_INCLUDE="$LIBGCC_BASE/include"
+LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/7.3.0"
 LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"

 # glibc
@@ -43,12 +43,16 @@ if test -z $PIC_BUILD; then
  LZ4_INCLUDE=" -I $LZ4_BASE/include/"
  LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
  CFLAGS+=" -DLZ4"
-
-  ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
-  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
-  CFLAGS+=" -DZSTD"
 fi

+ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
+if test -z $PIC_BUILD; then
+  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
+else
+  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a"
+fi
+CFLAGS+=" -DZSTD"
+
 # location of gflags headers and libraries
 GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
 if test -z $PIC_BUILD; then
@@ -56,7 +60,7 @@ if test -z $PIC_BUILD; then
 else
  GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
 fi
-CFLAGS+=" -DGFLAGS=google"
+CFLAGS+=" -DGFLAGS=gflags"

 # location of jemalloc
 JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
@@ -104,8 +108,8 @@ if [ -z "$USE_CLANG" ]; then
  CXX="$GCC_BASE/bin/g++"

  CFLAGS+=" -B$BINUTILS/gold"
-  CFLAGS+=" -isystem $GLIBC_INCLUDE"
  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
+  CFLAGS+=" -isystem $GLIBC_INCLUDE"
  JEMALLOC=1
 else
  # clang
@@ -116,8 +120,8 @@ else
  KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"

  CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.9.x "
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.9.x/x86_64-facebook-linux "
+  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x "
+  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x/x86_64-facebook-linux "
  CFLAGS+=" -isystem $GLIBC_INCLUDE"
  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
  CFLAGS+=" -isystem $CLANG_INCLUDE"
@@ -128,13 +132,14 @@ else
 fi

 CFLAGS+=" $DEPS_INCLUDE"
-CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE"
+CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
 CXXFLAGS+=" $CFLAGS"

 EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
-EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.9-glibc-2.20/lib/ld.so"
+EXEC_LDFLAGS+=" -B$BINUTILS/gold"
+EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform007/lib/ld.so"
 EXEC_LDFLAGS+=" $LIBUNWIND"
-EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/gcc-4.9-glibc-2.20/lib"
+EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform007/lib"
 # required by libtbb
 EXEC_LDFLAGS+=" -ldl"

@@ -144,12 +149,4 @@ EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GF

 VALGRIND_VER="$VALGRIND_BASE/bin/"

-LUA_PATH="$LUA_BASE"
-
-if test -z $PIC_BUILD; then
-  LUA_LIB=" $LUA_PATH/lib/liblua.a"
-else
-  LUA_LIB=" $LUA_PATH/lib/liblua_pic.a"
-fi
-
-export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB
+export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD 
@@ -9,7 +9,7 @@ source "$BASEDIR/dependencies_4.8.1.sh"

 # location of libgcc
 LIBGCC_INCLUDE="$LIBGCC_BASE/include"
-LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
+LIBGCC_LIBS=" -L $LIBGCC_BASE/libs"

 # location of glibc
 GLIBC_INCLUDE="$GLIBC_BASE/include"
@@ -48,17 +48,13 @@ NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
 # location of libunwind
 LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"

-# location of tbb
-TBB_INCLUDE=" -isystem $TBB_BASE/include/"
-TBB_LIBS="$TBB_BASE/lib/libtbb.a"
-
 # use Intel SSE support for checksum calculations
 export USE_SSE=1

 BINUTILS="$BINUTILS_BASE/bin"
 AR="$BINUTILS/ar"

-DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP2_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE"
+DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP2_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE"

 STDLIBS="-L $GCC_BASE/lib64"

@@ -94,15 +90,13 @@ fi

 CFLAGS+=" $DEPS_INCLUDE"
 CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE"
-CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2 -DLZ4 -DZSTD -DNUMA -DTBB"
+CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2 -DLZ4 -DZSTD -DNUMA"
 CXXFLAGS+=" $CFLAGS"

-EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP2_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
+EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP2_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB"
 EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib/ld.so"
 EXEC_LDFLAGS+=" $LIBUNWIND"
 EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib"
-# required by libtbb
-EXEC_LDFLAGS+=" -ldl"

 PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"

@@ -110,6 +104,4 @@ EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP2_LIBS $LZ4_LIBS $ZSTD_LIBS $G

 VALGRIND_VER="$VALGRIND_BASE/bin/"

-LUA_PATH="$LUA_BASE"
-
-export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE LUA_PATH
+export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE
@@ -9,12 +9,9 @@ fi
 # Check clang-format-diff.py
 if ! which $CLANG_FORMAT_DIFF &> /dev/null
 then
-  echo "You didn't have clang-format-diff.py and/or clang-format available in your computer!"
-  echo "You can download clang-format-diff.py by running: "
-  echo "    curl --location http://goo.gl/iUW1u2 -o ${CLANG_FORMAT_DIFF}"
-  echo "You can download clang-format by running: "
-  echo "    brew install clang-format"
-  echo "Then, move both files (i.e. ${CLANG_FORMAT_DIFF} and clang-format) to some directory within PATH=${PATH}"
+  echo "You didn't have clang-format-diff.py available in your computer!"
+  echo "You can download it by running: "
+  echo "    curl http://goo.gl/iUW1u2"
  exit 128
 fi

@@ -53,15 +50,14 @@ fi
 set -e

 uncommitted_code=`git diff HEAD`
-LAST_MASTER=`git merge-base master HEAD`

 # If there's no uncommitted changes, we assume user are doing post-commit
-# format check, in which case we'll check the modified lines since last commit
-# from master. Otherwise, we'll check format of the uncommitted code only.
+# format check, in which case we'll check the modified lines from latest commit.
+# Otherwise, we'll check format of the uncommitted code only.
 if [ -z "$uncommitted_code" ]
 then
  # Check the format of last commit
-  diffs=$(git diff -U0 $LAST_MASTER^ | $CLANG_FORMAT_DIFF -p 1)
+  diffs=$(git diff -U0 HEAD^ | $CLANG_FORMAT_DIFF -p 1)
 else
  # Check the format of uncommitted lines,
  diffs=$(git diff -U0 HEAD | $CLANG_FORMAT_DIFF -p 1)
@@ -101,12 +97,7 @@ then
 fi

 # Do in-place format adjustment.
-if [ -z "$uncommitted_code" ]
-then
-  git diff -U0 $LAST_MASTER^ | $CLANG_FORMAT_DIFF -i -p 1
-else
-  git diff -U0 HEAD^ | $CLANG_FORMAT_DIFF -i -p 1
-fi
+git diff -U0 HEAD^ | $CLANG_FORMAT_DIFF -i -p 1
 echo "Files reformatted!"

 # Amend to last commit if user do the post-commit format check
@@ -65,8 +65,8 @@ function main() {

  if [[ -d /vagrant ]]; then
    if [[ $OS = "ubuntu" ]]; then
-      package g++-4.8
-      export CXX=g++-4.8
+      package g++-4.7
+      export CXX=g++-4.7

      # the deb would depend on libgflags2, but the static lib is the only thing
      # installed by make install
@@ -1,4 +1,4 @@
-#!/usr/local/fbcode/gcc-4.9-glibc-2.20-fb/bin/python2.7
+#!/usr/local/fbcode/gcc-4.8.1-glibc-2.17-fb/bin/python2.7

 from __future__ import absolute_import
 from __future__ import division
@@ -12,15 +12,17 @@ import re
 import os
 import time

-
 #
 # Simple logger
 #

+
 class Log:

-    def __init__(self, filename):
-        self.filename = filename
+    LOG_FILE = "/tmp/precommit-check.log"
+
+    def __init__(self):
+        self.filename = Log.LOG_FILE
        self.f = open(self.filename, 'w+', 0)

    def caption(self, str):
@@ -50,9 +52,9 @@ class Log:

 class Env(object):

-    def __init__(self, logfile, tests):
+    def __init__(self, tests):
        self.tests = tests
-        self.log = Log(logfile)
+        self.log = Log()

    def shell(self, cmd, path=os.getcwd()):
        if path:
@@ -85,16 +87,15 @@ class Env(object):

 class PreCommitChecker(Env):

-    def __init__(self, args):
-        Env.__init__(self, args.logfile, args.tests)
-        self.ignore_failure = args.ignore_failure
+    def __init__(self, tests):
+        Env.__init__(self, tests)

    #
    #   Get commands for a given job from the determinator file
    #
    def get_commands(self, test):
        status, out = self.GetOutput(
-            "RATIO=1 build_tools/rocksdb-lego-determinator %s" % test, ".")
+            "build_tools/rocksdb-lego-determinator %s" % test, ".")
        return status, out

    #
@@ -143,7 +144,6 @@ class PreCommitChecker(Env):
        self.print_row("TEST", "RESULT")
        self.print_separator()

-        result = True
        for test in self.tests:
            start_time = time.time()
            self.print_test(test)
@@ -152,14 +152,11 @@ class PreCommitChecker(Env):
            if not result:
                self.log.error("Error running test %s" % test)
                self.print_result("FAIL (%dm)" % elapsed_min)
-                if not self.ignore_failure:
-                    return False
-                result = False
-            else:
-                self.print_result("PASS (%dm)" % elapsed_min)
+                return False
+            self.print_result("PASS (%dm)" % elapsed_min)

        self.print_separator()
-        return result
+        return True

    #
    # Print a line
@@ -185,24 +182,17 @@ class PreCommitChecker(Env):
 #
 parser = argparse.ArgumentParser(description='RocksDB pre-commit checker.')

-# --log <logfile>
-parser.add_argument('--logfile', default='/tmp/precommit-check.log',
-                    help='Log file. Default is /tmp/precommit-check.log')
-# --ignore_failure
-parser.add_argument('--ignore_failure', action='store_true', default=False,
-                    help='Stop when an error occurs')
 # <test ....>
-parser.add_argument('tests', nargs='+',
+parser.add_argument('test', nargs='+',
                    help='CI test(s) to run. e.g: unit punit asan tsan ubsan')

-args = parser.parse_args()
-checker = PreCommitChecker(args)
+print("Please follow log %s" % Log.LOG_FILE)

-print("Please follow log %s" % checker.log.filename)
+args = parser.parse_args()
+checker = PreCommitChecker(args.test)

 if not checker.run_tests():
-    print("Error running tests. Please check log file %s"
-          % checker.log.filename)
+    print("Error running tests. Please check log file %s" % Log.LOG_FILE)
    sys.exit(1)

 sys.exit(0)
@@ -55,6 +55,7 @@ make release
    --open_files=55000 \
    --statistics=1 \
    --histogram=1 \
+    --disable_data_sync=1 \
    --disable_wal=1 \
    --sync=0  > ${STAT_FILE}.fillseq

@@ -72,6 +73,7 @@ make release
    --open_files=55000 \
    --statistics=1 \
    --histogram=1 \
+    --disable_data_sync=1 \
    --disable_wal=1 \
    --sync=0 \
    --threads=8 > ${STAT_FILE}.overwrite
@@ -90,6 +92,7 @@ make release
    --open_files=55000 \
    --statistics=1 \
    --histogram=1 \
+    --disable_data_sync=1 \
    --disable_wal=1 \
    --sync=0 \
    --threads=1 > /dev/null
@@ -108,6 +111,7 @@ make release
    --open_files=55000 \
    --statistics=1 \
    --histogram=1 \
+    --disable_data_sync=1 \
    --disable_wal=1 \
    --sync=0 \
    --threads=16 > ${STAT_FILE}.readrandom
@@ -127,6 +131,7 @@ make release
    --use_tailing_iterator=1 \
    --statistics=1 \
    --histogram=1 \
+    --disable_data_sync=1 \
    --disable_wal=1 \
    --sync=0 \
    --threads=16 > ${STAT_FILE}.readrandomtailing
@@ -145,6 +150,7 @@ make release
    --open_files=55000 \
    --statistics=1 \
    --histogram=1 \
+    --disable_data_sync=1 \
    --disable_wal=1 \
    --sync=0 \
    --threads=16 > ${STAT_FILE}.readrandomsmallblockcache
@@ -165,6 +171,7 @@ make release
    --open_files=55000 \
    --statistics=1 \
    --histogram=1 \
+    --disable_data_sync=1 \
    --disable_wal=1 \
    --sync=0 \
    --threads=16 > ${STAT_FILE}.readrandom_mem_sst
@@ -184,6 +191,7 @@ make release
    --open_files=55000 \
    --statistics=1 \
    --histogram=1 \
+    --disable_data_sync=1 \
    --disable_wal=1 \
    --sync=0 \
    --threads=1 > /dev/null
@@ -202,6 +210,7 @@ make release
    --open_files=55000 \
    --statistics=1 \
    --histogram=1 \
+    --disable_data_sync=1 \
    --disable_wal=1 \
    --sync=0 \
    --threads=16 > /dev/null
@@ -221,6 +230,7 @@ make release
    --disable_auto_compactions=1 \
    --statistics=1 \
    --histogram=1 \
+    --disable_data_sync=1 \
    --disable_wal=1 \
    --sync=0 \
    --threads=16 > ${STAT_FILE}.readrandom_filluniquerandom
@@ -241,6 +251,7 @@ make release
    --open_files=55000 \
    --statistics=1 \
    --histogram=1 \
+    --disable_data_sync=1 \
    --disable_wal=1 \
    --sync=0 \
    --threads=16 > ${STAT_FILE}.readwhilewriting
@@ -259,6 +270,7 @@ make release
    --open_files=55000 \
    --statistics=1 \
    --histogram=1 \
+    --disable_data_sync=1 \
    --disable_wal=1 \
    --sync=0 \
    --value_size=10 \
@@ -283,6 +295,7 @@ common_in_mem_args="--db=/dev/shm/rocksdb \
    --disable_wal=0 \
    --wal_dir=/dev/shm/rocksdb \
    --sync=0 \
+    --disable_data_sync=1 \
    --verify_checksum=1 \
    --delete_obsolete_files_period_micros=314572800 \
    --max_grandparent_overlap_factor=10 \
@@ -343,6 +356,7 @@ common_in_mem_args="--db=/dev/shm/rocksdb \
    --open_files=55000 \
    --statistics=1 \
    --histogram=1 \
+    --disable_data_sync=1 \
    --disable_wal=1 \
    --sync=0  > ${STAT_FILE}.fillseq_lots_column_families

@@ -358,6 +372,7 @@ common_in_mem_args="--db=/dev/shm/rocksdb \
    --open_files=55000 \
    --statistics=1 \
    --histogram=1 \
+    --disable_data_sync=1 \
    --disable_wal=1 \
    --sync=0 \
    --threads=8 > ${STAT_FILE}.overwrite_lots_column_families
@@ -8,7 +8,7 @@
 # Input         Value
 # -------------------------------------------------------------------------
 # EMAIL         Email address to report on trigger conditions
-# ONCALL        Email address to raise a task on failure
+# ONCAL         Email address to raise a task on failure
 # TRIGGER       Trigger conditions for email. Valid values are fail, warn, all
 # SUBSCRIBER    Email addresss to add as subscriber for task
 #
@@ -42,9 +42,6 @@ if [ ! -z $ONCALL ]; then
  },"
 fi

-# For now, create the tasks using only the dedicated task creation tool.
-CREATE_TASK=
-
 REPORT=
 if [[ ! -z $REPORT_EMAIL || ! -z $CREATE_TASK ]]; then
  REPORT="'report': [
@@ -59,53 +56,23 @@ fi
 CLEANUP_ENV="
 {
    'name':'Cleanup environment',
-    'shell':'rm -rf /dev/shm/rocksdb && mkdir /dev/shm/rocksdb && (chmod +t /dev/shm || true)  && make clean',
+    'shell':'rm -rf /dev/shm/rocksdb && mkdir /dev/shm/rocksdb && make clean',
    'user':'root'
 }"

-# We will eventually set the RATIO to 1, but we want do this
-# in steps. RATIO=$(nproc) will make it work as J=1
-if [ -z $RATIO ]; then
-  RATIO=$(nproc)
-fi
-
-if [ -z $PARALLEL_J ]; then
-  PARALLEL_J="J=$(expr $(nproc) / ${RATIO})"
-fi
-
-if [ -z $PARALLEL_j ]; then
-  PARALLEL_j="-j$(expr $(nproc) / ${RATIO})"
-fi
-
-PARALLELISM="$PARALLEL_J $PARALLEL_j"
-
 DEBUG="OPT=-g"
 SHM="TEST_TMPDIR=/dev/shm/rocksdb"
-NON_SHM="TMPD=/tmp/rocksdb_test_tmp"
 GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1"
 ASAN="COMPILE_WITH_ASAN=1"
 CLANG="USE_CLANG=1"
-LITE="OPT=\"-DROCKSDB_LITE -g\""
+LITE="OPT=-DROCKSDB_LITE"
 TSAN="COMPILE_WITH_TSAN=1"
 UBSAN="COMPILE_WITH_UBSAN=1"
 DISABLE_JEMALLOC="DISABLE_JEMALLOC=1"
 HTTP_PROXY="https_proxy=http://fwdproxy.29.prn1:8080 http_proxy=http://fwdproxy.29.prn1:8080 ftp_proxy=http://fwdproxy.29.prn1:8080"
-SETUP_JAVA_ENV="export $HTTP_PROXY; export JAVA_HOME=/usr/local/jdk-8u60-64/; export PATH=\$JAVA_HOME/bin:\$PATH"
+SETUP_JAVA_ENV="export $HTTP_PROXY; export JAVA_HOME=/usr/local/jdk-7u10-64/; export PATH=\$PATH:\$JAVA_HOME/bin"
 PARSER="'parser':'python build_tools/error_filter.py $1'"

-CONTRUN_NAME="ROCKSDB_CONTRUN_NAME"
-
-# This code is getting called under various scenarios. What we care about is to
-# understand when it's called from nightly contruns because in that case we'll
-# create tasks for any failures. To follow the existing pattern, we'll check
-# the value of $ONCALL. If it's a diff then just call `false` to make sure
-# that errors will be properly propagated to the caller.
-if [ ! -z $ONCALL ]; then
-  TASK_CREATION_TOOL="/usr/local/bin/mysql_mtr_filter --rocksdb --oncall $ONCALL"
-else
-  TASK_CREATION_TOOL="false"
-fi
-
 ARTIFACTS=" 'artifacts': [
    {
        'name':'database',
@@ -143,14 +110,14 @@ PARALLEL_UNIT_TEST_COMMANDS="[
            $CLEANUP_ENV,
            {
                'name':'Build and test RocksDB debug version',
-                'shell':'$DEBUG make -j$(nproc) all && $SHM make check > /dev/null 2>&1 || $CONTRUN_NAME=punit_check $TASK_CREATION_TOOL || cat t/log-*',
+                'shell':'$DEBUG make -j$(nproc) all && $SHM make check > /dev/null 2>&1 || cat t/log-*',
                'user':'root',
                $PARSER
            },
            $CLEANUP_ENV,
            {
                'name':'Build and test RocksDB debug version under gcc-4.8.1',
-                'shell':'$GCC_481 $DEBUG make -j$(nproc) all && $SHM make check > /dev/null 2>&1 || $CONTRUN_NAME=punit_check_gcc481 $TASK_CREATION_TOOL || cat t/log-*',
+                'shell':'$GCC_481 $DEBUG make -j$(nproc) all && $SHM make check > /dev/null 2>&1 || cat t/log-*',
                'user':'root',
                $PARSER
            },
@@ -170,49 +137,7 @@ UNIT_TEST_COMMANDS="[
            $CLEANUP_ENV,
            {
                'name':'Build and test RocksDB debug version',
-                'shell':'$SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=check $TASK_CREATION_TOOL',
-                'user':'root',
-                $PARSER
-            },
-        ],
-        $REPORT
-    }
-]"
-
-#
-# RocksDB unit test not under /dev/shm
-#
-UNIT_TEST_NON_SHM_COMMANDS="[
-    {
-        'name':'Rocksdb Unit Test',
-        'oncall':'$ONCALL',
-        'timeout': 86400,
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'Build and test RocksDB debug version',
-                'timeout': 86400,
-                'shell':'$NON_SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=non_shm_check $TASK_CREATION_TOOL',
-                'user':'root',
-                $PARSER
-            },
-        ],
-        $REPORT
-    }
-]"
-
-#
-# RocksDB release build and unit tests
-#
-RELEASE_BUILD_COMMANDS="[
-    {
-        'name':'Rocksdb Release Build',
-        'oncall':'$ONCALL',
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'Build RocksDB release',
-                'shell':'make $PARALLEL_j release || $CONTRUN_NAME=release $TASK_CREATION_TOOL',
+                'shell':'$SHM $DEBUG make J=1 check',
                'user':'root',
                $PARSER
            },
@@ -232,27 +157,7 @@ UNIT_TEST_COMMANDS_481="[
            $CLEANUP_ENV,
            {
                'name':'Build and test RocksDB debug version',
-                'shell':'$SHM $GCC_481 $DEBUG make $PARALLELISM check || $CONTRUN_NAME=unit_gcc_481_check $TASK_CREATION_TOOL',
-                'user':'root',
-                $PARSER
-            },
-        ],
-        $REPORT
-    }
-]"
-
-#
-# RocksDB release build and unit tests
-#
-RELEASE_BUILD_COMMANDS_481="[
-    {
-        'name':'Rocksdb Release on GCC 4.8.1',
-        'oncall':'$ONCALL',
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'Build RocksDB release on GCC 4.8.1',
-                'shell':'$GCC_481 make $PARALLEL_j release || $CONTRUN_NAME=release_gcc481 $TASK_CREATION_TOOL',
+                'shell':'$SHM $GCC_481 $DEBUG make J=1 check',
                'user':'root',
                $PARSER
            },
@@ -272,27 +177,7 @@ CLANG_UNIT_TEST_COMMANDS="[
            $CLEANUP_ENV,
            {
                'name':'Build and test RocksDB debug',
-                'shell':'$CLANG $SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=clang_check $TASK_CREATION_TOOL',
-                'user':'root',
-                $PARSER
-            },
-        ],
-        $REPORT
-    }
-]"
-
-#
-# RocksDB release build with CLANG
-#
-CLANG_RELEASE_BUILD_COMMANDS="[
-    {
-        'name':'Rocksdb CLANG Release Build',
-        'oncall':'$ONCALL',
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'Build RocksDB release',
-                'shell':'$CLANG make $PARALLEL_j release|| $CONTRUN_NAME=clang_release $TASK_CREATION_TOOL',
+                'shell':'$CLANG $SHM $DEBUG make J=1 check',
                'user':'root',
                $PARSER
            },
@@ -312,7 +197,7 @@ CLANG_ANALYZE_COMMANDS="[
            $CLEANUP_ENV,
            {
                'name':'RocksDB build and analyze',
-                'shell':'$CLANG $SHM $DEBUG make $PARALLEL_j analyze || $CONTRUN_NAME=clang_analyze $TASK_CREATION_TOOL',
+                'shell':'$CLANG $SHM $DEBUG make J=1 analyze',
                'user':'root',
                $PARSER
            },
@@ -332,7 +217,7 @@ CODE_COV_COMMANDS="[
            $CLEANUP_ENV,
            {
                'name':'Build, test and collect code coverage info',
-                'shell':'$SHM $DEBUG make $PARALLELISM coverage || $CONTRUN_NAME=coverage $TASK_CREATION_TOOL',
+                'shell':'$SHM $DEBUG make J=1 coverage',
                'user':'root',
                $PARSER
            },
@@ -352,7 +237,7 @@ UNITY_COMMANDS="[
            $CLEANUP_ENV,
            {
                'name':'Build, test unity test',
-                'shell':'$SHM $DEBUG V=1 make J=1 unity_test || $CONTRUN_NAME=unity_test $TASK_CREATION_TOOL',
+                'shell':'$SHM $DEBUG V=1 make J=1 unity_test',
                'user':'root',
                $PARSER
            },
@@ -372,7 +257,7 @@ LITE_BUILD_COMMANDS="[
            $CLEANUP_ENV,
            {
                'name':'Build RocksDB debug version',
-                'shell':'$LITE make J=1 static_lib || $CONTRUN_NAME=lite_static_lib $TASK_CREATION_TOOL',
+                'shell':'$LITE $DEBUG make J=1 static_lib',
                'user':'root',
                $PARSER
            },
@@ -392,7 +277,7 @@ LITE_UNIT_TEST_COMMANDS="[
            $CLEANUP_ENV,
            {
                'name':'Build RocksDB debug version',
-                'shell':'$SHM $LITE make J=1 check || $CONTRUN_NAME=lite_check $TASK_CREATION_TOOL',
+                'shell':'$SHM $LITE $DEBUG make J=1 check',
                'user':'root',
                $PARSER
            },
@@ -413,14 +298,14 @@ STRESS_CRASH_TEST_COMMANDS="[
            $CLEANUP_ENV,
            {
                'name':'Build and run RocksDB debug stress tests',
-                'shell':'$SHM $DEBUG make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
+                'shell':'$SHM $DEBUG make J=1 db_stress',
                'user':'root',
                $PARSER
            },
            {
                'name':'Build and run RocksDB debug crash tests',
                'timeout': 86400,
-                'shell':'$SHM $DEBUG make J=1 crash_test || $CONTRUN_NAME=crash_test $TASK_CREATION_TOOL',
+                'shell':'$SHM $DEBUG make J=1 crash_test',
                'user':'root',
                $PARSER
            }
@@ -441,7 +326,7 @@ WRITE_STRESS_COMMANDS="[
            $CLEANUP_ENV,
            {
                'name':'Build and run RocksDB write stress tests',
-                'shell':'make write_stress && python tools/write_stress_runner.py --runtime_sec=3600 --db=/tmp/rocksdb_write_stress || $CONTRUN_NAME=write_stress $TASK_CREATION_TOOL',
+                'shell':'make write_stress && python tools/write_stress_runner.py --runtime_sec=3600 --db=/tmp/rocksdb_write_stress',
                'user':'root',
                $PARSER
            }
@@ -463,7 +348,7 @@ ASAN_TEST_COMMANDS="[
            $CLEANUP_ENV,
            {
                'name':'Test RocksDB debug under ASAN',
-'shell':'set -o pipefail && ($SHM $ASAN $DEBUG make $PARALLELISM asan_check || $CONTRUN_NAME=asan_check $TASK_CREATION_TOOL) |& /usr/facebook/ops/scripts/asan_symbolize.py -d',
+                'shell':'set -o pipefail && $SHM $ASAN $DEBUG make J=1 asan_check |& /usr/facebook/ops/scripts/asan_symbolize.py -d',
                'user':'root',
                $PARSER
            }
@@ -485,7 +370,7 @@ ASAN_CRASH_TEST_COMMANDS="[
            {
                'name':'Build and run RocksDB debug asan_crash_test',
                'timeout': 86400,
-                'shell':'$SHM $DEBUG make J=1 asan_crash_test || $CONTRUN_NAME=asan_crash_test $TASK_CREATION_TOOL',
+                'shell':'$SHM $DEBUG make J=1 asan_crash_test',
                'user':'root',
                $PARSER
            },
@@ -505,7 +390,7 @@ UBSAN_TEST_COMMANDS="[
            $CLEANUP_ENV,
            {
                'name':'Test RocksDB debug under UBSAN',
-                'shell':'set -o pipefail && $SHM $UBSAN $DEBUG make $PARALLELISM ubsan_check || $CONTRUN_NAME=ubsan_check $TASK_CREATION_TOOL',
+                'shell':'set -o pipefail && $SHM $UBSAN $DEBUG make J=1 ubsan_check',
                'user':'root',
                $PARSER
            }
@@ -527,7 +412,7 @@ UBSAN_CRASH_TEST_COMMANDS="[
            {
                'name':'Build and run RocksDB debug ubsan_crash_test',
                'timeout': 86400,
-                'shell':'$SHM $DEBUG make J=1 ubsan_crash_test || $CONTRUN_NAME=ubsan_crash_test $TASK_CREATION_TOOL',
+                'shell':'$SHM $DEBUG make J=1 ubsan_crash_test',
                'user':'root',
                $PARSER
            },
@@ -543,13 +428,11 @@ VALGRIND_TEST_COMMANDS="[
    {
        'name':'Rocksdb Unit Test under valgrind',
        'oncall':'$ONCALL',
-        'timeout': 86400,
        'steps': [
            $CLEANUP_ENV,
            {
                'name':'Run RocksDB debug unit tests',
-                'timeout': 86400,
-                'shell':'$SHM $DEBUG make $PARALLELISM valgrind_test || $CONTRUN_NAME=valgrind_check $TASK_CREATION_TOOL',
+                'shell':'$DISABLE_JEMALLOC $SHM $DEBUG make valgrind_check',
                'user':'root',
                $PARSER
            },
@@ -565,13 +448,11 @@ TSAN_UNIT_TEST_COMMANDS="[
    {
        'name':'Rocksdb Unit Test under TSAN',
        'oncall':'$ONCALL',
-        'timeout': 86400,
        'steps': [
            $CLEANUP_ENV,
            {
                'name':'Run RocksDB debug unit test',
-                'timeout': 86400,
-                'shell':'set -o pipefail && $SHM $DEBUG $TSAN make $PARALLELISM check || $CONTRUN_NAME=tsan_check $TASK_CREATION_TOOL',
+                'shell':'set -o pipefail && $SHM $DEBUG $TSAN make J=1 check',
                'user':'root',
                $PARSER
            },
@@ -593,7 +474,7 @@ TSAN_CRASH_TEST_COMMANDS="[
            {
                'name':'Compile and run',
                'timeout': 86400,
-                'shell':'set -o pipefail && $SHM $DEBUG $TSAN CRASH_TEST_KILL_ODD=1887 make J=1 crash_test || $CONTRUN_NAME=tsan_crash_test $TASK_CREATION_TOOL',
+                'shell':'set -o pipefail && $SHM $DEBUG $TSAN make J=1 crash_test',
                'user':'root',
                $PARSER
            },
@@ -602,6 +483,8 @@ TSAN_CRASH_TEST_COMMANDS="[
    }
 ]"

+TSAN_CRASH_TEST_COMMANDS=$DISABLE_COMMANDS
+
 #
 # RocksDB format compatible
 #
@@ -655,7 +538,7 @@ FORMAT_COMPATIBLE_COMMANDS="[
            $CLEANUP_ENV,
            {
                'name':'Run RocksDB debug unit test',
-                'shell':'build_tools/rocksdb-lego-determinator run_format_compatible || $CONTRUN_NAME=run_format_compatible $TASK_CREATION_TOOL',
+                'shell':'build_tools/rocksdb-lego-determinator run_format_compatible',
                'user':'root',
                $PARSER
            },
@@ -688,7 +571,7 @@ NO_COMPRESSION_COMMANDS="[
            $CLEANUP_ENV,
            {
                'name':'Run RocksDB debug unit test',
-                'shell':'build_tools/rocksdb-lego-determinator run_no_compression || $CONTRUN_NAME=run_no_compression $TASK_CREATION_TOOL',
+                'shell':'build_tools/rocksdb-lego-determinator run_no_compression',
                'user':'root',
                $PARSER
            },
@@ -745,7 +628,7 @@ REGRESSION_COMMANDS="[
            $CLEANUP_ENV,
            {
                'name':'Make and run script',
-                'shell':'build_tools/rocksdb-lego-determinator run_regression || $CONTRUN_NAME=run_regression $TASK_CREATION_TOOL',
+                'shell':'build_tools/rocksdb-lego-determinator run_regression',
                'user':'root',
                $PARSER
            },
@@ -765,7 +648,7 @@ JAVA_BUILD_TEST_COMMANDS="[
            $CLEANUP_ENV,
            {
                'name':'Build RocksDB for Java',
-                'shell':'$SETUP_JAVA_ENV; $SHM make rocksdbjava || $CONTRUN_NAME=rocksdbjava $TASK_CREATION_TOOL',
+                'shell':'$SETUP_JAVA_ENV; $SHM make rocksdbjava',
                'user':'root',
                $PARSER
            },
@@ -782,24 +665,12 @@ case $1 in
  unit)
    echo $UNIT_TEST_COMMANDS
    ;;
-  unit_non_shm)
-    echo $UNIT_TEST_NON_SHM_COMMANDS
-    ;;
-  release)
-    echo $RELEASE_BUILD_COMMANDS
-    ;;
  unit_481)
    echo $UNIT_TEST_COMMANDS_481
    ;;
-  release_481)
-    echo $RELEASE_BUILD_COMMANDS_481
-    ;;
  clang_unit)
    echo $CLANG_UNIT_TEST_COMMANDS
    ;;
-  clang_release)
-    echo $CLANG_RELEASE_BUILD_COMMANDS
-    ;;
  clang_analyze)
    echo $CLANG_ANALYZE_COMMANDS
    ;;
@@ -1,50 +1,24 @@
 # This script enables you running RocksDB tests by running
-# All the tests concurrently and utilizing all the cores
+# All the tests in parallel and utilizing all the cores
+# For db_test the script first lists and parses the tests
+# and then fires them up in parallel using async PS Job functionality
+# Run the script from the enlistment
 Param(
-  [switch]$EnableJE = $false,  # Look for and use _je executable, append _je to listed exclusions
-  [switch]$RunAll = $false,    # Will attempt discover all *_test[_je].exe binaries and run all
-                               # of them as Google suites. I.e. It will run test cases concurrently
-                               # except those mentioned as $Run, those will run as individual test cases
-                               # And any execlued with $ExcludeExes or $ExcludeCases
-                               # It will also not run any individual test cases
-                               # excluded but $ExcludeCasese
-  [string]$SuiteRun = "",      # Split test suites in test cases and run in parallel, not compatible with $RunAll
-  [string]$Run = "",           # Run specified executables in parallel but do not split to test cases
-  [string]$ExcludeCases = "",  # Exclude test cases, expects a comma separated list, no spaces
-                               # Takes effect when $RunAll or $SuiteRun is specified. Must have full
-                               # Test cases name including a group and a parameter if any
-  [string]$ExcludeExes = "",   # Exclude exes from consideration, expects a comma separated list,
-                               # no spaces. Takes effect only when $RunAll is specified
-  [string]$WorkFolder = "",    # Direct tests to use that folder. SSD or Ram drive are better options.
+  [switch]$EnableJE = $false,  # Use je executable
+  [switch]$EnableRerun = $false, # Rerun failed tests sequentially at the end
+  [string]$WorkFolder = "",  # Direct tests to use that folder
+  [int]$Limit = -1, # -1 means run all otherwise limit for testing purposes
+  [string]$Exclude = "", # Expect a comma separated list, no spaces
+  [string]$Run = "db_test",  # Run db_test|db_test2|tests|testname1,testname2...
   # Number of async tasks that would run concurrently. Recommend a number below 64.
   # However, CPU utlization really depends on the storage media. Recommend ram based disk.
   # a value of 1 will run everything serially
-  [int]$Concurrency = 8,
-  [int]$Limit = -1 # -1 means do not limit for test purposes
+  [int]$Concurrency = 16
 )

 # Folders and commands must be fullpath to run assuming
 # the current folder is at the root of the git enlistment
-$StartDate = (Get-Date)
-$StartDate
-
-
-$DebugPreference = "Continue"
-
-# These tests are not google test suites and we should guard
-# Against running them as suites
-$RunOnly = New-Object System.Collections.Generic.HashSet[string]
-$RunOnly.Add("c_test") | Out-Null
-$RunOnly.Add("compact_on_deletion_collector_test") | Out-Null
-$RunOnly.Add("merge_test") | Out-Null
-$RunOnly.Add("stringappend_test") | Out-Null # Apparently incorrectly written
-$RunOnly.Add("backupable_db_test") | Out-Null # Disabled
-
-
-if($RunAll -and $SuiteRun -ne "") {
-    Write-Error "$RunAll and $SuiteRun are not compatible"
-    exit 1
-}
+Get-Date

 # If running under Appveyor assume that root
 [string]$Appveyor = $Env:APPVEYOR_BUILD_FOLDER
@@ -73,32 +47,39 @@ if($WorkFolder -eq "") {
 }

 Write-Output "Root: $RootFolder, WorkFolder: $WorkFolder"
-Write-Output "BinariesFolder: $BinariesFolder, LogFolder: $LogFolder"
+
+# Use JEMALLOC executables
+if($Run -ceq "db_test" -or
+   $Run -ceq "db_test2" ) {
+
+   $file_name = $Run
+
+   if($EnableJE) {
+     $file_name += "_je"
+   }
+
+   $file_name += ".exe"
+
+   $db_test = -Join ($BinariesFolder, $file_name)
+
+   Write-Output "Binaries: $BinariesFolder db_test: $db_test"
+}
+
+
+#Exclusions that we do not want to run
+$ExcludeTests = New-Object System.Collections.Generic.HashSet[string]
+
+
+if($Exclude -ne "") {
+    Write-Host "Exclude: $Exclude"
+    $l = $Exclude -split ' '
+    ForEach($t in $l) { $ExcludeTests.Add($t) | Out-Null }
+}

 # Create test directories in the current folder
 md -Path $WorkFolder -ErrorAction Ignore | Out-Null
 md -Path $LogFolder -ErrorAction Ignore | Out-Null

-
-$ExcludeCasesSet = New-Object System.Collections.Generic.HashSet[string]
-if($ExcludeCases -ne "") {
-    Write-Host "ExcludeCases: $ExcludeCases"
-    $l = $ExcludeCases -split ' '
-    ForEach($t in $l) { 
-      $ExcludeCasesSet.Add($t) | Out-Null
-    }
-}
-
-$ExcludeExesSet = New-Object System.Collections.Generic.HashSet[string]
-if($ExcludeExes -ne "") {
-    Write-Host "ExcludeExe: $ExcludeExes"
-    $l = $ExcludeExes -split ' '
-    ForEach($t in $l) { 
-      $ExcludeExesSet.Add($t) | Out-Null
-    }
-}
-
-
 # Extract the names of its tests by running db_test with --gtest_list_tests.
 # This filter removes the "#"-introduced comments, and expands to
 # fully-qualified names by changing input like this:
@@ -116,37 +97,28 @@ if($ExcludeExes -ne "") {
 #   DBTest.WriteEmptyBatch
 #   MultiThreaded/MultiThreadedDBTest.MultiThreaded/0
 #   MultiThreaded/MultiThreadedDBTest.MultiThreaded/1
-#
 # Output into the parameter in a form TestName -> Log File Name
-function ExtractTestCases([string]$GTestExe, $HashTable) {
+function Normalize-DbTests($HashTable) {

    $Tests = @()
 # Run db_test to get a list of tests and store it into $a array
-    &$GTestExe --gtest_list_tests | tee -Variable Tests | Out-Null
+    &$db_test --gtest_list_tests | tee -Variable Tests | Out-Null

    # Current group
    $Group=""

    ForEach( $l in $Tests) {
-
-      # Leading whitespace is fine
-      $l = $l -replace '^\s+',''
-      # but no whitespace any other place
-      if($l -match "\s+") {
-        continue
-      }
-      # Trailing dot is a test group but no whitespace
-      elseif ( $l -match "\.$" ) {
+      # Trailing dot is a test group
+      if( $l -match "\.$") {
        $Group = $l
      }  else {
        # Otherwise it is a test name, remove leading space
-        $test = $l
+        $test = $l -replace '^\s+',''
        # remove trailing comment if any and create a log name
        $test = $test -replace '\s+\#.*',''
        $test = "$Group$test"

-        if($ExcludeCasesSet.Contains($test)) {
-            Write-Warning "$test case is excluded"
+        if($ExcludeTests.Contains($test)) {
            continue
        }

@@ -162,130 +134,58 @@ function ExtractTestCases([string]$GTestExe, $HashTable) {

 # The function removes trailing .exe siffix if any,
 # creates a name for the log file
-# Then adds the test name if it was not excluded into
-# a HashTable in a form of test_name -> log_path
 function MakeAndAdd([string]$token, $HashTable) {
-
    $test_name = $token -replace '.exe$', ''
    $log_name =  -join ($test_name, ".log")
    $log_path = -join ($LogFolder, $log_name)
-    $HashTable.Add($test_name, $log_path)
-}
-
-# This function takes a list of Suites to run
-# Lists all the test cases in each of the suite
-# and populates HashOfHashes
-# Ordered by suite(exe) @{ Exe = @{ TestCase = LogName }}
-function ProcessSuites($ListOfSuites, $HashOfHashes) {
-
-  $suite_list = $ListOfSuites
-  # Problem: if you run --gtest_list_tests on
-  # a non Google Test executable then it will start executing
-  # and we will get nowhere
-  ForEach($suite in $suite_list) {
-
-    if($RunOnly.Contains($suite)) {
-      Write-Warning "$suite is excluded from running as Google test suite"
-      continue
-    }
-
-    if($EnableJE) {
-      $suite += "_je"
-    }
-
-    $Cases = [ordered]@{}
-    $Cases.Clear()
-    $suite_exe = -Join ($BinariesFolder, $suite)
-    ExtractTestCases -GTestExe $suite_exe -HashTable $Cases
-    if($Cases.Count -gt 0) {
-      $HashOfHashes.Add($suite, $Cases);
-    }
-  }
-
-  # Make logs and run
-  if($CasesToRun.Count -lt 1) {
-     Write-Error "Failed to extract tests from $SuiteRun"
-     exit 1
-  }
-
-}
-
-# This will contain all test executables to run
-
-# Hash table that contains all non suite
-# Test executable to run
-$TestExes = [ordered]@{}
-
-# Check for test exe that are not
-# Google Test Suites
-# Since this is explicitely mentioned it is not subject
-# for exclusions
-if($Run -ne "") {
-
-  $test_list = $Run -split ' '
-
-  ForEach($t in $test_list) {
-
-    if($EnableJE) {
-      $t += "_je"
-    }
-
-    MakeAndAdd -token $t -HashTable $TestExes
-  }
-
-  if($TestExes.Count -lt 1) {
-     Write-Error "Failed to extract tests from $Run"
-     exit 1
-  }
-}
-
-# Ordered by exe @{ Exe = @{ TestCase = LogName }}
-$CasesToRun = [ordered]@{}
-
-if($SuiteRun -ne "") {
-  $suite_list = $SuiteRun -split ' '
-  ProcessSuites -ListOfSuites $suite_list -HashOfHashes $CasesToRun
-}
-
-if($RunAll) {
-# Discover all the test binaries
-  if($EnableJE) {
-    $pattern = "*_test_je.exe"
-  } else {
-    $pattern = "*_test.exe"
-  }
-
-
-  $search_path = -join ($BinariesFolder, $pattern)
-  Write-Host "Binaries Search Path: $search_path"
-
-  $ListOfExe = @()
-  dir -Path $search_path | ForEach-Object {
-     $ListOfExe += ($_.Name)     
-  }
-
-  # Exclude those in RunOnly from running as suites
-  $ListOfSuites = @()
-  ForEach($e in $ListOfExe) {
-
-    $e = $e -replace '.exe$', ''
-    $bare_name = $e -replace '_je$', ''
-
-    if($ExcludeExesSet.Contains($bare_name)) {
-      Write-Warning "Test $e is excluded"
-      continue
-    }
-
-    if($RunOnly.Contains($e)) {
-      MakeAndAdd -token $e -HashTable $TestExes
+    if(!$ExcludeTests.Contains($test_name)) {
+        $HashTable.Add($test_name, $log_path)
    } else {
-      $ListOfSuites += $bare_name
+        Write-Warning "Test $test_name is excluded"
    }
-  }
-
-  ProcessSuites -ListOfSuites $ListOfSuites -HashOfHashes $CasesToRun
 }

+# The function scans build\Debug folder to discover
+# Test executables. It then populates a table with
+# Test executable name -> Log file
+function Discover-TestBinaries([string]$Pattern, $HashTable) {
+
+    $Exclusions = @("db_test*", "db_sanity_test*")
+
+    $p = -join ($BinariesFolder, $pattern)
+
+    Write-Host "Path: $p"
+
+    dir -Path $p -Exclude $Exclusions | ForEach-Object {
+       MakeAndAdd -token ($_.Name) -HashTable $HashTable
+    }
+}
+
+$TestsToRun = [ordered]@{}
+
+if($Run -ceq "db_test" -or
+   $Run -ceq "db_test2") {
+    Normalize-DbTests -HashTable $TestsToRun
+} elseif($Run -ceq "tests") {
+    if($EnableJE) {
+        $pattern = "*_test_je.exe"
+    } else {
+        $pattern = "*_test.exe"
+    }
+    Discover-TestBinaries -Pattern $pattern -HashTable $TestsToRun
+} else {
+
+    $test_list = $Run -split ' '
+
+    ForEach($t in $test_list) {
+       MakeAndAdd -token $t -HashTable $TestsToRun
+    }
+}
+
+$NumTestsToStart = $TestsToRun.Count
+if($Limit -ge 0 -and $NumTestsToStart -gt $Limit) {
+    $NumTestsToStart = $Limit
+}

 Write-Host "Attempting to start: $NumTestsToStart tests"

@@ -303,13 +203,13 @@ $InvokeTestAsync = {

 # Hash that contains tests to rerun if any failed
 # Those tests will be rerun sequentially
-# $Rerun = [ordered]@{}
+$Rerun = [ordered]@{}
 # Test limiting factor here
-[int]$count = 0
+$count = 0
 # Overall status
 [bool]$success = $true;

-function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
+function RunJobs($TestToLog, [int]$ConcurrencyVal, [bool]$AddForRerun)
 {
    # Array to wait for any of the running jobs
    $jobs = @()
@@ -318,66 +218,37 @@ function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)

    # Wait for all to finish and get the results
    while(($JobToLog.Count -gt 0) -or
-          ($TestCmds.Count -gt 0) -or 
-           ($Suites.Count -gt 0)) {
+          ($TestToLog.Count -gt 0)) {

        # Make sure we have maximum concurrent jobs running if anything
        # and the $Limit either not set or allows to proceed
        while(($JobToLog.Count -lt $ConcurrencyVal) -and
-              ((($TestCmds.Count -gt 0) -or ($Suites.Count -gt 0)) -and
+              (($TestToLog.Count -gt 0) -and
              (($Limit -lt 0) -or ($count -lt $Limit)))) {

-            # We always favore suites to run if available
-            [string]$exe_name = ""
-            [string]$log_path = ""
-            $Cases = @{}

-            if($Suites.Count -gt 0) {
-              # Will teh first one
-              ForEach($e in $Suites.Keys) {
-                $exe_name = $e
-                $Cases = $Suites[$e]
+            # We only need the first key
+            foreach($key in $TestToLog.keys) {
+                $k = $key
                break
-              }
-              [string]$test_case = ""
-              [string]$log_path = ""
-              ForEach($c in $Cases.Keys) {
-                 $test_case = $c
-                 $log_path = $Cases[$c]
-                 break
-              }
-
-              Write-Host "Starting $exe_name::$test_case"
-              [string]$Exe =  -Join ($BinariesFolder, $exe_name)
-              $job = Start-Job -Name "$exe_name::$test_case" -ArgumentList @($Exe,$test_case,$log_path) -ScriptBlock $InvokeTestCase
-              $JobToLog.Add($job, $log_path)
-
-              $Cases.Remove($test_case)
-              if($Cases.Count -lt 1) {
-                $Suites.Remove($exe_name)
-              }
-
-            } elseif ($TestCmds.Count -gt 0) {
-
-               ForEach($e in $TestCmds.Keys) {
-                 $exe_name = $e
-                 $log_path = $TestCmds[$e]
-                 break
-               }
-
-              [string]$Exe =  -Join ($BinariesFolder, $exe_name)
-              $job = Start-Job -Name $exe_name -ScriptBlock $InvokeTestAsync -ArgumentList @($Exe,$log_path)
-              $JobToLog.Add($job, $log_path)
-
-              $TestCmds.Remove($exe_name)
-
-            } else {
-                Write-Error "In the job loop but nothing to run"
-                exit 1
            }

+            Write-Host "Starting $k"
+            $log_path = ($TestToLog.$k)
+
+            if($Run -ceq "db_test" -or
+               $Run -ceq "db_test2") {
+              $job = Start-Job -Name $k -ScriptBlock $InvokeTestCase -ArgumentList @($db_test,$k,$log_path)
+            } else {
+              [string]$Exe =  -Join ($BinariesFolder, $k)
+               $job = Start-Job -Name $k -ScriptBlock $InvokeTestAsync -ArgumentList @($exe,$log_path)
+            }
+
+            $JobToLog.Add($job, $log_path)
+            $TestToLog.Remove($k)
+
            ++$count
-        } # End of Job starting loop
+        }

        if($JobToLog.Count -lt 1) {
          break
@@ -422,6 +293,9 @@ function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
                $success = $false;
                Write-Warning $message
                $log_content | Write-Warning
+                if($AddForRerun) {
+                    $Rerun.Add($completed.Name, $log)
+                }
            } else {
                Write-Host $message
            }
@@ -433,14 +307,16 @@ function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
    }
 }

-RunJobs -Suites $CasesToRun -TestCmds $TestExes -ConcurrencyVal $Concurrency
+RunJobs -TestToLog $TestsToRun -ConcurrencyVal $Concurrency -AddForRerun $EnableRerun

-$EndDate = (Get-Date)
+if($Rerun.Count -gt 0) {
+    Write-Host "Rerunning " ($Rerun.Count) " tests sequentially"
+    $success = $true
+    $count = 0
+    RunJobs -TestToLog $Rerun -ConcurrencyVal 1 -AddForRerun $false
+}

-New-TimeSpan -Start $StartDate -End $EndDate | 
-  ForEach-Object { 
-    "Elapsed time: {0:g}" -f $_
-  }
+Get-Date


 if(!$success) {
@@ -448,9 +324,7 @@ if(!$success) {
 # So we simply exit
 #    Remove-Job -Job $jobs -Force
 # indicate failure using this exit code
-    exit 1
+    exit 12345
 }

- exit 0
-
 
@@ -65,15 +65,15 @@ echo "Writing dependencies to $OUTPUT"

 # Compilers locations
 GCC_BASE=`readlink -f $TP2_LATEST/gcc/4.9.x/centos6-native/*/`
-CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos6-native/*/`
+CLANG_BASE=`ls -d1 /mnt/gvfs/third-party2/clang/fc904e50a9266b9d7b98cae1993afa0c5aae1440/3.7.1/centos6-native/*/ | head -n1`

 log_variable GCC_BASE
 log_variable CLANG_BASE

 # Libraries locations
 get_lib_base libgcc     4.9.x
-get_lib_base glibc      2.20
-get_lib_base snappy     LATEST gcc-4.9-glibc-2.20
+get_lib_base glibc      2.20 
+get_lib_base snappy     LATEST
 get_lib_base zlib       LATEST
 get_lib_base bzip2      LATEST
 get_lib_base lz4        LATEST
@@ -82,12 +82,10 @@ get_lib_base gflags     LATEST
 get_lib_base jemalloc   LATEST
 get_lib_base numa       LATEST
 get_lib_base libunwind  LATEST
-get_lib_base tbb        4.0_update2 gcc-4.9-glibc-2.20

 get_lib_base kernel-headers LATEST 
 get_lib_base binutils   LATEST centos6-native 
-get_lib_base valgrind   3.10.0 gcc-4.9-glibc-2.20
-get_lib_base lua        5.2.3 gcc-4.9-glibc-2.20
+get_lib_base valgrind   LATEST

 git diff $OUTPUT

@@ -104,7 +102,7 @@ echo "Writing 4.8.1 dependencies to $OUTPUT"

 # Compilers locations
 GCC_BASE=`readlink -f $TP2_LATEST/gcc/4.8.1/centos6-native/*/`
-CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos6-native/*/`
+CLANG_BASE=`ls -d1 /mnt/gvfs/third-party2/clang/fc904e50a9266b9d7b98cae1993afa0c5aae1440/3.7.1/centos6-native/*/ | head -n1`

 log_variable GCC_BASE
 log_variable CLANG_BASE
@@ -121,11 +119,9 @@ get_lib_base gflags     LATEST gcc-4.8.1-glibc-2.17
 get_lib_base jemalloc   LATEST gcc-4.8.1-glibc-2.17
 get_lib_base numa       LATEST gcc-4.8.1-glibc-2.17
 get_lib_base libunwind  LATEST gcc-4.8.1-glibc-2.17
-get_lib_base tbb        4.0_update2 gcc-4.8.1-glibc-2.17

 get_lib_base kernel-headers LATEST gcc-4.8.1-glibc-2.17 
 get_lib_base binutils   LATEST centos6-native 
 get_lib_base valgrind   3.8.1  gcc-4.8.1-glibc-2.17
-get_lib_base lua        5.2.3 centos6-native

 git diff $OUTPUT
@@ -1,9 +1,8 @@
 #!/bin/sh
 if [ "$#" = "0" ]; then
-  echo "Usage: $0 major|minor|patch|full"
+  echo "Usage: $0 major|minor|patch"
  exit 1
 fi
-
 if [ "$1" = "major" ]; then
  cat include/rocksdb/version.h  | grep MAJOR | head -n1 | awk '{print $3}'
 fi
@@ -13,10 +12,3 @@ fi
 if [ "$1" = "patch" ]; then
  cat include/rocksdb/version.h  | grep PATCH | head -n1 | awk '{print $3}'
 fi
-if [ "$1" = "full" ]; then
-  awk '/#define ROCKSDB/ { env[$2] = $3 }
-       END { printf "%s.%s.%s\n", env["ROCKSDB_MAJOR"],
-                                  env["ROCKSDB_MINOR"],
-                                  env["ROCKSDB_PATCH"] }'  \
-      include/rocksdb/version.h
-fi
@@ -1,707 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "cache/clock_cache.h"
-
-#ifndef SUPPORT_CLOCK_CACHE
-
-namespace rocksdb {
-
-std::shared_ptr<Cache> NewClockCache(size_t capacity, int num_shard_bits,
-                                     bool strict_capacity_limit) {
-  // Clock cache not supported.
-  return nullptr;
-}
-
-}  // namespace rocksdb
-
-#else
-
-#include <assert.h>
-#include <atomic>
-#include <deque>
-#include <limits>
-
-#include "tbb/concurrent_hash_map.h"
-
-#include "cache/sharded_cache.h"
-#include "port/port.h"
-#include "util/autovector.h"
-#include "util/mutexlock.h"
-
-namespace rocksdb {
-
-namespace {
-
-// An implementation of the Cache interface based on CLOCK algorithm, with
-// better concurrent performance than LRUCache. The idea of CLOCK algorithm
-// is to maintain all cache entries in a circular list, and an iterator
-// (the "head") pointing to the last examined entry. Eviction starts from the
-// current head. Each entry is given a second chance before eviction, if it
-// has been access since last examine. In contrast to LRU, no modification
-// to the internal data-structure (except for flipping the usage bit) needs
-// to be done upon lookup. This gives us oppertunity to implement a cache
-// with better concurrency.
-//
-// Each cache entry is represented by a cache handle, and all the handles
-// are arranged in a circular list, as describe above. Upon erase of an entry,
-// we never remove the handle. Instead, the handle is put into a recycle bin
-// to be re-use. This is to avoid memory dealocation, which is hard to deal
-// with in concurrent environment.
-//
-// The cache also maintains a concurrent hash map for lookup. Any concurrent
-// hash map implementation should do the work. We currently use
-// tbb::concurrent_hash_map because it supports concurrent erase.
-//
-// Each cache handle has the following flags and counters, which are squeeze
-// in an atomic interger, to make sure the handle always be in a consistent
-// state:
-//
-//   * In-cache bit: whether the entry is reference by the cache itself. If
-//     an entry is in cache, its key would also be available in the hash map.
-//   * Usage bit: whether the entry has been access by user since last
-//     examine for eviction. Can be reset by eviction.
-//   * Reference count: reference count by user.
-//
-// An entry can be reference only when it's in cache. An entry can be evicted
-// only when it is in cache, has no usage since last examine, and reference
-// count is zero.
-//
-// The follow figure shows a possible layout of the cache. Boxes represents
-// cache handles and numbers in each box being in-cache bit, usage bit and
-// reference count respectively.
-//
-//    hash map:
-//      +-------+--------+
-//      |  key  | handle |
-//      +-------+--------+
-//      | "foo" |    5   |-------------------------------------+
-//      +-------+--------+                                     |
-//      | "bar" |    2   |--+                                  |
-//      +-------+--------+  |                                  |
-//                          |                                  |
-//                     head |                                  |
-//                       |  |                                  |
-//    circular list:     |  |                                  |
-//         +-------+   +-------+   +-------+   +-------+   +-------+   +-------
-//         |(0,0,0)|---|(1,1,0)|---|(0,0,0)|---|(0,1,3)|---|(1,0,0)|---|  ...
-//         +-------+   +-------+   +-------+   +-------+   +-------+   +-------
-//             |                       |
-//             +-------+   +-----------+
-//                     |   |
-//                   +---+---+
-//    recycle bin:   | 1 | 3 |
-//                   +---+---+
-//
-// Suppose we try to insert "baz" into the cache at this point and the cache is
-// full. The cache will first look for entries to evict, starting from where
-// head points to (the second entry). It resets usage bit of the second entry,
-// skips the third and fourth entry since they are not in cache, and finally
-// evict the fifth entry ("foo"). It looks at recycle bin for available handle,
-// grabs handle 3, and insert the key into the handle. The following figure
-// shows the resulting layout.
-//
-//    hash map:
-//      +-------+--------+
-//      |  key  | handle |
-//      +-------+--------+
-//      | "baz" |    3   |-------------+
-//      +-------+--------+             |
-//      | "bar" |    2   |--+          |
-//      +-------+--------+  |          |
-//                          |          |
-//                          |          |                                 head
-//                          |          |                                   |
-//    circular list:        |          |                                   |
-//         +-------+   +-------+   +-------+   +-------+   +-------+   +-------
-//         |(0,0,0)|---|(1,0,0)|---|(1,0,0)|---|(0,1,3)|---|(0,0,0)|---|  ...
-//         +-------+   +-------+   +-------+   +-------+   +-------+   +-------
-//             |                                               |
-//             +-------+   +-----------------------------------+
-//                     |   |
-//                   +---+---+
-//    recycle bin:   | 1 | 5 |
-//                   +---+---+
-//
-// A global mutex guards the circular list, the head, and the recycle bin.
-// We additionally require that modifying the hash map needs to hold the mutex.
-// As such, Modifying the cache (such as Insert() and Erase()) require to
-// hold the mutex. Lookup() only access the hash map and the flags associated
-// with each handle, and don't require explicit locking. Release() has to
-// acquire the mutex only when it releases the last reference to the entry and
-// the entry has been erased from cache explicitly. A future improvement could
-// be to remove the mutex completely.
-//
-// Benchmark:
-// We run readrandom db_bench on a test DB of size 13GB, with size of each
-// level:
-//
-//    Level    Files   Size(MB)
-//    -------------------------
-//      L0        1       0.01
-//      L1       18      17.32
-//      L2      230     182.94
-//      L3     1186    1833.63
-//      L4     4602    8140.30
-//
-// We test with both 32 and 16 read threads, with 2GB cache size (the whole DB
-// doesn't fits in) and 64GB cache size (the whole DB can fit in cache), and
-// whether to put index and filter blocks in block cache. The benchmark runs
-// with
-// with RocksDB 4.10. We got the following result:
-//
-// Threads Cache     Cache               ClockCache               LRUCache
-//         Size  Index/Filter Throughput(MB/s)   Hit Throughput(MB/s)    Hit
-//     32   2GB       yes               466.7  85.9%           433.7   86.5%
-//     32   2GB       no                529.9  72.7%           532.7   73.9%
-//     32  64GB       yes               649.9  99.9%           507.9   99.9%
-//     32  64GB       no                740.4  99.9%           662.8   99.9%
-//     16   2GB       yes               278.4  85.9%           283.4   86.5%
-//     16   2GB       no                318.6  72.7%           335.8   73.9%
-//     16  64GB       yes               391.9  99.9%           353.3   99.9%
-//     16  64GB       no                433.8  99.8%           419.4   99.8%
-
-// Cache entry meta data.
-struct CacheHandle {
-  Slice key;
-  uint32_t hash;
-  void* value;
-  size_t charge;
-  void (*deleter)(const Slice&, void* value);
-
-  // Flags and counters associated with the cache handle:
-  //   lowest bit: n-cache bit
-  //   second lowest bit: usage bit
-  //   the rest bits: reference count
-  // The handle is unused when flags equals to 0. The thread decreases the count
-  // to 0 is responsible to put the handle back to recycle_ and cleanup memory.
-  std::atomic<uint32_t> flags;
-
-  CacheHandle() = default;
-
-  CacheHandle(const CacheHandle& a) { *this = a; }
-
-  CacheHandle(const Slice& k, void* v,
-              void (*del)(const Slice& key, void* value))
-      : key(k), value(v), deleter(del) {}
-
-  CacheHandle& operator=(const CacheHandle& a) {
-    // Only copy members needed for deletion.
-    key = a.key;
-    value = a.value;
-    deleter = a.deleter;
-    return *this;
-  }
-};
-
-// Key of hash map. We store hash value with the key for convenience.
-struct CacheKey {
-  Slice key;
-  uint32_t hash_value;
-
-  CacheKey() = default;
-
-  CacheKey(const Slice& k, uint32_t h) {
-    key = k;
-    hash_value = h;
-  }
-
-  static bool equal(const CacheKey& a, const CacheKey& b) {
-    return a.hash_value == b.hash_value && a.key == b.key;
-  }
-
-  static size_t hash(const CacheKey& a) {
-    return static_cast<size_t>(a.hash_value);
-  }
-};
-
-struct CleanupContext {
-  // List of values to be deleted, along with the key and deleter.
-  autovector<CacheHandle> to_delete_value;
-
-  // List of keys to be deleted.
-  autovector<const char*> to_delete_key;
-};
-
-// A cache shard which maintains its own CLOCK cache.
-class ClockCacheShard : public CacheShard {
- public:
-  // Hash map type.
-  typedef tbb::concurrent_hash_map<CacheKey, CacheHandle*, CacheKey> HashTable;
-
-  ClockCacheShard();
-  ~ClockCacheShard();
-
-  // Interfaces
-  virtual void SetCapacity(size_t capacity) override;
-  virtual void SetStrictCapacityLimit(bool strict_capacity_limit) override;
-  virtual Status Insert(const Slice& key, uint32_t hash, void* value,
-                        size_t charge,
-                        void (*deleter)(const Slice& key, void* value),
-                        Cache::Handle** handle,
-                        Cache::Priority priority) override;
-  virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash) override;
-  // If the entry in in cache, increase reference count and return true.
-  // Return false otherwise.
-  //
-  // Not necessary to hold mutex_ before being called.
-  virtual bool Ref(Cache::Handle* handle) override;
-  virtual void Release(Cache::Handle* handle) override;
-  virtual void Erase(const Slice& key, uint32_t hash) override;
-  virtual size_t GetUsage() const override;
-  virtual size_t GetPinnedUsage() const override;
-  virtual void EraseUnRefEntries() override;
-  virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
-                                      bool thread_safe) override;
-
- private:
-  static const uint32_t kInCacheBit = 1;
-  static const uint32_t kUsageBit = 2;
-  static const uint32_t kRefsOffset = 2;
-  static const uint32_t kOneRef = 1 << kRefsOffset;
-
-  // Helper functions to extract cache handle flags and counters.
-  static bool InCache(uint32_t flags) { return flags & kInCacheBit; }
-  static bool HasUsage(uint32_t flags) { return flags & kUsageBit; }
-  static uint32_t CountRefs(uint32_t flags) { return flags >> kRefsOffset; }
-
-  // Decrease reference count of the entry. If this decreases the count to 0,
-  // recycle the entry. If set_usage is true, also set the usage bit.
-  //
-  // Not necessary to hold mutex_ before being called.
-  void Unref(CacheHandle* handle, bool set_usage, CleanupContext* context);
-
-  // Unset in-cache bit of the entry. Recycle the handle if necessary.
-  //
-  // Has to hold mutex_ before being called.
-  void UnsetInCache(CacheHandle* handle, CleanupContext* context);
-
-  // Put the handle back to recycle_ list, and put the value associated with
-  // it into to-be-deleted list. It doesn't cleanup the key as it might be
-  // reused by another handle.
-  //
-  // Has to hold mutex_ before being called.
-  void RecycleHandle(CacheHandle* handle, CleanupContext* context);
-
-  // Delete keys and values in to-be-deleted list. Call the method without
-  // holding mutex, as destructors can be expensive.
-  void Cleanup(const CleanupContext& context);
-
-  // Examine the handle for eviction. If the handle is in cache, usage bit is
-  // not set, and referece count is 0, evict it from cache. Otherwise unset
-  // the usage bit.
-  //
-  // Has to hold mutex_ before being called.
-  bool TryEvict(CacheHandle* value, CleanupContext* context);
-
-  // Scan through the circular list, evict entries until we get enough capacity
-  // for new cache entry of specific size. Return true if success, false
-  // otherwise.
-  //
-  // Has to hold mutex_ before being called.
-  bool EvictFromCache(size_t charge, CleanupContext* context);
-
-  CacheHandle* Insert(const Slice& key, uint32_t hash, void* value,
-                      size_t change,
-                      void (*deleter)(const Slice& key, void* value),
-                      bool hold_reference, CleanupContext* context);
-
-  // Guards list_, head_, and recycle_. In addition, updating table_ also has
-  // to hold the mutex, to avoid the cache being in inconsistent state.
-  mutable port::Mutex mutex_;
-
-  // The circular list of cache handles. Initially the list is empty. Once a
-  // handle is needed by insertion, and no more handles are available in
-  // recycle bin, one more handle is appended to the end.
-  //
-  // We use std::deque for the circular list because we want to make sure
-  // pointers to handles are valid through out the life-cycle of the cache
-  // (in contrast to std::vector), and be able to grow the list (in contrast
-  // to statically allocated arrays).
-  std::deque<CacheHandle> list_;
-
-  // Pointer to the next handle in the circular list to be examine for
-  // eviction.
-  size_t head_;
-
-  // Recycle bin of cache handles.
-  autovector<CacheHandle*> recycle_;
-
-  // Maximum cache size.
-  std::atomic<size_t> capacity_;
-
-  // Current total size of the cache.
-  std::atomic<size_t> usage_;
-
-  // Total un-released cache size.
-  std::atomic<size_t> pinned_usage_;
-
-  // Whether allow insert into cache if cache is full.
-  std::atomic<bool> strict_capacity_limit_;
-
-  // Hash table (tbb::concurrent_hash_map) for lookup.
-  HashTable table_;
-};
-
-ClockCacheShard::ClockCacheShard()
-    : head_(0), usage_(0), pinned_usage_(0), strict_capacity_limit_(false) {}
-
-ClockCacheShard::~ClockCacheShard() {
-  for (auto& handle : list_) {
-    uint32_t flags = handle.flags.load(std::memory_order_relaxed);
-    if (InCache(flags) || CountRefs(flags) > 0) {
-      (*handle.deleter)(handle.key, handle.value);
-      delete[] handle.key.data();
-    }
-  }
-}
-
-size_t ClockCacheShard::GetUsage() const {
-  return usage_.load(std::memory_order_relaxed);
-}
-
-size_t ClockCacheShard::GetPinnedUsage() const {
-  return pinned_usage_.load(std::memory_order_relaxed);
-}
-
-void ClockCacheShard::ApplyToAllCacheEntries(void (*callback)(void*, size_t),
-                                             bool thread_safe) {
-  if (thread_safe) {
-    mutex_.Lock();
-  }
-  for (auto& handle : list_) {
-    // Use relaxed semantics instead of acquire semantics since we are either
-    // holding mutex, or don't have thread safe requirement.
-    uint32_t flags = handle.flags.load(std::memory_order_relaxed);
-    if (InCache(flags)) {
-      callback(handle.value, handle.charge);
-    }
-  }
-  if (thread_safe) {
-    mutex_.Unlock();
-  }
-}
-
-void ClockCacheShard::RecycleHandle(CacheHandle* handle,
-                                    CleanupContext* context) {
-  mutex_.AssertHeld();
-  assert(!InCache(handle->flags) && CountRefs(handle->flags) == 0);
-  context->to_delete_key.push_back(handle->key.data());
-  context->to_delete_value.emplace_back(*handle);
-  handle->key.clear();
-  handle->value = nullptr;
-  handle->deleter = nullptr;
-  recycle_.push_back(handle);
-  usage_.fetch_sub(handle->charge, std::memory_order_relaxed);
-}
-
-void ClockCacheShard::Cleanup(const CleanupContext& context) {
-  for (const CacheHandle& handle : context.to_delete_value) {
-    if (handle.deleter) {
-      (*handle.deleter)(handle.key, handle.value);
-    }
-  }
-  for (const char* key : context.to_delete_key) {
-    delete[] key;
-  }
-}
-
-bool ClockCacheShard::Ref(Cache::Handle* h) {
-  auto handle = reinterpret_cast<CacheHandle*>(h);
-  // CAS loop to increase reference count.
-  uint32_t flags = handle->flags.load(std::memory_order_relaxed);
-  while (InCache(flags)) {
-    // Use acquire semantics on success, as further operations on the cache
-    // entry has to be order after reference count is increased.
-    if (handle->flags.compare_exchange_weak(flags, flags + kOneRef,
-                                            std::memory_order_acquire,
-                                            std::memory_order_relaxed)) {
-      if (CountRefs(flags) == 0) {
-        // No reference count before the operation.
-        pinned_usage_.fetch_add(handle->charge, std::memory_order_relaxed);
-      }
-      return true;
-    }
-  }
-  return false;
-}
-
-void ClockCacheShard::Unref(CacheHandle* handle, bool set_usage,
-                            CleanupContext* context) {
-  if (set_usage) {
-    handle->flags.fetch_or(kUsageBit, std::memory_order_relaxed);
-  }
-  // Use acquire-release semantics as previous operations on the cache entry
-  // has to be order before reference count is decreased, and potential cleanup
-  // of the entry has to be order after.
-  uint32_t flags = handle->flags.fetch_sub(kOneRef, std::memory_order_acq_rel);
-  assert(CountRefs(flags) > 0);
-  if (CountRefs(flags) == 1) {
-    // this is the last reference.
-    pinned_usage_.fetch_sub(handle->charge, std::memory_order_relaxed);
-    // Cleanup if it is the last reference.
-    if (!InCache(flags)) {
-      MutexLock l(&mutex_);
-      RecycleHandle(handle, context);
-    }
-  }
-}
-
-void ClockCacheShard::UnsetInCache(CacheHandle* handle,
-                                   CleanupContext* context) {
-  mutex_.AssertHeld();
-  // Use acquire-release semantics as previous operations on the cache entry
-  // has to be order before reference count is decreased, and potential cleanup
-  // of the entry has to be order after.
-  uint32_t flags =
-      handle->flags.fetch_and(~kInCacheBit, std::memory_order_acq_rel);
-  // Cleanup if it is the last reference.
-  if (InCache(flags) && CountRefs(flags) == 0) {
-    RecycleHandle(handle, context);
-  }
-}
-
-bool ClockCacheShard::TryEvict(CacheHandle* handle, CleanupContext* context) {
-  mutex_.AssertHeld();
-  uint32_t flags = kInCacheBit;
-  if (handle->flags.compare_exchange_strong(flags, 0, std::memory_order_acquire,
-                                            std::memory_order_relaxed)) {
-    bool erased __attribute__((__unused__)) =
-        table_.erase(CacheKey(handle->key, handle->hash));
-    assert(erased);
-    RecycleHandle(handle, context);
-    return true;
-  }
-  handle->flags.fetch_and(~kUsageBit, std::memory_order_relaxed);
-  return false;
-}
-
-bool ClockCacheShard::EvictFromCache(size_t charge, CleanupContext* context) {
-  size_t usage = usage_.load(std::memory_order_relaxed);
-  size_t capacity = capacity_.load(std::memory_order_relaxed);
-  if (usage == 0) {
-    return charge <= capacity;
-  }
-  size_t new_head = head_;
-  bool second_iteration = false;
-  while (usage + charge > capacity) {
-    assert(new_head < list_.size());
-    if (TryEvict(&list_[new_head], context)) {
-      usage = usage_.load(std::memory_order_relaxed);
-    }
-    new_head = (new_head + 1 >= list_.size()) ? 0 : new_head + 1;
-    if (new_head == head_) {
-      if (second_iteration) {
-        return false;
-      } else {
-        second_iteration = true;
-      }
-    }
-  }
-  head_ = new_head;
-  return true;
-}
-
-void ClockCacheShard::SetCapacity(size_t capacity) {
-  CleanupContext context;
-  {
-    MutexLock l(&mutex_);
-    capacity_.store(capacity, std::memory_order_relaxed);
-    EvictFromCache(0, &context);
-  }
-  Cleanup(context);
-}
-
-void ClockCacheShard::SetStrictCapacityLimit(bool strict_capacity_limit) {
-  strict_capacity_limit_.store(strict_capacity_limit,
-                               std::memory_order_relaxed);
-}
-
-CacheHandle* ClockCacheShard::Insert(
-    const Slice& key, uint32_t hash, void* value, size_t charge,
-    void (*deleter)(const Slice& key, void* value), bool hold_reference,
-    CleanupContext* context) {
-  MutexLock l(&mutex_);
-  bool success = EvictFromCache(charge, context);
-  bool strict = strict_capacity_limit_.load(std::memory_order_relaxed);
-  if (!success && (strict || !hold_reference)) {
-    context->to_delete_key.push_back(key.data());
-    if (!hold_reference) {
-      context->to_delete_value.emplace_back(key, value, deleter);
-    }
-    return nullptr;
-  }
-  // Grab available handle from recycle bin. If recycle bin is empty, create
-  // and append new handle to end of circular list.
-  CacheHandle* handle = nullptr;
-  if (!recycle_.empty()) {
-    handle = recycle_.back();
-    recycle_.pop_back();
-  } else {
-    list_.emplace_back();
-    handle = &list_.back();
-  }
-  // Fill handle.
-  handle->key = key;
-  handle->hash = hash;
-  handle->value = value;
-  handle->charge = charge;
-  handle->deleter = deleter;
-  uint32_t flags = hold_reference ? kInCacheBit + kOneRef : kInCacheBit;
-  handle->flags.store(flags, std::memory_order_relaxed);
-  HashTable::accessor accessor;
-  if (table_.find(accessor, CacheKey(key, hash))) {
-    CacheHandle* existing_handle = accessor->second;
-    table_.erase(accessor);
-    UnsetInCache(existing_handle, context);
-  }
-  table_.insert(HashTable::value_type(CacheKey(key, hash), handle));
-  if (hold_reference) {
-    pinned_usage_.fetch_add(charge, std::memory_order_relaxed);
-  }
-  usage_.fetch_add(charge, std::memory_order_relaxed);
-  return handle;
-}
-
-Status ClockCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
-                               size_t charge,
-                               void (*deleter)(const Slice& key, void* value),
-                               Cache::Handle** out_handle,
-                               Cache::Priority priority) {
-  CleanupContext context;
-  HashTable::accessor accessor;
-  char* key_data = new char[key.size()];
-  memcpy(key_data, key.data(), key.size());
-  Slice key_copy(key_data, key.size());
-  CacheHandle* handle = Insert(key_copy, hash, value, charge, deleter,
-                               out_handle != nullptr, &context);
-  Status s;
-  if (out_handle != nullptr) {
-    if (handle == nullptr) {
-      s = Status::Incomplete("Insert failed due to LRU cache being full.");
-    } else {
-      *out_handle = reinterpret_cast<Cache::Handle*>(handle);
-    }
-  }
-  Cleanup(context);
-  return s;
-}
-
-Cache::Handle* ClockCacheShard::Lookup(const Slice& key, uint32_t hash) {
-  HashTable::const_accessor accessor;
-  if (!table_.find(accessor, CacheKey(key, hash))) {
-    return nullptr;
-  }
-  CacheHandle* handle = accessor->second;
-  accessor.release();
-  // Ref() could fail if another thread sneak in and evict/erase the cache
-  // entry before we are able to hold reference.
-  if (!Ref(reinterpret_cast<Cache::Handle*>(handle))) {
-    return nullptr;
-  }
-  // Double check the key since the handle may now representing another key
-  // if other threads sneak in, evict/erase the entry and re-used the handle
-  // for another cache entry.
-  if (hash != handle->hash || key != handle->key) {
-    CleanupContext context;
-    Unref(handle, false, &context);
-    // It is possible Unref() delete the entry, so we need to cleanup.
-    Cleanup(context);
-    return nullptr;
-  }
-  return reinterpret_cast<Cache::Handle*>(handle);
-}
-
-void ClockCacheShard::Release(Cache::Handle* h) {
-  CleanupContext context;
-  CacheHandle* handle = reinterpret_cast<CacheHandle*>(h);
-  Unref(handle, true, &context);
-  Cleanup(context);
-}
-
-void ClockCacheShard::Erase(const Slice& key, uint32_t hash) {
-  CleanupContext context;
-  {
-    MutexLock l(&mutex_);
-    HashTable::accessor accessor;
-    if (table_.find(accessor, CacheKey(key, hash))) {
-      CacheHandle* handle = accessor->second;
-      table_.erase(accessor);
-      UnsetInCache(handle, &context);
-    }
-  }
-  Cleanup(context);
-}
-
-void ClockCacheShard::EraseUnRefEntries() {
-  CleanupContext context;
-  {
-    MutexLock l(&mutex_);
-    table_.clear();
-    for (auto& handle : list_) {
-      UnsetInCache(&handle, &context);
-    }
-  }
-  Cleanup(context);
-}
-
-class ClockCache : public ShardedCache {
- public:
-  ClockCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit)
-      : ShardedCache(capacity, num_shard_bits, strict_capacity_limit) {
-    int num_shards = 1 << num_shard_bits;
-    shards_ = new ClockCacheShard[num_shards];
-    SetCapacity(capacity);
-    SetStrictCapacityLimit(strict_capacity_limit);
-  }
-
-  virtual ~ClockCache() { delete[] shards_; }
-
-  virtual const char* Name() const override { return "ClockCache"; }
-
-  virtual CacheShard* GetShard(int shard) override {
-    return reinterpret_cast<CacheShard*>(&shards_[shard]);
-  }
-
-  virtual const CacheShard* GetShard(int shard) const override {
-    return reinterpret_cast<CacheShard*>(&shards_[shard]);
-  }
-
-  virtual void* Value(Handle* handle) override {
-    return reinterpret_cast<const CacheHandle*>(handle)->value;
-  }
-
-  virtual size_t GetCharge(Handle* handle) const override {
-    return reinterpret_cast<const CacheHandle*>(handle)->charge;
-  }
-
-  virtual uint32_t GetHash(Handle* handle) const override {
-    return reinterpret_cast<const CacheHandle*>(handle)->hash;
-  }
-
-  virtual void DisownData() override { shards_ = nullptr; }
-
- private:
-  ClockCacheShard* shards_;
-};
-
-}  // end anonymous namespace
-
-std::shared_ptr<Cache> NewClockCache(size_t capacity, int num_shard_bits,
-                                     bool strict_capacity_limit) {
-  if (num_shard_bits < 0) {
-    num_shard_bits = GetDefaultCacheShardBits(capacity);
-  }
-  return std::make_shared<ClockCache>(capacity, num_shard_bits,
-                                      strict_capacity_limit);
-}
-
-}  // namespace rocksdb
-
-#endif  // SUPPORT_CLOCK_CACHE
@@ -1,16 +0,0 @@
-// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree. An additional grant
-// of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#pragma once
-
-#include "rocksdb/cache.h"
-
-#if defined(TBB) && !defined(ROCKSDB_LITE)
-#define SUPPORT_CLOCK_CACHE
-#endif
@@ -1,490 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef __STDC_FORMAT_MACROS
-#define __STDC_FORMAT_MACROS
-#endif
-
-#include "cache/lru_cache.h"
-
-#include <assert.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string>
-
-#include "util/mutexlock.h"
-
-namespace rocksdb {
-
-LRUHandleTable::LRUHandleTable() : length_(0), elems_(0), list_(nullptr) {
-  Resize();
-}
-
-LRUHandleTable::~LRUHandleTable() {
-  ApplyToAllCacheEntries([](LRUHandle* h) {
-    if (h->refs == 1) {
-      h->Free();
-    }
-  });
-  delete[] list_;
-}
-
-LRUHandle* LRUHandleTable::Lookup(const Slice& key, uint32_t hash) {
-  return *FindPointer(key, hash);
-}
-
-LRUHandle* LRUHandleTable::Insert(LRUHandle* h) {
-  LRUHandle** ptr = FindPointer(h->key(), h->hash);
-  LRUHandle* old = *ptr;
-  h->next_hash = (old == nullptr ? nullptr : old->next_hash);
-  *ptr = h;
-  if (old == nullptr) {
-    ++elems_;
-    if (elems_ > length_) {
-      // Since each cache entry is fairly large, we aim for a small
-      // average linked list length (<= 1).
-      Resize();
-    }
-  }
-  return old;
-}
-
-LRUHandle* LRUHandleTable::Remove(const Slice& key, uint32_t hash) {
-  LRUHandle** ptr = FindPointer(key, hash);
-  LRUHandle* result = *ptr;
-  if (result != nullptr) {
-    *ptr = result->next_hash;
-    --elems_;
-  }
-  return result;
-}
-
-LRUHandle** LRUHandleTable::FindPointer(const Slice& key, uint32_t hash) {
-  LRUHandle** ptr = &list_[hash & (length_ - 1)];
-  while (*ptr != nullptr && ((*ptr)->hash != hash || key != (*ptr)->key())) {
-    ptr = &(*ptr)->next_hash;
-  }
-  return ptr;
-}
-
-void LRUHandleTable::Resize() {
-  uint32_t new_length = 16;
-  while (new_length < elems_ * 1.5) {
-    new_length *= 2;
-  }
-  LRUHandle** new_list = new LRUHandle*[new_length];
-  memset(new_list, 0, sizeof(new_list[0]) * new_length);
-  uint32_t count = 0;
-  for (uint32_t i = 0; i < length_; i++) {
-    LRUHandle* h = list_[i];
-    while (h != nullptr) {
-      LRUHandle* next = h->next_hash;
-      uint32_t hash = h->hash;
-      LRUHandle** ptr = &new_list[hash & (new_length - 1)];
-      h->next_hash = *ptr;
-      *ptr = h;
-      h = next;
-      count++;
-    }
-  }
-  assert(elems_ == count);
-  delete[] list_;
-  list_ = new_list;
-  length_ = new_length;
-}
-
-LRUCacheShard::LRUCacheShard()
-    : usage_(0), lru_usage_(0), high_pri_pool_usage_(0) {
-  // Make empty circular linked list
-  lru_.next = &lru_;
-  lru_.prev = &lru_;
-  lru_low_pri_ = &lru_;
-}
-
-LRUCacheShard::~LRUCacheShard() {}
-
-bool LRUCacheShard::Unref(LRUHandle* e) {
-  assert(e->refs > 0);
-  e->refs--;
-  return e->refs == 0;
-}
-
-// Call deleter and free
-
-void LRUCacheShard::EraseUnRefEntries() {
-  autovector<LRUHandle*> last_reference_list;
-  {
-    MutexLock l(&mutex_);
-    while (lru_.next != &lru_) {
-      LRUHandle* old = lru_.next;
-      assert(old->InCache());
-      assert(old->refs ==
-             1);  // LRU list contains elements which may be evicted
-      LRU_Remove(old);
-      table_.Remove(old->key(), old->hash);
-      old->SetInCache(false);
-      Unref(old);
-      usage_ -= old->charge;
-      last_reference_list.push_back(old);
-    }
-  }
-
-  for (auto entry : last_reference_list) {
-    entry->Free();
-  }
-}
-
-void LRUCacheShard::ApplyToAllCacheEntries(void (*callback)(void*, size_t),
-                                           bool thread_safe) {
-  if (thread_safe) {
-    mutex_.Lock();
-  }
-  table_.ApplyToAllCacheEntries(
-      [callback](LRUHandle* h) { callback(h->value, h->charge); });
-  if (thread_safe) {
-    mutex_.Unlock();
-  }
-}
-
-void LRUCacheShard::TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri) {
-  *lru = &lru_;
-  *lru_low_pri = lru_low_pri_;
-}
-
-void LRUCacheShard::LRU_Remove(LRUHandle* e) {
-  assert(e->next != nullptr);
-  assert(e->prev != nullptr);
-  if (lru_low_pri_ == e) {
-    lru_low_pri_ = e->prev;
-  }
-  e->next->prev = e->prev;
-  e->prev->next = e->next;
-  e->prev = e->next = nullptr;
-  lru_usage_ -= e->charge;
-  if (e->InHighPriPool()) {
-    assert(high_pri_pool_usage_ >= e->charge);
-    high_pri_pool_usage_ -= e->charge;
-  }
-}
-
-void LRUCacheShard::LRU_Insert(LRUHandle* e) {
-  assert(e->next == nullptr);
-  assert(e->prev == nullptr);
-  if (high_pri_pool_ratio_ > 0 && e->IsHighPri()) {
-    // Inset "e" to head of LRU list.
-    e->next = &lru_;
-    e->prev = lru_.prev;
-    e->prev->next = e;
-    e->next->prev = e;
-    e->SetInHighPriPool(true);
-    high_pri_pool_usage_ += e->charge;
-    MaintainPoolSize();
-  } else {
-    // Insert "e" to the head of low-pri pool. Note that when
-    // high_pri_pool_ratio is 0, head of low-pri pool is also head of LRU list.
-    e->next = lru_low_pri_->next;
-    e->prev = lru_low_pri_;
-    e->prev->next = e;
-    e->next->prev = e;
-    e->SetInHighPriPool(false);
-    lru_low_pri_ = e;
-  }
-  lru_usage_ += e->charge;
-}
-
-void LRUCacheShard::MaintainPoolSize() {
-  while (high_pri_pool_usage_ > high_pri_pool_capacity_) {
-    // Overflow last entry in high-pri pool to low-pri pool.
-    lru_low_pri_ = lru_low_pri_->next;
-    assert(lru_low_pri_ != &lru_);
-    lru_low_pri_->SetInHighPriPool(false);
-    high_pri_pool_usage_ -= lru_low_pri_->charge;
-  }
-}
-
-void LRUCacheShard::EvictFromLRU(size_t charge,
-                                 autovector<LRUHandle*>* deleted) {
-  while (usage_ + charge > capacity_ && lru_.next != &lru_) {
-    LRUHandle* old = lru_.next;
-    assert(old->InCache());
-    assert(old->refs == 1);  // LRU list contains elements which may be evicted
-    LRU_Remove(old);
-    table_.Remove(old->key(), old->hash);
-    old->SetInCache(false);
-    Unref(old);
-    usage_ -= old->charge;
-    deleted->push_back(old);
-  }
-}
-
-void LRUCacheShard::SetCapacity(size_t capacity) {
-  autovector<LRUHandle*> last_reference_list;
-  {
-    MutexLock l(&mutex_);
-    capacity_ = capacity;
-    high_pri_pool_capacity_ = capacity_ * high_pri_pool_ratio_;
-    EvictFromLRU(0, &last_reference_list);
-  }
-  // we free the entries here outside of mutex for
-  // performance reasons
-  for (auto entry : last_reference_list) {
-    entry->Free();
-  }
-}
-
-void LRUCacheShard::SetStrictCapacityLimit(bool strict_capacity_limit) {
-  MutexLock l(&mutex_);
-  strict_capacity_limit_ = strict_capacity_limit;
-}
-
-Cache::Handle* LRUCacheShard::Lookup(const Slice& key, uint32_t hash) {
-  MutexLock l(&mutex_);
-  LRUHandle* e = table_.Lookup(key, hash);
-  if (e != nullptr) {
-    assert(e->InCache());
-    if (e->refs == 1) {
-      LRU_Remove(e);
-    }
-    e->refs++;
-  }
-  return reinterpret_cast<Cache::Handle*>(e);
-}
-
-bool LRUCacheShard::Ref(Cache::Handle* h) {
-  LRUHandle* handle = reinterpret_cast<LRUHandle*>(h);
-  MutexLock l(&mutex_);
-  if (handle->InCache() && handle->refs == 1) {
-    LRU_Remove(handle);
-  }
-  handle->refs++;
-  return true;
-}
-
-void LRUCacheShard::SetHighPriorityPoolRatio(double high_pri_pool_ratio) {
-  MutexLock l(&mutex_);
-  high_pri_pool_ratio_ = high_pri_pool_ratio;
-  high_pri_pool_capacity_ = capacity_ * high_pri_pool_ratio_;
-  MaintainPoolSize();
-}
-
-void LRUCacheShard::Release(Cache::Handle* handle) {
-  if (handle == nullptr) {
-    return;
-  }
-  LRUHandle* e = reinterpret_cast<LRUHandle*>(handle);
-  bool last_reference = false;
-  {
-    MutexLock l(&mutex_);
-    last_reference = Unref(e);
-    if (last_reference) {
-      usage_ -= e->charge;
-    }
-    if (e->refs == 1 && e->InCache()) {
-      // The item is still in cache, and nobody else holds a reference to it
-      if (usage_ > capacity_) {
-        // the cache is full
-        // The LRU list must be empty since the cache is full
-        assert(lru_.next == &lru_);
-        // take this opportunity and remove the item
-        table_.Remove(e->key(), e->hash);
-        e->SetInCache(false);
-        Unref(e);
-        usage_ -= e->charge;
-        last_reference = true;
-      } else {
-        // put the item on the list to be potentially freed
-        LRU_Insert(e);
-      }
-    }
-  }
-
-  // free outside of mutex
-  if (last_reference) {
-    e->Free();
-  }
-}
-
-Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
-                             size_t charge,
-                             void (*deleter)(const Slice& key, void* value),
-                             Cache::Handle** handle, Cache::Priority priority) {
-  // Allocate the memory here outside of the mutex
-  // If the cache is full, we'll have to release it
-  // It shouldn't happen very often though.
-  LRUHandle* e = reinterpret_cast<LRUHandle*>(
-      new char[sizeof(LRUHandle) - 1 + key.size()]);
-  Status s;
-  autovector<LRUHandle*> last_reference_list;
-
-  e->value = value;
-  e->deleter = deleter;
-  e->charge = charge;
-  e->key_length = key.size();
-  e->hash = hash;
-  e->refs = (handle == nullptr
-                 ? 1
-                 : 2);  // One from LRUCache, one for the returned handle
-  e->next = e->prev = nullptr;
-  e->SetInCache(true);
-  e->SetPriority(priority);
-  memcpy(e->key_data, key.data(), key.size());
-
-  {
-    MutexLock l(&mutex_);
-
-    // Free the space following strict LRU policy until enough space
-    // is freed or the lru list is empty
-    EvictFromLRU(charge, &last_reference_list);
-
-    if (usage_ - lru_usage_ + charge > capacity_ &&
-        (strict_capacity_limit_ || handle == nullptr)) {
-      if (handle == nullptr) {
-        // Don't insert the entry but still return ok, as if the entry inserted
-        // into cache and get evicted immediately.
-        last_reference_list.push_back(e);
-      } else {
-        delete[] reinterpret_cast<char*>(e);
-        *handle = nullptr;
-        s = Status::Incomplete("Insert failed due to LRU cache being full.");
-      }
-    } else {
-      // insert into the cache
-      // note that the cache might get larger than its capacity if not enough
-      // space was freed
-      LRUHandle* old = table_.Insert(e);
-      usage_ += e->charge;
-      if (old != nullptr) {
-        old->SetInCache(false);
-        if (Unref(old)) {
-          usage_ -= old->charge;
-          // old is on LRU because it's in cache and its reference count
-          // was just 1 (Unref returned 0)
-          LRU_Remove(old);
-          last_reference_list.push_back(old);
-        }
-      }
-      if (handle == nullptr) {
-        LRU_Insert(e);
-      } else {
-        *handle = reinterpret_cast<Cache::Handle*>(e);
-      }
-      s = Status::OK();
-    }
-  }
-
-  // we free the entries here outside of mutex for
-  // performance reasons
-  for (auto entry : last_reference_list) {
-    entry->Free();
-  }
-
-  return s;
-}
-
-void LRUCacheShard::Erase(const Slice& key, uint32_t hash) {
-  LRUHandle* e;
-  bool last_reference = false;
-  {
-    MutexLock l(&mutex_);
-    e = table_.Remove(key, hash);
-    if (e != nullptr) {
-      last_reference = Unref(e);
-      if (last_reference) {
-        usage_ -= e->charge;
-      }
-      if (last_reference && e->InCache()) {
-        LRU_Remove(e);
-      }
-      e->SetInCache(false);
-    }
-  }
-
-  // mutex not held here
-  // last_reference will only be true if e != nullptr
-  if (last_reference) {
-    e->Free();
-  }
-}
-
-size_t LRUCacheShard::GetUsage() const {
-  MutexLock l(&mutex_);
-  return usage_;
-}
-
-size_t LRUCacheShard::GetPinnedUsage() const {
-  MutexLock l(&mutex_);
-  assert(usage_ >= lru_usage_);
-  return usage_ - lru_usage_;
-}
-
-std::string LRUCacheShard::GetPrintableOptions() const {
-  const int kBufferSize = 200;
-  char buffer[kBufferSize];
-  {
-    MutexLock l(&mutex_);
-    snprintf(buffer, kBufferSize, "    high_pri_pool_ratio: %.3lf\n",
-             high_pri_pool_ratio_);
-  }
-  return std::string(buffer);
-}
-
-LRUCache::LRUCache(size_t capacity, int num_shard_bits,
-                   bool strict_capacity_limit, double high_pri_pool_ratio)
-    : ShardedCache(capacity, num_shard_bits, strict_capacity_limit) {
-  int num_shards = 1 << num_shard_bits;
-  shards_ = new LRUCacheShard[num_shards];
-  SetCapacity(capacity);
-  SetStrictCapacityLimit(strict_capacity_limit);
-  for (int i = 0; i < num_shards; i++) {
-    shards_[i].SetHighPriorityPoolRatio(high_pri_pool_ratio);
-  }
-}
-
-LRUCache::~LRUCache() { delete[] shards_; }
-
-CacheShard* LRUCache::GetShard(int shard) {
-  return reinterpret_cast<CacheShard*>(&shards_[shard]);
-}
-
-const CacheShard* LRUCache::GetShard(int shard) const {
-  return reinterpret_cast<CacheShard*>(&shards_[shard]);
-}
-
-void* LRUCache::Value(Handle* handle) {
-  return reinterpret_cast<const LRUHandle*>(handle)->value;
-}
-
-size_t LRUCache::GetCharge(Handle* handle) const {
-  return reinterpret_cast<const LRUHandle*>(handle)->charge;
-}
-
-uint32_t LRUCache::GetHash(Handle* handle) const {
-  return reinterpret_cast<const LRUHandle*>(handle)->hash;
-}
-
-void LRUCache::DisownData() { shards_ = nullptr; }
-
-std::shared_ptr<Cache> NewLRUCache(size_t capacity, int num_shard_bits,
-                                   bool strict_capacity_limit,
-                                   double high_pri_pool_ratio) {
-  if (num_shard_bits >= 20) {
-    return nullptr;  // the cache cannot be sharded into too many fine pieces
-  }
-  if (high_pri_pool_ratio < 0.0 || high_pri_pool_ratio > 1.0) {
-    // invalid high_pri_pool_ratio
-    return nullptr;
-  }
-  if (num_shard_bits < 0) {
-    num_shard_bits = GetDefaultCacheShardBits(capacity);
-  }
-  return std::make_shared<LRUCache>(capacity, num_shard_bits,
-                                    strict_capacity_limit, high_pri_pool_ratio);
-}
-
-}  // namespace rocksdb
@@ -1,273 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-#pragma once
-
-#include <string>
-
-#include "cache/sharded_cache.h"
-
-#include "port/port.h"
-#include "util/autovector.h"
-
-namespace rocksdb {
-
-// LRU cache implementation
-
-// An entry is a variable length heap-allocated structure.
-// Entries are referenced by cache and/or by any external entity.
-// The cache keeps all its entries in table. Some elements
-// are also stored on LRU list.
-//
-// LRUHandle can be in these states:
-// 1. Referenced externally AND in hash table.
-//  In that case the entry is *not* in the LRU. (refs > 1 && in_cache == true)
-// 2. Not referenced externally and in hash table. In that case the entry is
-// in the LRU and can be freed. (refs == 1 && in_cache == true)
-// 3. Referenced externally and not in hash table. In that case the entry is
-// in not on LRU and not in table. (refs >= 1 && in_cache == false)
-//
-// All newly created LRUHandles are in state 1. If you call
-// LRUCacheShard::Release
-// on entry in state 1, it will go into state 2. To move from state 1 to
-// state 3, either call LRUCacheShard::Erase or LRUCacheShard::Insert with the
-// same key.
-// To move from state 2 to state 1, use LRUCacheShard::Lookup.
-// Before destruction, make sure that no handles are in state 1. This means
-// that any successful LRUCacheShard::Lookup/LRUCacheShard::Insert have a
-// matching
-// RUCache::Release (to move into state 2) or LRUCacheShard::Erase (for state 3)
-
-struct LRUHandle {
-  void* value;
-  void (*deleter)(const Slice&, void* value);
-  LRUHandle* next_hash;
-  LRUHandle* next;
-  LRUHandle* prev;
-  size_t charge;  // TODO(opt): Only allow uint32_t?
-  size_t key_length;
-  uint32_t refs;     // a number of refs to this entry
-                     // cache itself is counted as 1
-
-  // Include the following flags:
-  //   in_cache:    whether this entry is referenced by the hash table.
-  //   is_high_pri: whether this entry is high priority entry.
-  //   in_high_pro_pool: whether this entry is in high-pri pool.
-  char flags;
-
-  uint32_t hash;     // Hash of key(); used for fast sharding and comparisons
-
-  char key_data[1];  // Beginning of key
-
-  Slice key() const {
-    // For cheaper lookups, we allow a temporary Handle object
-    // to store a pointer to a key in "value".
-    if (next == this) {
-      return *(reinterpret_cast<Slice*>(value));
-    } else {
-      return Slice(key_data, key_length);
-    }
-  }
-
-  bool InCache() { return flags & 1; }
-  bool IsHighPri() { return flags & 2; }
-  bool InHighPriPool() { return flags & 4; }
-
-  void SetInCache(bool in_cache) {
-    if (in_cache) {
-      flags |= 1;
-    } else {
-      flags &= ~1;
-    }
-  }
-
-  void SetPriority(Cache::Priority priority) {
-    if (priority == Cache::Priority::HIGH) {
-      flags |= 2;
-    } else {
-      flags &= ~2;
-    }
-  }
-
-  void SetInHighPriPool(bool in_high_pri_pool) {
-    if (in_high_pri_pool) {
-      flags |= 4;
-    } else {
-      flags &= ~4;
-    }
-  }
-
-  void Free() {
-    assert((refs == 1 && InCache()) || (refs == 0 && !InCache()));
-    if (deleter) {
-      (*deleter)(key(), value);
-    }
-    delete[] reinterpret_cast<char*>(this);
-  }
-};
-
-// We provide our own simple hash table since it removes a whole bunch
-// of porting hacks and is also faster than some of the built-in hash
-// table implementations in some of the compiler/runtime combinations
-// we have tested.  E.g., readrandom speeds up by ~5% over the g++
-// 4.4.3's builtin hashtable.
-class LRUHandleTable {
- public:
-  LRUHandleTable();
-  ~LRUHandleTable();
-
-  LRUHandle* Lookup(const Slice& key, uint32_t hash);
-  LRUHandle* Insert(LRUHandle* h);
-  LRUHandle* Remove(const Slice& key, uint32_t hash);
-
-  template <typename T>
-  void ApplyToAllCacheEntries(T func) {
-    for (uint32_t i = 0; i < length_; i++) {
-      LRUHandle* h = list_[i];
-      while (h != nullptr) {
-        auto n = h->next_hash;
-        assert(h->InCache());
-        func(h);
-        h = n;
-      }
-    }
-  }
-
- private:
-  // Return a pointer to slot that points to a cache entry that
-  // matches key/hash.  If there is no such cache entry, return a
-  // pointer to the trailing slot in the corresponding linked list.
-  LRUHandle** FindPointer(const Slice& key, uint32_t hash);
-
-  void Resize();
-
-  // The table consists of an array of buckets where each bucket is
-  // a linked list of cache entries that hash into the bucket.
-  uint32_t length_;
-  uint32_t elems_;
-  LRUHandle** list_;
-};
-
-// A single shard of sharded cache.
-class LRUCacheShard : public CacheShard {
- public:
-  LRUCacheShard();
-  virtual ~LRUCacheShard();
-
-  // Separate from constructor so caller can easily make an array of LRUCache
-  // if current usage is more than new capacity, the function will attempt to
-  // free the needed space
-  virtual void SetCapacity(size_t capacity) override;
-
-  // Set the flag to reject insertion if cache if full.
-  virtual void SetStrictCapacityLimit(bool strict_capacity_limit) override;
-
-  // Set percentage of capacity reserved for high-pri cache entries.
-  void SetHighPriorityPoolRatio(double high_pri_pool_ratio);
-
-  // Like Cache methods, but with an extra "hash" parameter.
-  virtual Status Insert(const Slice& key, uint32_t hash, void* value,
-                        size_t charge,
-                        void (*deleter)(const Slice& key, void* value),
-                        Cache::Handle** handle,
-                        Cache::Priority priority) override;
-  virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash) override;
-  virtual bool Ref(Cache::Handle* handle) override;
-  virtual void Release(Cache::Handle* handle) override;
-  virtual void Erase(const Slice& key, uint32_t hash) override;
-
-  // Although in some platforms the update of size_t is atomic, to make sure
-  // GetUsage() and GetPinnedUsage() work correctly under any platform, we'll
-  // protect them with mutex_.
-
-  virtual size_t GetUsage() const override;
-  virtual size_t GetPinnedUsage() const override;
-
-  virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
-                                      bool thread_safe) override;
-
-  virtual void EraseUnRefEntries() override;
-
-  virtual std::string GetPrintableOptions() const override;
-
-  void TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri);
-
- private:
-  void LRU_Remove(LRUHandle* e);
-  void LRU_Insert(LRUHandle* e);
-
-  // Overflow the last entry in high-pri pool to low-pri pool until size of
-  // high-pri pool is no larger than the size specify by high_pri_pool_pct.
-  void MaintainPoolSize();
-
-  // Just reduce the reference count by 1.
-  // Return true if last reference
-  bool Unref(LRUHandle* e);
-
-  // Free some space following strict LRU policy until enough space
-  // to hold (usage_ + charge) is freed or the lru list is empty
-  // This function is not thread safe - it needs to be executed while
-  // holding the mutex_
-  void EvictFromLRU(size_t charge, autovector<LRUHandle*>* deleted);
-
-  // Initialized before use.
-  size_t capacity_;
-
-  // Memory size for entries residing in the cache
-  size_t usage_;
-
-  // Memory size for entries residing only in the LRU list
-  size_t lru_usage_;
-
-  // Memory size for entries in high-pri pool.
-  size_t high_pri_pool_usage_;
-
-  // Whether to reject insertion if cache reaches its full capacity.
-  bool strict_capacity_limit_;
-
-  // Ratio of capacity reserved for high priority cache entries.
-  double high_pri_pool_ratio_;
-
-  // High-pri pool size, equals to capacity * high_pri_pool_ratio.
-  // Remember the value to avoid recomputing each time.
-  double high_pri_pool_capacity_;
-
-  // mutex_ protects the following state.
-  // We don't count mutex_ as the cache's internal state so semantically we
-  // don't mind mutex_ invoking the non-const actions.
-  mutable port::Mutex mutex_;
-
-  // Dummy head of LRU list.
-  // lru.prev is newest entry, lru.next is oldest entry.
-  // LRU contains items which can be evicted, ie reference only by cache
-  LRUHandle lru_;
-
-  // Pointer to head of low-pri pool in LRU list.
-  LRUHandle* lru_low_pri_;
-
-  LRUHandleTable table_;
-};
-
-class LRUCache : public ShardedCache {
- public:
-  LRUCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit,
-           double high_pri_pool_ratio);
-  virtual ~LRUCache();
-  virtual const char* Name() const override { return "LRUCache"; }
-  virtual CacheShard* GetShard(int shard) override;
-  virtual const CacheShard* GetShard(int shard) const override;
-  virtual void* Value(Handle* handle) override;
-  virtual size_t GetCharge(Handle* handle) const override;
-  virtual uint32_t GetHash(Handle* handle) const override;
-  virtual void DisownData() override;
-
- private:
-  LRUCacheShard* shards_;
-};
-
-}  // namespace rocksdb
@@ -1,163 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-
-#include "cache/lru_cache.h"
-
-#include <string>
-#include <vector>
-#include "util/testharness.h"
-
-namespace rocksdb {
-
-class LRUCacheTest : public testing::Test {
- public:
-  LRUCacheTest() {}
-  ~LRUCacheTest() {}
-
-  void NewCache(size_t capacity, double high_pri_pool_ratio = 0.0) {
-    cache_.reset(new LRUCacheShard());
-    cache_->SetCapacity(capacity);
-    cache_->SetStrictCapacityLimit(false);
-    cache_->SetHighPriorityPoolRatio(high_pri_pool_ratio);
-  }
-
-  void Insert(const std::string& key,
-              Cache::Priority priority = Cache::Priority::LOW) {
-    cache_->Insert(key, 0 /*hash*/, nullptr /*value*/, 1 /*charge*/,
-                   nullptr /*deleter*/, nullptr /*handle*/, priority);
-  }
-
-  void Insert(char key, Cache::Priority priority = Cache::Priority::LOW) {
-    Insert(std::string(1, key), priority);
-  }
-
-  bool Lookup(const std::string& key) {
-    auto handle = cache_->Lookup(key, 0 /*hash*/);
-    if (handle) {
-      cache_->Release(handle);
-      return true;
-    }
-    return false;
-  }
-
-  bool Lookup(char key) { return Lookup(std::string(1, key)); }
-
-  void Erase(const std::string& key) { cache_->Erase(key, 0 /*hash*/); }
-
-  void ValidateLRUList(std::vector<std::string> keys,
-                       size_t num_high_pri_pool_keys = 0) {
-    LRUHandle* lru;
-    LRUHandle* lru_low_pri;
-    cache_->TEST_GetLRUList(&lru, &lru_low_pri);
-    LRUHandle* iter = lru;
-    bool in_high_pri_pool = false;
-    size_t high_pri_pool_keys = 0;
-    if (iter == lru_low_pri) {
-      in_high_pri_pool = true;
-    }
-    for (const auto& key : keys) {
-      iter = iter->next;
-      ASSERT_NE(lru, iter);
-      ASSERT_EQ(key, iter->key().ToString());
-      ASSERT_EQ(in_high_pri_pool, iter->InHighPriPool());
-      if (in_high_pri_pool) {
-        high_pri_pool_keys++;
-      }
-      if (iter == lru_low_pri) {
-        ASSERT_FALSE(in_high_pri_pool);
-        in_high_pri_pool = true;
-      }
-    }
-    ASSERT_EQ(lru, iter->next);
-    ASSERT_TRUE(in_high_pri_pool);
-    ASSERT_EQ(num_high_pri_pool_keys, high_pri_pool_keys);
-  }
-
- private:
-  std::unique_ptr<LRUCacheShard> cache_;
-};
-
-TEST_F(LRUCacheTest, BasicLRU) {
-  NewCache(5);
-  for (char ch = 'a'; ch <= 'e'; ch++) {
-    Insert(ch);
-  }
-  ValidateLRUList({"a", "b", "c", "d", "e"});
-  for (char ch = 'x'; ch <= 'z'; ch++) {
-    Insert(ch);
-  }
-  ValidateLRUList({"d", "e", "x", "y", "z"});
-  ASSERT_FALSE(Lookup("b"));
-  ValidateLRUList({"d", "e", "x", "y", "z"});
-  ASSERT_TRUE(Lookup("e"));
-  ValidateLRUList({"d", "x", "y", "z", "e"});
-  ASSERT_TRUE(Lookup("z"));
-  ValidateLRUList({"d", "x", "y", "e", "z"});
-  Erase("x");
-  ValidateLRUList({"d", "y", "e", "z"});
-  ASSERT_TRUE(Lookup("d"));
-  ValidateLRUList({"y", "e", "z", "d"});
-  Insert("u");
-  ValidateLRUList({"y", "e", "z", "d", "u"});
-  Insert("v");
-  ValidateLRUList({"e", "z", "d", "u", "v"});
-}
-
-TEST_F(LRUCacheTest, MidPointInsertion) {
-  // Allocate 2 cache entries to high-pri pool.
-  NewCache(5, 0.45);
-
-  Insert("a", Cache::Priority::LOW);
-  Insert("b", Cache::Priority::LOW);
-  Insert("c", Cache::Priority::LOW);
-  ValidateLRUList({"a", "b", "c"}, 0);
-
-  // Low-pri entries can take high-pri pool capacity if available
-  Insert("u", Cache::Priority::LOW);
-  Insert("v", Cache::Priority::LOW);
-  ValidateLRUList({"a", "b", "c", "u", "v"}, 0);
-
-  Insert("X", Cache::Priority::HIGH);
-  Insert("Y", Cache::Priority::HIGH);
-  ValidateLRUList({"c", "u", "v", "X", "Y"}, 2);
-
-  // High-pri entries can overflow to low-pri pool.
-  Insert("Z", Cache::Priority::HIGH);
-  ValidateLRUList({"u", "v", "X", "Y", "Z"}, 2);
-
-  // Low-pri entries will be inserted to head of low-pri pool.
-  Insert("a", Cache::Priority::LOW);
-  ValidateLRUList({"v", "X", "a", "Y", "Z"}, 2);
-
-  // Low-pri entries will be inserted to head of low-pri pool after lookup.
-  ASSERT_TRUE(Lookup("v"));
-  ValidateLRUList({"X", "a", "v", "Y", "Z"}, 2);
-
-  // High-pri entries will be inserted to the head of the list after lookup.
-  ASSERT_TRUE(Lookup("X"));
-  ValidateLRUList({"a", "v", "Y", "Z", "X"}, 2);
-  ASSERT_TRUE(Lookup("Z"));
-  ValidateLRUList({"a", "v", "Y", "X", "Z"}, 2);
-
-  Erase("Y");
-  ValidateLRUList({"a", "v", "X", "Z"}, 2);
-  Erase("X");
-  ValidateLRUList({"a", "v", "Z"}, 1);
-  Insert("d", Cache::Priority::LOW);
-  Insert("e", Cache::Priority::LOW);
-  ValidateLRUList({"a", "v", "d", "e", "Z"}, 1);
-  Insert("f", Cache::Priority::LOW);
-  Insert("g", Cache::Priority::LOW);
-  ValidateLRUList({"d", "e", "f", "g", "Z"}, 1);
-  ASSERT_TRUE(Lookup("d"));
-  ValidateLRUList({"e", "f", "g", "d", "Z"}, 1);
-}
-
-}  // namespace rocksdb
-
-int main(int argc, char** argv) {
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}
@@ -1,161 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef __STDC_FORMAT_MACROS
-#define __STDC_FORMAT_MACROS
-#endif
-
-#include "cache/sharded_cache.h"
-
-#include <string>
-
-#include "util/mutexlock.h"
-
-namespace rocksdb {
-
-ShardedCache::ShardedCache(size_t capacity, int num_shard_bits,
-                           bool strict_capacity_limit)
-    : num_shard_bits_(num_shard_bits),
-      capacity_(capacity),
-      strict_capacity_limit_(strict_capacity_limit),
-      last_id_(1) {}
-
-void ShardedCache::SetCapacity(size_t capacity) {
-  int num_shards = 1 << num_shard_bits_;
-  const size_t per_shard = (capacity + (num_shards - 1)) / num_shards;
-  MutexLock l(&capacity_mutex_);
-  for (int s = 0; s < num_shards; s++) {
-    GetShard(s)->SetCapacity(per_shard);
-  }
-  capacity_ = capacity;
-}
-
-void ShardedCache::SetStrictCapacityLimit(bool strict_capacity_limit) {
-  int num_shards = 1 << num_shard_bits_;
-  MutexLock l(&capacity_mutex_);
-  for (int s = 0; s < num_shards; s++) {
-    GetShard(s)->SetStrictCapacityLimit(strict_capacity_limit);
-  }
-  strict_capacity_limit_ = strict_capacity_limit;
-}
-
-Status ShardedCache::Insert(const Slice& key, void* value, size_t charge,
-                            void (*deleter)(const Slice& key, void* value),
-                            Handle** handle, Priority priority) {
-  uint32_t hash = HashSlice(key);
-  return GetShard(Shard(hash))
-      ->Insert(key, hash, value, charge, deleter, handle, priority);
-}
-
-Cache::Handle* ShardedCache::Lookup(const Slice& key, Statistics* stats) {
-  uint32_t hash = HashSlice(key);
-  return GetShard(Shard(hash))->Lookup(key, hash);
-}
-
-bool ShardedCache::Ref(Handle* handle) {
-  uint32_t hash = GetHash(handle);
-  return GetShard(Shard(hash))->Ref(handle);
-}
-
-void ShardedCache::Release(Handle* handle) {
-  uint32_t hash = GetHash(handle);
-  GetShard(Shard(hash))->Release(handle);
-}
-
-void ShardedCache::Erase(const Slice& key) {
-  uint32_t hash = HashSlice(key);
-  GetShard(Shard(hash))->Erase(key, hash);
-}
-
-uint64_t ShardedCache::NewId() {
-  return last_id_.fetch_add(1, std::memory_order_relaxed);
-}
-
-size_t ShardedCache::GetCapacity() const {
-  MutexLock l(&capacity_mutex_);
-  return capacity_;
-}
-
-bool ShardedCache::HasStrictCapacityLimit() const {
-  MutexLock l(&capacity_mutex_);
-  return strict_capacity_limit_;
-}
-
-size_t ShardedCache::GetUsage() const {
-  // We will not lock the cache when getting the usage from shards.
-  int num_shards = 1 << num_shard_bits_;
-  size_t usage = 0;
-  for (int s = 0; s < num_shards; s++) {
-    usage += GetShard(s)->GetUsage();
-  }
-  return usage;
-}
-
-size_t ShardedCache::GetUsage(Handle* handle) const {
-  return GetCharge(handle);
-}
-
-size_t ShardedCache::GetPinnedUsage() const {
-  // We will not lock the cache when getting the usage from shards.
-  int num_shards = 1 << num_shard_bits_;
-  size_t usage = 0;
-  for (int s = 0; s < num_shards; s++) {
-    usage += GetShard(s)->GetPinnedUsage();
-  }
-  return usage;
-}
-
-void ShardedCache::ApplyToAllCacheEntries(void (*callback)(void*, size_t),
-                                          bool thread_safe) {
-  int num_shards = 1 << num_shard_bits_;
-  for (int s = 0; s < num_shards; s++) {
-    GetShard(s)->ApplyToAllCacheEntries(callback, thread_safe);
-  }
-}
-
-void ShardedCache::EraseUnRefEntries() {
-  int num_shards = 1 << num_shard_bits_;
-  for (int s = 0; s < num_shards; s++) {
-    GetShard(s)->EraseUnRefEntries();
-  }
-}
-
-std::string ShardedCache::GetPrintableOptions() const {
-  std::string ret;
-  ret.reserve(20000);
-  const int kBufferSize = 200;
-  char buffer[kBufferSize];
-  {
-    MutexLock l(&capacity_mutex_);
-    snprintf(buffer, kBufferSize, "    capacity : %" ROCKSDB_PRIszt "\n",
-             capacity_);
-    ret.append(buffer);
-    snprintf(buffer, kBufferSize, "    num_shard_bits : %d\n", num_shard_bits_);
-    ret.append(buffer);
-    snprintf(buffer, kBufferSize, "    strict_capacity_limit : %d\n",
-             strict_capacity_limit_);
-    ret.append(buffer);
-  }
-  ret.append(GetShard(0)->GetPrintableOptions());
-  return ret;
-}
-int GetDefaultCacheShardBits(size_t capacity) {
-  int num_shard_bits = 0;
-  size_t min_shard_size = 512L * 1024L;  // Every shard is at least 512KB.
-  size_t num_shards = capacity / min_shard_size;
-  while (num_shards >>= 1) {
-    if (++num_shard_bits >= 6) {
-      // No more than 6.
-      return num_shard_bits;
-    }
-  }
-  return num_shard_bits;
-}
-
-}  // namespace rocksdb
@@ -1,102 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#pragma once
-
-#include <atomic>
-#include <string>
-
-#include "port/port.h"
-#include "rocksdb/cache.h"
-#include "util/hash.h"
-
-namespace rocksdb {
-
-// Single cache shard interface.
-class CacheShard {
- public:
-  CacheShard() = default;
-  virtual ~CacheShard() = default;
-
-  virtual Status Insert(const Slice& key, uint32_t hash, void* value,
-                        size_t charge,
-                        void (*deleter)(const Slice& key, void* value),
-                        Cache::Handle** handle, Cache::Priority priority) = 0;
-  virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash) = 0;
-  virtual bool Ref(Cache::Handle* handle) = 0;
-  virtual void Release(Cache::Handle* handle) = 0;
-  virtual void Erase(const Slice& key, uint32_t hash) = 0;
-  virtual void SetCapacity(size_t capacity) = 0;
-  virtual void SetStrictCapacityLimit(bool strict_capacity_limit) = 0;
-  virtual size_t GetUsage() const = 0;
-  virtual size_t GetPinnedUsage() const = 0;
-  virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
-                                      bool thread_safe) = 0;
-  virtual void EraseUnRefEntries() = 0;
-  virtual std::string GetPrintableOptions() const { return ""; }
-};
-
-// Generic cache interface which shards cache by hash of keys. 2^num_shard_bits
-// shards will be created, with capacity split evenly to each of the shards.
-// Keys are sharded by the highest num_shard_bits bits of hash value.
-class ShardedCache : public Cache {
- public:
-  ShardedCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit);
-  virtual ~ShardedCache() = default;
-  virtual const char* Name() const override = 0;
-  virtual CacheShard* GetShard(int shard) = 0;
-  virtual const CacheShard* GetShard(int shard) const = 0;
-  virtual void* Value(Handle* handle) override = 0;
-  virtual size_t GetCharge(Handle* handle) const = 0;
-  virtual uint32_t GetHash(Handle* handle) const = 0;
-  virtual void DisownData() override = 0;
-
-  virtual void SetCapacity(size_t capacity) override;
-  virtual void SetStrictCapacityLimit(bool strict_capacity_limit) override;
-
-  virtual Status Insert(const Slice& key, void* value, size_t charge,
-                        void (*deleter)(const Slice& key, void* value),
-                        Handle** handle, Priority priority) override;
-  virtual Handle* Lookup(const Slice& key, Statistics* stats) override;
-  virtual bool Ref(Handle* handle) override;
-  virtual void Release(Handle* handle) override;
-  virtual void Erase(const Slice& key) override;
-  virtual uint64_t NewId() override;
-  virtual size_t GetCapacity() const override;
-  virtual bool HasStrictCapacityLimit() const override;
-  virtual size_t GetUsage() const override;
-  virtual size_t GetUsage(Handle* handle) const override;
-  virtual size_t GetPinnedUsage() const override;
-  virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
-                                      bool thread_safe) override;
-  virtual void EraseUnRefEntries() override;
-  virtual std::string GetPrintableOptions() const override;
-
-  int GetNumShardBits() const { return num_shard_bits_; }
-
- private:
-  static inline uint32_t HashSlice(const Slice& s) {
-    return Hash(s.data(), s.size(), 0);
-  }
-
-  uint32_t Shard(uint32_t hash) {
-    // Note, hash >> 32 yields hash in gcc, not the zero we expect!
-    return (num_shard_bits_ > 0) ? (hash >> (32 - num_shard_bits_)) : 0;
-  }
-
-  int num_shard_bits_;
-  mutable port::Mutex capacity_mutex_;
-  size_t capacity_;
-  bool strict_capacity_limit_;
-  std::atomic<uint64_t> last_id_;
-};
-
-extern int GetDefaultCacheShardBits(size_t capacity);
-
-}  // namespace rocksdb
@@ -1,21 +0,0 @@
-# - Find JeMalloc library
-# Find the native JeMalloc includes and library
-#
-# JEMALLOC_INCLUDE_DIR - where to find jemalloc.h, etc.
-# JEMALLOC_LIBRARIES - List of libraries when using jemalloc.
-# JEMALLOC_FOUND - True if jemalloc found.
-
-find_path(JEMALLOC_INCLUDE_DIR
-  NAMES jemalloc/jemalloc.h
-  HINTS ${JEMALLOC_ROOT_DIR}/include)
-
-find_library(JEMALLOC_LIBRARIES
-  NAMES jemalloc
-  HINTS ${JEMALLOC_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(jemalloc DEFAULT_MSG JEMALLOC_LIBRARIES JEMALLOC_INCLUDE_DIR)
-
-mark_as_advanced(
-  JEMALLOC_LIBRARIES
-  JEMALLOC_INCLUDE_DIR)
@@ -1,21 +0,0 @@
-# - Find Snappy
-# Find the snappy compression library and includes
-#
-# SNAPPY_INCLUDE_DIR - where to find snappy.h, etc.
-# SNAPPY_LIBRARIES - List of libraries when using snappy.
-# SNAPPY_FOUND - True if snappy found.
-
-find_path(SNAPPY_INCLUDE_DIR
-  NAMES snappy.h
-  HINTS ${SNAPPY_ROOT_DIR}/include)
-
-find_library(SNAPPY_LIBRARIES
-  NAMES snappy
-  HINTS ${SNAPPY_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(snappy DEFAULT_MSG SNAPPY_LIBRARIES SNAPPY_INCLUDE_DIR)
-
-mark_as_advanced(
-  SNAPPY_LIBRARIES
-  SNAPPY_INCLUDE_DIR)
@@ -3,12 +3,13 @@
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
 //
-#include "util/auto_roll_logger.h"
+#include "db/auto_roll_logger.h"
 #include "util/mutexlock.h"

+using namespace std;
+
 namespace rocksdb {

-#ifndef ROCKSDB_LITE
 // -- AutoRollLogger
 Status AutoRollLogger::ResetLogger() {
  TEST_SYNC_POINT("AutoRollLogger::ResetLogger:BeforeNewLogger");
@@ -46,8 +47,7 @@ void AutoRollLogger::RollLogFile() {
  env_->RenameFile(log_fname_, old_fname);
 }

-std::string AutoRollLogger::ValistToString(const char* format,
-                                           va_list args) const {
+string AutoRollLogger::ValistToString(const char* format, va_list args) const {
  // Any log messages longer than 1024 will get truncated.
  // The user is responsible for chopping longer messages into multi line log
  static const int MAXBUFFERSIZE = 1024;
@@ -112,7 +112,7 @@ void AutoRollLogger::LogHeader(const char* format, va_list args) {
  // strings
  va_list tmp;
  va_copy(tmp, args);
-  std::string data = ValistToString(format, tmp);
+  string data = ValistToString(format, tmp);
  va_end(tmp);

  MutexLock l(&mutex_);
@@ -131,7 +131,6 @@ bool AutoRollLogger::LogExpired() {
  ++cached_now_access_count;
  return cached_now >= ctime_ + kLogFileTimeToRoll;
 }
-#endif  // !ROCKSDB_LITE

 Status CreateLoggerFromOptions(const std::string& dbname,
                               const DBOptions& options,
@@ -149,7 +148,6 @@ Status CreateLoggerFromOptions(const std::string& dbname,

  env->CreateDirIfMissing(dbname);  // In case it does not exist
  // Currently we only support roll by time-to-roll and log size
-#ifndef ROCKSDB_LITE
  if (options.log_file_time_to_roll > 0 || options.max_log_file_size > 0) {
    AutoRollLogger* result = new AutoRollLogger(
        env, dbname, options.db_log_dir, options.max_log_file_size,
@@ -161,17 +159,17 @@ Status CreateLoggerFromOptions(const std::string& dbname,
      logger->reset(result);
    }
    return s;
+  } else {
+    // Open a log file in the same directory as the db
+    env->RenameFile(
+        fname, OldInfoLogFileName(dbname, env->NowMicros(), db_absolute_path,
+                                  options.db_log_dir));
+    auto s = env->NewLogger(fname, logger);
+    if (logger->get() != nullptr) {
+      (*logger)->SetInfoLogLevel(options.info_log_level);
+    }
+    return s;
  }
-#endif  // !ROCKSDB_LITE
-  // Open a log file in the same directory as the db
-  env->RenameFile(fname,
-                  OldInfoLogFileName(dbname, env->NowMicros(), db_absolute_path,
-                                     options.db_log_dir));
-  auto s = env->NewLogger(fname, logger);
-  if (logger->get() != nullptr) {
-    (*logger)->SetInfoLogLevel(options.info_log_level);
-  }
-  return s;
 }

 }  // namespace rocksdb
@@ -10,15 +10,14 @@
 #include <list>
 #include <string>

+#include "db/filename.h"
 #include "port/port.h"
 #include "port/util_logger.h"
-#include "util/filename.h"
-#include "util/mutexlock.h"
 #include "util/sync_point.h"
+#include "util/mutexlock.h"

 namespace rocksdb {

-#ifndef ROCKSDB_LITE
 // Rolls the log file by size and/or time
 class AutoRollLogger : public Logger {
 public:
@@ -91,8 +90,6 @@ class AutoRollLogger : public Logger {
    return log_fname_;
  }

-  uint64_t TEST_ctime() const { return ctime_; }
-
 private:
  bool LogExpired();
  Status ResetLogger();
@@ -123,7 +120,6 @@ class AutoRollLogger : public Logger {
  uint64_t call_NowMicros_every_N_records_;
  mutable port::Mutex mutex_;
 };
-#endif  // !ROCKSDB_LITE

 // Facade to craete logger automatically
 Status CreateLoggerFromOptions(const std::string& dbname,
@@ -3,43 +3,25 @@
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
 //
-
-#ifndef ROCKSDB_LITE
-
-#include "util/auto_roll_logger.h"
-#include <errno.h>
-#include <sys/stat.h>
-#include <algorithm>
-#include <cmath>
-#include <fstream>
-#include <iostream>
-#include <iterator>
 #include <string>
 #include <thread>
 #include <vector>
+#include <cmath>
+#include <iostream>
+#include <fstream>
+#include <iterator>
+#include <algorithm>
+#include "db/auto_roll_logger.h"
 #include "port/port.h"
-#include "rocksdb/db.h"
-#include "util/logging.h"
 #include "util/sync_point.h"
 #include "util/testharness.h"
+#include "rocksdb/db.h"
+#include <sys/stat.h>
+#include <errno.h>
+
+using namespace std;

 namespace rocksdb {
-namespace {
-class NoSleepEnv : public EnvWrapper {
- public:
-  NoSleepEnv(Env* base) : EnvWrapper(base) {}
-  virtual void SleepForMicroseconds(int micros) override {
-    fake_time_ += static_cast<uint64_t>(micros);
-  }
-
-  virtual uint64_t NowNanos() override { return fake_time_ * 1000; }
-
-  virtual uint64_t NowMicros() override { return fake_time_; }
-
- private:
-  uint64_t fake_time_ = 6666666666;
-};
-}  // namespace

 class AutoRollLoggerTest : public testing::Test {
 public:
@@ -58,31 +40,32 @@ class AutoRollLoggerTest : public testing::Test {
    Env::Default()->CreateDir(kTestDir);
  }

-  void RollLogFileBySizeTest(AutoRollLogger* logger, size_t log_max_size,
-                             const std::string& log_message);
-  void RollLogFileByTimeTest(Env*, AutoRollLogger* logger, size_t time,
-                             const std::string& log_message);
+  void RollLogFileBySizeTest(AutoRollLogger* logger,
+                             size_t log_max_size,
+                             const string& log_message);
+  uint64_t RollLogFileByTimeTest(AutoRollLogger* logger,
+                                 size_t time,
+                                 const string& log_message);

-  static const std::string kSampleMessage;
-  static const std::string kTestDir;
-  static const std::string kLogFile;
-  static Env* default_env;
+  static const string kSampleMessage;
+  static const string kTestDir;
+  static const string kLogFile;
+  static Env* env;
 };

-const std::string AutoRollLoggerTest::kSampleMessage(
+const string AutoRollLoggerTest::kSampleMessage(
    "this is the message to be written to the log file!!");
-const std::string AutoRollLoggerTest::kTestDir(test::TmpDir() + "/db_log_test");
-const std::string AutoRollLoggerTest::kLogFile(test::TmpDir() +
-                                               "/db_log_test/LOG");
-Env* AutoRollLoggerTest::default_env = Env::Default();
+const string AutoRollLoggerTest::kTestDir(test::TmpDir() + "/db_log_test");
+const string AutoRollLoggerTest::kLogFile(test::TmpDir() + "/db_log_test/LOG");
+Env* AutoRollLoggerTest::env = Env::Default();

 // In this test we only want to Log some simple log message with
 // no format. LogMessage() provides such a simple interface and
 // avoids the [format-security] warning which occurs when you
-// call ROCKS_LOG_INFO(logger, log_message) directly.
+// call Log(logger, log_message) directly.
 namespace {
 void LogMessage(Logger* logger, const char* message) {
-  ROCKS_LOG_INFO(logger, "%s", message);
+  Log(logger, "%s", message);
 }

 void LogMessage(const InfoLogLevel log_level, Logger* logger,
@@ -91,9 +74,19 @@ void LogMessage(const InfoLogLevel log_level, Logger* logger,
 }
 }  // namespace

+namespace {
+void GetFileCreateTime(const std::string& fname, uint64_t* file_ctime) {
+  struct stat s;
+  if (stat(fname.c_str(), &s) != 0) {
+    *file_ctime = (uint64_t)0;
+  }
+  *file_ctime = static_cast<uint64_t>(s.st_ctime);
+}
+}  // namespace
+
 void AutoRollLoggerTest::RollLogFileBySizeTest(AutoRollLogger* logger,
                                               size_t log_max_size,
-                                               const std::string& log_message) {
+                                               const string& log_message) {
  logger->SetInfoLogLevel(InfoLogLevel::INFO_LEVEL);
  // measure the size of each message, which is supposed
  // to be equal or greater than log_message.size()
@@ -117,42 +110,45 @@ void AutoRollLoggerTest::RollLogFileBySizeTest(AutoRollLogger* logger,
  ASSERT_TRUE(message_size == logger->GetLogFileSize());
 }

-void AutoRollLoggerTest::RollLogFileByTimeTest(Env* env, AutoRollLogger* logger,
-                                               size_t time,
-                                               const std::string& log_message) {
-  uint64_t expected_ctime;
-  uint64_t actual_ctime;
-
+uint64_t AutoRollLoggerTest::RollLogFileByTimeTest(
+    AutoRollLogger* logger, size_t time, const string& log_message) {
+  uint64_t expected_create_time;
+  uint64_t actual_create_time;
  uint64_t total_log_size;
  EXPECT_OK(env->GetFileSize(kLogFile, &total_log_size));
-  expected_ctime = logger->TEST_ctime();
+  GetFileCreateTime(kLogFile, &expected_create_time);
  logger->SetCallNowMicrosEveryNRecords(0);

  // -- Write to the log for several times, which is supposed
  // to be finished before time.
  for (int i = 0; i < 10; ++i) {
-    env->SleepForMicroseconds(50000);
-    LogMessage(logger, log_message.c_str());
-    EXPECT_OK(logger->GetStatus());
-    // Make sure we always write to the same log file (by
-    // checking the create time);
+     LogMessage(logger, log_message.c_str());
+     EXPECT_OK(logger->GetStatus());
+     // Make sure we always write to the same log file (by
+     // checking the create time);
+     GetFileCreateTime(kLogFile, &actual_create_time);

-    actual_ctime = logger->TEST_ctime();
-
-    // Also make sure the log size is increasing.
-    EXPECT_EQ(expected_ctime, actual_ctime);
-    EXPECT_GT(logger->GetLogFileSize(), total_log_size);
-    total_log_size = logger->GetLogFileSize();
+     // Also make sure the log size is increasing.
+     EXPECT_EQ(expected_create_time, actual_create_time);
+     EXPECT_GT(logger->GetLogFileSize(), total_log_size);
+     total_log_size = logger->GetLogFileSize();
  }

  // -- Make the log file expire
-  env->SleepForMicroseconds(static_cast<int>(time * 1000000));
+#ifdef OS_WIN
+  Sleep(static_cast<unsigned int>(time) * 1000);
+#else
+  sleep(static_cast<unsigned int>(time));
+#endif
  LogMessage(logger, log_message.c_str());

  // At this time, the new log file should be created.
-  actual_ctime = logger->TEST_ctime();
-  EXPECT_LT(expected_ctime, actual_ctime);
+  GetFileCreateTime(kLogFile, &actual_create_time);
+  EXPECT_GT(actual_create_time, expected_create_time);
  EXPECT_LT(logger->GetLogFileSize(), total_log_size);
+  expected_create_time = actual_create_time;
+
+  return expected_create_time;
 }

 TEST_F(AutoRollLoggerTest, RollLogFileBySize) {
@@ -166,19 +162,16 @@ TEST_F(AutoRollLoggerTest, RollLogFileBySize) {
 }

 TEST_F(AutoRollLoggerTest, RollLogFileByTime) {
-  NoSleepEnv nse(Env::Default());
+    size_t time = 2;
+    size_t log_size = 1024 * 5;

-  size_t time = 2;
-  size_t log_size = 1024 * 5;
+    InitTestDb();
+    // -- Test the existence of file during the server restart.
+    ASSERT_EQ(Status::NotFound(), env->FileExists(kLogFile));
+    AutoRollLogger logger(Env::Default(), kTestDir, "", log_size, time);
+    ASSERT_OK(env->FileExists(kLogFile));

-  InitTestDb();
-  // -- Test the existence of file during the server restart.
-  ASSERT_EQ(Status::NotFound(), default_env->FileExists(kLogFile));
-  AutoRollLogger logger(&nse, kTestDir, "", log_size, time);
-  ASSERT_OK(default_env->FileExists(kLogFile));
-
-  RollLogFileByTimeTest(&nse, &logger, time,
-                        kSampleMessage + ":RollLogFileByTime");
+    RollLogFileByTimeTest(&logger, time, kSampleMessage + ":RollLogFileByTime");
 }

 TEST_F(AutoRollLoggerTest, OpenLogFilesMultipleTimesWithOptionLog_max_size) {
@@ -211,16 +204,16 @@ TEST_F(AutoRollLoggerTest, CompositeRollByTimeAndSizeLogger) {

  InitTestDb();

-  NoSleepEnv nse(Env::Default());
-  AutoRollLogger logger(&nse, kTestDir, "", log_max_size, time);
+  AutoRollLogger logger(Env::Default(), kTestDir, "", log_max_size, time);

  // Test the ability to roll by size
-  RollLogFileBySizeTest(&logger, log_max_size,
-                        kSampleMessage + ":CompositeRollByTimeAndSizeLogger");
+  RollLogFileBySizeTest(
+      &logger, log_max_size,
+      kSampleMessage + ":CompositeRollByTimeAndSizeLogger");

  // Test the ability to roll by Time
-  RollLogFileByTimeTest(&nse, &logger, time,
-                        kSampleMessage + ":CompositeRollByTimeAndSizeLogger");
+  RollLogFileByTimeTest( &logger, time,
+      kSampleMessage + ":CompositeRollByTimeAndSizeLogger");
 }

 #ifndef OS_WIN
@@ -228,7 +221,6 @@ TEST_F(AutoRollLoggerTest, CompositeRollByTimeAndSizeLogger) {
 // port
 TEST_F(AutoRollLoggerTest, CreateLoggerFromOptions) {
  DBOptions options;
-  NoSleepEnv nse(Env::Default());
  shared_ptr<Logger> logger;

  // Normal logger
@@ -247,15 +239,15 @@ TEST_F(AutoRollLoggerTest, CreateLoggerFromOptions) {
      kSampleMessage + ":CreateLoggerFromOptions - size");

  // Only roll by Time
-  options.env = &nse;
  InitTestDb();
  options.max_log_file_size = 0;
  options.log_file_time_to_roll = 2;
  ASSERT_OK(CreateLoggerFromOptions(kTestDir, options, &logger));
  auto_roll_logger =
    dynamic_cast<AutoRollLogger*>(logger.get());
-  RollLogFileByTimeTest(&nse, auto_roll_logger, options.log_file_time_to_roll,
-                        kSampleMessage + ":CreateLoggerFromOptions - time");
+  RollLogFileByTimeTest(
+      auto_roll_logger, options.log_file_time_to_roll,
+      kSampleMessage + ":CreateLoggerFromOptions - time");

  // roll by both Time and size
  InitTestDb();
@@ -264,10 +256,12 @@ TEST_F(AutoRollLoggerTest, CreateLoggerFromOptions) {
  ASSERT_OK(CreateLoggerFromOptions(kTestDir, options, &logger));
  auto_roll_logger =
    dynamic_cast<AutoRollLogger*>(logger.get());
-  RollLogFileBySizeTest(auto_roll_logger, options.max_log_file_size,
-                        kSampleMessage + ":CreateLoggerFromOptions - both");
-  RollLogFileByTimeTest(&nse, auto_roll_logger, options.log_file_time_to_roll,
-                        kSampleMessage + ":CreateLoggerFromOptions - both");
+  RollLogFileBySizeTest(
+      auto_roll_logger, options.max_log_file_size,
+      kSampleMessage + ":CreateLoggerFromOptions - both");
+  RollLogFileByTimeTest(
+      auto_roll_logger, options.log_file_time_to_roll,
+      kSampleMessage + ":CreateLoggerFromOptions - both");
 }

 TEST_F(AutoRollLoggerTest, LogFlushWhileRolling) {
@@ -280,29 +274,40 @@ TEST_F(AutoRollLoggerTest, LogFlushWhileRolling) {
  AutoRollLogger* auto_roll_logger =
      dynamic_cast<AutoRollLogger*>(logger.get());
  ASSERT_TRUE(auto_roll_logger);
-  rocksdb::port::Thread flush_thread;
+  std::thread flush_thread;

-  // Notes:
-  // (1) Need to pin the old logger before beginning the roll, as rolling grabs
-  //     the mutex, which would prevent us from accessing the old logger. This
-  //     also marks flush_thread with AutoRollLogger::Flush:PinnedLogger.
-  // (2) Need to reset logger during PosixLogger::Flush() to exercise a race
-  //     condition case, which is executing the flush with the pinned (old)
-  //     logger after auto-roll logger has cut over to a new logger.
-  // (3) PosixLogger::Flush() happens in both threads but its SyncPoints only
-  //     are enabled in flush_thread (the one pinning the old logger).
-  rocksdb::SyncPoint::GetInstance()->LoadDependencyAndMarkers(
-      {{"AutoRollLogger::Flush:PinnedLogger",
-        "AutoRollLoggerTest::LogFlushWhileRolling:PreRollAndPostThreadInit"},
-       {"PosixLogger::Flush:Begin1",
-        "AutoRollLogger::ResetLogger:BeforeNewLogger"},
-       {"AutoRollLogger::ResetLogger:AfterNewLogger",
-        "PosixLogger::Flush:Begin2"}},
-      {{"AutoRollLogger::Flush:PinnedLogger", "PosixLogger::Flush:Begin1"},
-       {"AutoRollLogger::Flush:PinnedLogger", "PosixLogger::Flush:Begin2"}});
+  rocksdb::SyncPoint::GetInstance()->LoadDependency({
+      // Need to pin the old logger before beginning the roll, as rolling grabs
+      // the mutex, which would prevent us from accessing the old logger.
+      {"AutoRollLogger::Flush:PinnedLogger",
+       "AutoRollLoggerTest::LogFlushWhileRolling:PreRollAndPostThreadInit"},
+      // Need to finish the flush thread init before this callback because the
+      // callback accesses flush_thread.get_id() in order to apply certain sync
+      // points only to the flush thread.
+      {"AutoRollLoggerTest::LogFlushWhileRolling:PreRollAndPostThreadInit",
+       "AutoRollLoggerTest::LogFlushWhileRolling:FlushCallbackBegin"},
+      // Need to reset logger at this point in Flush() to exercise a race
+      // condition case, which is executing the flush with the pinned (old)
+      // logger after the roll has cut over to a new logger.
+      {"AutoRollLoggerTest::LogFlushWhileRolling:FlushCallback1",
+       "AutoRollLogger::ResetLogger:BeforeNewLogger"},
+      {"AutoRollLogger::ResetLogger:AfterNewLogger",
+       "AutoRollLoggerTest::LogFlushWhileRolling:FlushCallback2"},
+  });
+  rocksdb::SyncPoint::GetInstance()->SetCallBack(
+      "PosixLogger::Flush:BeginCallback", [&](void* arg) {
+        TEST_SYNC_POINT(
+            "AutoRollLoggerTest::LogFlushWhileRolling:FlushCallbackBegin");
+        if (std::this_thread::get_id() == flush_thread.get_id()) {
+          TEST_SYNC_POINT(
+              "AutoRollLoggerTest::LogFlushWhileRolling:FlushCallback1");
+          TEST_SYNC_POINT(
+              "AutoRollLoggerTest::LogFlushWhileRolling:FlushCallback2");
+        }
+      });
  rocksdb::SyncPoint::GetInstance()->EnableProcessing();

-  flush_thread = port::Thread ([&]() { auto_roll_logger->Flush(); });
+  flush_thread = std::thread([&]() { auto_roll_logger->Flush(); });
  TEST_SYNC_POINT(
      "AutoRollLoggerTest::LogFlushWhileRolling:PreRollAndPostThreadInit");
  RollLogFileBySizeTest(auto_roll_logger, options.max_log_file_size,
@@ -338,12 +343,12 @@ TEST_F(AutoRollLoggerTest, InfoLogLevel) {
      logger.SetInfoLogLevel((InfoLogLevel)log_level);

      // again, messages with level smaller than log_level will not be logged.
-      ROCKS_LOG_HEADER(&logger, "%s", kSampleMessage.c_str());
-      ROCKS_LOG_DEBUG(&logger, "%s", kSampleMessage.c_str());
-      ROCKS_LOG_INFO(&logger, "%s", kSampleMessage.c_str());
-      ROCKS_LOG_WARN(&logger, "%s", kSampleMessage.c_str());
-      ROCKS_LOG_ERROR(&logger, "%s", kSampleMessage.c_str());
-      ROCKS_LOG_FATAL(&logger, "%s", kSampleMessage.c_str());
+      Log(InfoLogLevel::HEADER_LEVEL, &logger, "%s", kSampleMessage.c_str());
+      Debug(&logger, "%s", kSampleMessage.c_str());
+      Info(&logger, "%s", kSampleMessage.c_str());
+      Warn(&logger, "%s", kSampleMessage.c_str());
+      Error(&logger, "%s", kSampleMessage.c_str());
+      Fatal(&logger, "%s", kSampleMessage.c_str());
      log_lines += InfoLogLevel::HEADER_LEVEL - log_level + 1;
    }
  }
@@ -356,13 +361,13 @@ TEST_F(AutoRollLoggerTest, InfoLogLevel) {

 // Test the logger Header function for roll over logs
 // We expect the new logs creates as roll over to carry the headers specified
-static std::vector<std::string> GetOldFileNames(const std::string& path) {
-  std::vector<std::string> ret;
+static std::vector<string> GetOldFileNames(const string& path) {
+  std::vector<string> ret;

-  const std::string dirname = path.substr(/*start=*/0, path.find_last_of("/"));
-  const std::string fname = path.substr(path.find_last_of("/") + 1);
+  const string dirname = path.substr(/*start=*/ 0, path.find_last_of("/"));
+  const string fname = path.substr(path.find_last_of("/") + 1);

-  std::vector<std::string> children;
+  std::vector<string> children;
  Env::Default()->GetChildren(dirname, &children);

  // We know that the old log files are named [path]<something>
@@ -377,13 +382,12 @@ static std::vector<std::string> GetOldFileNames(const std::string& path) {
 }

 // Return the number of lines where a given pattern was found in the file
-static size_t GetLinesCount(const std::string& fname,
-                            const std::string& pattern) {
-  std::stringstream ssbuf;
-  std::string line;
+static size_t GetLinesCount(const string& fname, const string& pattern) {
+  stringstream ssbuf;
+  string line;
  size_t count = 0;

-  std::ifstream inFile(fname.c_str());
+  ifstream inFile(fname.c_str());
  ssbuf << inFile.rdbuf();

  while (getline(ssbuf, line)) {
@@ -417,11 +421,12 @@ TEST_F(AutoRollLoggerTest, LogHeaderTest) {
    } else if (test_num == 1) {
      // HEADER_LEVEL should make this behave like calling Header()
      for (size_t i = 0; i < MAX_HEADERS; i++) {
-        ROCKS_LOG_HEADER(&logger, "%s %d", HEADER_STR.c_str(), i);
+        Log(InfoLogLevel::HEADER_LEVEL, &logger, "%s %d",
+            HEADER_STR.c_str(), i);
      }
    }

-    const std::string newfname = logger.TEST_log_fname();
+    const string newfname = logger.TEST_log_fname();

    // Log enough data to cause a roll over
    int i = 0;
@@ -461,13 +466,13 @@ TEST_F(AutoRollLoggerTest, LogFileExistence) {
    [](char ch) { return ch == '/'; }, '\\');
  std::string deleteCmd = "if exist " + testDir + " rd /s /q " + testDir;
 #else
-  std::string deleteCmd = "rm -rf " + kTestDir;
+  string deleteCmd = "rm -rf " + kTestDir;
 #endif
  ASSERT_EQ(system(deleteCmd.c_str()), 0);
  options.max_log_file_size = 100 * 1024 * 1024;
  options.create_if_missing = true;
  ASSERT_OK(rocksdb::DB::Open(options, kTestDir, &db));
-  ASSERT_OK(default_env->FileExists(kLogFile));
+  ASSERT_OK(env->FileExists(kLogFile));
  delete db;
 }

@@ -477,14 +482,3 @@ int main(int argc, char** argv) {
  ::testing::InitGoogleTest(&argc, argv);
  return RUN_ALL_TESTS();
 }
-
-#else
-#include <stdio.h>
-
-int main(int argc, char** argv) {
-  fprintf(stderr,
-          "SKIPPED as AutoRollLogger is not supported in ROCKSDB_LITE\n");
-  return 0;
-}
-
-#endif  // !ROCKSDB_LITE
@@ -16,12 +16,11 @@
 #include "db/compaction_iterator.h"
 #include "db/dbformat.h"
 #include "db/event_helpers.h"
+#include "db/filename.h"
 #include "db/internal_stats.h"
 #include "db/merge_helper.h"
 #include "db/table_cache.h"
 #include "db/version_edit.h"
-#include "monitoring/iostats_context_imp.h"
-#include "monitoring/thread_status_util.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "rocksdb/iterator.h"
@@ -30,9 +29,9 @@
 #include "table/block_based_table_builder.h"
 #include "table/internal_iterator.h"
 #include "util/file_reader_writer.h"
-#include "util/filename.h"
+#include "util/iostats_context_imp.h"
 #include "util/stop_watch.h"
-#include "util/sync_point.h"
+#include "util/thread_status_util.h"

 namespace rocksdb {

@@ -46,7 +45,6 @@ TableBuilder* NewTableBuilder(
    uint32_t column_family_id, const std::string& column_family_name,
    WritableFileWriter* file, const CompressionType compression_type,
    const CompressionOptions& compression_opts,
-    int level,
    const std::string* compression_dict, const bool skip_filters) {
  assert((column_family_id ==
          TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
@@ -55,15 +53,14 @@ TableBuilder* NewTableBuilder(
      TableBuilderOptions(ioptions, internal_comparator,
                          int_tbl_prop_collector_factories, compression_type,
                          compression_opts, compression_dict, skip_filters,
-                          column_family_name, level),
+                          column_family_name),
      column_family_id, file);
 }

 Status BuildTable(
    const std::string& dbname, Env* env, const ImmutableCFOptions& ioptions,
-    const MutableCFOptions& mutable_cf_options, const EnvOptions& env_options,
-    TableCache* table_cache, InternalIterator* iter,
-    std::unique_ptr<InternalIterator> range_del_iter, FileMetaData* meta,
+    const EnvOptions& env_options, TableCache* table_cache,
+    InternalIterator* iter, FileMetaData* meta,
    const InternalKeyComparator& internal_comparator,
    const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
        int_tbl_prop_collector_factories,
@@ -83,13 +80,6 @@ Status BuildTable(
  Status s;
  meta->fd.file_size = 0;
  iter->SeekToFirst();
-  std::unique_ptr<RangeDelAggregator> range_del_agg(
-      new RangeDelAggregator(internal_comparator, snapshots));
-  s = range_del_agg->AddTombstones(std::move(range_del_iter));
-  if (!s.ok()) {
-    // may be non-ok if a range tombstone key is unparsable
-    return s;
-  }

  std::string fname = TableFileName(ioptions.db_paths, meta->fd.GetNumber(),
                                    meta->fd.GetPathId());
@@ -99,15 +89,11 @@ Status BuildTable(
 #endif  // !ROCKSDB_LITE
  TableProperties tp;

-  if (iter->Valid() || range_del_agg->ShouldAddTombstones()) {
+  if (iter->Valid()) {
    TableBuilder* builder;
    unique_ptr<WritableFileWriter> file_writer;
    {
      unique_ptr<WritableFile> file;
-#ifndef NDEBUG
-      bool use_direct_writes = env_options.use_direct_writes;
-      TEST_SYNC_POINT_CALLBACK("BuildTable:create_file", &use_direct_writes);
-#endif  // !NDEBUG
      s = NewWritableFile(env, fname, &file, env_options);
      if (!s.ok()) {
        EventHelpers::LogAndNotifyTableFileCreationFinished(
@@ -117,24 +103,24 @@ Status BuildTable(
      }
      file->SetIOPriority(io_priority);

-      file_writer.reset(new WritableFileWriter(std::move(file), env_options,
-                                               ioptions.statistics));
+      file_writer.reset(new WritableFileWriter(std::move(file), env_options));

      builder = NewTableBuilder(
          ioptions, internal_comparator, int_tbl_prop_collector_factories,
          column_family_id, column_family_name, file_writer.get(), compression,
-          compression_opts, level);
+          compression_opts);
    }

    MergeHelper merge(env, internal_comparator.user_comparator(),
                      ioptions.merge_operator, nullptr, ioptions.info_log,
+                      ioptions.min_partial_merge_operands,
                      true /* internal key corruption is not ok */,
                      snapshots.empty() ? 0 : snapshots.back());

-    CompactionIterator c_iter(
-        iter, internal_comparator.user_comparator(), &merge, kMaxSequenceNumber,
-        &snapshots, earliest_write_conflict_snapshot, env,
-        true /* internal key corruption is not ok */, range_del_agg.get());
+    CompactionIterator c_iter(iter, internal_comparator.user_comparator(),
+                              &merge, kMaxSequenceNumber, &snapshots,
+                              earliest_write_conflict_snapshot, env,
+                              true /* internal key corruption is not ok */);
    c_iter.SeekToFirst();
    for (; c_iter.Valid(); c_iter.Next()) {
      const Slice& key = c_iter.key();
@@ -149,9 +135,6 @@ Status BuildTable(
            ThreadStatus::FLUSH_BYTES_WRITTEN, IOSTATS(bytes_written));
      }
    }
-    // nullptr for table_{min,max} so all range tombstones will be flushed
-    range_del_agg->AddToBuilder(builder, nullptr /* lower_bound */,
-                                nullptr /* upper_bound */, meta);

    // Finish and check for builder errors
    bool empty = builder->NumEntries() == 0;
@@ -175,9 +158,9 @@ Status BuildTable(
    delete builder;

    // Finish and check for file errors
-    if (s.ok() && !empty) {
+    if (s.ok() && !empty && !ioptions.disable_data_sync) {
      StopWatch sw(env, ioptions.statistics, TABLE_SYNC_MICROS);
-      s = file_writer->Sync(ioptions.use_fsync);
+      file_writer->Sync(ioptions.use_fsync);
    }
    if (s.ok() && !empty) {
      s = file_writer->Close();
@@ -185,14 +168,8 @@ Status BuildTable(

    if (s.ok() && !empty) {
      // Verify that the table is usable
-      // We set for_compaction to false and don't OptimizeForCompactionTableRead
-      // here because this is a special case after we finish the table building
-      // No matter whether use_direct_io_for_flush_and_compaction is true,
-      // we will regrad this verification as user reads since the goal is
-      // to cache it here for further user reads
      std::unique_ptr<InternalIterator> it(table_cache->NewIterator(
-          ReadOptions(), env_options, internal_comparator, meta->fd,
-          nullptr /* range_del_agg */, nullptr,
+          ReadOptions(), env_options, internal_comparator, meta->fd, nullptr,
          (internal_stats == nullptr) ? nullptr
                                      : internal_stats->GetFileReadHist(0),
          false /* for_compaction */, nullptr /* arena */,
@@ -10,15 +10,14 @@
 #include <utility>
 #include <vector>
 #include "db/table_properties_collector.h"
-#include "options/cf_options.h"
 #include "rocksdb/comparator.h"
 #include "rocksdb/env.h"
+#include "rocksdb/immutable_options.h"
 #include "rocksdb/listener.h"
 #include "rocksdb/options.h"
 #include "rocksdb/status.h"
 #include "rocksdb/table_properties.h"
 #include "rocksdb/types.h"
-#include "table/scoped_arena_iterator.h"
 #include "util/event_logger.h"

 namespace rocksdb {
@@ -49,7 +48,6 @@ TableBuilder* NewTableBuilder(
    uint32_t column_family_id, const std::string& column_family_name,
    WritableFileWriter* file, const CompressionType compression_type,
    const CompressionOptions& compression_opts,
-    int level,
    const std::string* compression_dict = nullptr,
    const bool skip_filters = false);

@@ -63,9 +61,8 @@ TableBuilder* NewTableBuilder(
 //    by column_family_id, or empty string if unknown.
 extern Status BuildTable(
    const std::string& dbname, Env* env, const ImmutableCFOptions& options,
-    const MutableCFOptions& mutable_cf_options, const EnvOptions& env_options,
-    TableCache* table_cache, InternalIterator* iter,
-    std::unique_ptr<InternalIterator> range_del_iter, FileMetaData* meta,
+    const EnvOptions& env_options, TableCache* table_cache,
+    InternalIterator* iter, FileMetaData* meta,
    const InternalKeyComparator& internal_comparator,
    const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
        int_tbl_prop_collector_factories,
@@ -2,18 +2,17 @@
   Use of this source code is governed by a BSD-style license that can be
   found in the LICENSE file. See the AUTHORS file for names of contributors. */

-#include <stdio.h>
-
 #ifndef ROCKSDB_LITE  // Lite does not support C API

 #include "rocksdb/c.h"

 #include <stddef.h>
+#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <sys/types.h>
 #ifndef OS_WIN
-#include <unistd.h>
+#  include <unistd.h>
 #endif
 #include <inttypes.h>

@@ -22,10 +21,11 @@

 #include <Windows.h>

-#define snprintf _snprintf
+# define snprintf _snprintf

 // Ok for uniqueness
 int geteuid() {
+
  int result = 0;

  result = ((int)GetCurrentProcessId() << 16);
@@ -38,13 +38,13 @@ int geteuid() {

 const char* phase = "";
 static char dbname[200];
-static char sstfilename[200];
 static char dbbackupname[200];

 static void StartPhase(const char* name) {
  fprintf(stderr, "=== Test %s\n", name);
  phase = name;
 }
+
 static const char* GetTempDir(void) {
    const char* ret = getenv("TEST_TMPDIR");
    if (ret == NULL || ret[0] == '\0')
@@ -87,16 +87,6 @@ static void Free(char** ptr) {
  }
 }

-static void CheckValue(
-    char* err,
-    const char* expected,
-    char** actual,
-    size_t actual_length) {
-  CheckNoError(err);
-  CheckEqual(expected, *actual, actual_length);
-  Free(actual);
-}
-
 static void CheckGet(
    rocksdb_t* db,
    const rocksdb_readoptions_t* options,
@@ -298,11 +288,9 @@ int main(int argc, char** argv) {
  rocksdb_cache_t* cache;
  rocksdb_env_t* env;
  rocksdb_options_t* options;
-  rocksdb_compactoptions_t* coptions;
  rocksdb_block_based_table_options_t* table_options;
  rocksdb_readoptions_t* roptions;
  rocksdb_writeoptions_t* woptions;
-  rocksdb_ratelimiter_t* rate_limiter;
  char* err = NULL;
  int run = -1;

@@ -316,11 +304,6 @@ int main(int argc, char** argv) {
           GetTempDir(),
           ((int) geteuid()));

-  snprintf(sstfilename, sizeof(sstfilename),
-           "%s/rocksdb_c_test-%d-sst",
-           GetTempDir(),
-           ((int)geteuid()));
-
  StartPhase("create_objects");
  cmp = rocksdb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
  env = rocksdb_create_default_env();
@@ -334,7 +317,6 @@ int main(int argc, char** argv) {
  rocksdb_options_set_write_buffer_size(options, 100000);
  rocksdb_options_set_paranoid_checks(options, 1);
  rocksdb_options_set_max_open_files(options, 10);
-  rocksdb_options_set_base_background_compactions(options, 1);
  table_options = rocksdb_block_based_options_create();
  rocksdb_block_based_options_set_block_cache(table_options, cache);
  rocksdb_options_set_block_based_table_factory(options, table_options);
@@ -344,20 +326,14 @@ int main(int argc, char** argv) {
  int compression_levels[] = {rocksdb_no_compression, rocksdb_no_compression,
                              rocksdb_no_compression, rocksdb_no_compression};
  rocksdb_options_set_compression_per_level(options, compression_levels, 4);
-  rate_limiter = rocksdb_ratelimiter_create(1000 * 1024 * 1024, 100 * 1000, 10);
-  rocksdb_options_set_ratelimiter(options, rate_limiter);
-  rocksdb_ratelimiter_destroy(rate_limiter);

  roptions = rocksdb_readoptions_create();
  rocksdb_readoptions_set_verify_checksums(roptions, 1);
-  rocksdb_readoptions_set_fill_cache(roptions, 1);
+  rocksdb_readoptions_set_fill_cache(roptions, 0);

  woptions = rocksdb_writeoptions_create();
  rocksdb_writeoptions_set_sync(woptions, 1);

-  coptions = rocksdb_compactoptions_create();
-  rocksdb_compactoptions_set_exclusive_manual_compaction(coptions, 1);
-
  StartPhase("destroy");
  rocksdb_destroy_db(options, dbname, &err);
  Free(&err);
@@ -439,95 +415,6 @@ int main(int argc, char** argv) {
  rocksdb_compact_range(db, "a", 1, "z", 1);
  CheckGet(db, roptions, "foo", "hello");

-  StartPhase("compactallopt");
-  rocksdb_compact_range_opt(db, coptions, NULL, 0, NULL, 0);
-  CheckGet(db, roptions, "foo", "hello");
-
-  StartPhase("compactrangeopt");
-  rocksdb_compact_range_opt(db, coptions, "a", 1, "z", 1);
-  CheckGet(db, roptions, "foo", "hello");
-
-  // Simple check cache usage
-  StartPhase("cache_usage");
-  {
-    rocksdb_readoptions_set_pin_data(roptions, 1);
-    rocksdb_iterator_t* iter = rocksdb_create_iterator(db, roptions);
-    rocksdb_iter_seek(iter, "foo", 3);
-
-    size_t usage = rocksdb_cache_get_usage(cache);
-    CheckCondition(usage > 0);
-
-    size_t pin_usage = rocksdb_cache_get_pinned_usage(cache);
-    CheckCondition(pin_usage > 0);
-
-    rocksdb_iter_next(iter);
-    rocksdb_iter_destroy(iter);
-    rocksdb_readoptions_set_pin_data(roptions, 0);
-  }
-
-  StartPhase("addfile");
-  {
-    rocksdb_envoptions_t* env_opt = rocksdb_envoptions_create();
-    rocksdb_options_t* io_options = rocksdb_options_create();
-    rocksdb_sstfilewriter_t* writer =
-        rocksdb_sstfilewriter_create(env_opt, io_options);
-
-    unlink(sstfilename);
-    rocksdb_sstfilewriter_open(writer, sstfilename, &err);
-    CheckNoError(err);
-    rocksdb_sstfilewriter_add(writer, "sstk1", 5, "v1", 2, &err);
-    CheckNoError(err);
-    rocksdb_sstfilewriter_add(writer, "sstk2", 5, "v2", 2, &err);
-    CheckNoError(err);
-    rocksdb_sstfilewriter_add(writer, "sstk3", 5, "v3", 2, &err);
-    CheckNoError(err);
-    rocksdb_sstfilewriter_finish(writer, &err);
-    CheckNoError(err);
-
-    rocksdb_ingestexternalfileoptions_t* ing_opt =
-        rocksdb_ingestexternalfileoptions_create();
-    const char* file_list[1] = {sstfilename};
-    rocksdb_ingest_external_file(db, file_list, 1, ing_opt, &err);
-    CheckNoError(err);
-    CheckGet(db, roptions, "sstk1", "v1");
-    CheckGet(db, roptions, "sstk2", "v2");
-    CheckGet(db, roptions, "sstk3", "v3");
-
-    unlink(sstfilename);
-    rocksdb_sstfilewriter_open(writer, sstfilename, &err);
-    CheckNoError(err);
-    rocksdb_sstfilewriter_add(writer, "sstk2", 5, "v4", 2, &err);
-    CheckNoError(err);
-    rocksdb_sstfilewriter_add(writer, "sstk22", 6, "v5", 2, &err);
-    CheckNoError(err);
-    rocksdb_sstfilewriter_add(writer, "sstk3", 5, "v6", 2, &err);
-    CheckNoError(err);
-    rocksdb_sstfilewriter_finish(writer, &err);
-    CheckNoError(err);
-
-    rocksdb_ingest_external_file(db, file_list, 1, ing_opt, &err);
-    CheckNoError(err);
-    CheckGet(db, roptions, "sstk1", "v1");
-    CheckGet(db, roptions, "sstk2", "v4");
-    CheckGet(db, roptions, "sstk22", "v5");
-    CheckGet(db, roptions, "sstk3", "v6");
-
-    rocksdb_ingestexternalfileoptions_destroy(ing_opt);
-    rocksdb_sstfilewriter_destroy(writer);
-    rocksdb_options_destroy(io_options);
-    rocksdb_envoptions_destroy(env_opt);
-
-    // Delete all keys we just ingested
-    rocksdb_delete(db, woptions, "sstk1", 5, &err);
-    CheckNoError(err);
-    rocksdb_delete(db, woptions, "sstk2", 5, &err);
-    CheckNoError(err);
-    rocksdb_delete(db, woptions, "sstk22", 6, &err);
-    CheckNoError(err);
-    rocksdb_delete(db, woptions, "sstk3", 5, &err);
-    CheckNoError(err);
-  }
-
  StartPhase("writebatch");
  {
    rocksdb_writebatch_t* wb = rocksdb_writebatch_create();
@@ -544,24 +431,6 @@ int main(int argc, char** argv) {
    int pos = 0;
    rocksdb_writebatch_iterate(wb, &pos, CheckPut, CheckDel);
    CheckCondition(pos == 3);
-    rocksdb_writebatch_clear(wb);
-    rocksdb_writebatch_put(wb, "bar", 3, "b", 1);
-    rocksdb_writebatch_put(wb, "bay", 3, "d", 1);
-    rocksdb_writebatch_delete_range(wb, "bar", 3, "bay", 3);
-    rocksdb_write(db, woptions, wb, &err);
-    CheckNoError(err);
-    CheckGet(db, roptions, "bar", NULL);
-    CheckGet(db, roptions, "bay", "d");
-    rocksdb_writebatch_clear(wb);
-    const char* start_list[1] = {"bay"};
-    const size_t start_sizes[1] = {3};
-    const char* end_list[1] = {"baz"};
-    const size_t end_sizes[1] = {3};
-    rocksdb_writebatch_delete_rangev(wb, 1, start_list, start_sizes, end_list,
-                                     end_sizes);
-    rocksdb_write(db, woptions, wb, &err);
-    CheckNoError(err);
-    CheckGet(db, roptions, "bay", NULL);
    rocksdb_writebatch_destroy(wb);
  }

@@ -583,23 +452,6 @@ int main(int argc, char** argv) {
    rocksdb_writebatch_destroy(wb);
  }

-  StartPhase("writebatch_savepoint");
-  {
-    rocksdb_writebatch_t* wb = rocksdb_writebatch_create();
-    rocksdb_writebatch_set_save_point(wb);
-    const char* k_list[2] = {"z", "ap"};
-    const size_t k_sizes[2] = {1, 2};
-    const char* v_list[3] = {"x", "y", "z"};
-    const size_t v_sizes[3] = {1, 1, 1};
-    rocksdb_writebatch_putv(wb, 2, k_list, k_sizes, 3, v_list, v_sizes);
-    rocksdb_writebatch_rollback_to_save_point(wb, &err);
-    CheckNoError(err);
-    rocksdb_write(db, woptions, wb, &err);
-    CheckNoError(err);
-    CheckGet(db, roptions, "zap", NULL);
-    rocksdb_writebatch_destroy(wb);
-  }
-
  StartPhase("writebatch_rep");
  {
    rocksdb_writebatch_t* wb1 = rocksdb_writebatch_create();
@@ -618,90 +470,6 @@ int main(int argc, char** argv) {
    rocksdb_writebatch_destroy(wb2);
  }

-  StartPhase("writebatch_wi");
-  {
-    rocksdb_writebatch_wi_t* wbi = rocksdb_writebatch_wi_create(0, 1);
-    rocksdb_writebatch_wi_put(wbi, "foo", 3, "a", 1);
-    rocksdb_writebatch_wi_clear(wbi);
-    rocksdb_writebatch_wi_put(wbi, "bar", 3, "b", 1);
-    rocksdb_writebatch_wi_put(wbi, "box", 3, "c", 1);
-    rocksdb_writebatch_wi_delete(wbi, "bar", 3);
-    int count = rocksdb_writebatch_wi_count(wbi);
-    CheckCondition(count == 3);
-    size_t size;
-    char* value;
-    value = rocksdb_writebatch_wi_get_from_batch(wbi, options, "box", 3, &size, &err);
-    CheckValue(err, "c", &value, size);
-    value = rocksdb_writebatch_wi_get_from_batch(wbi, options, "bar", 3, &size, &err);
-    CheckValue(err, NULL, &value, size);
-    value = rocksdb_writebatch_wi_get_from_batch_and_db(wbi, db, roptions, "foo", 3, &size, &err);
-    CheckValue(err, "hello", &value, size);
-    value = rocksdb_writebatch_wi_get_from_batch_and_db(wbi, db, roptions, "box", 3, &size, &err);
-    CheckValue(err, "c", &value, size);
-    rocksdb_write_writebatch_wi(db, woptions, wbi, &err);
-    CheckNoError(err);
-    CheckGet(db, roptions, "foo", "hello");
-    CheckGet(db, roptions, "bar", NULL);
-    CheckGet(db, roptions, "box", "c");
-    int pos = 0;
-    rocksdb_writebatch_wi_iterate(wbi, &pos, CheckPut, CheckDel);
-    CheckCondition(pos == 3);
-    rocksdb_writebatch_wi_clear(wbi);
-    rocksdb_writebatch_wi_put(wbi, "bar", 3, "b", 1);
-    rocksdb_writebatch_wi_put(wbi, "bay", 3, "d", 1);
-    rocksdb_writebatch_wi_delete_range(wbi, "bar", 3, "bay", 3);
-    rocksdb_write_writebatch_wi(db, woptions, wbi, &err);
-    CheckNoError(err);
-    CheckGet(db, roptions, "bar", NULL);
-    CheckGet(db, roptions, "bay", "d");
-    rocksdb_writebatch_wi_clear(wbi);
-    const char* start_list[1] = {"bay"};
-    const size_t start_sizes[1] = {3};
-    const char* end_list[1] = {"baz"};
-    const size_t end_sizes[1] = {3};
-    rocksdb_writebatch_wi_delete_rangev(wbi, 1, start_list, start_sizes, end_list,
-                                     end_sizes);
-    rocksdb_write_writebatch_wi(db, woptions, wbi, &err);
-    CheckNoError(err);
-    CheckGet(db, roptions, "bay", NULL);
-    rocksdb_writebatch_wi_destroy(wbi);
-  }
-
-  StartPhase("writebatch_wi_vectors");
-  {
-    rocksdb_writebatch_wi_t* wb = rocksdb_writebatch_wi_create(0, 1);
-    const char* k_list[2] = { "z", "ap" };
-    const size_t k_sizes[2] = { 1, 2 };
-    const char* v_list[3] = { "x", "y", "z" };
-    const size_t v_sizes[3] = { 1, 1, 1 };
-    rocksdb_writebatch_wi_putv(wb, 2, k_list, k_sizes, 3, v_list, v_sizes);
-    rocksdb_write_writebatch_wi(db, woptions, wb, &err);
-    CheckNoError(err);
-    CheckGet(db, roptions, "zap", "xyz");
-    rocksdb_writebatch_wi_delete(wb, "zap", 3);
-    rocksdb_write_writebatch_wi(db, woptions, wb, &err);
-    CheckNoError(err);
-    CheckGet(db, roptions, "zap", NULL);
-    rocksdb_writebatch_wi_destroy(wb);
-  }
-
-  StartPhase("writebatch_wi_savepoint");
-  {
-    rocksdb_writebatch_wi_t* wb = rocksdb_writebatch_wi_create(0, 1);
-    rocksdb_writebatch_wi_set_save_point(wb);
-    const char* k_list[2] = {"z", "ap"};
-    const size_t k_sizes[2] = {1, 2};
-    const char* v_list[3] = {"x", "y", "z"};
-    const size_t v_sizes[3] = {1, 1, 1};
-    rocksdb_writebatch_wi_putv(wb, 2, k_list, k_sizes, 3, v_list, v_sizes);
-    rocksdb_writebatch_wi_rollback_to_save_point(wb, &err);
-    CheckNoError(err);
-    rocksdb_write_writebatch_wi(db, woptions, wb, &err);
-    CheckNoError(err);
-    CheckGet(db, roptions, "zap", NULL);
-    rocksdb_writebatch_wi_destroy(wb);
-  }
-
  StartPhase("iter");
  {
    rocksdb_iterator_t* iter = rocksdb_create_iterator(db, roptions);
@@ -719,46 +487,11 @@ int main(int argc, char** argv) {
    CheckIter(iter, "foo", "hello");
    rocksdb_iter_seek(iter, "b", 1);
    CheckIter(iter, "box", "c");
-    rocksdb_iter_seek_for_prev(iter, "g", 1);
-    CheckIter(iter, "foo", "hello");
-    rocksdb_iter_seek_for_prev(iter, "box", 3);
-    CheckIter(iter, "box", "c");
    rocksdb_iter_get_error(iter, &err);
    CheckNoError(err);
    rocksdb_iter_destroy(iter);
  }

-  StartPhase("wbwi_iter");
-  {
-    rocksdb_iterator_t* base_iter = rocksdb_create_iterator(db, roptions);
-    rocksdb_writebatch_wi_t* wbi = rocksdb_writebatch_wi_create(0, 1);
-    rocksdb_writebatch_wi_put(wbi, "bar", 3, "b", 1);
-    rocksdb_writebatch_wi_delete(wbi, "foo", 3);
-    rocksdb_iterator_t* iter = rocksdb_writebatch_wi_create_iterator_with_base(wbi, base_iter);
-    CheckCondition(!rocksdb_iter_valid(iter));
-    rocksdb_iter_seek_to_first(iter);
-    CheckCondition(rocksdb_iter_valid(iter));
-    CheckIter(iter, "bar", "b");
-    rocksdb_iter_next(iter);
-    CheckIter(iter, "box", "c");
-    rocksdb_iter_prev(iter);
-    CheckIter(iter, "bar", "b");
-    rocksdb_iter_prev(iter);
-    CheckCondition(!rocksdb_iter_valid(iter));
-    rocksdb_iter_seek_to_last(iter);
-    CheckIter(iter, "box", "c");
-    rocksdb_iter_seek(iter, "b", 1);
-    CheckIter(iter, "bar", "b");
-    rocksdb_iter_seek_for_prev(iter, "c", 1);
-    CheckIter(iter, "box", "c");
-    rocksdb_iter_seek_for_prev(iter, "box", 3);
-    CheckIter(iter, "box", "c");
-    rocksdb_iter_get_error(iter, &err);
-    CheckNoError(err);
-    rocksdb_iter_destroy(iter);
-    rocksdb_writebatch_wi_destroy(wbi);
-  }
-
  StartPhase("multiget");
  {
    const char* keys[3] = { "box", "foo", "notfound" };
@@ -842,7 +575,6 @@ int main(int argc, char** argv) {
    rocksdb_close(db);
    rocksdb_options_set_create_if_missing(options, 0);
    rocksdb_options_set_error_if_exists(options, 0);
-    rocksdb_options_set_wal_recovery_mode(options, 2);
    rocksdb_repair_db(options, dbname, &err);
    CheckNoError(err);
    db = rocksdb_open(options, dbname, &err);
@@ -965,7 +697,7 @@ int main(int argc, char** argv) {
  {
    rocksdb_close(db);
    rocksdb_destroy_db(options, dbname, &err);
-    CheckNoError(err);
+    CheckNoError(err)

    rocksdb_options_t* db_options = rocksdb_options_create();
    rocksdb_options_set_create_if_missing(db_options, 1);
@@ -1096,7 +828,6 @@ int main(int argc, char** argv) {
    rocksdb_options_set_prefix_extractor(options, rocksdb_slicetransform_create_fixed_prefix(3));
    rocksdb_options_set_hash_skip_list_rep(options, 5000, 4, 4);
    rocksdb_options_set_plain_table_factory(options, 4, 10, 0.75, 16);
-    rocksdb_options_set_allow_concurrent_memtable_write(options, 0);

    db = rocksdb_open(options, dbname, &err);
    CheckNoError(err);
@@ -1131,19 +862,6 @@ int main(int argc, char** argv) {
    CheckNoError(err);
    rocksdb_iter_destroy(iter);

-    rocksdb_readoptions_set_total_order_seek(roptions, 1);
-    iter = rocksdb_create_iterator(db, roptions);
-    CheckCondition(!rocksdb_iter_valid(iter));
-
-    rocksdb_iter_seek(iter, "ba", 2);
-    rocksdb_iter_get_error(iter, &err);
-    CheckNoError(err);
-    CheckCondition(rocksdb_iter_valid(iter));
-    CheckIter(iter, "bar1", "bar");
-
-    rocksdb_iter_destroy(iter);
-    rocksdb_readoptions_set_total_order_seek(roptions, 0);
-
    rocksdb_close(db);
    rocksdb_destroy_db(options, dbname, &err);
  }
@@ -1225,35 +943,12 @@ int main(int argc, char** argv) {
    }
  }

-  // Simple sanity check that setting memtable rep works.
-  StartPhase("memtable_reps");
-  {
-    // Create database with vector memtable.
-    rocksdb_close(db);
-    rocksdb_destroy_db(options, dbname, &err);
-    CheckNoError(err);
-
-    rocksdb_options_set_memtable_vector_rep(options);
-    db = rocksdb_open(options, dbname, &err);
-    CheckNoError(err);
-
-    // Create database with hash skiplist memtable.
-    rocksdb_close(db);
-    rocksdb_destroy_db(options, dbname, &err);
-    CheckNoError(err);
-
-    rocksdb_options_set_hash_skip_list_rep(options, 5000, 4, 4);
-    db = rocksdb_open(options, dbname, &err);
-    CheckNoError(err);
-  }
-
  StartPhase("cleanup");
  rocksdb_close(db);
  rocksdb_options_destroy(options);
  rocksdb_block_based_options_destroy(table_options);
  rocksdb_readoptions_destroy(roptions);
  rocksdb_writeoptions_destroy(woptions);
-  rocksdb_compactoptions_destroy(coptions);
  rocksdb_cache_destroy(cache);
  rocksdb_comparator_destroy(cmp);
  rocksdb_env_destroy(env);
@@ -1263,6 +958,7 @@ int main(int argc, char** argv) {
 }

 #else
+#include <stdio.h>

 int main() {
  fprintf(stderr, "SKIPPED\n");
@@ -20,19 +20,19 @@
 #include <limits>

 #include "db/compaction_picker.h"
-#include "db/compaction_picker_universal.h"
 #include "db/db_impl.h"
 #include "db/internal_stats.h"
 #include "db/job_context.h"
 #include "db/table_properties_collector.h"
 #include "db/version_set.h"
 #include "db/write_controller.h"
+#include "db/writebuffer.h"
 #include "memtable/hash_skiplist_rep.h"
-#include "monitoring/thread_status_util.h"
-#include "options/options_helper.h"
-#include "table/block_based_table_factory.h"
 #include "util/autovector.h"
 #include "util/compression.h"
+#include "util/options_helper.h"
+#include "util/thread_status_util.h"
+#include "util/xfunc.h"

 namespace rocksdb {

@@ -46,11 +46,6 @@ ColumnFamilyHandleImpl::ColumnFamilyHandleImpl(

 ColumnFamilyHandleImpl::~ColumnFamilyHandleImpl() {
  if (cfd_ != nullptr) {
-#ifndef ROCKSDB_LITE
-    for (auto& listener : cfd_->ioptions()->listeners) {
-      listener->OnColumnFamilyHandleDeletionStarted(this);
-    }
-#endif  // ROCKSDB_LITE
    // Job id == 0 means that this is not our background process, but rather
    // user thread
    JobContext job_context(0);
@@ -77,23 +72,26 @@ Status ColumnFamilyHandleImpl::GetDescriptor(ColumnFamilyDescriptor* desc) {
 #ifndef ROCKSDB_LITE
  // accessing mutable cf-options requires db mutex.
  InstrumentedMutexLock l(mutex_);
-  *desc = ColumnFamilyDescriptor(cfd()->GetName(), cfd()->GetLatestCFOptions());
+  *desc = ColumnFamilyDescriptor(
+      cfd()->GetName(),
+      BuildColumnFamilyOptions(*cfd()->options(),
+                               *cfd()->GetLatestMutableCFOptions()));
  return Status::OK();
 #else
  return Status::NotSupported();
 #endif  // !ROCKSDB_LITE
 }

-const Comparator* ColumnFamilyHandleImpl::GetComparator() const {
+const Comparator* ColumnFamilyHandleImpl::user_comparator() const {
  return cfd()->user_comparator();
 }

 void GetIntTblPropCollectorFactory(
-    const ImmutableCFOptions& ioptions,
+    const ColumnFamilyOptions& cf_options,
    std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
        int_tbl_prop_collector_factories) {
-  auto& collector_factories = ioptions.table_properties_collector_factories;
-  for (size_t i = 0; i < ioptions.table_properties_collector_factories.size();
+  auto& collector_factories = cf_options.table_properties_collector_factories;
+  for (size_t i = 0; i < cf_options.table_properties_collector_factories.size();
       ++i) {
    assert(collector_factories[i]);
    int_tbl_prop_collector_factories->emplace_back(
@@ -132,6 +130,11 @@ Status CheckConcurrentWritesSupported(const ColumnFamilyOptions& cf_options) {
        "In-place memtable updates (inplace_update_support) is not compatible "
        "with concurrent writes (allow_concurrent_memtable_write)");
  }
+  if (cf_options.filter_deletes) {
+    return Status::InvalidArgument(
+        "Delete filtering (filter_deletes) is not compatible with concurrent "
+        "memtable writes (allow_concurrent_memtable_writes)");
+  }
  if (!cf_options.memtable_factory->IsInsertConcurrentlySupported()) {
    return Status::InvalidArgument(
        "Memtable doesn't concurrent writes (allow_concurrent_memtable_write)");
@@ -139,9 +142,11 @@ Status CheckConcurrentWritesSupported(const ColumnFamilyOptions& cf_options) {
  return Status::OK();
 }

-ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
+ColumnFamilyOptions SanitizeOptions(const DBOptions& db_options,
+                                    const InternalKeyComparator* icmp,
                                    const ColumnFamilyOptions& src) {
  ColumnFamilyOptions result = src;
+  result.comparator = icmp;
  size_t clamp_max = std::conditional<
      sizeof(size_t) == 4, std::integral_constant<size_t, 0xffffffff>,
      std::integral_constant<uint64_t, 64ull << 30>>::type::value;
@@ -159,10 +164,6 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
  result.min_write_buffer_number_to_merge =
      std::min(result.min_write_buffer_number_to_merge,
               result.max_write_buffer_number - 1);
-  if (result.min_write_buffer_number_to_merge < 1) {
-    result.min_write_buffer_number_to_merge = 1;
-  }
-
  if (result.num_levels < 1) {
    result.num_levels = 1;
  }
@@ -176,12 +177,14 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
  if (result.max_write_buffer_number_to_maintain < 0) {
    result.max_write_buffer_number_to_maintain = result.max_write_buffer_number;
  }
-  // bloom filter size shouldn't exceed 1/4 of memtable size.
-  if (result.memtable_prefix_bloom_size_ratio > 0.25) {
-    result.memtable_prefix_bloom_size_ratio = 0.25;
-  } else if (result.memtable_prefix_bloom_size_ratio < 0) {
-    result.memtable_prefix_bloom_size_ratio = 0;
-  }
+  XFUNC_TEST("memtablelist_history", "transaction_xftest_SanitizeOptions",
+             xf_transaction_set_memtable_history1,
+             xf_transaction_set_memtable_history,
+             &result.max_write_buffer_number_to_maintain);
+  XFUNC_TEST("memtablelist_history_clear", "transaction_xftest_SanitizeOptions",
+             xf_transaction_clear_memtable_history1,
+             xf_transaction_clear_memtable_history,
+             &result.max_write_buffer_number_to_maintain);

  if (!result.prefix_extractor) {
    assert(result.memtable_factory);
@@ -201,13 +204,9 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
    result.level0_stop_writes_trigger = std::numeric_limits<int>::max();
  }

-  if (result.max_bytes_for_level_multiplier <= 0) {
-    result.max_bytes_for_level_multiplier = 1;
-  }
-
  if (result.level0_file_num_compaction_trigger == 0) {
-    ROCKS_LOG_WARN(db_options.info_log.get(),
-                   "level0_file_num_compaction_trigger cannot be 0");
+    Warn(db_options.info_log.get(),
+         "level0_file_num_compaction_trigger cannot be 0");
    result.level0_file_num_compaction_trigger = 1;
  }

@@ -215,14 +214,14 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
          result.level0_slowdown_writes_trigger ||
      result.level0_slowdown_writes_trigger <
          result.level0_file_num_compaction_trigger) {
-    ROCKS_LOG_WARN(db_options.info_log.get(),
-                   "This condition must be satisfied: "
-                   "level0_stop_writes_trigger(%d) >= "
-                   "level0_slowdown_writes_trigger(%d) >= "
-                   "level0_file_num_compaction_trigger(%d)",
-                   result.level0_stop_writes_trigger,
-                   result.level0_slowdown_writes_trigger,
-                   result.level0_file_num_compaction_trigger);
+    Warn(db_options.info_log.get(),
+         "This condition must be satisfied: "
+         "level0_stop_writes_trigger(%d) >= "
+         "level0_slowdown_writes_trigger(%d) >= "
+         "level0_file_num_compaction_trigger(%d)",
+         result.level0_stop_writes_trigger,
+         result.level0_slowdown_writes_trigger,
+         result.level0_file_num_compaction_trigger);
    if (result.level0_slowdown_writes_trigger <
        result.level0_file_num_compaction_trigger) {
      result.level0_slowdown_writes_trigger =
@@ -232,14 +231,14 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
        result.level0_slowdown_writes_trigger) {
      result.level0_stop_writes_trigger = result.level0_slowdown_writes_trigger;
    }
-    ROCKS_LOG_WARN(db_options.info_log.get(),
-                   "Adjust the value to "
-                   "level0_stop_writes_trigger(%d)"
-                   "level0_slowdown_writes_trigger(%d)"
-                   "level0_file_num_compaction_trigger(%d)",
-                   result.level0_stop_writes_trigger,
-                   result.level0_slowdown_writes_trigger,
-                   result.level0_file_num_compaction_trigger);
+    Warn(db_options.info_log.get(),
+         "Adjust the value to "
+         "level0_stop_writes_trigger(%d)"
+         "level0_slowdown_writes_trigger(%d)"
+         "level0_file_num_compaction_trigger(%d)",
+         result.level0_stop_writes_trigger,
+         result.level0_slowdown_writes_trigger,
+         result.level0_file_num_compaction_trigger);
  }

  if (result.soft_pending_compaction_bytes_limit == 0) {
@@ -263,10 +262,6 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
    }
  }

-  if (result.max_compaction_bytes == 0) {
-    result.max_compaction_bytes = result.target_file_size_base * 25;
-  }
-
  return result;
 }

@@ -334,8 +329,8 @@ void SuperVersionUnrefHandle(void* ptr) {

 ColumnFamilyData::ColumnFamilyData(
    uint32_t id, const std::string& name, Version* _dummy_versions,
-    Cache* _table_cache, WriteBufferManager* write_buffer_manager,
-    const ColumnFamilyOptions& cf_options, const ImmutableDBOptions& db_options,
+    Cache* _table_cache, WriteBuffer* write_buffer,
+    const ColumnFamilyOptions& cf_options, const DBOptions* db_options,
    const EnvOptions& env_options, ColumnFamilySet* column_family_set)
    : id_(id),
      name_(name),
@@ -344,15 +339,14 @@ ColumnFamilyData::ColumnFamilyData(
      refs_(0),
      dropped_(false),
      internal_comparator_(cf_options.comparator),
-      initial_cf_options_(SanitizeOptions(db_options, cf_options)),
-      ioptions_(db_options, initial_cf_options_),
-      mutable_cf_options_(initial_cf_options_),
-      is_delete_range_supported_(strcmp(cf_options.table_factory->Name(),
-                                        BlockBasedTableFactory().Name()) == 0),
-      write_buffer_manager_(write_buffer_manager),
+      options_(*db_options,
+               SanitizeOptions(*db_options, &internal_comparator_, cf_options)),
+      ioptions_(options_),
+      mutable_cf_options_(options_, ioptions_),
+      write_buffer_(write_buffer),
      mem_(nullptr),
-      imm_(ioptions_.min_write_buffer_number_to_merge,
-           ioptions_.max_write_buffer_number_to_maintain),
+      imm_(options_.min_write_buffer_number_to_merge,
+           options_.max_write_buffer_number_to_maintain),
      super_version_(nullptr),
      super_version_number_(0),
      local_sv_(new ThreadLocalPtr(&SuperVersionUnrefHandle)),
@@ -362,17 +356,16 @@ ColumnFamilyData::ColumnFamilyData(
      column_family_set_(column_family_set),
      pending_flush_(false),
      pending_compaction_(false),
-      prev_compaction_needed_bytes_(0),
-      allow_2pc_(db_options.allow_2pc) {
+      prev_compaction_needed_bytes_(0) {
  Ref();

  // Convert user defined table properties collector factories to internal ones.
-  GetIntTblPropCollectorFactory(ioptions_, &int_tbl_prop_collector_factories_);
+  GetIntTblPropCollectorFactory(options_, &int_tbl_prop_collector_factories_);

  // if _dummy_versions is nullptr, then this is a dummy column family.
  if (_dummy_versions != nullptr) {
    internal_stats_.reset(
-        new InternalStats(ioptions_.num_levels, db_options.env, this));
+        new InternalStats(ioptions_.num_levels, db_options->env, this));
    table_cache_.reset(new TableCache(ioptions_, env_options, _table_cache));
    if (ioptions_.compaction_style == kCompactionStyleLevel) {
      compaction_picker_.reset(
@@ -387,27 +380,27 @@ ColumnFamilyData::ColumnFamilyData(
    } else if (ioptions_.compaction_style == kCompactionStyleNone) {
      compaction_picker_.reset(new NullCompactionPicker(
          ioptions_, &internal_comparator_));
-      ROCKS_LOG_WARN(ioptions_.info_log,
-                     "Column family %s does not use any background compaction. "
-                     "Compactions can only be done via CompactFiles\n",
-                     GetName().c_str());
+      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
+          "Column family %s does not use any background compaction. "
+          "Compactions can only be done via CompactFiles\n",
+          GetName().c_str());
 #endif  // !ROCKSDB_LITE
    } else {
-      ROCKS_LOG_ERROR(ioptions_.info_log,
-                      "Unable to recognize the specified compaction style %d. "
-                      "Column family %s will use kCompactionStyleLevel.\n",
-                      ioptions_.compaction_style, GetName().c_str());
+      Log(InfoLogLevel::ERROR_LEVEL, ioptions_.info_log,
+          "Unable to recognize the specified compaction style %d. "
+          "Column family %s will use kCompactionStyleLevel.\n",
+          ioptions_.compaction_style, GetName().c_str());
      compaction_picker_.reset(
          new LevelCompactionPicker(ioptions_, &internal_comparator_));
    }

    if (column_family_set_->NumberOfColumnFamilies() < 10) {
-      ROCKS_LOG_INFO(ioptions_.info_log,
-                     "--------------- Options for column family [%s]:\n",
-                     name.c_str());
-      initial_cf_options_.Dump(ioptions_.info_log);
+      Log(InfoLogLevel::INFO_LEVEL, ioptions_.info_log,
+          "--------------- Options for column family [%s]:\n", name.c_str());
+      options_.DumpCFOptions(ioptions_.info_log);
    } else {
-      ROCKS_LOG_INFO(ioptions_.info_log, "\t(skipping printing options)\n");
+      Log(InfoLogLevel::INFO_LEVEL, ioptions_.info_log,
+          "\t(skipping printing options)\n");
    }
  }

@@ -481,43 +474,15 @@ void ColumnFamilyData::SetDropped() {
  column_family_set_->RemoveColumnFamily(this);
 }

-ColumnFamilyOptions ColumnFamilyData::GetLatestCFOptions() const {
-  return BuildColumnFamilyOptions(initial_cf_options_, mutable_cf_options_);
-}
-
-uint64_t ColumnFamilyData::OldestLogToKeep() {
-  auto current_log = GetLogNumber();
-
-  if (allow_2pc_) {
-    auto imm_prep_log = imm()->GetMinLogContainingPrepSection();
-    auto mem_prep_log = mem()->GetMinLogContainingPrepSection();
-
-    if (imm_prep_log > 0 && imm_prep_log < current_log) {
-      current_log = imm_prep_log;
-    }
-
-    if (mem_prep_log > 0 && mem_prep_log < current_log) {
-      current_log = mem_prep_log;
-    }
-  }
-
-  return current_log;
-}
-
-const double kIncSlowdownRatio = 0.8;
-const double kDecSlowdownRatio = 1 / kIncSlowdownRatio;
-const double kNearStopSlowdownRatio = 0.6;
-const double kDelayRecoverSlowdownRatio = 1.4;
+const double kSlowdownRatio = 1.2;

 namespace {
-// If penalize_stop is true, we further reduce slowdown rate.
 std::unique_ptr<WriteControllerToken> SetupDelay(
-    WriteController* write_controller, uint64_t compaction_needed_bytes,
-    uint64_t prev_compaction_need_bytes, bool penalize_stop,
+    uint64_t max_write_rate, WriteController* write_controller,
+    uint64_t compaction_needed_bytes, uint64_t prev_compaction_neeed_bytes,
    bool auto_comapctions_disabled) {
-  const uint64_t kMinWriteRate = 16 * 1024u;  // Minimum write rate 16KB/s.
+  const uint64_t kMinWriteRate = 1024u;  // Minimum write rate 1KB/s.

-  uint64_t max_write_rate = write_controller->max_delayed_write_rate();
  uint64_t write_rate = write_controller->delayed_write_rate();

  if (auto_comapctions_disabled) {
@@ -540,32 +505,19 @@ std::unique_ptr<WriteControllerToken> SetupDelay(
    // insert to mem tables, so we need to actively slow down before we get
    // feedback signal from compaction and flushes to avoid the full stop
    // because of hitting the max write buffer number.
-    //
-    // If DB just falled into the stop condition, we need to further reduce
-    // the write rate to avoid the stop condition.
-    if (penalize_stop) {
-      // Penalize the near stop or stop condition by more agressive slowdown.
-      // This is to provide the long term slowdown increase signal.
-      // The penalty is more than the reward of recovering to the normal
-      // condition.
-      write_rate = static_cast<uint64_t>(static_cast<double>(write_rate) *
-                                         kNearStopSlowdownRatio);
+    if (prev_compaction_neeed_bytes > 0 &&
+        prev_compaction_neeed_bytes <= compaction_needed_bytes) {
+      write_rate = static_cast<uint64_t>(static_cast<double>(write_rate) /
+                                         kSlowdownRatio);
      if (write_rate < kMinWriteRate) {
        write_rate = kMinWriteRate;
      }
-    } else if (prev_compaction_need_bytes > 0 &&
-               prev_compaction_need_bytes <= compaction_needed_bytes) {
-      write_rate = static_cast<uint64_t>(static_cast<double>(write_rate) *
-                                         kIncSlowdownRatio);
-      if (write_rate < kMinWriteRate) {
-        write_rate = kMinWriteRate;
-      }
-    } else if (prev_compaction_need_bytes > compaction_needed_bytes) {
+    } else if (prev_compaction_neeed_bytes > compaction_needed_bytes) {
      // We are speeding up by ratio of kSlowdownRatio when we have paid
      // compaction debt. But we'll never speed up to faster than the write rate
      // given by users.
      write_rate = static_cast<uint64_t>(static_cast<double>(write_rate) *
-                                         kDecSlowdownRatio);
+                                         kSlowdownRatio);
      if (write_rate > max_write_rate) {
        write_rate = max_write_rate;
      }
@@ -579,31 +531,14 @@ int GetL0ThresholdSpeedupCompaction(int level0_file_num_compaction_trigger,
  // SanitizeOptions() ensures it.
  assert(level0_file_num_compaction_trigger <= level0_slowdown_writes_trigger);

-  if (level0_file_num_compaction_trigger < 0) {
-    return std::numeric_limits<int>::max();
-  }
-
-  const int64_t twice_level0_trigger =
-      static_cast<int64_t>(level0_file_num_compaction_trigger) * 2;
-
-  const int64_t one_fourth_trigger_slowdown =
-      static_cast<int64_t>(level0_file_num_compaction_trigger) +
-      ((level0_slowdown_writes_trigger - level0_file_num_compaction_trigger) /
-       4);
-
-  assert(twice_level0_trigger >= 0);
-  assert(one_fourth_trigger_slowdown >= 0);
-
  // 1/4 of the way between L0 compaction trigger threshold and slowdown
  // condition.
  // Or twice as compaction trigger, if it is smaller.
-  int64_t res = std::min(twice_level0_trigger, one_fourth_trigger_slowdown);
-  if (res >= port::kMaxInt32) {
-    return port::kMaxInt32;
-  } else {
-    // res fits in int
-    return static_cast<int>(res);
-  }
+  return std::min(level0_file_num_compaction_trigger * 2,
+                  level0_file_num_compaction_trigger +
+                      (level0_slowdown_writes_trigger -
+                       level0_file_num_compaction_trigger) /
+                          4);
 }
 }  // namespace

@@ -615,39 +550,32 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
    uint64_t compaction_needed_bytes =
        vstorage->estimated_compaction_needed_bytes();

-    bool was_stopped = write_controller->IsStopped();
-    bool needed_delay = write_controller->NeedsDelay();
-
    if (imm()->NumNotFlushed() >= mutable_cf_options.max_write_buffer_number) {
      write_controller_token_ = write_controller->GetStopToken();
      internal_stats_->AddCFStats(InternalStats::MEMTABLE_COMPACTION, 1);
-      ROCKS_LOG_WARN(
-          ioptions_.info_log,
+      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
          "[%s] Stopping writes because we have %d immutable memtables "
          "(waiting for flush), max_write_buffer_number is set to %d",
          name_.c_str(), imm()->NumNotFlushed(),
          mutable_cf_options.max_write_buffer_number);
-    } else if (!mutable_cf_options.disable_auto_compactions &&
-               vstorage->l0_delay_trigger_count() >=
-                   mutable_cf_options.level0_stop_writes_trigger) {
+    } else if (vstorage->l0_delay_trigger_count() >=
+               mutable_cf_options.level0_stop_writes_trigger) {
      write_controller_token_ = write_controller->GetStopToken();
      internal_stats_->AddCFStats(InternalStats::LEVEL0_NUM_FILES_TOTAL, 1);
      if (compaction_picker_->IsLevel0CompactionInProgress()) {
        internal_stats_->AddCFStats(
            InternalStats::LEVEL0_NUM_FILES_WITH_COMPACTION, 1);
      }
-      ROCKS_LOG_WARN(ioptions_.info_log,
-                     "[%s] Stopping writes because we have %d level-0 files",
-                     name_.c_str(), vstorage->l0_delay_trigger_count());
-    } else if (!mutable_cf_options.disable_auto_compactions &&
-               mutable_cf_options.hard_pending_compaction_bytes_limit > 0 &&
+      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
+          "[%s] Stopping writes because we have %d level-0 files",
+          name_.c_str(), vstorage->l0_delay_trigger_count());
+    } else if (mutable_cf_options.hard_pending_compaction_bytes_limit > 0 &&
               compaction_needed_bytes >=
                   mutable_cf_options.hard_pending_compaction_bytes_limit) {
      write_controller_token_ = write_controller->GetStopToken();
      internal_stats_->AddCFStats(
          InternalStats::HARD_PENDING_COMPACTION_BYTES_LIMIT, 1);
-      ROCKS_LOG_WARN(
-          ioptions_.info_log,
+      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
          "[%s] Stopping writes because of estimated pending compaction "
          "bytes %" PRIu64,
          name_.c_str(), compaction_needed_bytes);
@@ -655,105 +583,73 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
               imm()->NumNotFlushed() >=
                   mutable_cf_options.max_write_buffer_number - 1) {
      write_controller_token_ =
-          SetupDelay(write_controller, compaction_needed_bytes,
-                     prev_compaction_needed_bytes_, was_stopped,
+          SetupDelay(ioptions_.delayed_write_rate, write_controller,
+                     compaction_needed_bytes, prev_compaction_needed_bytes_,
                     mutable_cf_options.disable_auto_compactions);
      internal_stats_->AddCFStats(InternalStats::MEMTABLE_SLOWDOWN, 1);
-      ROCKS_LOG_WARN(
-          ioptions_.info_log,
+      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
          "[%s] Stalling writes because we have %d immutable memtables "
          "(waiting for flush), max_write_buffer_number is set to %d "
          "rate %" PRIu64,
          name_.c_str(), imm()->NumNotFlushed(),
          mutable_cf_options.max_write_buffer_number,
          write_controller->delayed_write_rate());
-    } else if (!mutable_cf_options.disable_auto_compactions &&
-               mutable_cf_options.level0_slowdown_writes_trigger >= 0 &&
+    } else if (mutable_cf_options.level0_slowdown_writes_trigger >= 0 &&
               vstorage->l0_delay_trigger_count() >=
                   mutable_cf_options.level0_slowdown_writes_trigger) {
-      // L0 is the last two files from stopping.
-      bool near_stop = vstorage->l0_delay_trigger_count() >=
-                       mutable_cf_options.level0_stop_writes_trigger - 2;
      write_controller_token_ =
-          SetupDelay(write_controller, compaction_needed_bytes,
-                     prev_compaction_needed_bytes_, was_stopped || near_stop,
+          SetupDelay(ioptions_.delayed_write_rate, write_controller,
+                     compaction_needed_bytes, prev_compaction_needed_bytes_,
                     mutable_cf_options.disable_auto_compactions);
      internal_stats_->AddCFStats(InternalStats::LEVEL0_SLOWDOWN_TOTAL, 1);
      if (compaction_picker_->IsLevel0CompactionInProgress()) {
        internal_stats_->AddCFStats(
            InternalStats::LEVEL0_SLOWDOWN_WITH_COMPACTION, 1);
      }
-      ROCKS_LOG_WARN(ioptions_.info_log,
-                     "[%s] Stalling writes because we have %d level-0 files "
-                     "rate %" PRIu64,
-                     name_.c_str(), vstorage->l0_delay_trigger_count(),
-                     write_controller->delayed_write_rate());
-    } else if (!mutable_cf_options.disable_auto_compactions &&
-               mutable_cf_options.soft_pending_compaction_bytes_limit > 0 &&
+      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
+          "[%s] Stalling writes because we have %d level-0 files "
+          "rate %" PRIu64,
+          name_.c_str(), vstorage->l0_delay_trigger_count(),
+          write_controller->delayed_write_rate());
+    } else if (mutable_cf_options.soft_pending_compaction_bytes_limit > 0 &&
               vstorage->estimated_compaction_needed_bytes() >=
                   mutable_cf_options.soft_pending_compaction_bytes_limit) {
-      // If the distance to hard limit is less than 1/4 of the gap between soft
-      // and
-      // hard bytes limit, we think it is near stop and speed up the slowdown.
-      bool near_stop =
-          mutable_cf_options.hard_pending_compaction_bytes_limit > 0 &&
-          (compaction_needed_bytes -
-           mutable_cf_options.soft_pending_compaction_bytes_limit) >
-              3 * (mutable_cf_options.hard_pending_compaction_bytes_limit -
-                   mutable_cf_options.soft_pending_compaction_bytes_limit) /
-                  4;
-
      write_controller_token_ =
-          SetupDelay(write_controller, compaction_needed_bytes,
-                     prev_compaction_needed_bytes_, was_stopped || near_stop,
+          SetupDelay(ioptions_.delayed_write_rate, write_controller,
+                     compaction_needed_bytes, prev_compaction_needed_bytes_,
                     mutable_cf_options.disable_auto_compactions);
      internal_stats_->AddCFStats(
          InternalStats::SOFT_PENDING_COMPACTION_BYTES_LIMIT, 1);
-      ROCKS_LOG_WARN(
-          ioptions_.info_log,
+      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
          "[%s] Stalling writes because of estimated pending compaction "
          "bytes %" PRIu64 " rate %" PRIu64,
          name_.c_str(), vstorage->estimated_compaction_needed_bytes(),
          write_controller->delayed_write_rate());
+    } else if (vstorage->l0_delay_trigger_count() >=
+               GetL0ThresholdSpeedupCompaction(
+                   mutable_cf_options.level0_file_num_compaction_trigger,
+                   mutable_cf_options.level0_slowdown_writes_trigger)) {
+      write_controller_token_ = write_controller->GetCompactionPressureToken();
+      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
+          "[%s] Increasing compaction threads because we have %d level-0 "
+          "files ",
+          name_.c_str(), vstorage->l0_delay_trigger_count());
+    } else if (vstorage->estimated_compaction_needed_bytes() >=
+               mutable_cf_options.soft_pending_compaction_bytes_limit / 4) {
+      // Increase compaction threads if bytes needed for compaction exceeds
+      // 1/4 of threshold for slowing down.
+      // If soft pending compaction byte limit is not set, always speed up
+      // compaction.
+      write_controller_token_ = write_controller->GetCompactionPressureToken();
+      if (mutable_cf_options.soft_pending_compaction_bytes_limit > 0) {
+        Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
+            "[%s] Increasing compaction threads because of estimated pending "
+            "compaction "
+            "bytes %" PRIu64,
+            name_.c_str(), vstorage->estimated_compaction_needed_bytes());
+      }
    } else {
-      if (vstorage->l0_delay_trigger_count() >=
-          GetL0ThresholdSpeedupCompaction(
-              mutable_cf_options.level0_file_num_compaction_trigger,
-              mutable_cf_options.level0_slowdown_writes_trigger)) {
-        write_controller_token_ =
-            write_controller->GetCompactionPressureToken();
-        ROCKS_LOG_WARN(
-            ioptions_.info_log,
-            "[%s] Increasing compaction threads because we have %d level-0 "
-            "files ",
-            name_.c_str(), vstorage->l0_delay_trigger_count());
-      } else if (vstorage->estimated_compaction_needed_bytes() >=
-                 mutable_cf_options.soft_pending_compaction_bytes_limit / 4) {
-        // Increase compaction threads if bytes needed for compaction exceeds
-        // 1/4 of threshold for slowing down.
-        // If soft pending compaction byte limit is not set, always speed up
-        // compaction.
-        write_controller_token_ =
-            write_controller->GetCompactionPressureToken();
-        if (mutable_cf_options.soft_pending_compaction_bytes_limit > 0) {
-          ROCKS_LOG_WARN(
-              ioptions_.info_log,
-              "[%s] Increasing compaction threads because of estimated pending "
-              "compaction "
-              "bytes %" PRIu64,
-              name_.c_str(), vstorage->estimated_compaction_needed_bytes());
-        }
-      } else {
-        write_controller_token_.reset();
-      }
-      // If the DB recovers from delay conditions, we reward with reducing
-      // double the slowdown ratio. This is to balance the long term slowdown
-      // increase signal.
-      if (needed_delay) {
-        uint64_t write_rate = write_controller->delayed_write_rate();
-        write_controller->set_delayed_write_rate(static_cast<uint64_t>(
-            static_cast<double>(write_rate) * kDelayRecoverSlowdownRatio));
-      }
+      write_controller_token_.reset();
    }
    prev_compaction_needed_bytes_ = compaction_needed_bytes;
  }
@@ -779,7 +675,7 @@ MemTable* ColumnFamilyData::ConstructNewMemtable(
    const MutableCFOptions& mutable_cf_options, SequenceNumber earliest_seq) {
  assert(current_ != nullptr);
  return new MemTable(internal_comparator_, ioptions_, mutable_cf_options,
-                      write_buffer_manager_, earliest_seq);
+                      write_buffer_, earliest_seq);
 }

 void ColumnFamilyData::CreateNewMemtable(
@@ -805,13 +701,6 @@ Compaction* ColumnFamilyData::PickCompaction(
  return result;
 }

-bool ColumnFamilyData::RangeOverlapWithCompaction(
-    const Slice& smallest_user_key, const Slice& largest_user_key,
-    int level) const {
-  return compaction_picker_->RangeOverlapWithCompaction(
-      smallest_user_key, largest_user_key, level);
-}
-
 const int ColumnFamilyData::kCompactAllLevels = -1;
 const int ColumnFamilyData::kCompactToBaseLevel = -2;

@@ -962,21 +851,21 @@ Status ColumnFamilyData::SetOptions(
 #endif  // ROCKSDB_LITE

 ColumnFamilySet::ColumnFamilySet(const std::string& dbname,
-                                 const ImmutableDBOptions* db_options,
+                                 const DBOptions* db_options,
                                 const EnvOptions& env_options,
                                 Cache* table_cache,
-                                 WriteBufferManager* write_buffer_manager,
+                                 WriteBuffer* write_buffer,
                                 WriteController* write_controller)
    : max_column_family_(0),
      dummy_cfd_(new ColumnFamilyData(0, "", nullptr, nullptr, nullptr,
-                                      ColumnFamilyOptions(), *db_options,
+                                      ColumnFamilyOptions(), db_options,
                                      env_options, nullptr)),
      default_cfd_cache_(nullptr),
      db_name_(dbname),
      db_options_(db_options),
      env_options_(env_options),
      table_cache_(table_cache),
-      write_buffer_manager_(write_buffer_manager),
+      write_buffer_(write_buffer),
      write_controller_(write_controller) {
  // initialize linked list
  dummy_cfd_->prev_ = dummy_cfd_;
@@ -1039,9 +928,10 @@ ColumnFamilyData* ColumnFamilySet::CreateColumnFamily(
    const std::string& name, uint32_t id, Version* dummy_versions,
    const ColumnFamilyOptions& options) {
  assert(column_families_.find(name) == column_families_.end());
-  ColumnFamilyData* new_cfd = new ColumnFamilyData(
-      id, name, dummy_versions, table_cache_, write_buffer_manager_, options,
-      *db_options_, env_options_, this);
+  ColumnFamilyData* new_cfd =
+      new ColumnFamilyData(id, name, dummy_versions, table_cache_,
+                           write_buffer_, options, db_options_,
+                           env_options_, this);
  column_families_.insert({name, id});
  column_family_data_.insert({id, new_cfd});
  max_column_family_ = std::max(max_column_family_, id);
@@ -1118,7 +1008,8 @@ uint32_t GetColumnFamilyID(ColumnFamilyHandle* column_family) {
 const Comparator* GetColumnFamilyUserComparator(
    ColumnFamilyHandle* column_family) {
  if (column_family != nullptr) {
-    return column_family->GetComparator();
+    auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+    return cfh->user_comparator();
  }
  return nullptr;
 }
@@ -15,15 +15,15 @@
 #include <atomic>

 #include "db/memtable_list.h"
-#include "db/table_cache.h"
-#include "db/table_properties_collector.h"
 #include "db/write_batch_internal.h"
 #include "db/write_controller.h"
-#include "options/cf_options.h"
+#include "db/table_cache.h"
+#include "db/table_properties_collector.h"
 #include "rocksdb/compaction_job_stats.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "rocksdb/options.h"
+#include "util/mutable_cf_options.h"
 #include "util/thread_local.h"

 namespace rocksdb {
@@ -42,7 +42,7 @@ class LogBuffer;
 class InstrumentedMutex;
 class InstrumentedMutexLock;

-extern const double kIncSlowdownRatio;
+extern const double kSlowdownRatio;

 // ColumnFamilyHandleImpl is the class that clients use to access different
 // column families. It has non-trivial destructor, which gets called when client
@@ -55,11 +55,11 @@ class ColumnFamilyHandleImpl : public ColumnFamilyHandle {
  // destroy without mutex
  virtual ~ColumnFamilyHandleImpl();
  virtual ColumnFamilyData* cfd() const { return cfd_; }
+  virtual const Comparator* user_comparator() const;

  virtual uint32_t GetID() const override;
  virtual const std::string& GetName() const override;
  virtual Status GetDescriptor(ColumnFamilyDescriptor* desc) override;
-  virtual const Comparator* GetComparator() const override;

 private:
  ColumnFamilyData* cfd_;
@@ -136,13 +136,14 @@ extern Status CheckCompressionSupported(const ColumnFamilyOptions& cf_options);
 extern Status CheckConcurrentWritesSupported(
    const ColumnFamilyOptions& cf_options);

-extern ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
+extern ColumnFamilyOptions SanitizeOptions(const DBOptions& db_options,
+                                           const InternalKeyComparator* icmp,
                                           const ColumnFamilyOptions& src);
 // Wrap user defined table proproties collector factories `from cf_options`
 // into internal ones in int_tbl_prop_collector_factories. Add a system internal
 // one too.
 extern void GetIntTblPropCollectorFactory(
-    const ImmutableCFOptions& ioptions,
+    const ColumnFamilyOptions& cf_options,
    std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
        int_tbl_prop_collector_factories);

@@ -200,6 +201,9 @@ class ColumnFamilyData {
  void SetLogNumber(uint64_t log_number) { log_number_ = log_number; }
  uint64_t GetLogNumber() const { return log_number_; }

+  // !!! To be deprecated! Please don't not use this function anymore!
+  const Options* options() const { return &options_; }
+
  // thread-safe
  const EnvOptions* soptions() const;
  const ImmutableCFOptions* ioptions() const { return &ioptions_; }
@@ -214,14 +218,6 @@ class ColumnFamilyData {
  const MutableCFOptions* GetLatestMutableCFOptions() const {
    return &mutable_cf_options_;
  }
-
-  // REQUIRES: DB mutex held
-  // Build ColumnFamiliesOptions with immutable options and latest mutable
-  // options.
-  ColumnFamilyOptions GetLatestCFOptions() const;
-
-  bool is_delete_range_supported() { return is_delete_range_supported_; }
-
 #ifndef ROCKSDB_LITE
  // REQUIRES: DB mutex held
  Status SetOptions(
@@ -239,9 +235,6 @@ class ColumnFamilyData {
  uint64_t GetTotalSstFilesSize() const;  // REQUIRE: DB mutex held
  void SetMemtable(MemTable* new_mem) { mem_ = new_mem; }

-  // calculate the oldest log needed for the durability of this column family
-  uint64_t OldestLogToKeep();
-
  // See Memtable constructor for explanation of earliest_seq param.
  MemTable* ConstructNewMemtable(const MutableCFOptions& mutable_cf_options,
                                 SequenceNumber earliest_seq);
@@ -256,13 +249,6 @@ class ColumnFamilyData {
  // REQUIRES: DB mutex held
  Compaction* PickCompaction(const MutableCFOptions& mutable_options,
                             LogBuffer* log_buffer);
-
-  // Check if the passed range overlap with any running compactions.
-  // REQUIRES: DB mutex held
-  bool RangeOverlapWithCompaction(const Slice& smallest_user_key,
-                                  const Slice& largest_user_key,
-                                  int level) const;
-
  // A flag to tell a manual compaction is to compact all levels together
  // instad of for specific level.
  static const int kCompactAllLevels;
@@ -337,10 +323,9 @@ class ColumnFamilyData {
  friend class ColumnFamilySet;
  ColumnFamilyData(uint32_t id, const std::string& name,
                   Version* dummy_versions, Cache* table_cache,
-                   WriteBufferManager* write_buffer_manager,
+                   WriteBuffer* write_buffer,
                   const ColumnFamilyOptions& options,
-                   const ImmutableDBOptions& db_options,
-                   const EnvOptions& env_options,
+                   const DBOptions* db_options, const EnvOptions& env_options,
                   ColumnFamilySet* column_family_set);

  uint32_t id_;
@@ -355,17 +340,15 @@ class ColumnFamilyData {
  std::vector<std::unique_ptr<IntTblPropCollectorFactory>>
      int_tbl_prop_collector_factories_;

-  const ColumnFamilyOptions initial_cf_options_;
+  const Options options_;
  const ImmutableCFOptions ioptions_;
  MutableCFOptions mutable_cf_options_;

-  const bool is_delete_range_supported_;
-
  std::unique_ptr<TableCache> table_cache_;

  std::unique_ptr<InternalStats> internal_stats_;

-  WriteBufferManager* write_buffer_manager_;
+  WriteBuffer* write_buffer_;

  MemTable* mem_;
  MemTableList imm_;
@@ -407,9 +390,6 @@ class ColumnFamilyData {
  bool pending_compaction_;

  uint64_t prev_compaction_needed_bytes_;
-
-  // if the database was opened with 2pc enabled
-  bool allow_2pc_;
 };

 // ColumnFamilySet has interesting thread-safety requirements
@@ -456,11 +436,9 @@ class ColumnFamilySet {
    ColumnFamilyData* current_;
  };

-  ColumnFamilySet(const std::string& dbname,
-                  const ImmutableDBOptions* db_options,
+  ColumnFamilySet(const std::string& dbname, const DBOptions* db_options,
                  const EnvOptions& env_options, Cache* table_cache,
-                  WriteBufferManager* write_buffer_manager,
-                  WriteController* write_controller);
+                  WriteBuffer* write_buffer, WriteController* write_controller);
  ~ColumnFamilySet();

  ColumnFamilyData* GetDefault() const;
@@ -514,10 +492,10 @@ class ColumnFamilySet {
  ColumnFamilyData* default_cfd_cache_;

  const std::string db_name_;
-  const ImmutableDBOptions* const db_options_;
+  const DBOptions* const db_options_;
  const EnvOptions env_options_;
  Cache* table_cache_;
-  WriteBufferManager* write_buffer_manager_;
+  WriteBuffer* write_buffer_;
  WriteController* write_controller_;
 };

@@ -10,8 +10,6 @@
 #include <thread>
 #include <vector>

-#include "db/db_impl.h"
-#include "port/port.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "util/string_util.h"
@@ -51,10 +49,6 @@ class FlushedFileCollector : public EventListener {
    }
    return result;
  }
-  void ClearFlushedFiles() {
-    std::lock_guard<std::mutex> lock(mutex_);
-    flushed_files_.clear();
-  }

 private:
  std::vector<std::string> flushed_files_;
@@ -121,12 +115,13 @@ TEST_F(CompactFilesTest, L0ConflictsFiles) {
 TEST_F(CompactFilesTest, ObsoleteFiles) {
  Options options;
  // to trigger compaction more easily
-  const int kWriteBufferSize = 65536;
+  const int kWriteBufferSize = 10000;
  options.create_if_missing = true;
  // Disable RocksDB background compaction.
  options.compaction_style = kCompactionStyleNone;
-  options.level0_slowdown_writes_trigger = (1 << 30);
-  options.level0_stop_writes_trigger = (1 << 30);
+  // Small slowdown and stop trigger for experimental purpose.
+  options.level0_slowdown_writes_trigger = 20;
+  options.level0_stop_writes_trigger = 20;
  options.write_buffer_size = kWriteBufferSize;
  options.max_write_buffer_number = 2;
  options.compression = kNoCompression;
@@ -149,7 +144,6 @@ TEST_F(CompactFilesTest, ObsoleteFiles) {

  auto l0_files = collector->GetFlushedFiles();
  ASSERT_OK(db->CompactFiles(CompactionOptions(), l0_files, 1));
-  reinterpret_cast<DBImpl*>(db)->TEST_WaitForCompact();

  // verify all compaction input files are deleted
  for (auto fname : l0_files) {
@@ -158,46 +152,6 @@ TEST_F(CompactFilesTest, ObsoleteFiles) {
  delete db;
 }

-TEST_F(CompactFilesTest, NotCutOutputOnLevel0) {
-  Options options;
-  options.create_if_missing = true;
-  // Disable RocksDB background compaction.
-  options.compaction_style = kCompactionStyleNone;
-  options.level0_slowdown_writes_trigger = 1000;
-  options.level0_stop_writes_trigger = 1000;
-  options.write_buffer_size = 65536;
-  options.max_write_buffer_number = 2;
-  options.compression = kNoCompression;
-  options.max_compaction_bytes = 5000;
-
-  // Add listener
-  FlushedFileCollector* collector = new FlushedFileCollector();
-  options.listeners.emplace_back(collector);
-
-  DB* db = nullptr;
-  DestroyDB(db_name_, options);
-  Status s = DB::Open(options, db_name_, &db);
-  assert(s.ok());
-  assert(db);
-
-  // create couple files
-  for (int i = 0; i < 500; ++i) {
-    db->Put(WriteOptions(), ToString(i), std::string(1000, 'a' + (i % 26)));
-  }
-  reinterpret_cast<DBImpl*>(db)->TEST_WaitForFlushMemTable();
-  auto l0_files_1 = collector->GetFlushedFiles();
-  collector->ClearFlushedFiles();
-  for (int i = 0; i < 500; ++i) {
-    db->Put(WriteOptions(), ToString(i), std::string(1000, 'a' + (i % 26)));
-  }
-  reinterpret_cast<DBImpl*>(db)->TEST_WaitForFlushMemTable();
-  auto l0_files_2 = collector->GetFlushedFiles();
-  ASSERT_OK(db->CompactFiles(CompactionOptions(), l0_files_1, 0));
-  ASSERT_OK(db->CompactFiles(CompactionOptions(), l0_files_2, 0));
-  // no assertion failure
-  delete db;
-}
-
 TEST_F(CompactFilesTest, CapturingPendingFiles) {
  Options options;
  options.create_if_missing = true;
@@ -232,7 +186,7 @@ TEST_F(CompactFilesTest, CapturingPendingFiles) {
  rocksdb::SyncPoint::GetInstance()->EnableProcessing();

  // Start compacting files.
-  rocksdb::port::Thread compaction_thread(
+  std::thread compaction_thread(
      [&] { EXPECT_OK(db->CompactFiles(CompactionOptions(), l0_files, 1)); });

  // In the meantime flush another file.
@@ -254,61 +208,6 @@ TEST_F(CompactFilesTest, CapturingPendingFiles) {
  delete db;
 }

-TEST_F(CompactFilesTest, CompactionFilterWithGetSv) {
-  class FilterWithGet : public CompactionFilter {
-   public:
-    virtual bool Filter(int level, const Slice& key, const Slice& value,
-                        std::string* new_value,
-                        bool* value_changed) const override {
-      if (db_ == nullptr) {
-        return true;
-      }
-      std::string res;
-      db_->Get(ReadOptions(), "", &res);
-      return true;
-    }
-
-    void SetDB(DB* db) {
-      db_ = db;
-    }
-
-    virtual const char* Name() const override { return "FilterWithGet"; }
-
-   private:
-    DB* db_;
-  };
-
-
-  std::shared_ptr<FilterWithGet> cf(new FilterWithGet());
-
-  Options options;
-  options.create_if_missing = true;
-  options.compaction_filter = cf.get();
-
-  DB* db = nullptr;
-  DestroyDB(db_name_, options);
-  Status s = DB::Open(options, db_name_, &db);
-  ASSERT_OK(s);
-
-  cf->SetDB(db);
-
-  // Write one L0 file
-  db->Put(WriteOptions(), "K1", "V1");
-  db->Flush(FlushOptions());
-
-  // Compact all L0 files using CompactFiles
-  rocksdb::ColumnFamilyMetaData meta;
-  db->GetColumnFamilyMetaData(&meta);
-  for (auto& file : meta.levels[0].files) {
-    std::string fname = file.db_path + "/" + file.name;
-    ASSERT_OK(
-        db->CompactFiles(rocksdb::CompactionOptions(), {fname}, 0));
-  }
-
-
-  delete db;
-}
-
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -42,11 +42,11 @@ size_t CompactedDBImpl::FindFile(const Slice& key) {
  return right;
 }

-Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*,
-                            const Slice& key, PinnableSlice* value) {
+Status CompactedDBImpl::Get(const ReadOptions& options,
+     ColumnFamilyHandle*, const Slice& key, std::string* value) {
  GetContext get_context(user_comparator_, nullptr, nullptr, nullptr,
                         GetContext::kNotFound, key, value, nullptr, nullptr,
-                         nullptr, nullptr);
+                         nullptr);
  LookupKey lkey(key, kMaxSequenceNumber);
  files_.files[FindFile(key)].fd.table_reader->Get(
      options, lkey.internal_key(), &get_context);
@@ -75,14 +75,11 @@ std::vector<Status> CompactedDBImpl::MultiGet(const ReadOptions& options,
  int idx = 0;
  for (auto* r : reader_list) {
    if (r != nullptr) {
-      PinnableSlice pinnable_val;
-      std::string& value = (*values)[idx];
      GetContext get_context(user_comparator_, nullptr, nullptr, nullptr,
-                             GetContext::kNotFound, keys[idx], &pinnable_val,
-                             nullptr, nullptr, nullptr, nullptr);
+                             GetContext::kNotFound, keys[idx], &(*values)[idx],
+                             nullptr, nullptr, nullptr);
      LookupKey lkey(keys[idx], kMaxSequenceNumber);
      r->Get(options, lkey.internal_key(), &get_context);
-      value.assign(pinnable_val.data(), pinnable_val.size());
      if (get_context.State() == GetContext::kFound) {
        statuses[idx] = Status::OK();
      }
@@ -154,9 +151,9 @@ Status CompactedDBImpl::Open(const Options& options,
  std::unique_ptr<CompactedDBImpl> db(new CompactedDBImpl(db_options, dbname));
  Status s = db->Init(options);
  if (s.ok()) {
-    ROCKS_LOG_INFO(db->immutable_db_options_.info_log,
-                   "Opened the db as fully compacted mode");
-    LogFlush(db->immutable_db_options_.info_log);
+    Log(INFO_LEVEL, db->db_options_.info_log,
+        "Opened the db as fully compacted mode");
+    LogFlush(db->db_options_.info_log);
    *dbptr = db.release();
  }
  return s;
@@ -23,7 +23,7 @@ class CompactedDBImpl : public DBImpl {
  using DB::Get;
  virtual Status Get(const ReadOptions& options,
                     ColumnFamilyHandle* column_family, const Slice& key,
-                     PinnableSlice* value) override;
+                     std::string* value) override;
  using DB::MultiGet;
  virtual std::vector<Status> MultiGet(
      const ReadOptions& options,
@@ -76,13 +76,6 @@ class CompactedDBImpl : public DBImpl {
                       ColumnFamilyHandle* column_family) override {
    return Status::NotSupported("Not supported in compacted db mode.");
  }
-  using DB::IngestExternalFile;
-  virtual Status IngestExternalFile(
-      ColumnFamilyHandle* column_family,
-      const std::vector<std::string>& external_files,
-      const IngestExternalFileOptions& ingestion_options) override {
-    return Status::NotSupported("Not supported in compacted db mode.");
-  }

 private:
  friend class DB;
@@ -16,9 +16,9 @@
 #include <inttypes.h>
 #include <vector>

-#include "db/column_family.h"
 #include "rocksdb/compaction_filter.h"
-#include "util/string_util.h"
+#include "db/column_family.h"
+#include "util/logging.h"
 #include "util/sync_point.h"

 namespace rocksdb {
@@ -140,22 +140,19 @@ bool Compaction::IsFullCompaction(
 }

 Compaction::Compaction(VersionStorageInfo* vstorage,
-                       const ImmutableCFOptions& _immutable_cf_options,
                       const MutableCFOptions& _mutable_cf_options,
                       std::vector<CompactionInputFiles> _inputs,
                       int _output_level, uint64_t _target_file_size,
-                       uint64_t _max_compaction_bytes, uint32_t _output_path_id,
-                       CompressionType _compression,
+                       uint64_t _max_grandparent_overlap_bytes,
+                       uint32_t _output_path_id, CompressionType _compression,
                       std::vector<FileMetaData*> _grandparents,
                       bool _manual_compaction, double _score,
                       bool _deletion_compaction,
                       CompactionReason _compaction_reason)
-    : input_vstorage_(vstorage),
-      start_level_(_inputs[0].level),
+    : start_level_(_inputs[0].level),
      output_level_(_output_level),
      max_output_file_size_(_target_file_size),
-      max_compaction_bytes_(_max_compaction_bytes),
-      immutable_cf_options_(_immutable_cf_options),
+      max_grandparent_overlap_bytes_(_max_grandparent_overlap_bytes),
      mutable_cf_options_(_mutable_cf_options),
      input_version_(nullptr),
      number_levels_(vstorage->num_levels()),
@@ -190,7 +187,8 @@ Compaction::Compaction(VersionStorageInfo* vstorage,
    }
  }

-  GetBoundaryKeys(vstorage, inputs_, &smallest_user_key_, &largest_user_key_);
+  Slice smallest_user_key;
+  GetBoundaryKeys(vstorage, inputs_, &smallest_user_key, &largest_user_key_);
 }

 Compaction::~Compaction() {
@@ -205,9 +203,8 @@ Compaction::~Compaction() {
 }

 bool Compaction::InputCompressionMatchesOutput() const {
-  int base_level = input_vstorage_->base_level();
-  bool matches = (GetCompressionType(immutable_cf_options_, input_vstorage_,
-                                     mutable_cf_options_, start_level_,
+  int base_level = input_version_->storage_info()->base_level();
+  bool matches = (GetCompressionType(*cfd_->ioptions(), start_level_,
                                     base_level) == output_compression_);
  if (matches) {
    TEST_SYNC_POINT("Compaction::InputCompressionMatchesOutput:Matches");
@@ -225,14 +222,15 @@ bool Compaction::IsTrivialMove() const {
  // filter to be applied to that level, and thus cannot be a trivial move.

  // Check if start level have files with overlapping ranges
-  if (start_level_ == 0 && input_vstorage_->level0_non_overlapping() == false) {
+  if (start_level_ == 0 &&
+      input_version_->storage_info()->level0_non_overlapping() == false) {
    // We cannot move files from L0 to L1 if the files are overlapping
    return false;
  }

  if (is_manual_compaction_ &&
-      (immutable_cf_options_.compaction_filter != nullptr ||
-       immutable_cf_options_.compaction_filter_factory != nullptr)) {
+      (cfd_->ioptions()->compaction_filter != nullptr ||
+       cfd_->ioptions()->compaction_filter_factory != nullptr)) {
    // This is a manual compaction and we have a compaction filter that should
    // be executed, we cannot do a trivial move
    return false;
@@ -240,34 +238,15 @@ bool Compaction::IsTrivialMove() const {

  // Used in universal compaction, where trivial move can be done if the
  // input files are non overlapping
-  if ((immutable_cf_options_.compaction_options_universal.allow_trivial_move) &&
+  if ((cfd_->ioptions()->compaction_options_universal.allow_trivial_move) &&
      (output_level_ != 0)) {
    return is_trivial_move_;
  }

-  if (!(start_level_ != output_level_ && num_input_levels() == 1 &&
+  return (start_level_ != output_level_ && num_input_levels() == 1 &&
          input(0, 0)->fd.GetPathId() == output_path_id() &&
-          InputCompressionMatchesOutput())) {
-    return false;
-  }
-
-  // assert inputs_.size() == 1
-
-  for (const auto& file : inputs_.front().files) {
-    std::vector<FileMetaData*> file_grand_parents;
-    if (output_level_ + 1 >= number_levels_) {
-      continue;
-    }
-    input_vstorage_->GetOverlappingInputs(output_level_ + 1, &file->smallest,
-                                          &file->largest, &file_grand_parents);
-    const auto compaction_size =
-        file->fd.GetFileSize() + TotalFileSize(file_grand_parents);
-    if (compaction_size > max_compaction_bytes_) {
-      return false;
-    }
-  }
-
-  return true;
+          InputCompressionMatchesOutput() &&
+          TotalFileSize(grandparents_) <= max_grandparent_overlap_bytes_);
 }

 void Compaction::AddInputDeletions(VersionEdit* out_edit) {
@@ -290,7 +269,8 @@ bool Compaction::KeyNotExistsBeyondOutputLevel(
  // Maybe use binary search to find right entry instead of linear search?
  const Comparator* user_cmp = cfd_->user_comparator();
  for (int lvl = output_level_ + 1; lvl < number_levels_; lvl++) {
-    const std::vector<FileMetaData*>& files = input_vstorage_->LevelFiles(lvl);
+    const std::vector<FileMetaData*>& files =
+        input_version_->storage_info()->LevelFiles(lvl);
    for (; level_ptrs->at(lvl) < files.size(); level_ptrs->at(lvl)++) {
      auto* f = files[level_ptrs->at(lvl)];
      if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) {
@@ -311,8 +291,8 @@ bool Compaction::KeyNotExistsBeyondOutputLevel(
 void Compaction::MarkFilesBeingCompacted(bool mark_as_compacted) {
  for (size_t i = 0; i < num_input_levels(); i++) {
    for (size_t j = 0; j < inputs_[i].size(); j++) {
-      assert(mark_as_compacted ? !inputs_[i][j]->being_compacted
-                               : inputs_[i][j]->being_compacted);
+      assert(mark_as_compacted ? !inputs_[i][j]->being_compacted :
+                                  inputs_[i][j]->being_compacted);
      inputs_[i][j]->being_compacted = mark_as_compacted;
    }
  }
@@ -362,7 +342,7 @@ void Compaction::ReleaseCompactionFiles(Status status) {

 void Compaction::ResetNextCompactionIndex() {
  assert(input_version_ != nullptr);
-  input_vstorage_->ResetNextCompactionIndex(start_level_);
+  input_version_->storage_info()->ResetNextCompactionIndex(start_level_);
 }

 namespace {
@@ -387,8 +367,10 @@ int InputSummary(const std::vector<FileMetaData*>& files, char* output,

 void Compaction::Summary(char* output, int len) {
  int write =
-      snprintf(output, len, "Base version %" PRIu64 " Base level %d, inputs: [",
-               input_version_->GetVersionNumber(), start_level_);
+      snprintf(output, len, "Base version %" PRIu64
+                            " Base level %d, inputs: [",
+               input_version_->GetVersionNumber(),
+               start_level_);
  if (write < 0 || write >= len) {
    return;
  }
@@ -413,15 +395,14 @@ void Compaction::Summary(char* output, int len) {
 uint64_t Compaction::OutputFilePreallocationSize() const {
  uint64_t preallocation_size = 0;

-  if (max_output_file_size_ != port::kMaxUint64 &&
-      (cfd_->ioptions()->compaction_style == kCompactionStyleLevel ||
-       output_level() > 0)) {
+  if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel ||
+      output_level() > 0) {
    preallocation_size = max_output_file_size_;
  } else {
-    for (const auto& level_files : inputs_) {
-      for (const auto& file : level_files.files) {
-        preallocation_size += file->fd.GetFileSize();
-      }
+    // output_level() == 0
+    assert(num_input_levels() > 0);
+    for (const auto& f : inputs_[0].files) {
+      preallocation_size += f->fd.GetFileSize();
    }
  }
  // Over-estimate slightly so we don't end up just barely crossing
@@ -447,11 +428,11 @@ bool Compaction::IsOutputLevelEmpty() const {
 }

 bool Compaction::ShouldFormSubcompactions() const {
-  if (immutable_cf_options_.max_subcompactions <= 1 || cfd_ == nullptr) {
+  if (mutable_cf_options_.max_subcompactions <= 1 || cfd_ == nullptr) {
    return false;
  }
  if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
-    return start_level_ == 0 && output_level_ > 0 && !IsOutputLevelEmpty();
+    return start_level_ == 0 && !IsOutputLevelEmpty();
  } else if (cfd_->ioptions()->compaction_style == kCompactionStyleUniversal) {
    return number_levels_ > 1 && output_level_ > 0;
  } else {
@@ -8,10 +8,10 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.

 #pragma once
-#include "db/version_set.h"
-#include "options/cf_options.h"
 #include "util/arena.h"
 #include "util/autovector.h"
+#include "util/mutable_cf_options.h"
+#include "db/version_set.h"

 namespace rocksdb {

@@ -35,10 +35,9 @@ class CompactionFilter;
 class Compaction {
 public:
  Compaction(VersionStorageInfo* input_version,
-             const ImmutableCFOptions& immutable_cf_options,
             const MutableCFOptions& mutable_cf_options,
             std::vector<CompactionInputFiles> inputs, int output_level,
-             uint64_t target_file_size, uint64_t max_compaction_bytes,
+             uint64_t target_file_size, uint64_t max_grandparent_overlap_bytes,
             uint32_t output_path_id, CompressionType compression,
             std::vector<FileMetaData*> grandparents,
             bool manual_compaction = false, double score = -1,
@@ -104,8 +103,6 @@ class Compaction {
    return &inputs_[compaction_input_level].files;
  }

-  const std::vector<CompactionInputFiles>* inputs() { return &inputs_; }
-
  // Returns the LevelFilesBrief of the specified compaction input level.
  const LevelFilesBrief* input_levels(size_t compaction_input_level) const {
    return &input_levels_[compaction_input_level];
@@ -174,12 +171,6 @@ class Compaction {
  // How many total levels are there?
  int number_levels() const { return number_levels_; }

-  // Return the ImmutableCFOptions that should be used throughout the compaction
-  // procedure
-  const ImmutableCFOptions* immutable_cf_options() const {
-    return &immutable_cf_options_;
-  }
-
  // Return the MutableCFOptions that should be used throughout the compaction
  // procedure
  const MutableCFOptions* mutable_cf_options() const {
@@ -228,8 +219,6 @@ class Compaction {
    output_table_properties_ = std::move(tp);
  }

-  Slice GetSmallestUserKey() const { return smallest_user_key_; }
-
  Slice GetLargestUserKey() const { return largest_user_key_; }

  CompactionReason compaction_reason() { return compaction_reason_; }
@@ -238,7 +227,9 @@ class Compaction {
    return grandparents_;
  }

-  uint64_t max_compaction_bytes() const { return max_compaction_bytes_; }
+  uint64_t max_grandparent_overlap_bytes() const {
+    return max_grandparent_overlap_bytes_;
+  }

 private:
  // mark (or clear) all files that are being compacted
@@ -258,14 +249,11 @@ class Compaction {
  static bool IsFullCompaction(VersionStorageInfo* vstorage,
                               const std::vector<CompactionInputFiles>& inputs);

-  VersionStorageInfo* input_vstorage_;
-
  const int start_level_;    // the lowest level to be compacted
  const int output_level_;  // levels to which output files are stored
  uint64_t max_output_file_size_;
-  uint64_t max_compaction_bytes_;
-  const ImmutableCFOptions immutable_cf_options_;
-  const MutableCFOptions mutable_cf_options_;
+  uint64_t max_grandparent_overlap_bytes_;
+  MutableCFOptions mutable_cf_options_;
  Version* input_version_;
  VersionEdit edit_;
  const int number_levels_;
@@ -306,9 +294,6 @@ class Compaction {
  // table properties of output files
  TablePropertiesCollection output_table_properties_;

-  // smallest user keys in compaction
-  Slice smallest_user_key_;
-
  // largest user keys in compaction
  Slice largest_user_key_;

@@ -1,33 +0,0 @@
-//  Copyright (c) 2016-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-
-#pragma once
-
-struct CompactionIterationStats {
-  // Compaction statistics
-
-  // Doesn't include records skipped because of
-  // CompactionFilter::Decision::kRemoveAndSkipUntil.
-  int64_t num_record_drop_user = 0;
-
-  int64_t num_record_drop_hidden = 0;
-  int64_t num_record_drop_obsolete = 0;
-  int64_t num_record_drop_range_del = 0;
-  int64_t num_range_del_drop_obsolete = 0;
-  uint64_t total_filter_time = 0;
-
-  // Input statistics
-  // TODO(noetzli): The stats are incomplete. They are lacking everything
-  // consumed by MergeHelper.
-  uint64_t num_input_records = 0;
-  uint64_t num_input_deletion_records = 0;
-  uint64_t num_input_corrupt_records = 0;
-  uint64_t total_input_raw_key_bytes = 0;
-  uint64_t total_input_raw_value_bytes = 0;
-
-  // Single-Delete diagnostics for exceptional situations
-  uint64_t num_single_del_fallthru = 0;
-  uint64_t num_single_del_mismatch = 0;
-};
@@ -10,54 +10,12 @@

 namespace rocksdb {

-#ifndef ROCKSDB_LITE
-CompactionEventListener::CompactionListenerValueType fromInternalValueType(
-    ValueType vt) {
-  switch (vt) {
-    case kTypeDeletion:
-      return CompactionEventListener::CompactionListenerValueType::kDelete;
-    case kTypeValue:
-      return CompactionEventListener::CompactionListenerValueType::kValue;
-    case kTypeMerge:
-      return CompactionEventListener::CompactionListenerValueType::
-          kMergeOperand;
-    case kTypeSingleDeletion:
-      return CompactionEventListener::CompactionListenerValueType::
-          kSingleDelete;
-    case kTypeRangeDeletion:
-      return CompactionEventListener::CompactionListenerValueType::kRangeDelete;
-    default:
-      assert(false);
-      return CompactionEventListener::CompactionListenerValueType::kInvalid;
-  }
-}
-#endif  // ROCKSDB_LITE
-
 CompactionIterator::CompactionIterator(
    InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
    SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
    SequenceNumber earliest_write_conflict_snapshot, Env* env,
-    bool expect_valid_internal_key, RangeDelAggregator* range_del_agg,
-    const Compaction* compaction, const CompactionFilter* compaction_filter,
-    CompactionEventListener* compaction_listener,
-    const std::atomic<bool>* shutting_down)
-    : CompactionIterator(
-          input, cmp, merge_helper, last_sequence, snapshots,
-          earliest_write_conflict_snapshot, env, expect_valid_internal_key,
-          range_del_agg,
-          std::unique_ptr<CompactionProxy>(
-              compaction ? new CompactionProxy(compaction) : nullptr),
-          compaction_filter, compaction_listener, shutting_down) {}
-
-CompactionIterator::CompactionIterator(
-    InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
-    SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
-    SequenceNumber earliest_write_conflict_snapshot, Env* env,
-    bool expect_valid_internal_key, RangeDelAggregator* range_del_agg,
-    std::unique_ptr<CompactionProxy> compaction,
-    const CompactionFilter* compaction_filter,
-    CompactionEventListener* compaction_listener,
-    const std::atomic<bool>* shutting_down)
+    bool expect_valid_internal_key, const Compaction* compaction,
+    const CompactionFilter* compaction_filter, LogBuffer* log_buffer)
    : input_(input),
      cmp_(cmp),
      merge_helper_(merge_helper),
@@ -65,12 +23,9 @@ CompactionIterator::CompactionIterator(
      earliest_write_conflict_snapshot_(earliest_write_conflict_snapshot),
      env_(env),
      expect_valid_internal_key_(expect_valid_internal_key),
-      range_del_agg_(range_del_agg),
-      compaction_(std::move(compaction)),
+      compaction_(compaction),
      compaction_filter_(compaction_filter),
-      compaction_listener_(compaction_listener),
-      shutting_down_(shutting_down),
-      ignore_snapshots_(false),
+      log_buffer_(log_buffer),
      merge_out_iter_(merge_helper_) {
  assert(compaction_filter_ == nullptr || compaction_ != nullptr);
  bottommost_level_ =
@@ -81,35 +36,25 @@ CompactionIterator::CompactionIterator(

  if (snapshots_->size() == 0) {
    // optimize for fast path if there are no snapshots
-    visible_at_tip_ = true;
-    earliest_snapshot_ = last_sequence;
+    visible_at_tip_ = last_sequence;
+    earliest_snapshot_ = visible_at_tip_;
    latest_snapshot_ = 0;
  } else {
-    visible_at_tip_ = false;
+    visible_at_tip_ = 0;
    earliest_snapshot_ = snapshots_->at(0);
    latest_snapshot_ = snapshots_->back();
  }
-  if (compaction_filter_ != nullptr) {
-    if (compaction_filter_->IgnoreSnapshots()) {
-      ignore_snapshots_ = true;
-    }
+  if (compaction_filter_ != nullptr && compaction_filter_->IgnoreSnapshots()) {
+    ignore_snapshots_ = true;
  } else {
    ignore_snapshots_ = false;
  }
-  input_->SetPinnedItersMgr(&pinned_iters_mgr_);
-}
-
-CompactionIterator::~CompactionIterator() {
-  // input_ Iteartor lifetime is longer than pinned_iters_mgr_ lifetime
-  input_->SetPinnedItersMgr(nullptr);
 }

 void CompactionIterator::ResetRecordCounts() {
  iter_stats_.num_record_drop_user = 0;
  iter_stats_.num_record_drop_hidden = 0;
  iter_stats_.num_record_drop_obsolete = 0;
-  iter_stats_.num_record_drop_range_del = 0;
-  iter_stats_.num_range_del_drop_obsolete = 0;
 }

 void CompactionIterator::SeekToFirst() {
@@ -134,12 +79,10 @@ void CompactionIterator::Next() {
      assert(valid_key);
      // Keep current_key_ in sync.
      current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type);
-      key_ = current_key_.GetInternalKey();
+      key_ = current_key_.GetKey();
      ikey_.user_key = current_key_.GetUserKey();
      valid_ = true;
    } else {
-      // We consumed all pinned merge operands, release pinned iterators
-      pinned_iters_mgr_.ReleasePinnedData();
      // MergeHelper moves the iterator to the first record after the merged
      // records, so even though we reached the end of the merge output, we do
      // not want to advance the iterator.
@@ -155,7 +98,7 @@ void CompactionIterator::Next() {
  }

  if (valid_) {
-    // Record that we've outputted a record for the current key.
+    // Record that we've ouputted a record for the current key.
    has_outputted_key_ = true;
  }

@@ -166,7 +109,7 @@ void CompactionIterator::NextFromInput() {
  at_next_ = false;
  valid_ = false;

-  while (!valid_ && input_->Valid() && !IsShuttingDown()) {
+  while (!valid_ && input_->Valid()) {
    key_ = input_->key();
    value_ = input_->value();
    iter_stats_.num_input_records++;
@@ -180,7 +123,7 @@ void CompactionIterator::NextFromInput() {
        status_ = Status::Corruption("Corrupted internal key not expected.");
        break;
      }
-      key_ = current_key_.SetInternalKey(key_);
+      key_ = current_key_.SetKey(key_);
      has_current_user_key_ = false;
      current_user_key_sequence_ = kMaxSequenceNumber;
      current_user_key_snapshot_ = 0;
@@ -196,35 +139,19 @@ void CompactionIterator::NextFromInput() {
    iter_stats_.total_input_raw_key_bytes += key_.size();
    iter_stats_.total_input_raw_value_bytes += value_.size();

-    // If need_skip is true, we should seek the input iterator
-    // to internal key skip_until and continue from there.
-    bool need_skip = false;
-    // Points either into compaction_filter_skip_until_ or into
-    // merge_helper_->compaction_filter_skip_until_.
-    Slice skip_until;
-
    // Check whether the user key changed. After this if statement current_key_
    // is a copy of the current input key (maybe converted to a delete by the
    // compaction filter). ikey_.user_key is pointing to the copy.
    if (!has_current_user_key_ ||
        !cmp_->Equal(ikey_.user_key, current_user_key_)) {
      // First occurrence of this user key
-      // Copy key for output
-      key_ = current_key_.SetInternalKey(key_, &ikey_);
+      key_ = current_key_.SetKey(key_, &ikey_);
      current_user_key_ = ikey_.user_key;
      has_current_user_key_ = true;
      has_outputted_key_ = false;
      current_user_key_sequence_ = kMaxSequenceNumber;
      current_user_key_snapshot_ = 0;

-#ifndef ROCKSDB_LITE
-      if (compaction_listener_) {
-        compaction_listener_->OnCompaction(compaction_->level(), ikey_.user_key,
-                                           fromInternalValueType(ikey_.type),
-                                           value_, ikey_.sequence, true);
-      }
-#endif  // ROCKSDB_LITE
-
      // apply the compaction filter to the first occurrence of the user key
      if (compaction_filter_ != nullptr && ikey_.type == kTypeValue &&
          (visible_at_tip_ || ikey_.sequence > latest_snapshot_ ||
@@ -233,60 +160,36 @@ void CompactionIterator::NextFromInput() {
        // number is greater than any external snapshot, then invoke the
        // filter. If the return value of the compaction filter is true,
        // replace the entry with a deletion marker.
-        CompactionFilter::Decision filter;
+        bool value_changed = false;
+        bool to_delete = false;
        compaction_filter_value_.clear();
-        compaction_filter_skip_until_.Clear();
        {
          StopWatchNano timer(env_, true);
-          filter = compaction_filter_->FilterV2(
-              compaction_->level(), ikey_.user_key,
-              CompactionFilter::ValueType::kValue, value_,
-              &compaction_filter_value_, compaction_filter_skip_until_.rep());
+          to_delete = compaction_filter_->Filter(
+              compaction_->level(), ikey_.user_key, value_,
+              &compaction_filter_value_, &value_changed);
          iter_stats_.total_filter_time +=
              env_ != nullptr ? timer.ElapsedNanos() : 0;
        }
-
-        if (filter == CompactionFilter::Decision::kRemoveAndSkipUntil &&
-            cmp_->Compare(*compaction_filter_skip_until_.rep(),
-                          ikey_.user_key) <= 0) {
-          // Can't skip to a key smaller than the current one.
-          // Keep the key as per FilterV2 documentation.
-          filter = CompactionFilter::Decision::kKeep;
-        }
-
-        if (filter == CompactionFilter::Decision::kRemove) {
-          // convert the current key to a delete; key_ is pointing into
-          // current_key_ at this point, so updating current_key_ updates key()
+        if (to_delete) {
+          // convert the current key to a delete
          ikey_.type = kTypeDeletion;
          current_key_.UpdateInternalKey(ikey_.sequence, kTypeDeletion);
          // no value associated with delete
          value_.clear();
          iter_stats_.num_record_drop_user++;
-        } else if (filter == CompactionFilter::Decision::kChangeValue) {
+        } else if (value_changed) {
          value_ = compaction_filter_value_;
-        } else if (filter == CompactionFilter::Decision::kRemoveAndSkipUntil) {
-          need_skip = true;
-          compaction_filter_skip_until_.ConvertFromUserKey(kMaxSequenceNumber,
-                                                           kValueTypeForSeek);
-          skip_until = compaction_filter_skip_until_.Encode();
        }
      }
    } else {
-#ifndef ROCKSDB_LITE
-      if (compaction_listener_) {
-        compaction_listener_->OnCompaction(compaction_->level(), ikey_.user_key,
-                                           fromInternalValueType(ikey_.type),
-                                           value_, ikey_.sequence, false);
-      }
-#endif  // ROCKSDB_LITE
-
      // Update the current key to reflect the new sequence number/type without
      // copying the user key.
      // TODO(rven): Compaction filter does not process keys in this path
      // Need to have the compaction filter process multiple versions
      // if we have versions on both sides of a snapshot
      current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type);
-      key_ = current_key_.GetInternalKey();
+      key_ = current_key_.GetKey();
      ikey_.user_key = current_key_.GetUserKey();
    }

@@ -299,13 +202,10 @@ void CompactionIterator::NextFromInput() {
    SequenceNumber last_snapshot = current_user_key_snapshot_;
    SequenceNumber prev_snapshot = 0;  // 0 means no previous snapshot
    current_user_key_snapshot_ =
-        visible_at_tip_
-            ? earliest_snapshot_
-            : findEarliestVisibleSnapshot(ikey_.sequence, &prev_snapshot);
+        visible_at_tip_ ? visible_at_tip_ : findEarliestVisibleSnapshot(
+                                                ikey_.sequence, &prev_snapshot);

-    if (need_skip) {
-      // This case is handled below.
-    } else if (clear_and_output_next_key_) {
+    if (clear_and_output_next_key_) {
      // In the previous iteration we encountered a single delete that we could
      // not compact out.  We will keep this Put, but can drop it's data.
      // (See Optimization 3, below.)
@@ -346,7 +246,6 @@ void CompactionIterator::NextFromInput() {
      // those operations for a given key is documented as being undefined.  So
      // we can choose how to handle such a combinations of operations.  We will
      // try to compact out as much as we can in these cases.
-      // We will report counts on these anomalous cases.

      // The easiest way to process a SingleDelete during iteration is to peek
      // ahead at the next key.
@@ -369,7 +268,6 @@ void CompactionIterator::NextFromInput() {
            // First SingleDelete has been skipped since we already called
            // input_->Next().
            ++iter_stats_.num_record_drop_obsolete;
-            ++iter_stats_.num_single_del_mismatch;
          } else if ((ikey_.sequence <= earliest_write_conflict_snapshot_) ||
                     has_outputted_key_) {
            // Found a matching value, we can drop the single delete and the
@@ -379,12 +277,7 @@ void CompactionIterator::NextFromInput() {

            // Note: it doesn't matter whether the second key is a Put or if it
            // is an unexpected Merge or Delete.  We will compact it out
-            // either way. We will maintain counts of how many mismatches
-            // happened
-            if (next_ikey.type != kTypeValue) {
-              ++iter_stats_.num_single_del_mismatch;
-            }
-
+            // either way.
            ++iter_stats_.num_record_drop_hidden;
            ++iter_stats_.num_record_drop_obsolete;
            // Already called input_->Next() once.  Call it a second time to
@@ -396,6 +289,7 @@ void CompactionIterator::NextFromInput() {
            // to know that a write happened in this snapshot (Rule 2 above).
            // Clear the value and output the SingleDelete. (The value will be
            // outputted on the next iteration.)
+            ++iter_stats_.num_record_drop_hidden;

            // Setting valid_ to true will output the current SingleDelete
            valid_ = true;
@@ -411,7 +305,7 @@ void CompactionIterator::NextFromInput() {
        }
      } else {
        // We are at the end of the input, could not parse the next key, or hit
-        // a different key. The iterator returns the single delete if the key
+        // the next key. The iterator returns the single delete if the key
        // possibly exists beyond the current output level.  We set
        // has_current_user_key to false so that if the iterator is at the next
        // key, we do not compare it again against the previous key at the next
@@ -424,7 +318,6 @@ void CompactionIterator::NextFromInput() {
          // Key doesn't exist outside of this range.
          // Can compact out this SingleDelete.
          ++iter_stats_.num_record_drop_obsolete;
-          ++iter_stats_.num_single_del_fallthru;
        } else {
          // Output SingleDelete
          valid_ = true;
@@ -439,7 +332,7 @@ void CompactionIterator::NextFromInput() {
      // is the same as the visibility of a previous instance of the
      // same key, then this kv is not visible in any snapshot.
      // Hidden by an newer entry for same user key
-      // TODO(noetzli): why not > ?
+      // TODO: why not > ?
      //
      // Note: Dropping this key will not affect TransactionDB write-conflict
      // checking since there has already been a record returned for this key
@@ -469,24 +362,20 @@ void CompactionIterator::NextFromInput() {
      input_->Next();
    } else if (ikey_.type == kTypeMerge) {
      if (!merge_helper_->HasOperator()) {
+        LogToBuffer(log_buffer_, "Options::merge_operator is null.");
        status_ = Status::InvalidArgument(
            "merge_operator is not properly initialized.");
        return;
      }

-      pinned_iters_mgr_.StartPinning();
      // We know the merge type entry is not hidden, otherwise we would
      // have hit (A)
      // We encapsulate the merge related state machine in a different
      // object to minimize change to the existing flow.
-      Status s = merge_helper_->MergeUntil(input_, range_del_agg_,
-                                           prev_snapshot, bottommost_level_);
+      merge_helper_->MergeUntil(input_, prev_snapshot, bottommost_level_);
      merge_out_iter_.SeekToFirst();

-      if (!s.ok() && !s.IsMergeInProgress()) {
-        status_ = s;
-        return;
-      } else if (merge_out_iter_.Valid()) {
+      if (merge_out_iter_.Valid()) {
        // NOTE: key, value, and ikey_ refer to old entries.
        //       These will be correctly set below.
        key_ = merge_out_iter_.key();
@@ -498,7 +387,7 @@ void CompactionIterator::NextFromInput() {
        assert(valid_key);
        // Keep current_key_ in sync.
        current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type);
-        key_ = current_key_.GetInternalKey();
+        key_ = current_key_.GetKey();
        ikey_.user_key = current_key_.GetUserKey();
        valid_ = true;
      } else {
@@ -506,33 +395,10 @@ void CompactionIterator::NextFromInput() {
        // batch consumed by the merge operator should not shadow any keys
        // coming after the merges
        has_current_user_key_ = false;
-        pinned_iters_mgr_.ReleasePinnedData();
-
-        if (merge_helper_->FilteredUntil(&skip_until)) {
-          need_skip = true;
-        }
      }
    } else {
-      // 1. new user key -OR-
-      // 2. different snapshot stripe
-      bool should_delete = range_del_agg_->ShouldDelete(
-          key_, RangeDelAggregator::RangePositioningMode::kForwardTraversal);
-      if (should_delete) {
-        ++iter_stats_.num_record_drop_hidden;
-        ++iter_stats_.num_record_drop_range_del;
-        input_->Next();
-      } else {
-        valid_ = true;
-      }
+      valid_ = true;
    }
-
-    if (need_skip) {
-      input_->Seek(skip_until);
-    }
-  }
-
-  if (!valid_ && IsShuttingDown()) {
-    status_ = Status::ShutdownInProgress();
  }
 }

@@ -12,72 +12,39 @@
 #include <vector>

 #include "db/compaction.h"
-#include "db/compaction_iteration_stats.h"
 #include "db/merge_helper.h"
-#include "db/pinned_iterators_manager.h"
-#include "db/range_del_aggregator.h"
 #include "rocksdb/compaction_filter.h"
+#include "util/log_buffer.h"

 namespace rocksdb {

-class CompactionEventListener;
+struct CompactionIteratorStats {
+  // Compaction statistics
+  int64_t num_record_drop_user = 0;
+  int64_t num_record_drop_hidden = 0;
+  int64_t num_record_drop_obsolete = 0;
+  uint64_t total_filter_time = 0;
+
+  // Input statistics
+  // TODO(noetzli): The stats are incomplete. They are lacking everything
+  // consumed by MergeHelper.
+  uint64_t num_input_records = 0;
+  uint64_t num_input_deletion_records = 0;
+  uint64_t num_input_corrupt_records = 0;
+  uint64_t total_input_raw_key_bytes = 0;
+  uint64_t total_input_raw_value_bytes = 0;
+};

 class CompactionIterator {
 public:
-  // A wrapper around Compaction. Has a much smaller interface, only what
-  // CompactionIterator uses. Tests can override it.
-  class CompactionProxy {
-   public:
-    explicit CompactionProxy(const Compaction* compaction)
-        : compaction_(compaction) {}
-
-    virtual ~CompactionProxy() = default;
-    virtual int level(size_t compaction_input_level = 0) const {
-      return compaction_->level();
-    }
-    virtual bool KeyNotExistsBeyondOutputLevel(
-        const Slice& user_key, std::vector<size_t>* level_ptrs) const {
-      return compaction_->KeyNotExistsBeyondOutputLevel(user_key, level_ptrs);
-    }
-    virtual bool bottommost_level() const {
-      return compaction_->bottommost_level();
-    }
-    virtual int number_levels() const { return compaction_->number_levels(); }
-    virtual Slice GetLargestUserKey() const {
-      return compaction_->GetLargestUserKey();
-    }
-
-   protected:
-    CompactionProxy() = default;
-
-   private:
-    const Compaction* compaction_;
-  };
-
  CompactionIterator(InternalIterator* input, const Comparator* cmp,
                     MergeHelper* merge_helper, SequenceNumber last_sequence,
                     std::vector<SequenceNumber>* snapshots,
                     SequenceNumber earliest_write_conflict_snapshot, Env* env,
                     bool expect_valid_internal_key,
-                     RangeDelAggregator* range_del_agg,
                     const Compaction* compaction = nullptr,
                     const CompactionFilter* compaction_filter = nullptr,
-                     CompactionEventListener* compaction_listener = nullptr,
-                     const std::atomic<bool>* shutting_down = nullptr);
-
-  // Constructor with custom CompactionProxy, used for tests.
-  CompactionIterator(InternalIterator* input, const Comparator* cmp,
-                     MergeHelper* merge_helper, SequenceNumber last_sequence,
-                     std::vector<SequenceNumber>* snapshots,
-                     SequenceNumber earliest_write_conflict_snapshot, Env* env,
-                     bool expect_valid_internal_key,
-                     RangeDelAggregator* range_del_agg,
-                     std::unique_ptr<CompactionProxy> compaction,
-                     const CompactionFilter* compaction_filter = nullptr,
-                     CompactionEventListener* compaction_listener = nullptr,
-                     const std::atomic<bool>* shutting_down = nullptr);
-
-  ~CompactionIterator();
+                     LogBuffer* log_buffer = nullptr);

  void ResetRecordCounts();

@@ -98,7 +65,7 @@ class CompactionIterator {
  const ParsedInternalKey& ikey() const { return ikey_; }
  bool Valid() const { return valid_; }
  const Slice& user_key() const { return current_user_key_; }
-  const CompactionIterationStats& iter_stats() const { return iter_stats_; }
+  const CompactionIteratorStats& iter_stats() const { return iter_stats_; }

 private:
  // Processes the input stream to find the next output
@@ -125,14 +92,12 @@ class CompactionIterator {
  const SequenceNumber earliest_write_conflict_snapshot_;
  Env* env_;
  bool expect_valid_internal_key_;
-  RangeDelAggregator* range_del_agg_;
-  std::unique_ptr<CompactionProxy> compaction_;
+  const Compaction* compaction_;
  const CompactionFilter* compaction_filter_;
-  CompactionEventListener* compaction_listener_;
-  const std::atomic<bool>* shutting_down_;
+  LogBuffer* log_buffer_;
  bool bottommost_level_;
  bool valid_ = false;
-  bool visible_at_tip_;
+  SequenceNumber visible_at_tip_;
  SequenceNumber earliest_snapshot_;
  SequenceNumber latest_snapshot_;
  bool ignore_snapshots_;
@@ -171,11 +136,7 @@ class CompactionIterator {
  bool clear_and_output_next_key_ = false;

  MergeOutputIterator merge_out_iter_;
-  // PinnedIteratorsManager used to pin input_ Iterator blocks while reading
-  // merge operands and then releasing them after consuming them.
-  PinnedIteratorsManager pinned_iters_mgr_;
  std::string compaction_filter_value_;
-  InternalKey compaction_filter_skip_until_;
  // "level_ptrs" holds indices that remember which file of an associated
  // level we were last checking during the last call to compaction->
  // KeyNotExistsBeyondOutputLevel(). This allows future calls to the function
@@ -183,11 +144,6 @@ class CompactionIterator {
  // increasing so a later call to the function must be looking for a key that
  // is in or beyond the last file checked during the previous call
  std::vector<size_t> level_ptrs_;
-  CompactionIterationStats iter_stats_;
-
-  bool IsShuttingDown() {
-    // This is a best-effort facility, so memory_order_relaxed is sufficient.
-    return shutting_down_ && shutting_down_->load(std::memory_order_relaxed);
-  }
+  CompactionIteratorStats iter_stats_;
 };
 }  // namespace rocksdb
@@ -4,213 +4,40 @@
 //  of patent rights can be found in the PATENTS file in the same directory.

 #include "db/compaction_iterator.h"
-
-#include <string>
-#include <vector>
-
-#include "port/port.h"
 #include "util/testharness.h"
 #include "util/testutil.h"

 namespace rocksdb {

-// Expects no merging attempts.
-class NoMergingMergeOp : public MergeOperator {
- public:
-  bool FullMergeV2(const MergeOperationInput& merge_in,
-                   MergeOperationOutput* merge_out) const override {
-    ADD_FAILURE();
-    return false;
-  }
-  bool PartialMergeMulti(const Slice& key,
-                         const std::deque<Slice>& operand_list,
-                         std::string* new_value,
-                         Logger* logger) const override {
-    ADD_FAILURE();
-    return false;
-  }
-  const char* Name() const override {
-    return "CompactionIteratorTest NoMergingMergeOp";
-  }
-};
-
-// Compaction filter that gets stuck when it sees a particular key,
-// then gets unstuck when told to.
-// Always returns Decition::kRemove.
-class StallingFilter : public CompactionFilter {
- public:
-  virtual Decision FilterV2(int level, const Slice& key, ValueType t,
-                            const Slice& existing_value, std::string* new_value,
-                            std::string* skip_until) const override {
-    int k = std::atoi(key.ToString().c_str());
-    last_seen.store(k);
-    while (k >= stall_at.load()) {
-      std::this_thread::yield();
-    }
-    return Decision::kRemove;
-  }
-
-  const char* Name() const override {
-    return "CompactionIteratorTest StallingFilter";
-  }
-
-  // Wait until the filter sees a key >= k and stalls at that key.
-  // If `exact`, asserts that the seen key is equal to k.
-  void WaitForStall(int k, bool exact = true) {
-    stall_at.store(k);
-    while (last_seen.load() < k) {
-      std::this_thread::yield();
-    }
-    if (exact) {
-      EXPECT_EQ(k, last_seen.load());
-    }
-  }
-
-  // Filter will stall on key >= stall_at. Advance stall_at to unstall.
-  mutable std::atomic<int> stall_at{0};
-  // Last key the filter was called with.
-  mutable std::atomic<int> last_seen{0};
-};
-
-class LoggingForwardVectorIterator : public InternalIterator {
- public:
-  struct Action {
-    enum class Type {
-      SEEK_TO_FIRST,
-      SEEK,
-      NEXT,
-    };
-
-    Type type;
-    std::string arg;
-
-    explicit Action(Type _type, std::string _arg = "")
-        : type(_type), arg(_arg) {}
-
-    bool operator==(const Action& rhs) const {
-      return std::tie(type, arg) == std::tie(rhs.type, rhs.arg);
-    }
-  };
-
-  LoggingForwardVectorIterator(const std::vector<std::string>& keys,
-                               const std::vector<std::string>& values)
-      : keys_(keys), values_(values), current_(keys.size()) {
-    assert(keys_.size() == values_.size());
-  }
-
-  virtual bool Valid() const override { return current_ < keys_.size(); }
-
-  virtual void SeekToFirst() override {
-    log.emplace_back(Action::Type::SEEK_TO_FIRST);
-    current_ = 0;
-  }
-  virtual void SeekToLast() override { assert(false); }
-
-  virtual void Seek(const Slice& target) override {
-    log.emplace_back(Action::Type::SEEK, target.ToString());
-    current_ = std::lower_bound(keys_.begin(), keys_.end(), target.ToString()) -
-               keys_.begin();
-  }
-
-  virtual void SeekForPrev(const Slice& target) override { assert(false); }
-
-  virtual void Next() override {
-    assert(Valid());
-    log.emplace_back(Action::Type::NEXT);
-    current_++;
-  }
-  virtual void Prev() override { assert(false); }
-
-  virtual Slice key() const override {
-    assert(Valid());
-    return Slice(keys_[current_]);
-  }
-  virtual Slice value() const override {
-    assert(Valid());
-    return Slice(values_[current_]);
-  }
-
-  virtual Status status() const override { return Status::OK(); }
-
-  std::vector<Action> log;
-
- private:
-  std::vector<std::string> keys_;
-  std::vector<std::string> values_;
-  size_t current_;
-};
-
-class FakeCompaction : public CompactionIterator::CompactionProxy {
- public:
-  FakeCompaction() = default;
-
-  virtual int level(size_t compaction_input_level) const { return 0; }
-  virtual bool KeyNotExistsBeyondOutputLevel(
-      const Slice& user_key, std::vector<size_t>* level_ptrs) const {
-    return key_not_exists_beyond_output_level;
-  }
-  virtual bool bottommost_level() const { return false; }
-  virtual int number_levels() const { return 1; }
-  virtual Slice GetLargestUserKey() const {
-    return "\xff\xff\xff\xff\xff\xff\xff\xff\xff";
-  }
-
-  bool key_not_exists_beyond_output_level = false;
-};
-
 class CompactionIteratorTest : public testing::Test {
 public:
-  CompactionIteratorTest()
-      : cmp_(BytewiseComparator()), icmp_(cmp_), snapshots_({}) {}
+  CompactionIteratorTest() : cmp_(BytewiseComparator()), snapshots_({}) {}

-  void InitIterators(const std::vector<std::string>& ks,
-                     const std::vector<std::string>& vs,
-                     const std::vector<std::string>& range_del_ks,
-                     const std::vector<std::string>& range_del_vs,
-                     SequenceNumber last_sequence,
-                     MergeOperator* merge_op = nullptr,
-                     CompactionFilter* filter = nullptr) {
-    std::unique_ptr<InternalIterator> range_del_iter(
-        new test::VectorIterator(range_del_ks, range_del_vs));
-    range_del_agg_.reset(new RangeDelAggregator(icmp_, snapshots_));
-    ASSERT_OK(range_del_agg_->AddTombstones(std::move(range_del_iter)));
-
-    std::unique_ptr<CompactionIterator::CompactionProxy> compaction;
-    if (filter) {
-      compaction_proxy_ = new FakeCompaction();
-      compaction.reset(compaction_proxy_);
-    }
-
-    merge_helper_.reset(new MergeHelper(Env::Default(), cmp_, merge_op, filter,
-                                        nullptr, false, 0, 0, nullptr,
-                                        &shutting_down_));
-    iter_.reset(new LoggingForwardVectorIterator(ks, vs));
+  void InitIterator(const std::vector<std::string>& ks,
+                    const std::vector<std::string>& vs,
+                    SequenceNumber last_sequence) {
+    merge_helper_.reset(new MergeHelper(Env::Default(), cmp_, nullptr, nullptr,
+                                        nullptr, 0U, false, 0));
+    iter_.reset(new test::VectorIterator(ks, vs));
    iter_->SeekToFirst();
    c_iter_.reset(new CompactionIterator(
        iter_.get(), cmp_, merge_helper_.get(), last_sequence, &snapshots_,
-        kMaxSequenceNumber, Env::Default(), false, range_del_agg_.get(),
-        std::move(compaction), filter, nullptr, &shutting_down_));
+        kMaxSequenceNumber, Env::Default(), false));
  }

-  void AddSnapshot(SequenceNumber snapshot) { snapshots_.push_back(snapshot); }
-
  const Comparator* cmp_;
-  const InternalKeyComparator icmp_;
  std::vector<SequenceNumber> snapshots_;
  std::unique_ptr<MergeHelper> merge_helper_;
-  std::unique_ptr<LoggingForwardVectorIterator> iter_;
+  std::unique_ptr<test::VectorIterator> iter_;
  std::unique_ptr<CompactionIterator> c_iter_;
-  std::unique_ptr<RangeDelAggregator> range_del_agg_;
-  std::atomic<bool> shutting_down_{false};
-  FakeCompaction* compaction_proxy_;
 };

 // It is possible that the output of the compaction iterator is empty even if
 // the input is not.
 TEST_F(CompactionIteratorTest, EmptyResult) {
-  InitIterators({test::KeyStr("a", 5, kTypeSingleDeletion),
-                 test::KeyStr("a", 3, kTypeValue)},
-                {"", "val"}, {}, {}, 5);
+  InitIterator({test::KeyStr("a", 5, kTypeSingleDeletion),
+                test::KeyStr("a", 3, kTypeValue)},
+               {"", "val"}, 5);
  c_iter_->SeekToFirst();
  ASSERT_FALSE(c_iter_->Valid());
 }
@@ -218,10 +45,10 @@ TEST_F(CompactionIteratorTest, EmptyResult) {
 // If there is a corruption after a single deletion, the corrupted key should
 // be preserved.
 TEST_F(CompactionIteratorTest, CorruptionAfterSingleDeletion) {
-  InitIterators({test::KeyStr("a", 5, kTypeSingleDeletion),
-                 test::KeyStr("a", 3, kTypeValue, true),
-                 test::KeyStr("b", 10, kTypeValue)},
-                {"", "val", "val2"}, {}, {}, 10);
+  InitIterator({test::KeyStr("a", 5, kTypeSingleDeletion),
+                test::KeyStr("a", 3, kTypeValue, true),
+                test::KeyStr("b", 10, kTypeValue)},
+               {"", "val", "val2"}, 10);
  c_iter_->SeekToFirst();
  ASSERT_TRUE(c_iter_->Valid());
  ASSERT_EQ(test::KeyStr("a", 5, kTypeSingleDeletion),
@@ -236,224 +63,6 @@ TEST_F(CompactionIteratorTest, CorruptionAfterSingleDeletion) {
  ASSERT_FALSE(c_iter_->Valid());
 }

-TEST_F(CompactionIteratorTest, SimpleRangeDeletion) {
-  InitIterators({test::KeyStr("morning", 5, kTypeValue),
-                 test::KeyStr("morning", 2, kTypeValue),
-                 test::KeyStr("night", 3, kTypeValue)},
-                {"zao", "zao", "wan"},
-                {test::KeyStr("ma", 4, kTypeRangeDeletion)}, {"mz"}, 5);
-  c_iter_->SeekToFirst();
-  ASSERT_TRUE(c_iter_->Valid());
-  ASSERT_EQ(test::KeyStr("morning", 5, kTypeValue), c_iter_->key().ToString());
-  c_iter_->Next();
-  ASSERT_TRUE(c_iter_->Valid());
-  ASSERT_EQ(test::KeyStr("night", 3, kTypeValue), c_iter_->key().ToString());
-  c_iter_->Next();
-  ASSERT_FALSE(c_iter_->Valid());
-}
-
-TEST_F(CompactionIteratorTest, RangeDeletionWithSnapshots) {
-  AddSnapshot(10);
-  std::vector<std::string> ks1;
-  ks1.push_back(test::KeyStr("ma", 28, kTypeRangeDeletion));
-  std::vector<std::string> vs1{"mz"};
-  std::vector<std::string> ks2{test::KeyStr("morning", 15, kTypeValue),
-                               test::KeyStr("morning", 5, kTypeValue),
-                               test::KeyStr("night", 40, kTypeValue),
-                               test::KeyStr("night", 20, kTypeValue)};
-  std::vector<std::string> vs2{"zao 15", "zao 5", "wan 40", "wan 20"};
-  InitIterators(ks2, vs2, ks1, vs1, 40);
-  c_iter_->SeekToFirst();
-  ASSERT_TRUE(c_iter_->Valid());
-  ASSERT_EQ(test::KeyStr("morning", 5, kTypeValue), c_iter_->key().ToString());
-  c_iter_->Next();
-  ASSERT_TRUE(c_iter_->Valid());
-  ASSERT_EQ(test::KeyStr("night", 40, kTypeValue), c_iter_->key().ToString());
-  c_iter_->Next();
-  ASSERT_FALSE(c_iter_->Valid());
-}
-
-TEST_F(CompactionIteratorTest, CompactionFilterSkipUntil) {
-  class Filter : public CompactionFilter {
-    virtual Decision FilterV2(int level, const Slice& key, ValueType t,
-                              const Slice& existing_value,
-                              std::string* new_value,
-                              std::string* skip_until) const override {
-      std::string k = key.ToString();
-      std::string v = existing_value.ToString();
-      // See InitIterators() call below for the sequence of keys and their
-      // filtering decisions. Here we closely assert that compaction filter is
-      // called with the expected keys and only them, and with the right values.
-      if (k == "a") {
-        EXPECT_EQ(ValueType::kValue, t);
-        EXPECT_EQ("av50", v);
-        return Decision::kKeep;
-      }
-      if (k == "b") {
-        EXPECT_EQ(ValueType::kValue, t);
-        EXPECT_EQ("bv60", v);
-        *skip_until = "d+";
-        return Decision::kRemoveAndSkipUntil;
-      }
-      if (k == "e") {
-        EXPECT_EQ(ValueType::kMergeOperand, t);
-        EXPECT_EQ("em71", v);
-        return Decision::kKeep;
-      }
-      if (k == "f") {
-        if (v == "fm65") {
-          EXPECT_EQ(ValueType::kMergeOperand, t);
-          *skip_until = "f";
-        } else {
-          EXPECT_EQ("fm30", v);
-          EXPECT_EQ(ValueType::kMergeOperand, t);
-          *skip_until = "g+";
-        }
-        return Decision::kRemoveAndSkipUntil;
-      }
-      if (k == "h") {
-        EXPECT_EQ(ValueType::kValue, t);
-        EXPECT_EQ("hv91", v);
-        return Decision::kKeep;
-      }
-      if (k == "i") {
-        EXPECT_EQ(ValueType::kMergeOperand, t);
-        EXPECT_EQ("im95", v);
-        *skip_until = "z";
-        return Decision::kRemoveAndSkipUntil;
-      }
-      ADD_FAILURE();
-      return Decision::kKeep;
-    }
-
-    const char* Name() const override {
-      return "CompactionIteratorTest.CompactionFilterSkipUntil::Filter";
-    }
-  };
-
-  NoMergingMergeOp merge_op;
-  Filter filter;
-  InitIterators(
-      {test::KeyStr("a", 50, kTypeValue),  // keep
-       test::KeyStr("a", 45, kTypeMerge),
-       test::KeyStr("b", 60, kTypeValue),  // skip to "d+"
-       test::KeyStr("b", 40, kTypeValue), test::KeyStr("c", 35, kTypeValue),
-       test::KeyStr("d", 70, kTypeMerge),
-       test::KeyStr("e", 71, kTypeMerge),  // keep
-       test::KeyStr("f", 65, kTypeMerge),  // skip to "f", aka keep
-       test::KeyStr("f", 30, kTypeMerge),  // skip to "g+"
-       test::KeyStr("f", 25, kTypeValue), test::KeyStr("g", 90, kTypeValue),
-       test::KeyStr("h", 91, kTypeValue),  // keep
-       test::KeyStr("i", 95, kTypeMerge),  // skip to "z"
-       test::KeyStr("j", 99, kTypeValue)},
-      {"av50", "am45", "bv60", "bv40", "cv35", "dm70", "em71", "fm65", "fm30",
-       "fv25", "gv90", "hv91", "im95", "jv99"},
-      {}, {}, kMaxSequenceNumber, &merge_op, &filter);
-
-  // Compaction should output just "a", "e" and "h" keys.
-  c_iter_->SeekToFirst();
-  ASSERT_TRUE(c_iter_->Valid());
-  ASSERT_EQ(test::KeyStr("a", 50, kTypeValue), c_iter_->key().ToString());
-  ASSERT_EQ("av50", c_iter_->value().ToString());
-  c_iter_->Next();
-  ASSERT_TRUE(c_iter_->Valid());
-  ASSERT_EQ(test::KeyStr("e", 71, kTypeMerge), c_iter_->key().ToString());
-  ASSERT_EQ("em71", c_iter_->value().ToString());
-  c_iter_->Next();
-  ASSERT_TRUE(c_iter_->Valid());
-  ASSERT_EQ(test::KeyStr("h", 91, kTypeValue), c_iter_->key().ToString());
-  ASSERT_EQ("hv91", c_iter_->value().ToString());
-  c_iter_->Next();
-  ASSERT_FALSE(c_iter_->Valid());
-
-  // Check that the compaction iterator did the correct sequence of calls on
-  // the underlying iterator.
-  using A = LoggingForwardVectorIterator::Action;
-  using T = A::Type;
-  std::vector<A> expected_actions = {
-      A(T::SEEK_TO_FIRST),
-      A(T::NEXT),
-      A(T::NEXT),
-      A(T::SEEK, test::KeyStr("d+", kMaxSequenceNumber, kValueTypeForSeek)),
-      A(T::NEXT),
-      A(T::NEXT),
-      A(T::SEEK, test::KeyStr("g+", kMaxSequenceNumber, kValueTypeForSeek)),
-      A(T::NEXT),
-      A(T::SEEK, test::KeyStr("z", kMaxSequenceNumber, kValueTypeForSeek))};
-  ASSERT_EQ(expected_actions, iter_->log);
-}
-
-TEST_F(CompactionIteratorTest, ShuttingDownInFilter) {
-  NoMergingMergeOp merge_op;
-  StallingFilter filter;
-  InitIterators(
-      {test::KeyStr("1", 1, kTypeValue), test::KeyStr("2", 2, kTypeValue),
-       test::KeyStr("3", 3, kTypeValue), test::KeyStr("4", 4, kTypeValue)},
-      {"v1", "v2", "v3", "v4"}, {}, {}, kMaxSequenceNumber, &merge_op, &filter);
-  // Don't leave tombstones (kTypeDeletion) for filtered keys.
-  compaction_proxy_->key_not_exists_beyond_output_level = true;
-
-  std::atomic<bool> seek_done{false};
-  rocksdb::port::Thread compaction_thread([&] {
-    c_iter_->SeekToFirst();
-    EXPECT_FALSE(c_iter_->Valid());
-    EXPECT_TRUE(c_iter_->status().IsShutdownInProgress());
-    seek_done.store(true);
-  });
-
-  // Let key 1 through.
-  filter.WaitForStall(1);
-
-  // Shutdown during compaction filter call for key 2.
-  filter.WaitForStall(2);
-  shutting_down_.store(true);
-  EXPECT_FALSE(seek_done.load());
-
-  // Unstall filter and wait for SeekToFirst() to return.
-  filter.stall_at.store(3);
-  compaction_thread.join();
-  assert(seek_done.load());
-
-  // Check that filter was never called again.
-  EXPECT_EQ(2, filter.last_seen.load());
-}
-
-// Same as ShuttingDownInFilter, but shutdown happens during filter call for
-// a merge operand, not for a value.
-TEST_F(CompactionIteratorTest, ShuttingDownInMerge) {
-  NoMergingMergeOp merge_op;
-  StallingFilter filter;
-  InitIterators(
-      {test::KeyStr("1", 1, kTypeValue), test::KeyStr("2", 2, kTypeMerge),
-       test::KeyStr("3", 3, kTypeMerge), test::KeyStr("4", 4, kTypeValue)},
-      {"v1", "v2", "v3", "v4"}, {}, {}, kMaxSequenceNumber, &merge_op, &filter);
-  compaction_proxy_->key_not_exists_beyond_output_level = true;
-
-  std::atomic<bool> seek_done{false};
-  rocksdb::port::Thread compaction_thread([&] {
-    c_iter_->SeekToFirst();
-    ASSERT_FALSE(c_iter_->Valid());
-    ASSERT_TRUE(c_iter_->status().IsShutdownInProgress());
-    seek_done.store(true);
-  });
-
-  // Let key 1 through.
-  filter.WaitForStall(1);
-
-  // Shutdown during compaction filter call for key 2.
-  filter.WaitForStall(2);
-  shutting_down_.store(true);
-  EXPECT_FALSE(seek_done.load());
-
-  // Unstall filter and wait for SeekToFirst() to return.
-  filter.stall_at.store(3);
-  compaction_thread.join();
-  assert(seek_done.load());
-
-  // Check that filter was never called again.
-  EXPECT_EQ(2, filter.last_seen.load());
-}
-
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -28,6 +28,7 @@
 #include "db/db_iter.h"
 #include "db/dbformat.h"
 #include "db/event_helpers.h"
+#include "db/filename.h"
 #include "db/log_reader.h"
 #include "db/log_writer.h"
 #include "db/memtable.h"
@@ -35,9 +36,6 @@
 #include "db/merge_context.h"
 #include "db/merge_helper.h"
 #include "db/version_set.h"
-#include "monitoring/iostats_context_imp.h"
-#include "monitoring/perf_context_imp.h"
-#include "monitoring/thread_status_util.h"
 #include "port/likely.h"
 #include "port/port.h"
 #include "rocksdb/db.h"
@@ -47,19 +45,20 @@
 #include "rocksdb/table.h"
 #include "table/block.h"
 #include "table/block_based_table_factory.h"
-#include "table/merging_iterator.h"
+#include "table/merger.h"
 #include "table/table_builder.h"
 #include "util/coding.h"
 #include "util/file_reader_writer.h"
-#include "util/filename.h"
+#include "util/iostats_context_imp.h"
 #include "util/log_buffer.h"
 #include "util/logging.h"
-#include "util/mutexlock.h"
-#include "util/random.h"
 #include "util/sst_file_manager_impl.h"
+#include "util/mutexlock.h"
+#include "util/perf_context_imp.h"
 #include "util/stop_watch.h"
 #include "util/string_util.h"
 #include "util/sync_point.h"
+#include "util/thread_status_util.h"

 namespace rocksdb {

@@ -100,8 +99,6 @@ struct CompactionJob::SubcompactionState {
    }
  }

-  uint64_t current_output_file_size;
-
  // State during the subcompaction
  uint64_t total_bytes;
  uint64_t num_input_records;
@@ -124,7 +121,6 @@ struct CompactionJob::SubcompactionState {
        end(_end),
        outfile(nullptr),
        builder(nullptr),
-        current_output_file_size(0),
        total_bytes(0),
        num_input_records(0),
        num_output_records(0),
@@ -146,7 +142,6 @@ struct CompactionJob::SubcompactionState {
    outputs = std::move(o.outputs);
    outfile = std::move(o.outfile);
    builder = std::move(o.builder);
-    current_output_file_size = std::move(o.current_output_file_size);
    total_bytes = std::move(o.total_bytes);
    num_input_records = std::move(o.num_input_records);
    num_output_records = std::move(o.num_output_records);
@@ -166,7 +161,7 @@ struct CompactionJob::SubcompactionState {

  // Returns true iff we should stop building the current output
  // before processing "internal_key".
-  bool ShouldStopBefore(const Slice& internal_key, uint64_t curr_file_size) {
+  bool ShouldStopBefore(const Slice& internal_key) {
    const InternalKeyComparator* icmp =
        &compaction->column_family_data()->internal_comparator();
    const std::vector<FileMetaData*>& grandparents = compaction->grandparents();
@@ -187,8 +182,7 @@ struct CompactionJob::SubcompactionState {
    }
    seen_key = true;

-    if (overlapped_bytes + curr_file_size >
-        compaction->max_compaction_bytes()) {
+    if (overlapped_bytes > compaction->max_grandparent_overlap_bytes()) {
      // Too much overlap for current output; start new output
      overlapped_bytes = 0;
      return true;
@@ -263,9 +257,9 @@ void CompactionJob::AggregateStatistics() {
 }

 CompactionJob::CompactionJob(
-    int job_id, Compaction* compaction, const ImmutableDBOptions& db_options,
+    int job_id, Compaction* compaction, const DBOptions& db_options,
    const EnvOptions& env_options, VersionSet* versions,
-    const std::atomic<bool>* shutting_down, LogBuffer* log_buffer,
+    std::atomic<bool>* shutting_down, LogBuffer* log_buffer,
    Directory* db_directory, Directory* output_directory, Statistics* stats,
    InstrumentedMutex* db_mutex, Status* db_bg_error,
    std::vector<SequenceNumber> existing_snapshots,
@@ -298,7 +292,7 @@ CompactionJob::CompactionJob(
  assert(log_buffer_ != nullptr);
  const auto* cfd = compact_->compaction->column_family_data();
  ThreadStatusUtil::SetColumnFamily(cfd, cfd->ioptions()->env,
-                                    db_options_.enable_thread_tracking);
+                                    cfd->options()->enable_thread_tracking);
  ThreadStatusUtil::SetThreadOperation(ThreadStatus::OP_COMPACTION);
  ReportStartedCompaction(compaction);
 }
@@ -312,7 +306,7 @@ void CompactionJob::ReportStartedCompaction(
    Compaction* compaction) {
  const auto* cfd = compact_->compaction->column_family_data();
  ThreadStatusUtil::SetColumnFamily(cfd, cfd->ioptions()->env,
-                                    db_options_.enable_thread_tracking);
+                                    cfd->options()->enable_thread_tracking);

  ThreadStatusUtil::SetThreadOperationProperty(
      ThreadStatus::COMPACTION_JOB_ID,
@@ -477,16 +471,17 @@ void CompactionJob::GenSubcompactionBoundaries() {

  // Group the ranges into subcompactions
  const double min_file_fill_percent = 4.0 / 5;
-  uint64_t max_output_files = static_cast<uint64_t>(
-      std::ceil(sum / min_file_fill_percent /
-                c->mutable_cf_options()->MaxFileSizeForLevel(out_lvl)));
+  uint64_t max_output_files = static_cast<uint64_t>(std::ceil(
+      sum / min_file_fill_percent /
+      cfd->GetCurrentMutableCFOptions()->MaxFileSizeForLevel(out_lvl)));
  uint64_t subcompactions =
      std::min({static_cast<uint64_t>(ranges.size()),
                static_cast<uint64_t>(db_options_.max_subcompactions),
                max_output_files});

+  double mean = sum * 1.0 / subcompactions;
+
  if (subcompactions > 1) {
-    double mean = sum * 1.0 / subcompactions;
    // Greedily add ranges to the subcompaction until the sum of the ranges'
    // sizes becomes >= the expected mean size of a subcompaction
    sum = 0;
@@ -523,7 +518,7 @@ Status CompactionJob::Run() {
  const uint64_t start_micros = env_->NowMicros();

  // Launch a thread for each of subcompactions 1...num_threads-1
-  std::vector<port::Thread> thread_pool;
+  std::vector<std::thread> thread_pool;
  thread_pool.reserve(num_threads - 1);
  for (size_t i = 1; i < compact_->sub_compact_states.size(); i++) {
    thread_pool.emplace_back(&CompactionJob::ProcessKeyValueCompaction, this,
@@ -539,7 +534,7 @@ Status CompactionJob::Run() {
    thread.join();
  }

-  if (output_directory_) {
+  if (output_directory_ && !db_options_.disableDataSync) {
    output_directory_->Fsync();
  }

@@ -591,17 +586,7 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) {
  VersionStorageInfo::LevelSummaryStorage tmp;
  auto vstorage = cfd->current()->storage_info();
  const auto& stats = compaction_stats_;
-
-  double read_write_amp = 0.0;
-  double write_amp = 0.0;
-  if (stats.bytes_read_non_output_levels > 0) {
-    read_write_amp = (stats.bytes_written + stats.bytes_read_output_level +
-                      stats.bytes_read_non_output_levels) /
-                     static_cast<double>(stats.bytes_read_non_output_levels);
-    write_amp = stats.bytes_written /
-                static_cast<double>(stats.bytes_read_non_output_levels);
-  }
-  ROCKS_LOG_BUFFER(
+  LogToBuffer(
      log_buffer_,
      "[%s] compacted to: %s, MB/sec: %.1f rd, %.1f wr, level %d, "
      "files in(%d, %d) out(%d) "
@@ -613,10 +598,16 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) {
      stats.bytes_written / static_cast<double>(stats.micros),
      compact_->compaction->output_level(),
      stats.num_input_files_in_non_output_levels,
-      stats.num_input_files_in_output_level, stats.num_output_files,
+      stats.num_input_files_in_output_level,
+      stats.num_output_files,
      stats.bytes_read_non_output_levels / 1048576.0,
      stats.bytes_read_output_level / 1048576.0,
-      stats.bytes_written / 1048576.0, read_write_amp, write_amp,
+      stats.bytes_written / 1048576.0,
+      (stats.bytes_written + stats.bytes_read_output_level +
+       stats.bytes_read_non_output_levels) /
+          static_cast<double>(stats.bytes_read_non_output_levels),
+      stats.bytes_written /
+          static_cast<double>(stats.bytes_read_non_output_levels),
      status.ToString().c_str(), stats.num_input_records,
      stats.num_dropped_records);

@@ -633,13 +624,6 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) {
         << "num_output_records" << compact_->num_output_records
         << "num_subcompactions" << compact_->sub_compact_states.size();

-  if (compaction_job_stats_ != nullptr) {
-    stream << "num_single_delete_mismatches"
-           << compaction_job_stats_->num_single_del_mismatch;
-    stream << "num_single_delete_fallthrough"
-           << compaction_job_stats_->num_single_del_fallthru;
-  }
-
  if (measure_io_stats_ && compaction_job_stats_ != nullptr) {
    stream << "file_write_nanos" << compaction_job_stats_->file_write_nanos;
    stream << "file_range_sync_nanos"
@@ -662,11 +646,8 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) {

 void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
  assert(sub_compact != nullptr);
-  ColumnFamilyData* cfd = sub_compact->compaction->column_family_data();
-  std::unique_ptr<RangeDelAggregator> range_del_agg(
-      new RangeDelAggregator(cfd->internal_comparator(), existing_snapshots_));
-  std::unique_ptr<InternalIterator> input(versions_->MakeInputIterator(
-      sub_compact->compaction, range_del_agg.get()));
+  std::unique_ptr<InternalIterator> input(
+      versions_->MakeInputIterator(sub_compact->compaction));

  AutoThreadOperationStageUpdater stage_updater(
      ThreadStatus::STAGE_COMPACTION_PROCESS_KV);
@@ -687,6 +668,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
    prev_prepare_write_nanos = IOSTATS(prepare_write_nanos);
  }

+  ColumnFamilyData* cfd = sub_compact->compaction->column_family_data();
  const MutableCFOptions* mutable_cf_options =
      sub_compact->compaction->mutable_cf_options();

@@ -722,10 +704,10 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
  MergeHelper merge(
      env_, cfd->user_comparator(), cfd->ioptions()->merge_operator,
      compaction_filter, db_options_.info_log.get(),
+      cfd->ioptions()->min_partial_merge_operands,
      false /* internal key corruption is expected */,
      existing_snapshots_.empty() ? 0 : existing_snapshots_.back(),
-      compact_->compaction->level(), db_options_.statistics.get(),
-      shutting_down_);
+      compact_->compaction->level(), db_options_.statistics.get());

  TEST_SYNC_POINT("CompactionJob::Run():Inprogress");

@@ -734,38 +716,18 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
  if (start != nullptr) {
    IterKey start_iter;
    start_iter.SetInternalKey(*start, kMaxSequenceNumber, kValueTypeForSeek);
-    input->Seek(start_iter.GetInternalKey());
+    input->Seek(start_iter.GetKey());
  } else {
    input->SeekToFirst();
  }

-  // we allow only 1 compaction event listener. Used by blob storage
-  CompactionEventListener* comp_event_listener = nullptr;
-#ifndef ROCKSDB_LITE
-  for (auto& celitr : cfd->ioptions()->listeners) {
-    comp_event_listener = celitr->GetCompactionEventListener();
-    if (comp_event_listener != nullptr) {
-      break;
-    }
-  }
-#endif  // ROCKSDB_LITE
-
  Status status;
  sub_compact->c_iter.reset(new CompactionIterator(
      input.get(), cfd->user_comparator(), &merge, versions_->LastSequence(),
      &existing_snapshots_, earliest_write_conflict_snapshot_, env_, false,
-      range_del_agg.get(), sub_compact->compaction, compaction_filter,
-      comp_event_listener, shutting_down_));
+      sub_compact->compaction, compaction_filter));
  auto c_iter = sub_compact->c_iter.get();
  c_iter->SeekToFirst();
-  if (c_iter->Valid() &&
-      sub_compact->compaction->output_level() != 0) {
-    // ShouldStopBefore() maintains state based on keys processed so far. The
-    // compaction loop always calls it on the "next" key, thus won't tell it the
-    // first key. So we do that here.
-    sub_compact->ShouldStopBefore(
-      c_iter->key(), sub_compact->current_output_file_size);
-  }
  const auto& c_iter_stats = c_iter->iter_stats();
  auto sample_begin_offset_iter = sample_begin_offsets.cbegin();
  // data_begin_offset and compression_dict are only valid while generating
@@ -774,7 +736,10 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
  std::string compression_dict;
  compression_dict.reserve(cfd->ioptions()->compression_opts.max_dict_bytes);

-  while (status.ok() && !cfd->IsDropped() && c_iter->Valid()) {
+  // TODO(noetzli): check whether we could check !shutting_down_->... only
+  // only occasionally (see diff D42687)
+  while (status.ok() && !shutting_down_->load(std::memory_order_acquire) &&
+         !cfd->IsDropped() && c_iter->Valid()) {
    // Invariant: c_iter.status() is guaranteed to be OK if c_iter->Valid()
    // returns true.
    const Slice& key = c_iter->key();
@@ -785,7 +750,14 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
    if (end != nullptr &&
        cfd->user_comparator()->Compare(c_iter->user_key(), *end) >= 0) {
      break;
+    } else if (sub_compact->ShouldStopBefore(key) &&
+               sub_compact->builder != nullptr) {
+      status = FinishCompactionOutputFile(input->status(), sub_compact);
+      if (!status.ok()) {
+        break;
+      }
    }
+
    if (c_iter_stats.num_input_records % kRecordStatsEvery ==
        kRecordStatsEvery - 1) {
      RecordDroppedKeys(c_iter_stats, &sub_compact->compaction_job_stats);
@@ -803,7 +775,6 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
    assert(sub_compact->builder != nullptr);
    assert(sub_compact->current_output() != nullptr);
    sub_compact->builder->Add(key, value);
-    sub_compact->current_output_file_size = sub_compact->builder->FileSize();
    sub_compact->current_output()->meta.UpdateBoundaries(
        key, c_iter->ikey().sequence);
    sub_compact->num_output_records++;
@@ -857,53 +828,21 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
      }
    }

-    // Close output file if it is big enough. Two possibilities determine it's
-    // time to close it: (1) the current key should be this file's last key, (2)
-    // the next key should not be in this file.
-    //
+    // Close output file if it is big enough
    // TODO(aekmekji): determine if file should be closed earlier than this
    // during subcompactions (i.e. if output size, estimated by input size, is
    // going to be 1.2MB and max_output_file_size = 1MB, prefer to have 0.6MB
    // and 0.6MB instead of 1MB and 0.2MB)
-    bool output_file_ended = false;
-    Status input_status;
-    if (sub_compact->compaction->output_level() != 0 &&
-        sub_compact->current_output_file_size >=
-            sub_compact->compaction->max_output_file_size()) {
-      // (1) this key terminates the file. For historical reasons, the iterator
-      // status before advancing will be given to FinishCompactionOutputFile().
-      input_status = input->status();
-      output_file_ended = true;
-    }
-    c_iter->Next();
-    if (!output_file_ended && c_iter->Valid() &&
-        sub_compact->compaction->output_level() != 0 &&
-        sub_compact->ShouldStopBefore(
-          c_iter->key(), sub_compact->current_output_file_size) &&
-        sub_compact->builder != nullptr) {
-      // (2) this key belongs to the next file. For historical reasons, the
-      // iterator status after advancing will be given to
-      // FinishCompactionOutputFile().
-      input_status = input->status();
-      output_file_ended = true;
-    }
-    if (output_file_ended) {
-      const Slice* next_key = nullptr;
-      if (c_iter->Valid()) {
-        next_key = &c_iter->key();
-      }
-      CompactionIterationStats range_del_out_stats;
-      status = FinishCompactionOutputFile(input_status, sub_compact,
-                                          range_del_agg.get(),
-                                          &range_del_out_stats, next_key);
-      RecordDroppedKeys(range_del_out_stats,
-                        &sub_compact->compaction_job_stats);
+    if (sub_compact->builder->FileSize() >=
+        sub_compact->compaction->max_output_file_size()) {
+      status = FinishCompactionOutputFile(input->status(), sub_compact);
      if (sub_compact->outputs.size() == 1) {
        // Use dictionary from first output file for compression of subsequent
        // files.
        sub_compact->compression_dict = std::move(compression_dict);
      }
    }
+    c_iter->Next();
  }

  sub_compact->num_input_records = c_iter_stats.num_input_records;
@@ -911,10 +850,6 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
      c_iter_stats.num_input_deletion_records;
  sub_compact->compaction_job_stats.num_corrupt_keys =
      c_iter_stats.num_input_corrupt_records;
-  sub_compact->compaction_job_stats.num_single_del_fallthru =
-      c_iter_stats.num_single_del_fallthru;
-  sub_compact->compaction_job_stats.num_single_del_mismatch =
-      c_iter_stats.num_single_del_mismatch;
  sub_compact->compaction_job_stats.total_input_raw_key_bytes +=
      c_iter_stats.total_input_raw_key_bytes;
  sub_compact->compaction_job_stats.total_input_raw_value_bytes +=
@@ -925,36 +860,17 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
  RecordDroppedKeys(c_iter_stats, &sub_compact->compaction_job_stats);
  RecordCompactionIOStats();

-  if (status.ok() && (shutting_down_->load(std::memory_order_relaxed) ||
-                      cfd->IsDropped())) {
+  if (status.ok() &&
+      (shutting_down_->load(std::memory_order_acquire) || cfd->IsDropped())) {
    status = Status::ShutdownInProgress(
        "Database shutdown or Column family drop during compaction");
  }
+  if (status.ok() && sub_compact->builder != nullptr) {
+    status = FinishCompactionOutputFile(input->status(), sub_compact);
+  }
  if (status.ok()) {
    status = input->status();
  }
-  if (status.ok()) {
-    status = c_iter->status();
-  }
-
-  if (status.ok() && sub_compact->builder == nullptr &&
-      sub_compact->outputs.size() == 0 &&
-      range_del_agg->ShouldAddTombstones(bottommost_level_)) {
-    // handle subcompaction containing only range deletions
-    status = OpenCompactionOutputFile(sub_compact);
-  }
-
-  // Call FinishCompactionOutputFile() even if status is not ok: it needs to
-  // close the output file.
-  if (sub_compact->builder != nullptr) {
-    CompactionIterationStats range_del_out_stats;
-    Status s = FinishCompactionOutputFile(
-        status, sub_compact, range_del_agg.get(), &range_del_out_stats);
-    if (status.ok()) {
-      status = s;
-    }
-    RecordDroppedKeys(range_del_out_stats, &sub_compact->compaction_job_stats);
-  }

  if (measure_io_stats_) {
    sub_compact->compaction_job_stats.file_write_nanos +=
@@ -976,7 +892,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
 }

 void CompactionJob::RecordDroppedKeys(
-    const CompactionIterationStats& c_iter_stats,
+    const CompactionIteratorStats& c_iter_stats,
    CompactionJobStats* compaction_job_stats) {
  if (c_iter_stats.num_record_drop_user > 0) {
    RecordTick(stats_, COMPACTION_KEY_DROP_USER,
@@ -998,21 +914,10 @@ void CompactionJob::RecordDroppedKeys(
          c_iter_stats.num_record_drop_obsolete;
    }
  }
-  if (c_iter_stats.num_record_drop_range_del > 0) {
-    RecordTick(stats_, COMPACTION_KEY_DROP_RANGE_DEL,
-               c_iter_stats.num_record_drop_range_del);
-  }
-  if (c_iter_stats.num_range_del_drop_obsolete > 0) {
-    RecordTick(stats_, COMPACTION_RANGE_DEL_DROP_OBSOLETE,
-               c_iter_stats.num_range_del_drop_obsolete);
-  }
 }

 Status CompactionJob::FinishCompactionOutputFile(
-    const Status& input_status, SubcompactionState* sub_compact,
-    RangeDelAggregator* range_del_agg,
-    CompactionIterationStats* range_del_out_stats,
-    const Slice* next_table_min_key /* = nullptr */) {
+    const Status& input_status, SubcompactionState* sub_compact) {
  AutoThreadOperationStageUpdater stage_updater(
      ThreadStatus::STAGE_COMPACTION_SYNC_FILE);
  assert(sub_compact != nullptr);
@@ -1027,36 +932,6 @@ Status CompactionJob::FinishCompactionOutputFile(
  // Check for iterator errors
  Status s = input_status;
  auto meta = &sub_compact->current_output()->meta;
-  if (s.ok()) {
-    Slice lower_bound_guard, upper_bound_guard;
-    const Slice *lower_bound, *upper_bound;
-    if (sub_compact->outputs.size() == 1) {
-      // For the first output table, include range tombstones before the min key
-      // but after the subcompaction boundary.
-      lower_bound = sub_compact->start;
-    } else if (meta->smallest.size() > 0) {
-      // For subsequent output tables, only include range tombstones from min
-      // key onwards since the previous file was extended to contain range
-      // tombstones falling before min key.
-      lower_bound_guard = meta->smallest.user_key();
-      lower_bound = &lower_bound_guard;
-    } else {
-      lower_bound = nullptr;
-    }
-    if (next_table_min_key != nullptr) {
-      // This isn't the last file in the subcompaction, so extend until the next
-      // file starts.
-      upper_bound_guard = ExtractUserKey(*next_table_min_key);
-      upper_bound = &upper_bound_guard;
-    } else {
-      // This is the last file in the subcompaction, so extend until the
-      // subcompaction ends.
-      upper_bound = sub_compact->end;
-    }
-    range_del_agg->AddToBuilder(sub_compact->builder.get(), lower_bound,
-                                upper_bound, meta, range_del_out_stats,
-                                bottommost_level_);
-  }
  const uint64_t current_entries = sub_compact->builder->NumEntries();
  meta->marked_for_compaction = sub_compact->builder->NeedCompact();
  if (s.ok()) {
@@ -1070,7 +945,7 @@ Status CompactionJob::FinishCompactionOutputFile(
  sub_compact->total_bytes += current_bytes;

  // Finish and check for file errors
-  if (s.ok()) {
+  if (s.ok() && !db_options_.disableDataSync) {
    StopWatch sw(env_, stats_, COMPACTION_OUTFILE_SYNC_MICROS);
    s = sub_compact->outfile->Sync(db_options_.use_fsync);
  }
@@ -1083,16 +958,10 @@ Status CompactionJob::FinishCompactionOutputFile(
  TableProperties tp;
  if (s.ok() && current_entries > 0) {
    // Verify that the table is usable
-    // We set for_compaction to false and don't OptimizeForCompactionTableRead
-    // here because this is a special case after we finish the table building
-    // No matter whether use_direct_io_for_flush_and_compaction is true,
-    // we will regrad this verification as user reads since the goal is
-    // to cache it here for further user reads
    InternalIterator* iter = cfd->table_cache()->NewIterator(
        ReadOptions(), env_options_, cfd->internal_comparator(), meta->fd,
-        nullptr /* range_del_agg */, nullptr,
-        cfd->internal_stats()->GetFileReadHist(
-            compact_->compaction->output_level()),
+        nullptr, cfd->internal_stats()->GetFileReadHist(
+                     compact_->compaction->output_level()),
        false);
    s = iter->status();

@@ -1108,12 +977,12 @@ Status CompactionJob::FinishCompactionOutputFile(
      tp = sub_compact->builder->GetTableProperties();
      sub_compact->current_output()->table_properties =
          std::make_shared<TableProperties>(tp);
-      ROCKS_LOG_INFO(db_options_.info_log,
-                     "[%s] [JOB %d] Generated table #%" PRIu64 ": %" PRIu64
-                     " keys, %" PRIu64 " bytes%s",
-                     cfd->GetName().c_str(), job_id_, output_number,
-                     current_entries, current_bytes,
-                     meta->marked_for_compaction ? " (need compaction)" : "");
+      Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
+          "[%s] [JOB %d] Generated table #%" PRIu64 ": %" PRIu64
+          " keys, %" PRIu64 " bytes%s",
+          cfd->GetName().c_str(), job_id_, output_number, current_entries,
+          current_bytes,
+          meta->marked_for_compaction ? " (need compaction)" : "");
    }
  }
  std::string fname = TableFileName(db_options_.db_paths, meta->fd.GetNumber(),
@@ -1122,7 +991,6 @@ Status CompactionJob::FinishCompactionOutputFile(
      event_logger_, cfd->ioptions()->listeners, dbname_, cfd->GetName(), fname,
      job_id_, meta->fd, tp, TableFileCreationReason::kCompaction, s);

-#ifndef ROCKSDB_LITE
  // Report new file to SstFileManagerImpl
  auto sfm =
      static_cast<SstFileManagerImpl*>(db_options_.sst_file_manager.get());
@@ -1140,10 +1008,8 @@ Status CompactionJob::FinishCompactionOutputFile(
      }
    }
  }
-#endif

  sub_compact->builder.reset();
-  sub_compact->current_output_file_size = 0;
  return s;
 }

@@ -1159,16 +1025,17 @@ Status CompactionJob::InstallCompactionResults(
  if (!versions_->VerifyCompactionFileConsistency(compaction)) {
    Compaction::InputLevelSummaryBuffer inputs_summary;

-    ROCKS_LOG_ERROR(db_options_.info_log, "[%s] [JOB %d] Compaction %s aborted",
-                    compaction->column_family_data()->GetName().c_str(),
-                    job_id_, compaction->InputLevelSummary(&inputs_summary));
+    Log(InfoLogLevel::ERROR_LEVEL, db_options_.info_log,
+        "[%s] [JOB %d] Compaction %s aborted",
+        compaction->column_family_data()->GetName().c_str(), job_id_,
+        compaction->InputLevelSummary(&inputs_summary));
    return Status::Corruption("Compaction input files inconsistent");
  }

  {
    Compaction::InputLevelSummaryBuffer inputs_summary;
-    ROCKS_LOG_INFO(
-        db_options_.info_log, "[%s] [JOB %d] Compacted %s => %" PRIu64 " bytes",
+    Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
+        "[%s] [JOB %d] Compacted %s => %" PRIu64 " bytes",
        compaction->column_family_data()->GetName().c_str(), job_id_,
        compaction->InputLevelSummary(&inputs_summary), compact_->total_bytes);
  }
@@ -1214,14 +1081,9 @@ Status CompactionJob::OpenCompactionOutputFile(
 #endif  // !ROCKSDB_LITE
  // Make the output file
  unique_ptr<WritableFile> writable_file;
-  EnvOptions opt_env_opts =
-      env_->OptimizeForCompactionTableWrite(env_options_, db_options_);
-  TEST_SYNC_POINT_CALLBACK("CompactionJob::OpenCompactionOutputFile",
-                           &opt_env_opts.use_direct_writes);
-  Status s = NewWritableFile(env_, fname, &writable_file, opt_env_opts);
+  Status s = NewWritableFile(env_, fname, &writable_file, env_options_);
  if (!s.ok()) {
-    ROCKS_LOG_ERROR(
-        db_options_.info_log,
+    Log(InfoLogLevel::ERROR_LEVEL, db_options_.info_log,
        "[%s] [JOB %d] OpenCompactionOutputFiles for table #%" PRIu64
        " fails at NewWritableFile with status %s",
        sub_compact->compaction->column_family_data()->GetName().c_str(),
@@ -1243,8 +1105,8 @@ Status CompactionJob::OpenCompactionOutputFile(
  writable_file->SetIOPriority(Env::IO_LOW);
  writable_file->SetPreallocationBlockSize(static_cast<size_t>(
      sub_compact->compaction->OutputFilePreallocationSize()));
-  sub_compact->outfile.reset(new WritableFileWriter(
-      std::move(writable_file), env_options_, db_options_.statistics.get()));
+  sub_compact->outfile.reset(
+      new WritableFileWriter(std::move(writable_file), env_options_));

  // If the Column family flag is to only optimize filters for hits,
  // we can skip creating filters if this is the bottommost_level where
@@ -1255,9 +1117,7 @@ Status CompactionJob::OpenCompactionOutputFile(
      *cfd->ioptions(), cfd->internal_comparator(),
      cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
      sub_compact->outfile.get(), sub_compact->compaction->output_compression(),
-      cfd->ioptions()->compression_opts,
-      sub_compact->compaction->output_level(),
-      &sub_compact->compression_dict,
+      cfd->ioptions()->compression_opts, &sub_compact->compression_dict,
      skip_filters));
  LogFlush(db_options_.info_log);
  return s;
@@ -1305,7 +1165,8 @@ void CompactionJob::UpdateCompactionStats() {
  for (int input_level = 0;
       input_level < static_cast<int>(compaction->num_input_levels());
       ++input_level) {
-    if (compaction->level(input_level) != compaction->output_level()) {
+    if (compaction->start_level() + input_level
+        != compaction->output_level()) {
      UpdateCompactionInputStatsHelper(
          &compaction_stats_.num_input_files_in_non_output_levels,
          &compaction_stats_.bytes_read_non_output_levels,
@@ -1397,14 +1258,14 @@ void CompactionJob::LogCompaction() {
  // we're not logging
  if (db_options_.info_log_level <= InfoLogLevel::INFO_LEVEL) {
    Compaction::InputLevelSummaryBuffer inputs_summary;
-    ROCKS_LOG_INFO(
-        db_options_.info_log, "[%s] [JOB %d] Compacting %s, score %.2f",
-        cfd->GetName().c_str(), job_id_,
-        compaction->InputLevelSummary(&inputs_summary), compaction->score());
+    Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
+        "[%s] [JOB %d] Compacting %s, score %.2f", cfd->GetName().c_str(),
+        job_id_, compaction->InputLevelSummary(&inputs_summary),
+        compaction->score());
    char scratch[2345];
    compaction->Summary(scratch, sizeof(scratch));
-    ROCKS_LOG_INFO(db_options_.info_log, "[%s] Compaction start summary: %s\n",
-                   cfd->GetName().c_str(), scratch);
+    Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
+        "[%s] Compaction start summary: %s\n", cfd->GetName().c_str(), scratch);
    // build event logger report
    auto stream = event_logger_->Log();
    stream << "job" << job_id_ << "event"
@@ -25,11 +25,9 @@
 #include "db/job_context.h"
 #include "db/log_writer.h"
 #include "db/memtable_list.h"
-#include "db/range_del_aggregator.h"
 #include "db/version_edit.h"
 #include "db/write_controller.h"
 #include "db/write_thread.h"
-#include "options/db_options.h"
 #include "port/port.h"
 #include "rocksdb/compaction_filter.h"
 #include "rocksdb/compaction_job_stats.h"
@@ -54,10 +52,9 @@ class Arena;

 class CompactionJob {
 public:
-  CompactionJob(int job_id, Compaction* compaction,
-                const ImmutableDBOptions& db_options,
+  CompactionJob(int job_id, Compaction* compaction, const DBOptions& db_options,
                const EnvOptions& env_options, VersionSet* versions,
-                const std::atomic<bool>* shutting_down, LogBuffer* log_buffer,
+                std::atomic<bool>* shutting_down, LogBuffer* log_buffer,
                Directory* db_directory, Directory* output_directory,
                Statistics* stats, InstrumentedMutex* db_mutex,
                Status* db_bg_error,
@@ -96,18 +93,15 @@ class CompactionJob {
  // kv-pairs
  void ProcessKeyValueCompaction(SubcompactionState* sub_compact);

-  Status FinishCompactionOutputFile(
-      const Status& input_status, SubcompactionState* sub_compact,
-      RangeDelAggregator* range_del_agg,
-      CompactionIterationStats* range_del_out_stats,
-      const Slice* next_table_min_key = nullptr);
+  Status FinishCompactionOutputFile(const Status& input_status,
+                                    SubcompactionState* sub_compact);
  Status InstallCompactionResults(const MutableCFOptions& mutable_cf_options);
  void RecordCompactionIOStats();
  Status OpenCompactionOutputFile(SubcompactionState* sub_compact);
  void CleanupCompaction();
  void UpdateCompactionJobStats(
    const InternalStats::CompactionStats& stats) const;
-  void RecordDroppedKeys(const CompactionIterationStats& c_iter_stats,
+  void RecordDroppedKeys(const CompactionIteratorStats& c_iter_stats,
                         CompactionJobStats* compaction_job_stats = nullptr);

  void UpdateCompactionStats();
@@ -126,12 +120,11 @@ class CompactionJob {

  // DBImpl state
  const std::string& dbname_;
-  const ImmutableDBOptions& db_options_;
+  const DBOptions& db_options_;
  const EnvOptions& env_options_;
-
  Env* env_;
  VersionSet* versions_;
-  const std::atomic<bool>* shutting_down_;
+  std::atomic<bool>* shutting_down_;
  LogBuffer* log_buffer_;
  Directory* db_directory_;
  Directory* output_directory_;
@@ -23,13 +23,11 @@

 #include "db/db_impl.h"
 #include "db/dbformat.h"
+#include "db/filename.h"
 #include "db/job_context.h"
 #include "db/version_set.h"
 #include "db/write_batch_internal.h"
-#include "env/mock_env.h"
 #include "memtable/hash_linklist_rep.h"
-#include "monitoring/statistics.h"
-#include "monitoring/thread_status_util.h"
 #include "port/stack_trace.h"
 #include "rocksdb/cache.h"
 #include "rocksdb/compaction_filter.h"
@@ -52,15 +50,18 @@
 #include "table/plain_table_factory.h"
 #include "table/scoped_arena_iterator.h"
 #include "util/compression.h"
-#include "util/filename.h"
 #include "util/hash.h"
 #include "util/logging.h"
+#include "util/mock_env.h"
 #include "util/mutexlock.h"
 #include "util/rate_limiter.h"
+#include "util/statistics.h"
 #include "util/string_util.h"
 #include "util/sync_point.h"
 #include "util/testharness.h"
 #include "util/testutil.h"
+#include "util/thread_status_util.h"
+#include "util/xfunc.h"
 #include "utilities/merge_operators.h"

 #if !defined(IOS_CROSS_COMPILE)
@@ -956,7 +957,7 @@ TEST_P(CompactionJobStatsTest, UniversalCompactionTest) {
  uint64_t key_base = 100000000l;
  // Note: key_base must be multiple of num_keys_per_L0_file
  int num_keys_per_table = 100;
-  const uint32_t kTestScale = 6;
+  const uint32_t kTestScale = 8;
  const int kKeySize = 10;
  const int kValueSize = 900;
  double compression_ratio = 1.0;
@@ -1007,9 +1008,8 @@ TEST_P(CompactionJobStatsTest, UniversalCompactionTest) {
            num_input_units,
            num_keys_per_table * num_input_units,
            1.0, 0, false));
-    dbfull()->TEST_WaitForCompact();
  }
-  ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 3U);
+  ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 4U);

  for (uint64_t start_key = key_base;
                start_key <= key_base * kTestScale;
@@ -10,13 +10,13 @@
 #include <string>
 #include <tuple>

-#include "db/column_family.h"
 #include "db/compaction_job.h"
+#include "db/column_family.h"
 #include "db/version_set.h"
+#include "db/writebuffer.h"
 #include "rocksdb/cache.h"
 #include "rocksdb/db.h"
 #include "rocksdb/options.h"
-#include "rocksdb/write_buffer_manager.h"
 #include "table/mock_table.h"
 #include "util/file_reader_writer.h"
 #include "util/string_util.h"
@@ -68,12 +68,11 @@ class CompactionJobTest : public testing::Test {
  CompactionJobTest()
      : env_(Env::Default()),
        dbname_(test::TmpDir() + "/compaction_job_test"),
-        db_options_(),
-        mutable_cf_options_(cf_options_),
+        mutable_cf_options_(Options(), ImmutableCFOptions(Options())),
        table_cache_(NewLRUCache(50000, 16)),
-        write_buffer_manager_(db_options_.db_write_buffer_size),
+        write_buffer_(db_options_.db_write_buffer_size),
        versions_(new VersionSet(dbname_, &db_options_, env_options_,
-                                 table_cache_.get(), &write_buffer_manager_,
+                                 table_cache_.get(), &write_buffer_,
                                 &write_controller_)),
        shutting_down_(false),
        mock_table_factory_(new mock::MockTableFactory()) {
@@ -240,10 +239,10 @@ class CompactionJobTest : public testing::Test {
      num_input_files += level_files.size();
    }

-    Compaction compaction(cfd->current()->storage_info(), *cfd->ioptions(),
+    Compaction compaction(cfd->current()->storage_info(),
                          *cfd->GetLatestMutableCFOptions(),
-                          compaction_input_files, 1, 1024 * 1024,
-                          10 * 1024 * 1024, 0, kNoCompression, {}, true);
+                          compaction_input_files, 1, 1024 * 1024, 10, 0,
+                          kNoCompression, {}, true);
    compaction.SetInputVersion(cfd->current());

    LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, db_options_.info_log.get());
@@ -281,12 +280,12 @@ class CompactionJobTest : public testing::Test {
  Env* env_;
  std::string dbname_;
  EnvOptions env_options_;
-  ImmutableDBOptions db_options_;
-  ColumnFamilyOptions cf_options_;
  MutableCFOptions mutable_cf_options_;
  std::shared_ptr<Cache> table_cache_;
  WriteController write_controller_;
-  WriteBufferManager write_buffer_manager_;
+  DBOptions db_options_;
+  ColumnFamilyOptions cf_options_;
+  WriteBuffer write_buffer_;
  std::unique_ptr<VersionSet> versions_;
  InstrumentedMutex mutex_;
  std::atomic<bool> shutting_down_;
@@ -17,10 +17,11 @@

 #include "db/compaction.h"
 #include "db/version_set.h"
-#include "options/cf_options.h"
 #include "rocksdb/env.h"
 #include "rocksdb/options.h"
 #include "rocksdb/status.h"
+#include "util/mutable_cf_options.h"
+

 namespace rocksdb {

@@ -62,20 +63,23 @@ class CompactionPicker {
      InternalKey** compaction_end, bool* manual_conflict);

  // The maximum allowed output level.  Default value is NumberLevels() - 1.
-  virtual int MaxOutputLevel() const { return NumberLevels() - 1; }
+  virtual int MaxOutputLevel() const {
+    return NumberLevels() - 1;
+  }

  virtual bool NeedsCompaction(const VersionStorageInfo* vstorage) const = 0;

-// Sanitize the input set of compaction input files.
-// When the input parameters do not describe a valid compaction, the
-// function will try to fix the input_files by adding necessary
-// files.  If it's not possible to conver an invalid input_files
-// into a valid one by adding more files, the function will return a
-// non-ok status with specific reason.
+  // Sanitize the input set of compaction input files.
+  // When the input parameters do not describe a valid compaction, the
+  // function will try to fix the input_files by adding necessary
+  // files.  If it's not possible to conver an invalid input_files
+  // into a valid one by adding more files, the function will return a
+  // non-ok status with specific reason.
 #ifndef ROCKSDB_LITE
-  Status SanitizeCompactionInputFiles(std::unordered_set<uint64_t>* input_files,
-                                      const ColumnFamilyMetaData& cf_meta,
-                                      const int output_level) const;
+  Status SanitizeCompactionInputFiles(
+      std::unordered_set<uint64_t>* input_files,
+      const ColumnFamilyMetaData& cf_meta,
+      const int output_level) const;
 #endif  // ROCKSDB_LITE

  // Free up the files that participated in a compaction
@@ -84,15 +88,15 @@ class CompactionPicker {
  void ReleaseCompactionFiles(Compaction* c, Status status);

  // Returns true if any one of the specified files are being compacted
-  bool AreFilesInCompaction(const std::vector<FileMetaData*>& files);
+  bool FilesInCompaction(const std::vector<FileMetaData*>& files);

  // Takes a list of CompactionInputFiles and returns a (manual) Compaction
  // object.
-  Compaction* CompactFiles(const CompactionOptions& compact_options,
-                           const std::vector<CompactionInputFiles>& input_files,
-                           int output_level, VersionStorageInfo* vstorage,
-                           const MutableCFOptions& mutable_cf_options,
-                           uint32_t output_path_id);
+  Compaction* FormCompaction(
+      const CompactionOptions& compact_options,
+      const std::vector<CompactionInputFiles>& input_files, int output_level,
+      VersionStorageInfo* vstorage, const MutableCFOptions& mutable_cf_options,
+      uint32_t output_path_id);

  // Converts a set of compaction input file numbers into
  // a list of CompactionInputFiles.
@@ -102,37 +106,32 @@ class CompactionPicker {
      const VersionStorageInfo* vstorage,
      const CompactionOptions& compact_options) const;

+  // Used in universal compaction when the enabled_trivial_move
+  // option is set. Checks whether there are any overlapping files
+  // in the input. Returns true if the input files are non
+  // overlapping.
+  bool IsInputNonOverlapping(Compaction* c);
+
  // Is there currently a compaction involving level 0 taking place
  bool IsLevel0CompactionInProgress() const {
    return !level0_compactions_in_progress_.empty();
  }

-  // Return true if the passed key range overlap with a compaction output
-  // that is currently running.
-  bool RangeOverlapWithCompaction(const Slice& smallest_user_key,
-                                  const Slice& largest_user_key,
-                                  int level) const;
+ protected:
+  int NumberLevels() const { return ioptions_.num_levels; }

  // Stores the minimal range that covers all entries in inputs in
  // *smallest, *largest.
  // REQUIRES: inputs is not empty
-  void GetRange(const CompactionInputFiles& inputs, InternalKey* smallest,
-                InternalKey* largest) const;
+  void GetRange(const CompactionInputFiles& inputs,
+                InternalKey* smallest, InternalKey* largest);

  // Stores the minimal range that covers all entries in inputs1 and inputs2
  // in *smallest, *largest.
  // REQUIRES: inputs is not empty
  void GetRange(const CompactionInputFiles& inputs1,
-                const CompactionInputFiles& inputs2, InternalKey* smallest,
-                InternalKey* largest) const;
-
-  // Stores the minimal range that covers all entries in inputs
-  // in *smallest, *largest.
-  // REQUIRES: inputs is not empty (at least on entry have one file)
-  void GetRange(const std::vector<CompactionInputFiles>& inputs,
-                InternalKey* smallest, InternalKey* largest) const;
-
-  int NumberLevels() const { return ioptions_.num_levels; }
+                const CompactionInputFiles& inputs2,
+                InternalKey* smallest, InternalKey* largest);

  // Add more files to the inputs on "level" to make sure that
  // no newer version of a key is compacted to "level+1" while leaving an older
@@ -144,19 +143,14 @@ class CompactionPicker {
  // populated.
  //
  // Will return false if it is impossible to apply this compaction.
-  bool ExpandInputsToCleanCut(const std::string& cf_name,
+  bool ExpandWhileOverlapping(const std::string& cf_name,
                              VersionStorageInfo* vstorage,
                              CompactionInputFiles* inputs);

  // Returns true if any one of the parent files are being compacted
-  bool IsRangeInCompaction(VersionStorageInfo* vstorage,
-                           const InternalKey* smallest,
-                           const InternalKey* largest, int level, int* index);
-
-  // Returns true if the key range that `inputs` files cover overlap with the
-  // key range of a currently running compaction.
-  bool FilesRangeOverlapWithCompaction(
-      const std::vector<CompactionInputFiles>& inputs, int level) const;
+  bool RangeInCompaction(VersionStorageInfo* vstorage,
+                         const InternalKey* smallest,
+                         const InternalKey* largest, int level, int* index);

  bool SetupOtherInputs(const std::string& cf_name,
                        const MutableCFOptions& mutable_cf_options,
@@ -170,38 +164,21 @@ class CompactionPicker {
                       const CompactionInputFiles& output_level_inputs,
                       std::vector<FileMetaData*>* grandparents);

-  // Register this compaction in the set of running compactions
-  void RegisterCompaction(Compaction* c);
-
-  // Remove this compaction from the set of running compactions
-  void UnregisterCompaction(Compaction* c);
-
-  std::set<Compaction*>* level0_compactions_in_progress() {
-    return &level0_compactions_in_progress_;
-  }
-  std::unordered_set<Compaction*>* compactions_in_progress() {
-    return &compactions_in_progress_;
-  }
-
- protected:
  const ImmutableCFOptions& ioptions_;

-// A helper function to SanitizeCompactionInputFiles() that
-// sanitizes "input_files" by adding necessary files.
+  // A helper function to SanitizeCompactionInputFiles() that
+  // sanitizes "input_files" by adding necessary files.
 #ifndef ROCKSDB_LITE
  virtual Status SanitizeCompactionInputFilesForAllLevels(
      std::unordered_set<uint64_t>* input_files,
-      const ColumnFamilyMetaData& cf_meta, const int output_level) const;
+      const ColumnFamilyMetaData& cf_meta,
+      const int output_level) const;
 #endif  // ROCKSDB_LITE

  // Keeps track of all compactions that are running on Level0.
-  // Protected by DB mutex
+  // It is protected by DB mutex
  std::set<Compaction*> level0_compactions_in_progress_;

-  // Keeps track of all compactions that are running.
-  // Protected by DB mutex
-  std::unordered_set<Compaction*> compactions_in_progress_;
-
  const InternalKeyComparator* const icmp_;
 };

@@ -215,11 +192,103 @@ class LevelCompactionPicker : public CompactionPicker {
                                     VersionStorageInfo* vstorage,
                                     LogBuffer* log_buffer) override;

-  virtual bool NeedsCompaction(
-      const VersionStorageInfo* vstorage) const override;
+  virtual bool NeedsCompaction(const VersionStorageInfo* vstorage) const
+      override;
+
+  // Pick a path ID to place a newly generated file, with its level
+  static uint32_t GetPathId(const ImmutableCFOptions& ioptions,
+                            const MutableCFOptions& mutable_cf_options,
+                            int level);
+
+ private:
+  // For the specfied level, pick a file that we want to compact.
+  // Returns false if there is no file to compact.
+  // If it returns true, inputs->files.size() will be exactly one.
+  // If level is 0 and there is already a compaction on that level, this
+  // function will return false.
+  bool PickCompactionBySize(VersionStorageInfo* vstorage, int level,
+                            int output_level, CompactionInputFiles* inputs,
+                            int* parent_index, int* base_index);
+
+  // If there is any file marked for compaction, put put it into inputs.
+  // This is still experimental. It will return meaningful results only if
+  // clients call experimental feature SuggestCompactRange()
+  void PickFilesMarkedForCompactionExperimental(const std::string& cf_name,
+                                                VersionStorageInfo* vstorage,
+                                                CompactionInputFiles* inputs,
+                                                int* level, int* output_level);
 };

 #ifndef ROCKSDB_LITE
+class UniversalCompactionPicker : public CompactionPicker {
+ public:
+  UniversalCompactionPicker(const ImmutableCFOptions& ioptions,
+                            const InternalKeyComparator* icmp)
+      : CompactionPicker(ioptions, icmp) {}
+  virtual Compaction* PickCompaction(const std::string& cf_name,
+                                     const MutableCFOptions& mutable_cf_options,
+                                     VersionStorageInfo* vstorage,
+                                     LogBuffer* log_buffer) override;
+
+  virtual int MaxOutputLevel() const override { return NumberLevels() - 1; }
+
+  virtual bool NeedsCompaction(const VersionStorageInfo* vstorage) const
+      override;
+
+ private:
+  struct SortedRun {
+    SortedRun(int _level, FileMetaData* _file, uint64_t _size,
+              uint64_t _compensated_file_size, bool _being_compacted)
+        : level(_level),
+          file(_file),
+          size(_size),
+          compensated_file_size(_compensated_file_size),
+          being_compacted(_being_compacted) {
+      assert(compensated_file_size > 0);
+      assert(level != 0 || file != nullptr);
+    }
+
+    void Dump(char* out_buf, size_t out_buf_size,
+              bool print_path = false) const;
+
+    // sorted_run_count is added into the string to print
+    void DumpSizeInfo(char* out_buf, size_t out_buf_size,
+                      size_t sorted_run_count) const;
+
+    int level;
+    // `file` Will be null for level > 0. For level = 0, the sorted run is
+    // for this file.
+    FileMetaData* file;
+    // For level > 0, `size` and `compensated_file_size` are sum of sizes all
+    // files in the level. `being_compacted` should be the same for all files
+    // in a non-zero level. Use the value here.
+    uint64_t size;
+    uint64_t compensated_file_size;
+    bool being_compacted;
+  };
+
+  // Pick Universal compaction to limit read amplification
+  Compaction* PickCompactionUniversalReadAmp(
+      const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
+      VersionStorageInfo* vstorage, double score, unsigned int ratio,
+      unsigned int num_files, const std::vector<SortedRun>& sorted_runs,
+      LogBuffer* log_buffer);
+
+  // Pick Universal compaction to limit space amplification.
+  Compaction* PickCompactionUniversalSizeAmp(
+      const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
+      VersionStorageInfo* vstorage, double score,
+      const std::vector<SortedRun>& sorted_runs, LogBuffer* log_buffer);
+
+  static std::vector<SortedRun> CalculateSortedRuns(
+      const VersionStorageInfo& vstorage, const ImmutableCFOptions& ioptions);
+
+  // Pick a path ID to place a newly generated file, with its estimated file
+  // size.
+  static uint32_t GetPathId(const ImmutableCFOptions& ioptions,
+                            uint64_t file_size);
+};
+
 class FIFOCompactionPicker : public CompactionPicker {
 public:
  FIFOCompactionPicker(const ImmutableCFOptions& ioptions,
@@ -238,17 +307,19 @@ class FIFOCompactionPicker : public CompactionPicker {
      InternalKey** compaction_end, bool* manual_conflict) override;

  // The maximum allowed output level.  Always returns 0.
-  virtual int MaxOutputLevel() const override { return 0; }
+  virtual int MaxOutputLevel() const override {
+    return 0;
+  }

-  virtual bool NeedsCompaction(
-      const VersionStorageInfo* vstorage) const override;
+  virtual bool NeedsCompaction(const VersionStorageInfo* vstorage) const
+      override;
 };

 class NullCompactionPicker : public CompactionPicker {
 public:
  NullCompactionPicker(const ImmutableCFOptions& ioptions,
-                       const InternalKeyComparator* icmp)
-      : CompactionPicker(ioptions, icmp) {}
+                       const InternalKeyComparator* icmp) :
+      CompactionPicker(ioptions, icmp) {}
  virtual ~NullCompactionPicker() {}

  // Always return "nullptr"
@@ -271,16 +342,14 @@ class NullCompactionPicker : public CompactionPicker {
  }

  // Always returns false.
-  virtual bool NeedsCompaction(
-      const VersionStorageInfo* vstorage) const override {
+  virtual bool NeedsCompaction(const VersionStorageInfo* vstorage) const
+      override {
    return false;
  }
 };
 #endif  // !ROCKSDB_LITE

 CompressionType GetCompressionType(const ImmutableCFOptions& ioptions,
-                                   const VersionStorageInfo* vstorage,
-                                   const MutableCFOptions& mutable_cf_options,
                                   int level, int base_level,
                                   const bool enable_compression = true);

@@ -3,12 +3,11 @@
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

+#include "db/compaction.h"
 #include "db/compaction_picker.h"
 #include <limits>
 #include <string>
 #include <utility>
-#include "db/compaction.h"
-#include "db/compaction_picker_universal.h"

 #include "util/logging.h"
 #include "util/string_util.h"
@@ -49,7 +48,7 @@ class CompactionPickerTest : public testing::Test {
      : ucmp_(BytewiseComparator()),
        icmp_(ucmp_),
        ioptions_(options_),
-        mutable_cf_options_(options_),
+        mutable_cf_options_(options_, ioptions_),
        level_compaction_picker(ioptions_, &icmp_),
        cf_name_("dummy"),
        log_buffer_(InfoLogLevel::INFO_LEVEL, &logger_),
@@ -67,8 +66,8 @@ class CompactionPickerTest : public testing::Test {
  void NewVersionStorage(int num_levels, CompactionStyle style) {
    DeleteVersionStorage();
    options_.num_levels = num_levels;
-    vstorage_.reset(new VersionStorageInfo(&icmp_, ucmp_, options_.num_levels,
-                                           style, nullptr, false));
+    vstorage_.reset(new VersionStorageInfo(
+        &icmp_, ucmp_, options_.num_levels, style, nullptr));
    vstorage_->CalculateBaseBytes(ioptions_, mutable_cf_options_);
  }

@@ -116,13 +115,12 @@ class CompactionPickerTest : public testing::Test {

  void UpdateVersionStorageInfo() {
    vstorage_->CalculateBaseBytes(ioptions_, mutable_cf_options_);
-    vstorage_->UpdateFilesByCompactionPri(ioptions_.compaction_pri);
+    vstorage_->UpdateFilesByCompactionPri(mutable_cf_options_);
    vstorage_->UpdateNumNonEmptyLevels();
    vstorage_->GenerateFileIndexer();
    vstorage_->GenerateLevelFilesBrief();
-    vstorage_->ComputeCompactionScore(ioptions_, mutable_cf_options_);
+    vstorage_->ComputeCompactionScore(mutable_cf_options_, fifo_options_);
    vstorage_->GenerateLevel0NonOverlapping();
-    vstorage_->ComputeFilesMarkedForCompaction();
    vstorage_->SetFinalized();
  }
 };
@@ -198,7 +196,7 @@ TEST_F(CompactionPickerTest, LevelMaxScore) {
  mutable_cf_options_.target_file_size_base = 10000000;
  mutable_cf_options_.target_file_size_multiplier = 10;
  mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
-  Add(0, 1U, "150", "200", 1000000U);
+  Add(0, 1U, "150", "200", 1000000000U);
  // Level 1 score 1.2
  Add(1, 66U, "150", "200", 6000000U);
  Add(1, 88U, "201", "300", 6000000U);
@@ -352,10 +350,10 @@ TEST_F(CompactionPickerTest, Level0TriggerDynamic4) {
 TEST_F(CompactionPickerTest, LevelTriggerDynamic4) {
  int num_levels = ioptions_.num_levels;
  ioptions_.level_compaction_dynamic_level_bytes = true;
-  ioptions_.compaction_pri = kMinOverlappingRatio;
  mutable_cf_options_.level0_file_num_compaction_trigger = 2;
  mutable_cf_options_.max_bytes_for_level_base = 200;
  mutable_cf_options_.max_bytes_for_level_multiplier = 10;
+  mutable_cf_options_.compaction_pri = kMinOverlappingRatio;
  NewVersionStorage(num_levels, kCompactionStyleLevel);
  Add(0, 1U, "150", "200");
  Add(num_levels - 1, 3U, "200", "250", 300U);
@@ -469,6 +467,7 @@ TEST_F(CompactionPickerTest, NeedsCompactionFIFO) {
  fifo_options_.max_table_files_size = kMaxSize;
  ioptions_.compaction_options_fifo = fifo_options_;
  FIFOCompactionPicker fifo_compaction_picker(ioptions_, &icmp_);
+
  UpdateVersionStorageInfo();
  // must return false when there's no files.
  ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()), false);
@@ -491,10 +490,10 @@ TEST_F(CompactionPickerTest, NeedsCompactionFIFO) {

 TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) {
  NewVersionStorage(6, kCompactionStyleLevel);
-  ioptions_.compaction_pri = kMinOverlappingRatio;
  mutable_cf_options_.target_file_size_base = 10000000;
  mutable_cf_options_.target_file_size_multiplier = 10;
  mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
+  mutable_cf_options_.compaction_pri = kMinOverlappingRatio;

  Add(2, 6U, "150", "179", 50000000U);
  Add(2, 7U, "180", "220", 50000000U);
@@ -518,10 +517,10 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) {

 TEST_F(CompactionPickerTest, CompactionPriMinOverlapping2) {
  NewVersionStorage(6, kCompactionStyleLevel);
-  ioptions_.compaction_pri = kMinOverlappingRatio;
  mutable_cf_options_.target_file_size_base = 10000000;
  mutable_cf_options_.target_file_size_multiplier = 10;
  mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
+  mutable_cf_options_.compaction_pri = kMinOverlappingRatio;

  Add(2, 6U, "150", "175",
      60000000U);  // Overlaps with file 26, 27, total size 521M
@@ -548,9 +547,9 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping2) {

 TEST_F(CompactionPickerTest, CompactionPriMinOverlapping3) {
  NewVersionStorage(6, kCompactionStyleLevel);
-  ioptions_.compaction_pri = kMinOverlappingRatio;
  mutable_cf_options_.max_bytes_for_level_base = 10000000;
  mutable_cf_options_.max_bytes_for_level_multiplier = 10;
+  mutable_cf_options_.compaction_pri = kMinOverlappingRatio;

  // file 7 and 8 over lap with the same file, but file 8 is smaller so
  // it will be picked.
@@ -600,7 +599,7 @@ TEST_F(CompactionPickerTest, ParentIndexResetBug) {
 // ranges (with different sequence numbers) in the input files.
 TEST_F(CompactionPickerTest, OverlappingUserKeys) {
  NewVersionStorage(6, kCompactionStyleLevel);
-  ioptions_.compaction_pri = kByCompensatedSize;
+  mutable_cf_options_.compaction_pri = kByCompensatedSize;

  Add(1, 1U, "100", "150", 1U);
  // Overlapping user keys
@@ -623,9 +622,9 @@ TEST_F(CompactionPickerTest, OverlappingUserKeys2) {
  // Overlapping user keys on same level and output level
  Add(1, 1U, "200", "400", 1000000000U);
  Add(1, 2U, "400", "500", 1U, 0, 0);
-  Add(2, 3U, "000", "100", 1U);
-  Add(2, 4U, "100", "600", 1U, 0, 0);
-  Add(2, 5U, "600", "700", 1U, 0, 0);
+  Add(2, 3U, "400", "600", 1U);
+  // The following file is not in the compaction despite overlapping user keys
+  Add(2, 4U, "600", "700", 1U, 0, 0);
  UpdateVersionStorageInfo();

  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
@@ -633,12 +632,10 @@ TEST_F(CompactionPickerTest, OverlappingUserKeys2) {
  ASSERT_TRUE(compaction.get() != nullptr);
  ASSERT_EQ(2U, compaction->num_input_levels());
  ASSERT_EQ(2U, compaction->num_input_files(0));
-  ASSERT_EQ(3U, compaction->num_input_files(1));
+  ASSERT_EQ(1U, compaction->num_input_files(1));
  ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
  ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber());
  ASSERT_EQ(3U, compaction->input(1, 0)->fd.GetNumber());
-  ASSERT_EQ(4U, compaction->input(1, 1)->fd.GetNumber());
-  ASSERT_EQ(5U, compaction->input(1, 2)->fd.GetNumber());
 }

 TEST_F(CompactionPickerTest, OverlappingUserKeys3) {
@@ -670,159 +667,6 @@ TEST_F(CompactionPickerTest, OverlappingUserKeys3) {
  ASSERT_EQ(7U, compaction->input(1, 1)->fd.GetNumber());
 }

-TEST_F(CompactionPickerTest, OverlappingUserKeys4) {
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.max_bytes_for_level_base = 1000000;
-
-  Add(1, 1U, "100", "150", 1U);
-  Add(1, 2U, "150", "199", 1U, 0, 0);
-  Add(1, 3U, "200", "250", 1100000U, 0, 0);
-  Add(1, 4U, "251", "300", 1U, 0, 0);
-  Add(1, 5U, "300", "350", 1U, 0, 0);
-
-  Add(2, 6U, "100", "115", 1U);
-  Add(2, 7U, "125", "325", 1U);
-  Add(2, 8U, "350", "400", 1U);
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(2U, compaction->num_input_levels());
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(1U, compaction->num_input_files(1));
-  ASSERT_EQ(3U, compaction->input(0, 0)->fd.GetNumber());
-  ASSERT_EQ(7U, compaction->input(1, 0)->fd.GetNumber());
-}
-
-TEST_F(CompactionPickerTest, OverlappingUserKeys5) {
-  NewVersionStorage(6, kCompactionStyleLevel);
-  // Overlapping user keys on same level and output level
-  Add(1, 1U, "200", "400", 1000000000U);
-  Add(1, 2U, "400", "500", 1U, 0, 0);
-  Add(2, 3U, "000", "100", 1U);
-  Add(2, 4U, "100", "600", 1U, 0, 0);
-  Add(2, 5U, "600", "700", 1U, 0, 0);
-
-  vstorage_->LevelFiles(2)[2]->being_compacted = true;
-
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() == nullptr);
-}
-
-TEST_F(CompactionPickerTest, OverlappingUserKeys6) {
-  NewVersionStorage(6, kCompactionStyleLevel);
-  // Overlapping user keys on same level and output level
-  Add(1, 1U, "200", "400", 1U, 0, 0);
-  Add(1, 2U, "401", "500", 1U, 0, 0);
-  Add(2, 3U, "000", "100", 1U);
-  Add(2, 4U, "100", "300", 1U, 0, 0);
-  Add(2, 5U, "305", "450", 1U, 0, 0);
-  Add(2, 6U, "460", "600", 1U, 0, 0);
-  Add(2, 7U, "600", "700", 1U, 0, 0);
-
-  vstorage_->LevelFiles(1)[0]->marked_for_compaction = true;
-  vstorage_->LevelFiles(1)[1]->marked_for_compaction = true;
-
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(2U, compaction->num_input_levels());
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(3U, compaction->num_input_files(1));
-}
-
-TEST_F(CompactionPickerTest, OverlappingUserKeys7) {
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.max_compaction_bytes = 100000000000u;
-  // Overlapping user keys on same level and output level
-  Add(1, 1U, "200", "400", 1U, 0, 0);
-  Add(1, 2U, "401", "500", 1000000000U, 0, 0);
-  Add(2, 3U, "100", "250", 1U);
-  Add(2, 4U, "300", "600", 1U, 0, 0);
-  Add(2, 5U, "600", "800", 1U, 0, 0);
-
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(2U, compaction->num_input_levels());
-  ASSERT_GE(1U, compaction->num_input_files(0));
-  ASSERT_GE(2U, compaction->num_input_files(1));
-  // File 5 has to be included in the compaction
-  ASSERT_EQ(5U, compaction->inputs(1)->back()->fd.GetNumber());
-}
-
-TEST_F(CompactionPickerTest, OverlappingUserKeys8) {
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.max_compaction_bytes = 100000000000u;
-  // grow the number of inputs in "level" without
-  // changing the number of "level+1" files we pick up
-  // Expand input level as much as possible
-  // no overlapping case
-  Add(1, 1U, "101", "150", 1U);
-  Add(1, 2U, "151", "200", 1U);
-  Add(1, 3U, "201", "300", 1000000000U);
-  Add(1, 4U, "301", "400", 1U);
-  Add(1, 5U, "401", "500", 1U);
-  Add(2, 6U, "150", "200", 1U);
-  Add(2, 7U, "200", "450", 1U, 0, 0);
-  Add(2, 8U, "500", "600", 1U);
-
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(2U, compaction->num_input_levels());
-  ASSERT_EQ(3U, compaction->num_input_files(0));
-  ASSERT_EQ(2U, compaction->num_input_files(1));
-  ASSERT_EQ(2U, compaction->input(0, 0)->fd.GetNumber());
-  ASSERT_EQ(3U, compaction->input(0, 1)->fd.GetNumber());
-  ASSERT_EQ(4U, compaction->input(0, 2)->fd.GetNumber());
-  ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber());
-  ASSERT_EQ(7U, compaction->input(1, 1)->fd.GetNumber());
-}
-
-TEST_F(CompactionPickerTest, OverlappingUserKeys9) {
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.max_compaction_bytes = 100000000000u;
-  // grow the number of inputs in "level" without
-  // changing the number of "level+1" files we pick up
-  // Expand input level as much as possible
-  // overlapping case
-  Add(1, 1U, "121", "150", 1U);
-  Add(1, 2U, "151", "200", 1U);
-  Add(1, 3U, "201", "300", 1000000000U);
-  Add(1, 4U, "301", "400", 1U);
-  Add(1, 5U, "401", "500", 1U);
-  Add(2, 6U, "100", "120", 1U);
-  Add(2, 7U, "150", "200", 1U);
-  Add(2, 8U, "200", "450", 1U, 0, 0);
-  Add(2, 9U, "501", "600", 1U);
-
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(2U, compaction->num_input_levels());
-  ASSERT_EQ(5U, compaction->num_input_files(0));
-  ASSERT_EQ(2U, compaction->num_input_files(1));
-  ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
-  ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber());
-  ASSERT_EQ(3U, compaction->input(0, 2)->fd.GetNumber());
-  ASSERT_EQ(4U, compaction->input(0, 3)->fd.GetNumber());
-  ASSERT_EQ(7U, compaction->input(1, 0)->fd.GetNumber());
-  ASSERT_EQ(8U, compaction->input(1, 1)->fd.GetNumber());
-}
-
 TEST_F(CompactionPickerTest, NotScheduleL1IfL0WithHigherPri1) {
  NewVersionStorage(6, kCompactionStyleLevel);
  mutable_cf_options_.level0_file_num_compaction_trigger = 2;
@@ -932,22 +776,19 @@ TEST_F(CompactionPickerTest, EstimateCompactionBytesNeeded1) {
  Add(1, 4U, "400", "500", 600);
  Add(1, 5U, "600", "700", 600);
  // Level 2 is less than target 10000 even added size of level 1
-  // Size ratio of L2/L1 is 9600 / 1200 = 8
  Add(2, 6U, "150", "200", 2500);
  Add(2, 7U, "201", "210", 2000);
-  Add(2, 8U, "300", "310", 2600);
+  Add(2, 8U, "300", "310", 2500);
  Add(2, 9U, "400", "500", 2500);
  // Level 3 exceeds target 100,000 of 1000
  Add(3, 10U, "400", "500", 101000);
-  // Level 4 exceeds target 1,000,000 by 900 after adding size from level 3
-  // Size ratio L4/L3 is 9.9
-  // After merge from L3, L4 size is 1000900
-  Add(4, 11U, "400", "500", 999900);
-  Add(5, 11U, "400", "500", 8007200);
+  // Level 4 exceeds target 1,000,000 of 500 after adding size from level 3
+  Add(4, 11U, "400", "500", 999500);
+  Add(5, 11U, "400", "500", 8000000);

  UpdateVersionStorageInfo();

-  ASSERT_EQ(200u * 9u + 10900u + 900u * 9,
+  ASSERT_EQ(2200u + 11000u + 5500u,
            vstorage_->estimated_compaction_needed_bytes());
 }

@@ -963,42 +804,17 @@ TEST_F(CompactionPickerTest, EstimateCompactionBytesNeeded2) {
  Add(0, 4U, "150", "200", 200);
  Add(0, 5U, "150", "200", 200);
  Add(0, 6U, "150", "200", 200);
-  // Level 1 size will be 1400 after merging with L0
+  // Level 1 is over target by
  Add(1, 7U, "400", "500", 200);
  Add(1, 8U, "600", "700", 200);
  // Level 2 is less than target 10000 even added size of level 1
-  Add(2, 9U, "150", "200", 9100);
-  // Level 3 over the target, but since level 4 is empty, we assume it will be
-  // a trivial move.
+  Add(2, 9U, "150", "200", 9500);
  Add(3, 10U, "400", "500", 101000);

  UpdateVersionStorageInfo();

-  // estimated L1->L2 merge: 400 * (9100.0 / 1400.0 + 1.0)
-  ASSERT_EQ(1400u + 3000u, vstorage_->estimated_compaction_needed_bytes());
-}
-
-TEST_F(CompactionPickerTest, EstimateCompactionBytesNeeded3) {
-  int num_levels = ioptions_.num_levels;
-  ioptions_.level_compaction_dynamic_level_bytes = false;
-  mutable_cf_options_.level0_file_num_compaction_trigger = 3;
-  mutable_cf_options_.max_bytes_for_level_base = 1000;
-  mutable_cf_options_.max_bytes_for_level_multiplier = 10;
-  NewVersionStorage(num_levels, kCompactionStyleLevel);
-  Add(0, 1U, "150", "200", 2000);
-  Add(0, 2U, "150", "200", 2000);
-  Add(0, 4U, "150", "200", 2000);
-  Add(0, 5U, "150", "200", 2000);
-  Add(0, 6U, "150", "200", 1000);
-  // Level 1 size will be 10000 after merging with L0
-  Add(1, 7U, "400", "500", 500);
-  Add(1, 8U, "600", "700", 500);
-
-  Add(2, 9U, "150", "200", 10000);
-
-  UpdateVersionStorageInfo();
-
-  ASSERT_EQ(10000u + 18000u, vstorage_->estimated_compaction_needed_bytes());
+  ASSERT_EQ(1400u + 4400u + 11000u,
+            vstorage_->estimated_compaction_needed_bytes());
 }

 TEST_F(CompactionPickerTest, EstimateCompactionBytesNeededDynamicLevel) {
@@ -1022,14 +838,12 @@ TEST_F(CompactionPickerTest, EstimateCompactionBytesNeededDynamicLevel) {
  // num_levels - 3 is over target by 100 + 1000
  Add(num_levels - 3, 7U, "400", "500", 300);
  Add(num_levels - 3, 8U, "600", "700", 300);
-  // num_levels - 2 is over target by 1100 + 200
-  Add(num_levels - 2, 9U, "150", "200", 5200);
+  // Level 2 is over target by 1100 + 100
+  Add(num_levels - 2, 9U, "150", "200", 5100);

  UpdateVersionStorageInfo();

-  // Merging to the second last level: (5200 / 1600 + 1) * 1100
-  // Merging to the last level: (50000 / 6300 + 1) * 1300
-  ASSERT_EQ(1600u + 4675u + 11617u,
+  ASSERT_EQ(1600u + 12100u + 13200u,
            vstorage_->estimated_compaction_needed_bytes());
 }

@@ -1191,101 +1005,6 @@ TEST_F(CompactionPickerTest, IsBottommostLevelTest) {
  DeleteVersionStorage();
 }

-TEST_F(CompactionPickerTest, MaxCompactionBytesHit) {
-  mutable_cf_options_.max_bytes_for_level_base = 1000000u;
-  mutable_cf_options_.max_compaction_bytes = 800000u;
-  ioptions_.level_compaction_dynamic_level_bytes = false;
-  NewVersionStorage(6, kCompactionStyleLevel);
-  // A compaction should be triggered and pick file 2 and 5.
-  // It can expand because adding file 1 and 3, the compaction size will
-  // exceed mutable_cf_options_.max_bytes_for_level_base.
-  Add(1, 1U, "100", "150", 300000U);
-  Add(1, 2U, "151", "200", 300001U, 0, 0);
-  Add(1, 3U, "201", "250", 300000U, 0, 0);
-  Add(1, 4U, "251", "300", 300000U, 0, 0);
-  Add(2, 5U, "100", "256", 1U);
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(2U, compaction->num_input_levels());
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(1U, compaction->num_input_files(1));
-  ASSERT_EQ(2U, compaction->input(0, 0)->fd.GetNumber());
-  ASSERT_EQ(5U, compaction->input(1, 0)->fd.GetNumber());
-}
-
-TEST_F(CompactionPickerTest, MaxCompactionBytesNotHit) {
-  mutable_cf_options_.max_bytes_for_level_base = 800000u;
-  mutable_cf_options_.max_compaction_bytes = 1000000u;
-  ioptions_.level_compaction_dynamic_level_bytes = false;
-  NewVersionStorage(6, kCompactionStyleLevel);
-  // A compaction should be triggered and pick file 2 and 5.
-  // and it expands to file 1 and 3 too.
-  Add(1, 1U, "100", "150", 300000U);
-  Add(1, 2U, "151", "200", 300001U, 0, 0);
-  Add(1, 3U, "201", "250", 300000U, 0, 0);
-  Add(1, 4U, "251", "300", 300000U, 0, 0);
-  Add(2, 5U, "000", "251", 1U);
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(2U, compaction->num_input_levels());
-  ASSERT_EQ(3U, compaction->num_input_files(0));
-  ASSERT_EQ(1U, compaction->num_input_files(1));
-  ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
-  ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber());
-  ASSERT_EQ(3U, compaction->input(0, 2)->fd.GetNumber());
-  ASSERT_EQ(5U, compaction->input(1, 0)->fd.GetNumber());
-}
-
-TEST_F(CompactionPickerTest, IsTrivialMoveOn) {
-  mutable_cf_options_.max_bytes_for_level_base = 10000u;
-  mutable_cf_options_.max_compaction_bytes = 10001u;
-  ioptions_.level_compaction_dynamic_level_bytes = false;
-  NewVersionStorage(6, kCompactionStyleLevel);
-  // A compaction should be triggered and pick file 2
-  Add(1, 1U, "100", "150", 3000U);
-  Add(1, 2U, "151", "200", 3001U);
-  Add(1, 3U, "201", "250", 3000U);
-  Add(1, 4U, "251", "300", 3000U);
-
-  Add(3, 5U, "120", "130", 7000U);
-  Add(3, 6U, "170", "180", 7000U);
-  Add(3, 5U, "220", "230", 7000U);
-  Add(3, 5U, "270", "280", 7000U);
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-    cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_TRUE(compaction->IsTrivialMove());
-}
-
-TEST_F(CompactionPickerTest, IsTrivialMoveOff) {
-  mutable_cf_options_.max_bytes_for_level_base = 1000000u;
-  mutable_cf_options_.max_compaction_bytes = 10000u;
-  ioptions_.level_compaction_dynamic_level_bytes = false;
-  NewVersionStorage(6, kCompactionStyleLevel);
-  // A compaction should be triggered and pick all files from level 1
-  Add(1, 1U, "100", "150", 300000U, 0, 0);
-  Add(1, 2U, "150", "200", 300000U, 0, 0);
-  Add(1, 3U, "200", "250", 300000U, 0, 0);
-  Add(1, 4U, "250", "300", 300000U, 0, 0);
-
-  Add(3, 5U, "120", "130", 6000U);
-  Add(3, 6U, "140", "150", 6000U);
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-    cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_FALSE(compaction->IsTrivialMove());
-}
-
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -1,735 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/compaction_picker_universal.h"
-#ifndef ROCKSDB_LITE
-
-#ifndef __STDC_FORMAT_MACROS
-#define __STDC_FORMAT_MACROS
-#endif
-
-#include <inttypes.h>
-#include <limits>
-#include <queue>
-#include <string>
-#include <utility>
-#include "db/column_family.h"
-#include "monitoring/statistics.h"
-#include "util/filename.h"
-#include "util/log_buffer.h"
-#include "util/random.h"
-#include "util/string_util.h"
-#include "util/sync_point.h"
-
-namespace rocksdb {
-namespace {
-// Used in universal compaction when trivial move is enabled.
-// This structure is used for the construction of min heap
-// that contains the file meta data, the level of the file
-// and the index of the file in that level
-
-struct InputFileInfo {
-  InputFileInfo() : f(nullptr) {}
-
-  FileMetaData* f;
-  size_t level;
-  size_t index;
-};
-
-// Used in universal compaction when trivial move is enabled.
-// This comparator is used for the construction of min heap
-// based on the smallest key of the file.
-struct SmallestKeyHeapComparator {
-  explicit SmallestKeyHeapComparator(const Comparator* ucmp) { ucmp_ = ucmp; }
-
-  bool operator()(InputFileInfo i1, InputFileInfo i2) const {
-    return (ucmp_->Compare(i1.f->smallest.user_key(),
-                           i2.f->smallest.user_key()) > 0);
-  }
-
- private:
-  const Comparator* ucmp_;
-};
-
-typedef std::priority_queue<InputFileInfo, std::vector<InputFileInfo>,
-                            SmallestKeyHeapComparator>
-    SmallestKeyHeap;
-
-// This function creates the heap that is used to find if the files are
-// overlapping during universal compaction when the allow_trivial_move
-// is set.
-SmallestKeyHeap create_level_heap(Compaction* c, const Comparator* ucmp) {
-  SmallestKeyHeap smallest_key_priority_q =
-      SmallestKeyHeap(SmallestKeyHeapComparator(ucmp));
-
-  InputFileInfo input_file;
-
-  for (size_t l = 0; l < c->num_input_levels(); l++) {
-    if (c->num_input_files(l) != 0) {
-      if (l == 0 && c->start_level() == 0) {
-        for (size_t i = 0; i < c->num_input_files(0); i++) {
-          input_file.f = c->input(0, i);
-          input_file.level = 0;
-          input_file.index = i;
-          smallest_key_priority_q.push(std::move(input_file));
-        }
-      } else {
-        input_file.f = c->input(l, 0);
-        input_file.level = l;
-        input_file.index = 0;
-        smallest_key_priority_q.push(std::move(input_file));
-      }
-    }
-  }
-  return smallest_key_priority_q;
-}
-
-#ifndef NDEBUG
-// smallest_seqno and largest_seqno are set iff. `files` is not empty.
-void GetSmallestLargestSeqno(const std::vector<FileMetaData*>& files,
-                             SequenceNumber* smallest_seqno,
-                             SequenceNumber* largest_seqno) {
-  bool is_first = true;
-  for (FileMetaData* f : files) {
-    assert(f->smallest_seqno <= f->largest_seqno);
-    if (is_first) {
-      is_first = false;
-      *smallest_seqno = f->smallest_seqno;
-      *largest_seqno = f->largest_seqno;
-    } else {
-      if (f->smallest_seqno < *smallest_seqno) {
-        *smallest_seqno = f->smallest_seqno;
-      }
-      if (f->largest_seqno > *largest_seqno) {
-        *largest_seqno = f->largest_seqno;
-      }
-    }
-  }
-}
-#endif
-}  // namespace
-
-// Algorithm that checks to see if there are any overlapping
-// files in the input
-bool UniversalCompactionPicker::IsInputFilesNonOverlapping(Compaction* c) {
-  auto comparator = icmp_->user_comparator();
-  int first_iter = 1;
-
-  InputFileInfo prev, curr, next;
-
-  SmallestKeyHeap smallest_key_priority_q =
-      create_level_heap(c, icmp_->user_comparator());
-
-  while (!smallest_key_priority_q.empty()) {
-    curr = smallest_key_priority_q.top();
-    smallest_key_priority_q.pop();
-
-    if (first_iter) {
-      prev = curr;
-      first_iter = 0;
-    } else {
-      if (comparator->Compare(prev.f->largest.user_key(),
-                              curr.f->smallest.user_key()) >= 0) {
-        // found overlapping files, return false
-        return false;
-      }
-      assert(comparator->Compare(curr.f->largest.user_key(),
-                                 prev.f->largest.user_key()) > 0);
-      prev = curr;
-    }
-
-    next.f = nullptr;
-
-    if (curr.level != 0 && curr.index < c->num_input_files(curr.level) - 1) {
-      next.f = c->input(curr.level, curr.index + 1);
-      next.level = curr.level;
-      next.index = curr.index + 1;
-    }
-
-    if (next.f) {
-      smallest_key_priority_q.push(std::move(next));
-    }
-  }
-  return true;
-}
-
-bool UniversalCompactionPicker::NeedsCompaction(
-    const VersionStorageInfo* vstorage) const {
-  const int kLevel0 = 0;
-  return vstorage->CompactionScore(kLevel0) >= 1;
-}
-
-void UniversalCompactionPicker::SortedRun::Dump(char* out_buf,
-                                                size_t out_buf_size,
-                                                bool print_path) const {
-  if (level == 0) {
-    assert(file != nullptr);
-    if (file->fd.GetPathId() == 0 || !print_path) {
-      snprintf(out_buf, out_buf_size, "file %" PRIu64, file->fd.GetNumber());
-    } else {
-      snprintf(out_buf, out_buf_size, "file %" PRIu64
-                                      "(path "
-                                      "%" PRIu32 ")",
-               file->fd.GetNumber(), file->fd.GetPathId());
-    }
-  } else {
-    snprintf(out_buf, out_buf_size, "level %d", level);
-  }
-}
-
-void UniversalCompactionPicker::SortedRun::DumpSizeInfo(
-    char* out_buf, size_t out_buf_size, size_t sorted_run_count) const {
-  if (level == 0) {
-    assert(file != nullptr);
-    snprintf(out_buf, out_buf_size,
-             "file %" PRIu64 "[%" ROCKSDB_PRIszt
-             "] "
-             "with size %" PRIu64 " (compensated size %" PRIu64 ")",
-             file->fd.GetNumber(), sorted_run_count, file->fd.GetFileSize(),
-             file->compensated_file_size);
-  } else {
-    snprintf(out_buf, out_buf_size,
-             "level %d[%" ROCKSDB_PRIszt
-             "] "
-             "with size %" PRIu64 " (compensated size %" PRIu64 ")",
-             level, sorted_run_count, size, compensated_file_size);
-  }
-}
-
-std::vector<UniversalCompactionPicker::SortedRun>
-UniversalCompactionPicker::CalculateSortedRuns(
-    const VersionStorageInfo& vstorage, const ImmutableCFOptions& ioptions) {
-  std::vector<UniversalCompactionPicker::SortedRun> ret;
-  for (FileMetaData* f : vstorage.LevelFiles(0)) {
-    ret.emplace_back(0, f, f->fd.GetFileSize(), f->compensated_file_size,
-                     f->being_compacted);
-  }
-  for (int level = 1; level < vstorage.num_levels(); level++) {
-    uint64_t total_compensated_size = 0U;
-    uint64_t total_size = 0U;
-    bool being_compacted = false;
-    bool is_first = true;
-    for (FileMetaData* f : vstorage.LevelFiles(level)) {
-      total_compensated_size += f->compensated_file_size;
-      total_size += f->fd.GetFileSize();
-      if (ioptions.compaction_options_universal.allow_trivial_move == true) {
-        if (f->being_compacted) {
-          being_compacted = f->being_compacted;
-        }
-      } else {
-        // Compaction always includes all files for a non-zero level, so for a
-        // non-zero level, all the files should share the same being_compacted
-        // value.
-        // This assumption is only valid when
-        // ioptions.compaction_options_universal.allow_trivial_move is false
-        assert(is_first || f->being_compacted == being_compacted);
-      }
-      if (is_first) {
-        being_compacted = f->being_compacted;
-        is_first = false;
-      }
-    }
-    if (total_compensated_size > 0) {
-      ret.emplace_back(level, nullptr, total_size, total_compensated_size,
-                       being_compacted);
-    }
-  }
-  return ret;
-}
-
-// Universal style of compaction. Pick files that are contiguous in
-// time-range to compact.
-//
-Compaction* UniversalCompactionPicker::PickCompaction(
-    const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-    VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
-  const int kLevel0 = 0;
-  double score = vstorage->CompactionScore(kLevel0);
-  std::vector<SortedRun> sorted_runs =
-      CalculateSortedRuns(*vstorage, ioptions_);
-
-  if (sorted_runs.size() == 0 ||
-      sorted_runs.size() <
-          (unsigned int)mutable_cf_options.level0_file_num_compaction_trigger) {
-    ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: nothing to do\n",
-                     cf_name.c_str());
-    TEST_SYNC_POINT_CALLBACK("UniversalCompactionPicker::PickCompaction:Return",
-                             nullptr);
-    return nullptr;
-  }
-  VersionStorageInfo::LevelSummaryStorage tmp;
-  ROCKS_LOG_BUFFER_MAX_SZ(
-      log_buffer, 3072,
-      "[%s] Universal: sorted runs files(%" ROCKSDB_PRIszt "): %s\n",
-      cf_name.c_str(), sorted_runs.size(), vstorage->LevelSummary(&tmp));
-
-  // Check for size amplification first.
-  Compaction* c;
-  if ((c = PickCompactionToReduceSizeAmp(cf_name, mutable_cf_options, vstorage,
-                                         score, sorted_runs, log_buffer)) !=
-      nullptr) {
-    ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: compacting for size amp\n",
-                     cf_name.c_str());
-  } else {
-    // Size amplification is within limits. Try reducing read
-    // amplification while maintaining file size ratios.
-    unsigned int ratio = ioptions_.compaction_options_universal.size_ratio;
-
-    if ((c = PickCompactionToReduceSortedRuns(
-             cf_name, mutable_cf_options, vstorage, score, ratio, UINT_MAX,
-             sorted_runs, log_buffer)) != nullptr) {
-      ROCKS_LOG_BUFFER(log_buffer,
-                       "[%s] Universal: compacting for size ratio\n",
-                       cf_name.c_str());
-    } else {
-      // Size amplification and file size ratios are within configured limits.
-      // If max read amplification is exceeding configured limits, then force
-      // compaction without looking at filesize ratios and try to reduce
-      // the number of files to fewer than level0_file_num_compaction_trigger.
-      // This is guaranteed by NeedsCompaction()
-      assert(sorted_runs.size() >=
-             static_cast<size_t>(
-                 mutable_cf_options.level0_file_num_compaction_trigger));
-      // Get the total number of sorted runs that are not being compacted
-      int num_sr_not_compacted = 0;
-      for (size_t i = 0; i < sorted_runs.size(); i++) {
-        if (sorted_runs[i].being_compacted == false) {
-          num_sr_not_compacted++;
-        }
-      }
-
-      // The number of sorted runs that are not being compacted is greater than
-      // the maximum allowed number of sorted runs
-      if (num_sr_not_compacted >
-          mutable_cf_options.level0_file_num_compaction_trigger) {
-        unsigned int num_files =
-            num_sr_not_compacted -
-            mutable_cf_options.level0_file_num_compaction_trigger + 1;
-        if ((c = PickCompactionToReduceSortedRuns(
-                 cf_name, mutable_cf_options, vstorage, score, UINT_MAX,
-                 num_files, sorted_runs, log_buffer)) != nullptr) {
-          ROCKS_LOG_BUFFER(log_buffer,
-                           "[%s] Universal: compacting for file num -- %u\n",
-                           cf_name.c_str(), num_files);
-        }
-      }
-    }
-  }
-  if (c == nullptr) {
-    TEST_SYNC_POINT_CALLBACK("UniversalCompactionPicker::PickCompaction:Return",
-                             nullptr);
-    return nullptr;
-  }
-
-  if (ioptions_.compaction_options_universal.allow_trivial_move == true) {
-    c->set_is_trivial_move(IsInputFilesNonOverlapping(c));
-  }
-
-// validate that all the chosen files of L0 are non overlapping in time
-#ifndef NDEBUG
-  SequenceNumber prev_smallest_seqno = 0U;
-  bool is_first = true;
-
-  size_t level_index = 0U;
-  if (c->start_level() == 0) {
-    for (auto f : *c->inputs(0)) {
-      assert(f->smallest_seqno <= f->largest_seqno);
-      if (is_first) {
-        is_first = false;
-      } else {
-        assert(prev_smallest_seqno > f->largest_seqno);
-      }
-      prev_smallest_seqno = f->smallest_seqno;
-    }
-    level_index = 1U;
-  }
-  for (; level_index < c->num_input_levels(); level_index++) {
-    if (c->num_input_files(level_index) != 0) {
-      SequenceNumber smallest_seqno = 0U;
-      SequenceNumber largest_seqno = 0U;
-      GetSmallestLargestSeqno(*(c->inputs(level_index)), &smallest_seqno,
-                              &largest_seqno);
-      if (is_first) {
-        is_first = false;
-      } else if (prev_smallest_seqno > 0) {
-        // A level is considered as the bottommost level if there are
-        // no files in higher levels or if files in higher levels do
-        // not overlap with the files being compacted. Sequence numbers
-        // of files in bottommost level can be set to 0 to help
-        // compression. As a result, the following assert may not hold
-        // if the prev_smallest_seqno is 0.
-        assert(prev_smallest_seqno > largest_seqno);
-      }
-      prev_smallest_seqno = smallest_seqno;
-    }
-  }
-#endif
-  // update statistics
-  MeasureTime(ioptions_.statistics, NUM_FILES_IN_SINGLE_COMPACTION,
-              c->inputs(0)->size());
-
-  RegisterCompaction(c);
-
-  TEST_SYNC_POINT_CALLBACK("UniversalCompactionPicker::PickCompaction:Return",
-                           c);
-  return c;
-}
-
-uint32_t UniversalCompactionPicker::GetPathId(
-    const ImmutableCFOptions& ioptions, uint64_t file_size) {
-  // Two conditions need to be satisfied:
-  // (1) the target path needs to be able to hold the file's size
-  // (2) Total size left in this and previous paths need to be not
-  //     smaller than expected future file size before this new file is
-  //     compacted, which is estimated based on size_ratio.
-  // For example, if now we are compacting files of size (1, 1, 2, 4, 8),
-  // we will make sure the target file, probably with size of 16, will be
-  // placed in a path so that eventually when new files are generated and
-  // compacted to (1, 1, 2, 4, 8, 16), all those files can be stored in or
-  // before the path we chose.
-  //
-  // TODO(sdong): now the case of multiple column families is not
-  // considered in this algorithm. So the target size can be violated in
-  // that case. We need to improve it.
-  uint64_t accumulated_size = 0;
-  uint64_t future_size =
-      file_size * (100 - ioptions.compaction_options_universal.size_ratio) /
-      100;
-  uint32_t p = 0;
-  assert(!ioptions.db_paths.empty());
-  for (; p < ioptions.db_paths.size() - 1; p++) {
-    uint64_t target_size = ioptions.db_paths[p].target_size;
-    if (target_size > file_size &&
-        accumulated_size + (target_size - file_size) > future_size) {
-      return p;
-    }
-    accumulated_size += target_size;
-  }
-  return p;
-}
-
-//
-// Consider compaction files based on their size differences with
-// the next file in time order.
-//
-Compaction* UniversalCompactionPicker::PickCompactionToReduceSortedRuns(
-    const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-    VersionStorageInfo* vstorage, double score, unsigned int ratio,
-    unsigned int max_number_of_files_to_compact,
-    const std::vector<SortedRun>& sorted_runs, LogBuffer* log_buffer) {
-  unsigned int min_merge_width =
-      ioptions_.compaction_options_universal.min_merge_width;
-  unsigned int max_merge_width =
-      ioptions_.compaction_options_universal.max_merge_width;
-
-  const SortedRun* sr = nullptr;
-  bool done = false;
-  size_t start_index = 0;
-  unsigned int candidate_count = 0;
-
-  unsigned int max_files_to_compact =
-      std::min(max_merge_width, max_number_of_files_to_compact);
-  min_merge_width = std::max(min_merge_width, 2U);
-
-  // Caller checks the size before executing this function. This invariant is
-  // important because otherwise we may have a possible integer underflow when
-  // dealing with unsigned types.
-  assert(sorted_runs.size() > 0);
-
-  // Considers a candidate file only if it is smaller than the
-  // total size accumulated so far.
-  for (size_t loop = 0; loop < sorted_runs.size(); loop++) {
-    candidate_count = 0;
-
-    // Skip files that are already being compacted
-    for (sr = nullptr; loop < sorted_runs.size(); loop++) {
-      sr = &sorted_runs[loop];
-
-      if (!sr->being_compacted) {
-        candidate_count = 1;
-        break;
-      }
-      char file_num_buf[kFormatFileNumberBufSize];
-      sr->Dump(file_num_buf, sizeof(file_num_buf));
-      ROCKS_LOG_BUFFER(log_buffer,
-                       "[%s] Universal: %s"
-                       "[%d] being compacted, skipping",
-                       cf_name.c_str(), file_num_buf, loop);
-
-      sr = nullptr;
-    }
-
-    // This file is not being compacted. Consider it as the
-    // first candidate to be compacted.
-    uint64_t candidate_size = sr != nullptr ? sr->compensated_file_size : 0;
-    if (sr != nullptr) {
-      char file_num_buf[kFormatFileNumberBufSize];
-      sr->Dump(file_num_buf, sizeof(file_num_buf), true);
-      ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: Possible candidate %s[%d].",
-                       cf_name.c_str(), file_num_buf, loop);
-    }
-
-    // Check if the succeeding files need compaction.
-    for (size_t i = loop + 1;
-         candidate_count < max_files_to_compact && i < sorted_runs.size();
-         i++) {
-      const SortedRun* succeeding_sr = &sorted_runs[i];
-      if (succeeding_sr->being_compacted) {
-        break;
-      }
-      // Pick files if the total/last candidate file size (increased by the
-      // specified ratio) is still larger than the next candidate file.
-      // candidate_size is the total size of files picked so far with the
-      // default kCompactionStopStyleTotalSize; with
-      // kCompactionStopStyleSimilarSize, it's simply the size of the last
-      // picked file.
-      double sz = candidate_size * (100.0 + ratio) / 100.0;
-      if (sz < static_cast<double>(succeeding_sr->size)) {
-        break;
-      }
-      if (ioptions_.compaction_options_universal.stop_style ==
-          kCompactionStopStyleSimilarSize) {
-        // Similar-size stopping rule: also check the last picked file isn't
-        // far larger than the next candidate file.
-        sz = (succeeding_sr->size * (100.0 + ratio)) / 100.0;
-        if (sz < static_cast<double>(candidate_size)) {
-          // If the small file we've encountered begins a run of similar-size
-          // files, we'll pick them up on a future iteration of the outer
-          // loop. If it's some lonely straggler, it'll eventually get picked
-          // by the last-resort read amp strategy which disregards size ratios.
-          break;
-        }
-        candidate_size = succeeding_sr->compensated_file_size;
-      } else {  // default kCompactionStopStyleTotalSize
-        candidate_size += succeeding_sr->compensated_file_size;
-      }
-      candidate_count++;
-    }
-
-    // Found a series of consecutive files that need compaction.
-    if (candidate_count >= (unsigned int)min_merge_width) {
-      start_index = loop;
-      done = true;
-      break;
-    } else {
-      for (size_t i = loop;
-           i < loop + candidate_count && i < sorted_runs.size(); i++) {
-        const SortedRun* skipping_sr = &sorted_runs[i];
-        char file_num_buf[256];
-        skipping_sr->DumpSizeInfo(file_num_buf, sizeof(file_num_buf), loop);
-        ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: Skipping %s",
-                         cf_name.c_str(), file_num_buf);
-      }
-    }
-  }
-  if (!done || candidate_count <= 1) {
-    return nullptr;
-  }
-  size_t first_index_after = start_index + candidate_count;
-  // Compression is enabled if files compacted earlier already reached
-  // size ratio of compression.
-  bool enable_compression = true;
-  int ratio_to_compress =
-      ioptions_.compaction_options_universal.compression_size_percent;
-  if (ratio_to_compress >= 0) {
-    uint64_t total_size = 0;
-    for (auto& sorted_run : sorted_runs) {
-      total_size += sorted_run.compensated_file_size;
-    }
-
-    uint64_t older_file_size = 0;
-    for (size_t i = sorted_runs.size() - 1; i >= first_index_after; i--) {
-      older_file_size += sorted_runs[i].size;
-      if (older_file_size * 100L >= total_size * (long)ratio_to_compress) {
-        enable_compression = false;
-        break;
-      }
-    }
-  }
-
-  uint64_t estimated_total_size = 0;
-  for (unsigned int i = 0; i < first_index_after; i++) {
-    estimated_total_size += sorted_runs[i].size;
-  }
-  uint32_t path_id = GetPathId(ioptions_, estimated_total_size);
-  int start_level = sorted_runs[start_index].level;
-  int output_level;
-  if (first_index_after == sorted_runs.size()) {
-    output_level = vstorage->num_levels() - 1;
-  } else if (sorted_runs[first_index_after].level == 0) {
-    output_level = 0;
-  } else {
-    output_level = sorted_runs[first_index_after].level - 1;
-  }
-
-  std::vector<CompactionInputFiles> inputs(vstorage->num_levels());
-  for (size_t i = 0; i < inputs.size(); ++i) {
-    inputs[i].level = start_level + static_cast<int>(i);
-  }
-  for (size_t i = start_index; i < first_index_after; i++) {
-    auto& picking_sr = sorted_runs[i];
-    if (picking_sr.level == 0) {
-      FileMetaData* picking_file = picking_sr.file;
-      inputs[0].files.push_back(picking_file);
-    } else {
-      auto& files = inputs[picking_sr.level - start_level].files;
-      for (auto* f : vstorage->LevelFiles(picking_sr.level)) {
-        files.push_back(f);
-      }
-    }
-    char file_num_buf[256];
-    picking_sr.DumpSizeInfo(file_num_buf, sizeof(file_num_buf), i);
-    ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: Picking %s", cf_name.c_str(),
-                     file_num_buf);
-  }
-
-  CompactionReason compaction_reason;
-  if (max_number_of_files_to_compact == UINT_MAX) {
-    compaction_reason = CompactionReason::kUniversalSortedRunNum;
-  } else {
-    compaction_reason = CompactionReason::kUniversalSizeRatio;
-  }
-  return new Compaction(
-      vstorage, ioptions_, mutable_cf_options, std::move(inputs), output_level,
-      mutable_cf_options.MaxFileSizeForLevel(output_level), LLONG_MAX, path_id,
-      GetCompressionType(ioptions_, vstorage, mutable_cf_options, start_level,
-                         1, enable_compression),
-      /* grandparents */ {}, /* is manual */ false, score,
-      false /* deletion_compaction */, compaction_reason);
-}
-
-// Look at overall size amplification. If size amplification
-// exceeeds the configured value, then do a compaction
-// of the candidate files all the way upto the earliest
-// base file (overrides configured values of file-size ratios,
-// min_merge_width and max_merge_width).
-//
-Compaction* UniversalCompactionPicker::PickCompactionToReduceSizeAmp(
-    const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-    VersionStorageInfo* vstorage, double score,
-    const std::vector<SortedRun>& sorted_runs, LogBuffer* log_buffer) {
-  // percentage flexibilty while reducing size amplification
-  uint64_t ratio =
-      ioptions_.compaction_options_universal.max_size_amplification_percent;
-
-  unsigned int candidate_count = 0;
-  uint64_t candidate_size = 0;
-  size_t start_index = 0;
-  const SortedRun* sr = nullptr;
-
-  // Skip files that are already being compacted
-  for (size_t loop = 0; loop < sorted_runs.size() - 1; loop++) {
-    sr = &sorted_runs[loop];
-    if (!sr->being_compacted) {
-      start_index = loop;  // Consider this as the first candidate.
-      break;
-    }
-    char file_num_buf[kFormatFileNumberBufSize];
-    sr->Dump(file_num_buf, sizeof(file_num_buf), true);
-    ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: skipping %s[%d] compacted %s",
-                     cf_name.c_str(), file_num_buf, loop,
-                     " cannot be a candidate to reduce size amp.\n");
-    sr = nullptr;
-  }
-
-  if (sr == nullptr) {
-    return nullptr;  // no candidate files
-  }
-  {
-    char file_num_buf[kFormatFileNumberBufSize];
-    sr->Dump(file_num_buf, sizeof(file_num_buf), true);
-    ROCKS_LOG_BUFFER(
-        log_buffer,
-        "[%s] Universal: First candidate %s[%" ROCKSDB_PRIszt "] %s",
-        cf_name.c_str(), file_num_buf, start_index, " to reduce size amp.\n");
-  }
-
-  // keep adding up all the remaining files
-  for (size_t loop = start_index; loop < sorted_runs.size() - 1; loop++) {
-    sr = &sorted_runs[loop];
-    if (sr->being_compacted) {
-      char file_num_buf[kFormatFileNumberBufSize];
-      sr->Dump(file_num_buf, sizeof(file_num_buf), true);
-      ROCKS_LOG_BUFFER(
-          log_buffer, "[%s] Universal: Possible candidate %s[%d] %s",
-          cf_name.c_str(), file_num_buf, start_index,
-          " is already being compacted. No size amp reduction possible.\n");
-      return nullptr;
-    }
-    candidate_size += sr->compensated_file_size;
-    candidate_count++;
-  }
-  if (candidate_count == 0) {
-    return nullptr;
-  }
-
-  // size of earliest file
-  uint64_t earliest_file_size = sorted_runs.back().size;
-
-  // size amplification = percentage of additional size
-  if (candidate_size * 100 < ratio * earliest_file_size) {
-    ROCKS_LOG_BUFFER(
-        log_buffer,
-        "[%s] Universal: size amp not needed. newer-files-total-size %" PRIu64
-        " earliest-file-size %" PRIu64,
-        cf_name.c_str(), candidate_size, earliest_file_size);
-    return nullptr;
-  } else {
-    ROCKS_LOG_BUFFER(
-        log_buffer,
-        "[%s] Universal: size amp needed. newer-files-total-size %" PRIu64
-        " earliest-file-size %" PRIu64,
-        cf_name.c_str(), candidate_size, earliest_file_size);
-  }
-  assert(start_index < sorted_runs.size() - 1);
-
-  // Estimate total file size
-  uint64_t estimated_total_size = 0;
-  for (size_t loop = start_index; loop < sorted_runs.size(); loop++) {
-    estimated_total_size += sorted_runs[loop].size;
-  }
-  uint32_t path_id = GetPathId(ioptions_, estimated_total_size);
-  int start_level = sorted_runs[start_index].level;
-
-  std::vector<CompactionInputFiles> inputs(vstorage->num_levels());
-  for (size_t i = 0; i < inputs.size(); ++i) {
-    inputs[i].level = start_level + static_cast<int>(i);
-  }
-  // We always compact all the files, so always compress.
-  for (size_t loop = start_index; loop < sorted_runs.size(); loop++) {
-    auto& picking_sr = sorted_runs[loop];
-    if (picking_sr.level == 0) {
-      FileMetaData* f = picking_sr.file;
-      inputs[0].files.push_back(f);
-    } else {
-      auto& files = inputs[picking_sr.level - start_level].files;
-      for (auto* f : vstorage->LevelFiles(picking_sr.level)) {
-        files.push_back(f);
-      }
-    }
-    char file_num_buf[256];
-    picking_sr.DumpSizeInfo(file_num_buf, sizeof(file_num_buf), loop);
-    ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: size amp picking %s",
-                     cf_name.c_str(), file_num_buf);
-  }
-
-  return new Compaction(
-      vstorage, ioptions_, mutable_cf_options, std::move(inputs),
-      vstorage->num_levels() - 1,
-      mutable_cf_options.MaxFileSizeForLevel(vstorage->num_levels() - 1),
-      /* max_grandparent_overlap_bytes */ LLONG_MAX, path_id,
-      GetCompressionType(ioptions_, vstorage, mutable_cf_options,
-                         vstorage->num_levels() - 1, 1),
-      /* grandparents */ {}, /* is manual */ false, score,
-      false /* deletion_compaction */,
-      CompactionReason::kUniversalSizeAmplification);
-}
-}  // namespace rocksdb
-
-#endif  // !ROCKSDB_LITE
@@ -1,91 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#pragma once
-#ifndef ROCKSDB_LITE
-
-#include "db/compaction_picker.h"
-
-namespace rocksdb {
-class UniversalCompactionPicker : public CompactionPicker {
- public:
-  UniversalCompactionPicker(const ImmutableCFOptions& ioptions,
-                            const InternalKeyComparator* icmp)
-      : CompactionPicker(ioptions, icmp) {}
-  virtual Compaction* PickCompaction(const std::string& cf_name,
-                                     const MutableCFOptions& mutable_cf_options,
-                                     VersionStorageInfo* vstorage,
-                                     LogBuffer* log_buffer) override;
-
-  virtual int MaxOutputLevel() const override { return NumberLevels() - 1; }
-
-  virtual bool NeedsCompaction(
-      const VersionStorageInfo* vstorage) const override;
-
- private:
-  struct SortedRun {
-    SortedRun(int _level, FileMetaData* _file, uint64_t _size,
-              uint64_t _compensated_file_size, bool _being_compacted)
-        : level(_level),
-          file(_file),
-          size(_size),
-          compensated_file_size(_compensated_file_size),
-          being_compacted(_being_compacted) {
-      assert(compensated_file_size > 0);
-      assert(level != 0 || file != nullptr);
-    }
-
-    void Dump(char* out_buf, size_t out_buf_size,
-              bool print_path = false) const;
-
-    // sorted_run_count is added into the string to print
-    void DumpSizeInfo(char* out_buf, size_t out_buf_size,
-                      size_t sorted_run_count) const;
-
-    int level;
-    // `file` Will be null for level > 0. For level = 0, the sorted run is
-    // for this file.
-    FileMetaData* file;
-    // For level > 0, `size` and `compensated_file_size` are sum of sizes all
-    // files in the level. `being_compacted` should be the same for all files
-    // in a non-zero level. Use the value here.
-    uint64_t size;
-    uint64_t compensated_file_size;
-    bool being_compacted;
-  };
-
-  // Pick Universal compaction to limit read amplification
-  Compaction* PickCompactionToReduceSortedRuns(
-      const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-      VersionStorageInfo* vstorage, double score, unsigned int ratio,
-      unsigned int num_files, const std::vector<SortedRun>& sorted_runs,
-      LogBuffer* log_buffer);
-
-  // Pick Universal compaction to limit space amplification.
-  Compaction* PickCompactionToReduceSizeAmp(
-      const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-      VersionStorageInfo* vstorage, double score,
-      const std::vector<SortedRun>& sorted_runs, LogBuffer* log_buffer);
-
-  // Used in universal compaction when the enabled_trivial_move
-  // option is set. Checks whether there are any overlapping files
-  // in the input. Returns true if the input files are non
-  // overlapping.
-  bool IsInputFilesNonOverlapping(Compaction* c);
-
-  static std::vector<SortedRun> CalculateSortedRuns(
-      const VersionStorageInfo& vstorage, const ImmutableCFOptions& ioptions);
-
-  // Pick a path ID to place a newly generated file, with its estimated file
-  // size.
-  static uint32_t GetPathId(const ImmutableCFOptions& ioptions,
-                            uint64_t file_size);
-};
-}  // namespace rocksdb
-#endif  // !ROCKSDB_LITE
@@ -40,10 +40,6 @@ class KVIter : public Iterator {
  virtual void Seek(const Slice& k) override {
    iter_ = map_->lower_bound(k.ToString());
  }
-  virtual void SeekForPrev(const Slice& k) override {
-    iter_ = map_->upper_bound(k.ToString());
-    Prev();
-  }
  virtual void Next() override { ++iter_; }
  virtual void Prev() override {
    if (iter_ == map_->begin()) {
@@ -11,20 +11,20 @@

 #include "rocksdb/db.h"

+#include <inttypes.h>
 #include <errno.h>
 #include <fcntl.h>
-#include <inttypes.h>
 #include <sys/stat.h>
 #include <sys/types.h>
-#include "db/db_impl.h"
-#include "db/log_format.h"
-#include "db/version_set.h"
 #include "rocksdb/cache.h"
 #include "rocksdb/env.h"
 #include "rocksdb/table.h"
 #include "rocksdb/write_batch.h"
-#include "util/filename.h"
-#include "util/string_util.h"
+#include "db/db_impl.h"
+#include "db/filename.h"
+#include "db/log_format.h"
+#include "db/version_set.h"
+#include "util/logging.h"
 #include "util/testharness.h"
 #include "util/testutil.h"

@@ -41,10 +41,7 @@ class CorruptionTest : public testing::Test {
  DB* db_;

  CorruptionTest() {
-    // If LRU cache shard bit is smaller than 2 (or -1 which will automatically
-    // set it to 0), test SequenceNumberRecovery will fail, likely because of a
-    // bug in recovery code. Keep it 4 for now to make the test passes.
-    tiny_cache_ = NewLRUCache(100, 4);
+    tiny_cache_ = NewLRUCache(100);
    options_.wal_recovery_mode = WALRecoveryMode::kTolerateCorruptedTailRecords;
    options_.env = &env_;
    dbname_ = test::TmpDir() + "/corruption_test";
@@ -8,10 +8,9 @@
 #include "db/db_impl.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
+#include "table/meta_blocks.h"
 #include "table/cuckoo_table_factory.h"
 #include "table/cuckoo_table_reader.h"
-#include "table/meta_blocks.h"
-#include "util/string_util.h"
 #include "util/testharness.h"
 #include "util/testutil.h"

@@ -42,7 +41,6 @@ class CuckooTableDBTest : public testing::Test {
    options.memtable_factory.reset(NewHashLinkListRepFactory(4, 0, 3, true));
    options.allow_mmap_reads = true;
    options.create_if_missing = true;
-    options.allow_concurrent_memtable_write = false;
    return options;
  }

@@ -1,852 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-#include "db/db_test_util.h"
-#include "port/stack_trace.h"
-#include "rocksdb/perf_context.h"
-#if !defined(ROCKSDB_LITE)
-#include "util/sync_point.h"
-#endif
-
-namespace rocksdb {
-
-class DBBasicTest : public DBTestBase {
- public:
-  DBBasicTest() : DBTestBase("/db_basic_test") {}
-};
-
-TEST_F(DBBasicTest, OpenWhenOpen) {
-  Options options = CurrentOptions();
-  options.env = env_;
-  rocksdb::DB* db2 = nullptr;
-  rocksdb::Status s = DB::Open(options, dbname_, &db2);
-
-  ASSERT_EQ(Status::Code::kIOError, s.code());
-  ASSERT_EQ(Status::SubCode::kNone, s.subcode());
-  ASSERT_TRUE(strstr(s.getState(), "lock ") != nullptr);
-
-  delete db2;
-}
-
-#ifndef ROCKSDB_LITE
-TEST_F(DBBasicTest, ReadOnlyDB) {
-  ASSERT_OK(Put("foo", "v1"));
-  ASSERT_OK(Put("bar", "v2"));
-  ASSERT_OK(Put("foo", "v3"));
-  Close();
-
-  auto options = CurrentOptions();
-  assert(options.env = env_);
-  ASSERT_OK(ReadOnlyReopen(options));
-  ASSERT_EQ("v3", Get("foo"));
-  ASSERT_EQ("v2", Get("bar"));
-  Iterator* iter = db_->NewIterator(ReadOptions());
-  int count = 0;
-  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
-    ASSERT_OK(iter->status());
-    ++count;
-  }
-  ASSERT_EQ(count, 2);
-  delete iter;
-  Close();
-
-  // Reopen and flush memtable.
-  Reopen(options);
-  Flush();
-  Close();
-  // Now check keys in read only mode.
-  ASSERT_OK(ReadOnlyReopen(options));
-  ASSERT_EQ("v3", Get("foo"));
-  ASSERT_EQ("v2", Get("bar"));
-  ASSERT_TRUE(db_->SyncWAL().IsNotSupported());
-}
-
-TEST_F(DBBasicTest, CompactedDB) {
-  const uint64_t kFileSize = 1 << 20;
-  Options options = CurrentOptions();
-  options.disable_auto_compactions = true;
-  options.write_buffer_size = kFileSize;
-  options.target_file_size_base = kFileSize;
-  options.max_bytes_for_level_base = 1 << 30;
-  options.compression = kNoCompression;
-  Reopen(options);
-  // 1 L0 file, use CompactedDB if max_open_files = -1
-  ASSERT_OK(Put("aaa", DummyString(kFileSize / 2, '1')));
-  Flush();
-  Close();
-  ASSERT_OK(ReadOnlyReopen(options));
-  Status s = Put("new", "value");
-  ASSERT_EQ(s.ToString(),
-            "Not implemented: Not supported operation in read only mode.");
-  ASSERT_EQ(DummyString(kFileSize / 2, '1'), Get("aaa"));
-  Close();
-  options.max_open_files = -1;
-  ASSERT_OK(ReadOnlyReopen(options));
-  s = Put("new", "value");
-  ASSERT_EQ(s.ToString(),
-            "Not implemented: Not supported in compacted db mode.");
-  ASSERT_EQ(DummyString(kFileSize / 2, '1'), Get("aaa"));
-  Close();
-  Reopen(options);
-  // Add more L0 files
-  ASSERT_OK(Put("bbb", DummyString(kFileSize / 2, '2')));
-  Flush();
-  ASSERT_OK(Put("aaa", DummyString(kFileSize / 2, 'a')));
-  Flush();
-  ASSERT_OK(Put("bbb", DummyString(kFileSize / 2, 'b')));
-  ASSERT_OK(Put("eee", DummyString(kFileSize / 2, 'e')));
-  Flush();
-  Close();
-
-  ASSERT_OK(ReadOnlyReopen(options));
-  // Fallback to read-only DB
-  s = Put("new", "value");
-  ASSERT_EQ(s.ToString(),
-            "Not implemented: Not supported operation in read only mode.");
-  Close();
-
-  // Full compaction
-  Reopen(options);
-  // Add more keys
-  ASSERT_OK(Put("fff", DummyString(kFileSize / 2, 'f')));
-  ASSERT_OK(Put("hhh", DummyString(kFileSize / 2, 'h')));
-  ASSERT_OK(Put("iii", DummyString(kFileSize / 2, 'i')));
-  ASSERT_OK(Put("jjj", DummyString(kFileSize / 2, 'j')));
-  db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
-  ASSERT_EQ(3, NumTableFilesAtLevel(1));
-  Close();
-
-  // CompactedDB
-  ASSERT_OK(ReadOnlyReopen(options));
-  s = Put("new", "value");
-  ASSERT_EQ(s.ToString(),
-            "Not implemented: Not supported in compacted db mode.");
-  ASSERT_EQ("NOT_FOUND", Get("abc"));
-  ASSERT_EQ(DummyString(kFileSize / 2, 'a'), Get("aaa"));
-  ASSERT_EQ(DummyString(kFileSize / 2, 'b'), Get("bbb"));
-  ASSERT_EQ("NOT_FOUND", Get("ccc"));
-  ASSERT_EQ(DummyString(kFileSize / 2, 'e'), Get("eee"));
-  ASSERT_EQ(DummyString(kFileSize / 2, 'f'), Get("fff"));
-  ASSERT_EQ("NOT_FOUND", Get("ggg"));
-  ASSERT_EQ(DummyString(kFileSize / 2, 'h'), Get("hhh"));
-  ASSERT_EQ(DummyString(kFileSize / 2, 'i'), Get("iii"));
-  ASSERT_EQ(DummyString(kFileSize / 2, 'j'), Get("jjj"));
-  ASSERT_EQ("NOT_FOUND", Get("kkk"));
-
-  // MultiGet
-  std::vector<std::string> values;
-  std::vector<Status> status_list = dbfull()->MultiGet(
-      ReadOptions(),
-      std::vector<Slice>({Slice("aaa"), Slice("ccc"), Slice("eee"),
-                          Slice("ggg"), Slice("iii"), Slice("kkk")}),
-      &values);
-  ASSERT_EQ(status_list.size(), static_cast<uint64_t>(6));
-  ASSERT_EQ(values.size(), static_cast<uint64_t>(6));
-  ASSERT_OK(status_list[0]);
-  ASSERT_EQ(DummyString(kFileSize / 2, 'a'), values[0]);
-  ASSERT_TRUE(status_list[1].IsNotFound());
-  ASSERT_OK(status_list[2]);
-  ASSERT_EQ(DummyString(kFileSize / 2, 'e'), values[2]);
-  ASSERT_TRUE(status_list[3].IsNotFound());
-  ASSERT_OK(status_list[4]);
-  ASSERT_EQ(DummyString(kFileSize / 2, 'i'), values[4]);
-  ASSERT_TRUE(status_list[5].IsNotFound());
-
-  Reopen(options);
-  // Add a key
-  ASSERT_OK(Put("fff", DummyString(kFileSize / 2, 'f')));
-  Close();
-  ASSERT_OK(ReadOnlyReopen(options));
-  s = Put("new", "value");
-  ASSERT_EQ(s.ToString(),
-            "Not implemented: Not supported operation in read only mode.");
-}
-
-TEST_F(DBBasicTest, LevelLimitReopen) {
-  Options options = CurrentOptions();
-  CreateAndReopenWithCF({"pikachu"}, options);
-
-  const std::string value(1024 * 1024, ' ');
-  int i = 0;
-  while (NumTableFilesAtLevel(2, 1) == 0) {
-    ASSERT_OK(Put(1, Key(i++), value));
-  }
-
-  options.num_levels = 1;
-  options.max_bytes_for_level_multiplier_additional.resize(1, 1);
-  Status s = TryReopenWithColumnFamilies({"default", "pikachu"}, options);
-  ASSERT_EQ(s.IsInvalidArgument(), true);
-  ASSERT_EQ(s.ToString(),
-            "Invalid argument: db has more levels than options.num_levels");
-
-  options.num_levels = 10;
-  options.max_bytes_for_level_multiplier_additional.resize(10, 1);
-  ASSERT_OK(TryReopenWithColumnFamilies({"default", "pikachu"}, options));
-}
-#endif  // ROCKSDB_LITE
-
-TEST_F(DBBasicTest, PutDeleteGet) {
-  do {
-    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
-    ASSERT_OK(Put(1, "foo", "v1"));
-    ASSERT_EQ("v1", Get(1, "foo"));
-    ASSERT_OK(Put(1, "foo", "v2"));
-    ASSERT_EQ("v2", Get(1, "foo"));
-    ASSERT_OK(Delete(1, "foo"));
-    ASSERT_EQ("NOT_FOUND", Get(1, "foo"));
-  } while (ChangeOptions());
-}
-
-TEST_F(DBBasicTest, PutSingleDeleteGet) {
-  do {
-    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
-    ASSERT_OK(Put(1, "foo", "v1"));
-    ASSERT_EQ("v1", Get(1, "foo"));
-    ASSERT_OK(Put(1, "foo2", "v2"));
-    ASSERT_EQ("v2", Get(1, "foo2"));
-    ASSERT_OK(SingleDelete(1, "foo"));
-    ASSERT_EQ("NOT_FOUND", Get(1, "foo"));
-    // Skip HashCuckooRep as it does not support single delete. FIFO and
-    // universal compaction do not apply to the test case. Skip MergePut
-    // because single delete does not get removed when it encounters a merge.
-  } while (ChangeOptions(kSkipHashCuckoo | kSkipFIFOCompaction |
-                         kSkipUniversalCompaction | kSkipMergePut));
-}
-
-TEST_F(DBBasicTest, EmptyFlush) {
-  // It is possible to produce empty flushes when using single deletes. Tests
-  // whether empty flushes cause issues.
-  do {
-    Random rnd(301);
-
-    Options options = CurrentOptions();
-    options.disable_auto_compactions = true;
-    CreateAndReopenWithCF({"pikachu"}, options);
-
-    Put(1, "a", Slice());
-    SingleDelete(1, "a");
-    ASSERT_OK(Flush(1));
-
-    ASSERT_EQ("[ ]", AllEntriesFor("a", 1));
-    // Skip HashCuckooRep as it does not support single delete. FIFO and
-    // universal compaction do not apply to the test case. Skip MergePut
-    // because merges cannot be combined with single deletions.
-  } while (ChangeOptions(kSkipHashCuckoo | kSkipFIFOCompaction |
-                         kSkipUniversalCompaction | kSkipMergePut));
-}
-
-TEST_F(DBBasicTest, GetFromVersions) {
-  do {
-    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
-    ASSERT_OK(Put(1, "foo", "v1"));
-    ASSERT_OK(Flush(1));
-    ASSERT_EQ("v1", Get(1, "foo"));
-    ASSERT_EQ("NOT_FOUND", Get(0, "foo"));
-  } while (ChangeOptions());
-}
-
-#ifndef ROCKSDB_LITE
-TEST_F(DBBasicTest, GetSnapshot) {
-  anon::OptionsOverride options_override;
-  options_override.skip_policy = kSkipNoSnapshot;
-  do {
-    CreateAndReopenWithCF({"pikachu"}, CurrentOptions(options_override));
-    // Try with both a short key and a long key
-    for (int i = 0; i < 2; i++) {
-      std::string key = (i == 0) ? std::string("foo") : std::string(200, 'x');
-      ASSERT_OK(Put(1, key, "v1"));
-      const Snapshot* s1 = db_->GetSnapshot();
-      if (option_config_ == kHashCuckoo) {
-        // Unsupported case.
-        ASSERT_TRUE(s1 == nullptr);
-        break;
-      }
-      ASSERT_OK(Put(1, key, "v2"));
-      ASSERT_EQ("v2", Get(1, key));
-      ASSERT_EQ("v1", Get(1, key, s1));
-      ASSERT_OK(Flush(1));
-      ASSERT_EQ("v2", Get(1, key));
-      ASSERT_EQ("v1", Get(1, key, s1));
-      db_->ReleaseSnapshot(s1);
-    }
-  } while (ChangeOptions());
-}
-#endif  // ROCKSDB_LITE
-
-TEST_F(DBBasicTest, CheckLock) {
-  do {
-    DB* localdb;
-    Options options = CurrentOptions();
-    ASSERT_OK(TryReopen(options));
-
-    // second open should fail
-    ASSERT_TRUE(!(DB::Open(options, dbname_, &localdb)).ok());
-  } while (ChangeCompactOptions());
-}
-
-TEST_F(DBBasicTest, FlushMultipleMemtable) {
-  do {
-    Options options = CurrentOptions();
-    WriteOptions writeOpt = WriteOptions();
-    writeOpt.disableWAL = true;
-    options.max_write_buffer_number = 4;
-    options.min_write_buffer_number_to_merge = 3;
-    options.max_write_buffer_number_to_maintain = -1;
-    CreateAndReopenWithCF({"pikachu"}, options);
-    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v1"));
-    ASSERT_OK(Flush(1));
-    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1"));
-
-    ASSERT_EQ("v1", Get(1, "foo"));
-    ASSERT_EQ("v1", Get(1, "bar"));
-    ASSERT_OK(Flush(1));
-  } while (ChangeCompactOptions());
-}
-
-TEST_F(DBBasicTest, FlushEmptyColumnFamily) {
-  // Block flush thread and disable compaction thread
-  env_->SetBackgroundThreads(1, Env::HIGH);
-  env_->SetBackgroundThreads(1, Env::LOW);
-  test::SleepingBackgroundTask sleeping_task_low;
-  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
-                 Env::Priority::LOW);
-  test::SleepingBackgroundTask sleeping_task_high;
-  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
-                 &sleeping_task_high, Env::Priority::HIGH);
-
-  Options options = CurrentOptions();
-  // disable compaction
-  options.disable_auto_compactions = true;
-  WriteOptions writeOpt = WriteOptions();
-  writeOpt.disableWAL = true;
-  options.max_write_buffer_number = 2;
-  options.min_write_buffer_number_to_merge = 1;
-  options.max_write_buffer_number_to_maintain = 1;
-  CreateAndReopenWithCF({"pikachu"}, options);
-
-  // Compaction can still go through even if no thread can flush the
-  // mem table.
-  ASSERT_OK(Flush(0));
-  ASSERT_OK(Flush(1));
-
-  // Insert can go through
-  ASSERT_OK(dbfull()->Put(writeOpt, handles_[0], "foo", "v1"));
-  ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1"));
-
-  ASSERT_EQ("v1", Get(0, "foo"));
-  ASSERT_EQ("v1", Get(1, "bar"));
-
-  sleeping_task_high.WakeUp();
-  sleeping_task_high.WaitUntilDone();
-
-  // Flush can still go through.
-  ASSERT_OK(Flush(0));
-  ASSERT_OK(Flush(1));
-
-  sleeping_task_low.WakeUp();
-  sleeping_task_low.WaitUntilDone();
-}
-
-TEST_F(DBBasicTest, FLUSH) {
-  do {
-    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
-    WriteOptions writeOpt = WriteOptions();
-    writeOpt.disableWAL = true;
-    SetPerfLevel(kEnableTime);
-    ;
-    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v1"));
-    // this will now also flush the last 2 writes
-    ASSERT_OK(Flush(1));
-    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1"));
-
-    perf_context.Reset();
-    Get(1, "foo");
-    ASSERT_TRUE((int)perf_context.get_from_output_files_time > 0);
-
-    ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
-    ASSERT_EQ("v1", Get(1, "foo"));
-    ASSERT_EQ("v1", Get(1, "bar"));
-
-    writeOpt.disableWAL = true;
-    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v2"));
-    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v2"));
-    ASSERT_OK(Flush(1));
-
-    ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
-    ASSERT_EQ("v2", Get(1, "bar"));
-    perf_context.Reset();
-    ASSERT_EQ("v2", Get(1, "foo"));
-    ASSERT_TRUE((int)perf_context.get_from_output_files_time > 0);
-
-    writeOpt.disableWAL = false;
-    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v3"));
-    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v3"));
-    ASSERT_OK(Flush(1));
-
-    ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
-    // 'foo' should be there because its put
-    // has WAL enabled.
-    ASSERT_EQ("v3", Get(1, "foo"));
-    ASSERT_EQ("v3", Get(1, "bar"));
-
-    SetPerfLevel(kDisable);
-  } while (ChangeCompactOptions());
-}
-
-TEST_F(DBBasicTest, ManifestRollOver) {
-  do {
-    Options options;
-    options.max_manifest_file_size = 10;  // 10 bytes
-    options = CurrentOptions(options);
-    CreateAndReopenWithCF({"pikachu"}, options);
-    {
-      ASSERT_OK(Put(1, "manifest_key1", std::string(1000, '1')));
-      ASSERT_OK(Put(1, "manifest_key2", std::string(1000, '2')));
-      ASSERT_OK(Put(1, "manifest_key3", std::string(1000, '3')));
-      uint64_t manifest_before_flush = dbfull()->TEST_Current_Manifest_FileNo();
-      ASSERT_OK(Flush(1));  // This should trigger LogAndApply.
-      uint64_t manifest_after_flush = dbfull()->TEST_Current_Manifest_FileNo();
-      ASSERT_GT(manifest_after_flush, manifest_before_flush);
-      ReopenWithColumnFamilies({"default", "pikachu"}, options);
-      ASSERT_GT(dbfull()->TEST_Current_Manifest_FileNo(), manifest_after_flush);
-      // check if a new manifest file got inserted or not.
-      ASSERT_EQ(std::string(1000, '1'), Get(1, "manifest_key1"));
-      ASSERT_EQ(std::string(1000, '2'), Get(1, "manifest_key2"));
-      ASSERT_EQ(std::string(1000, '3'), Get(1, "manifest_key3"));
-    }
-  } while (ChangeCompactOptions());
-}
-
-TEST_F(DBBasicTest, IdentityAcrossRestarts) {
-  do {
-    std::string id1;
-    ASSERT_OK(db_->GetDbIdentity(id1));
-
-    Options options = CurrentOptions();
-    Reopen(options);
-    std::string id2;
-    ASSERT_OK(db_->GetDbIdentity(id2));
-    // id1 should match id2 because identity was not regenerated
-    ASSERT_EQ(id1.compare(id2), 0);
-
-    std::string idfilename = IdentityFileName(dbname_);
-    ASSERT_OK(env_->DeleteFile(idfilename));
-    Reopen(options);
-    std::string id3;
-    ASSERT_OK(db_->GetDbIdentity(id3));
-    // id1 should NOT match id3 because identity was regenerated
-    ASSERT_NE(id1.compare(id3), 0);
-  } while (ChangeCompactOptions());
-}
-
-#ifndef ROCKSDB_LITE
-TEST_F(DBBasicTest, Snapshot) {
-  anon::OptionsOverride options_override;
-  options_override.skip_policy = kSkipNoSnapshot;
-  do {
-    CreateAndReopenWithCF({"pikachu"}, CurrentOptions(options_override));
-    Put(0, "foo", "0v1");
-    Put(1, "foo", "1v1");
-
-    const Snapshot* s1 = db_->GetSnapshot();
-    ASSERT_EQ(1U, GetNumSnapshots());
-    uint64_t time_snap1 = GetTimeOldestSnapshots();
-    ASSERT_GT(time_snap1, 0U);
-    Put(0, "foo", "0v2");
-    Put(1, "foo", "1v2");
-
-    env_->addon_time_.fetch_add(1);
-
-    const Snapshot* s2 = db_->GetSnapshot();
-    ASSERT_EQ(2U, GetNumSnapshots());
-    ASSERT_EQ(time_snap1, GetTimeOldestSnapshots());
-    Put(0, "foo", "0v3");
-    Put(1, "foo", "1v3");
-
-    {
-      ManagedSnapshot s3(db_);
-      ASSERT_EQ(3U, GetNumSnapshots());
-      ASSERT_EQ(time_snap1, GetTimeOldestSnapshots());
-
-      Put(0, "foo", "0v4");
-      Put(1, "foo", "1v4");
-      ASSERT_EQ("0v1", Get(0, "foo", s1));
-      ASSERT_EQ("1v1", Get(1, "foo", s1));
-      ASSERT_EQ("0v2", Get(0, "foo", s2));
-      ASSERT_EQ("1v2", Get(1, "foo", s2));
-      ASSERT_EQ("0v3", Get(0, "foo", s3.snapshot()));
-      ASSERT_EQ("1v3", Get(1, "foo", s3.snapshot()));
-      ASSERT_EQ("0v4", Get(0, "foo"));
-      ASSERT_EQ("1v4", Get(1, "foo"));
-    }
-
-    ASSERT_EQ(2U, GetNumSnapshots());
-    ASSERT_EQ(time_snap1, GetTimeOldestSnapshots());
-    ASSERT_EQ("0v1", Get(0, "foo", s1));
-    ASSERT_EQ("1v1", Get(1, "foo", s1));
-    ASSERT_EQ("0v2", Get(0, "foo", s2));
-    ASSERT_EQ("1v2", Get(1, "foo", s2));
-    ASSERT_EQ("0v4", Get(0, "foo"));
-    ASSERT_EQ("1v4", Get(1, "foo"));
-
-    db_->ReleaseSnapshot(s1);
-    ASSERT_EQ("0v2", Get(0, "foo", s2));
-    ASSERT_EQ("1v2", Get(1, "foo", s2));
-    ASSERT_EQ("0v4", Get(0, "foo"));
-    ASSERT_EQ("1v4", Get(1, "foo"));
-    ASSERT_EQ(1U, GetNumSnapshots());
-    ASSERT_LT(time_snap1, GetTimeOldestSnapshots());
-
-    db_->ReleaseSnapshot(s2);
-    ASSERT_EQ(0U, GetNumSnapshots());
-    ASSERT_EQ("0v4", Get(0, "foo"));
-    ASSERT_EQ("1v4", Get(1, "foo"));
-  } while (ChangeOptions(kSkipHashCuckoo));
-}
-
-#endif  // ROCKSDB_LITE
-
-TEST_F(DBBasicTest, CompactBetweenSnapshots) {
-  anon::OptionsOverride options_override;
-  options_override.skip_policy = kSkipNoSnapshot;
-  do {
-    Options options = CurrentOptions(options_override);
-    options.disable_auto_compactions = true;
-    CreateAndReopenWithCF({"pikachu"}, options);
-    Random rnd(301);
-    FillLevels("a", "z", 1);
-
-    Put(1, "foo", "first");
-    const Snapshot* snapshot1 = db_->GetSnapshot();
-    Put(1, "foo", "second");
-    Put(1, "foo", "third");
-    Put(1, "foo", "fourth");
-    const Snapshot* snapshot2 = db_->GetSnapshot();
-    Put(1, "foo", "fifth");
-    Put(1, "foo", "sixth");
-
-    // All entries (including duplicates) exist
-    // before any compaction or flush is triggered.
-    ASSERT_EQ(AllEntriesFor("foo", 1),
-              "[ sixth, fifth, fourth, third, second, first ]");
-    ASSERT_EQ("sixth", Get(1, "foo"));
-    ASSERT_EQ("fourth", Get(1, "foo", snapshot2));
-    ASSERT_EQ("first", Get(1, "foo", snapshot1));
-
-    // After a flush, "second", "third" and "fifth" should
-    // be removed
-    ASSERT_OK(Flush(1));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ sixth, fourth, first ]");
-
-    // after we release the snapshot1, only two values left
-    db_->ReleaseSnapshot(snapshot1);
-    FillLevels("a", "z", 1);
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-
-    // We have only one valid snapshot snapshot2. Since snapshot1 is
-    // not valid anymore, "first" should be removed by a compaction.
-    ASSERT_EQ("sixth", Get(1, "foo"));
-    ASSERT_EQ("fourth", Get(1, "foo", snapshot2));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ sixth, fourth ]");
-
-    // after we release the snapshot2, only one value should be left
-    db_->ReleaseSnapshot(snapshot2);
-    FillLevels("a", "z", 1);
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-    ASSERT_EQ("sixth", Get(1, "foo"));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ sixth ]");
-    // skip HashCuckooRep as it does not support snapshot
-  } while (ChangeOptions(kSkipHashCuckoo | kSkipFIFOCompaction));
-}
-
-TEST_F(DBBasicTest, DBOpen_Options) {
-  Options options = CurrentOptions();
-  std::string dbname = test::TmpDir(env_) + "/db_options_test";
-  ASSERT_OK(DestroyDB(dbname, options));
-
-  // Does not exist, and create_if_missing == false: error
-  DB* db = nullptr;
-  options.create_if_missing = false;
-  Status s = DB::Open(options, dbname, &db);
-  ASSERT_TRUE(strstr(s.ToString().c_str(), "does not exist") != nullptr);
-  ASSERT_TRUE(db == nullptr);
-
-  // Does not exist, and create_if_missing == true: OK
-  options.create_if_missing = true;
-  s = DB::Open(options, dbname, &db);
-  ASSERT_OK(s);
-  ASSERT_TRUE(db != nullptr);
-
-  delete db;
-  db = nullptr;
-
-  // Does exist, and error_if_exists == true: error
-  options.create_if_missing = false;
-  options.error_if_exists = true;
-  s = DB::Open(options, dbname, &db);
-  ASSERT_TRUE(strstr(s.ToString().c_str(), "exists") != nullptr);
-  ASSERT_TRUE(db == nullptr);
-
-  // Does exist, and error_if_exists == false: OK
-  options.create_if_missing = true;
-  options.error_if_exists = false;
-  s = DB::Open(options, dbname, &db);
-  ASSERT_OK(s);
-  ASSERT_TRUE(db != nullptr);
-
-  delete db;
-  db = nullptr;
-}
-
-TEST_F(DBBasicTest, CompactOnFlush) {
-  anon::OptionsOverride options_override;
-  options_override.skip_policy = kSkipNoSnapshot;
-  do {
-    Options options = CurrentOptions(options_override);
-    options.disable_auto_compactions = true;
-    CreateAndReopenWithCF({"pikachu"}, options);
-
-    Put(1, "foo", "v1");
-    ASSERT_OK(Flush(1));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v1 ]");
-
-    // Write two new keys
-    Put(1, "a", "begin");
-    Put(1, "z", "end");
-    Flush(1);
-
-    // Case1: Delete followed by a put
-    Delete(1, "foo");
-    Put(1, "foo", "v2");
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2, DEL, v1 ]");
-
-    // After the current memtable is flushed, the DEL should
-    // have been removed
-    ASSERT_OK(Flush(1));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2, v1 ]");
-
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2 ]");
-
-    // Case 2: Delete followed by another delete
-    Delete(1, "foo");
-    Delete(1, "foo");
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, DEL, v2 ]");
-    ASSERT_OK(Flush(1));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, v2 ]");
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]");
-
-    // Case 3: Put followed by a delete
-    Put(1, "foo", "v3");
-    Delete(1, "foo");
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, v3 ]");
-    ASSERT_OK(Flush(1));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL ]");
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]");
-
-    // Case 4: Put followed by another Put
-    Put(1, "foo", "v4");
-    Put(1, "foo", "v5");
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v5, v4 ]");
-    ASSERT_OK(Flush(1));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v5 ]");
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v5 ]");
-
-    // clear database
-    Delete(1, "foo");
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]");
-
-    // Case 5: Put followed by snapshot followed by another Put
-    // Both puts should remain.
-    Put(1, "foo", "v6");
-    const Snapshot* snapshot = db_->GetSnapshot();
-    Put(1, "foo", "v7");
-    ASSERT_OK(Flush(1));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v7, v6 ]");
-    db_->ReleaseSnapshot(snapshot);
-
-    // clear database
-    Delete(1, "foo");
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]");
-
-    // Case 5: snapshot followed by a put followed by another Put
-    // Only the last put should remain.
-    const Snapshot* snapshot1 = db_->GetSnapshot();
-    Put(1, "foo", "v8");
-    Put(1, "foo", "v9");
-    ASSERT_OK(Flush(1));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v9 ]");
-    db_->ReleaseSnapshot(snapshot1);
-  } while (ChangeCompactOptions());
-}
-
-TEST_F(DBBasicTest, FlushOneColumnFamily) {
-  Options options = CurrentOptions();
-  CreateAndReopenWithCF({"pikachu", "ilya", "muromec", "dobrynia", "nikitich",
-                         "alyosha", "popovich"},
-                        options);
-
-  ASSERT_OK(Put(0, "Default", "Default"));
-  ASSERT_OK(Put(1, "pikachu", "pikachu"));
-  ASSERT_OK(Put(2, "ilya", "ilya"));
-  ASSERT_OK(Put(3, "muromec", "muromec"));
-  ASSERT_OK(Put(4, "dobrynia", "dobrynia"));
-  ASSERT_OK(Put(5, "nikitich", "nikitich"));
-  ASSERT_OK(Put(6, "alyosha", "alyosha"));
-  ASSERT_OK(Put(7, "popovich", "popovich"));
-
-  for (int i = 0; i < 8; ++i) {
-    Flush(i);
-    auto tables = ListTableFiles(env_, dbname_);
-    ASSERT_EQ(tables.size(), i + 1U);
-  }
-}
-
-TEST_F(DBBasicTest, MultiGetSimple) {
-  do {
-    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
-    ASSERT_OK(Put(1, "k1", "v1"));
-    ASSERT_OK(Put(1, "k2", "v2"));
-    ASSERT_OK(Put(1, "k3", "v3"));
-    ASSERT_OK(Put(1, "k4", "v4"));
-    ASSERT_OK(Delete(1, "k4"));
-    ASSERT_OK(Put(1, "k5", "v5"));
-    ASSERT_OK(Delete(1, "no_key"));
-
-    std::vector<Slice> keys({"k1", "k2", "k3", "k4", "k5", "no_key"});
-
-    std::vector<std::string> values(20, "Temporary data to be overwritten");
-    std::vector<ColumnFamilyHandle*> cfs(keys.size(), handles_[1]);
-
-    std::vector<Status> s = db_->MultiGet(ReadOptions(), cfs, keys, &values);
-    ASSERT_EQ(values.size(), keys.size());
-    ASSERT_EQ(values[0], "v1");
-    ASSERT_EQ(values[1], "v2");
-    ASSERT_EQ(values[2], "v3");
-    ASSERT_EQ(values[4], "v5");
-
-    ASSERT_OK(s[0]);
-    ASSERT_OK(s[1]);
-    ASSERT_OK(s[2]);
-    ASSERT_TRUE(s[3].IsNotFound());
-    ASSERT_OK(s[4]);
-    ASSERT_TRUE(s[5].IsNotFound());
-  } while (ChangeCompactOptions());
-}
-
-TEST_F(DBBasicTest, MultiGetEmpty) {
-  do {
-    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
-    // Empty Key Set
-    std::vector<Slice> keys;
-    std::vector<std::string> values;
-    std::vector<ColumnFamilyHandle*> cfs;
-    std::vector<Status> s = db_->MultiGet(ReadOptions(), cfs, keys, &values);
-    ASSERT_EQ(s.size(), 0U);
-
-    // Empty Database, Empty Key Set
-    Options options = CurrentOptions();
-    options.create_if_missing = true;
-    DestroyAndReopen(options);
-    CreateAndReopenWithCF({"pikachu"}, options);
-    s = db_->MultiGet(ReadOptions(), cfs, keys, &values);
-    ASSERT_EQ(s.size(), 0U);
-
-    // Empty Database, Search for Keys
-    keys.resize(2);
-    keys[0] = "a";
-    keys[1] = "b";
-    cfs.push_back(handles_[0]);
-    cfs.push_back(handles_[1]);
-    s = db_->MultiGet(ReadOptions(), cfs, keys, &values);
-    ASSERT_EQ(static_cast<int>(s.size()), 2);
-    ASSERT_TRUE(s[0].IsNotFound() && s[1].IsNotFound());
-  } while (ChangeCompactOptions());
-}
-
-TEST_F(DBBasicTest, ChecksumTest) {
-  BlockBasedTableOptions table_options;
-  Options options = CurrentOptions();
-
-  table_options.checksum = kCRC32c;
-  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-  Reopen(options);
-  ASSERT_OK(Put("a", "b"));
-  ASSERT_OK(Put("c", "d"));
-  ASSERT_OK(Flush());  // table with crc checksum
-
-  table_options.checksum = kxxHash;
-  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-  Reopen(options);
-  ASSERT_OK(Put("e", "f"));
-  ASSERT_OK(Put("g", "h"));
-  ASSERT_OK(Flush());  // table with xxhash checksum
-
-  table_options.checksum = kCRC32c;
-  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-  Reopen(options);
-  ASSERT_EQ("b", Get("a"));
-  ASSERT_EQ("d", Get("c"));
-  ASSERT_EQ("f", Get("e"));
-  ASSERT_EQ("h", Get("g"));
-
-  table_options.checksum = kCRC32c;
-  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-  Reopen(options);
-  ASSERT_EQ("b", Get("a"));
-  ASSERT_EQ("d", Get("c"));
-  ASSERT_EQ("f", Get("e"));
-  ASSERT_EQ("h", Get("g"));
-}
-
-// On Windows you can have either memory mapped file or a file
-// with unbuffered access. So this asserts and does not make
-// sense to run
-#ifndef OS_WIN
-TEST_F(DBBasicTest, MmapAndBufferOptions) {
-  Options options = CurrentOptions();
-
-  options.use_direct_reads = true;
-  options.allow_mmap_reads = true;
-  ASSERT_NOK(TryReopen(options));
-
-  // All other combinations are acceptable
-  options.use_direct_reads = false;
-  ASSERT_OK(TryReopen(options));
-
-  if (IsDirectIOSupported()) {
-    options.use_direct_reads = true;
-    options.allow_mmap_reads = false;
-    ASSERT_OK(TryReopen(options));
-  }
-
-  options.use_direct_reads = false;
-  ASSERT_OK(TryReopen(options));
-}
-#endif
-
-}  // namespace rocksdb
-
-int main(int argc, char** argv) {
-  rocksdb::port::InstallStackTraceHandler();
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}
@@ -7,7 +7,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 #include <cstdlib>
-#include "cache/lru_cache.h"
 #include "db/db_test_util.h"
 #include "port/stack_trace.h"

@@ -40,7 +39,6 @@ class DBBlockCacheTest : public DBTestBase {
  Options GetOptions(const BlockBasedTableOptions& table_options) {
    Options options = CurrentOptions();
    options.create_if_missing = true;
-    options.avoid_flush_during_recovery = false;
    // options.compression = kNoCompression;
    options.statistics = rocksdb::CreateDBStatistics();
    options.table_factory.reset(new BlockBasedTableFactory(table_options));
@@ -172,7 +170,7 @@ TEST_F(DBBlockCacheTest, TestWithCompressedBlockCache) {
  InitTable(options);

  std::shared_ptr<Cache> cache = NewLRUCache(0, 0, false);
-  std::shared_ptr<Cache> compressed_cache = NewLRUCache(1 << 25, 0, false);
+  std::shared_ptr<Cache> compressed_cache = NewLRUCache(0, 0, false);
  table_options.block_cache = cache;
  table_options.block_cache_compressed = compressed_cache;
  options.table_factory.reset(new BlockBasedTableFactory(table_options));
@@ -204,6 +202,9 @@ TEST_F(DBBlockCacheTest, TestWithCompressedBlockCache) {
  cache->SetCapacity(usage);
  cache->SetStrictCapacityLimit(true);
  ASSERT_EQ(usage, cache->GetPinnedUsage());
+  // compressed_cache->SetCapacity(compressed_usage);
+  compressed_cache->SetCapacity(0);
+  // compressed_cache->SetStrictCapacityLimit(true);
  iter = db_->NewIterator(read_options);
  iter->Seek(ToString(kNumBlocks - 1));
  ASSERT_TRUE(iter->status().IsIncomplete());
@@ -268,142 +269,16 @@ TEST_F(DBBlockCacheTest, IndexAndFilterBlocksOfNewTableAddedToCache) {
  ASSERT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));

  // Make sure index block is in cache.
-  auto index_block_hit = TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT);
+  auto index_block_hit = TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT);
  value = Get(1, "key");
-  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
+  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
  ASSERT_EQ(index_block_hit + 1,
-            TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
+            TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));

  value = Get(1, "key");
-  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
+  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
  ASSERT_EQ(index_block_hit + 2,
-            TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
-}
-
-TEST_F(DBBlockCacheTest, IndexAndFilterBlocksStats) {
-  Options options = CurrentOptions();
-  options.create_if_missing = true;
-  options.statistics = rocksdb::CreateDBStatistics();
-  BlockBasedTableOptions table_options;
-  table_options.cache_index_and_filter_blocks = true;
-  // 200 bytes are enough to hold the first two blocks
-  std::shared_ptr<Cache> cache = NewLRUCache(200, 0, false);
-  table_options.block_cache = cache;
-  table_options.filter_policy.reset(NewBloomFilterPolicy(20));
-  options.table_factory.reset(new BlockBasedTableFactory(table_options));
-  CreateAndReopenWithCF({"pikachu"}, options);
-
-  ASSERT_OK(Put(1, "key", "val"));
-  // Create a new table
-  ASSERT_OK(Flush(1));
-  size_t index_bytes_insert =
-      TestGetTickerCount(options, BLOCK_CACHE_INDEX_BYTES_INSERT);
-  size_t filter_bytes_insert =
-      TestGetTickerCount(options, BLOCK_CACHE_FILTER_BYTES_INSERT);
-  ASSERT_GT(index_bytes_insert, 0);
-  ASSERT_GT(filter_bytes_insert, 0);
-  ASSERT_EQ(cache->GetUsage(), index_bytes_insert + filter_bytes_insert);
-  // set the cache capacity to the current usage
-  cache->SetCapacity(index_bytes_insert + filter_bytes_insert);
-  ASSERT_EQ(TestGetTickerCount(options, BLOCK_CACHE_INDEX_BYTES_EVICT), 0);
-  ASSERT_EQ(TestGetTickerCount(options, BLOCK_CACHE_FILTER_BYTES_EVICT), 0);
-  ASSERT_OK(Put(1, "key2", "val"));
-  // Create a new table
-  ASSERT_OK(Flush(1));
-  // cache evicted old index and block entries
-  ASSERT_GT(TestGetTickerCount(options, BLOCK_CACHE_INDEX_BYTES_INSERT),
-            index_bytes_insert);
-  ASSERT_GT(TestGetTickerCount(options, BLOCK_CACHE_FILTER_BYTES_INSERT),
-            filter_bytes_insert);
-  ASSERT_EQ(TestGetTickerCount(options, BLOCK_CACHE_INDEX_BYTES_EVICT),
-            index_bytes_insert);
-  ASSERT_EQ(TestGetTickerCount(options, BLOCK_CACHE_FILTER_BYTES_EVICT),
-            filter_bytes_insert);
-}
-
-namespace {
-
-// A mock cache wraps LRUCache, and record how many entries have been
-// inserted for each priority.
-class MockCache : public LRUCache {
- public:
-  static uint32_t high_pri_insert_count;
-  static uint32_t low_pri_insert_count;
-
-  MockCache() : LRUCache(1 << 25, 0, false, 0.0) {}
-
-  virtual Status Insert(const Slice& key, void* value, size_t charge,
-                        void (*deleter)(const Slice& key, void* value),
-                        Handle** handle, Priority priority) override {
-    if (priority == Priority::LOW) {
-      low_pri_insert_count++;
-    } else {
-      high_pri_insert_count++;
-    }
-    return LRUCache::Insert(key, value, charge, deleter, handle, priority);
-  }
-};
-
-uint32_t MockCache::high_pri_insert_count = 0;
-uint32_t MockCache::low_pri_insert_count = 0;
-
-}  // anonymous namespace
-
-TEST_F(DBBlockCacheTest, IndexAndFilterBlocksCachePriority) {
-  for (auto priority : {Cache::Priority::LOW, Cache::Priority::HIGH}) {
-    Options options = CurrentOptions();
-    options.create_if_missing = true;
-    options.statistics = rocksdb::CreateDBStatistics();
-    BlockBasedTableOptions table_options;
-    table_options.cache_index_and_filter_blocks = true;
-    table_options.block_cache.reset(new MockCache());
-    table_options.filter_policy.reset(NewBloomFilterPolicy(20));
-    table_options.cache_index_and_filter_blocks_with_high_priority =
-        priority == Cache::Priority::HIGH ? true : false;
-    options.table_factory.reset(new BlockBasedTableFactory(table_options));
-    DestroyAndReopen(options);
-
-    MockCache::high_pri_insert_count = 0;
-    MockCache::low_pri_insert_count = 0;
-
-    // Create a new table.
-    ASSERT_OK(Put("foo", "value"));
-    ASSERT_OK(Put("bar", "value"));
-    ASSERT_OK(Flush());
-    ASSERT_EQ(1, NumTableFilesAtLevel(0));
-
-    // index/filter blocks added to block cache right after table creation.
-    ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
-    ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
-    ASSERT_EQ(2, /* only index/filter were added */
-              TestGetTickerCount(options, BLOCK_CACHE_ADD));
-    ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_MISS));
-    if (priority == Cache::Priority::LOW) {
-      ASSERT_EQ(0, MockCache::high_pri_insert_count);
-      ASSERT_EQ(2, MockCache::low_pri_insert_count);
-    } else {
-      ASSERT_EQ(2, MockCache::high_pri_insert_count);
-      ASSERT_EQ(0, MockCache::low_pri_insert_count);
-    }
-
-    // Access data block.
-    ASSERT_EQ("value", Get("foo"));
-
-    ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
-    ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
-    ASSERT_EQ(3, /*adding data block*/
-              TestGetTickerCount(options, BLOCK_CACHE_ADD));
-    ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_MISS));
-
-    // Data block should be inserted with low priority.
-    if (priority == Cache::Priority::LOW) {
-      ASSERT_EQ(0, MockCache::high_pri_insert_count);
-      ASSERT_EQ(3, MockCache::low_pri_insert_count);
-    } else {
-      ASSERT_EQ(2, MockCache::high_pri_insert_count);
-      ASSERT_EQ(1, MockCache::low_pri_insert_count);
-    }
-  }
+            TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
 }

 TEST_F(DBBlockCacheTest, ParanoidFileChecks) {
@@ -20,46 +20,16 @@ class DBBloomFilterTest : public DBTestBase {
  DBBloomFilterTest() : DBTestBase("/db_bloom_filter_test") {}
 };

-class DBBloomFilterTestWithParam
-    : public DBTestBase,
-      public testing::WithParamInterface<std::tuple<bool, bool>> {
-  //                             public testing::WithParamInterface<bool> {
- protected:
-  bool use_block_based_filter_;
-  bool partition_filters_;
-
- public:
-  DBBloomFilterTestWithParam() : DBTestBase("/db_bloom_filter_tests") {}
-
-  ~DBBloomFilterTestWithParam() {}
-
-  void SetUp() override {
-    use_block_based_filter_ = std::get<0>(GetParam());
-    partition_filters_ = std::get<1>(GetParam());
-  }
-};
-
 // KeyMayExist can lead to a few false positives, but not false negatives.
 // To make test deterministic, use a much larger number of bits per key-20 than
 // bits in the key, so that false positives are eliminated
-TEST_P(DBBloomFilterTestWithParam, KeyMayExist) {
+TEST_F(DBBloomFilterTest, KeyMayExist) {
  do {
    ReadOptions ropts;
    std::string value;
    anon::OptionsOverride options_override;
-    options_override.filter_policy.reset(
-        NewBloomFilterPolicy(20, use_block_based_filter_));
-    options_override.partition_filters = partition_filters_;
-    options_override.metadata_block_size = 32;
+    options_override.filter_policy.reset(NewBloomFilterPolicy(20));
    Options options = CurrentOptions(options_override);
-    if (partition_filters_ &&
-        static_cast<BlockBasedTableOptions*>(
-            options.table_factory->GetOptions())
-                ->index_type != BlockBasedTableOptions::kTwoLevelIndexSearch) {
-      // In the current implementation partitioned filters depend on partitioned
-      // indexes
-      continue;
-    }
    options.statistics = rocksdb::CreateDBStatistics();
    CreateAndReopenWithCF({"pikachu"}, options);

@@ -117,208 +87,235 @@ TEST_P(DBBloomFilterTestWithParam, KeyMayExist) {
      ChangeOptions(kSkipPlainTable | kSkipHashIndex | kSkipFIFOCompaction));
 }

+// A delete is skipped for key if KeyMayExist(key) returns False
+// Tests Writebatch consistency and proper delete behaviour
+TEST_F(DBBloomFilterTest, FilterDeletes) {
+  do {
+    anon::OptionsOverride options_override;
+    options_override.filter_policy.reset(NewBloomFilterPolicy(20));
+    Options options = CurrentOptions(options_override);
+    options.filter_deletes = true;
+    CreateAndReopenWithCF({"pikachu"}, options);
+    WriteBatch batch;
+
+    batch.Delete(handles_[1], "a");
+    dbfull()->Write(WriteOptions(), &batch);
+    ASSERT_EQ(AllEntriesFor("a", 1), "[ ]");  // Delete skipped
+    batch.Clear();
+
+    batch.Put(handles_[1], "a", "b");
+    batch.Delete(handles_[1], "a");
+    dbfull()->Write(WriteOptions(), &batch);
+    ASSERT_EQ(Get(1, "a"), "NOT_FOUND");
+    ASSERT_EQ(AllEntriesFor("a", 1), "[ DEL, b ]");  // Delete issued
+    batch.Clear();
+
+    batch.Delete(handles_[1], "c");
+    batch.Put(handles_[1], "c", "d");
+    dbfull()->Write(WriteOptions(), &batch);
+    ASSERT_EQ(Get(1, "c"), "d");
+    ASSERT_EQ(AllEntriesFor("c", 1), "[ d ]");  // Delete skipped
+    batch.Clear();
+
+    ASSERT_OK(Flush(1));  // A stray Flush
+
+    batch.Delete(handles_[1], "c");
+    dbfull()->Write(WriteOptions(), &batch);
+    ASSERT_EQ(AllEntriesFor("c", 1), "[ DEL, d ]");  // Delete issued
+    batch.Clear();
+  } while (ChangeCompactOptions());
+}
+
 TEST_F(DBBloomFilterTest, GetFilterByPrefixBloom) {
-  for (bool partition_filters : {true, false}) {
-    Options options = last_options_;
-    options.prefix_extractor.reset(NewFixedPrefixTransform(8));
-    options.statistics = rocksdb::CreateDBStatistics();
-    BlockBasedTableOptions bbto;
-    bbto.filter_policy.reset(NewBloomFilterPolicy(10, false));
-    if (partition_filters) {
-      bbto.partition_filters = true;
-      bbto.index_type = BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
-    }
-    bbto.whole_key_filtering = false;
-    options.table_factory.reset(NewBlockBasedTableFactory(bbto));
-    DestroyAndReopen(options);
+  Options options = last_options_;
+  options.prefix_extractor.reset(NewFixedPrefixTransform(8));
+  options.statistics = rocksdb::CreateDBStatistics();
+  BlockBasedTableOptions bbto;
+  bbto.filter_policy.reset(NewBloomFilterPolicy(10, false));
+  bbto.whole_key_filtering = false;
+  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+  DestroyAndReopen(options);

-    WriteOptions wo;
-    ReadOptions ro;
-    FlushOptions fo;
-    fo.wait = true;
-    std::string value;
+  WriteOptions wo;
+  ReadOptions ro;
+  FlushOptions fo;
+  fo.wait = true;
+  std::string value;

-    ASSERT_OK(dbfull()->Put(wo, "barbarbar", "foo"));
-    ASSERT_OK(dbfull()->Put(wo, "barbarbar2", "foo2"));
-    ASSERT_OK(dbfull()->Put(wo, "foofoofoo", "bar"));
+  ASSERT_OK(dbfull()->Put(wo, "barbarbar", "foo"));
+  ASSERT_OK(dbfull()->Put(wo, "barbarbar2", "foo2"));
+  ASSERT_OK(dbfull()->Put(wo, "foofoofoo", "bar"));

-    dbfull()->Flush(fo);
+  dbfull()->Flush(fo);

-    ASSERT_EQ("foo", Get("barbarbar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
-    ASSERT_EQ("foo2", Get("barbarbar2"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
-    ASSERT_EQ("NOT_FOUND", Get("barbarbar3"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
+  ASSERT_EQ("foo", Get("barbarbar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
+  ASSERT_EQ("foo2", Get("barbarbar2"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
+  ASSERT_EQ("NOT_FOUND", Get("barbarbar3"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);

-    ASSERT_EQ("NOT_FOUND", Get("barfoofoo"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
+  ASSERT_EQ("NOT_FOUND", Get("barfoofoo"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);

-    ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 2);
+  ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 2);

-    ro.total_order_seek = true;
-    ASSERT_TRUE(db_->Get(ro, "foobarbar", &value).IsNotFound());
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 2);
-  }
+  ro.total_order_seek = true;
+  ASSERT_TRUE(db_->Get(ro, "foobarbar", &value).IsNotFound());
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 2);
 }

 TEST_F(DBBloomFilterTest, WholeKeyFilterProp) {
-  for (bool partition_filters : {true, false}) {
-    Options options = last_options_;
-    options.prefix_extractor.reset(NewFixedPrefixTransform(3));
-    options.statistics = rocksdb::CreateDBStatistics();
+  Options options = last_options_;
+  options.prefix_extractor.reset(NewFixedPrefixTransform(3));
+  options.statistics = rocksdb::CreateDBStatistics();

-    BlockBasedTableOptions bbto;
-    bbto.filter_policy.reset(NewBloomFilterPolicy(10, false));
-    bbto.whole_key_filtering = false;
-    if (partition_filters) {
-      bbto.partition_filters = true;
-      bbto.index_type = BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
-    }
-    options.table_factory.reset(NewBlockBasedTableFactory(bbto));
-    DestroyAndReopen(options);
+  BlockBasedTableOptions bbto;
+  bbto.filter_policy.reset(NewBloomFilterPolicy(10, false));
+  bbto.whole_key_filtering = false;
+  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+  DestroyAndReopen(options);

-    WriteOptions wo;
-    ReadOptions ro;
-    FlushOptions fo;
-    fo.wait = true;
-    std::string value;
+  WriteOptions wo;
+  ReadOptions ro;
+  FlushOptions fo;
+  fo.wait = true;
+  std::string value;

-    ASSERT_OK(dbfull()->Put(wo, "foobar", "foo"));
-    // Needs insert some keys to make sure files are not filtered out by key
-    // ranges.
-    ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
-    ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
-    dbfull()->Flush(fo);
+  ASSERT_OK(dbfull()->Put(wo, "foobar", "foo"));
+  // Needs insert some keys to make sure files are not filtered out by key
+  // ranges.
+  ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
+  ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
+  dbfull()->Flush(fo);

-    Reopen(options);
-    ASSERT_EQ("NOT_FOUND", Get("foo"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
-    ASSERT_EQ("NOT_FOUND", Get("bar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
-    ASSERT_EQ("foo", Get("foobar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
+  Reopen(options);
+  ASSERT_EQ("NOT_FOUND", Get("foo"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
+  ASSERT_EQ("NOT_FOUND", Get("bar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
+  ASSERT_EQ("foo", Get("foobar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);

-    // Reopen with whole key filtering enabled and prefix extractor
-    // NULL. Bloom filter should be off for both of whole key and
-    // prefix bloom.
-    bbto.whole_key_filtering = true;
-    options.table_factory.reset(NewBlockBasedTableFactory(bbto));
-    options.prefix_extractor.reset();
-    Reopen(options);
+  // Reopen with whole key filtering enabled and prefix extractor
+  // NULL. Bloom filter should be off for both of whole key and
+  // prefix bloom.
+  bbto.whole_key_filtering = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+  options.prefix_extractor.reset();
+  Reopen(options);

-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
-    ASSERT_EQ("NOT_FOUND", Get("foo"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
-    ASSERT_EQ("NOT_FOUND", Get("bar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
-    ASSERT_EQ("foo", Get("foobar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
-    // Write DB with only full key filtering.
-    ASSERT_OK(dbfull()->Put(wo, "foobar", "foo"));
-    // Needs insert some keys to make sure files are not filtered out by key
-    // ranges.
-    ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
-    ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
-    db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
+  ASSERT_EQ("NOT_FOUND", Get("foo"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
+  ASSERT_EQ("NOT_FOUND", Get("bar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
+  ASSERT_EQ("foo", Get("foobar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
+  // Write DB with only full key filtering.
+  ASSERT_OK(dbfull()->Put(wo, "foobar", "foo"));
+  // Needs insert some keys to make sure files are not filtered out by key
+  // ranges.
+  ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
+  ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
+  db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);

-    // Reopen with both of whole key off and prefix extractor enabled.
-    // Still no bloom filter should be used.
-    options.prefix_extractor.reset(NewFixedPrefixTransform(3));
-    bbto.whole_key_filtering = false;
-    options.table_factory.reset(NewBlockBasedTableFactory(bbto));
-    Reopen(options);
+  // Reopen with both of whole key off and prefix extractor enabled.
+  // Still no bloom filter should be used.
+  options.prefix_extractor.reset(NewFixedPrefixTransform(3));
+  bbto.whole_key_filtering = false;
+  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+  Reopen(options);

-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
-    ASSERT_EQ("NOT_FOUND", Get("foo"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
-    ASSERT_EQ("NOT_FOUND", Get("bar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
-    ASSERT_EQ("foo", Get("foobar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
+  ASSERT_EQ("NOT_FOUND", Get("foo"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
+  ASSERT_EQ("NOT_FOUND", Get("bar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
+  ASSERT_EQ("foo", Get("foobar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);

-    // Try to create a DB with mixed files:
-    ASSERT_OK(dbfull()->Put(wo, "foobar", "foo"));
-    // Needs insert some keys to make sure files are not filtered out by key
-    // ranges.
-    ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
-    ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
-    db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
+  // Try to create a DB with mixed files:
+  ASSERT_OK(dbfull()->Put(wo, "foobar", "foo"));
+  // Needs insert some keys to make sure files are not filtered out by key
+  // ranges.
+  ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
+  ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
+  db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);

-    options.prefix_extractor.reset();
-    bbto.whole_key_filtering = true;
-    options.table_factory.reset(NewBlockBasedTableFactory(bbto));
-    Reopen(options);
+  options.prefix_extractor.reset();
+  bbto.whole_key_filtering = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+  Reopen(options);

-    // Try to create a DB with mixed files.
-    ASSERT_OK(dbfull()->Put(wo, "barfoo", "bar"));
-    // In this case needs insert some keys to make sure files are
-    // not filtered out by key ranges.
-    ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
-    ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
-    Flush();
+  // Try to create a DB with mixed files.
+  ASSERT_OK(dbfull()->Put(wo, "barfoo", "bar"));
+  // In this case needs insert some keys to make sure files are
+  // not filtered out by key ranges.
+  ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
+  ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
+  Flush();

-    // Now we have two files:
-    // File 1: An older file with prefix bloom.
-    // File 2: A newer file with whole bloom filter.
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
-    ASSERT_EQ("NOT_FOUND", Get("foo"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 2);
-    ASSERT_EQ("NOT_FOUND", Get("bar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 3);
-    ASSERT_EQ("foo", Get("foobar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 4);
-    ASSERT_EQ("bar", Get("barfoo"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 4);
+  // Now we have two files:
+  // File 1: An older file with prefix bloom.
+  // File 2: A newer file with whole bloom filter.
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
+  ASSERT_EQ("NOT_FOUND", Get("foo"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 2);
+  ASSERT_EQ("NOT_FOUND", Get("bar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 3);
+  ASSERT_EQ("foo", Get("foobar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 4);
+  ASSERT_EQ("bar", Get("barfoo"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 4);

-    // Reopen with the same setting: only whole key is used
-    Reopen(options);
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 4);
-    ASSERT_EQ("NOT_FOUND", Get("foo"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 5);
-    ASSERT_EQ("NOT_FOUND", Get("bar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 6);
-    ASSERT_EQ("foo", Get("foobar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 7);
-    ASSERT_EQ("bar", Get("barfoo"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 7);
+  // Reopen with the same setting: only whole key is used
+  Reopen(options);
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 4);
+  ASSERT_EQ("NOT_FOUND", Get("foo"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 5);
+  ASSERT_EQ("NOT_FOUND", Get("bar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 6);
+  ASSERT_EQ("foo", Get("foobar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 7);
+  ASSERT_EQ("bar", Get("barfoo"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 7);

-    // Restart with both filters are allowed
-    options.prefix_extractor.reset(NewFixedPrefixTransform(3));
-    bbto.whole_key_filtering = true;
-    options.table_factory.reset(NewBlockBasedTableFactory(bbto));
-    Reopen(options);
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 7);
-    // File 1 will has it filtered out.
-    // File 2 will not, as prefix `foo` exists in the file.
-    ASSERT_EQ("NOT_FOUND", Get("foo"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 8);
-    ASSERT_EQ("NOT_FOUND", Get("bar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 10);
-    ASSERT_EQ("foo", Get("foobar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 11);
-    ASSERT_EQ("bar", Get("barfoo"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 11);
+  // Restart with both filters are allowed
+  options.prefix_extractor.reset(NewFixedPrefixTransform(3));
+  bbto.whole_key_filtering = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+  Reopen(options);
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 7);
+  // File 1 will has it filtered out.
+  // File 2 will not, as prefix `foo` exists in the file.
+  ASSERT_EQ("NOT_FOUND", Get("foo"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 8);
+  ASSERT_EQ("NOT_FOUND", Get("bar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 10);
+  ASSERT_EQ("foo", Get("foobar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 11);
+  ASSERT_EQ("bar", Get("barfoo"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 11);

-    // Restart with only prefix bloom is allowed.
-    options.prefix_extractor.reset(NewFixedPrefixTransform(3));
-    bbto.whole_key_filtering = false;
-    options.table_factory.reset(NewBlockBasedTableFactory(bbto));
-    Reopen(options);
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 11);
-    ASSERT_EQ("NOT_FOUND", Get("foo"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 11);
-    ASSERT_EQ("NOT_FOUND", Get("bar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 12);
-    ASSERT_EQ("foo", Get("foobar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 12);
-    ASSERT_EQ("bar", Get("barfoo"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 12);
-  }
+  // Restart with only prefix bloom is allowed.
+  options.prefix_extractor.reset(NewFixedPrefixTransform(3));
+  bbto.whole_key_filtering = false;
+  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+  Reopen(options);
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 11);
+  ASSERT_EQ("NOT_FOUND", Get("foo"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 11);
+  ASSERT_EQ("NOT_FOUND", Get("bar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 12);
+  ASSERT_EQ("foo", Get("foobar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 12);
+  ASSERT_EQ("bar", Get("barfoo"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 12);
 }

-TEST_P(DBBloomFilterTestWithParam, BloomFilter) {
+TEST_F(DBBloomFilterTest, BloomFilter) {
  do {
    Options options = CurrentOptions();
    env_->count_random_reads_ = true;
@@ -327,14 +324,7 @@ TEST_P(DBBloomFilterTestWithParam, BloomFilter) {
    // trigger reset of table_factory
    BlockBasedTableOptions table_options;
    table_options.no_block_cache = true;
-    table_options.filter_policy.reset(
-        NewBloomFilterPolicy(10, use_block_based_filter_));
-    table_options.partition_filters = partition_filters_;
-    if (partition_filters_) {
-      table_options.index_type =
-          BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
-    }
-    table_options.metadata_block_size = 32;
+    table_options.filter_policy.reset(NewBloomFilterPolicy(10));
    options.table_factory.reset(NewBlockBasedTableFactory(table_options));

    CreateAndReopenWithCF({"pikachu"}, options);
@@ -361,13 +351,7 @@ TEST_P(DBBloomFilterTestWithParam, BloomFilter) {
    int reads = env_->random_read_counter_.Read();
    fprintf(stderr, "%d present => %d reads\n", N, reads);
    ASSERT_GE(reads, N);
-    if (partition_filters_) {
-      // Without block cache, we read an extra partition filter per each
-      // level*read and a partition index per each read
-      ASSERT_LE(reads, 4 * N + 2 * N / 100);
-    } else {
-      ASSERT_LE(reads, N + 2 * N / 100);
-    }
+    ASSERT_LE(reads, N + 2 * N / 100);

    // Lookup present keys.  Should rarely read from either sstable.
    env_->random_read_counter_.Reset();
@@ -376,24 +360,13 @@ TEST_P(DBBloomFilterTestWithParam, BloomFilter) {
    }
    reads = env_->random_read_counter_.Read();
    fprintf(stderr, "%d missing => %d reads\n", N, reads);
-    if (partition_filters_) {
-      // With partitioned filter we read one extra filter per level per each
-      // missed read.
-      ASSERT_LE(reads, 2 * N + 3 * N / 100);
-    } else {
-      ASSERT_LE(reads, 3 * N / 100);
-    }
+    ASSERT_LE(reads, 3 * N / 100);

    env_->delay_sstable_sync_.store(false, std::memory_order_release);
    Close();
  } while (ChangeCompactOptions());
 }

-INSTANTIATE_TEST_CASE_P(DBBloomFilterTestWithParam, DBBloomFilterTestWithParam,
-                        ::testing::Values(std::make_tuple(true, false),
-                                          std::make_tuple(false, true),
-                                          std::make_tuple(false, false)));
-
 TEST_F(DBBloomFilterTest, BloomFilterRate) {
  while (ChangeFilterOptions()) {
    Options options = CurrentOptions();
@@ -449,13 +422,28 @@ TEST_F(DBBloomFilterTest, BloomFilterCompatibility) {
    ASSERT_EQ(Key(i), Get(1, Key(i)));
  }
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
+}

-  // Check db with partitioned full filter
-  table_options.partition_filters = true;
-  table_options.index_type =
-      BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
+TEST_F(DBBloomFilterTest, BloomFilterReverseCompatibility) {
+  Options options = CurrentOptions();
+  options.statistics = rocksdb::CreateDBStatistics();
+  BlockBasedTableOptions table_options;
  table_options.filter_policy.reset(NewBloomFilterPolicy(10, false));
  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+
+  // Create with full filter
+  CreateAndReopenWithCF({"pikachu"}, options);
+
+  const int maxKey = 10000;
+  for (int i = 0; i < maxKey; i++) {
+    ASSERT_OK(Put(1, Key(i), Key(i)));
+  }
+  ASSERT_OK(Put(1, Key(maxKey + 55555), Key(maxKey + 55555)));
+  Flush(1);
+
+  // Check db with block_based filter
+  table_options.filter_policy.reset(NewBloomFilterPolicy(10, true));
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
  ReopenWithColumnFamilies({"default", "pikachu"}, options);

  // Check if they can be found
@@ -465,43 +453,6 @@ TEST_F(DBBloomFilterTest, BloomFilterCompatibility) {
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
 }

-TEST_F(DBBloomFilterTest, BloomFilterReverseCompatibility) {
-  for (bool partition_filters : {true, false}) {
-    Options options = CurrentOptions();
-    options.statistics = rocksdb::CreateDBStatistics();
-    BlockBasedTableOptions table_options;
-    if (partition_filters) {
-      table_options.partition_filters = true;
-      table_options.index_type =
-          BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
-    }
-    table_options.filter_policy.reset(NewBloomFilterPolicy(10, false));
-    options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-    DestroyAndReopen(options);
-
-    // Create with full filter
-    CreateAndReopenWithCF({"pikachu"}, options);
-
-    const int maxKey = 10000;
-    for (int i = 0; i < maxKey; i++) {
-      ASSERT_OK(Put(1, Key(i), Key(i)));
-    }
-    ASSERT_OK(Put(1, Key(maxKey + 55555), Key(maxKey + 55555)));
-    Flush(1);
-
-    // Check db with block_based filter
-    table_options.filter_policy.reset(NewBloomFilterPolicy(10, true));
-    options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-    ReopenWithColumnFamilies({"default", "pikachu"}, options);
-
-    // Check if they can be found
-    for (int i = 0; i < maxKey; i++) {
-      ASSERT_EQ(Key(i), Get(1, Key(i)));
-    }
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
-  }
-}
-
 namespace {
 // A wrapped bloom over default FilterPolicy
 class WrappedBloom : public FilterPolicy {
@@ -656,35 +607,25 @@ TEST_F(DBBloomFilterTest, PrefixExtractorBlockFilter) {
 #ifndef ROCKSDB_LITE
 class BloomStatsTestWithParam
    : public DBBloomFilterTest,
-      public testing::WithParamInterface<std::tuple<bool, bool, bool>> {
+      public testing::WithParamInterface<std::tuple<bool, bool>> {
 public:
  BloomStatsTestWithParam() {
    use_block_table_ = std::get<0>(GetParam());
    use_block_based_builder_ = std::get<1>(GetParam());
-    partition_filters_ = std::get<2>(GetParam());

    options_.create_if_missing = true;
    options_.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(4));
-    options_.memtable_prefix_bloom_size_ratio =
-        8.0 * 1024.0 / static_cast<double>(options_.write_buffer_size);
+    options_.memtable_prefix_bloom_bits = 8 * 1024;
    if (use_block_table_) {
      BlockBasedTableOptions table_options;
      table_options.hash_index_allow_collision = false;
-      if (partition_filters_) {
-        assert(!use_block_based_builder_);
-        table_options.partition_filters = partition_filters_;
-        table_options.index_type =
-            BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
-      }
      table_options.filter_policy.reset(
          NewBloomFilterPolicy(10, use_block_based_builder_));
      options_.table_factory.reset(NewBlockBasedTableFactory(table_options));
    } else {
-      assert(!partition_filters_);  // not supported in plain table
      PlainTableOptions table_options;
      options_.table_factory.reset(NewPlainTableFactory(table_options));
    }
-    options_.env = env_;

    perf_context.Reset();
    DestroyAndReopen(options_);
@@ -701,7 +642,6 @@ class BloomStatsTestWithParam

  bool use_block_table_;
  bool use_block_based_builder_;
-  bool partition_filters_;
  Options options_;
 };

@@ -744,10 +684,13 @@ TEST_P(BloomStatsTestWithParam, BloomStatsTest) {
  ASSERT_EQ(0, perf_context.bloom_sst_miss_count);

  // check SST bloom stats
+  // NOTE: hits per get differs because of code paths differences
+  // in BlockBasedTable::Get()
+  int hits_per_get = use_block_table_ && !use_block_based_builder_ ? 2 : 1;
  ASSERT_EQ(value1, Get(key1));
-  ASSERT_EQ(1, perf_context.bloom_sst_hit_count);
+  ASSERT_EQ(hits_per_get, perf_context.bloom_sst_hit_count);
  ASSERT_EQ(value3, Get(key3));
-  ASSERT_EQ(2, perf_context.bloom_sst_hit_count);
+  ASSERT_EQ(2 * hits_per_get, perf_context.bloom_sst_hit_count);

  ASSERT_EQ("NOT_FOUND", Get(key2));
  ASSERT_EQ(1, perf_context.bloom_sst_miss_count);
@@ -796,44 +739,25 @@ TEST_P(BloomStatsTestWithParam, BloomStatsTestWithIter) {
  ASSERT_OK(iter->status());
  ASSERT_TRUE(iter->Valid());
  ASSERT_EQ(value1, iter->value().ToString());
-  if (partition_filters_) {
-    ASSERT_EQ(0, perf_context.bloom_sst_hit_count);   // no_io
-    ASSERT_EQ(0, perf_context.bloom_sst_miss_count);  // no_io
-  } else {
-    ASSERT_EQ(1, perf_context.bloom_sst_hit_count);
-  }
+  ASSERT_EQ(1, perf_context.bloom_sst_hit_count);

  iter->Seek(key3);
  ASSERT_OK(iter->status());
  ASSERT_TRUE(iter->Valid());
  ASSERT_EQ(value3, iter->value().ToString());
-  if (partition_filters_) {
-    ASSERT_EQ(0, perf_context.bloom_sst_hit_count);   // no_io
-    ASSERT_EQ(0, perf_context.bloom_sst_miss_count);  // no_io
-  } else {
-    ASSERT_EQ(2, perf_context.bloom_sst_hit_count);
-  }
+  ASSERT_EQ(2, perf_context.bloom_sst_hit_count);

  iter->Seek(key2);
  ASSERT_OK(iter->status());
-  if (partition_filters_) {
-    // iter is still valid since filter did not reject the key2
-    ASSERT_TRUE(iter->Valid());
-    ASSERT_EQ(0, perf_context.bloom_sst_hit_count);   // no_io
-    ASSERT_EQ(0, perf_context.bloom_sst_miss_count);  // no_io
-  } else {
-    ASSERT_TRUE(!iter->Valid());
-    ASSERT_EQ(1, perf_context.bloom_sst_miss_count);
-    ASSERT_EQ(2, perf_context.bloom_sst_hit_count);
-  }
+  ASSERT_TRUE(!iter->Valid());
+  ASSERT_EQ(1, perf_context.bloom_sst_miss_count);
+  ASSERT_EQ(2, perf_context.bloom_sst_hit_count);
 }

 INSTANTIATE_TEST_CASE_P(BloomStatsTestWithParam, BloomStatsTestWithParam,
-                        ::testing::Values(std::make_tuple(true, true, false),
-                                          std::make_tuple(true, false, false),
-                                          std::make_tuple(true, false, true),
-                                          std::make_tuple(false, false,
-                                                          false)));
+                        ::testing::Values(std::make_tuple(true, true),
+                                          std::make_tuple(true, false),
+                                          std::make_tuple(false, false)));

 namespace {
 void PrefixScanInit(DBBloomFilterTest* dbtest) {
@@ -887,6 +811,8 @@ void PrefixScanInit(DBBloomFilterTest* dbtest) {
 }  // namespace

 TEST_F(DBBloomFilterTest, PrefixScan) {
+  XFUNC_TEST("", "dbtest_prefix", prefix_skip1, XFuncPoint::SetSkip,
+             kSkipNoPrefix);
  while (ChangeFilterOptions()) {
    int count;
    Slice prefix;
@@ -907,7 +833,6 @@ TEST_F(DBBloomFilterTest, PrefixScan) {
    options.max_background_compactions = 2;
    options.create_if_missing = true;
    options.memtable_factory.reset(NewHashSkipListRepFactory(16));
-    options.allow_concurrent_memtable_write = false;

    BlockBasedTableOptions table_options;
    table_options.no_block_cache = true;
@@ -933,6 +858,7 @@ TEST_F(DBBloomFilterTest, PrefixScan) {
    ASSERT_EQ(env_->random_read_counter_.Read(), 2);
    Close();
  }  // end of while
+  XFUNC_TEST("", "dbtest_prefix", prefix_skip1, XFuncPoint::SetSkip, 0);
 }

 TEST_F(DBBloomFilterTest, OptimizeFiltersForHits) {
@@ -13,7 +13,6 @@
 namespace rocksdb {

 static int cfilter_count = 0;
-static int cfilter_skips = 0;

 // This is a static filter used for filtering
 // kvs during the compaction process.
@@ -66,30 +65,6 @@ class DeleteISFilter : public CompactionFilter {
  virtual const char* Name() const override { return "DeleteFilter"; }
 };

-// Skip x if floor(x/10) is even, use range skips. Requires that keys are
-// zero-padded to length 10.
-class SkipEvenFilter : public CompactionFilter {
- public:
-  virtual Decision FilterV2(int level, const Slice& key, ValueType value_type,
-                            const Slice& existing_value, std::string* new_value,
-                            std::string* skip_until) const override {
-    cfilter_count++;
-    int i = std::stoi(key.ToString());
-    if (i / 10 % 2 == 0) {
-      char key_str[100];
-      snprintf(key_str, sizeof(key), "%010d", i / 10 * 10 + 10);
-      *skip_until = key_str;
-      ++cfilter_skips;
-      return Decision::kRemoveAndSkipUntil;
-    }
-    return Decision::kKeep;
-  }
-
-  virtual bool IgnoreSnapshots() const override { return true; }
-
-  virtual const char* Name() const override { return "DeleteFilter"; }
-};
-
 class DelayFilter : public CompactionFilter {
 public:
  explicit DelayFilter(DBTestBase* d) : db_test(d) {}
@@ -199,20 +174,6 @@ class DeleteISFilterFactory : public CompactionFilterFactory {
  virtual const char* Name() const override { return "DeleteFilterFactory"; }
 };

-class SkipEvenFilterFactory : public CompactionFilterFactory {
- public:
-  virtual std::unique_ptr<CompactionFilter> CreateCompactionFilter(
-      const CompactionFilter::Context& context) override {
-    if (context.is_manual_compaction) {
-      return std::unique_ptr<CompactionFilter>(new SkipEvenFilter());
-    } else {
-      return std::unique_ptr<CompactionFilter>(nullptr);
-    }
-  }
-
-  virtual const char* Name() const override { return "SkipEvenFilterFactory"; }
-};
-
 class DelayFilterFactory : public CompactionFilterFactory {
 public:
  explicit DelayFilterFactory(DBTestBase* d) : db_test(d) {}
@@ -300,10 +261,8 @@ TEST_F(DBTestCompactionFilter, CompactionFilter) {
  int total = 0;
  Arena arena;
  {
-    InternalKeyComparator icmp(options.comparator);
-    RangeDelAggregator range_del_agg(icmp, {} /* snapshots */);
    ScopedArenaIterator iter(
-        dbfull()->NewInternalIterator(&arena, &range_del_agg, handles_[1]));
+        dbfull()->NewInternalIterator(&arena, handles_[1]));
    iter->SeekToFirst();
    ASSERT_OK(iter->status());
    while (iter->Valid()) {
@@ -390,10 +349,8 @@ TEST_F(DBTestCompactionFilter, CompactionFilter) {
  // level Lmax because this record is at the tip
  count = 0;
  {
-    InternalKeyComparator icmp(options.comparator);
-    RangeDelAggregator range_del_agg(icmp, {} /* snapshots */);
    ScopedArenaIterator iter(
-        dbfull()->NewInternalIterator(&arena, &range_del_agg, handles_[1]));
+        dbfull()->NewInternalIterator(&arena, handles_[1]));
    iter->SeekToFirst();
    ASSERT_OK(iter->status());
    while (iter->Valid()) {
@@ -609,10 +566,7 @@ TEST_F(DBTestCompactionFilter, CompactionFilterContextManual) {
    int count = 0;
    int total = 0;
    Arena arena;
-    InternalKeyComparator icmp(options.comparator);
-    RangeDelAggregator range_del_agg(icmp, {} /* snapshots */);
-    ScopedArenaIterator iter(
-        dbfull()->NewInternalIterator(&arena, &range_del_agg));
+    ScopedArenaIterator iter(dbfull()->NewInternalIterator(&arena));
    iter->SeekToFirst();
    ASSERT_OK(iter->status());
    while (iter->Valid()) {
@@ -758,47 +712,6 @@ TEST_F(DBTestCompactionFilter, CompactionFilterIgnoreSnapshot) {
 }
 #endif  // ROCKSDB_LITE

-TEST_F(DBTestCompactionFilter, SkipUntil) {
-  Options options = CurrentOptions();
-  options.compaction_filter_factory = std::make_shared<SkipEvenFilterFactory>();
-  options.disable_auto_compactions = true;
-  options.create_if_missing = true;
-  DestroyAndReopen(options);
-
-  // Write 100K keys, these are written to a few files in L0.
-  for (int table = 0; table < 4; ++table) {
-    // Key ranges in tables are [0, 38], [106, 149], [212, 260], [318, 371].
-    for (int i = table * 6; i < 39 + table * 11; ++i) {
-      char key[100];
-      snprintf(key, sizeof(key), "%010d", table * 100 + i);
-      Put(key, std::to_string(table * 1000 + i));
-    }
-    Flush();
-  }
-
-  cfilter_skips = 0;
-  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
-  // Numberof skips in tables: 2, 3, 3, 3.
-  ASSERT_EQ(11, cfilter_skips);
-
-  for (int table = 0; table < 4; ++table) {
-    for (int i = table * 6; i < 39 + table * 11; ++i) {
-      int k = table * 100 + i;
-      char key[100];
-      snprintf(key, sizeof(key), "%010d", table * 100 + i);
-      auto expected = std::to_string(table * 1000 + i);
-      std::string val;
-      Status s = db_->Get(ReadOptions(), key, &val);
-      if (k / 10 % 2 == 0) {
-        ASSERT_TRUE(s.IsNotFound());
-      } else {
-        ASSERT_OK(s);
-        ASSERT_EQ(expected, val);
-      }
-    }
-  }
-}
-
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -9,7 +9,6 @@

 #include "db/db_test_util.h"
 #include "port/stack_trace.h"
-#include "port/port.h"
 #include "rocksdb/experimental.h"
 #include "rocksdb/utilities/convenience.h"
 #include "util/sync_point.h"
@@ -40,12 +39,6 @@ class DBCompactionTestWithParam
  bool exclusive_manual_compaction_;
 };

-class DBCompactionDirectIOTest : public DBCompactionTest,
-                                 public ::testing::WithParamInterface<bool> {
- public:
-  DBCompactionDirectIOTest() : DBCompactionTest() {}
-};
-
 namespace {

 class FlushedFileCollector : public EventListener {
@@ -219,8 +212,8 @@ TEST_P(DBCompactionTestWithParam, CompactionDeletionTrigger) {
 }

 TEST_F(DBCompactionTest, SkipStatsUpdateTest) {
-  // This test verify UpdateAccumulatedStats is not on
-  // if options.skip_stats_update_on_db_open = true
+  // This test verify UpdateAccumulatedStats is not on by observing
+  // the compaction behavior when there are many of deletion entries.
  // The test will need to be updated if the internal behavior changes.

  Options options = DeletionTriggerOptions(CurrentOptions());
@@ -237,6 +230,10 @@ TEST_F(DBCompactionTest, SkipStatsUpdateTest) {
  dbfull()->TEST_WaitForFlushMemTable();
  dbfull()->TEST_WaitForCompact();

+  for (int k = 0; k < kTestSize; ++k) {
+    ASSERT_OK(Delete(Key(k)));
+  }
+
  // Reopen the DB with stats-update disabled
  options.skip_stats_update_on_db_open = true;
  env_->random_file_open_counter_.store(0);
@@ -312,8 +309,7 @@ TEST_F(DBCompactionTest, TestTableReaderForCompaction) {
  dbfull()->TEST_WaitForCompact();
  // Preloading iterator issues one table cache lookup and creates
  // a new table reader. One file is created for flush and one for compaction.
-  // Compaction inputs make no table cache look-up for data/range deletion
-  // iterators
+  // Compaction inputs make no table cache look-up.
  ASSERT_EQ(num_table_cache_lookup, 2);
  // Create new iterator for:
  // (1) 1 for verifying flush results
@@ -334,8 +330,7 @@ TEST_F(DBCompactionTest, TestTableReaderForCompaction) {
  cro.target_level = 2;
  cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
  db_->CompactRange(cro, nullptr, nullptr);
-  // Only verifying compaction outputs issues one table cache lookup
-  // for both data block and range deletion block).
+  // Only verifying compaction outputs issues one table cache lookup.
  ASSERT_EQ(num_table_cache_lookup, 1);
  // One for compaction input, one for verifying compaction results.
  ASSERT_EQ(num_new_table_reader, 2);
@@ -658,72 +653,6 @@ TEST_F(DBCompactionTest, MinorCompactionsHappen) {
  } while (ChangeCompactOptions());
 }

-TEST_F(DBCompactionTest, UserKeyCrossFile1) {
-  Options options = CurrentOptions();
-  options.compaction_style = kCompactionStyleLevel;
-  options.level0_file_num_compaction_trigger = 3;
-
-  DestroyAndReopen(options);
-
-  // create first file and flush to l0
-  Put("4", "A");
-  Put("3", "A");
-  Flush();
-  dbfull()->TEST_WaitForFlushMemTable();
-
-  Put("2", "A");
-  Delete("3");
-  Flush();
-  dbfull()->TEST_WaitForFlushMemTable();
-  ASSERT_EQ("NOT_FOUND", Get("3"));
-
-  // move both files down to l1
-  dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
-  ASSERT_EQ("NOT_FOUND", Get("3"));
-
-  for (int i = 0; i < 3; i++) {
-    Put("2", "B");
-    Flush();
-    dbfull()->TEST_WaitForFlushMemTable();
-  }
-  dbfull()->TEST_WaitForCompact();
-
-  ASSERT_EQ("NOT_FOUND", Get("3"));
-}
-
-TEST_F(DBCompactionTest, UserKeyCrossFile2) {
-  Options options = CurrentOptions();
-  options.compaction_style = kCompactionStyleLevel;
-  options.level0_file_num_compaction_trigger = 3;
-
-  DestroyAndReopen(options);
-
-  // create first file and flush to l0
-  Put("4", "A");
-  Put("3", "A");
-  Flush();
-  dbfull()->TEST_WaitForFlushMemTable();
-
-  Put("2", "A");
-  SingleDelete("3");
-  Flush();
-  dbfull()->TEST_WaitForFlushMemTable();
-  ASSERT_EQ("NOT_FOUND", Get("3"));
-
-  // move both files down to l1
-  dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
-  ASSERT_EQ("NOT_FOUND", Get("3"));
-
-  for (int i = 0; i < 3; i++) {
-    Put("2", "B");
-    Flush();
-    dbfull()->TEST_WaitForFlushMemTable();
-  }
-  dbfull()->TEST_WaitForCompact();
-
-  ASSERT_EQ("NOT_FOUND", Get("3"));
-}
-
 TEST_F(DBCompactionTest, ZeroSeqIdCompaction) {
  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleLevel;
@@ -778,41 +707,6 @@ TEST_F(DBCompactionTest, ZeroSeqIdCompaction) {
  ASSERT_OK(Put("", ""));
 }

-TEST_F(DBCompactionTest, ManualCompactionUnknownOutputSize) {
-  // github issue #2249
-  Options options = CurrentOptions();
-  options.compaction_style = kCompactionStyleLevel;
-  options.level0_file_num_compaction_trigger = 3;
-  DestroyAndReopen(options);
-
-  // create two files in l1 that we can compact
-  for (int i = 0; i < 2; ++i) {
-    for (int j = 0; j < options.level0_file_num_compaction_trigger; j++) {
-      // make l0 files' ranges overlap to avoid trivial move
-      Put(std::to_string(2 * i), std::string(1, 'A'));
-      Put(std::to_string(2 * i + 1), std::string(1, 'A'));
-      Flush();
-      dbfull()->TEST_WaitForFlushMemTable();
-    }
-    dbfull()->TEST_WaitForCompact();
-    ASSERT_EQ(NumTableFilesAtLevel(0, 0), 0);
-    ASSERT_EQ(NumTableFilesAtLevel(1, 0), i + 1);
-  }
-
-  ColumnFamilyMetaData cf_meta;
-  dbfull()->GetColumnFamilyMetaData(dbfull()->DefaultColumnFamily(), &cf_meta);
-  ASSERT_EQ(2, cf_meta.levels[1].files.size());
-  std::vector<std::string> input_filenames;
-  for (const auto& sst_file : cf_meta.levels[1].files) {
-    input_filenames.push_back(sst_file.name);
-  }
-
-  // note CompactionOptions::output_file_size_limit is unset.
-  CompactionOptions compact_opt;
-  compact_opt.compression = kNoCompression;
-  dbfull()->CompactFiles(compact_opt, input_filenames, 1);
-}
-
 // Check that writes done during a memtable compaction are recovered
 // if the database is shutdown during the memtable compaction.
 TEST_F(DBCompactionTest, RecoverDuringMemtableCompaction) {
@@ -1059,21 +953,19 @@ TEST_P(DBCompactionTestWithParam, ManualCompactionPartial) {
      "DBImpl::BackgroundCompaction:NonTrivial",
      [&](void* arg) { non_trivial_move++; });
  bool first = true;
-  // Purpose of dependencies:
-  // 4 -> 1: ensure the order of two non-trivial compactions
-  // 5 -> 2 and 5 -> 3: ensure we do a check before two non-trivial compactions
-  // are installed
+  bool second = true;
  rocksdb::SyncPoint::GetInstance()->LoadDependency(
      {{"DBCompaction::ManualPartial:4", "DBCompaction::ManualPartial:1"},
-       {"DBCompaction::ManualPartial:5", "DBCompaction::ManualPartial:2"},
-       {"DBCompaction::ManualPartial:5", "DBCompaction::ManualPartial:3"}});
+       {"DBCompaction::ManualPartial:2", "DBCompaction::ManualPartial:3"},
+       {"DBCompaction::ManualPartial:5",
+        "DBImpl::BackgroundCompaction:NonTrivial:AfterRun"}});
  rocksdb::SyncPoint::GetInstance()->SetCallBack(
      "DBImpl::BackgroundCompaction:NonTrivial:AfterRun", [&](void* arg) {
        if (first) {
-          first = false;
          TEST_SYNC_POINT("DBCompaction::ManualPartial:4");
+          first = false;
          TEST_SYNC_POINT("DBCompaction::ManualPartial:3");
-        } else {  // second non-trivial compaction
+        } else if (second) {
          TEST_SYNC_POINT("DBCompaction::ManualPartial:2");
        }
      });
@@ -1143,14 +1035,13 @@ TEST_P(DBCompactionTestWithParam, ManualCompactionPartial) {
  ASSERT_EQ(trivial_move, 6);
  ASSERT_EQ(non_trivial_move, 0);

-  rocksdb::port::Thread threads([&] {
+  std::thread threads([&] {
    compact_options.change_level = false;
    compact_options.exclusive_manual_compaction = false;
    std::string begin_string = Key(0);
    std::string end_string = Key(199);
    Slice begin(begin_string);
    Slice end(end_string);
-    // First non-trivial compaction is triggered
    ASSERT_OK(db_->CompactRange(compact_options, &begin, &end));
  });

@@ -1174,17 +1065,15 @@ TEST_P(DBCompactionTestWithParam, ManualCompactionPartial) {
    values[i] = RandomString(&rnd, value_size);
    ASSERT_OK(Put(Key(i), values[i]));
  }
-  // Second non-trivial compaction is triggered
  ASSERT_OK(Flush());

-  // Before two non-trivial compactions are installed, there are 3 files in L0
+  // 3 files in L0
  ASSERT_EQ("3,0,0,0,0,1,2", FilesPerLevel(0));
  TEST_SYNC_POINT("DBCompaction::ManualPartial:5");

+  // 1 file in L6, 1 file in L1
  dbfull()->TEST_WaitForFlushMemTable();
  dbfull()->TEST_WaitForCompact();
-  // After two non-trivial compactions are installed, there is 1 file in L6, and
-  // 1 file in L1
  ASSERT_EQ("0,1,0,0,0,0,1", FilesPerLevel(0));
  threads.join();

@@ -1275,7 +1164,7 @@ TEST_F(DBCompactionTest, ManualPartialFill) {
  ASSERT_EQ(trivial_move, 2);
  ASSERT_EQ(non_trivial_move, 0);

-  rocksdb::port::Thread threads([&] {
+  std::thread threads([&] {
    compact_options.change_level = false;
    compact_options.exclusive_manual_compaction = false;
    std::string begin_string = Key(0);
@@ -1302,8 +1191,7 @@ TEST_F(DBCompactionTest, ManualPartialFill) {
  uint64_t target_size = 4 * options.max_bytes_for_level_base;
  for (int32_t i = 1; i < options.num_levels; i++) {
    ASSERT_LE(SizeAtLevel(i), target_size);
-    target_size = static_cast<uint64_t>(target_size *
-                                        options.max_bytes_for_level_multiplier);
+    target_size *= options.max_bytes_for_level_multiplier;
  }

  TEST_SYNC_POINT("DBCompaction::PartialFill:2");
@@ -1380,8 +1268,7 @@ TEST_F(DBCompactionTest, DeleteFileRange) {
  uint64_t target_size = 4 * options.max_bytes_for_level_base;
  for (int32_t i = 1; i < options.num_levels; i++) {
    ASSERT_LE(SizeAtLevel(i), target_size);
-    target_size = static_cast<uint64_t>(target_size *
-                                        options.max_bytes_for_level_multiplier);
+    target_size *= options.max_bytes_for_level_multiplier;
  }

  size_t old_num_files = CountFiles();
@@ -1884,47 +1771,6 @@ TEST_F(DBCompactionTest, L0_CompactionBug_Issue44_b) {
  } while (ChangeCompactOptions());
 }

-TEST_F(DBCompactionTest, ManualAutoRace) {
-  CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
-  rocksdb::SyncPoint::GetInstance()->LoadDependency(
-      {{"DBImpl::BGWorkCompaction", "DBCompactionTest::ManualAutoRace:1"},
-       {"DBImpl::RunManualCompaction:WaitScheduled",
-        "BackgroundCallCompaction:0"}});
-
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-
-  Put(1, "foo", "");
-  Put(1, "bar", "");
-  Flush(1);
-  Put(1, "foo", "");
-  Put(1, "bar", "");
-  // Generate four files in CF 0, which should trigger an auto compaction
-  Put("foo", "");
-  Put("bar", "");
-  Flush();
-  Put("foo", "");
-  Put("bar", "");
-  Flush();
-  Put("foo", "");
-  Put("bar", "");
-  Flush();
-  Put("foo", "");
-  Put("bar", "");
-  Flush();
-
-  // The auto compaction is scheduled but waited until here
-  TEST_SYNC_POINT("DBCompactionTest::ManualAutoRace:1");
-  // The auto compaction will wait until the the manual compaction is registerd
-  // before processing so that it will be cancelled.
-  dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr, nullptr);
-  ASSERT_EQ("0,1", FilesPerLevel(1));
-
-  // Eventually the cancelled compaction will be rescheduled and executed.
-  dbfull()->TEST_WaitForCompact();
-  ASSERT_EQ("0,1", FilesPerLevel(0));
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-}
-
 TEST_P(DBCompactionTestWithParam, ManualCompaction) {
  Options options = CurrentOptions();
  options.max_subcompactions = max_subcompactions_;
@@ -2574,107 +2420,12 @@ TEST_P(DBCompactionTestWithParam, ForceBottommostLevelCompaction) {
  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
 }

-TEST_P(DBCompactionTestWithParam, IntraL0Compaction) {
-  Options options = CurrentOptions();
-  options.compression = kNoCompression;
-  options.level0_file_num_compaction_trigger = 5;
-  options.max_background_compactions = 2;
-  options.max_subcompactions = max_subcompactions_;
-  DestroyAndReopen(options);
-
-  const size_t kValueSize = 1 << 20;
-  Random rnd(301);
-  std::string value(RandomString(&rnd, kValueSize));
-
-  rocksdb::SyncPoint::GetInstance()->LoadDependency(
-      {{"LevelCompactionPicker::PickCompactionBySize:0",
-        "CompactionJob::Run():Start"}});
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-
-  // index:   0   1   2   3   4   5   6   7   8   9
-  // size:  1MB 1MB 1MB 1MB 1MB 2MB 1MB 1MB 1MB 1MB
-  // score:                     1.5 1.3 1.5 2.0 inf
-  //
-  // Files 0-4 will be included in an L0->L1 compaction.
-  //
-  // L0->L0 will be triggered since the sync points guarantee compaction to base
-  // level is still blocked when files 5-9 trigger another compaction.
-  //
-  // Files 6-9 are the longest span of available files for which
-  // work-per-deleted-file decreases (see "score" row above).
-  for (int i = 0; i < 10; ++i) {
-    for (int j = 0; j < 2; ++j) {
-      ASSERT_OK(Put(Key(0), ""));  // prevents trivial move
-      if (i == 5) {
-        ASSERT_OK(Put(Key(i + 1), value + value));
-      } else {
-        ASSERT_OK(Put(Key(i + 1), value));
-      }
-    }
-    ASSERT_OK(Flush());
-  }
-  dbfull()->TEST_WaitForCompact();
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-
-  std::vector<std::vector<FileMetaData>> level_to_files;
-  dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(),
-                                  &level_to_files);
-  ASSERT_GE(level_to_files.size(), 2);  // at least L0 and L1
-  // L0 has the 2MB file (not compacted) and 4MB file (output of L0->L0)
-  ASSERT_EQ(2, level_to_files[0].size());
-  ASSERT_GT(level_to_files[1].size(), 0);
-  for (int i = 0; i < 2; ++i) {
-    ASSERT_GE(level_to_files[0][0].fd.file_size, 1 << 21);
-  }
-}
-
 INSTANTIATE_TEST_CASE_P(DBCompactionTestWithParam, DBCompactionTestWithParam,
                        ::testing::Values(std::make_tuple(1, true),
                                          std::make_tuple(1, false),
                                          std::make_tuple(4, true),
                                          std::make_tuple(4, false)));

-TEST_P(DBCompactionDirectIOTest, DirectIO) {
-  Options options = CurrentOptions();
-  Destroy(options);
-  options.create_if_missing = true;
-  options.disable_auto_compactions = true;
-  options.use_direct_io_for_flush_and_compaction = GetParam();
-  options.env = new MockEnv(Env::Default());
-  Reopen(options);
-  bool readahead = false;
-  SyncPoint::GetInstance()->SetCallBack(
-      "TableCache::NewIterator:for_compaction", [&](void* arg) {
-        bool* use_direct_reads = static_cast<bool*>(arg);
-        ASSERT_EQ(*use_direct_reads,
-                  options.use_direct_io_for_flush_and_compaction);
-      });
-  SyncPoint::GetInstance()->SetCallBack(
-      "CompactionJob::OpenCompactionOutputFile", [&](void* arg) {
-        bool* use_direct_writes = static_cast<bool*>(arg);
-        ASSERT_EQ(*use_direct_writes,
-                  options.use_direct_io_for_flush_and_compaction);
-      });
-  if (options.use_direct_io_for_flush_and_compaction) {
-    SyncPoint::GetInstance()->SetCallBack(
-        "SanitizeOptions:direct_io", [&](void* arg) {
-          readahead = true;
-        });
-  }
-  SyncPoint::GetInstance()->EnableProcessing();
-  CreateAndReopenWithCF({"pikachu"}, options);
-  MakeTables(3, "p", "q", 1);
-  ASSERT_EQ("1,1,1", FilesPerLevel(1));
-  Compact(1, "p1", "p9");
-  ASSERT_FALSE(readahead ^ options.use_direct_io_for_flush_and_compaction);
-  ASSERT_EQ("0,0,1", FilesPerLevel(1));
-  Destroy(options);
-  delete options.env;
-}
-
-INSTANTIATE_TEST_CASE_P(DBCompactionDirectIOTest, DBCompactionDirectIOTest,
-                        testing::Bool());
-
 class CompactionPriTest : public DBTestBase,
                          public testing::WithParamInterface<uint32_t> {
 public:
@@ -13,7 +13,6 @@
 #if !defined(ROCKSDB_LITE)

 #include "db/db_test_util.h"
-#include "port/port.h"
 #include "port/stack_trace.h"

 namespace rocksdb {
@@ -75,7 +74,6 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase) {
      options.compression_per_level[0] = kNoCompression;
      options.compression_per_level[1] = kLZ4Compression;
      options.compression_per_level[2] = kSnappyCompression;
-      options.env = env_;

      DestroyAndReopen(options);

@@ -127,19 +125,19 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase2) {

  Options options = CurrentOptions();
  options.create_if_missing = true;
-  options.db_write_buffer_size = 204800;
-  options.write_buffer_size = 20480;
+  options.db_write_buffer_size = 2048;
+  options.write_buffer_size = 2048;
  options.max_write_buffer_number = 2;
  options.level0_file_num_compaction_trigger = 2;
  options.level0_slowdown_writes_trigger = 9999;
  options.level0_stop_writes_trigger = 9999;
-  options.target_file_size_base = 9102;
+  options.target_file_size_base = 2048;
  options.level_compaction_dynamic_level_bytes = true;
-  options.max_bytes_for_level_base = 40960;
+  options.max_bytes_for_level_base = 10240;
  options.max_bytes_for_level_multiplier = 4;
  options.max_background_compactions = 2;
  options.num_levels = 5;
-  options.max_compaction_bytes = 0;  // Force not expanding in compactions
+  options.expanded_compaction_factor = 0;  // Force not expanding in compactions
  BlockBasedTableOptions table_options;
  table_options.block_size = 1024;
  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
@@ -156,10 +154,10 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase2) {
  ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
  ASSERT_EQ(4U, int_prop);

-  // Put about 28K to L0
+  // Put about 7K to L0
  for (int i = 0; i < 70; i++) {
    ASSERT_OK(Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))),
-                  RandomString(&rnd, 380)));
+                  RandomString(&rnd, 80)));
  }
  ASSERT_OK(dbfull()->SetOptions({
      {"disable_auto_compactions", "false"},
@@ -169,14 +167,14 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase2) {
  ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
  ASSERT_EQ(4U, int_prop);

-  // Insert extra about 28K to L0. After they are compacted to L4, base level
+  // Insert extra about 3.5K to L0. After they are compacted to L4, base level
  // should be changed to L3.
  ASSERT_OK(dbfull()->SetOptions({
      {"disable_auto_compactions", "true"},
  }));
  for (int i = 0; i < 70; i++) {
    ASSERT_OK(Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))),
-                  RandomString(&rnd, 380)));
+                  RandomString(&rnd, 80)));
  }

  ASSERT_OK(dbfull()->SetOptions({
@@ -201,10 +199,10 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase2) {
  ASSERT_OK(dbfull()->SetOptions({
      {"disable_auto_compactions", "true"},
  }));
-  // Write about 40K more
+  // Write about 10K more
  for (int i = 0; i < 100; i++) {
    ASSERT_OK(Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))),
-                  RandomString(&rnd, 380)));
+                  RandomString(&rnd, 80)));
  }
  ASSERT_OK(dbfull()->SetOptions({
      {"disable_auto_compactions", "false"},
@@ -220,16 +218,15 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase2) {

  // Trigger a condition that the compaction changes base level and L0->Lbase
  // happens at the same time.
-  // We try to make last levels' targets to be 40K, 160K, 640K, add triggers
+  // We try to make last levels' targets to be 10K, 40K, 160K, add triggers
  // another compaction from 40K->160K.
  ASSERT_OK(dbfull()->SetOptions({
      {"disable_auto_compactions", "true"},
  }));
-  // Write about 650K more.
-  // Each file is about 11KB, with 9KB of data.
-  for (int i = 0; i < 1300; i++) {
+  // Write about 150K more
+  for (int i = 0; i < 1350; i++) {
    ASSERT_OK(Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))),
-                  RandomString(&rnd, 380)));
+                  RandomString(&rnd, 80)));
  }
  ASSERT_OK(dbfull()->SetOptions({
      {"disable_auto_compactions", "false"},
@@ -239,40 +236,30 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase2) {
  ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
  ASSERT_EQ(2U, int_prop);

-  // A manual compaction will trigger the base level to become L2
  // Keep Writing data until base level changed 2->1. There will be L0->L2
  // compaction going on at the same time.
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-  rocksdb::SyncPoint::GetInstance()->ClearAllCallBacks();
-
-  rocksdb::SyncPoint::GetInstance()->LoadDependency({
-      {"CompactionJob::Run():Start", "DynamicLevelMaxBytesBase2:0"},
-      {"DynamicLevelMaxBytesBase2:1", "CompactionJob::Run():End"},
-      {"DynamicLevelMaxBytesBase2:compact_range_finish",
-       "FlushJob::WriteLevel0Table"},
-  });
  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+  for (int attempt = 0; attempt <= 20; attempt++) {
+    // Write about 5K more data with two flushes. It should be flush to level 2
+    // but when it is applied, base level is already 1.
+    for (int i = 0; i < 50; i++) {
+      ASSERT_OK(Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))),
+                    RandomString(&rnd, 80)));
+    }
+    Flush();

-  rocksdb::port::Thread thread([this] {
-    TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:compact_range_start");
-    ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
-    TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:compact_range_finish");
-  });
-
-  TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:0");
-  for (int i = 0; i < 2; i++) {
-    ASSERT_OK(Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))),
-                  RandomString(&rnd, 380)));
+    ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
+    if (int_prop == 2U) {
+      env_->SleepForMicroseconds(50000);
+    } else {
+      break;
+    }
  }
-  TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:1");
-
-  Flush();
-
-  thread.join();
-
  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
  rocksdb::SyncPoint::GetInstance()->ClearAllCallBacks();

+  env_->SleepForMicroseconds(200000);
+
  ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
  ASSERT_EQ(1U, int_prop);
 }
@@ -297,7 +284,7 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesCompactRange) {
  options.max_background_compactions = 1;
  const int kNumLevels = 5;
  options.num_levels = kNumLevels;
-  options.max_compaction_bytes = 1;  // Force not expanding in compactions
+  options.expanded_compaction_factor = 0;  // Force not expanding in compactions
  BlockBasedTableOptions table_options;
  table_options.block_size = 1024;
  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
@@ -408,7 +395,7 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBaseInc) {
  env_->SetBackgroundThreads(1, Env::HIGH);
 }

-TEST_F(DBTestDynamicLevel, DISABLED_MigrateToDynamicLevelMaxBytesBase) {
+TEST_F(DBTestDynamicLevel, MigrateToDynamicLevelMaxBytesBase) {
  Random rnd(301);
  const int kMaxKey = 2000;

@@ -464,7 +451,7 @@ TEST_F(DBTestDynamicLevel, DISABLED_MigrateToDynamicLevelMaxBytesBase) {
  compaction_finished = false;
  // Issue manual compaction in one thread and still verify DB state
  // in main thread.
-  rocksdb::port::Thread t([&]() {
+  std::thread t([&]() {
    CompactRangeOptions compact_options;
    compact_options.change_level = true;
    compact_options.target_level = options.num_levels - 1;
@@ -14,19 +14,19 @@
 #endif

 #include <inttypes.h>
-#include <stdint.h>
 #include <algorithm>
 #include <string>
+#include <stdint.h>
 #include "db/db_impl.h"
+#include "db/filename.h"
 #include "db/job_context.h"
 #include "db/version_set.h"
-#include "port/port.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
-#include "util/file_util.h"
-#include "util/filename.h"
+#include "port/port.h"
 #include "util/mutexlock.h"
 #include "util/sync_point.h"
+#include "util/file_util.h"

 namespace rocksdb {

@@ -34,11 +34,12 @@ Status DBImpl::DisableFileDeletions() {
  InstrumentedMutexLock l(&mutex_);
  ++disable_delete_obsolete_files_;
  if (disable_delete_obsolete_files_ == 1) {
-    ROCKS_LOG_INFO(immutable_db_options_.info_log, "File Deletions Disabled");
+    Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
+        "File Deletions Disabled");
  } else {
-    ROCKS_LOG_WARN(immutable_db_options_.info_log,
-                   "File Deletions Disabled, but already disabled. Counter: %d",
-                   disable_delete_obsolete_files_);
+    Log(InfoLogLevel::WARN_LEVEL, db_options_.info_log,
+        "File Deletions Disabled, but already disabled. Counter: %d",
+        disable_delete_obsolete_files_);
  }
  return Status::OK();
 }
@@ -57,12 +58,12 @@ Status DBImpl::EnableFileDeletions(bool force) {
      --disable_delete_obsolete_files_;
    }
    if (disable_delete_obsolete_files_ == 0)  {
-      ROCKS_LOG_INFO(immutable_db_options_.info_log, "File Deletions Enabled");
+      Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
+          "File Deletions Enabled");
      should_purge_files = true;
      FindObsoleteFiles(&job_context, true);
    } else {
-      ROCKS_LOG_WARN(
-          immutable_db_options_.info_log,
+      Log(InfoLogLevel::WARN_LEVEL, db_options_.info_log,
          "File Deletions Enable, but not really enabled. Counter: %d",
          disable_delete_obsolete_files_);
    }
@@ -71,7 +72,7 @@ Status DBImpl::EnableFileDeletions(bool force) {
    PurgeObsoleteFiles(job_context);
  }
  job_context.Clean();
-  LogFlush(immutable_db_options_.info_log);
+  LogFlush(db_options_.info_log);
  return Status::OK();
 }

@@ -82,6 +83,7 @@ int DBImpl::IsFileDeletionsEnabled() const {
 Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
                            uint64_t* manifest_file_size,
                            bool flush_memtable) {
+
  *manifest_file_size = 0;

  mutex_.Lock();
@@ -108,8 +110,8 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,

    if (!status.ok()) {
      mutex_.Unlock();
-      ROCKS_LOG_ERROR(immutable_db_options_.info_log, "Cannot Flush data %s\n",
-                      status.ToString().c_str());
+      Log(InfoLogLevel::ERROR_LEVEL, db_options_.info_log,
+          "Cannot Flush data %s\n", status.ToString().c_str());
      return status;
    }
  }
@@ -124,7 +126,7 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
  }

  ret.clear();
-  ret.reserve(live.size() + 3);  // *.sst + CURRENT + MANIFEST + OPTIONS
+  ret.reserve(live.size() + 2); //*.sst + CURRENT + MANIFEST

  // create names of the live files. The names are not absolute
  // paths, instead they are relative to dbname_;
@@ -134,7 +136,6 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,

  ret.push_back(CurrentFileName(""));
  ret.push_back(DescriptorFileName("", versions_->manifest_file_number()));
-  ret.push_back(OptionsFileName("", versions_->options_file_number()));

  // find length of manifest file while holding the mutex lock
  *manifest_file_size = versions_->manifest_file_size();
@@ -1,125 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/db_test_util.h"
-#include "port/stack_trace.h"
-#include "util/fault_injection_test_env.h"
-#include "util/sync_point.h"
-
-namespace rocksdb {
-
-class DBFlushTest : public DBTestBase {
- public:
-  DBFlushTest() : DBTestBase("/db_flush_test") {}
-};
-
-class DBFlushDirectIOTest : public DBFlushTest,
-                            public ::testing::WithParamInterface<bool> {
- public:
-  DBFlushDirectIOTest() : DBFlushTest() {}
-};
-
-// We had issue when two background threads trying to flush at the same time,
-// only one of them get committed. The test verifies the issue is fixed.
-TEST_F(DBFlushTest, FlushWhileWritingManifest) {
-  Options options;
-  options.disable_auto_compactions = true;
-  options.max_background_flushes = 2;
-  options.env = env_;
-  Reopen(options);
-  FlushOptions no_wait;
-  no_wait.wait = false;
-
-  SyncPoint::GetInstance()->LoadDependency(
-      {{"VersionSet::LogAndApply:WriteManifest",
-        "DBFlushTest::FlushWhileWritingManifest:1"},
-       {"MemTableList::InstallMemtableFlushResults:InProgress",
-        "VersionSet::LogAndApply:WriteManifestDone"}});
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  ASSERT_OK(Put("foo", "v"));
-  ASSERT_OK(dbfull()->Flush(no_wait));
-  TEST_SYNC_POINT("DBFlushTest::FlushWhileWritingManifest:1");
-  ASSERT_OK(Put("bar", "v"));
-  ASSERT_OK(dbfull()->Flush(no_wait));
-  // If the issue is hit we will wait here forever.
-  dbfull()->TEST_WaitForFlushMemTable();
-#ifndef ROCKSDB_LITE
-  ASSERT_EQ(2, TotalTableFiles());
-#endif  // ROCKSDB_LITE
-}
-
-TEST_F(DBFlushTest, SyncFail) {
-  std::unique_ptr<FaultInjectionTestEnv> fault_injection_env(
-      new FaultInjectionTestEnv(env_));
-  Options options;
-  options.disable_auto_compactions = true;
-  options.env = fault_injection_env.get();
-
-  SyncPoint::GetInstance()->LoadDependency(
-      {{"DBFlushTest::SyncFail:1", "DBImpl::SyncClosedLogs:Start"},
-       {"DBImpl::SyncClosedLogs:Failed", "DBFlushTest::SyncFail:2"}});
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  Reopen(options);
-  Put("key", "value");
-  auto* cfd =
-      reinterpret_cast<ColumnFamilyHandleImpl*>(db_->DefaultColumnFamily())
-          ->cfd();
-  int refs_before = cfd->current()->TEST_refs();
-  FlushOptions flush_options;
-  flush_options.wait = false;
-  ASSERT_OK(dbfull()->Flush(flush_options));
-  fault_injection_env->SetFilesystemActive(false);
-  TEST_SYNC_POINT("DBFlushTest::SyncFail:1");
-  TEST_SYNC_POINT("DBFlushTest::SyncFail:2");
-  fault_injection_env->SetFilesystemActive(true);
-  dbfull()->TEST_WaitForFlushMemTable();
-#ifndef ROCKSDB_LITE
-  ASSERT_EQ("", FilesPerLevel());  // flush failed.
-#endif                             // ROCKSDB_LITE
-  // Flush job should release ref count to current version.
-  ASSERT_EQ(refs_before, cfd->current()->TEST_refs());
-  Destroy(options);
-}
-
-TEST_P(DBFlushDirectIOTest, DirectIO) {
-  Options options;
-  options.create_if_missing = true;
-  options.disable_auto_compactions = true;
-  options.max_background_flushes = 2;
-  options.use_direct_io_for_flush_and_compaction = GetParam();
-  options.env = new MockEnv(Env::Default());
-  SyncPoint::GetInstance()->SetCallBack(
-      "BuildTable:create_file", [&](void* arg) {
-        bool* use_direct_writes = static_cast<bool*>(arg);
-        ASSERT_EQ(*use_direct_writes,
-                  options.use_direct_io_for_flush_and_compaction);
-      });
-
-  SyncPoint::GetInstance()->EnableProcessing();
-  Reopen(options);
-  ASSERT_OK(Put("foo", "v"));
-  FlushOptions flush_options;
-  flush_options.wait = true;
-  ASSERT_OK(dbfull()->Flush(flush_options));
-  Destroy(options);
-  delete options.env;
-}
-
-INSTANTIATE_TEST_CASE_P(DBFlushDirectIOTest, DBFlushDirectIOTest,
-                        testing::Bool());
-
-}  // namespace rocksdb
-
-int main(int argc, char** argv) {
-  rocksdb::port::InstallStackTraceHandler();
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}
@@ -10,11 +10,8 @@

 #include <atomic>
 #include <deque>
-#include <functional>
 #include <limits>
 #include <list>
-#include <map>
-#include <queue>
 #include <set>
 #include <string>
 #include <utility>
@@ -23,7 +20,6 @@
 #include "db/column_family.h"
 #include "db/compaction_job.h"
 #include "db/dbformat.h"
-#include "db/external_sst_file_ingestion_job.h"
 #include "db/flush_job.h"
 #include "db/flush_scheduler.h"
 #include "db/internal_stats.h"
@@ -33,20 +29,18 @@
 #include "db/wal_manager.h"
 #include "db/write_controller.h"
 #include "db/write_thread.h"
+#include "db/writebuffer.h"
 #include "memtable_list.h"
-#include "monitoring/instrumented_mutex.h"
-#include "options/db_options.h"
 #include "port/port.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "rocksdb/memtablerep.h"
-#include "rocksdb/status.h"
 #include "rocksdb/transaction_log.h"
-#include "rocksdb/write_buffer_manager.h"
 #include "table/scoped_arena_iterator.h"
 #include "util/autovector.h"
 #include "util/event_logger.h"
 #include "util/hash.h"
+#include "util/instrumented_mutex.h"
 #include "util/stop_watch.h"
 #include "util/thread_local.h"

@@ -61,7 +55,6 @@ class Arena;
 class WriteCallback;
 struct JobContext;
 struct ExternalSstFileInfo;
-struct MemTableInfo;

 class DBImpl : public DB {
 public:
@@ -92,7 +85,7 @@ class DBImpl : public DB {
  using DB::Get;
  virtual Status Get(const ReadOptions& options,
                     ColumnFamilyHandle* column_family, const Slice& key,
-                     PinnableSlice* value) override;
+                     std::string* value) override;
  using DB::MultiGet;
  virtual std::vector<Status> MultiGet(
      const ReadOptions& options,
@@ -100,19 +93,10 @@ class DBImpl : public DB {
      const std::vector<Slice>& keys,
      std::vector<std::string>* values) override;

-  virtual Status CreateColumnFamily(const ColumnFamilyOptions& cf_options,
+  virtual Status CreateColumnFamily(const ColumnFamilyOptions& options,
                                    const std::string& column_family,
                                    ColumnFamilyHandle** handle) override;
-  virtual Status CreateColumnFamilies(
-      const ColumnFamilyOptions& cf_options,
-      const std::vector<std::string>& column_family_names,
-      std::vector<ColumnFamilyHandle*>* handles) override;
-  virtual Status CreateColumnFamilies(
-      const std::vector<ColumnFamilyDescriptor>& column_families,
-      std::vector<ColumnFamilyHandle*>* handles) override;
  virtual Status DropColumnFamily(ColumnFamilyHandle* column_family) override;
-  virtual Status DropColumnFamilies(
-      const std::vector<ColumnFamilyHandle*>& column_families) override;

  // Returns false if key doesn't exist in the database and true if it may.
  // If value_found is not passed in as null, then return the value if found in
@@ -135,10 +119,6 @@ class DBImpl : public DB {
  using DB::GetProperty;
  virtual bool GetProperty(ColumnFamilyHandle* column_family,
                           const Slice& property, std::string* value) override;
-  using DB::GetMapProperty;
-  virtual bool GetMapProperty(ColumnFamilyHandle* column_family,
-                              const Slice& property,
-                              std::map<std::string, double>* value) override;
  using DB::GetIntProperty;
  virtual bool GetIntProperty(ColumnFamilyHandle* column_family,
                              const Slice& property, uint64_t* value) override;
@@ -148,13 +128,7 @@ class DBImpl : public DB {
  using DB::GetApproximateSizes;
  virtual void GetApproximateSizes(ColumnFamilyHandle* column_family,
                                   const Range* range, int n, uint64_t* sizes,
-                                   uint8_t include_flags
-                                   = INCLUDE_FILES) override;
-  using DB::GetApproximateMemTableStats;
-  virtual void GetApproximateMemTableStats(ColumnFamilyHandle* column_family,
-                                           const Range& range,
-                                           uint64_t* const count,
-                                           uint64_t* const size) override;
+                                   bool include_memtable = false) override;
  using DB::CompactRange;
  virtual Status CompactRange(const CompactRangeOptions& options,
                              ColumnFamilyHandle* column_family,
@@ -178,9 +152,6 @@ class DBImpl : public DB {
      ColumnFamilyHandle* column_family,
      const std::unordered_map<std::string, std::string>& options_map) override;

-  virtual Status SetDBOptions(
-      const std::unordered_map<std::string, std::string>& options_map) override;
-
  using DB::NumberLevels;
  virtual int NumberLevels(ColumnFamilyHandle* column_family) override;
  using DB::MaxMemCompactionLevel;
@@ -191,9 +162,10 @@ class DBImpl : public DB {
  virtual const std::string& GetName() const override;
  virtual Env* GetEnv() const override;
  using DB::GetOptions;
-  virtual Options GetOptions(ColumnFamilyHandle* column_family) const override;
+  virtual const Options& GetOptions(
+      ColumnFamilyHandle* column_family) const override;
  using DB::GetDBOptions;
-  virtual DBOptions GetDBOptions() const override;
+  virtual const DBOptions& GetDBOptions() const override;
  using DB::Flush;
  virtual Status Flush(const FlushOptions& options,
                       ColumnFamilyHandle* column_family) override;
@@ -201,11 +173,7 @@ class DBImpl : public DB {

  virtual SequenceNumber GetLatestSequenceNumber() const override;

-  bool HasActiveSnapshotLaterThanSN(SequenceNumber sn);
-
 #ifndef ROCKSDB_LITE
-  using DB::ResetStats;
-  virtual Status ResetStats() override;
  virtual Status DisableFileDeletions() override;
  virtual Status EnableFileDeletions(bool force) override;
  virtual int IsFileDeletionsEnabled() const;
@@ -281,16 +249,16 @@ class DBImpl : public DB {
  //
  // Returns OK or NotFound on success,
  // other status on unexpected error.
-  // TODO(andrewkr): this API need to be aware of range deletion operations
  Status GetLatestSequenceForKey(SuperVersion* sv, const Slice& key,
                                 bool cache_only, SequenceNumber* seq,
                                 bool* found_record_for_key);

-  using DB::IngestExternalFile;
-  virtual Status IngestExternalFile(
-      ColumnFamilyHandle* column_family,
-      const std::vector<std::string>& external_files,
-      const IngestExternalFileOptions& ingestion_options) override;
+  using DB::AddFile;
+  virtual Status AddFile(ColumnFamilyHandle* column_family,
+                         const ExternalSstFileInfo* file_info,
+                         bool move_file) override;
+  virtual Status AddFile(ColumnFamilyHandle* column_family,
+                         const std::string& file_path, bool move_file) override;

 #endif  // ROCKSDB_LITE

@@ -316,8 +284,7 @@ class DBImpl : public DB {
  // The keys of this iterator are internal keys (see format.h).
  // The returned iterator should be deleted when no longer needed.
  InternalIterator* NewInternalIterator(
-      Arena* arena, RangeDelAggregator* range_del_agg,
-      ColumnFamilyHandle* column_family = nullptr);
+      Arena* arena, ColumnFamilyHandle* column_family = nullptr);

 #ifndef NDEBUG
  // Extra methods (for testing) that are not in the public DB interface
@@ -328,19 +295,8 @@ class DBImpl : public DB {
                           ColumnFamilyHandle* column_family = nullptr,
                           bool disallow_trivial_move = false);

-  void TEST_HandleWALFull();
-
-  bool TEST_UnableToFlushOldestLog() {
-    return unable_to_flush_oldest_log_;
-  }
-
-  bool TEST_IsLogGettingFlushed() {
-    return alive_log_files_.begin()->getting_flushed;
-  }
-
  // Force current memtable contents to be flushed.
-  Status TEST_FlushMemTable(bool wait = true,
-                            ColumnFamilyHandle* cfh = nullptr);
+  Status TEST_FlushMemTable(bool wait = true);

  // Wait for memtable compaction
  Status TEST_WaitForFlushMemTable(ColumnFamilyHandle* column_family = nullptr);
@@ -381,37 +337,20 @@ class DBImpl : public DB {

  uint64_t TEST_LogfileNumber();

-  uint64_t TEST_total_log_size() const { return total_log_size_; }
-
  // Returns column family name to ImmutableCFOptions map.
  Status TEST_GetAllImmutableCFOptions(
      std::unordered_map<std::string, const ImmutableCFOptions*>* iopts_map);

-  // Return the lastest MutableCFOptions of of a column family
-  Status TEST_GetLatestMutableCFOptions(ColumnFamilyHandle* column_family,
-                                        MutableCFOptions* mutable_cf_opitons);
-
  Cache* TEST_table_cache() { return table_cache_.get(); }

  WriteController& TEST_write_controler() { return write_controller_; }

-  uint64_t TEST_FindMinLogContainingOutstandingPrep();
-  uint64_t TEST_FindMinPrepLogReferencedByMemTable();
-
-  int TEST_BGCompactionsAllowed() const;
-
 #endif  // NDEBUG

-  // Return maximum background compaction allowed to be scheduled based on
+  // Return maximum background compaction alowed to be scheduled based on
  // compaction status.
  int BGCompactionsAllowed() const;

-  // move logs pending closing from job_context to the DB queue and
-  // schedule a purge
-  void ScheduleBgLogWriterClose(JobContext* job_context);
-
-  uint64_t MinLogNumberToKeep();
-
  // Returns the list of live files in 'live' and the list
  // of all files in the filesystem in 'candidate_files'.
  // If force == false and the last call was less than
@@ -424,19 +363,12 @@ class DBImpl : public DB {
  // belong to live files are posibly removed. Also, removes all the
  // files in sst_delete_files and log_delete_files.
  // It is not necessary to hold the mutex when invoking this method.
-  void PurgeObsoleteFiles(const JobContext& background_contet,
-                          bool schedule_only = false);
-
-  void SchedulePurge();
+  void PurgeObsoleteFiles(const JobContext& background_contet);

  ColumnFamilyHandle* DefaultColumnFamily() const override;

  const SnapshotList& snapshots() const { return snapshots_; }

-  const ImmutableDBOptions& immutable_db_options() const {
-    return immutable_db_options_;
-  }
-
  void CancelAllBackgroundWork(bool wait);

  // Find Super version and reference it. Based on options, it might return
@@ -450,6 +382,9 @@ class DBImpl : public DB {
  // mutex is held.
  SuperVersion* GetAndRefSuperVersion(uint32_t column_family_id);

+  // Same as above, should called without mutex held and not on write thread.
+  SuperVersion* GetAndRefSuperVersionUnlocked(uint32_t column_family_id);
+
  // Un-reference the super version and return it to thread local cache if
  // needed. If it is the last reference of the super version. Clean it up
  // after un-referencing it.
@@ -460,6 +395,10 @@ class DBImpl : public DB {
  // REQUIRED: this function should only be called on the write thread.
  void ReturnAndCleanupSuperVersion(uint32_t colun_family_id, SuperVersion* sv);

+  // Same as above, should called without mutex held and not on write thread.
+  void ReturnAndCleanupSuperVersionUnlocked(uint32_t colun_family_id,
+                                            SuperVersion* sv);
+
  // REQUIRED: this function should only be called on the write thread or if the
  // mutex is held.  Return value only valid until next call to this function or
  // mutex is released.
@@ -482,91 +421,22 @@ class DBImpl : public DB {
    return num_running_compactions_;
  }

-  const WriteController& write_controller() { return write_controller_; }
-
-  // hollow transactions shell used for recovery.
-  // these will then be passed to TransactionDB so that
-  // locks can be reacquired before writing can resume.
-  struct RecoveredTransaction {
-    uint64_t log_number_;
-    std::string name_;
-    WriteBatch* batch_;
-    explicit RecoveredTransaction(const uint64_t log, const std::string& name,
-                                  WriteBatch* batch)
-        : log_number_(log), name_(name), batch_(batch) {}
-
-    ~RecoveredTransaction() { delete batch_; }
-  };
-
-  bool allow_2pc() const { return immutable_db_options_.allow_2pc; }
-
-  std::unordered_map<std::string, RecoveredTransaction*>
-  recovered_transactions() {
-    return recovered_transactions_;
-  }
-
-  RecoveredTransaction* GetRecoveredTransaction(const std::string& name) {
-    auto it = recovered_transactions_.find(name);
-    if (it == recovered_transactions_.end()) {
-      return nullptr;
-    } else {
-      return it->second;
-    }
-  }
-
-  void InsertRecoveredTransaction(const uint64_t log, const std::string& name,
-                                  WriteBatch* batch) {
-    recovered_transactions_[name] = new RecoveredTransaction(log, name, batch);
-    MarkLogAsContainingPrepSection(log);
-  }
-
-  void DeleteRecoveredTransaction(const std::string& name) {
-    auto it = recovered_transactions_.find(name);
-    assert(it != recovered_transactions_.end());
-    auto* trx = it->second;
-    recovered_transactions_.erase(it);
-    MarkLogAsHavingPrepSectionFlushed(trx->log_number_);
-    delete trx;
-  }
-
-  void DeleteAllRecoveredTransactions() {
-    for (auto it = recovered_transactions_.begin();
-         it != recovered_transactions_.end(); it++) {
-      delete it->second;
-    }
-    recovered_transactions_.clear();
-  }
-
-  void MarkLogAsHavingPrepSectionFlushed(uint64_t log);
-  void MarkLogAsContainingPrepSection(uint64_t log);
-  void AddToLogsToFreeQueue(log::Writer* log_writer) {
-    logs_to_free_queue_.push_back(log_writer);
-  }
-
-  Status NewDB();
-
 protected:
  Env* const env_;
  const std::string dbname_;
  unique_ptr<VersionSet> versions_;
-  const DBOptions initial_db_options_;
-  const ImmutableDBOptions immutable_db_options_;
-  MutableDBOptions mutable_db_options_;
+  const DBOptions db_options_;
  Statistics* stats_;
-  std::unordered_map<std::string, RecoveredTransaction*>
-      recovered_transactions_;

  InternalIterator* NewInternalIterator(const ReadOptions&,
                                        ColumnFamilyData* cfd,
                                        SuperVersion* super_version,
-                                        Arena* arena,
-                                        RangeDelAggregator* range_del_agg);
+                                        Arena* arena);

  // Except in DB::Open(), WriteOptionsFile can only be called when:
-  // Persist options to options file.
-  // If need_mutex_lock = false, the method will lock DB mutex.
-  // If need_enter_write_thread = false, the method will enter write thread.
-  Status WriteOptionsFile(bool need_mutex_lock, bool need_enter_write_thread);
+  // 1. WriteThread::Writer::EnterUnbatched() is used.
+  // 2. db_mutex is held
+  Status WriteOptionsFile();

  // The following two functions can only be called when:
  // 1. WriteThread::Writer::EnterUnbatched() is used.
@@ -574,10 +444,6 @@ class DBImpl : public DB {
  Status RenameTempFileToOptionsFile(const std::string& file_name);
  Status DeleteObsoleteOptionsFiles();

-  void NotifyOnFlushBegin(ColumnFamilyData* cfd, FileMetaData* file_meta,
-                          const MutableCFOptions& mutable_cf_options,
-                          int job_id, TableProperties prop);
-
  void NotifyOnFlushCompleted(ColumnFamilyData* cfd, FileMetaData* file_meta,
                              const MutableCFOptions& mutable_cf_options,
                              int job_id, TableProperties prop);
@@ -586,13 +452,6 @@ class DBImpl : public DB {
                                   Compaction *c, const Status &st,
                                   const CompactionJobStats& job_stats,
                                   int job_id);
-  void NotifyOnMemTableSealed(ColumnFamilyData* cfd,
-                              const MemTableInfo& mem_table_info);
-
-#ifndef ROCKSDB_LITE
-  void NotifyOnExternalFileIngested(
-      ColumnFamilyData* cfd, const ExternalSstFileIngestionJob& ingestion_job);
-#endif  // !ROCKSDB_LITE

  void NewThreadStatusCfInfo(ColumnFamilyData* cfd) const;

@@ -601,17 +460,11 @@ class DBImpl : public DB {
  void EraseThreadStatusDbInfo() const;

  Status WriteImpl(const WriteOptions& options, WriteBatch* updates,
-                   WriteCallback* callback = nullptr,
-                   uint64_t* log_used = nullptr, uint64_t log_ref = 0,
-                   bool disable_memtable = false);
-
-  uint64_t FindMinLogContainingOutstandingPrep();
-  uint64_t FindMinPrepLogReferencedByMemTable();
+                   WriteCallback* callback);

 private:
  friend class DB;
  friend class InternalStats;
-  friend class TransactionImpl;
 #ifndef ROCKSDB_LITE
  friend class ForwardIterator;
 #endif
@@ -622,21 +475,9 @@ class DBImpl : public DB {
 #endif
  struct CompactionState;

-  struct WriteContext {
-    autovector<SuperVersion*> superversions_to_free_;
-    autovector<MemTable*> memtables_to_free_;
+  struct WriteContext;

-    ~WriteContext() {
-      for (auto& sv : superversions_to_free_) {
-        delete sv;
-      }
-      for (auto& m : memtables_to_free_) {
-        delete m;
-      }
-    }
-  };
-
-  struct PurgeFileInfo;
+  Status NewDB();

  // Recover the descriptor from persistent storage.  May do a significant
  // amount of work to recover recently logged updates.  Any changes to
@@ -649,18 +490,8 @@ class DBImpl : public DB {

  const Status CreateArchivalDirectory();

-  Status CreateColumnFamilyImpl(const ColumnFamilyOptions& cf_options,
-                                const std::string& cf_name,
-                                ColumnFamilyHandle** handle);
-
-  Status DropColumnFamilyImpl(ColumnFamilyHandle* column_family);
-
  // Delete any unneeded files and stale in-memory entries.
  void DeleteObsoleteFiles();
-  // Delete obsolete files and log status and information of file deletion
-  void DeleteObsoleteFileImpl(Status file_deletion_status, int job_id,
-                              const std::string& fname, FileType type,
-                              uint64_t number, uint32_t path_id);

  // Background process needs to call
  //     auto x = CaptureCurrentFileNumberInPendingOutputs()
@@ -681,8 +512,6 @@ class DBImpl : public DB {
  // and blocked by any other pending_outputs_ calls)
  void ReleaseFileNumberFromPendingOutputs(std::list<uint64_t>::iterator v);

-  Status SyncClosedLogs(JobContext* job_context);
-
  // Flush the in-memory write buffer to storage.  Switches to a new
  // log-file/memtable and writes a new descriptor iff successful.
  Status FlushMemTableToOutputFile(ColumnFamilyData* cfd,
@@ -692,7 +521,7 @@ class DBImpl : public DB {

  // REQUIRES: log_numbers are sorted in ascending order
  Status RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
-                         SequenceNumber* next_sequence, bool read_only);
+                         SequenceNumber* max_sequence, bool read_only);

  // The following two methods are used to flush a memtable to
  // storage. The first one is used at database RecoveryTime (when the
@@ -704,49 +533,24 @@ class DBImpl : public DB {

  // num_bytes: for slowdown case, delay time is calculated based on
  //            `num_bytes` going through.
-  Status DelayWrite(uint64_t num_bytes, const WriteOptions& write_options);
+  Status DelayWrite(uint64_t num_bytes);

  Status ScheduleFlushes(WriteContext* context);

  Status SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context);

  // Force current memtable contents to be flushed.
-  Status FlushMemTable(ColumnFamilyData* cfd, const FlushOptions& options,
-                       bool writes_stopped = false);
+  Status FlushMemTable(ColumnFamilyData* cfd, const FlushOptions& options);

  // Wait for memtable flushed
  Status WaitForFlushMemTable(ColumnFamilyData* cfd);

-  // REQUIRES: mutex locked
-  Status HandleWALFull(WriteContext* write_context);
-
-  // REQUIRES: mutex locked
-  Status HandleWriteBufferFull(WriteContext* write_context);
-
-  // REQUIRES: mutex locked
-  Status PreprocessWrite(const WriteOptions& write_options, bool need_log_sync,
-                         bool* logs_getting_syned, WriteContext* write_context);
-
-  Status WriteToWAL(const autovector<WriteThread::Writer*>& write_group,
-                    log::Writer* log_writer, bool need_log_sync,
-                    bool need_log_dir_sync, SequenceNumber sequence);
-
 #ifndef ROCKSDB_LITE
-
-  Status CompactFilesImpl(const CompactionOptions& compact_options,
-                          ColumnFamilyData* cfd, Version* version,
-                          const std::vector<std::string>& input_file_names,
-                          const int output_level, int output_path_id,
-                          JobContext* job_context, LogBuffer* log_buffer);
-
-  // Wait for current IngestExternalFile() calls to finish.
-  // REQUIRES: mutex_ held
-  void WaitForIngestFile();
-
-#else
-  // IngestExternalFile is not supported in ROCKSDB_LITE so this function
-  // will be no-op
-  void WaitForIngestFile() {}
+  Status CompactFilesImpl(
+      const CompactionOptions& compact_options, ColumnFamilyData* cfd,
+      Version* version, const std::vector<std::string>& input_file_names,
+      const int output_level, int output_path_id, JobContext* job_context,
+      LogBuffer* log_buffer);
 #endif  // ROCKSDB_LITE

  ColumnFamilyData* GetColumnFamilyDataByName(const std::string& cf_name);
@@ -754,15 +558,11 @@ class DBImpl : public DB {
  void MaybeScheduleFlushOrCompaction();
  void SchedulePendingFlush(ColumnFamilyData* cfd);
  void SchedulePendingCompaction(ColumnFamilyData* cfd);
-  void SchedulePendingPurge(std::string fname, FileType type, uint64_t number,
-                            uint32_t path_id, int job_id);
  static void BGWorkCompaction(void* arg);
  static void BGWorkFlush(void* db);
-  static void BGWorkPurge(void* arg);
  static void UnscheduleCallback(void* arg);
  void BackgroundCallCompaction(void* arg);
  void BackgroundCallFlush();
-  void BackgroundCallPurge();
  Status BackgroundCompaction(bool* madeProgress, JobContext* job_context,
                              LogBuffer* log_buffer, void* m = 0);
  Status BackgroundFlush(bool* madeProgress, JobContext* job_context,
@@ -794,8 +594,6 @@ class DBImpl : public DB {

  const Snapshot* GetSnapshotImpl(bool is_write_conflict_boundary);

-  uint64_t GetMaxTotalWalSize() const;
-
  // table_cache_ provides its own synchronization
  std::shared_ptr<Cache> table_cache_;

@@ -807,7 +605,7 @@ class DBImpl : public DB {
  //       same time.
  InstrumentedMutex options_files_mutex_;
  // State below is protected by mutex_
-  mutable InstrumentedMutex mutex_;
+  InstrumentedMutex mutex_;

  std::atomic<bool> shutting_down_;
  // This condition variable is signaled on these conditions:
@@ -815,10 +613,9 @@ class DBImpl : public DB {
  // * if AnyManualCompaction, whenever a compaction finishes, even if it hasn't
  // made any progress
  // * whenever a compaction made any progress
-  // * whenever bg_flush_scheduled_ or bg_purge_scheduled_ value decreases
-  // (i.e. whenever a flush is done, even if it didn't make any progress)
-  // * whenever there is an error in background purge, flush or compaction
-  // * whenever num_running_ingest_file_ goes to 0.
+  // * whenever bg_flush_scheduled_ value decreases (i.e. whenever a flush is
+  // done, even if it didn't make any progress)
+  // * whenever there is an error in background flush or compaction
  InstrumentedCondVar bg_cv_;
  uint64_t logfile_number_;
  std::deque<uint64_t>
@@ -869,7 +666,7 @@ class DBImpl : public DB {
  std::deque<LogWriterNumber> logs_;
  // Signaled when getting_synced becomes false for some of the logs_.
  InstrumentedCondVar log_sync_cv_;
-  std::atomic<uint64_t> total_log_size_;
+  uint64_t total_log_size_;
  // only used for dynamically adjusting max_total_wal_size. it is a sum of
  // [write_buffer_size * max_write_buffer_number] over all column families
  uint64_t max_total_in_memory_state_;
@@ -911,7 +708,7 @@ class DBImpl : public DB {

  Directories directories_;

-  WriteBufferManager* write_buffer_manager_;
+  WriteBuffer write_buffer_;

  WriteThread write_thread_;

@@ -938,19 +735,6 @@ class DBImpl : public DB {
  // State is protected with db mutex.
  std::list<uint64_t> pending_outputs_;

-  // PurgeFileInfo is a structure to hold information of files to be deleted in
-  // purge_queue_
-  struct PurgeFileInfo {
-    std::string fname;
-    FileType type;
-    uint64_t number;
-    uint32_t path_id;
-    int job_id;
-    PurgeFileInfo(std::string fn, FileType t, uint64_t num, uint32_t pid,
-                  int jid)
-        : fname(fn), type(t), number(num), path_id(pid), job_id(jid) {}
-  };
-
  // flush_queue_ and compaction_queue_ hold column families that we need to
  // flush and compact, respectively.
  // A column family is inserted into flush_queue_ when it satisfies condition
@@ -975,12 +759,6 @@ class DBImpl : public DB {
  // invariant(column family present in compaction_queue_ <==>
  // ColumnFamilyData::pending_compaction_ == true)
  std::deque<ColumnFamilyData*> compaction_queue_;
-
-  // A queue to store filenames of the files to be purged
-  std::deque<PurgeFileInfo> purge_queue_;
-
-  // A queue to store log writers to close
-  std::deque<log::Writer*> logs_to_free_queue_;
  int unscheduled_flushes_;
  int unscheduled_compactions_;

@@ -996,9 +774,6 @@ class DBImpl : public DB {
  // stores the number of flushes are currently running
  int num_running_flushes_;

-  // number of background obsolete file purge jobs, submitted to the HIGH pool
-  int bg_purge_scheduled_;
-
  // Information for a manual compaction
  struct ManualCompaction {
    ColumnFamilyData* cfd;
@@ -1036,9 +811,8 @@ class DBImpl : public DB {
  // without any synchronization
  int disable_delete_obsolete_files_;

-  // last time when DeleteObsoleteFiles with full scan was executed. Originaly
-  // initialized with startup time.
-  uint64_t delete_obsolete_files_last_run_;
+  // next time when we should run DeleteObsoleteFiles with full scan
+  uint64_t delete_obsolete_files_next_run_;

  // last time stats were dumped to LOG
  std::atomic<uint64_t> last_stats_dump_time_microsec_;
@@ -1050,15 +824,7 @@ class DBImpl : public DB {
  // A flag indicating whether the current rocksdb database has any
  // data that is not yet persisted into either WAL or SST file.
  // Used when disableWAL is true.
-  std::atomic<bool> has_unpersisted_data_;
-
-  // if an attempt was made to flush all column families that
-  // the oldest log depends on but uncommited data in the oldest
-  // log prevents the log from being released.
-  // We must attempt to free the dependent memtables again
-  // at a later time after the transaction in the oldest
-  // log is fully commited.
-  bool unable_to_flush_oldest_log_;
+  bool has_unpersisted_data_;

  static const int KEEP_LOG_FILE_NUM = 1000;
  // MSVC version 1800 still does not have constexpr for ::max()
@@ -1069,10 +835,6 @@ class DBImpl : public DB {
  // The options to access storage files
  const EnvOptions env_options_;

-  // Number of running IngestExternalFile() calls.
-  // REQUIRES: mutex held
-  int num_running_ingest_file_;
-
 #ifndef ROCKSDB_LITE
  WalManager wal_manager_;
 #endif  // ROCKSDB_LITE
@@ -1092,32 +854,17 @@ class DBImpl : public DB {
  // Indicate DB was opened successfully
  bool opened_successfully_;

-  // minmum log number still containing prepared data.
-  // this is used by FindObsoleteFiles to determine which
-  // flushed logs we must keep around because they still
-  // contain prepared data which has not been flushed or rolled back
-  std::priority_queue<uint64_t, std::vector<uint64_t>, std::greater<uint64_t>>
-      min_log_with_prep_;
-
-  // to be used in conjunction with min_log_with_prep_.
-  // once a transaction with data in log L is committed or rolled back
-  // rather than removing the value from the heap we add that value
-  // to prepared_section_completed_ which maps LOG -> instance_count
-  // since a log could contain multiple prepared sections
-  //
-  // when trying to determine the minmum log still active we first
-  // consult min_log_with_prep_. while that root value maps to
-  // a value > 0 in prepared_section_completed_ we decrement the
-  // instance_count for that log and pop the root value in
-  // min_log_with_prep_. This will work the same as a min_heap
-  // where we are deleteing arbitrary elements and the up heaping.
-  std::unordered_map<uint64_t, uint64_t> prepared_section_completed_;
-  std::mutex prep_heap_mutex_;
-
  // No copying allowed
  DBImpl(const DBImpl&);
  void operator=(const DBImpl&);

+  // Return the earliest snapshot where seqno is visible.
+  // Store the snapshot right before that, if any, in prev_snapshot
+  inline SequenceNumber findEarliestVisibleSnapshot(
+    SequenceNumber in,
+    std::vector<SequenceNumber>& snapshots,
+    SequenceNumber* prev_snapshot);
+
  // Background threads call this function, which is just a wrapper around
  // the InstallSuperVersion() function. Background threads carry
  // job_context which can have new_superversion already
@@ -1147,7 +894,7 @@ class DBImpl : public DB {
  // Function that Get and KeyMayExist call with no_io true or false
  // Note: 'value_found' from KeyMayExist propagates here
  Status GetImpl(const ReadOptions& options, ColumnFamilyHandle* column_family,
-                 const Slice& key, PinnableSlice* value,
+                 const Slice& key, std::string* value,
                 bool* value_found = nullptr);

  bool GetIntPropertyInternal(ColumnFamilyData* cfd,
@@ -1161,19 +908,15 @@ class DBImpl : public DB {
  bool ShouldntRunManualCompaction(ManualCompaction* m);
  bool HaveManualCompaction(ColumnFamilyData* cfd);
  bool MCOverlap(ManualCompaction* m, ManualCompaction* m1);
-
-  size_t GetWalPreallocateBlockSize(uint64_t write_buffer_size) const;
 };

+// Sanitize db options.  The caller should delete result.info_log if
+// it is not equal to src.info_log.
 extern Options SanitizeOptions(const std::string& db,
+                               const InternalKeyComparator* icmp,
                               const Options& src);
-
 extern DBOptions SanitizeOptions(const std::string& db, const DBOptions& src);

-extern CompressionType GetCompressionFlush(
-    const ImmutableCFOptions& ioptions,
-    const MutableCFOptions& mutable_cf_options);
- 
 // Fix user-supplied options to be reasonable
 template <class T, class V>
 static void ClipToRange(T* ptr, V minvalue, V maxvalue) {
@@ -10,7 +10,7 @@
 #ifndef NDEBUG

 #include "db/db_impl.h"
-#include "monitoring/thread_status_updater.h"
+#include "util/thread_status_updater.h"

 namespace rocksdb {

@@ -19,12 +19,6 @@ uint64_t DBImpl::TEST_GetLevel0TotalSize() {
  return default_cf_handle_->cfd()->current()->storage_info()->NumLevelBytes(0);
 }

-void DBImpl::TEST_HandleWALFull() {
-  WriteContext write_context;
-  InstrumentedMutexLock l(&mutex_);
-  HandleWALFull(&write_context);
-}
-
 int64_t DBImpl::TEST_MaxNextLevelOverlappingBytes(
    ColumnFamilyHandle* column_family) {
  ColumnFamilyData* cfd;
@@ -80,17 +74,10 @@ Status DBImpl::TEST_CompactRange(int level, const Slice* begin,
                             disallow_trivial_move);
 }

-Status DBImpl::TEST_FlushMemTable(bool wait, ColumnFamilyHandle* cfh) {
+Status DBImpl::TEST_FlushMemTable(bool wait) {
  FlushOptions fo;
  fo.wait = wait;
-  ColumnFamilyData* cfd;
-  if (cfh == nullptr) {
-    cfd = default_cf_handle_->cfd();
-  } else {
-    auto cfhi = reinterpret_cast<ColumnFamilyHandleImpl*>(cfh);
-    cfd = cfhi->cfd();
-  }
-  return FlushMemTable(cfd, fo);
+  return FlushMemTable(default_cf_handle_->cfd(), fo);
 }

 Status DBImpl::TEST_WaitForFlushMemTable(ColumnFamilyHandle* column_family) {
@@ -167,27 +154,5 @@ Status DBImpl::TEST_GetAllImmutableCFOptions(
  return Status::OK();
 }

-uint64_t DBImpl::TEST_FindMinLogContainingOutstandingPrep() {
-  return FindMinLogContainingOutstandingPrep();
-}
-
-uint64_t DBImpl::TEST_FindMinPrepLogReferencedByMemTable() {
-  return FindMinPrepLogReferencedByMemTable();
-}
-
-Status DBImpl::TEST_GetLatestMutableCFOptions(
-    ColumnFamilyHandle* column_family, MutableCFOptions* mutable_cf_options) {
-  InstrumentedMutexLock l(&mutex_);
-
-  auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
-  *mutable_cf_options = *cfh->cfd()->GetLatestMutableCFOptions();
-  return Status::OK();
-}
-
-int DBImpl::TEST_BGCompactionsAllowed() const {
-  InstrumentedMutexLock l(&mutex_);
-  return BGCompactionsAllowed();
-}
-
 }  // namespace rocksdb
 #endif  // NDEBUG
@@ -49,8 +49,8 @@ Status DBImpl::SuggestCompactRange(ColumnFamilyHandle* column_family,
    }
    // Since we have some more files to compact, we should also recompute
    // compaction score
-    vstorage->ComputeCompactionScore(*cfd->ioptions(),
-                                     *cfd->GetLatestMutableCFOptions());
+    vstorage->ComputeCompactionScore(*cfd->GetLatestMutableCFOptions(),
+                                     CompactionOptionsFIFO());
    SchedulePendingCompaction(cfd);
    MaybeScheduleFlushOrCompaction();
  }
@@ -61,8 +61,8 @@ Status DBImpl::PromoteL0(ColumnFamilyHandle* column_family, int target_level) {
  assert(column_family);

  if (target_level < 1) {
-    ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                   "PromoteL0 FAILED. Invalid target level %d\n", target_level);
+    Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
+        "PromoteL0 FAILED. Invalid target level %d\n", target_level);
    return Status::InvalidArgument("Invalid target level");
  }

@@ -75,9 +75,8 @@ Status DBImpl::PromoteL0(ColumnFamilyHandle* column_family, int target_level) {
    const auto* vstorage = cfd->current()->storage_info();

    if (target_level >= vstorage->num_levels()) {
-      ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                     "PromoteL0 FAILED. Target level %d does not exist\n",
-                     target_level);
+      Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
+          "PromoteL0 FAILED. Target level %d does not exist\n", target_level);
      job_context.Clean();
      return Status::InvalidArgument("Target level does not exist");
    }
@@ -95,9 +94,9 @@ Status DBImpl::PromoteL0(ColumnFamilyHandle* column_family, int target_level) {
    for (size_t i = 0; i < l0_files.size(); ++i) {
      auto f = l0_files[i];
      if (f->being_compacted) {
-        ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                       "PromoteL0 FAILED. File %" PRIu64 " being compacted\n",
-                       f->fd.GetNumber());
+        Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
+            "PromoteL0 FAILED. File %" PRIu64 " being compacted\n",
+            f->fd.GetNumber());
        job_context.Clean();
        return Status::InvalidArgument("PromoteL0 called during L0 compaction");
      }
@@ -105,10 +104,10 @@ Status DBImpl::PromoteL0(ColumnFamilyHandle* column_family, int target_level) {
      if (i == 0) continue;
      auto prev_f = l0_files[i - 1];
      if (icmp->Compare(prev_f->largest, f->smallest) >= 0) {
-        ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                       "PromoteL0 FAILED. Files %" PRIu64 " and %" PRIu64
-                       " have overlapping ranges\n",
-                       prev_f->fd.GetNumber(), f->fd.GetNumber());
+        Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
+            "PromoteL0 FAILED. Files %" PRIu64 " and %" PRIu64
+            " have overlapping ranges\n",
+            prev_f->fd.GetNumber(), f->fd.GetNumber());
        job_context.Clean();
        return Status::InvalidArgument("L0 has overlapping files");
      }
@@ -117,8 +116,8 @@ Status DBImpl::PromoteL0(ColumnFamilyHandle* column_family, int target_level) {
    // Check that all levels up to target_level are empty.
    for (int level = 1; level <= target_level; ++level) {
      if (vstorage->NumLevelFiles(level) > 0) {
-        ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                       "PromoteL0 FAILED. Level %d not empty\n", level);
+        Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
+            "PromoteL0 FAILED. Level %d not empty\n", level);
        job_context.Clean();
        return Status::InvalidArgument(
            "All levels up to target_level "
@@ -142,7 +141,7 @@ Status DBImpl::PromoteL0(ColumnFamilyHandle* column_family, int target_level) {
          cfd, &job_context, *cfd->GetLatestMutableCFOptions());
    }
  }  // lock released here
-  LogFlush(immutable_db_options_.info_log);
+  LogFlush(db_options_.info_log);
  job_context.Clean();

  return status;
@@ -1,536 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-#include "db/db_impl.h"
-
-#ifndef __STDC_FORMAT_MACROS
-#define __STDC_FORMAT_MACROS
-#endif
-#include <inttypes.h>
-#include "db/event_helpers.h"
-#include "util/file_util.h"
-#include "util/sst_file_manager_impl.h"
-
-
-namespace rocksdb {
-uint64_t DBImpl::FindMinPrepLogReferencedByMemTable() {
-  if (!allow_2pc()) {
-    return 0;
-  }
-
-  uint64_t min_log = 0;
-
-  // we must look through the memtables for two phase transactions
-  // that have been committed but not yet flushed
-  for (auto loop_cfd : *versions_->GetColumnFamilySet()) {
-    if (loop_cfd->IsDropped()) {
-      continue;
-    }
-
-    auto log = loop_cfd->imm()->GetMinLogContainingPrepSection();
-
-    if (log > 0 && (min_log == 0 || log < min_log)) {
-      min_log = log;
-    }
-
-    log = loop_cfd->mem()->GetMinLogContainingPrepSection();
-
-    if (log > 0 && (min_log == 0 || log < min_log)) {
-      min_log = log;
-    }
-  }
-
-  return min_log;
-}
-
-void DBImpl::MarkLogAsHavingPrepSectionFlushed(uint64_t log) {
-  assert(log != 0);
-  std::lock_guard<std::mutex> lock(prep_heap_mutex_);
-  auto it = prepared_section_completed_.find(log);
-  assert(it != prepared_section_completed_.end());
-  it->second += 1;
-}
-
-void DBImpl::MarkLogAsContainingPrepSection(uint64_t log) {
-  assert(log != 0);
-  std::lock_guard<std::mutex> lock(prep_heap_mutex_);
-  min_log_with_prep_.push(log);
-  auto it = prepared_section_completed_.find(log);
-  if (it == prepared_section_completed_.end()) {
-    prepared_section_completed_[log] = 0;
-  }
-}
-
-uint64_t DBImpl::FindMinLogContainingOutstandingPrep() {
-
-  if (!allow_2pc()) {
-    return 0;
-  }
-
-  std::lock_guard<std::mutex> lock(prep_heap_mutex_);
-  uint64_t min_log = 0;
-
-  // first we look in the prepared heap where we keep
-  // track of transactions that have been prepared (written to WAL)
-  // but not yet committed.
-  while (!min_log_with_prep_.empty()) {
-    min_log = min_log_with_prep_.top();
-
-    auto it = prepared_section_completed_.find(min_log);
-
-    // value was marked as 'deleted' from heap
-    if (it != prepared_section_completed_.end() && it->second > 0) {
-      it->second -= 1;
-      min_log_with_prep_.pop();
-
-      // back to squere one...
-      min_log = 0;
-      continue;
-    } else {
-      // found a valid value
-      break;
-    }
-  }
-
-  return min_log;
-}
-
-uint64_t DBImpl::MinLogNumberToKeep() {
-  uint64_t log_number = versions_->MinLogNumber();
-
-  if (allow_2pc()) {
-    // if are 2pc we must consider logs containing prepared
-    // sections of outstanding transactions.
-    //
-    // We must check min logs with outstanding prep before we check
-    // logs referneces by memtables because a log referenced by the
-    // first data structure could transition to the second under us.
-    //
-    // TODO(horuff): iterating over all column families under db mutex.
-    // should find more optimial solution
-    auto min_log_in_prep_heap = FindMinLogContainingOutstandingPrep();
-
-    if (min_log_in_prep_heap != 0 && min_log_in_prep_heap < log_number) {
-      log_number = min_log_in_prep_heap;
-    }
-
-    auto min_log_refed_by_mem = FindMinPrepLogReferencedByMemTable();
-
-    if (min_log_refed_by_mem != 0 && min_log_refed_by_mem < log_number) {
-      log_number = min_log_refed_by_mem;
-    }
-  }
-  return log_number;
-}
-
-// * Returns the list of live files in 'sst_live'
-// If it's doing full scan:
-// * Returns the list of all files in the filesystem in
-// 'full_scan_candidate_files'.
-// Otherwise, gets obsolete files from VersionSet.
-// no_full_scan = true -- never do the full scan using GetChildren()
-// force = false -- don't force the full scan, except every
-//  mutable_db_options_.delete_obsolete_files_period_micros
-// force = true -- force the full scan
-void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
-                               bool no_full_scan) {
-  mutex_.AssertHeld();
-
-  // if deletion is disabled, do nothing
-  if (disable_delete_obsolete_files_ > 0) {
-    return;
-  }
-
-  bool doing_the_full_scan = false;
-
-  // logic for figurint out if we're doing the full scan
-  if (no_full_scan) {
-    doing_the_full_scan = false;
-  } else if (force ||
-             mutable_db_options_.delete_obsolete_files_period_micros == 0) {
-    doing_the_full_scan = true;
-  } else {
-    const uint64_t now_micros = env_->NowMicros();
-    if ((delete_obsolete_files_last_run_ +
-         mutable_db_options_.delete_obsolete_files_period_micros) <
-        now_micros) {
-      doing_the_full_scan = true;
-      delete_obsolete_files_last_run_ = now_micros;
-    }
-  }
-
-  // don't delete files that might be currently written to from compaction
-  // threads
-  // Since job_context->min_pending_output is set, until file scan finishes,
-  // mutex_ cannot be released. Otherwise, we might see no min_pending_output
-  // here but later find newer generated unfinalized files while scannint.
-  if (!pending_outputs_.empty()) {
-    job_context->min_pending_output = *pending_outputs_.begin();
-  } else {
-    // delete all of them
-    job_context->min_pending_output = std::numeric_limits<uint64_t>::max();
-  }
-
-  // Get obsolete files.  This function will also update the list of
-  // pending files in VersionSet().
-  versions_->GetObsoleteFiles(&job_context->sst_delete_files,
-                              &job_context->manifest_delete_files,
-                              job_context->min_pending_output);
-
-  // store the current filenum, lognum, etc
-  job_context->manifest_file_number = versions_->manifest_file_number();
-  job_context->pending_manifest_file_number =
-      versions_->pending_manifest_file_number();
-  job_context->log_number = MinLogNumberToKeep();
-
-  job_context->prev_log_number = versions_->prev_log_number();
-
-  versions_->AddLiveFiles(&job_context->sst_live);
-  if (doing_the_full_scan) {
-    for (size_t path_id = 0; path_id < immutable_db_options_.db_paths.size();
-         path_id++) {
-      // set of all files in the directory. We'll exclude files that are still
-      // alive in the subsequent processings.
-      std::vector<std::string> files;
-      env_->GetChildren(immutable_db_options_.db_paths[path_id].path,
-                        &files);  // Ignore errors
-      for (std::string file : files) {
-        // TODO(icanadi) clean up this mess to avoid having one-off "/" prefixes
-        job_context->full_scan_candidate_files.emplace_back(
-            "/" + file, static_cast<uint32_t>(path_id));
-      }
-    }
-
-    // Add log files in wal_dir
-    if (immutable_db_options_.wal_dir != dbname_) {
-      std::vector<std::string> log_files;
-      env_->GetChildren(immutable_db_options_.wal_dir,
-                        &log_files);  // Ignore errors
-      for (std::string log_file : log_files) {
-        job_context->full_scan_candidate_files.emplace_back(log_file, 0);
-      }
-    }
-    // Add info log files in db_log_dir
-    if (!immutable_db_options_.db_log_dir.empty() &&
-        immutable_db_options_.db_log_dir != dbname_) {
-      std::vector<std::string> info_log_files;
-      // Ignore errors
-      env_->GetChildren(immutable_db_options_.db_log_dir, &info_log_files);
-      for (std::string log_file : info_log_files) {
-        job_context->full_scan_candidate_files.emplace_back(log_file, 0);
-      }
-    }
-  }
-
-  // logs_ is empty when called during recovery, in which case there can't yet
-  // be any tracked obsolete logs
-  if (!alive_log_files_.empty() && !logs_.empty()) {
-    uint64_t min_log_number = job_context->log_number;
-    size_t num_alive_log_files = alive_log_files_.size();
-    // find newly obsoleted log files
-    while (alive_log_files_.begin()->number < min_log_number) {
-      auto& earliest = *alive_log_files_.begin();
-      if (immutable_db_options_.recycle_log_file_num >
-          log_recycle_files.size()) {
-        ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                       "adding log %" PRIu64 " to recycle list\n",
-                       earliest.number);
-        log_recycle_files.push_back(earliest.number);
-      } else {
-        job_context->log_delete_files.push_back(earliest.number);
-      }
-      if (job_context->size_log_to_delete == 0) {
-        job_context->prev_total_log_size = total_log_size_;
-        job_context->num_alive_log_files = num_alive_log_files;
-      }
-      job_context->size_log_to_delete += earliest.size;
-      total_log_size_ -= earliest.size;
-      alive_log_files_.pop_front();
-      // Current log should always stay alive since it can't have
-      // number < MinLogNumber().
-      assert(alive_log_files_.size());
-    }
-    while (!logs_.empty() && logs_.front().number < min_log_number) {
-      auto& log = logs_.front();
-      if (log.getting_synced) {
-        log_sync_cv_.Wait();
-        // logs_ could have changed while we were waiting.
-        continue;
-      }
-      logs_to_free_.push_back(log.ReleaseWriter());
-      logs_.pop_front();
-    }
-    // Current log cannot be obsolete.
-    assert(!logs_.empty());
-  }
-
-  // We're just cleaning up for DB::Write().
-  assert(job_context->logs_to_free.empty());
-  job_context->logs_to_free = logs_to_free_;
-  job_context->log_recycle_files.assign(log_recycle_files.begin(),
-                                        log_recycle_files.end());
-  logs_to_free_.clear();
-}
-
-namespace {
-bool CompareCandidateFile(const JobContext::CandidateFileInfo& first,
-                          const JobContext::CandidateFileInfo& second) {
-  if (first.file_name > second.file_name) {
-    return true;
-  } else if (first.file_name < second.file_name) {
-    return false;
-  } else {
-    return (first.path_id > second.path_id);
-  }
-}
-};  // namespace
-
-// Delete obsolete files and log status and information of file deletion
-void DBImpl::DeleteObsoleteFileImpl(Status file_deletion_status, int job_id,
-                                    const std::string& fname, FileType type,
-                                    uint64_t number, uint32_t path_id) {
-  if (type == kTableFile) {
-    file_deletion_status =
-        DeleteSSTFile(&immutable_db_options_, fname, path_id);
-  } else {
-    file_deletion_status = env_->DeleteFile(fname);
-  }
-  if (file_deletion_status.ok()) {
-    ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
-                    "[JOB %d] Delete %s type=%d #%" PRIu64 " -- %s\n", job_id,
-                    fname.c_str(), type, number,
-                    file_deletion_status.ToString().c_str());
-  } else if (env_->FileExists(fname).IsNotFound()) {
-    ROCKS_LOG_INFO(
-        immutable_db_options_.info_log,
-        "[JOB %d] Tried to delete a non-existing file %s type=%d #%" PRIu64
-        " -- %s\n",
-        job_id, fname.c_str(), type, number,
-        file_deletion_status.ToString().c_str());
-  } else {
-    ROCKS_LOG_ERROR(immutable_db_options_.info_log,
-                    "[JOB %d] Failed to delete %s type=%d #%" PRIu64 " -- %s\n",
-                    job_id, fname.c_str(), type, number,
-                    file_deletion_status.ToString().c_str());
-  }
-  if (type == kTableFile) {
-    EventHelpers::LogAndNotifyTableFileDeletion(
-        &event_logger_, job_id, number, fname, file_deletion_status, GetName(),
-        immutable_db_options_.listeners);
-  }
-}
-
-// Diffs the files listed in filenames and those that do not
-// belong to live files are posibly removed. Also, removes all the
-// files in sst_delete_files and log_delete_files.
-// It is not necessary to hold the mutex when invoking this method.
-void DBImpl::PurgeObsoleteFiles(const JobContext& state, bool schedule_only) {
-  // we'd better have sth to delete
-  assert(state.HaveSomethingToDelete());
-
-  // this checks if FindObsoleteFiles() was run before. If not, don't do
-  // PurgeObsoleteFiles(). If FindObsoleteFiles() was run, we need to also
-  // run PurgeObsoleteFiles(), even if disable_delete_obsolete_files_ is true
-  if (state.manifest_file_number == 0) {
-    return;
-  }
-
-  // Now, convert live list to an unordered map, WITHOUT mutex held;
-  // set is slow.
-  std::unordered_map<uint64_t, const FileDescriptor*> sst_live_map;
-  for (const FileDescriptor& fd : state.sst_live) {
-    sst_live_map[fd.GetNumber()] = &fd;
-  }
-  std::unordered_set<uint64_t> log_recycle_files_set(
-      state.log_recycle_files.begin(), state.log_recycle_files.end());
-
-  auto candidate_files = state.full_scan_candidate_files;
-  candidate_files.reserve(
-      candidate_files.size() + state.sst_delete_files.size() +
-      state.log_delete_files.size() + state.manifest_delete_files.size());
-  // We may ignore the dbname when generating the file names.
-  const char* kDumbDbName = "";
-  for (auto file : state.sst_delete_files) {
-    candidate_files.emplace_back(
-        MakeTableFileName(kDumbDbName, file->fd.GetNumber()),
-        file->fd.GetPathId());
-    delete file;
-  }
-
-  for (auto file_num : state.log_delete_files) {
-    if (file_num > 0) {
-      candidate_files.emplace_back(LogFileName(kDumbDbName, file_num), 0);
-    }
-  }
-  for (const auto& filename : state.manifest_delete_files) {
-    candidate_files.emplace_back(filename, 0);
-  }
-
-  // dedup state.candidate_files so we don't try to delete the same
-  // file twice
-  std::sort(candidate_files.begin(), candidate_files.end(),
-            CompareCandidateFile);
-  candidate_files.erase(
-      std::unique(candidate_files.begin(), candidate_files.end()),
-      candidate_files.end());
-
-  if (state.prev_total_log_size > 0) {
-    ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                   "[JOB %d] Try to delete WAL files size %" PRIu64
-                   ", prev total WAL file size %" PRIu64
-                   ", number of live WAL files %" ROCKSDB_PRIszt ".\n",
-                   state.job_id, state.size_log_to_delete,
-                   state.prev_total_log_size, state.num_alive_log_files);
-  }
-
-  std::vector<std::string> old_info_log_files;
-  InfoLogPrefix info_log_prefix(!immutable_db_options_.db_log_dir.empty(),
-                                dbname_);
-  for (const auto& candidate_file : candidate_files) {
-    std::string to_delete = candidate_file.file_name;
-    uint32_t path_id = candidate_file.path_id;
-    uint64_t number;
-    FileType type;
-    // Ignore file if we cannot recognize it.
-    if (!ParseFileName(to_delete, &number, info_log_prefix.prefix, &type)) {
-      continue;
-    }
-
-    bool keep = true;
-    switch (type) {
-      case kLogFile:
-        keep = ((number >= state.log_number) ||
-                (number == state.prev_log_number) ||
-                (log_recycle_files_set.find(number) !=
-                 log_recycle_files_set.end()));
-        break;
-      case kDescriptorFile:
-        // Keep my manifest file, and any newer incarnations'
-        // (can happen during manifest roll)
-        keep = (number >= state.manifest_file_number);
-        break;
-      case kTableFile:
-        // If the second condition is not there, this makes
-        // DontDeletePendingOutputs fail
-        keep = (sst_live_map.find(number) != sst_live_map.end()) ||
-               number >= state.min_pending_output;
-        break;
-      case kTempFile:
-        // Any temp files that are currently being written to must
-        // be recorded in pending_outputs_, which is inserted into "live".
-        // Also, SetCurrentFile creates a temp file when writing out new
-        // manifest, which is equal to state.pending_manifest_file_number. We
-        // should not delete that file
-        //
-        // TODO(yhchiang): carefully modify the third condition to safely
-        //                 remove the temp options files.
-        keep = (sst_live_map.find(number) != sst_live_map.end()) ||
-               (number == state.pending_manifest_file_number) ||
-               (to_delete.find(kOptionsFileNamePrefix) != std::string::npos);
-        break;
-      case kInfoLogFile:
-        keep = true;
-        if (number != 0) {
-          old_info_log_files.push_back(to_delete);
-        }
-        break;
-      case kCurrentFile:
-      case kDBLockFile:
-      case kIdentityFile:
-      case kMetaDatabase:
-      case kOptionsFile:
-      case kBlobFile:
-        keep = true;
-        break;
-    }
-
-    if (keep) {
-      continue;
-    }
-
-    std::string fname;
-    if (type == kTableFile) {
-      // evict from cache
-      TableCache::Evict(table_cache_.get(), number);
-      fname = TableFileName(immutable_db_options_.db_paths, number, path_id);
-    } else {
-      fname = ((type == kLogFile) ? immutable_db_options_.wal_dir : dbname_) +
-              "/" + to_delete;
-    }
-
-#ifndef ROCKSDB_LITE
-    if (type == kLogFile && (immutable_db_options_.wal_ttl_seconds > 0 ||
-                             immutable_db_options_.wal_size_limit_mb > 0)) {
-      wal_manager_.ArchiveWALFile(fname, number);
-      continue;
-    }
-#endif  // !ROCKSDB_LITE
-
-    Status file_deletion_status;
-    if (schedule_only) {
-      InstrumentedMutexLock guard_lock(&mutex_);
-      SchedulePendingPurge(fname, type, number, path_id, state.job_id);
-    } else {
-      DeleteObsoleteFileImpl(file_deletion_status, state.job_id, fname, type,
-                             number, path_id);
-    }
-  }
-
-  // Delete old info log files.
-  size_t old_info_log_file_count = old_info_log_files.size();
-  if (old_info_log_file_count != 0 &&
-      old_info_log_file_count >= immutable_db_options_.keep_log_file_num) {
-    std::sort(old_info_log_files.begin(), old_info_log_files.end());
-    size_t end =
-        old_info_log_file_count - immutable_db_options_.keep_log_file_num;
-    for (unsigned int i = 0; i <= end; i++) {
-      std::string& to_delete = old_info_log_files.at(i);
-      std::string full_path_to_delete =
-          (immutable_db_options_.db_log_dir.empty()
-               ? dbname_
-               : immutable_db_options_.db_log_dir) +
-          "/" + to_delete;
-      ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                     "[JOB %d] Delete info log file %s\n", state.job_id,
-                     full_path_to_delete.c_str());
-      Status s = env_->DeleteFile(full_path_to_delete);
-      if (!s.ok()) {
-        if (env_->FileExists(full_path_to_delete).IsNotFound()) {
-          ROCKS_LOG_INFO(
-              immutable_db_options_.info_log,
-              "[JOB %d] Tried to delete non-existing info log file %s FAILED "
-              "-- %s\n",
-              state.job_id, to_delete.c_str(), s.ToString().c_str());
-        } else {
-          ROCKS_LOG_ERROR(immutable_db_options_.info_log,
-                          "[JOB %d] Delete info log file %s FAILED -- %s\n",
-                          state.job_id, to_delete.c_str(),
-                          s.ToString().c_str());
-        }
-      }
-    }
-  }
-#ifndef ROCKSDB_LITE
-  wal_manager_.PurgeObsoleteWALFiles();
-#endif  // ROCKSDB_LITE
-  LogFlush(immutable_db_options_.info_log);
-}
-
-void DBImpl::DeleteObsoleteFiles() {
-  mutex_.AssertHeld();
-  JobContext job_context(next_job_id_.fetch_add(1));
-  FindObsoleteFiles(&job_context, true);
-
-  mutex_.Unlock();
-  if (job_context.HaveSomethingToDelete()) {
-    PurgeObsoleteFiles(job_context);
-  }
-  job_context.Clean();
-  mutex_.Lock();
-}
-}  // namespace rocksdb
@@ -8,10 +8,9 @@

 #include "db/compacted_db_impl.h"
 #include "db/db_impl.h"
-#include "db/db_iter.h"
 #include "db/merge_context.h"
-#include "db/range_del_aggregator.h"
-#include "monitoring/perf_context_imp.h"
+#include "db/db_iter.h"
+#include "util/perf_context_imp.h"

 namespace rocksdb {

@@ -20,9 +19,8 @@ namespace rocksdb {
 DBImplReadOnly::DBImplReadOnly(const DBOptions& db_options,
                               const std::string& dbname)
    : DBImpl(db_options, dbname) {
-  ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                 "Opening the db in read only mode");
-  LogFlush(immutable_db_options_.info_log);
+  Log(INFO_LEVEL, db_options_.info_log, "Opening the db in read only mode");
+  LogFlush(db_options_.info_log);
 }

 DBImplReadOnly::~DBImplReadOnly() {
@@ -31,23 +29,18 @@ DBImplReadOnly::~DBImplReadOnly() {
 // Implementations of the DB interface
 Status DBImplReadOnly::Get(const ReadOptions& read_options,
                           ColumnFamilyHandle* column_family, const Slice& key,
-                           PinnableSlice* pinnable_val) {
-  assert(pinnable_val != nullptr);
+                           std::string* value) {
  Status s;
  SequenceNumber snapshot = versions_->LastSequence();
  auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
  auto cfd = cfh->cfd();
  SuperVersion* super_version = cfd->GetSuperVersion();
  MergeContext merge_context;
-  RangeDelAggregator range_del_agg(cfd->internal_comparator(), snapshot);
  LookupKey lkey(key, snapshot);
-  if (super_version->mem->Get(lkey, pinnable_val->GetSelf(), &s, &merge_context,
-                              &range_del_agg, read_options)) {
-    pinnable_val->PinSelf();
+  if (super_version->mem->Get(lkey, value, &s, &merge_context)) {
  } else {
    PERF_TIMER_GUARD(get_from_output_files_time);
-    super_version->current->Get(read_options, lkey, pinnable_val, &s,
-                                &merge_context, &range_del_agg);
+    super_version->current->Get(read_options, lkey, value, &s, &merge_context);
  }
  return s;
 }
@@ -59,16 +52,15 @@ Iterator* DBImplReadOnly::NewIterator(const ReadOptions& read_options,
  SuperVersion* super_version = cfd->GetSuperVersion()->Ref();
  SequenceNumber latest_snapshot = versions_->LastSequence();
  auto db_iter = NewArenaWrappedDbIterator(
-      env_, read_options, *cfd->ioptions(), cfd->user_comparator(),
+      env_, *cfd->ioptions(), cfd->user_comparator(),
      (read_options.snapshot != nullptr
           ? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
                 ->number_
           : latest_snapshot),
      super_version->mutable_cf_options.max_sequential_skip_in_iterations,
      super_version->version_number);
-  auto internal_iter =
-      NewInternalIterator(read_options, cfd, super_version, db_iter->GetArena(),
-                          db_iter->GetRangeDelAggregator());
+  auto internal_iter = NewInternalIterator(
+      read_options, cfd, super_version, db_iter->GetArena());
  db_iter->SetIterUnderDBIter(internal_iter);
  return db_iter;
 }
@@ -88,16 +80,15 @@ Status DBImplReadOnly::NewIterators(
    auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(cfh)->cfd();
    auto* sv = cfd->GetSuperVersion()->Ref();
    auto* db_iter = NewArenaWrappedDbIterator(
-        env_, read_options, *cfd->ioptions(), cfd->user_comparator(),
+        env_, *cfd->ioptions(), cfd->user_comparator(),
        (read_options.snapshot != nullptr
             ? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
                   ->number_
             : latest_snapshot),
        sv->mutable_cf_options.max_sequential_skip_in_iterations,
        sv->version_number);
-    auto* internal_iter =
-        NewInternalIterator(read_options, cfd, sv, db_iter->GetArena(),
-                            db_iter->GetRangeDelAggregator());
+    auto* internal_iter = NewInternalIterator(
+        read_options, cfd, sv, db_iter->GetArena());
    db_iter->SetIterUnderDBIter(internal_iter);
    iterators->push_back(db_iter);
  }
@@ -22,7 +22,7 @@ class DBImplReadOnly : public DBImpl {
  using DB::Get;
  virtual Status Get(const ReadOptions& options,
                     ColumnFamilyHandle* column_family, const Slice& key,
-                     PinnableSlice* value) override;
+                     std::string* value) override;

  // TODO: Implement ReadOnly MultiGet?

@@ -79,6 +79,7 @@ class DBImplReadOnly : public DBImpl {
    return Status::NotSupported("Not supported operation in read only mode.");
  }

+#ifndef ROCKSDB_LITE
  virtual Status DisableFileDeletions() override {
    return Status::NotSupported("Not supported operation in read only mode.");
  }
@@ -91,6 +92,7 @@ class DBImplReadOnly : public DBImpl {
                              bool flush_memtable = true) override {
    return Status::NotSupported("Not supported operation in read only mode.");
  }
+#endif  // ROCKSDB_LITE

  using DBImpl::Flush;
  virtual Status Flush(const FlushOptions& options,
@@ -103,14 +105,6 @@ class DBImplReadOnly : public DBImpl {
    return Status::NotSupported("Not supported operation in read only mode.");
  }

-  using DB::IngestExternalFile;
-  virtual Status IngestExternalFile(
-      ColumnFamilyHandle* column_family,
-      const std::vector<std::string>& external_files,
-      const IngestExternalFileOptions& ingestion_options) override {
-    return Status::NotSupported("Not supported operation in read only mode.");
-  }
-
 private:
  friend class DB;
Author	SHA1	Message	Date
sdong	52a2cc5a6d	Disable error as warning	2019-11-05 11:04:14 -08:00
sdong	d60acee274	Add one more #include<functional>	2019-11-05 11:04:14 -08:00
sdong	84b27f480f	Add some include<functional>	2019-10-31 14:28:16 -07:00
sdong	92bf1f5773	[FB Internal] Point to the latest tool chain.	2019-10-31 14:28:08 -07:00
sdong	6672cd0770	[fb only] revert unintended change of USE_SSE The previuos change that use gcc-5 set USE_SSE to wrong flag by mistake. Fix it.	2017-07-17 22:22:16 -07:00
sdong	1d0562e7c8	[FB Only] use gcc-5	2017-07-17 21:53:13 -07:00
Islam AbdelRahman	3f96ed111e	backport fbcode gcc path fix	2016-06-08 13:08:36 -07:00
Yi Wu	0645e8b42c	Fix win build Summary: Fixing error with win build where we compare int64_t with size_t. Test Plan: make check Reviewers: andrewkr Reviewed By: andrewkr Subscribers: andrewkr, dhruba, leveldb Differential Revision: https://reviews.facebook.net/D57885	2016-05-09 13:43:05 -07:00
Islam AbdelRahman	6504c31cb6	Fix valgrind (DBIteratorTest.ReadAhead) Summary: This test is failing under valgrind because we dont delete the Env that we allocated Test Plan: run the test under valgrind Reviewers: andrewkr, yhchiang, yiwu, sdong Reviewed By: sdong Subscribers: andrewkr, dhruba Differential Revision: https://reviews.facebook.net/D57693	2016-05-09 13:42:56 -07:00
Andrew Kryczka	53a86bf61e	Make max_dict_bytes optional in options string Summary: For backwards compatibility with older option strings, the parser needs to treat this argument as optional. Test Plan: Updated unit test to cover case where compression_opts is present but max_dict_bytes is omitted. Reviewers: MarkCallaghan, sdong, IslamAbdelRahman Reviewed By: IslamAbdelRahman Subscribers: andrewkr, dhruba, leveldb Differential Revision: https://reviews.facebook.net/D57759	2016-05-06 11:39:26 -07:00
sdong	8a5ec0ec6f	OptimizeForSmallDb(): revert some options whose defaults were just changed Summary: We changed default options of max_open_files and max_file_opening_threads but didn't revert it in OptimizeForSmallDb(). Test Plan: Add a unit test Reviewers: igor, yhchiang, IslamAbdelRahman Reviewed By: IslamAbdelRahman Subscribers: leveldb, andrewkr, dhruba Differential Revision: https://reviews.facebook.net/D57675	2016-05-06 10:18:34 -07:00
sdong	b7dbbdf783	BlockBasedTable::Get() not to use prefix bloom if read_options.total_order_seek = true Summary: This is to provide a way for users to skip prefix bloom in point look-up. Test Plan: Add a new unit test scenario. Reviewers: IslamAbdelRahman Subscribers: leveldb, andrewkr, dhruba Differential Revision: https://reviews.facebook.net/D57747	2016-05-06 10:18:03 -07:00
Yi Wu	ec64b24a0c	Fixing lite build Summary: Fixing lite build broke in unit test. `FilesPerLevel()` depends on `DB::GetProperty()`, which lite build doesn't support. Test Plan: OPT=-DROCKSDB_LITE make check -j64 Reviewers: sdong Reviewed By: sdong Subscribers: andrewkr, dhruba, leveldb Differential Revision: https://reviews.facebook.net/D57651	2016-05-05 10:09:16 -07:00
Yi Wu	2182bf2828	Enable configurable readahead for iterators Summary: Add an option `iterator_readahead_size` to `ReadOptions` to enable configurable readahead for iterators similar to the corresponding option for compaction. Test Plan: ``` make commit_prereq ``` Reviewers: kumar.rangarajan, ott, igor, sdong Reviewed By: sdong Subscribers: yiwu, andrewkr, dhruba Differential Revision: https://reviews.facebook.net/D55419	2016-05-05 10:09:16 -07:00
Islam AbdelRahman	2137377f0e	Fix Iterator::Prev memory pinning bug Summary: We should not use IterKey::SetKey with copy = false except if we are pinning the iterator thru it's life time, otherwise we may release the temporarily pinned blocks and in this case the IterKey will be pointing to freed memory Test Plan: added a new test Reviewers: sdong, andrewkr Reviewed By: andrewkr Subscribers: andrewkr, dhruba Differential Revision: https://reviews.facebook.net/D57561	2016-05-05 10:09:16 -07:00
Patrick Chan	a9f71f1b6e	sst_dump won't print size for unsupported compression type	2016-05-05 10:09:16 -07:00
Islam AbdelRahman	1f9508953f	Eliminate memcpy in Iterator::Prev() by pinning blocks for keys spanning multiple blocks Summary: This diff is stacked on top of this diff https://reviews.facebook.net/D56493 The current Iterator::Prev() implementation need to copy every value since the underlying Iterator may move after reading the value. This can be optimized by making sure that the block containing the value is pinned until the Iterator move. which will improve the throughput by up to 1.5X master ``` ==> 1000000_Keys_100Byte.txt <== readreverse : 0.449 micros/op 2225887 ops/sec; 246.2 MB/s readreverse : 0.433 micros/op 2311508 ops/sec; 255.7 MB/s readreverse : 0.436 micros/op 2294335 ops/sec; 253.8 MB/s readreverse : 0.471 micros/op 2121295 ops/sec; 234.7 MB/s readreverse : 0.465 micros/op 2152227 ops/sec; 238.1 MB/s readreverse : 0.454 micros/op 2203011 ops/sec; 243.7 MB/s readreverse : 0.451 micros/op 2216095 ops/sec; 245.2 MB/s readreverse : 0.462 micros/op 2162447 ops/sec; 239.2 MB/s readreverse : 0.476 micros/op 2099151 ops/sec; 232.2 MB/s readreverse : 0.472 micros/op 2120710 ops/sec; 234.6 MB/s avg : 242.34 MB/s ==> 1000000_Keys_1KB.txt <== readreverse : 1.013 micros/op 986793 ops/sec; 978.7 MB/s readreverse : 0.942 micros/op 1061136 ops/sec; 1052.5 MB/s readreverse : 0.951 micros/op 1051901 ops/sec; 1043.3 MB/s readreverse : 0.932 micros/op 1072894 ops/sec; 1064.1 MB/s readreverse : 1.024 micros/op 976720 ops/sec; 968.7 MB/s readreverse : 0.935 micros/op 1069169 ops/sec; 1060.4 MB/s readreverse : 1.012 micros/op 988132 ops/sec; 980.1 MB/s readreverse : 0.962 micros/op 1039579 ops/sec; 1031.1 MB/s readreverse : 0.991 micros/op 1008924 ops/sec; 1000.7 MB/s readreverse : 1.004 micros/op 996144 ops/sec; 988.0 MB/s avg : 1016.76 MB/s ==> 1000000_Keys_10KB.txt <== readreverse : 4.167 micros/op 239952 ops/sec; 2346.9 MB/s readreverse : 4.070 micros/op 245713 ops/sec; 2403.3 MB/s readreverse : 4.572 micros/op 218733 ops/sec; 2139.4 MB/s readreverse : 4.497 micros/op 222388 ops/sec; 2175.2 MB/s readreverse : 4.203 micros/op 237920 ops/sec; 2327.1 MB/s readreverse : 4.206 micros/op 237756 ops/sec; 2325.5 MB/s readreverse : 4.181 micros/op 239149 ops/sec; 2339.1 MB/s readreverse : 4.157 micros/op 240552 ops/sec; 2352.8 MB/s readreverse : 4.187 micros/op 238848 ops/sec; 2336.1 MB/s readreverse : 4.106 micros/op 243575 ops/sec; 2382.4 MB/s avg : 2312.78 MB/s ==> 100000_Keys_100KB.txt <== readreverse : 41.281 micros/op 24224 ops/sec; 2366.0 MB/s readreverse : 39.722 micros/op 25175 ops/sec; 2458.9 MB/s readreverse : 40.319 micros/op 24802 ops/sec; 2422.5 MB/s readreverse : 39.762 micros/op 25149 ops/sec; 2456.4 MB/s readreverse : 40.916 micros/op 24440 ops/sec; 2387.1 MB/s readreverse : 41.188 micros/op 24278 ops/sec; 2371.4 MB/s readreverse : 40.061 micros/op 24962 ops/sec; 2438.1 MB/s readreverse : 40.221 micros/op 24862 ops/sec; 2428.4 MB/s readreverse : 40.084 micros/op 24947 ops/sec; 2436.7 MB/s readreverse : 40.655 micros/op 24597 ops/sec; 2402.4 MB/s avg : 2416.79 MB/s ==> 10000_Keys_1MB.txt <== readreverse : 298.038 micros/op 3355 ops/sec; 3355.3 MB/s readreverse : 335.001 micros/op 2985 ops/sec; 2985.1 MB/s readreverse : 286.956 micros/op 3484 ops/sec; 3484.9 MB/s readreverse : 329.954 micros/op 3030 ops/sec; 3030.8 MB/s readreverse : 306.428 micros/op 3263 ops/sec; 3263.5 MB/s readreverse : 330.749 micros/op 3023 ops/sec; 3023.5 MB/s readreverse : 328.903 micros/op 3040 ops/sec; 3040.5 MB/s readreverse : 324.853 micros/op 3078 ops/sec; 3078.4 MB/s readreverse : 320.488 micros/op 3120 ops/sec; 3120.3 MB/s readreverse : 320.536 micros/op 3119 ops/sec; 3119.8 MB/s avg : 3150.21 MB/s ``` After memcpy elimination ``` ==> 1000000_Keys_100Byte.txt <== readreverse : 0.395 micros/op 2529890 ops/sec; 279.9 MB/s readreverse : 0.368 micros/op 2715922 ops/sec; 300.5 MB/s readreverse : 0.384 micros/op 2603929 ops/sec; 288.1 MB/s readreverse : 0.375 micros/op 2663286 ops/sec; 294.6 MB/s readreverse : 0.357 micros/op 2802180 ops/sec; 310.0 MB/s readreverse : 0.363 micros/op 2757684 ops/sec; 305.1 MB/s readreverse : 0.372 micros/op 2689603 ops/sec; 297.5 MB/s readreverse : 0.379 micros/op 2638599 ops/sec; 291.9 MB/s readreverse : 0.375 micros/op 2663803 ops/sec; 294.7 MB/s readreverse : 0.375 micros/op 2665579 ops/sec; 294.9 MB/s avg: 295.72 MB/s (1.22 X) ==> 1000000_Keys_1KB.txt <== readreverse : 0.879 micros/op 1138112 ops/sec; 1128.8 MB/s readreverse : 0.842 micros/op 1187998 ops/sec; 1178.3 MB/s readreverse : 0.837 micros/op 1194915 ops/sec; 1185.1 MB/s readreverse : 0.845 micros/op 1182983 ops/sec; 1173.3 MB/s readreverse : 0.877 micros/op 1140308 ops/sec; 1131.0 MB/s readreverse : 0.849 micros/op 1177581 ops/sec; 1168.0 MB/s readreverse : 0.915 micros/op 1093284 ops/sec; 1084.3 MB/s readreverse : 0.863 micros/op 1159418 ops/sec; 1149.9 MB/s readreverse : 0.895 micros/op 1117670 ops/sec; 1108.5 MB/s readreverse : 0.852 micros/op 1174116 ops/sec; 1164.5 MB/s avg: 1147.17 MB/s (1.12 X) ==> 1000000_Keys_10KB.txt <== readreverse : 3.870 micros/op 258386 ops/sec; 2527.2 MB/s readreverse : 3.568 micros/op 280296 ops/sec; 2741.5 MB/s readreverse : 4.005 micros/op 249694 ops/sec; 2442.2 MB/s readreverse : 3.550 micros/op 281719 ops/sec; 2755.5 MB/s readreverse : 3.562 micros/op 280758 ops/sec; 2746.1 MB/s readreverse : 3.507 micros/op 285125 ops/sec; 2788.8 MB/s readreverse : 3.463 micros/op 288739 ops/sec; 2824.1 MB/s readreverse : 3.428 micros/op 291734 ops/sec; 2853.4 MB/s readreverse : 3.553 micros/op 281491 ops/sec; 2753.2 MB/s readreverse : 3.535 micros/op 282885 ops/sec; 2766.9 MB/s avg : 2719.89 MB/s (1.17 X) ==> 100000_Keys_100KB.txt <== readreverse : 22.815 micros/op 43830 ops/sec; 4281.0 MB/s readreverse : 29.957 micros/op 33381 ops/sec; 3260.4 MB/s readreverse : 25.334 micros/op 39473 ops/sec; 3855.4 MB/s readreverse : 23.037 micros/op 43409 ops/sec; 4239.8 MB/s readreverse : 27.810 micros/op 35958 ops/sec; 3512.1 MB/s readreverse : 30.327 micros/op 32973 ops/sec; 3220.6 MB/s readreverse : 29.704 micros/op 33665 ops/sec; 3288.2 MB/s readreverse : 29.423 micros/op 33987 ops/sec; 3319.6 MB/s readreverse : 23.334 micros/op 42856 ops/sec; 4185.9 MB/s readreverse : 29.969 micros/op 33368 ops/sec; 3259.1 MB/s avg : 3642.21 MB/s (1.5 X) ==> 10000_Keys_1MB.txt <== readreverse : 244.748 micros/op 4085 ops/sec; 4085.9 MB/s readreverse : 230.208 micros/op 4343 ops/sec; 4344.0 MB/s readreverse : 235.655 micros/op 4243 ops/sec; 4243.6 MB/s readreverse : 235.730 micros/op 4242 ops/sec; 4242.2 MB/s readreverse : 237.346 micros/op 4213 ops/sec; 4213.3 MB/s readreverse : 227.306 micros/op 4399 ops/sec; 4399.4 MB/s readreverse : 194.957 micros/op 5129 ops/sec; 5129.4 MB/s readreverse : 238.359 micros/op 4195 ops/sec; 4195.4 MB/s readreverse : 221.588 micros/op 4512 ops/sec; 4513.0 MB/s readreverse : 235.911 micros/op 4238 ops/sec; 4239.0 MB/s avg : 4360.52 MB/s (1.38 X) ``` Test Plan: COMPILE_WITH_ASAN=1 make check -j64 Reviewers: andrewkr, yhchiang, sdong Reviewed By: sdong Subscribers: andrewkr, dhruba Differential Revision: https://reviews.facebook.net/D56511	2016-05-05 10:09:16 -07:00
Yi Wu	39d156b120	Release RocksDB 4.8.0 Summary: Release RocksDB 4.8.0 Test Plan: N/A Reviewers: sdong, IslamAbdelRahman Reviewed By: IslamAbdelRahman Subscribers: andrewkr, dhruba, leveldb Differential Revision: https://reviews.facebook.net/D57525	2016-05-02 15:09:38 -07:00
Warren Falk	cb08423712	Fix #1110 , 32-bit build failure on Mac OSX (#1112 ) Using explicit 64-bit type in conditional in platforms above 32-bits This appears to be necessary on Mac OSX as std::conditional does not appear to short circuit and evaluates the third template arg Making the third template arg be 64 bits explicitly works around this problem and will work on both 32 bit and 64+ bit platforms.	2016-05-02 15:09:38 -07:00
Islam AbdelRahman	d9a7d8a769	Fix calling GetCurrentMutableCFOptions in CompactionJob::ProcessKeyValueCompaction() Summary: GetCurrentMutableCFOptions() can only be called when DB mutex is held so we cannot call it in CompactionJob::ProcessKeyValueCompaction() since it's not holding the db mutex Test Plan: make check -j64 Reviewers: sdong, andrewkr Reviewed By: andrewkr Subscribers: andrewkr, dhruba Differential Revision: https://reviews.facebook.net/D57471	2016-05-02 15:09:38 -07:00
Dmitri Smirnov	5407b7c6d9	Fix multiple issues with WinMmapFile fo sequential writing (#1108 ) make preallocation inline with other writable files make sure that we map no more than pre-allocated size.	2016-05-02 15:09:38 -07:00