Update version and HISTORY for 6.22.3

Don't hold DB mutex for block cache entry stat scans (#8538 )
Summary: I previously didn't notice the DB mutex was being held during block cache entry stat scans, probably because I primarily checked for read performance regressions, because they require the block cache and are traditionally latency-sensitive. This change does some refactoring to avoid holding DB mutex and to avoid triggering and waiting for a scan in GetProperty("rocksdb.cfstats"). Some tests have to be updated because now the stats collector is populated in the Cache aggressively on DB startup rather than lazily. (I hope to clean up some of this added complexity in the future.) This change also ensures proper treatment of need_out_of_mutex for non-int DB properties. Pull Request resolved: https://github.com/facebook/rocksdb/pull/8538 Test Plan: Added unit test logic that uses sync points to fail if the DB mutex is held during a scan, covering the various ways that a scan might be triggered. Performance test - the known impact to holding the DB mutex is on TransactionDB, and the easiest way to see the impact is to hack the scan code to almost always miss and take an artificially long time scanning. Here I've injected an unconditional 5s sleep at the call to ApplyToAllEntries. Before (hacked): $ TEST_TMPDIR=/dev/shm ./db_bench.base_xxx -benchmarks=randomtransaction,stats -cache_index_and_filter_blocks=1 -bloom_bits=10 -partition_index_and_filters=1 -duration=30 -stats_dump_period_sec=12 -cache_size=100000000 -statistics -transaction_db 2>&1 | egrep 'db.db.write.micros|micros/op' randomtransaction : 433.219 micros/op 2308 ops/sec; 0.1 MB/s ( transactions:78999 aborts:0) rocksdb.db.write.micros P50 : 16.135883 P95 : 36.622503 P99 : 66.036115 P100 : 5000614.000000 COUNT : 149677 SUM : 8364856 $ TEST_TMPDIR=/dev/shm ./db_bench.base_xxx -benchmarks=randomtransaction,stats -cache_index_and_filter_blocks=1 -bloom_bits=10 -partition_index_and_filters=1 -duration=30 -stats_dump_period_sec=12 -cache_size=100000000 -statistics -transaction_db 2>&1 | egrep 'db.db.write.micros|micros/op' randomtransaction : 448.802 micros/op 2228 ops/sec; 0.1 MB/s ( transactions:75999 aborts:0) rocksdb.db.write.micros P50 : 16.629221 P95 : 37.320607 P99 : 72.144341 P100 : 5000871.000000 COUNT : 143995 SUM : 13472323 Notice the 5s P100 write time. After (hacked): $ TEST_TMPDIR=/dev/shm ./db_bench.new_xxx -benchmarks=randomtransaction,stats -cache_index_and_filter_blocks=1 -bloom_bits=10 -partition_index_and_filters=1 -duration=30 -stats_dump_period_sec=12 -cache_size=100000000 -statistics -transaction_db 2>&1 | egrep 'db.db.write.micros|micros/op' randomtransaction : 303.645 micros/op 3293 ops/sec; 0.1 MB/s ( transactions:98999 aborts:0) rocksdb.db.write.micros P50 : 16.061871 P95 : 33.978834 P99 : 60.018017 P100 : 616315.000000 COUNT : 187619 SUM : 4097407 $ TEST_TMPDIR=/dev/shm ./db_bench.new_xxx -benchmarks=randomtransaction,stats -cache_index_and_filter_blocks=1 -bloom_bits=10 -partition_index_and_filters=1 -duration=30 -stats_dump_period_sec=12 -cache_size=100000000 -statistics -transaction_db 2>&1 | egrep 'db.db.write.micros|micros/op' randomtransaction : 310.383 micros/op 3221 ops/sec; 0.1 MB/s ( transactions:96999 aborts:0) rocksdb.db.write.micros P50 : 16.270026 P95 : 35.786844 P99 : 64.302878 P100 : 603088.000000 COUNT : 183819 SUM : 4095918 P100 write is now ~0.6s. Not good, but it's the same even if I completely bypass all the scanning code: $ TEST_TMPDIR=/dev/shm ./db_bench.new_skip -benchmarks=randomtransaction,stats -cache_index_and_filter_blocks=1 -bloom_bits=10 -partition_index_and_filters=1 -duration=30 -stats_dump_period_sec=12 -cache_size=100000000 -statistics -transaction_db 2>&1 | egrep 'db.db.write.micros|micros/op' randomtransaction : 311.365 micros/op 3211 ops/sec; 0.1 MB/s ( transactions:96999 aborts:0) rocksdb.db.write.micros P50 : 16.274362 P95 : 36.221184 P99 : 68.809783 P100 : 649808.000000 COUNT : 183819 SUM : 4156767 $ TEST_TMPDIR=/dev/shm ./db_bench.new_skip -benchmarks=randomtransaction,stats -cache_index_and_filter_blocks=1 -bloom_bits=10 -partition_index_and_filters=1 -duration=30 -stats_dump_period_sec=12 -cache_size=100000000 -statistics -transaction_db 2>&1 | egrep 'db.db.write.micros|micros/op' randomtransaction : 308.395 micros/op 3242 ops/sec; 0.1 MB/s ( transactions:97999 aborts:0) rocksdb.db.write.micros P50 : 16.106222 P95 : 37.202403 P99 : 67.081875 P100 : 598091.000000 COUNT : 185714 SUM : 4098832 No substantial difference. Reviewed By: siying Differential Revision: D29738847 Pulled By: pdillinger fbshipit-source-id: 1c5c155f5a1b62e4fea0fd4eeb515a8b7474027b
2026-11-06 11:02:36 -05:00 · 2021-07-19 08:42:01 -07:00 · 2021-07-19 08:37:38 -07:00 · 2021-07-19 08:37:07 -07:00 · 2021-07-19 08:36:55 -07:00 · 2021-07-14 22:08:10 -07:00
334 changed files with 15507 additions and 5285 deletions
@@ -49,6 +49,9 @@ commands:
            echo "export GTEST_OUTPUT=\"xml:/tmp/test-results/\"" >> $BASH_ENV
            echo "export SKIP_FORMAT_BUCK_CHECKS=1" >> $BASH_ENV
            echo "export PRINT_PARALLEL_OUTPUTS=1" >> $BASH_ENV
+            echo "export GTEST_COLOR=1" >> $BASH_ENV
+            echo "export CTEST_OUTPUT_ON_FAILURE=1" >> $BASH_ENV
+            echo "export CTEST_TEST_TIMEOUT=300" >> $BASH_ENV

  pre-steps-macos:
      steps:
@@ -88,6 +91,14 @@ commands:
          command: |
            sudo apt-get update -y && sudo apt-get install -y libgflags-dev

+  upgrade-cmake:
+    steps:
+      - run:
+          name: Upgrade cmake
+          command: |
+            sudo apt remove --purge cmake
+            sudo snap install cmake --classic
+
  install-gflags-on-macos:
    steps:
      - run:
@@ -120,7 +131,8 @@ executors:
 jobs:
  build-macos:
    macos:
-      xcode: 9.4.1
+      xcode: 11.3.0
+    resource_class: large
    steps:
      - increase-max-open-files-on-macos
      - install-pyenv-on-macos
@@ -131,19 +143,20 @@ jobs:

  build-macos-cmake:
    macos:
-      xcode: 9.4.1
+      xcode: 11.3.0
+    resource_class: large
    steps:
      - increase-max-open-files-on-macos
      - install-pyenv-on-macos
      - install-cmake-on-macos
      - install-gflags-on-macos
      - pre-steps-macos
-      - run: ulimit -S -n 1048576 && (mkdir build && cd build && cmake -DWITH_GFLAGS=0 .. && make V=1 -j32) | .circleci/cat_ignore_eagain
+      - run: ulimit -S -n 1048576 && (mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. && make V=1 -j32 && ctest -j10) | .circleci/cat_ignore_eagain
      - post-steps

  build-linux:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
@@ -153,7 +166,7 @@ jobs:

  build-linux-mem-env:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
@@ -163,7 +176,7 @@ jobs:

  build-linux-encrypted-env:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
@@ -173,7 +186,7 @@ jobs:

  build-linux-shared_lib-alt_namespace-status_checked:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
@@ -183,7 +196,7 @@ jobs:

  build-linux-release:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: large
    steps:
      - checkout # check out the code in the project directory
@@ -210,7 +223,7 @@ jobs:

  build-linux-lite:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
@@ -220,7 +233,7 @@ jobs:

  build-linux-lite-release:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: large
    steps:
      - checkout # check out the code in the project directory
@@ -233,17 +246,17 @@ jobs:

  build-linux-clang-no_test_run:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: xlarge
    steps:
      - checkout # check out the code in the project directory
-      - run: sudo apt-get update -y && sudo apt-get install -y clang libgflags-dev
+      - run: sudo apt-get update -y && sudo apt-get install -y clang libgflags-dev libtbb-dev
      - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j16 all | .circleci/cat_ignore_eagain
      - post-steps

  build-linux-clang10-asan:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
@@ -254,7 +267,7 @@ jobs:

  build-linux-clang10-mini-tsan:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
@@ -265,7 +278,7 @@ jobs:

  build-linux-clang10-ubsan:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
@@ -276,7 +289,7 @@ jobs:

  build-linux-clang10-clang-analyze:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
@@ -286,13 +299,25 @@ jobs:
      - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path.
      - post-steps

-  build-linux-cmake-no_test_run:
+  build-linux-cmake:
    machine:
-      image: ubuntu-1604:202007-01
-    resource_class: large
+      image: ubuntu-1604:202104-01
+    resource_class: 2xlarge
    steps:
      - checkout # check out the code in the project directory
-      - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=0 .. && make V=1 -j8) | .circleci/cat_ignore_eagain
+      - install-gflags
+      - upgrade-cmake
+      - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. && make V=1 -j20 && ctest -j20) | .circleci/cat_ignore_eagain
+      - post-steps
+
+  build-linux-cmake-ubuntu-20:
+    machine:
+      image: ubuntu-2004:202104-01
+    resource_class: 2xlarge
+    steps:
+      - checkout # check out the code in the project directory
+      - install-gflags
+      - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. && make V=1 -j20 && ctest -j20) | .circleci/cat_ignore_eagain
      - post-steps

  build-linux-unity:
@@ -307,7 +332,7 @@ jobs:

  build-linux-gcc-4_8-no_test_run:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: large
    steps:
      - pre-steps
@@ -380,6 +405,8 @@ jobs:
              echo "Installing VS2015..."
              powershell .circleci/vs2015_install.ps1
            fi
+      - store_artifacts:
+          path: \Users\circleci\AppData\Local\Temp\vslogs.zip
      - run:
          name: "Install thirdparty dependencies"
          command: |
@@ -413,7 +440,7 @@ jobs:

  build-linux-java:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: large
    environment:
      JAVA_HOME: /usr/lib/jvm/java-1.8.0-openjdk-amd64
@@ -437,7 +464,7 @@ jobs:

  build-linux-java-static:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: large
    environment:
      JAVA_HOME: /usr/lib/jvm/java-1.8.0-openjdk-amd64
@@ -458,10 +485,10 @@ jobs:

  build-macos-java:
    macos:
-      xcode: 9.4.1
+      xcode: 11.3.0
    resource_class: medium
    environment:
-      JAVA_HOME: /Library/Java/JavaVirtualMachines/jdk1.8.0_172.jdk/Contents/Home
+      JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
    steps:
      - increase-max-open-files-on-macos
      - install-pyenv-on-macos
@@ -484,10 +511,10 @@ jobs:

  build-macos-java-static:
    macos:
-      xcode: 9.4.1
+      xcode: 11.3.0
    resource_class: medium
    environment:
-      JAVA_HOME: /Library/Java/JavaVirtualMachines/jdk1.8.0_172.jdk/Contents/Home
+      JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
    steps:
      - increase-max-open-files-on-macos
      - install-pyenv-on-macos
@@ -508,7 +535,7 @@ jobs:

  build-examples:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: large
    steps:
      - pre-steps
@@ -521,7 +548,7 @@ jobs:

  build-cmake-mingw:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
@@ -544,7 +571,7 @@ jobs:

  build-linux-non-shm:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    parameters:
      start_test:
@@ -600,6 +627,7 @@ jobs:
      JAVA_HOME: /usr/lib/jvm/java-8-openjdk-arm64
    steps:
      - pre-steps
+      - install-gflags
      - run:
          name: "Set Java Environment"
          command: |
@@ -612,7 +640,7 @@ jobs:
          command: |
            mkdir build
            cd build
-            cmake -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=0 -DWITH_GFLAGS=0 -DWITH_BENCHMARK_TOOLS=0 -DWITH_TOOLS=0 -DWITH_CORE_TOOLS=1 ..
+            cmake -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=0 -DWITH_GFLAGS=1 -DWITH_BENCHMARK_TOOLS=0 -DWITH_TOOLS=0 -DWITH_CORE_TOOLS=1 ..
            make -j4
      - run:
          name: "Build Java with cmake"
@@ -620,13 +648,13 @@ jobs:
            rm -rf build
            mkdir build
            cd build
-            cmake -DJNI=1 -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=0 ..
+            cmake -DJNI=1 -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=1 ..
            make -j4 rocksdb rocksdbjni
      - post-steps

  build-format-compatible:
    machine:
-      image: ubuntu-1604:202007-01
+      image: ubuntu-1604:202104-01
    resource_class: 2xlarge
    steps:
      - pre-steps
@@ -646,6 +674,10 @@ workflows:
  build-linux:
    jobs:
      - build-linux
+  build-linux-cmake:
+    jobs:
+      - build-linux-cmake
+      - build-linux-cmake-ubuntu-20
  build-linux-mem-env:
    jobs:
      - build-linux-mem-env
@@ -697,12 +729,12 @@ workflows:
          name: "build-windows-vs2017"
          vs_year: "2017"
          cmake_generator: "Visual Studio 15 Win64"
-  build-windows-vs2015:
-    jobs:
-      - build-windows:
-          name: "build-windows-vs2015"
-          vs_year: "2015"
-          cmake_generator: "Visual Studio 14 Win64"
+#  build-windows-vs2015:
+#    jobs:
+#      - build-windows:
+#          name: "build-windows-vs2015"
+#          vs_year: "2015"
+#          cmake_generator: "Visual Studio 14 Win64"
  build-java:
    jobs:
      - build-linux-java
@@ -729,7 +761,6 @@ workflows:
  build-linux-compilers-no_test_run:
    jobs:
      - build-linux-clang-no_test_run
-      - build-linux-cmake-no_test_run
      - build-linux-gcc-4_8-no_test_run
      - build-linux-gcc-8-no_test_run
      - build-linux-gcc-9-no_test_run
@@ -32,10 +32,11 @@
 # 3. cmake ..
 # 4. make -j

-cmake_minimum_required(VERSION 3.5.1)
+cmake_minimum_required(VERSION 3.10)

 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules/")
 include(ReadVersion)
+include(GoogleTest)
 get_rocksdb_version(rocksdb_VERSION)
 project(rocksdb
  VERSION ${rocksdb_VERSION}
@@ -62,6 +63,7 @@ if(CCACHE_FOUND)
 endif(CCACHE_FOUND)

 option(WITH_JEMALLOC "build with JeMalloc" OFF)
+option(WITH_LIBURING "build with liburing" ON)
 option(WITH_SNAPPY "build with SNAPPY" OFF)
 option(WITH_LZ4 "build with lz4" OFF)
 option(WITH_ZLIB "build with zlib" OFF)
@@ -70,6 +72,12 @@ option(WITH_WINDOWS_UTF8_FILENAMES "use UTF8 as characterset for opening files,
 if (WITH_WINDOWS_UTF8_FILENAMES)
  add_definitions(-DROCKSDB_WINDOWS_UTF8_FILENAMES)
 endif()
+
+if ($ENV{CIRCLECI})
+  message(STATUS "Build for CircieCI env, a few tests may be disabled")
+  add_definitions(-DCIRCLECI)
+endif()
+
 # third-party/folly is only validated to work on Linux and Windows for now.
 # So only turn it on there by default.
 if(CMAKE_SYSTEM_NAME MATCHES "Linux|Windows")
@@ -122,11 +130,11 @@ else()
        set(GFLAGS_LIB ${GFLAGS_TARGET})
      else()
        # Config with GFLAGS_LIBRARIES available since gflags 2.1.0
-        set(GFLAGS_LIB ${GFLAGS_LIBRARIES})
+        set(GFLAGS_LIB ${gflags_LIBRARIES})
      endif()
    else()
      find_package(gflags REQUIRED)
-        set(GFLAGS_LIB gflags::gflags)
+      set(GFLAGS_LIB gflags::gflags)
    endif()
    include_directories(${GFLAGS_INCLUDE_DIR})
    list(APPEND THIRDPARTY_LIBS ${GFLAGS_LIB})
@@ -210,7 +218,7 @@ if(MSVC)
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W4 /wd4127 /wd4800 /wd4996 /wd4351 /wd4100 /wd4204 /wd4324")
 else()
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall -pthread")
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wshadow -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers -Wno-strict-aliasing")
  if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wstrict-prototypes")
@@ -338,6 +346,22 @@ if (NOT BUILTIN_ATOMIC)
 endif()
 endif()

+if (WITH_LIBURING)
+  set(CMAKE_REQUIRED_FLAGS "-luring")
+  CHECK_CXX_SOURCE_COMPILES("
+#include <liburing.h>
+int main() {
+  struct io_uring ring;
+  io_uring_queue_init(1, &ring, 0);
+  return 0;
+}
+" HAS_LIBURING)
+  if (HAS_LIBURING)
+    add_definitions(-DROCKSDB_IOURING_PRESENT)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -luring")
+  endif()
+endif()
+
 # Reset the required flags
 set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQUIRED_FLAGS})

@@ -599,10 +623,12 @@ find_package(Threads REQUIRED)

 set(SOURCES
        cache/cache.cc
+        cache/cache_entry_roles.cc
        cache/clock_cache.cc
        cache/lru_cache.cc
        cache/sharded_cache.cc
        db/arena_wrapped_db_iter.cc
+        db/blob/blob_fetcher.cc
        db/blob/blob_file_addition.cc
        db/blob/blob_file_builder.cc
        db/blob/blob_file_cache.cc
@@ -1108,10 +1134,12 @@ if(WITH_TESTS)
        db/blob/db_blob_index_test.cc
        db/column_family_test.cc
        db/compact_files_test.cc
+        db/compaction/clipping_iterator_test.cc
        db/compaction/compaction_job_stats_test.cc
        db/compaction/compaction_job_test.cc
        db/compaction/compaction_iterator_test.cc
        db/compaction/compaction_picker_test.cc
+        db/compaction/compaction_service_test.cc
        db/comparator_db_test.cc
        db/corruption_test.cc
        db/cuckoo_table_db_test.cc
@@ -1302,21 +1330,21 @@ if(WITH_TESTS)

  foreach(sourcefile ${TESTS})
      get_filename_component(exename ${sourcefile} NAME_WE)
-      add_executable(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} ${sourcefile})
-      set_target_properties(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX}
+      add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile})
+      set_target_properties(${exename}${ARTIFACT_SUFFIX}
        PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
        EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
        EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
        OUTPUT_NAME ${exename}${ARTIFACT_SUFFIX}
      )
-      target_link_libraries(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} testharness gtest ${THIRDPARTY_LIBS} ${ROCKSDB_LIB})
+      target_link_libraries(${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} testharness gtest ${THIRDPARTY_LIBS} ${ROCKSDB_LIB})
      if(NOT "${exename}" MATCHES "db_sanity_test")
-        add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
-        add_dependencies(check ${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX})
+        gtest_discover_tests(${exename} DISCOVERY_TIMEOUT 120)
+        add_dependencies(check ${exename}${ARTIFACT_SUFFIX})
      endif()
      if("${exename}" MATCHES "env_librados_test")
        # env_librados_test.cc uses librados directly
-        target_link_libraries(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} rados)
+        target_link_libraries(${exename}${ARTIFACT_SUFFIX} rados)
      endif()
  endforeach(sourcefile ${TESTS})

@@ -1342,6 +1370,7 @@ endif()

 if(WITH_BENCHMARK_TOOLS)
  add_executable(db_bench${ARTIFACT_SUFFIX}
+    tools/simulated_hybrid_file_system.cc
    tools/db_bench.cc
    tools/db_bench_tool.cc)
  target_link_libraries(db_bench${ARTIFACT_SUFFIX}
@@ -1,23 +1,84 @@
 # Rocksdb Change Log
-## Unreleased
+## 6.22.3 (2021-07-19)
+### Bug Fixes
+* Fixed block cache entry stat scans not to hold the DB mutex, which was a serious performance bug for tail latencies in TransactionDB and likely elsewhere.
+
+## 6.22.2 (2021-07-14)
+### Bug Fixes
+* Fix continuous logging of an existing background error on every user write
+
+## 6.22.1 (2021-06-25)
+### Bug Fixes
+* `GetLiveFilesMetaData()` now populates the `temperature`, `oldest_ancester_time`, and `file_creation_time` fields of its `LiveFileMetaData` results when the information is available. Previously these fields always contained zero indicating unknown.
+
+## 6.22.0 (2021-06-18)
+### Behavior Changes
+* Added two additional tickers, MEMTABLE_PAYLOAD_BYTES_AT_FLUSH and MEMTABLE_GARBAGE_BYTES_AT_FLUSH. These stats can be used to estimate the ratio of "garbage" (outdated) bytes in the memtable that are discarded at flush time.
+* Added API comments clarifying safe usage of Disable/EnableManualCompaction and EventListener callbacks for compaction.
+
+### Bug Fixes
+* fs_posix.cc GetFreeSpace() always report disk space available to root even when running as non-root.  Linux defaults often have disk mounts with 5 to 10 percent of total space reserved only for root.  Out of space could result for non-root users.
+* Subcompactions are now disabled when user-defined timestamps are used, since the subcompaction boundary picking logic is currently not timestamp-aware, which could lead to incorrect results when different subcompactions process keys that only differ by timestamp.
+* Fix an issue that `DeleteFilesInRange()` may cause ongoing compaction reports corruption exception, or ASSERT for debug build. There's no actual data loss or corruption that we find.
+* Fixed confusingly duplicated output in LOG for periodic stats ("DUMPING STATS"), including "Compaction Stats" and "File Read Latency Histogram By Level".
+* Fixed performance bugs in background gathering of block cache entry statistics, that could consume a lot of CPU when there are many column families with a shared block cache.
+
+### New Features
+* Marked the Ribbon filter and optimize_filters_for_memory features as production-ready, each enabling memory savings for Bloom-like filters. Use `NewRibbonFilterPolicy` in place of `NewBloomFilterPolicy` to use Ribbon filters instead of Bloom, or `ribbonfilter` in place of `bloomfilter` in configuration string.
+* Allow `DBWithTTL` to use `DeleteRange` api just like other DBs. `DeleteRangeCF()` which executes `WriteBatchInternal::DeleteRange()` has been added to the handler in `DBWithTTLImpl::Write()` to implement it.
+* Add BlockBasedTableOptions.prepopulate_block_cache.  If enabled, it prepopulate warm/hot data blocks which are already in memory into block cache at the time of flush. On a flush, the data block that is in memory (in memtables) get flushed to the device. If using Direct IO, additional IO is incurred to read this data back into memory again, which is avoided by enabling this option and it also helps with Distributed FileSystem. More details in include/rocksdb/table.h.
+* Added a `cancel` field to `CompactRangeOptions`, allowing individual in-process manual range compactions to be cancelled.
+
+## 6.21.0 (2021-05-21)
 ### Bug Fixes
 * Fixed a bug in handling file rename error in distributed/network file systems when the server succeeds but client returns error. The bug can cause CURRENT file to point to non-existing MANIFEST file, thus DB cannot be opened.
 * Fixed a bug where ingested files were written with incorrect boundary key metadata. In rare cases this could have led to a level's files being wrongly ordered and queries for the boundary keys returning wrong results.
 * Fixed a data race between insertion into memtables and the retrieval of the DB properties `rocksdb.cur-size-active-mem-table`, `rocksdb.cur-size-all-mem-tables`, and `rocksdb.size-all-mem-tables`.
 * Fixed the false-positive alert when recovering from the WAL file. Avoid reporting "SST file is ahead of WAL" on a newly created empty column family, if the previous WAL file is corrupted.
+* Fixed a bug where `GetLiveFiles()` output included a non-existent file called "OPTIONS-000000". Backups and checkpoints, which use `GetLiveFiles()`, failed on DBs impacted by this bug. Read-write DBs were impacted when the latest OPTIONS file failed to write and `fail_if_options_file_error == false`. Read-only DBs were impacted when no OPTIONS files existed.
+* Handle return code by io_uring_submit_and_wait() and io_uring_wait_cqe().
+* In the IngestExternalFile() API, only try to sync the ingested file if the file is linked and the FileSystem/Env supports reopening a writable file.
+* Fixed a bug that `AdvancedColumnFamilyOptions.max_compaction_bytes` is under-calculated for manual compaction (`CompactRange()`). Manual compaction is split to multiple compactions if the compaction size exceed the `max_compaction_bytes`. The bug creates much larger compaction which size exceed the user setting. On the other hand, larger manual compaction size can increase the subcompaction parallelism, you can tune that by setting `max_compaction_bytes`.

 ### Behavior Changes
 * Due to the fix of false-postive alert of "SST file is ahead of WAL", all the CFs with no SST file (CF empty) will bypass the consistency check. We fixed a false-positive, but introduced a very rare true-negative which will be triggered in the following conditions: A CF with some delete operations in the last a few queries which will result in an empty CF (those are flushed to SST file and a compaction triggered which combines this file and all other SST files and generates an empty CF, or there is another reason to write a manifest entry for this CF after a flush that generates no SST file from an empty CF). The deletion entries are logged in a WAL and this WAL was corrupted, while the CF's log number points to the next WAL (due to the flush). Therefore, the DB can only recover to the point without these trailing deletions and cause the inconsistent DB status.

 ### New Features
 * Add new option allow_stall passed during instance creation of WriteBufferManager. When allow_stall is set, WriteBufferManager will stall all writers shared across multiple DBs and columns if memory usage goes beyond specified WriteBufferManager::buffer_size (soft limit). Stall will be cleared when memory is freed after flush and memory usage goes down below buffer_size.
+* Allow `CompactionFilter`s to apply in more table file creation scenarios such as flush and recovery. For compatibility, `CompactionFilter`s by default apply during compaction. Users can customize this behavior by overriding `CompactionFilterFactory::ShouldFilterTableFileCreation()`.
+* Added more fields to FilterBuildingContext with LSM details, for custom filter policies that vary behavior based on where they are in the LSM-tree.
+* Added DB::Properties::kBlockCacheEntryStats for querying statistics on what percentage of block cache is used by various kinds of blocks, etc. using DB::GetProperty and DB::GetMapProperty. The same information is now dumped to info LOG periodically according to `stats_dump_period_sec`.
+* Add an experimental Remote Compaction feature, which allows the user to run Compaction on a different host or process. The feature is still under development, currently only works on some basic use cases. The interface will be changed without backward/forward compatibility support.
+* RocksDB would validate total entries read in flush, and compare with counter inserted into it. If flush_verify_memtable_count = true (default), flush will fail. Otherwise, only log to info logs.
+* Add `TableProperties::num_filter_entries`, which can be used with `TableProperties::filter_size` to calculate the effective bits per filter entry (unique user key or prefix) for a table file.

-## 6.20.0 (04/16/2021)
+### Performance Improvements
+* BlockPrefetcher is used by iterators to prefetch data if they anticipate more data to be used in future. It is enabled implicitly by rocksdb. Added change to take in account read pattern if reads are sequential. This would disable prefetching for random reads in MultiGet and iterators as readahead_size is increased exponential doing large prefetches.
+
+### Public API change
+* Removed a parameter from TableFactory::NewTableBuilder, which should not be called by user code because TableBuilder is not a public API.
+* Removed unused structure `CompactionFilterContext`.
+* The `skip_filters` parameter to SstFileWriter is now considered deprecated. Use `BlockBasedTableOptions::filter_policy` to control generation of filters.
+* ClockCache is known to have bugs that could lead to crash or corruption, so should not be used until fixed. Use NewLRUCache instead.
+* Added a new pure virtual function `ApplyToAllEntries` to `Cache`, to replace `ApplyToAllCacheEntries`. Custom `Cache` implementations must add an implementation. Because this function is for gathering statistics, an empty implementation could be acceptable for some applications.
+* Added the ObjectRegistry to the ConfigOptions class.  This registry instance will be used to find any customizable loadable objects during initialization.
+* Expanded the ObjectRegistry functionality to allow nested ObjectRegistry instances.  Added methods to register a set of functions with the registry/library as a group.
+* Deprecated backupable_db.h and BackupableDBOptions in favor of new versions with appropriate names: backup_engine.h and BackupEngineOptions. Old API compatibility is preserved.
+
+### Default Option Change
+* When options.arena_block_size <= 0 (default value 0), still use writer_buffer_size / 8 but cap to 1MB. Too large alloation size might not be friendly to allocator and might cause performance issues in extreme cases.
+
+### Build
+* By default, try to build with liburing. For make, if ROCKSDB_USE_IO_URING is not set, treat as enable, which means RocksDB will try to build with liburing. Users can disable it with ROCKSDB_USE_IO_URING=0. For cmake, add WITH_LIBURING to control it, with default on.
+
+## 6.20.0 (2021-04-16)
 ### Behavior Changes
 * `ColumnFamilyOptions::sample_for_compression` now takes effect for creation of all block-based tables. Previously it only took effect for block-based tables created by flush.
 * `CompactFiles()` can no longer compact files from lower level to up level, which has the risk to corrupt DB (details: #8063). The validation is also added to all compactions.
 * Fixed some cases in which DB::OpenForReadOnly() could write to the filesystem. If you want a Logger with a read-only DB, you must now set DBOptions::info_log yourself, such as using CreateLoggerFromOptions().
 * get_iostats_context() will never return nullptr. If thread-local support is not available, and user does not opt-out iostats context, then compilation will fail. The same applies to perf context as well.
+* Added support for WriteBatchWithIndex::NewIteratorWithBase when overwrite_key=false.  Previously, this combination was not supported and would assert or return nullptr.
+* Improve the behavior of WriteBatchWithIndex for Merge operations.  Now more operations may be stored in order to return the correct merged result.

 ### Bug Fixes
 * Use thread-safe `strerror_r()` to get error messages.
@@ -43,8 +104,7 @@
 * Added BackupEngine::GetBackupInfo / GetLatestBackupInfo for querying individual backups.
 * Made the Ribbon filter a long-term supported feature in terms of the SST schema(compatible with version >= 6.15.0) though the API for enabling it is expected to change.

-
-## 6.19.0 (03/21/2021)
+## 6.19.0 (2021-03-21)
 ### Bug Fixes
 * Fixed the truncation error found in APIs/tools when dumping block-based SST files in a human-readable format. After fix, the block-based table can be fully dumped as a readable file.
 * When hitting a write slowdown condition, no write delay (previously 1 millisecond) is imposed until `delayed_write_rate` is actually exceeded, with an initial burst allowance of 1 millisecond worth of bytes. Also, beyond the initial burst allowance, `delayed_write_rate` is now more strictly enforced, especially with multiple column families.
@@ -73,7 +133,7 @@
 ### Behavior Changes
 * During flush, only WAL sync retryable IO error is mapped to hard error, which will stall the writes. When WAL is used but only SST file write has retryable IO error, it will be mapped to soft error and write will not be affected.

-## 6.18.0 (02/19/2021)
+## 6.18.0 (2021-02-19)
 ### Behavior Changes
 * When retryable IO error occurs during compaction, it is mapped to soft error and set the BG error. However, auto resume is not called to clean the soft error since compaction will reschedule by itself. In this change, When retryable IO error occurs during compaction, BG error is not set. User will be informed the error via EventHelper.
 * Introduce a new trace file format for query tracing and replay and trace file version is bump up to 0.2. A payload map is added as the first portion of the payload. We will not have backward compatible issues when adding new entries to trace records. Added the iterator_upper_bound and iterator_lower_bound in Seek and SeekForPrev tracing function. Added them as the new payload member for iterator tracing.
@@ -97,7 +157,7 @@
 * Add an option, `CompressionOptions::max_dict_buffer_bytes`, to limit the in-memory buffering for selecting samples for generating/training a dictionary. The limit is currently loosely adhered to.


-## 6.17.0 (01/15/2021)
+## 6.17.0 (2021-01-15)
 ### Behavior Changes
 * When verifying full file checksum with `DB::VerifyFileChecksums()`, we now fail with `Status::InvalidArgument` if the name of the checksum generator used for verification does not match the name of the checksum generator used for protecting the file when it was created.
 * Since RocksDB does not continue write the same file if a file write fails for any reason, the file scope write IO error is treated the same as retryable IO error. More information about error handling of file scope IO error is included in `ErrorHandler::SetBGError`.
@@ -111,7 +171,7 @@
 * Add a SystemClock class that contains the time-related methods from Env.  The original methods in Env may be deprecated in a future release.  This class will allow easier testing, development, and expansion of time-related features.
 * Add a public API GetRocksBuildProperties and GetRocksBuildInfoAsString to get properties about the current build.  These properties may include settings related to the GIT settings (branch, timestamp).  This change also sets the "build date" based on the GIT properties, rather than the actual build time, thereby enabling more reproducible builds.

-## 6.16.0 (12/18/2020)
+## 6.16.0 (2020-12-18)
 ### Behavior Changes
 * Attempting to write a merge operand without explicitly configuring `merge_operator` now fails immediately, causing the DB to enter read-only mode. Previously, failure was deferred until the `merge_operator` was needed by a user read or a background operation.

@@ -137,7 +197,7 @@
 * `rocksdb_approximate_sizes` and `rocksdb_approximate_sizes_cf` in the C API now requires an error pointer (`char** errptr`) for receiving any error.
 * All overloads of DB::GetApproximateSizes now return Status, so that any failure to obtain the sizes is indicated to the caller.

-## 6.15.0 (11/13/2020)
+## 6.15.0 (2020-11-13)
 ### Bug Fixes
 * Fixed a bug in the following combination of features: indexes with user keys (`format_version >= 3`), indexes are partitioned (`index_type == kTwoLevelIndexSearch`), and some index partitions are pinned in memory (`BlockBasedTableOptions::pin_l0_filter_and_index_blocks_in_cache`). The bug could cause keys to be truncated when read from the index leading to wrong read results or other unexpected behavior.
 * Fixed a bug when indexes are partitioned (`index_type == kTwoLevelIndexSearch`), some index partitions are pinned in memory (`BlockBasedTableOptions::pin_l0_filter_and_index_blocks_in_cache`), and partitions reads could be mixed between block cache and directly from the file (e.g., with `enable_index_compression == 1` and `mmap_read == 1`, partitions that were stored uncompressed due to poor compression ratio would be read directly from the file via mmap, while partitions that were stored compressed would be read from block cache). The bug could cause index partitions to be mistakenly considered empty during reads leading to wrong read results.
@@ -165,7 +225,7 @@
 ### New Features
 * An EXPERIMENTAL new Bloom alternative that saves about 30% space compared to Bloom filters, with about 3-4x construction time and similar query times is available using NewExperimentalRibbonFilterPolicy.

-## 6.14 (10/09/2020)
+## 6.14 (2020-10-09)
 ### Bug fixes
 * Fixed a bug after a `CompactRange()` with `CompactRangeOptions::change_level` set fails due to a conflict in the level change step, which caused all subsequent calls to `CompactRange()` with `CompactRangeOptions::change_level` set to incorrectly fail with a `Status::NotSupported("another thread is refitting")` error.
 * Fixed a bug that the bottom most level compaction could still be a trivial move even if `BottommostLevelCompaction.kForce` or `kForceOptimized` is set.
@@ -189,7 +249,7 @@
 * SST files have a new table property called db_host_id, which is set to the hostname by default. A new option in DBOptions, db_host_id, allows the property value to be overridden with a user specified string, or disable it completely by making the option string empty.
 * Methods to create customizable extensions -- such as TableFactory -- are exposed directly through the Customizable base class (from which these objects inherit).  This change will allow these Customizable classes to be loaded and configured in a standard way (via CreateFromString).  More information on how to write and use Customizable classes is in the customizable.h header file.

-## 6.13 (09/12/2020)
+## 6.13 (2020-09-12)
 ### Bug fixes
 * Fix a performance regression introduced in 6.4 that makes a upper bound check for every Next() even if keys are within a data block that is within the upper bound.
 * Fix a possible corruption to the LSM state (overlapping files within a level) when a `CompactRange()` for refitting levels (`CompactRangeOptions::change_level == true`) and another manual compaction are executed in parallel.
@@ -269,7 +329,7 @@
 * Reduce key comparisons during random access in all block-based tables.
 * BackupEngine avoids unnecessary repeated checksum computation for backing up a table file to the `shared_checksum` directory when using `share_files_with_checksum_naming = kUseDbSessionId` (new default), except on SST files generated before this version of RocksDB, which fall back on using `kLegacyCrc32cAndFileSize`.

-## 6.11 (6/12/2020)
+## 6.11 (2020-06-12)
 ### Bug Fixes
 * Fix consistency checking error swallowing in some cases when options.force_consistency_checks = true.
 * Fix possible false NotFound status from batched MultiGet using index type kHashSearch.
@@ -304,7 +364,7 @@
 ### Performance Improvements
 * Eliminate redundant key comparisons during random access in block-based tables.

-## 6.10 (5/2/2020)
+## 6.10 (2020-05-02)
 ### Bug Fixes
 * Fix wrong result being read from ingested file. May happen when a key in the file happen to be prefix of another key also in the file. The issue can further cause more data corruption. The issue exists with rocksdb >= 5.0.0 since DB::IngestExternalFile() was introduced.
 * Finish implementation of BlockBasedTableOptions::IndexType::kBinarySearchWithFirstKey. It's now ready for use. Significantly reduces read amplification in some setups, especially for iterator seeks.
@@ -336,7 +396,7 @@
 * Improve performance of batch MultiGet with partitioned filters, by sharing block cache lookups to applicable filter blocks.
 * Reduced memory copies when fetching and uncompressing compressed blocks from sst files.

-## 6.9.0 (03/29/2020)
+## 6.9.0 (2020-03-29)
 ### Behavior changes
 * Since RocksDB 6.8, ttl-based FIFO compaction can drop a file whose oldest key becomes older than options.ttl while others have not. This fix reverts this and makes ttl-based FIFO compaction use the file's flush time as the criterion. This fix also requires that max_open_files = -1 and compaction_options_fifo.allow_compaction = false to function properly.

@@ -363,7 +423,7 @@
 * Added a new option, best_efforts_recovery (default: false), to allow database to open in a db dir with missing table files. During best efforts recovery, missing table files are ignored, and database recovers to the most recent state without missing table file. Cross-column-family consistency is not guaranteed even if WAL is enabled.
 * options.bottommost_compression, options.compression_opts and options.bottommost_compression_opts are now dynamically changeable.

-## 6.8.0 (02/24/2020)
+## 6.8.0 (2020-02-24)
 ### Java API Changes
 * Major breaking changes to Java comparators, toward standardizing on ByteBuffer for performant, locale-neutral operations on keys (#6252).
 * Added overloads of common API methods using direct ByteBuffers for keys and values (#2283).
@@ -390,7 +450,7 @@
 * `db_bench` now supports `value_size_distribution_type`, `value_size_min`, `value_size_max` options for generating random variable sized value. Added `blob_db_compression_type` option for BlobDB to enable blob compression.
 * Replace RocksDB namespace "rocksdb" with flag "ROCKSDB_NAMESPACE" which if is not defined, defined as "rocksdb" in header file rocksdb_namespace.h.

-## 6.7.0 (01/21/2020)
+## 6.7.0 (2020-01-21)
 ### Public API Change
 * Added a rocksdb::FileSystem class in include/rocksdb/file_system.h to encapsulate file creation/read/write operations, and an option DBOptions::file_system to allow a user to pass in an instance of rocksdb::FileSystem. If its a non-null value, this will take precendence over DBOptions::env for file operations. A new API rocksdb::FileSystem::Default() returns a platform default object. The DBOptions::env option and Env::Default() API will continue to be used for threading and other OS related functions, and where DBOptions::file_system is not specified, for file operations. For storage developers who are accustomed to rocksdb::Env, the interface in rocksdb::FileSystem is new and will probably undergo some changes as more storage systems are ported to it from rocksdb::Env. As of now, no env other than Posix has been ported to the new interface.
 * A new rocksdb::NewSstFileManager() API that allows the caller to pass in separate Env and FileSystem objects.
@@ -415,11 +475,11 @@
 * Introduce ReadOptions.auto_prefix_mode. When set to true, iterator will return the same result as total order seek, but may choose to use prefix seek internally based on seek key and iterator upper bound.
 * MultiGet() can use IO Uring to parallelize read from the same SST file. This featuer is by default disabled. It can be enabled with environment variable ROCKSDB_USE_IO_URING.

-## 6.6.2 (01/13/2020)
+## 6.6.2 (2020-01-13)
 ### Bug Fixes
 * Fixed a bug where non-L0 compaction input files were not considered to compute the `creation_time` of new compaction outputs.

-## 6.6.1 (01/02/2020)
+## 6.6.1 (2020-01-02)
 ### Bug Fixes
 * Fix a bug in WriteBatchWithIndex::MultiGetFromBatchAndDB, which is called by Transaction::MultiGet, that causes due to stale pointer access when the number of keys is > 32
 * Fixed two performance issues related to memtable history trimming. First, a new SuperVersion is now created only if some memtables were actually trimmed. Second, trimming is only scheduled if there is at least one flushed memtable that is kept in memory for the purposes of transaction conflict checking.
@@ -429,7 +489,7 @@
 * Delete superversions in BackgroundCallPurge.
 * Fix use-after-free and double-deleting files in BackgroundCallPurge().

-## 6.6.0 (11/25/2019)
+## 6.6.0 (2019-11-25)
 ### Bug Fixes
 * Fix data corruption caused by output of intra-L0 compaction on ingested file not being placed in correct order in L0.
 * Fix a data race between Version::GetColumnFamilyMetaData() and Compaction::MarkFilesBeingCompacted() for access to being_compacted (#6056). The current fix acquires the db mutex during Version::GetColumnFamilyMetaData(), which may cause regression.
@@ -482,19 +542,19 @@
 * For 64-bit hashing, RocksDB is standardizing on a slightly modified preview version of XXH3. This function is now used for many non-persisted hashes, along with fastrange64() in place of the modulus operator, and some benchmarks show a slight improvement.
 * Level iterator to invlidate the iterator more often in prefix seek and the level is filtered out by prefix bloom.

-## 6.5.2 (11/15/2019)
+## 6.5.2 (2019-11-15)
 ### Bug Fixes
 * Fix a assertion failure in MultiGet() when BlockBasedTableOptions::no_block_cache is true and there is no compressed block cache
 * Fix a buffer overrun problem in BlockBasedTable::MultiGet() when compression is enabled and no compressed block cache is configured.
 * If a call to BackupEngine::PurgeOldBackups or BackupEngine::DeleteBackup suffered a crash, power failure, or I/O error, files could be left over from old backups that could only be purged with a call to GarbageCollect. Any call to PurgeOldBackups, DeleteBackup, or GarbageCollect should now suffice to purge such files.

-## 6.5.1 (10/16/2019)
+## 6.5.1 (2019-10-16)
 ### Bug Fixes
 * Revert the feature "Merging iterator to avoid child iterator reseek for some cases (#5286)" since it might cause strange results when reseek happens with a different iterator upper bound.
 * Fix a bug in BlockBasedTableIterator that might return incorrect results when reseek happens with a different iterator upper bound.
 * Fix a bug when partitioned filters and prefix search are used in conjunction, ::SeekForPrev could return invalid for an existing prefix. ::SeekForPrev might be called by the user, or internally on ::Prev, or within ::Seek if the return value involves Delete or a Merge operand.

-## 6.5.0 (9/13/2019)
+## 6.5.0 (2019-09-13)
 ### Bug Fixes
 * Fixed a number of data races in BlobDB.
 * Fix a bug where the compaction snapshot refresh feature is not disabled as advertised when `snap_refresh_nanos` is set to 0..
@@ -515,7 +575,7 @@
 ### Performance Improvements
 * Improve the speed of the MemTable Bloom filter, reducing the write overhead of enabling it by 1/3 to 1/2, with similar benefit to read performance.

-## 6.4.0 (7/30/2019)
+## 6.4.0 (2019-07-30)
 ### Default Option Change
 * LRUCacheOptions.high_pri_pool_ratio is set to 0.5 (previously 0.0) by default, which means that by default midpoint insertion is enabled. The same change is made for the default value of high_pri_pool_ratio argument in NewLRUCache(). When block cache is not explicitly created, the small block cache created by BlockBasedTable will still has this option to be 0.0.
 * Change BlockBasedTableOptions.cache_index_and_filter_blocks_with_high_priority's default value from false to true.
@@ -551,7 +611,7 @@
 * Fixed a regression where the fill_cache read option also affected index blocks.
 * Fixed an issue where using cache_index_and_filter_blocks==false affected partitions of partitioned indexes/filters as well.

-## 6.3.2 (8/15/2019)
+## 6.3.2 (2019-08-15)
 ### Public API Change
 * The semantics of the per-block-type block read counts in the performance context now match those of the generic block_read_count.

@@ -559,11 +619,11 @@
 * Fixed a regression where the fill_cache read option also affected index blocks.
 * Fixed an issue where using cache_index_and_filter_blocks==false affected partitions of partitioned indexes as well.

-## 6.3.1 (7/24/2019)
+## 6.3.1 (2019-07-24)
 ### Bug Fixes
 * Fix auto rolling bug introduced in 6.3.0, which causes segfault if log file creation fails.

-## 6.3.0 (6/18/2019)
+## 6.3.0 (2019-06-18)
 ### Public API Change
 * Now DB::Close() will return Aborted() error when there is unreleased snapshot. Users can retry after all snapshots are released.
 * Index blocks are now handled similarly to data blocks with regards to the block cache: instead of storing objects in the cache, only the blocks themselves are cached. In addition, index blocks no longer get evicted from the cache when a table is closed, can now use the compressed block cache (if any), and can be shared among multiple table readers.
@@ -600,7 +660,7 @@
 * Fix a bug caused by secondary not skipping the beginning of new MANIFEST.
 * On DB open, delete WAL trash files left behind in wal_dir

-## 6.2.0 (4/30/2019)
+## 6.2.0 (2019-04-30)
 ### New Features
 * Add an option `strict_bytes_per_sync` that causes a file-writing thread to block rather than exceed the limit on bytes pending writeback specified by `bytes_per_sync` or `wal_bytes_per_sync`.
 * Improve range scan performance by avoiding per-key upper bound check in BlockBasedTableIterator.
@@ -622,7 +682,7 @@
 * Close a WAL file before another thread deletes it.
 * Fix an assertion failure `IsFlushPending() == true` caused by one bg thread releasing the db mutex in ~ColumnFamilyData and another thread clearing `flush_requested_` flag.

-## 6.1.1 (4/9/2019)
+## 6.1.1 (2019-04-09)
 ### New Features
 * When reading from option file/string/map, customized comparators and/or merge operators can be filled according to object registry.

@@ -632,7 +692,7 @@
 * Fix a bug in 2PC where a sequence of txn prepare, memtable flush, and crash could result in losing the prepared transaction.
 * Fix a bug in Encryption Env which could cause encrypted files to be read beyond file boundaries.

-## 6.1.0 (3/27/2019)
+## 6.1.0 (2019-03-27)
 ### New Features
 * Introduce two more stats levels, kExceptHistogramOrTimers and kExceptTimers.
 * Added a feature to perform data-block sampling for compressibility, and report stats to user.
@@ -650,7 +710,7 @@
 * Fix JEMALLOC_CXX_THROW macro missing from older Jemalloc versions, causing build failures on some platforms.
 * Fix SstFileReader not able to open file ingested with write_glbal_seqno=true.

-## 6.0.0 (2/19/2019)
+## 6.0.0 (2019-02-19)
 ### New Features
 * Enabled checkpoint on readonly db (DBImplReadOnly).
 * Make DB ignore dropped column families while committing results of atomic flush.
@@ -692,7 +752,7 @@
 ### Change Default Options
 * Change options.compaction_pri's default to kMinOverlappingRatio

-## 5.18.0 (11/30/2018)
+## 5.18.0 (2018-11-30)
 ### New Features
 * Introduced `JemallocNodumpAllocator` memory allocator. When being use, block cache will be excluded from core dump.
 * Introduced `PerfContextByLevel` as part of `PerfContext` which allows storing perf context at each level. Also replaced `__thread` with `thread_local` keyword for perf_context. Added per-level perf context for bloom filter and `Get` query.
@@ -720,7 +780,7 @@
 * Start populating `NO_FILE_CLOSES` ticker statistic, which was always zero previously.
 * The default value of NewBloomFilterPolicy()'s argument use_block_based_builder is changed to false. Note that this new default may cause large temp memory usage when building very large SST files.

-## 5.17.0 (10/05/2018)
+## 5.17.0 (2018-10-05)
 ### Public API Change
 * `OnTableFileCreated` will now be called for empty files generated during compaction. In that case, `TableFileCreationInfo::file_path` will be "(nil)" and `TableFileCreationInfo::file_size` will be zero.
 * Add `FlushOptions::allow_write_stall`, which controls whether Flush calls start working immediately, even if it causes user writes to stall, or will wait until flush can be performed without causing write stall (similar to `CompactRangeOptions::allow_write_stall`). Note that the default value is false, meaning we add delay to Flush calls until stalling can be avoided when possible. This is behavior change compared to previous RocksDB versions, where Flush calls didn't check if they might cause stall or not.
@@ -734,21 +794,21 @@
 * Avoid creating empty SSTs and subsequently deleting them in certain cases during compaction.
 * Sync CURRENT file contents during checkpoint.

-## 5.16.3 (10/1/2018)
+## 5.16.3 (2018-10-01)
 ### Bug Fixes
 * Fix crash caused when `CompactFiles` run with `CompactionOptions::compression == CompressionType::kDisableCompressionOption`. Now that setting causes the compression type to be chosen according to the column family-wide compression options.

-## 5.16.2 (9/21/2018)
+## 5.16.2 (2018-09-21)
 ### Bug Fixes
 * Fix bug in partition filters with format_version=4.

-## 5.16.1 (9/17/2018)
+## 5.16.1 (2018-09-17)
 ### Bug Fixes
 * Remove trace_analyzer_tool from rocksdb_lib target in TARGETS file.
 * Fix RocksDB Java build and tests.
 * Remove sync point in Block destructor.

-## 5.16.0 (8/21/2018)
+## 5.16.0 (2018-08-21)
 ### Public API Change
 * The merge operands are passed to `MergeOperator::ShouldMerge` in the reversed order relative to how they were merged (passed to FullMerge or FullMergeV2) for performance reasons
 * GetAllKeyVersions() to take an extra argument of `max_num_ikeys`.
@@ -762,7 +822,7 @@
 ### Bug Fixes
 * Fix a bug in misreporting the estimated partition index size in properties block.

-## 5.15.0 (7/17/2018)
+## 5.15.0 (2018-07-17)
 ### Public API Change
 * Remove managed iterator. ReadOptions.managed is not effective anymore.
 * For bottommost_compression, a compatible CompressionOptions is added via `bottommost_compression_opts`. To keep backward compatible, a new boolean `enabled` is added to CompressionOptions. For compression_opts, it will be always used no matter what value of `enabled` is. For bottommost_compression_opts, it will only be used when user set `enabled=true`, otherwise, compression_opts will be used for bottommost_compression as default.
@@ -788,7 +848,7 @@
 * Fix a bug caused by not copying the block trailer with compressed SST file, direct IO, prefetcher and no compressed block cache.
 * Fix write can stuck indefinitely if enable_pipelined_write=true. The issue exists since pipelined write was introduced in 5.5.0.

-## 5.14.0 (5/16/2018)
+## 5.14.0 (2018-05-16)
 ### Public API Change
 * Add a BlockBasedTableOption to align uncompressed data blocks on the smaller of block size or page size boundary, to reduce flash reads by avoiding reads spanning 4K pages.
 * The background thread naming convention changed (on supporting platforms) to "rocksdb:<thread pool priority><thread number>", e.g., "rocksdb:low0".
@@ -821,7 +881,7 @@
 * Add `BlockBasedTableConfig.setBlockCache` to allow sharing a block cache across DB instances.
 * Added SstFileManager to the Java API to allow managing SST files across DB instances.

-## 5.13.0 (3/20/2018)
+## 5.13.0 (2018-03-20)
 ### Public API Change
 * RocksDBOptionsParser::Parse()'s `ignore_unknown_options` argument will only be effective if the option file shows it is generated using a higher version of RocksDB than the current version.
 * Remove CompactionEventListener.
@@ -837,7 +897,7 @@
 * Fix a leak in prepared_section_completed_ where the zeroed entries would not removed from the map.
 * Fix WAL corruption caused by race condition between user write thread and backup/checkpoint thread.

-## 5.12.0 (2/14/2018)
+## 5.12.0 (2018-02-14)
 ### Public API Change
 * Iterator::SeekForPrev is now a pure virtual method. This is to prevent user who implement the Iterator interface fail to implement SeekForPrev by mistake.
 * Add `include_end` option to make the range end exclusive when `include_end == false` in `DeleteFilesInRange()`.
@@ -859,7 +919,7 @@
 * Fix advance reservation of arena block addresses.
 * Fix handling of empty string as checkpoint directory.

-## 5.11.0 (01/08/2018)
+## 5.11.0 (2018-01-08)
 ### Public API Change
 * Add `autoTune` and `getBytesPerSecond()` to RocksJava RateLimiter

@@ -876,7 +936,7 @@
 * Fix a mislabel bug for bottom-pri compaction threads.
 * Fix DB::Flush() keep waiting after flush finish under certain condition.

-## 5.10.0 (12/11/2017)
+## 5.10.0 (2017-12-11)
 ### Public API Change
 * When running `make` with environment variable `USE_SSE` set and `PORTABLE` unset, will use all machine features available locally. Previously this combination only compiled SSE-related features.

@@ -891,7 +951,7 @@
 * Fix performance issue in `IngestExternalFile()` affecting databases with large number of SST files.
 * Fix possible corruption to LSM structure when `DeleteFilesInRange()` deletes a subset of files spanned by a `DeleteRange()` marker.

-## 5.9.0 (11/1/2017)
+## 5.9.0 (2017-11-01)
 ### Public API Change
 * `BackupableDBOptions::max_valid_backups_to_open == 0` now means no backups will be opened during BackupEngine initialization. Previously this condition disabled limiting backups opened.
 * `DBOptions::preserve_deletes` is a new option that allows one to specify that DB should not drop tombstones for regular deletes if they have sequence number larger than what was set by the new API call `DB::SetPreserveDeletesSequenceNumber(SequenceNumber seqnum)`. Disabled by default.
@@ -918,7 +978,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Fix a potential data inconsistency issue during point-in-time recovery. `DB:Open()` will abort if column family inconsistency is found during PIT recovery.
 * Fix possible metadata corruption in databases using `DeleteRange()`.

-## 5.8.0 (08/30/2017)
+## 5.8.0 (2017-08-30)
 ### Public API Change
 * Users of `Statistics::getHistogramString()` will see fewer histogram buckets and different bucket endpoints.
 * `Slice::compare` and BytewiseComparator `Compare` no longer accept `Slice`s containing nullptr.
@@ -938,7 +998,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Fix transient reappearance of keys covered by range deletions when memtable prefix bloom filter is enabled.
 * Fix potentially wrong file smallest key when range deletions separated by snapshot are written together.

-## 5.7.0 (07/13/2017)
+## 5.7.0 (2017-07-13)
 ### Public API Change
 * DB property "rocksdb.sstables" now prints keys in hex form.

@@ -953,7 +1013,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 ### Bug Fixes
 * Fix discarding empty compaction output files when `DeleteRange()` is used together with subcompactions.

-## 5.6.0 (06/06/2017)
+## 5.6.0 (2017-06-06)
 ### Public API Change
 * Scheduling flushes and compactions in the same thread pool is no longer supported by setting `max_background_flushes=0`. Instead, users can achieve this by configuring their high-pri thread pool to have zero threads.
 * Replace `Options::max_background_flushes`, `Options::max_background_compactions`, and `Options::base_background_compactions` all with `Options::max_background_jobs`, which automatically decides how many threads to allocate towards flush/compaction.
@@ -970,7 +1030,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 ### Bug Fixes
 * Shouldn't ignore return value of fsync() in flush.

-## 5.5.0 (05/17/2017)
+## 5.5.0 (2017-05-17)
 ### New Features
 * FIFO compaction to support Intra L0 compaction too with CompactionOptionsFIFO.allow_compaction=true.
 * DB::ResetStats() to reset internal stats.
@@ -987,7 +1047,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 ### Bug Fixes
 * Fix the bug that Direct I/O uses direct reads for non-SST file

-## 5.4.0 (04/11/2017)
+## 5.4.0 (2017-04-11)
 ### Public API Change
 * random_access_max_buffer_size no longer has any effect
 * Removed Env::EnableReadAhead(), Env::ShouldForwardRawRequest()
@@ -1004,7 +1064,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Introduce level-based L0->L0 compactions to reduce file count, so write delays are incurred less often.
 * (Experimental) Partitioning filters which creates an index on the partitions. The feature can be enabled by setting partition_filters when using kFullFilter. Currently the feature also requires two-level indexing to be enabled. Number of partitions is the same as the number of partitions for indexes, which is controlled by metadata_block_size.

-## 5.3.0 (03/08/2017)
+## 5.3.0 (2017-03-08)
 ### Public API Change
 * Remove disableDataSync option.
 * Remove timeout_hint_us option from WriteOptions. The option has been deprecated and has no effect since 3.13.0.
@@ -1014,7 +1074,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 ### Bug Fixes
 * Fix the bug that iterator may skip keys

-## 5.2.0 (02/08/2017)
+## 5.2.0 (2017-02-08)
 ### Public API Change
 * NewLRUCache() will determine number of shard bits automatically based on capacity, if the user doesn't pass one. This also impacts the default block cache when the user doesn't explicit provide one.
 * Change the default of delayed slowdown value to 16MB/s and further increase the L0 stop condition to 36 files.
@@ -1032,7 +1092,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Some fixes related to 2PC.
 * Fix bugs of data corruption in direct I/O

-## 5.1.0 (01/13/2017)
+## 5.1.0 (2017-01-13)
 * Support dynamically change `delete_obsolete_files_period_micros` option via SetDBOptions().
 * Added EventListener::OnExternalFileIngested which will be called when IngestExternalFile() add a file successfully.
 * BackupEngine::Open and BackupEngineReadOnly::Open now always return error statuses matching those of the backup Env.
@@ -1041,7 +1101,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Fix the bug that if 2PC is enabled, checkpoints may loss some recent transactions.
 * When file copying is needed when creating checkpoints or bulk loading files, fsync the file after the file copying.

-## 5.0.0 (11/17/2016)
+## 5.0.0 (2016-11-17)
 ### Public API Change
 * Options::max_bytes_for_level_multiplier is now a double along with all getters and setters.
 * Support dynamically change `delayed_write_rate` and `max_total_wal_size` options via SetDBOptions().
@@ -1060,7 +1120,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Add LuaCompactionFilter in utilities.  This allows developers to write compaction filters in Lua.  To use this feature, LUA_PATH needs to be set to the root directory of Lua.
 * No longer populate "LATEST_BACKUP" file in backup directory, which formerly contained the number of the latest backup. The latest backup can be determined by finding the highest numbered file in the "meta/" subdirectory.

-## 4.13.0 (10/18/2016)
+## 4.13.0 (2016-10-18)
 ### Public API Change
 * DB::GetOptions() reflect dynamic changed options (i.e. through DB::SetOptions()) and return copy of options instead of reference.
 * Added Statistics::getAndResetTickerCount().
@@ -1069,7 +1129,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Add DB::SetDBOptions() to dynamic change base_background_compactions and max_background_compactions.
 * Added Iterator::SeekForPrev(). This new API will seek to the last key that less than or equal to the target key.

-## 4.12.0 (9/12/2016)
+## 4.12.0 (2016-09-12)
 ### Public API Change
 * CancelAllBackgroundWork() flushes all memtables for databases containing writes that have bypassed the WAL (writes issued with WriteOptions::disableWAL=true) before shutting down background threads.
 * Merge options source_compaction_factor, max_grandparent_overlap_bytes and expanded_compaction_factor into max_compaction_bytes.
@@ -1081,7 +1141,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Change ticker/histogram statistics implementations to accumulate data in thread-local storage, which improves CPU performance by reducing cache coherency costs. Callers of CreateDBStatistics do not need to change anything to use this feature.
 * Block cache mid-point insertion, where index and filter block are inserted into LRU block cache with higher priority. The feature can be enabled by setting BlockBasedTableOptions::cache_index_and_filter_blocks_with_high_priority to true and high_pri_pool_ratio > 0 when creating NewLRUCache.

-## 4.11.0 (8/1/2016)
+## 4.11.0 (2016-08-01)
 ### Public API Change
 * options.memtable_prefix_bloom_huge_page_tlb_size => memtable_huge_page_size. When it is set, RocksDB will try to allocate memory from huge page for memtable too, rather than just memtable bloom filter.

@@ -1089,7 +1149,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * A tool to migrate DB after options change. See include/rocksdb/utilities/option_change_migration.h.
 * Add ReadOptions.background_purge_on_iterator_cleanup. If true, we avoid file deletion when destroying iterators.

-## 4.10.0 (7/5/2016)
+## 4.10.0 (2016-07-05)
 ### Public API Change
 * options.memtable_prefix_bloom_bits changes to options.memtable_prefix_bloom_bits_ratio and deprecate options.memtable_prefix_bloom_probes
 * enum type CompressionType and PerfLevel changes from char to unsigned char. Value of all PerfLevel shift by one.
@@ -1101,7 +1161,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * RepairDB support for column families. RepairDB now associates data with non-default column families using information embedded in the SST/WAL files (4.7 or later). For data written by 4.6 or earlier, RepairDB associates it with the default column family.
 * Add options.write_buffer_manager which allows users to control total memtable sizes across multiple DB instances.

-## 4.9.0 (6/9/2016)
+## 4.9.0 (2016-06-09)
 ### Public API changes
 * Add bottommost_compression option, This option can be used to set a specific compression algorithm for the bottommost level (Last level containing files in the DB).
 * Introduce CompactionJobInfo::compression, This field state the compression algorithm used to generate the output files of the compaction.
@@ -1111,7 +1171,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 ### New Features
 * Introduce NewSimCache() in rocksdb/utilities/sim_cache.h. This function creates a block cache that is able to give simulation results (mainly hit rate) of simulating block behavior with a configurable cache size.

-## 4.8.0 (5/2/2016)
+## 4.8.0 (2016-05-02)
 ### Public API Change
 * Allow preset compression dictionary for improved compression of block-based tables. This is supported for zlib, zstd, and lz4. The compression dictionary's size is configurable via CompressionOptions::max_dict_bytes.
 * Delete deprecated classes for creating backups (BackupableDB) and restoring from backups (RestoreBackupableDB). Now, BackupEngine should be used for creating backups, and BackupEngineReadOnly should be used for restorations. For more details, see https://github.com/facebook/rocksdb/wiki/How-to-backup-RocksDB%3F
@@ -1121,12 +1181,12 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 ### New Features
 * Add ReadOptions::readahead_size. If non-zero, NewIterator will create a new table reader which performs reads of the given size.

-## 4.7.0 (4/8/2016)
+## 4.7.0 (2016-04-08)
 ### Public API Change
 * rename options compaction_measure_io_stats to report_bg_io_stats and include flush too.
 * Change some default options. Now default options will optimize for server-workloads. Also enable slowdown and full stop triggers for pending compaction bytes. These changes may cause sub-optimal performance or significant increase of resource usage. To avoid these risks, users can open existing RocksDB with options extracted from RocksDB option files. See https://github.com/facebook/rocksdb/wiki/RocksDB-Options-File for how to use RocksDB option files. Or you can call Options.OldDefaults() to recover old defaults. DEFAULT_OPTIONS_HISTORY.md will track change history of default options.

-## 4.6.0 (3/10/2016)
+## 4.6.0 (2016-03-10)
 ### Public API Changes
 * Change default of BlockBasedTableOptions.format_version to 2. It means default DB created by 4.6 or up cannot be opened by RocksDB version 3.9 or earlier.
 * Added strict_capacity_limit option to NewLRUCache. If the flag is set to true, insert to cache will fail if no enough capacity can be free. Signature of Cache::Insert() is updated accordingly.
@@ -1137,7 +1197,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Add CompactionPri::kMinOverlappingRatio, a compaction picking mode friendly to write amplification.
 * Deprecate Iterator::IsKeyPinned() and replace it with Iterator::GetProperty() with prop_name="rocksdb.iterator.is.key.pinned"

-## 4.5.0 (2/5/2016)
+## 4.5.0 (2016-02-05)
 ### Public API Changes
 * Add a new perf context level between kEnableCount and kEnableTime. Level 2 now does not include timers for mutexes.
 * Statistics of mutex operation durations will not be measured by default. If you want to have them enabled, you need to set Statistics::stats_level_ to kAll.
@@ -1148,7 +1208,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Add kPersistedTier to ReadTier.  This option allows Get and MultiGet to read only the persited data and skip mem-tables if writes were done with disableWAL = true.
 * Add DBOptions::sst_file_manager. Use NewSstFileManager() in include/rocksdb/sst_file_manager.h to create a SstFileManager that can be used to track the total size of SST files and control the SST files deletion rate.

-## 4.4.0 (1/14/2016)
+## 4.4.0 (2016-01-14)
 ### Public API Changes
 * Change names in CompactionPri and add a new one.
 * Deprecate options.soft_rate_limit and add options.soft_pending_compaction_bytes_limit.
@@ -1158,7 +1218,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Increase default options.delayed_write_rate to 2MB/s.
 * Added a new parameter --path to ldb tool. --path accepts the name of either MANIFEST, SST or a WAL file. Either --db or --path can be used when calling ldb.

-## 4.3.0 (12/8/2015)
+## 4.3.0 (2015-12-08)
 ### New Features
 * CompactionFilter has new member function called IgnoreSnapshots which allows CompactionFilter to be called even if there are snapshots later than the key.
 * RocksDB will now persist options under the same directory as the RocksDB database on successful DB::Open, CreateColumnFamily, DropColumnFamily, and SetOptions.
@@ -1168,7 +1228,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 ### Public API Changes
 * When options.db_write_buffer_size triggers, only the column family with the largest column family size will be flushed, not all the column families.

-## 4.2.0 (11/9/2015)
+## 4.2.0 (2015-11-09)
 ### New Features
 * Introduce CreateLoggerFromOptions(), this function create a Logger for provided DBOptions.
 * Add GetAggregatedIntProperty(), which returns the sum of the GetIntProperty of all the column families.
@@ -1181,7 +1241,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Remove DefaultCompactionFilterFactory.


-## 4.1.0 (10/8/2015)
+## 4.1.0 (2015-10-08)
 ### New Features
 * Added single delete operation as a more efficient way to delete keys that have not been overwritten.
 * Added experimental AddFile() to DB interface that allow users to add files created by SstFileWriter into an empty Database, see include/rocksdb/sst_file_writer.h and DB::AddFile() for more info.
@@ -1195,7 +1255,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * CompactionFilter has a new method FilterMergeOperand() that RocksDB applies to every merge operand during compaction to decide whether to filter the operand.
 * We removed CompactionFilterV2 interfaces from include/rocksdb/compaction_filter.h. The functionality was deprecated already in version 3.13.

-## 4.0.0 (9/9/2015)
+## 4.0.0 (2015-09-09)
 ### New Features
 * Added support for transactions.  See include/rocksdb/utilities/transaction.h for more info.
 * DB::GetProperty() now accepts "rocksdb.aggregated-table-properties" and "rocksdb.aggregated-table-properties-at-levelN", in which case it returns aggregated table properties of the target column family, or the aggregated table properties of the specified level N if the "at-level" version is used.
@@ -1208,7 +1268,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Added Equal() method to the Comparator interface that can optionally be overwritten in cases where equality comparisons can be done more efficiently than three-way comparisons.
 * Previous 'experimental' OptimisticTransaction class has been replaced by Transaction class.

-## 3.13.0 (8/6/2015)
+## 3.13.0 (2015-08-06)
 ### New Features
 * RollbackToSavePoint() in WriteBatch/WriteBatchWithIndex
 * Add NewCompactOnDeletionCollectorFactory() in utilities/table_properties_collectors, which allows rocksdb to mark a SST file as need-compaction when it observes at least D deletion entries in any N consecutive entries in that SST file.  Note that this feature depends on an experimental NeedCompact() API --- the result of this API will not persist after DB restart.
@@ -1223,7 +1283,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Add statistics::getHistogramString() to print detailed distribution of a histogram metric.
 * Add DBOptions::skip_stats_update_on_db_open.  When it is on, DB::Open() will run faster as it skips the random reads required for loading necessary stats from SST files to optimize compaction.

-## 3.12.0 (7/2/2015)
+## 3.12.0 (2015-07-02)
 ### New Features
 * Added experimental support for optimistic transactions.  See include/rocksdb/utilities/optimistic_transaction.h for more info.
 * Added a new way to report QPS from db_bench (check out --report_file and --report_interval_seconds)
@@ -1253,7 +1313,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Add BackupEngineImpl.options_.max_background_operations to specify the maximum number of operations that may be performed in parallel. Add support for parallelized backup and restore.
 * Add DB::SyncWAL() that does a WAL sync without blocking writers.

-## 3.11.0 (5/19/2015)
+## 3.11.0 (2015-05-19)
 ### New Features
 * Added a new API Cache::SetCapacity(size_t capacity) to dynamically change the maximum configured capacity of the cache. If the new capacity is less than the existing cache usage, the implementation will try to lower the usage by evicting the necessary number of elements following a strict LRU policy.
 * Added an experimental API for handling flashcache devices (blacklists background threads from caching their reads) -- NewFlashcacheAwareEnv
@@ -1264,7 +1324,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * TablePropertiesCollector::AddUserKey() is added to replace TablePropertiesCollector::Add(). AddUserKey() exposes key type, sequence number and file size up to now to users.
 * DBOptions::bytes_per_sync used to apply to both WAL and table files. As of 3.11 it applies only to table files. If you want to use this option to sync WAL in the background, please use wal_bytes_per_sync

-## 3.10.0 (3/24/2015)
+## 3.10.0 (2015-03-24)
 ### New Features
 * GetThreadStatus() is now able to report detailed thread status, including:
 - Thread Operation including flush and compaction.
@@ -1299,7 +1359,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * lz4 compression is now included in rocksjava static library when running `make rocksdbjavastatic`.
 * Overflowing a size_t when setting rocksdb options now throws an IllegalArgumentException, which removes the necessity for a developer to catch these Exceptions explicitly.

-## 3.9.0 (12/8/2014)
+## 3.9.0 (2014-12-08)

 ### New Features
 * Add rocksdb::GetThreadList(), which in the future will return the current status of all
@@ -1318,7 +1378,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 ### Improvements
 * RocksDBLite library now becomes smaller and will be compiled with -fno-exceptions flag.

-## 3.8.0 (11/14/2014)
+## 3.8.0 (2014-11-14)

 ### Public API changes
 * BackupEngine::NewBackupEngine() was deprecated; please use BackupEngine::Open() from now on.
@@ -1332,14 +1392,14 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * CompactFiles and EventListener, although they are still in experimental state
 * Full ColumnFamily support in RocksJava.

-## 3.7.0 (11/6/2014)
+## 3.7.0 (2014-11-06)
 ### Public API changes
 * Introduce SetOptions() API to allow adjusting a subset of options dynamically online
 * Introduce 4 new convenient functions for converting Options from string: GetColumnFamilyOptionsFromMap(), GetColumnFamilyOptionsFromString(), GetDBOptionsFromMap(), GetDBOptionsFromString()
 * Remove WriteBatchWithIndex.Delete() overloads using SliceParts
 * When opening a DB, if options.max_background_compactions is larger than the existing low pri pool of options.env, it will enlarge it. Similarly, options.max_background_flushes is larger than the existing high pri pool of options.env, it will enlarge it.

-## 3.6.0 (10/7/2014)
+## 3.6.0 (2014-10-07)
 ### Disk format changes
 * If you're using RocksDB on ARM platforms and you're using default bloom filter, there is a disk format change you need to be aware of. There are three steps you need to do when you convert to new release: 1. turn off filter policy, 2. compact the whole database, 3. turn on filter policy

@@ -1352,7 +1412,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Change target_file_size_base type to uint64_t from int.
 * Remove allow_thread_local. This feature was proved to be stable, so we are turning it always-on.

-## 3.5.0 (9/3/2014)
+## 3.5.0 (2014-09-03)
 ### New Features
 * Add include/utilities/write_batch_with_index.h, providing a utility class to query data out of WriteBatch when building it.
 * Move BlockBasedTable related options to BlockBasedTableOptions from Options. Change corresponding JNI interface. Options affected include:
@@ -1363,7 +1423,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 ### Public API changes
 * The Prefix Extractor used with V2 compaction filters is now passed user key to SliceTransform::Transform instead of unparsed RocksDB key.

-## 3.4.0 (8/18/2014)
+## 3.4.0 (2014-08-18)
 ### New Features
 * Support Multiple DB paths in universal style compactions
 * Add feature of storing plain table index and bloom filter in SST file.
@@ -1379,7 +1439,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Add DB::GetIntProperty(), which returns DB properties that are integer as uint64_t.
 * The Prefix Extractor used with V2 compaction filters is now passed user key to SliceTransform::Transform instead of unparsed RocksDB key.

-## 3.3.0 (7/10/2014)
+## 3.3.0 (2014-07-10)
 ### New Features
 * Added JSON API prototype.
 * HashLinklist reduces performance outlier caused by skewed bucket by switching data in the bucket from linked list to skip list. Add parameter threshold_use_skiplist in NewHashLinkListRepFactory().
@@ -1390,7 +1450,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 ### Public API changes
 * Removed NewTotalOrderPlainTableFactory because it is not used and implemented semantically incorrect.

-## 3.2.0 (06/20/2014)
+## 3.2.0 (2014-06-20)

 ### Public API changes
 * We removed seek compaction as a concept from RocksDB because:
@@ -1408,7 +1468,7 @@ Because of that, Options::disable_seek_compaction is now obsolete. It is still a
 ### Performance Improvements
 * Tailing Iterator re-implemeted with ForwardIterator + Cascading Search Hint , see ~20% throughput improvement.

-## 3.1.0 (05/21/2014)
+## 3.1.0 (2014-05-21)

 ### Public API changes
 * Replaced ColumnFamilyOptions::table_properties_collectors with ColumnFamilyOptions::table_properties_collector_factories
@@ -1417,7 +1477,7 @@ Because of that, Options::disable_seek_compaction is now obsolete. It is still a
 * Hash index for block-based table will be materialized and reconstructed more efficiently. Previously hash index is constructed by scanning the whole table during every table open.
 * FIFO compaction style

-## 3.0.0 (05/05/2014)
+## 3.0.0 (2014-05-05)

 ### Public API changes
 * Added _LEVEL to all InfoLogLevel enums
@@ -1429,7 +1489,7 @@ Because of that, Options::disable_seek_compaction is now obsolete. It is still a
 * Added an option to use different checksum functions in BlockBasedTableOptions
 * Added ApplyToAllCacheEntries() function to Cache

-## 2.8.0 (04/04/2014)
+## 2.8.0 (2014-04-04)

 * Removed arena.h from public header files.
 * By default, checksums are verified on every read from database
@@ -1458,7 +1518,7 @@ Because of that, Options::disable_seek_compaction is now obsolete. It is still a
 * Now compaction filter has a V2 interface. It buffers the kv-pairs sharing the same key prefix, process them in batches, and return the batched results back to DB. The new interface uses a new structure CompactionFilterContext for the same purpose as CompactionFilter::Context in V1.
 * Geo-spatial support for locations and radial-search.

-## 2.7.0 (01/28/2014)
+## 2.7.0 (2014-01-28)

 ### Public API changes

@@ -497,7 +497,7 @@ ANALYZE_OBJECTS = $(patsubst %.cc, $(OBJ_DIR)/%.o, $(ANALYZER_LIB_SOURCES))
 STRESS_OBJECTS =  $(patsubst %.cc, $(OBJ_DIR)/%.o, $(STRESS_LIB_SOURCES))

 # Exclude build_version.cc -- a generated source file -- from all sources.  Not needed for dependencies
-ALL_SOURCES  = $(LIB_SOURCES) $(TEST_LIB_SOURCES) $(MOCK_LIB_SOURCES) $(GTEST_DIR)/gtest/gtest-all.cc
+ALL_SOURCES  = $(filter-out util/build_version.cc, $(LIB_SOURCES)) $(TEST_LIB_SOURCES) $(MOCK_LIB_SOURCES) $(GTEST_DIR)/gtest/gtest-all.cc
 ALL_SOURCES += $(TOOL_LIB_SOURCES) $(BENCH_LIB_SOURCES) $(CACHE_BENCH_LIB_SOURCES) $(ANALYZER_LIB_SOURCES) $(STRESS_LIB_SOURCES)
 ALL_SOURCES += $(TEST_MAIN_SOURCES) $(TOOL_MAIN_SOURCES) $(BENCH_MAIN_SOURCES)

@@ -516,8 +516,10 @@ endif
 ifdef ASSERT_STATUS_CHECKED
 	# TODO: finish fixing all tests to pass this check
 	TESTS_FAILING_ASC = \
+		c_test \
 		db_test \
 		db_test2 \
+		env_test \
 		range_locking_test \
 		testutil_test \

@@ -1540,6 +1542,9 @@ compaction_job_test: $(OBJ_DIR)/db/compaction/compaction_job_test.o $(TEST_LIBRA
 compaction_job_stats_test: $(OBJ_DIR)/db/compaction/compaction_job_stats_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)

+compaction_service_test: $(OBJ_DIR)/db/compaction/compaction_service_test.o $(TEST_LIBRARY) $(LIBRARY)
+	$(AM_LINK)
+
 compact_on_deletion_collector_test: $(OBJ_DIR)/utilities/table_properties_collectors/compact_on_deletion_collector_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)

@@ -1863,6 +1868,10 @@ db_blob_corruption_test: $(OBJ_DIR)/db/blob/db_blob_corruption_test.o $(TEST_LIB

 db_write_buffer_manager_test: $(OBJ_DIR)/db/db_write_buffer_manager_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
+
+clipping_iterator_test: $(OBJ_DIR)/db/compaction/clipping_iterator_test.o $(TEST_LIBRARY) $(LIBRARY)
+	$(AM_LINK)
+
 #-------------------------------------------------
 # make install related stuff
 PREFIX ?= /usr/local
@@ -10,7 +10,7 @@ load(":defs.bzl", "test_binary")

 REPO_PATH = package_name() + "/"

-ROCKSDB_COMPILER_FLAGS = [
+ROCKSDB_COMPILER_FLAGS_0 = [
    "-fno-builtin-memcmp",
    # Needed to compile in fbcode
    "-Wno-expansion-to-defined",
@@ -28,7 +28,7 @@ ROCKSDB_EXTERNAL_DEPS = [
    ("zstd", None, "zstd"),
 ]

-ROCKSDB_OS_DEPS = [
+ROCKSDB_OS_DEPS_0 = [
    (
        "linux",
        ["third-party//numa:numa", "third-party//liburing:uring", "third-party//tbb:tbb"],
@@ -39,7 +39,7 @@ ROCKSDB_OS_DEPS = [
    ),
 ]

-ROCKSDB_OS_PREPROCESSOR_FLAGS = [
+ROCKSDB_OS_PREPROCESSOR_FLAGS_0 = [
    (
        "linux",
        [
@@ -107,18 +107,18 @@ is_opt_mode = build_mode.startswith("opt")

 # -DNDEBUG is added by default in opt mode in fbcode. But adding it twice
 # doesn't harm and avoid forgetting to add it.
-ROCKSDB_COMPILER_FLAGS += (["-DNDEBUG"] if is_opt_mode else [])
+ROCKSDB_COMPILER_FLAGS = ROCKSDB_COMPILER_FLAGS_0 + (["-DNDEBUG"] if is_opt_mode else [])

 sanitizer = read_config("fbcode", "sanitizer")

 # Do not enable jemalloc if sanitizer presents. RocksDB will further detect
 # whether the binary is linked with jemalloc at runtime.
-ROCKSDB_OS_PREPROCESSOR_FLAGS += ([(
+ROCKSDB_OS_PREPROCESSOR_FLAGS = ROCKSDB_OS_PREPROCESSOR_FLAGS_0 + ([(
    "linux",
    ["-DROCKSDB_JEMALLOC"],
 )] if sanitizer == "" else [])

-ROCKSDB_OS_DEPS += ([(
+ROCKSDB_OS_DEPS = ROCKSDB_OS_DEPS_0 + ([(
    "linux",
    ["third-party//jemalloc:headers"],
 )] if sanitizer == "" else [])
@@ -132,10 +132,12 @@ cpp_library(
    name = "rocksdb_lib",
    srcs = [
        "cache/cache.cc",
+        "cache/cache_entry_roles.cc",
        "cache/clock_cache.cc",
        "cache/lru_cache.cc",
        "cache/sharded_cache.cc",
        "db/arena_wrapped_db_iter.cc",
+        "db/blob/blob_fetcher.cc",
        "db/blob/blob_file_addition.cc",
        "db/blob/blob_file_builder.cc",
        "db/blob/blob_file_cache.cc",
@@ -442,10 +444,12 @@ cpp_library(
    name = "rocksdb_whole_archive_lib",
    srcs = [
        "cache/cache.cc",
+        "cache/cache_entry_roles.cc",
        "cache/clock_cache.cc",
        "cache/lru_cache.cc",
        "cache/sharded_cache.cc",
        "db/arena_wrapped_db_iter.cc",
+        "db/blob/blob_fetcher.cc",
        "db/blob/blob_file_addition.cc",
        "db/blob/blob_file_builder.cc",
        "db/blob/blob_file_cache.cc",
@@ -780,6 +784,7 @@ cpp_library(
        "test_util/testutil.cc",
        "tools/block_cache_analyzer/block_cache_trace_analyzer.cc",
        "tools/db_bench_tool.cc",
+        "tools/simulated_hybrid_file_system.cc",
        "tools/trace_analyzer_tool.cc",
    ],
    auto_headers = AutoHeaders.RECURSIVE_GLOB,
@@ -1048,6 +1053,13 @@ ROCKS_TESTS = [
        [],
        [],
    ],
+    [
+        "clipping_iterator_test",
+        "db/compaction/clipping_iterator_test.cc",
+        "parallel",
+        [],
+        [],
+    ],
    [
        "coding_test",
        "util/coding_test.cc",
@@ -1104,6 +1116,13 @@ ROCKS_TESTS = [
        [],
        [],
    ],
+    [
+        "compaction_service_test",
+        "db/compaction/compaction_service_test.cc",
+        "parallel",
+        [],
+        [],
+    ],
    [
        "comparator_db_test",
        "db/comparator_db_test.cc",
@@ -17,7 +17,7 @@ load(":defs.bzl", "test_binary")

 REPO_PATH = package_name() + "/"

-ROCKSDB_COMPILER_FLAGS = [
+ROCKSDB_COMPILER_FLAGS_0 = [
    "-fno-builtin-memcmp",
    # Needed to compile in fbcode
    "-Wno-expansion-to-defined",
@@ -35,7 +35,7 @@ ROCKSDB_EXTERNAL_DEPS = [
    ("zstd", None, "zstd"),
 ]

-ROCKSDB_OS_DEPS = [
+ROCKSDB_OS_DEPS_0 = [
    (
        "linux",
        ["third-party//numa:numa", "third-party//liburing:uring", "third-party//tbb:tbb"],
@@ -46,7 +46,7 @@ ROCKSDB_OS_DEPS = [
    ),
 ]

-ROCKSDB_OS_PREPROCESSOR_FLAGS = [
+ROCKSDB_OS_PREPROCESSOR_FLAGS_0 = [
    (
        "linux",
        [
@@ -114,18 +114,18 @@ is_opt_mode = build_mode.startswith("opt")

 # -DNDEBUG is added by default in opt mode in fbcode. But adding it twice
 # doesn't harm and avoid forgetting to add it.
-ROCKSDB_COMPILER_FLAGS += (["-DNDEBUG"] if is_opt_mode else [])
+ROCKSDB_COMPILER_FLAGS = ROCKSDB_COMPILER_FLAGS_0 + (["-DNDEBUG"] if is_opt_mode else [])

 sanitizer = read_config("fbcode", "sanitizer")

 # Do not enable jemalloc if sanitizer presents. RocksDB will further detect
 # whether the binary is linked with jemalloc at runtime.
-ROCKSDB_OS_PREPROCESSOR_FLAGS += ([(
+ROCKSDB_OS_PREPROCESSOR_FLAGS = ROCKSDB_OS_PREPROCESSOR_FLAGS_0 + ([(
    "linux",
    ["-DROCKSDB_JEMALLOC"],
 )] if sanitizer == "" else [])

-ROCKSDB_OS_DEPS += ([(
+ROCKSDB_OS_DEPS = ROCKSDB_OS_DEPS_0 + ([(
    "linux",
    ["third-party//jemalloc:headers"],
 )] if sanitizer == "" else [])
@@ -69,10 +69,6 @@ if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
      source "$PWD/build_tools/fbcode_config_platform007.sh"
    elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_PLATFORM009" ]; then
      source "$PWD/build_tools/fbcode_config_platform009.sh"
-    elif [ -z "$USE_CLANG" ]; then
-        # Still use platform007 for gcc by default for build break on
-        # some hosts.
-      source "$PWD/build_tools/fbcode_config_platform007.sh"
    else
      source "$PWD/build_tools/fbcode_config_platform009.sh"
    fi
@@ -175,7 +171,10 @@ case "$TARGET_OS" in
            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic"
        fi
        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt -ldl"
-        if test $ROCKSDB_USE_IO_URING; then
+        if test -z "$ROCKSDB_USE_IO_URING"; then
+            ROCKSDB_USE_IO_URING=1
+        fi
+        if test "$ROCKSDB_USE_IO_URING" -ne 0; then
            # check for liburing
            $CXX $PLATFORM_CXXFLAGS -x c++ - -luring -o /dev/null 2>/dev/null  <<EOF
              #include <liburing.h>
@@ -120,7 +120,7 @@ if [ -z "$USE_CLANG" ]; then
  CXX="$GCC_BASE/bin/g++"
  AR="$GCC_BASE/bin/gcc-ar"

-  CFLAGS+=" -B$BINUTILS/gold"
+  CFLAGS+=" -B$BINUTILS"
  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
  CFLAGS+=" -isystem $GLIBC_INCLUDE"
  JEMALLOC=1
@@ -133,7 +133,7 @@ else

  KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"

-  CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
+  CFLAGS+=" -B$BINUTILS -nostdinc -nostdlib"
  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/9.x "
  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/9.x/x86_64-facebook-linux "
  CFLAGS+=" -isystem $GLIBC_INCLUDE"
@@ -150,10 +150,11 @@ CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PR
 CXXFLAGS+=" $CFLAGS"

 EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS $LIBURING_LIBS"
-EXEC_LDFLAGS+=" -B$BINUTILS/gold"
+EXEC_LDFLAGS+=" -B$BINUTILS"
 EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform009/lib/ld.so"
 EXEC_LDFLAGS+=" $LIBUNWIND"
 EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform009/lib"
+EXEC_LDFLAGS+=" -Wl,-rpath=$GCC_BASE/lib64"
 # required by libtbb
 EXEC_LDFLAGS+=" -ldl"

@@ -106,9 +106,7 @@ NON_SHM="TMPD=/tmp/rocksdb_test_tmp"
 GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1"
 ASAN="COMPILE_WITH_ASAN=1"
 CLANG="USE_CLANG=1"
-# in gcc-5 there are known problems with TSAN like https://gcc.gnu.org/bugzilla/show_bug.cgi?id=71090.
-# using platform007 gives us gcc-8 or higher which has that bug fixed.
-TSAN="ROCKSDB_FBCODE_BUILD_WITH_PLATFORM007=1 COMPILE_WITH_TSAN=1"
+TSAN="COMPILE_WITH_TSAN=1"
 UBSAN="COMPILE_WITH_UBSAN=1"
 TSAN_CRASH='CRASH_TEST_EXT_ARGS="--compression_type=zstd --log2_keys_per_lock=22"'
 NON_TSAN_CRASH="CRASH_TEST_EXT_ARGS=--compression_type=zstd"
@@ -44,9 +44,9 @@ Status Cache::CreateFromString(const ConfigOptions& config_options,
  } else {
 #ifndef ROCKSDB_LITE
    LRUCacheOptions cache_opts;
-    status = OptionTypeInfo::ParseStruct(
-        config_options, "", &lru_cache_options_type_info, "", value,
-        reinterpret_cast<char*>(&cache_opts));
+    status = OptionTypeInfo::ParseStruct(config_options, "",
+                                         &lru_cache_options_type_info, "",
+                                         value, &cache_opts);
    if (status.ok()) {
      cache = NewLRUCache(cache_opts);
    }
@@ -6,7 +6,6 @@
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
-
 #ifndef GFLAGS
 #include <cstdio>
 int main() {
@@ -4,29 +4,28 @@
 //  (found in the LICENSE.Apache file in the root directory).

 #ifdef GFLAGS
-#include "rocksdb/cache_bench_tool.h"
-
-#include <sys/types.h>
-
 #include <cinttypes>
 #include <cstdio>
 #include <limits>
+#include <set>
+#include <sstream>

-#include "options/configurable_helper.h"
+#include "monitoring/histogram.h"
 #include "port/port.h"
 #include "rocksdb/cache.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
+#include "rocksdb/secondary_cache.h"
 #include "rocksdb/system_clock.h"
-#include "rocksdb/tiered_cache.h"
 #include "rocksdb/utilities/object_registry.h"
-#include "rocksdb/utilities/options_type.h"
+#include "table/block_based/cachable_entry.h"
 #include "util/coding.h"
 #include "util/gflags_compat.h"
 #include "util/hash.h"
 #include "util/mutexlock.h"
 #include "util/random.h"
 #include "util/stop_watch.h"
+#include "util/string_util.h"

 using GFLAGS_NAMESPACE::ParseCommandLineFlags;

@@ -41,8 +40,7 @@ DEFINE_uint32(num_shard_bits, 6, "shard_bits.");

 DEFINE_double(resident_ratio, 0.25,
              "Ratio of keys fitting in cache to keyspace.");
-DEFINE_uint64(ops_per_thread, 0,
-              "Number of operations per thread. (Default: 5 * keyspace size)");
+DEFINE_uint64(ops_per_thread, 2000000U, "Number of operations per thread.");
 DEFINE_uint32(value_bytes, 8 * KiB, "Size of each value added.");

 DEFINE_uint32(skew, 5, "Degree of skew in key selection");
@@ -57,19 +55,27 @@ DEFINE_uint32(lookup_percent, 10,
              "Ratio of lookup to total workload (expressed as a percentage)");
 DEFINE_uint32(erase_percent, 1,
              "Ratio of erase to total workload (expressed as a percentage)");
+DEFINE_bool(gather_stats, false,
+            "Whether to periodically simulate gathering block cache stats, "
+            "using one more thread.");
+DEFINE_uint32(
+    gather_stats_sleep_ms, 1000,
+    "How many milliseconds to sleep between each gathering of stats.");
+
+DEFINE_uint32(gather_stats_entries_per_lock, 256,
+              "For Cache::ApplyToAllEntries");
+DEFINE_bool(skewed, false, "If true, skew the key access distribution");
+#ifndef ROCKSDB_LITE
+DEFINE_string(secondary_cache_uri, "",
+              "Full URI for creating a custom secondary cache object");
+static class std::shared_ptr<ROCKSDB_NAMESPACE::SecondaryCache> secondary_cache;
+#endif  // ROCKSDB_LITE

 DEFINE_bool(use_clock_cache, false, "");

-DEFINE_bool(skewed, false, "If true, skew the key access distribution");
-#ifndef ROCKSDB_LITE
-DEFINE_string(tiered_cache_uri, "",
-              "Full URI for creating a custom NVM cache object");
-static class std::shared_ptr<ROCKSDB_NAMESPACE::TieredCache> tiered_cache;
-#endif
-
 namespace ROCKSDB_NAMESPACE {
-class CacheBench;

+class CacheBench;
 namespace {
 // State shared by all concurrent executions of the same benchmark.
 class SharedState {
@@ -112,40 +118,13 @@ class SharedState {
  CacheBench* cache_bench_;
 };

-class Stats {
- private:
-  uint64_t hits;
-  uint64_t misses;
-  uint64_t lookup_us;
-  uint64_t insert_us;
-
- public:
-  Stats() : hits(0), misses(0), lookup_us(0), insert_us(0) {}
-
-  void AddHits(uint64_t inc) { hits += inc; }
-  void AddMisses(uint64_t inc) { misses += inc; }
-  void AddLookupUs(uint64_t lookup) { lookup_us += lookup; }
-  void AddInsertUs(uint64_t insert) { insert_us += insert; }
-  void Merge(const Stats& other) {
-    hits += other.hits;
-    misses += other.misses;
-    lookup_us += other.lookup_us;
-    insert_us += other.insert_us;
-  }
-  void Report() {
-    fprintf(stdout, "%llu hits, %llu misses, %llu lookup_us, %llu insert_us\n",
-            (unsigned long long)hits, (unsigned long long)misses,
-            (unsigned long long)lookup_us, (unsigned long long)insert_us);
-    fflush(stdout);
-  }
-};
-
 // Per-thread state for concurrent executions of the same benchmark.
 struct ThreadState {
  uint32_t tid;
  Random64 rnd;
-  Stats stats;
  SharedState* shared;
+  HistogramImpl latency_ns_hist;
+  uint64_t duration_us = 0;

  ThreadState(uint32_t index, SharedState* _shared)
      : tid(index), rnd(1000 + index), shared(_shared) {}
@@ -190,25 +169,29 @@ char* createValue(Random64& rnd) {
  return rv;
 }

-void helperCallback(Cache::SizeCallback* size_cb,
-                    Cache::SaveToCallback* save_cb,
-                    Cache::DeletionCallback* del_cb) {
-  if (size_cb) {
-    *size_cb = [](void* /*obj*/) -> size_t { return FLAGS_value_bytes; };
-  }
-  if (save_cb) {
-    *save_cb = [](void* obj, size_t /*offset*/, size_t size,
-                  void* out) -> Status {
-      memcpy(out, obj, size);
-      return Status::OK();
-    };
-  }
-  if (del_cb) {
-    *del_cb = [](const Slice& /*key*/, void* obj) -> void {
-      delete[] static_cast<char*>(obj);
-    };
-  }
+// Callbacks for secondary cache
+size_t SizeFn(void* /*obj*/) { return FLAGS_value_bytes; }
+
+Status SaveToFn(void* obj, size_t /*offset*/, size_t size, void* out) {
+  memcpy(out, obj, size);
+  return Status::OK();
 }
+
+// Different deleters to simulate using deleter to gather
+// stats on the code origin and kind of cache entries.
+void deleter1(const Slice& /*key*/, void* value) {
+  delete[] static_cast<char*>(value);
+}
+void deleter2(const Slice& /*key*/, void* value) {
+  delete[] static_cast<char*>(value);
+}
+void deleter3(const Slice& /*key*/, void* value) {
+  delete[] static_cast<char*>(value);
+}
+
+Cache::CacheItemHelper helper1(SizeFn, SaveToFn, deleter1);
+Cache::CacheItemHelper helper2(SizeFn, SaveToFn, deleter2);
+Cache::CacheItemHelper helper3(SizeFn, SaveToFn, deleter3);
 }  // namespace

 class CacheBench {
@@ -249,24 +232,23 @@ class CacheBench {
    } else {
      LRUCacheOptions opts(FLAGS_cache_size, FLAGS_num_shard_bits, false, 0.5);
 #ifndef ROCKSDB_LITE
-      if (!FLAGS_tiered_cache_uri.empty()) {
-        Status s = ObjectRegistry::NewInstance()->NewSharedObject<TieredCache>(
-            FLAGS_tiered_cache_uri, &tiered_cache);
-        if (tiered_cache == nullptr) {
-          fprintf(stderr,
-                  "No tiered cache registered matching string: %s status=%s\n",
-                  FLAGS_tiered_cache_uri.c_str(), s.ToString().c_str());
+      if (!FLAGS_secondary_cache_uri.empty()) {
+        Status s =
+            ObjectRegistry::NewInstance()->NewSharedObject<SecondaryCache>(
+                FLAGS_secondary_cache_uri, &secondary_cache);
+        if (secondary_cache == nullptr) {
+          fprintf(
+              stderr,
+              "No secondary cache registered matching string: %s status=%s\n",
+              FLAGS_secondary_cache_uri.c_str(), s.ToString().c_str());
          exit(1);
        }
-        opts.tiered_cache = tiered_cache;
+        opts.secondary_cache = secondary_cache;
      }
-#endif
+#endif  // ROCKSDB_LITE

      cache_ = NewLRUCache(opts);
    }
-    if (FLAGS_ops_per_thread == 0) {
-      FLAGS_ops_per_thread = 5 * max_key_;
-    }
  }

  ~CacheBench() {}
@@ -274,30 +256,35 @@ class CacheBench {
  void PopulateCache() {
    Random64 rnd(1);
    KeyGen keygen;
-    for (uint64_t i = 0; i < 10 * FLAGS_cache_size; i += FLAGS_value_bytes) {
+    for (uint64_t i = 0; i < 2 * FLAGS_cache_size; i += FLAGS_value_bytes) {
      cache_->Insert(keygen.GetRand(rnd, max_key_, max_log_), createValue(rnd),
-                     helperCallback, FLAGS_value_bytes);
+                     &helper1, FLAGS_value_bytes);
    }
  }

  bool Run() {
-    ROCKSDB_NAMESPACE::Env* env = ROCKSDB_NAMESPACE::Env::Default();
-    const auto& clock = env->GetSystemClock();
+    const auto clock = SystemClock::Default().get();

    PrintEnv();
    SharedState shared(this);
    std::vector<std::unique_ptr<ThreadState> > threads(FLAGS_threads);
    for (uint32_t i = 0; i < FLAGS_threads; i++) {
      threads[i].reset(new ThreadState(i, &shared));
-      env->StartThread(ThreadBody, threads[i].get());
+      std::thread(ThreadBody, threads[i].get()).detach();
    }
+
+    HistogramImpl stats_hist;
+    std::string stats_report;
+    std::thread stats_thread(StatsBody, &shared, &stats_hist, &stats_report);
+
+    uint64_t start_time;
    {
      MutexLock l(shared.GetMutex());
      while (!shared.AllInitialized()) {
        shared.GetCondVar()->Wait();
      }
      // Record start time
-      uint64_t start_time = clock->NowMicros();
+      start_time = clock->NowMicros();

      // Start all threads
      shared.SetStart();
@@ -307,20 +294,44 @@ class CacheBench {
      while (!shared.AllDone()) {
        shared.GetCondVar()->Wait();
      }
-
-      // Record end time
-      uint64_t end_time = clock->NowMicros();
-      double elapsed = static_cast<double>(end_time - start_time) * 1e-6;
-      uint32_t qps = static_cast<uint32_t>(
-          static_cast<double>(FLAGS_threads * FLAGS_ops_per_thread) / elapsed);
-      fprintf(stdout, "Complete in %.3f s; QPS = %u\n", elapsed, qps);
-
-      Stats merge_stats;
-      for (uint32_t i = 0; i < FLAGS_threads; ++i) {
-        merge_stats.Merge(threads[i]->stats);
-      }
-      merge_stats.Report();
    }
+
+    // Stats gathering is considered background work. This time measurement
+    // is for foreground work, and not really ideal for that. See below.
+    uint64_t end_time = clock->NowMicros();
+    stats_thread.join();
+
+    // Wall clock time - includes idle time if threads
+    // finish at different times (not ideal).
+    double elapsed_secs = static_cast<double>(end_time - start_time) * 1e-6;
+    uint32_t ops_per_sec = static_cast<uint32_t>(
+        1.0 * FLAGS_threads * FLAGS_ops_per_thread / elapsed_secs);
+    printf("Complete in %.3f s; Rough parallel ops/sec = %u\n", elapsed_secs,
+           ops_per_sec);
+
+    // Total time in each thread (more accurate throughput measure)
+    elapsed_secs = 0;
+    for (uint32_t i = 0; i < FLAGS_threads; i++) {
+      elapsed_secs += threads[i]->duration_us * 1e-6;
+    }
+    ops_per_sec = static_cast<uint32_t>(1.0 * FLAGS_threads *
+                                        FLAGS_ops_per_thread / elapsed_secs);
+    printf("Thread ops/sec = %u\n", ops_per_sec);
+
+    printf("\nOperation latency (ns):\n");
+    HistogramImpl combined;
+    for (uint32_t i = 0; i < FLAGS_threads; i++) {
+      combined.Merge(threads[i]->latency_ns_hist);
+    }
+    printf("%s", combined.ToString().c_str());
+
+    if (FLAGS_gather_stats) {
+      printf("\nGather stats latency (us):\n");
+      printf("%s", stats_hist.ToString().c_str());
+    }
+
+    printf("\n%s", stats_report.c_str());
+
    return true;
  }

@@ -335,8 +346,77 @@ class CacheBench {
  const bool skewed_;
  int max_log_;

-  static void ThreadBody(void* v) {
-    ThreadState* thread = static_cast<ThreadState*>(v);
+  // A benchmark version of gathering stats on an active block cache by
+  // iterating over it. The primary purpose is to measure the impact of
+  // gathering stats with ApplyToAllEntries on throughput- and
+  // latency-sensitive Cache users. Performance of stats gathering is
+  // also reported. The last set of gathered stats is also reported, for
+  // manual sanity checking for logical errors or other unexpected
+  // behavior of cache_bench or the underlying Cache.
+  static void StatsBody(SharedState* shared, HistogramImpl* stats_hist,
+                        std::string* stats_report) {
+    if (!FLAGS_gather_stats) {
+      return;
+    }
+    const auto clock = SystemClock::Default().get();
+    uint64_t total_key_size = 0;
+    uint64_t total_charge = 0;
+    uint64_t total_entry_count = 0;
+    std::set<Cache::DeleterFn> deleters;
+    StopWatchNano timer(clock);
+
+    for (;;) {
+      uint64_t time;
+      time = clock->NowMicros();
+      uint64_t deadline = time + uint64_t{FLAGS_gather_stats_sleep_ms} * 1000;
+
+      {
+        MutexLock l(shared->GetMutex());
+        for (;;) {
+          if (shared->AllDone()) {
+            std::ostringstream ostr;
+            ostr << "Most recent cache entry stats:\n"
+                 << "Number of entries: " << total_entry_count << "\n"
+                 << "Total charge: " << BytesToHumanString(total_charge) << "\n"
+                 << "Average key size: "
+                 << (1.0 * total_key_size / total_entry_count) << "\n"
+                 << "Average charge: "
+                 << BytesToHumanString(1.0 * total_charge / total_entry_count)
+                 << "\n"
+                 << "Unique deleters: " << deleters.size() << "\n";
+            *stats_report = ostr.str();
+            return;
+          }
+          if (clock->NowMicros() >= deadline) {
+            break;
+          }
+          uint64_t diff = deadline - std::min(clock->NowMicros(), deadline);
+          shared->GetCondVar()->TimedWait(diff + 1);
+        }
+      }
+
+      // Now gather stats, outside of mutex
+      total_key_size = 0;
+      total_charge = 0;
+      total_entry_count = 0;
+      deleters.clear();
+      auto fn = [&](const Slice& key, void* /*value*/, size_t charge,
+                    Cache::DeleterFn deleter) {
+        total_key_size += key.size();
+        total_charge += charge;
+        ++total_entry_count;
+        // Something slightly more expensive as in (future) stats by category
+        deleters.insert(deleter);
+      };
+      timer.Start();
+      Cache::ApplyToAllEntriesOptions opts;
+      opts.average_entries_per_lock = FLAGS_gather_stats_entries_per_lock;
+      shared->GetCacheBench()->cache_->ApplyToAllEntries(fn, opts);
+      stats_hist->Add(timer.ElapsedNanos() / 1000);
+    }
+  }
+
+  static void ThreadBody(ThreadState* thread) {
    SharedState* shared = thread->shared;

    {
@@ -366,9 +446,12 @@ class CacheBench {
    // To hold handles for a non-trivial amount of time
    Cache::Handle* handle = nullptr;
    KeyGen gen;
-    ROCKSDB_NAMESPACE::Env* env = ROCKSDB_NAMESPACE::Env::Default();
-    SystemClock* clock = env->GetSystemClock().get();
+    const auto clock = SystemClock::Default().get();
+    uint64_t start_time = clock->NowMicros();
+    StopWatchNano timer(clock);
+
    for (uint64_t i = 0; i < FLAGS_ops_per_thread; i++) {
+      timer.Start();
      Slice key = gen.GetRand(thread->rnd, max_key_, max_log_);
      uint64_t random_op = thread->rnd.Next();
      Cache::CreateCallback create_cb =
@@ -385,32 +468,16 @@ class CacheBench {
          handle = nullptr;
        }
        // do lookup
-        {
-          uint64_t elapsed = 0;
-          {
-            StopWatch sw(clock, nullptr, 0, &elapsed);
-            handle = cache_->Lookup(key, helperCallback, create_cb,
-                                    Cache::Priority::LOW, true);
-          }
-          thread->stats.AddLookupUs(elapsed);
-        }
+        handle = cache_->Lookup(key, &helper2, create_cb, Cache::Priority::LOW,
+                                true);
        if (handle) {
-          thread->stats.AddHits(1);
          // do something with the data
          result += NPHash64(static_cast<char*>(cache_->Value(handle)),
                             FLAGS_value_bytes);
        } else {
-          thread->stats.AddMisses(1);
          // do insert
-          {
-            uint64_t elapsed = 0;
-            {
-              StopWatch sw(clock, nullptr, 0, &elapsed);
-              cache_->Insert(key, createValue(thread->rnd), helperCallback,
-                             FLAGS_value_bytes, &handle);
-            }
-            thread->stats.AddInsertUs(elapsed);
-          }
+          cache_->Insert(key, createValue(thread->rnd), &helper2,
+                         FLAGS_value_bytes, &handle);
        }
      } else if (random_op < insert_threshold_) {
        if (handle) {
@@ -418,32 +485,17 @@ class CacheBench {
          handle = nullptr;
        }
        // do insert
-        {
-          uint64_t elapsed = 0;
-          {
-            StopWatch sw(clock, nullptr, 0, &elapsed);
-            cache_->Insert(key, createValue(thread->rnd), helperCallback,
-                           FLAGS_value_bytes, &handle);
-          }
-          thread->stats.AddInsertUs(elapsed);
-        }
+        cache_->Insert(key, createValue(thread->rnd), &helper3,
+                       FLAGS_value_bytes, &handle);
      } else if (random_op < lookup_threshold_) {
        if (handle) {
          cache_->Release(handle);
          handle = nullptr;
        }
        // do lookup
-        {
-          uint64_t elapsed = 0;
-          {
-            StopWatch sw(clock, nullptr, 0, &elapsed);
-            handle = cache_->Lookup(key, helperCallback, create_cb,
-                                    Cache::Priority::LOW, true);
-          }
-          thread->stats.AddLookupUs(elapsed);
-        }
+        handle = cache_->Lookup(key, &helper2, create_cb, Cache::Priority::LOW,
+                                true);
        if (handle) {
-          thread->stats.AddHits(1);
          // do something with the data
          result += NPHash64(static_cast<char*>(cache_->Value(handle)),
                             FLAGS_value_bytes);
@@ -455,18 +507,26 @@ class CacheBench {
        // Should be extremely unlikely (noop)
        assert(random_op >= kHundredthUint64 * 100U);
      }
+      thread->latency_ns_hist.Add(timer.ElapsedNanos());
    }
    if (handle) {
      cache_->Release(handle);
      handle = nullptr;
    }
+    // Ensure computations on `result` are not optimized away.
+    if (result == 1) {
+      printf("You are extremely unlucky(2). Try again.\n");
+      exit(1);
+    }
+    thread->duration_us = clock->NowMicros() - start_time;
  }

  void PrintEnv() const {
    printf("RocksDB version     : %d.%d\n", kMajorVersion, kMinorVersion);
    printf("Number of threads   : %u\n", FLAGS_threads);
    printf("Ops per thread      : %" PRIu64 "\n", FLAGS_ops_per_thread);
-    printf("Cache size          : %" PRIu64 "\n", FLAGS_cache_size);
+    printf("Cache size          : %s\n",
+           BytesToHumanString(FLAGS_cache_size).c_str());
    printf("Num shard bits      : %u\n", FLAGS_num_shard_bits);
    printf("Max key             : %" PRIu64 "\n", max_key_);
    printf("Resident ratio      : %g\n", FLAGS_resident_ratio);
@@ -476,6 +536,14 @@ class CacheBench {
    printf("Insert percentage   : %u%%\n", FLAGS_insert_percent);
    printf("Lookup percentage   : %u%%\n", FLAGS_lookup_percent);
    printf("Erase percentage    : %u%%\n", FLAGS_erase_percent);
+    std::ostringstream stats;
+    if (FLAGS_gather_stats) {
+      stats << "enabled (" << FLAGS_gather_stats_sleep_ms << "ms, "
+            << FLAGS_gather_stats_entries_per_lock << "/lock)";
+    } else {
+      stats << "disabled";
+    }
+    printf("Gather stats        : %s\n", stats.str().c_str());
    printf("----------------------------\n");
  }
 };
@@ -499,6 +567,7 @@ int cache_bench_tool(int argc, char** argv) {
  } else {
    return 1;
  }
-}
 }  // namespace ROCKSDB_NAMESPACE
-#endif
+}  // namespace ROCKSDB_NAMESPACE
+
+#endif  // GFLAGS
@@ -0,0 +1,66 @@
+//  Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#include "cache/cache_entry_roles.h"
+
+#include <mutex>
+
+#include "port/lang.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+std::array<const char*, kNumCacheEntryRoles> kCacheEntryRoleToCamelString{{
+    "DataBlock",
+    "FilterBlock",
+    "FilterMetaBlock",
+    "DeprecatedFilterBlock",
+    "IndexBlock",
+    "OtherBlock",
+    "WriteBuffer",
+    "Misc",
+}};
+
+std::array<const char*, kNumCacheEntryRoles> kCacheEntryRoleToHyphenString{{
+    "data-block",
+    "filter-block",
+    "filter-meta-block",
+    "deprecated-filter-block",
+    "index-block",
+    "other-block",
+    "write-buffer",
+    "misc",
+}};
+
+namespace {
+
+struct Registry {
+  std::mutex mutex;
+  std::unordered_map<Cache::DeleterFn, CacheEntryRole> role_map;
+  void Register(Cache::DeleterFn fn, CacheEntryRole role) {
+    std::lock_guard<std::mutex> lock(mutex);
+    role_map[fn] = role;
+  }
+  std::unordered_map<Cache::DeleterFn, CacheEntryRole> Copy() {
+    std::lock_guard<std::mutex> lock(mutex);
+    return role_map;
+  }
+};
+
+Registry& GetRegistry() {
+  STATIC_AVOID_DESTRUCTION(Registry, registry);
+  return registry;
+}
+
+}  // namespace
+
+void RegisterCacheDeleterRole(Cache::DeleterFn fn, CacheEntryRole role) {
+  GetRegistry().Register(fn, role);
+}
+
+std::unordered_map<Cache::DeleterFn, CacheEntryRole> CopyCacheDeleterRoleMap() {
+  return GetRegistry().Copy();
+}
+
+}  // namespace ROCKSDB_NAMESPACE
@@ -0,0 +1,122 @@
+//  Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include <array>
+#include <cstdint>
+#include <unordered_map>
+
+#include "rocksdb/cache.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+// Classifications of block cache entries, for reporting statistics
+enum class CacheEntryRole {
+  // Block-based table data block
+  kDataBlock,
+  // Block-based table filter block (full or partitioned)
+  kFilterBlock,
+  // Block-based table metadata block for partitioned filter
+  kFilterMetaBlock,
+  // Block-based table deprecated filter block (old "block-based" filter)
+  kDeprecatedFilterBlock,
+  // Block-based table index block
+  kIndexBlock,
+  // Other kinds of block-based table block
+  kOtherBlock,
+  // WriteBufferManager reservations to account for memtable usage
+  kWriteBuffer,
+  // Default bucket, for miscellaneous cache entries. Do not use for
+  // entries that could potentially add up to large usage.
+  kMisc,
+};
+constexpr uint32_t kNumCacheEntryRoles =
+    static_cast<uint32_t>(CacheEntryRole::kMisc) + 1;
+
+extern std::array<const char*, kNumCacheEntryRoles>
+    kCacheEntryRoleToCamelString;
+extern std::array<const char*, kNumCacheEntryRoles>
+    kCacheEntryRoleToHyphenString;
+
+// To associate cache entries with their role, we use a hack on the
+// existing Cache interface. Because the deleter of an entry can authenticate
+// the code origin of an entry, we can elaborate the choice of deleter to
+// also encode role information, without inferring false role information
+// from entries not choosing to encode a role.
+//
+// The rest of this file is for handling mappings between deleters and
+// roles.
+
+// To infer a role from a deleter, the deleter must be registered. This
+// can be done "manually" with this function. This function is thread-safe,
+// and the registration mappings go into private but static storage. (Note
+// that DeleterFn is a function pointer, not std::function. Registrations
+// should not be too many.)
+void RegisterCacheDeleterRole(Cache::DeleterFn fn, CacheEntryRole role);
+
+// Gets a copy of the registered deleter -> role mappings. This is the only
+// function for reading the mappings made with RegisterCacheDeleterRole.
+// Why only this interface for reading?
+// * This function has to be thread safe, which could incur substantial
+// overhead. We should not pay this overhead for every deleter look-up.
+// * This is suitable for preparing for batch operations, like with
+// CacheEntryStatsCollector.
+// * The number of mappings should be sufficiently small (dozens).
+std::unordered_map<Cache::DeleterFn, CacheEntryRole> CopyCacheDeleterRoleMap();
+
+// ************************************************************** //
+// An automatic registration infrastructure. This enables code
+// to simply ask for a deleter associated with a particular type
+// and role, and registration is automatic. In a sense, this is
+// a small dependency injection infrastructure, because linking
+// in new deleter instantiations is essentially sufficient for
+// making stats collection (using CopyCacheDeleterRoleMap) aware
+// of them.
+
+namespace cache_entry_roles_detail {
+
+template <typename T, CacheEntryRole R>
+struct RegisteredDeleter {
+  RegisteredDeleter() { RegisterCacheDeleterRole(Delete, R); }
+
+  // These have global linkage to help ensure compiler optimizations do not
+  // break uniqueness for each <T,R>
+  static void Delete(const Slice& /* key */, void* value) {
+    delete static_cast<T*>(value);
+  }
+};
+
+template <CacheEntryRole R>
+struct RegisteredNoopDeleter {
+  RegisteredNoopDeleter() { RegisterCacheDeleterRole(Delete, R); }
+
+  static void Delete(const Slice& /* key */, void* value) {
+    (void)value;
+    assert(value == nullptr);
+  }
+};
+
+}  // namespace cache_entry_roles_detail
+
+// Get an automatically registered deleter for value type T and role R.
+// Based on C++ semantics, registration is invoked exactly once in a
+// thread-safe way on first call to this function, for each <T, R>.
+template <typename T, CacheEntryRole R>
+Cache::DeleterFn GetCacheEntryDeleterForRole() {
+  static cache_entry_roles_detail::RegisteredDeleter<T, R> reg;
+  return reg.Delete;
+}
+
+// Get an automatically registered no-op deleter (value should be nullptr)
+// and associated with role R. This is used for Cache "reservation" entries
+// such as for WriteBufferManager.
+template <CacheEntryRole R>
+Cache::DeleterFn GetNoopDeleterForRole() {
+  static cache_entry_roles_detail::RegisteredNoopDeleter<R> reg;
+  return reg.Delete;
+}
+
+}  // namespace ROCKSDB_NAMESPACE
@@ -0,0 +1,180 @@
+//  Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include <array>
+#include <cstdint>
+#include <memory>
+#include <mutex>
+
+#include "cache/cache_helpers.h"
+#include "port/lang.h"
+#include "rocksdb/cache.h"
+#include "rocksdb/status.h"
+#include "rocksdb/system_clock.h"
+#include "util/coding_lean.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+// A generic helper object for gathering stats about cache entries by
+// iterating over them with ApplyToAllEntries. This class essentially
+// solves the problem of slowing down a Cache with too many stats
+// collectors that could be sharing stat results, such as from multiple
+// column families or multiple DBs sharing a Cache. We employ a few
+// mitigations:
+// * Only one collector for a particular kind of Stats is alive
+// for each Cache. This is guaranteed using the Cache itself to hold
+// the collector.
+// * A mutex ensures only one thread is gathering stats for this
+// collector.
+// * The most recent gathered stats are saved and simply copied to
+// satisfy requests within a time window (default: 3 minutes) of
+// completion of the most recent stat gathering.
+//
+// Template parameter Stats must be copyable and trivially constructable,
+// as well as...
+// concept Stats {
+//   // Notification before applying callback to all entries
+//   void BeginCollection(Cache*, SystemClock*, uint64_t start_time_micros);
+//   // Get the callback to apply to all entries. `callback`
+//   // type must be compatible with Cache::ApplyToAllEntries
+//   callback GetEntryCallback();
+//   // Notification after applying callback to all entries
+//   void EndCollection(Cache*, SystemClock*, uint64_t end_time_micros);
+//   // Notification that a collection was skipped because of
+//   // sufficiently recent saved results.
+//   void SkippedCollection();
+// }
+template <class Stats>
+class CacheEntryStatsCollector {
+ public:
+  // Gather and save stats if saved stats are too old. (Use GetStats() to
+  // read saved stats.)
+  //
+  // Maximum allowed age for a "hit" on saved results is determined by the
+  // two interval parameters. Both set to 0 forces a re-scan. For example
+  // with min_interval_seconds=300 and min_interval_factor=100, if the last
+  // scan took 10s, we would only rescan ("miss") if the age in seconds of
+  // the saved results is > max(300, 100*10).
+  // Justification: scans can vary wildly in duration, e.g. from 0.02 sec
+  // to as much as 20 seconds, so we want to be able to cap the absolute
+  // and relative frequency of scans.
+  void CollectStats(int min_interval_seconds, int min_interval_factor) {
+    // Waits for any pending reader or writer (collector)
+    std::lock_guard<std::mutex> lock(working_mutex_);
+
+    uint64_t max_age_micros =
+        static_cast<uint64_t>(std::max(min_interval_seconds, 0)) * 1000000U;
+
+    if (last_end_time_micros_ > last_start_time_micros_ &&
+        min_interval_factor > 0) {
+      max_age_micros = std::max(
+          max_age_micros, min_interval_factor * (last_end_time_micros_ -
+                                                 last_start_time_micros_));
+    }
+
+    uint64_t start_time_micros = clock_->NowMicros();
+    if ((start_time_micros - last_end_time_micros_) > max_age_micros) {
+      last_start_time_micros_ = start_time_micros;
+      working_stats_.BeginCollection(cache_, clock_, start_time_micros);
+
+      cache_->ApplyToAllEntries(working_stats_.GetEntryCallback(), {});
+      TEST_SYNC_POINT_CALLBACK(
+          "CacheEntryStatsCollector::GetStats:AfterApplyToAllEntries", nullptr);
+
+      uint64_t end_time_micros = clock_->NowMicros();
+      last_end_time_micros_ = end_time_micros;
+      working_stats_.EndCollection(cache_, clock_, end_time_micros);
+    } else {
+      working_stats_.SkippedCollection();
+    }
+
+    // Save so that we don't need to wait for an outstanding collection in
+    // order to make of copy of the last saved stats
+    std::lock_guard<std::mutex> lock2(saved_mutex_);
+    saved_stats_ = working_stats_;
+  }
+
+  // Gets saved stats, regardless of age
+  void GetStats(Stats *stats) {
+    std::lock_guard<std::mutex> lock(saved_mutex_);
+    *stats = saved_stats_;
+  }
+
+  Cache *GetCache() const { return cache_; }
+
+  // Gets or creates a shared instance of CacheEntryStatsCollector in the
+  // cache itself, and saves into `ptr`. This shared_ptr will hold the
+  // entry in cache until all refs are destroyed.
+  static Status GetShared(Cache *cache, SystemClock *clock,
+                          std::shared_ptr<CacheEntryStatsCollector> *ptr) {
+    std::array<uint64_t, 3> cache_key_data{
+        {// First 16 bytes == md5 of class name
+         0x7eba5a8fb5437c90U, 0x8ca68c9b11655855U,
+         // Last 8 bytes based on a function pointer to make unique for each
+         // template instantiation
+         reinterpret_cast<uint64_t>(&CacheEntryStatsCollector::GetShared)}};
+    Slice cache_key = GetSlice(&cache_key_data);
+
+    Cache::Handle *h = cache->Lookup(cache_key);
+    if (h == nullptr) {
+      // Not yet in cache, but Cache doesn't provide a built-in way to
+      // avoid racing insert. So we double-check under a shared mutex,
+      // inspired by TableCache.
+      STATIC_AVOID_DESTRUCTION(std::mutex, static_mutex);
+      std::lock_guard<std::mutex> lock(static_mutex);
+
+      h = cache->Lookup(cache_key);
+      if (h == nullptr) {
+        auto new_ptr = new CacheEntryStatsCollector(cache, clock);
+        // TODO: non-zero charge causes some tests that count block cache
+        // usage to go flaky. Fix the problem somehow so we can use an
+        // accurate charge.
+        size_t charge = 0;
+        Status s = cache->Insert(cache_key, new_ptr, charge, Deleter, &h,
+                                 Cache::Priority::HIGH);
+        if (!s.ok()) {
+          assert(h == nullptr);
+          delete new_ptr;
+          return s;
+        }
+      }
+    }
+    // If we reach here, shared entry is in cache with handle `h`.
+    assert(cache->GetDeleter(h) == Deleter);
+
+    // Build an aliasing shared_ptr that keeps `ptr` in cache while there
+    // are references.
+    *ptr = MakeSharedCacheHandleGuard<CacheEntryStatsCollector>(cache, h);
+    return Status::OK();
+  }
+
+ private:
+  explicit CacheEntryStatsCollector(Cache *cache, SystemClock *clock)
+      : saved_stats_(),
+        working_stats_(),
+        last_start_time_micros_(0),
+        last_end_time_micros_(/*pessimistic*/ 10000000),
+        cache_(cache),
+        clock_(clock) {}
+
+  static void Deleter(const Slice &, void *value) {
+    delete static_cast<CacheEntryStatsCollector *>(value);
+  }
+
+  std::mutex saved_mutex_;
+  Stats saved_stats_;
+
+  std::mutex working_mutex_;
+  Stats working_stats_;
+  uint64_t last_start_time_micros_;
+  uint64_t last_end_time_micros_;
+
+  Cache *const cache_;
+  SystemClock *const clock_;
+};
+
+}  // namespace ROCKSDB_NAMESPACE
@@ -111,4 +111,15 @@ class CacheHandleGuard {
  T* value_ = nullptr;
 };

+// Build an aliasing shared_ptr that keeps `handle` in cache while there
+// are references, but the pointer is to the value for that cache entry,
+// which must be of type T. This is copyable, unlike CacheHandleGuard, but
+// does not provide access to caching details.
+template <typename T>
+std::shared_ptr<T> MakeSharedCacheHandleGuard(Cache* cache,
+                                              Cache::Handle* handle) {
+  auto wrapper = std::make_shared<CacheHandleGuard<T>>(cache, handle);
+  return std::shared_ptr<T>(wrapper, static_cast<T*>(cache->Value(handle)));
+}
+
 }  // namespace ROCKSDB_NAMESPACE
@@ -712,25 +712,98 @@ TEST_P(CacheTest, OverCapacity) {
 }

 namespace {
-std::vector<std::pair<int, int>> callback_state;
-void callback(void* entry, size_t charge) {
-  callback_state.push_back({DecodeValue(entry), static_cast<int>(charge)});
+std::vector<std::pair<int, int>> legacy_callback_state;
+void legacy_callback(void* value, size_t charge) {
+  legacy_callback_state.push_back(
+      {DecodeValue(value), static_cast<int>(charge)});
 }
 };

-TEST_P(CacheTest, ApplyToAllCacheEntiresTest) {
+TEST_P(CacheTest, ApplyToAllCacheEntriesTest) {
  std::vector<std::pair<int, int>> inserted;
-  callback_state.clear();
+  legacy_callback_state.clear();

  for (int i = 0; i < 10; ++i) {
    Insert(i, i * 2, i + 1);
    inserted.push_back({i * 2, i + 1});
  }
-  cache_->ApplyToAllCacheEntries(callback, true);
+  cache_->ApplyToAllCacheEntries(legacy_callback, true);
+
+  std::sort(inserted.begin(), inserted.end());
+  std::sort(legacy_callback_state.begin(), legacy_callback_state.end());
+  ASSERT_EQ(inserted.size(), legacy_callback_state.size());
+  for (size_t i = 0; i < inserted.size(); ++i) {
+    EXPECT_EQ(inserted[i], legacy_callback_state[i]);
+  }
+}
+
+TEST_P(CacheTest, ApplyToAllEntriesTest) {
+  std::vector<std::string> callback_state;
+  const auto callback = [&](const Slice& key, void* value, size_t charge,
+                            Cache::DeleterFn deleter) {
+    callback_state.push_back(ToString(DecodeKey(key)) + "," +
+                             ToString(DecodeValue(value)) + "," +
+                             ToString(charge));
+    assert(deleter == &CacheTest::Deleter);
+  };
+
+  std::vector<std::string> inserted;
+  callback_state.clear();
+
+  for (int i = 0; i < 10; ++i) {
+    Insert(i, i * 2, i + 1);
+    inserted.push_back(ToString(i) + "," + ToString(i * 2) + "," +
+                       ToString(i + 1));
+  }
+  cache_->ApplyToAllEntries(callback, /*opts*/ {});

  std::sort(inserted.begin(), inserted.end());
  std::sort(callback_state.begin(), callback_state.end());
-  ASSERT_TRUE(inserted == callback_state);
+  ASSERT_EQ(inserted.size(), callback_state.size());
+  for (size_t i = 0; i < inserted.size(); ++i) {
+    EXPECT_EQ(inserted[i], callback_state[i]);
+  }
+}
+
+TEST_P(CacheTest, ApplyToAllEntriesDuringResize) {
+  // This is a mini-stress test of ApplyToAllEntries, to ensure
+  // items in the cache that are neither added nor removed
+  // during ApplyToAllEntries are counted exactly once.
+
+  // Insert some entries that we expect to be seen exactly once
+  // during iteration.
+  constexpr int kSpecialCharge = 2;
+  constexpr int kNotSpecialCharge = 1;
+  constexpr int kSpecialCount = 100;
+  for (int i = 0; i < kSpecialCount; ++i) {
+    Insert(i, i * 2, kSpecialCharge);
+  }
+
+  // For callback
+  int special_count = 0;
+  const auto callback = [&](const Slice&, void*, size_t charge,
+                            Cache::DeleterFn) {
+    if (charge == static_cast<size_t>(kSpecialCharge)) {
+      ++special_count;
+    }
+  };
+
+  // Start counting
+  std::thread apply_thread([&]() {
+    // Use small average_entries_per_lock to make the problem difficult
+    Cache::ApplyToAllEntriesOptions opts;
+    opts.average_entries_per_lock = 2;
+    cache_->ApplyToAllEntries(callback, opts);
+  });
+
+  // In parallel, add more entries, enough to cause resize but not enough
+  // to cause ejections
+  for (int i = kSpecialCount * 1; i < kSpecialCount * 6; ++i) {
+    Insert(i, i * 2, kNotSpecialCharge);
+  }
+
+  apply_thread.join();
+  ASSERT_EQ(special_count, kSpecialCount);
 }

 TEST_P(CacheTest, DefaultShardBits) {
@@ -749,11 +822,12 @@ TEST_P(CacheTest, DefaultShardBits) {
  ASSERT_EQ(6, sc->GetNumShardBits());
 }

-TEST_P(CacheTest, GetCharge) {
+TEST_P(CacheTest, GetChargeAndDeleter) {
  Insert(1, 2);
  Cache::Handle* h1 = cache_->Lookup(EncodeKey(1));
  ASSERT_EQ(2, DecodeValue(cache_->Value(h1)));
  ASSERT_EQ(1, cache_->GetCharge(h1));
+  ASSERT_EQ(&CacheTest::Deleter, cache_->GetDeleter(h1));
  cache_->Release(h1);
 }

@@ -176,10 +176,13 @@ namespace {
 // Cache entry meta data.
 struct CacheHandle {
  Slice key;
-  uint32_t hash;
  void* value;
  size_t charge;
-  void (*deleter)(const Slice&, void* value);
+  Cache::DeleterFn deleter;
+  uint32_t hash;
+
+  // Addition to "charge" to get "total charge" under metadata policy.
+  uint32_t meta_charge;

  // Flags and counters associated with the cache handle:
  //   lowest bit: in-cache bit
@@ -205,9 +208,8 @@ struct CacheHandle {
    return *this;
  }

-  inline static size_t CalcTotalCharge(
-      Slice key, size_t charge,
-      CacheMetadataChargePolicy metadata_charge_policy) {
+  inline static uint32_t CalcMetadataCharge(
+      Slice key, CacheMetadataChargePolicy metadata_charge_policy) {
    size_t meta_charge = 0;
    if (metadata_charge_policy == kFullChargeCacheMetadata) {
      meta_charge += sizeof(CacheHandle);
@@ -218,13 +220,11 @@ struct CacheHandle {
      meta_charge += key.size();
 #endif
    }
-    return charge + meta_charge;
+    assert(meta_charge <= UINT32_MAX);
+    return static_cast<uint32_t>(meta_charge);
  }

-  inline size_t CalcTotalCharge(
-      CacheMetadataChargePolicy metadata_charge_policy) {
-    return CalcTotalCharge(key, charge, metadata_charge_policy);
-  }
+  inline size_t GetTotalCharge() { return charge + meta_charge; }
 };

 // Key of hash map. We store hash value with the key for convenience.
@@ -272,15 +272,13 @@ class ClockCacheShard final : public CacheShard {
                void (*deleter)(const Slice& key, void* value),
                Cache::Handle** handle, Cache::Priority priority) override;
  Status Insert(const Slice& key, uint32_t hash, void* value,
-                Cache::CacheItemHelperCallback helper_cb, size_t charge,
+                const Cache::CacheItemHelper* helper, size_t charge,
                Cache::Handle** handle, Cache::Priority priority) override {
-    Cache::DeletionCallback delete_cb;
-    (*helper_cb)(nullptr, nullptr, &delete_cb);
-    return Insert(key, hash, value, charge, delete_cb, handle, priority);
+    return Insert(key, hash, value, charge, helper->del_cb, handle, priority);
  }
  Cache::Handle* Lookup(const Slice& key, uint32_t hash) override;
  Cache::Handle* Lookup(const Slice& key, uint32_t hash,
-                        Cache::CacheItemHelperCallback /*helper_cb*/,
+                        const Cache::CacheItemHelper* /*helper*/,
                        const Cache::CreateCallback& /*create_cb*/,
                        Cache::Priority /*priority*/, bool /*wait*/) override {
    return Lookup(key, hash);
@@ -289,7 +287,7 @@ class ClockCacheShard final : public CacheShard {
               bool force_erase) override {
    return Release(handle, force_erase);
  }
-  bool isReady(Cache::Handle* /*handle*/) override { return true; }
+  bool IsReady(Cache::Handle* /*handle*/) override { return true; }
  void Wait(Cache::Handle* /*handle*/) override {}

  // If the entry in in cache, increase reference count and return true.
@@ -304,8 +302,10 @@ class ClockCacheShard final : public CacheShard {
  size_t GetUsage() const override;
  size_t GetPinnedUsage() const override;
  void EraseUnRefEntries() override;
-  void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
-                              bool thread_safe) override;
+  void ApplyToSomeEntries(
+      const std::function<void(const Slice& key, void* value, size_t charge,
+                               DeleterFn deleter)>& callback,
+      uint32_t average_entries_per_lock, uint32_t* state) override;

 private:
  static const uint32_t kInCacheBit = 1;
@@ -424,22 +424,46 @@ size_t ClockCacheShard::GetPinnedUsage() const {
  return pinned_usage_.load(std::memory_order_relaxed);
 }

-void ClockCacheShard::ApplyToAllCacheEntries(void (*callback)(void*, size_t),
-                                             bool thread_safe) {
-  if (thread_safe) {
-    mutex_.Lock();
+void ClockCacheShard::ApplyToSomeEntries(
+    const std::function<void(const Slice& key, void* value, size_t charge,
+                             DeleterFn deleter)>& callback,
+    uint32_t average_entries_per_lock, uint32_t* state) {
+  assert(average_entries_per_lock > 0);
+  MutexLock lock(&mutex_);
+
+  // Figure out the range to iterate, update `state`
+  size_t list_size = list_.size();
+  size_t start_idx = *state;
+  size_t end_idx = start_idx + average_entries_per_lock;
+  if (start_idx > list_size) {
+    // Shouldn't reach here, but recoverable
+    assert(false);
+    // Mark finished with all
+    *state = UINT32_MAX;
+    return;
  }
-  for (auto& handle : list_) {
-    // Use relaxed semantics instead of acquire semantics since we are either
-    // holding mutex, or don't have thread safe requirement.
+  if (end_idx >= list_size || end_idx >= UINT32_MAX) {
+    // This also includes the hypothetical case of >4 billion
+    // cache handles.
+    end_idx = list_size;
+    // Mark finished with all
+    *state = UINT32_MAX;
+  } else {
+    *state = static_cast<uint32_t>(end_idx);
+  }
+
+  // Do the iteration
+  auto cur = list_.begin() + start_idx;
+  auto end = list_.begin() + end_idx;
+  for (; cur != end; ++cur) {
+    const CacheHandle& handle = *cur;
+    // Use relaxed semantics instead of acquire semantics since we are
+    // holding mutex
    uint32_t flags = handle.flags.load(std::memory_order_relaxed);
    if (InCache(flags)) {
-      callback(handle.value, handle.charge);
+      callback(handle.key, handle.value, handle.charge, handle.deleter);
    }
  }
-  if (thread_safe) {
-    mutex_.Unlock();
-  }
 }

 void ClockCacheShard::RecycleHandle(CacheHandle* handle,
@@ -448,10 +472,8 @@ void ClockCacheShard::RecycleHandle(CacheHandle* handle,
  assert(!InCache(handle->flags) && CountRefs(handle->flags) == 0);
  context->to_delete_key.push_back(handle->key.data());
  context->to_delete_value.emplace_back(*handle);
-  size_t total_charge = handle->CalcTotalCharge(metadata_charge_policy_);
-  handle->key.clear();
-  handle->value = nullptr;
-  handle->deleter = nullptr;
+  size_t total_charge = handle->GetTotalCharge();
+  // clearing `handle` fields would go here but not strictly required
  recycle_.push_back(handle);
  usage_.fetch_sub(total_charge, std::memory_order_relaxed);
 }
@@ -479,7 +501,7 @@ bool ClockCacheShard::Ref(Cache::Handle* h) {
                                            std::memory_order_relaxed)) {
      if (CountRefs(flags) == 0) {
        // No reference count before the operation.
-        size_t total_charge = handle->CalcTotalCharge(metadata_charge_policy_);
+        size_t total_charge = handle->GetTotalCharge();
        pinned_usage_.fetch_add(total_charge, std::memory_order_relaxed);
      }
      return true;
@@ -493,6 +515,11 @@ bool ClockCacheShard::Unref(CacheHandle* handle, bool set_usage,
  if (set_usage) {
    handle->flags.fetch_or(kUsageBit, std::memory_order_relaxed);
  }
+  // If the handle reaches state refs=0 and InCache=true after this
+  // atomic operation then we cannot access `handle` afterward, because
+  // it could be evicted before we access the `handle`.
+  size_t total_charge = handle->GetTotalCharge();
+
  // Use acquire-release semantics as previous operations on the cache entry
  // has to be order before reference count is decreased, and potential cleanup
  // of the entry has to be order after.
@@ -500,7 +527,6 @@ bool ClockCacheShard::Unref(CacheHandle* handle, bool set_usage,
  assert(CountRefs(flags) > 0);
  if (CountRefs(flags) == 1) {
    // this is the last reference.
-    size_t total_charge = handle->CalcTotalCharge(metadata_charge_policy_);
    pinned_usage_.fetch_sub(total_charge, std::memory_order_relaxed);
    // Cleanup if it is the last reference.
    if (!InCache(flags)) {
@@ -587,8 +613,9 @@ CacheHandle* ClockCacheShard::Insert(
    void (*deleter)(const Slice& key, void* value), bool hold_reference,
    CleanupContext* context, bool* overwritten) {
  assert(overwritten != nullptr && *overwritten == false);
-  size_t total_charge =
-      CacheHandle::CalcTotalCharge(key, charge, metadata_charge_policy_);
+  uint32_t meta_charge =
+      CacheHandle::CalcMetadataCharge(key, metadata_charge_policy_);
+  size_t total_charge = charge + meta_charge;
  MutexLock l(&mutex_);
  bool success = EvictFromCache(total_charge, context);
  bool strict = strict_capacity_limit_.load(std::memory_order_relaxed);
@@ -614,8 +641,18 @@ CacheHandle* ClockCacheShard::Insert(
  handle->hash = hash;
  handle->value = value;
  handle->charge = charge;
+  handle->meta_charge = meta_charge;
  handle->deleter = deleter;
  uint32_t flags = hold_reference ? kInCacheBit + kOneRef : kInCacheBit;
+
+  // TODO investigate+fix suspected race condition:
+  // [thread 1] Lookup starts, up to Ref()
+  // [thread 2] Erase/evict the entry just looked up
+  // [thread 1] Ref() the handle, even though it's in the recycle bin
+  // [thread 2] Insert with recycling that handle
+  // Here we obliterate the other thread's Ref
+  // Possible fix: never blindly overwrite the flags, but only make
+  // relative updates (fetch_add, etc).
  handle->flags.store(flags, std::memory_order_relaxed);
  HashTable::accessor accessor;
  if (table_.find(accessor, CacheKey(key, hash))) {
@@ -746,11 +783,11 @@ class ClockCache final : public ShardedCache {

  const char* Name() const override { return "ClockCache"; }

-  CacheShard* GetShard(int shard) override {
+  CacheShard* GetShard(uint32_t shard) override {
    return reinterpret_cast<CacheShard*>(&shards_[shard]);
  }

-  const CacheShard* GetShard(int shard) const override {
+  const CacheShard* GetShard(uint32_t shard) const override {
    return reinterpret_cast<CacheShard*>(&shards_[shard]);
  }

@@ -766,7 +803,15 @@ class ClockCache final : public ShardedCache {
    return reinterpret_cast<const CacheHandle*>(handle)->hash;
  }

-  void DisownData() override { shards_ = nullptr; }
+  DeleterFn GetDeleter(Handle* handle) const override {
+    return reinterpret_cast<const CacheHandle*>(handle)->deleter;
+  }
+
+  void DisownData() override {
+#ifndef MUST_FREE_HEAP_ALLOCATIONS
+    shards_ = nullptr;
+#endif
+  }

  void WaitAll(std::vector<Handle*>& /*handles*/) override {}

@@ -10,24 +10,27 @@
 #include "cache/lru_cache.h"

 #include <cassert>
+#include <cstdint>
 #include <cstdio>
-#include <string>

 #include "util/mutexlock.h"

 namespace ROCKSDB_NAMESPACE {

-LRUHandleTable::LRUHandleTable() : list_(nullptr), length_(0), elems_(0) {
-  Resize();
-}
+LRUHandleTable::LRUHandleTable(int max_upper_hash_bits)
+    : length_bits_(/* historical starting size*/ 4),
+      list_(new LRUHandle* [size_t{1} << length_bits_] {}),
+      elems_(0),
+      max_length_bits_(max_upper_hash_bits) {}

 LRUHandleTable::~LRUHandleTable() {
-  ApplyToAllCacheEntries([](LRUHandle* h) {
-    if (!h->HasRefs()) {
-      h->Free();
-    }
-  });
-  delete[] list_;
+  ApplyToEntriesRange(
+      [](LRUHandle* h) {
+        if (!h->HasRefs()) {
+          h->Free();
+        }
+      },
+      0, uint32_t{1} << length_bits_);
 }

 LRUHandle* LRUHandleTable::Lookup(const Slice& key, uint32_t hash) {
@@ -41,7 +44,7 @@ LRUHandle* LRUHandleTable::Insert(LRUHandle* h) {
  *ptr = h;
  if (old == nullptr) {
    ++elems_;
-    if (elems_ > length_) {
+    if ((elems_ >> length_bits_) > 0) {  // elems_ >= length
      // Since each cache entry is fairly large, we aim for a small
      // average linked list length (<= 1).
      Resize();
@@ -61,7 +64,7 @@ LRUHandle* LRUHandleTable::Remove(const Slice& key, uint32_t hash) {
 }

 LRUHandle** LRUHandleTable::FindPointer(const Slice& key, uint32_t hash) {
-  LRUHandle** ptr = &list_[hash & (length_ - 1)];
+  LRUHandle** ptr = &list_[hash >> (32 - length_bits_)];
  while (*ptr != nullptr && ((*ptr)->hash != hash || key != (*ptr)->key())) {
    ptr = &(*ptr)->next_hash;
  }
@@ -69,19 +72,29 @@ LRUHandle** LRUHandleTable::FindPointer(const Slice& key, uint32_t hash) {
 }

 void LRUHandleTable::Resize() {
-  uint32_t new_length = 16;
-  while (new_length < elems_ * 1.5) {
-    new_length *= 2;
+  if (length_bits_ >= max_length_bits_) {
+    // Due to reaching limit of hash information, if we made the table
+    // bigger, we would allocate more addresses but only the same
+    // number would be used.
+    return;
  }
-  LRUHandle** new_list = new LRUHandle*[new_length];
-  memset(new_list, 0, sizeof(new_list[0]) * new_length);
+  if (length_bits_ >= 31) {
+    // Avoid undefined behavior shifting uint32_t by 32
+    return;
+  }
+
+  uint32_t old_length = uint32_t{1} << length_bits_;
+  int new_length_bits = length_bits_ + 1;
+  std::unique_ptr<LRUHandle* []> new_list {
+    new LRUHandle* [size_t{1} << new_length_bits] {}
+  };
  uint32_t count = 0;
-  for (uint32_t i = 0; i < length_; i++) {
+  for (uint32_t i = 0; i < old_length; i++) {
    LRUHandle* h = list_[i];
    while (h != nullptr) {
      LRUHandle* next = h->next_hash;
      uint32_t hash = h->hash;
-      LRUHandle** ptr = &new_list[hash & (new_length - 1)];
+      LRUHandle** ptr = &new_list[hash >> (32 - new_length_bits)];
      h->next_hash = *ptr;
      *ptr = h;
      h = next;
@@ -89,25 +102,25 @@ void LRUHandleTable::Resize() {
    }
  }
  assert(elems_ == count);
-  delete[] list_;
-  list_ = new_list;
-  length_ = new_length;
+  list_ = std::move(new_list);
+  length_bits_ = new_length_bits;
 }

-LRUCacheShard::LRUCacheShard(size_t capacity, bool strict_capacity_limit,
-                             double high_pri_pool_ratio,
-                             bool use_adaptive_mutex,
-                             CacheMetadataChargePolicy metadata_charge_policy,
-                             const std::shared_ptr<TieredCache>& tiered_cache)
+LRUCacheShard::LRUCacheShard(
+    size_t capacity, bool strict_capacity_limit, double high_pri_pool_ratio,
+    bool use_adaptive_mutex, CacheMetadataChargePolicy metadata_charge_policy,
+    int max_upper_hash_bits,
+    const std::shared_ptr<SecondaryCache>& secondary_cache)
    : capacity_(0),
      high_pri_pool_usage_(0),
      strict_capacity_limit_(strict_capacity_limit),
      high_pri_pool_ratio_(high_pri_pool_ratio),
      high_pri_pool_capacity_(0),
+      table_(max_upper_hash_bits),
      usage_(0),
      lru_usage_(0),
      mutex_(use_adaptive_mutex),
-      tiered_cache_(tiered_cache) {
+      secondary_cache_(secondary_cache) {
  set_metadata_charge_policy(metadata_charge_policy);
  // Make empty circular linked list
  lru_.next = &lru_;
@@ -139,19 +152,40 @@ void LRUCacheShard::EraseUnRefEntries() {
  }
 }

-void LRUCacheShard::ApplyToAllCacheEntries(void (*callback)(void*, size_t),
-                                           bool thread_safe) {
-  const auto applyCallback = [&]() {
-    table_.ApplyToAllCacheEntries(
-        [callback](LRUHandle* h) { callback(h->value, h->charge); });
-  };
+void LRUCacheShard::ApplyToSomeEntries(
+    const std::function<void(const Slice& key, void* value, size_t charge,
+                             DeleterFn deleter)>& callback,
+    uint32_t average_entries_per_lock, uint32_t* state) {
+  // The state is essentially going to be the starting hash, which works
+  // nicely even if we resize between calls because we use upper-most
+  // hash bits for table indexes.
+  MutexLock l(&mutex_);
+  uint32_t length_bits = table_.GetLengthBits();
+  uint32_t length = uint32_t{1} << length_bits;

-  if (thread_safe) {
-    MutexLock l(&mutex_);
-    applyCallback();
+  assert(average_entries_per_lock > 0);
+  // Assuming we are called with same average_entries_per_lock repeatedly,
+  // this simplifies some logic (index_end will not overflow)
+  assert(average_entries_per_lock < length || *state == 0);
+
+  uint32_t index_begin = *state >> (32 - length_bits);
+  uint32_t index_end = index_begin + average_entries_per_lock;
+  if (index_end >= length) {
+    // Going to end
+    index_end = length;
+    *state = UINT32_MAX;
  } else {
-    applyCallback();
+    *state = index_end << (32 - length_bits);
  }
+
+  table_.ApplyToEntriesRange(
+      [callback](LRUHandle* h) {
+        DeleterFn deleter = h->IsSecondaryCacheCompatible()
+                                ? h->info_.helper->del_cb
+                                : h->info_.deleter;
+        callback(h->key(), h->value, h->charge, deleter);
+      },
+      index_begin, index_end);
 }

 void LRUCacheShard::TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri) {
@@ -261,9 +295,9 @@ void LRUCacheShard::SetCapacity(size_t capacity) {
  // Try to insert the evicted entries into tiered cache
  // Free the entries outside of mutex for performance reasons
  for (auto entry : last_reference_list) {
-    if (tiered_cache_ && entry->IsTieredCacheCompatible() &&
+    if (secondary_cache_ && entry->IsSecondaryCacheCompatible() &&
        !entry->IsPromoted()) {
-      tiered_cache_->Insert(entry->key(), entry->value, entry->info_.helper_cb)
+      secondary_cache_->Insert(entry->key(), entry->value, entry->info_.helper)
          .PermitUncheckedError();
    }
    entry->Free();
@@ -275,7 +309,8 @@ void LRUCacheShard::SetStrictCapacityLimit(bool strict_capacity_limit) {
  strict_capacity_limit_ = strict_capacity_limit;
 }

-Status LRUCacheShard::InsertItem(LRUHandle* e, Cache::Handle** handle) {
+Status LRUCacheShard::InsertItem(LRUHandle* e, Cache::Handle** handle,
+                                 bool free_handle_on_fail) {
  Status s = Status::OK();
  autovector<LRUHandle*> last_reference_list;
  size_t total_charge = e->CalcTotalCharge(metadata_charge_policy_);
@@ -289,14 +324,16 @@ Status LRUCacheShard::InsertItem(LRUHandle* e, Cache::Handle** handle) {

    if ((usage_ + total_charge) > capacity_ &&
        (strict_capacity_limit_ || handle == nullptr)) {
+      e->SetInCache(false);
      if (handle == nullptr) {
        // Don't insert the entry but still return ok, as if the entry inserted
        // into cache and get evicted immediately.
-        e->SetInCache(false);
        last_reference_list.push_back(e);
      } else {
-        delete[] reinterpret_cast<char*>(e);
-        *handle = nullptr;
+        if (free_handle_on_fail) {
+          delete[] reinterpret_cast<char*>(e);
+          *handle = nullptr;
+        }
        s = Status::Incomplete("Insert failed due to LRU cache being full.");
      }
    } else {
@@ -327,12 +364,12 @@ Status LRUCacheShard::InsertItem(LRUHandle* e, Cache::Handle** handle) {
    }
  }

-  // Try to insert the evicted entries into NVM cache
+  // Try to insert the evicted entries into the secondary cache
  // Free the entries here outside of mutex for performance reasons
  for (auto entry : last_reference_list) {
-    if (tiered_cache_ && entry->IsTieredCacheCompatible() &&
+    if (secondary_cache_ && entry->IsSecondaryCacheCompatible() &&
        !entry->IsPromoted()) {
-      tiered_cache_->Insert(entry->key(), entry->value, entry->info_.helper_cb)
+      secondary_cache_->Insert(entry->key(), entry->value, entry->info_.helper)
          .PermitUncheckedError();
    }
    entry->Free();
@@ -341,9 +378,46 @@ Status LRUCacheShard::InsertItem(LRUHandle* e, Cache::Handle** handle) {
  return s;
 }

+void LRUCacheShard::Promote(LRUHandle* e) {
+  SecondaryCacheResultHandle* secondary_handle = e->sec_handle;
+
+  assert(secondary_handle->IsReady());
+  e->SetIncomplete(false);
+  e->SetInCache(true);
+  e->SetPromoted(true);
+  e->value = secondary_handle->Value();
+  e->charge = secondary_handle->Size();
+  delete secondary_handle;
+
+  // This call could fail if the cache is over capacity and
+  // strict_capacity_limit_ is true. In such a case, we don't want
+  // InsertItem() to free the handle, since the item is already in memory
+  // and the caller will most likely just read from disk if we erase it here.
+  if (e->value) {
+    Cache::Handle* handle = reinterpret_cast<Cache::Handle*>(e);
+    Status s = InsertItem(e, &handle, /*free_handle_on_fail=*/false);
+    if (s.ok()) {
+      // InsertItem would have taken a reference on the item, so decrement it
+      // here as we expect the caller to already hold a reference
+      e->Unref();
+    } else {
+      // Item is in memory, but not accounted against the cache capacity.
+      // When the handle is released, the item should get deleted
+      assert(!e->InCache());
+    }
+  } else {
+    // Since the secondary cache lookup failed, mark the item as not in cache
+    // and charge the cache only for metadata usage, i.e handle, key etc
+    MutexLock l(&mutex_);
+    e->charge = 0;
+    e->SetInCache(false);
+    usage_ += e->CalcTotalCharge(metadata_charge_policy_);
+  }
+}
+
 Cache::Handle* LRUCacheShard::Lookup(
    const Slice& key, uint32_t hash,
-    ShardedCache::CacheItemHelperCallback helper_cb,
+    const ShardedCache::CacheItemHelper* helper,
    const ShardedCache::CreateCallback& create_cb, Cache::Priority priority,
    bool wait) {
  LRUHandle* e = nullptr;
@@ -362,38 +436,48 @@ Cache::Handle* LRUCacheShard::Lookup(
  }

  // If handle table lookup failed, then allocate a handle outside the
-  // mutex if we're going to lookup in the NVM cache
+  // mutex if we're going to lookup in the secondary cache
  // Only support synchronous for now
-  // TODO: Support asynchronous lookup in NVM cache
-  if (!e && tiered_cache_ && helper_cb && wait) {
-    assert(create_cb);
-    std::unique_ptr<TieredCacheHandle> tiered_handle =
-        tiered_cache_->Lookup(key, create_cb, wait);
-    if (tiered_handle != nullptr) {
+  // TODO: Support asynchronous lookup in secondary cache
+  if (!e && secondary_cache_ && helper && helper->saveto_cb) {
+    // For objects from the secondary cache, we expect the caller to provide
+    // a way to create/delete the primary cache object. The only case where
+    // a deleter would not be required is for dummy entries inserted for
+    // accounting purposes, which we won't demote to the secondary cache
+    // anyway.
+    assert(create_cb && helper->del_cb);
+    std::unique_ptr<SecondaryCacheResultHandle> secondary_handle =
+        secondary_cache_->Lookup(key, create_cb, wait);
+    if (secondary_handle != nullptr) {
      e = reinterpret_cast<LRUHandle*>(
          new char[sizeof(LRUHandle) - 1 + key.size()]);

      e->flags = 0;
-      e->SetPromoted(true);
-      e->SetTieredCacheCompatible(true);
-      e->info_.helper_cb = helper_cb;
+      e->SetSecondaryCacheCompatible(true);
+      e->info_.helper = helper;
      e->key_length = key.size();
      e->hash = hash;
      e->refs = 0;
      e->next = e->prev = nullptr;
-      e->SetInCache(true);
      e->SetPriority(priority);
      memcpy(e->key_data, key.data(), key.size());
+      e->value = nullptr;
+      e->sec_handle = secondary_handle.release();
+      e->Ref();

-      e->value = tiered_handle->Value();
-      e->charge = tiered_handle->Size();
-
-      // This call could nullify e if the cache is over capacity and
-      // strict_capacity_limit_ is true. In such a case, the caller will try
-      // to insert later, which might again fail, but its ok as this should
-      // not be common
-      InsertItem(e, reinterpret_cast<Cache::Handle**>(&e))
-          .PermitUncheckedError();
+      if (wait) {
+        Promote(e);
+        if (!e->value) {
+          // The secondary cache returned a handle, but the lookup failed
+          e->Unref();
+          e->Free();
+          e = nullptr;
+        }
+      } else {
+        // If wait is false, we always return a handle and let the caller
+        // release the handle after checking for success or failure
+        e->SetIncomplete(true);
+      }
    }
  }
  return reinterpret_cast<Cache::Handle*>(e);
@@ -455,7 +539,7 @@ bool LRUCacheShard::Release(Cache::Handle* handle, bool force_erase) {
 Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
                             size_t charge,
                             void (*deleter)(const Slice& key, void* value),
-                             Cache::CacheItemHelperCallback helper_cb,
+                             const Cache::CacheItemHelper* helper,
                             Cache::Handle** handle, Cache::Priority priority) {
  // Allocate the memory here outside of the mutex
  // If the cache is full, we'll have to release it
@@ -465,10 +549,13 @@ Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value,

  e->value = value;
  e->flags = 0;
-  if (helper_cb) {
-    e->SetTieredCacheCompatible(true);
-    e->info_.helper_cb = helper_cb;
+  if (helper) {
+    e->SetSecondaryCacheCompatible(true);
+    e->info_.helper = helper;
  } else {
+#ifdef __SANITIZE_THREAD__
+    e->is_secondary_cache_compatible_for_tsan = false;
+#endif  // __SANITIZE_THREAD__
    e->info_.deleter = deleter;
  }
  e->charge = charge;
@@ -480,7 +567,7 @@ Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
  e->SetPriority(priority);
  memcpy(e->key_data, key.data(), key.size());

-  return InsertItem(e, handle);
+  return InsertItem(e, handle, /* free_handle_on_fail */ true);
 }

 void LRUCacheShard::Erase(const Slice& key, uint32_t hash) {
@@ -510,6 +597,18 @@ void LRUCacheShard::Erase(const Slice& key, uint32_t hash) {
  }
 }

+bool LRUCacheShard::IsReady(Cache::Handle* handle) {
+  LRUHandle* e = reinterpret_cast<LRUHandle*>(handle);
+  MutexLock l(&mutex_);
+  bool ready = true;
+  if (e->IsPending()) {
+    assert(secondary_cache_);
+    assert(e->sec_handle);
+    ready = e->sec_handle->IsReady();
+  }
+  return ready;
+}
+
 size_t LRUCacheShard::GetUsage() const {
  MutexLock l(&mutex_);
  return usage_;
@@ -537,7 +636,7 @@ LRUCache::LRUCache(size_t capacity, int num_shard_bits,
                   std::shared_ptr<MemoryAllocator> allocator,
                   bool use_adaptive_mutex,
                   CacheMetadataChargePolicy metadata_charge_policy,
-                   const std::shared_ptr<TieredCache>& tiered_cache)
+                   const std::shared_ptr<SecondaryCache>& secondary_cache)
    : ShardedCache(capacity, num_shard_bits, strict_capacity_limit,
                   std::move(allocator)) {
  num_shards_ = 1 << num_shard_bits;
@@ -545,10 +644,12 @@ LRUCache::LRUCache(size_t capacity, int num_shard_bits,
      port::cacheline_aligned_alloc(sizeof(LRUCacheShard) * num_shards_));
  size_t per_shard = (capacity + (num_shards_ - 1)) / num_shards_;
  for (int i = 0; i < num_shards_; i++) {
-    new (&shards_[i])
-        LRUCacheShard(per_shard, strict_capacity_limit, high_pri_pool_ratio,
-                      use_adaptive_mutex, metadata_charge_policy, tiered_cache);
+    new (&shards_[i]) LRUCacheShard(
+        per_shard, strict_capacity_limit, high_pri_pool_ratio,
+        use_adaptive_mutex, metadata_charge_policy,
+        /* max_upper_hash_bits */ 32 - num_shard_bits, secondary_cache);
  }
+  secondary_cache_ = secondary_cache;
 }

 LRUCache::~LRUCache() {
@@ -561,11 +662,11 @@ LRUCache::~LRUCache() {
  }
 }

-CacheShard* LRUCache::GetShard(int shard) {
+CacheShard* LRUCache::GetShard(uint32_t shard) {
  return reinterpret_cast<CacheShard*>(&shards_[shard]);
 }

-const CacheShard* LRUCache::GetShard(int shard) const {
+const CacheShard* LRUCache::GetShard(uint32_t shard) const {
  return reinterpret_cast<CacheShard*>(&shards_[shard]);
 }

@@ -577,23 +678,25 @@ size_t LRUCache::GetCharge(Handle* handle) const {
  return reinterpret_cast<const LRUHandle*>(handle)->charge;
 }

+Cache::DeleterFn LRUCache::GetDeleter(Handle* handle) const {
+  auto h = reinterpret_cast<const LRUHandle*>(handle);
+  if (h->IsSecondaryCacheCompatible()) {
+    return h->info_.helper->del_cb;
+  } else {
+    return h->info_.deleter;
+  }
+}
+
 uint32_t LRUCache::GetHash(Handle* handle) const {
  return reinterpret_cast<const LRUHandle*>(handle)->hash;
 }

 void LRUCache::DisownData() {
 // Do not drop data if compile with ASAN to suppress leak warning.
-#if defined(__clang__)
-#if !defined(__has_feature) || !__has_feature(address_sanitizer)
+#ifndef MUST_FREE_HEAP_ALLOCATIONS
  shards_ = nullptr;
  num_shards_ = 0;
 #endif
-#else  // __clang__
-#ifndef __SANITIZE_ADDRESS__
-  shards_ = nullptr;
-  num_shards_ = 0;
-#endif  // !__SANITIZE_ADDRESS__
-#endif  // __clang__
 }

 size_t LRUCache::TEST_GetLRUSize() {
@@ -612,12 +715,42 @@ double LRUCache::GetHighPriPoolRatio() {
  return result;
 }

+void LRUCache::WaitAll(std::vector<Handle*>& handles) {
+  if (secondary_cache_) {
+    std::vector<SecondaryCacheResultHandle*> sec_handles;
+    sec_handles.reserve(handles.size());
+    for (Handle* handle : handles) {
+      if (!handle) {
+        continue;
+      }
+      LRUHandle* lru_handle = reinterpret_cast<LRUHandle*>(handle);
+      if (!lru_handle->IsPending()) {
+        continue;
+      }
+      sec_handles.emplace_back(lru_handle->sec_handle);
+    }
+    secondary_cache_->WaitAll(sec_handles);
+    for (Handle* handle : handles) {
+      if (!handle) {
+        continue;
+      }
+      LRUHandle* lru_handle = reinterpret_cast<LRUHandle*>(handle);
+      if (!lru_handle->IsPending()) {
+        continue;
+      }
+      uint32_t hash = GetHash(handle);
+      LRUCacheShard* shard = static_cast<LRUCacheShard*>(GetShard(Shard(hash)));
+      shard->Promote(lru_handle);
+    }
+  }
+}
+
 std::shared_ptr<Cache> NewLRUCache(
    size_t capacity, int num_shard_bits, bool strict_capacity_limit,
    double high_pri_pool_ratio,
    std::shared_ptr<MemoryAllocator> memory_allocator, bool use_adaptive_mutex,
    CacheMetadataChargePolicy metadata_charge_policy,
-    const std::shared_ptr<TieredCache>& tiered_cache) {
+    const std::shared_ptr<SecondaryCache>& secondary_cache) {
  if (num_shard_bits >= 20) {
    return nullptr;  // the cache cannot be sharded into too many fine pieces
  }
@@ -631,7 +764,7 @@ std::shared_ptr<Cache> NewLRUCache(
  return std::make_shared<LRUCache>(
      capacity, num_shard_bits, strict_capacity_limit, high_pri_pool_ratio,
      std::move(memory_allocator), use_adaptive_mutex, metadata_charge_policy,
-      tiered_cache);
+      secondary_cache);
 }

 std::shared_ptr<Cache> NewLRUCache(const LRUCacheOptions& cache_opts) {
@@ -639,7 +772,7 @@ std::shared_ptr<Cache> NewLRUCache(const LRUCacheOptions& cache_opts) {
      cache_opts.capacity, cache_opts.num_shard_bits,
      cache_opts.strict_capacity_limit, cache_opts.high_pri_pool_ratio,
      cache_opts.memory_allocator, cache_opts.use_adaptive_mutex,
-      cache_opts.metadata_charge_policy, cache_opts.tiered_cache);
+      cache_opts.metadata_charge_policy, cache_opts.secondary_cache);
 }

 std::shared_ptr<Cache> NewLRUCache(
@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved
 //  This source code is licensed under both the GPLv2 (found in the
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
@@ -8,12 +8,14 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 #pragma once

+#include <memory>
 #include <string>

 #include "cache/sharded_cache.h"
+#include "port/lang.h"
 #include "port/malloc.h"
 #include "port/port.h"
-#include "rocksdb/tiered_cache.h"
+#include "rocksdb/secondary_cache.h"
 #include "util/autovector.h"

 namespace ROCKSDB_NAMESPACE {
@@ -52,12 +54,15 @@ struct LRUHandle {
  union Info {
    Info() {}
    ~Info() {}
-    void (*deleter)(const Slice&, void* value);
-    ShardedCache::CacheItemHelperCallback helper_cb;
-    // This needs to be explicitly constructed and destructed
-    std::unique_ptr<TieredCacheHandle> tiered_handle;
+    Cache::DeleterFn deleter;
+    const ShardedCache::CacheItemHelper* helper;
  } info_;
-  LRUHandle* next_hash;
+  // An entry is not added to the LRUHandleTable until the secondary cache
+  // lookup is complete, so its safe to have this union.
+  union {
+    LRUHandle* next_hash;
+    SecondaryCacheResultHandle* sec_handle;
+  };
  LRUHandle* next;
  LRUHandle* prev;
  size_t charge;  // TODO(opt): Only allow uint32_t?
@@ -76,8 +81,8 @@ struct LRUHandle {
    IN_HIGH_PRI_POOL = (1 << 2),
    // Whether this entry has had any lookups (hits).
    HAS_HIT = (1 << 3),
-    // Can this be inserted into the tiered cache
-    IS_TIERED_CACHE_COMPATIBLE = (1 << 4),
+    // Can this be inserted into the secondary cache
+    IS_SECONDARY_CACHE_COMPATIBLE = (1 << 4),
    // Is the handle still being read from a lower tier
    IS_PENDING = (1 << 5),
    // Has the item been promoted from a lower tier
@@ -86,6 +91,14 @@ struct LRUHandle {

  uint8_t flags;

+#ifdef __SANITIZE_THREAD__
+  // TSAN can report a false data race on flags, where one thread is writing
+  // to one of the mutable bits and another thread is reading this immutable
+  // bit. So precisely suppress that TSAN warning, we separate out this bit
+  // during TSAN runs.
+  bool is_secondary_cache_compatible_for_tsan;
+#endif  // __SANITIZE_THREAD__
+
  // Beginning of the key (MUST BE THE LAST FIELD IN THIS STRUCT!)
  char key_data[1];

@@ -108,8 +121,12 @@ struct LRUHandle {
  bool IsHighPri() const { return flags & IS_HIGH_PRI; }
  bool InHighPriPool() const { return flags & IN_HIGH_PRI_POOL; }
  bool HasHit() const { return flags & HAS_HIT; }
-  bool IsTieredCacheCompatible() const {
-    return flags & IS_TIERED_CACHE_COMPATIBLE;
+  bool IsSecondaryCacheCompatible() const {
+#ifdef __SANITIZE_THREAD__
+    return is_secondary_cache_compatible_for_tsan;
+#else
+    return flags & IS_SECONDARY_CACHE_COMPATIBLE;
+#endif  // __SANITIZE_THREAD__
  }
  bool IsPending() const { return flags & IS_PENDING; }
  bool IsPromoted() const { return flags & IS_PROMOTED; }
@@ -140,12 +157,15 @@ struct LRUHandle {

  void SetHit() { flags |= HAS_HIT; }

-  void SetTieredCacheCompatible(bool tiered) {
-    if (tiered) {
-      flags |= IS_TIERED_CACHE_COMPATIBLE;
+  void SetSecondaryCacheCompatible(bool compat) {
+    if (compat) {
+      flags |= IS_SECONDARY_CACHE_COMPATIBLE;
    } else {
-      flags &= ~IS_TIERED_CACHE_COMPATIBLE;
+      flags &= ~IS_SECONDARY_CACHE_COMPATIBLE;
    }
+#ifdef __SANITIZE_THREAD__
+    is_secondary_cache_compatible_for_tsan = compat;
+#endif  // __SANITIZE_THREAD__
  }

  void SetIncomplete(bool incomp) {
@@ -166,12 +186,24 @@ struct LRUHandle {

  void Free() {
    assert(refs == 0);
-    if (!IsTieredCacheCompatible() && info_.deleter) {
+#ifdef __SANITIZE_THREAD__
+    // Here we can safely assert they are the same without a data race reported
+    assert(((flags & IS_SECONDARY_CACHE_COMPATIBLE) != 0) ==
+           is_secondary_cache_compatible_for_tsan);
+#endif  // __SANITIZE_THREAD__
+    if (!IsSecondaryCacheCompatible() && info_.deleter) {
      (*info_.deleter)(key(), value);
-    } else if (IsTieredCacheCompatible()) {
-      ShardedCache::DeletionCallback del_cb;
-      (*info_.helper_cb)(nullptr, nullptr, &del_cb);
-      (*del_cb)(key(), value);
+    } else if (IsSecondaryCacheCompatible()) {
+      if (IsPending()) {
+        assert(sec_handle != nullptr);
+        SecondaryCacheResultHandle* tmp_sec_handle = sec_handle;
+        tmp_sec_handle->Wait();
+        value = tmp_sec_handle->Value();
+        delete tmp_sec_handle;
+      }
+      if (value) {
+        (*info_.helper->del_cb)(key(), value);
+      }
    }
    delete[] reinterpret_cast<char*>(this);
  }
@@ -199,7 +231,10 @@ struct LRUHandle {
 // 4.4.3's builtin hashtable.
 class LRUHandleTable {
 public:
-  LRUHandleTable();
+  // If the table uses more hash bits than `max_upper_hash_bits`,
+  // it will eat into the bits used for sharding, which are constant
+  // for a given LRUHandleTable.
+  explicit LRUHandleTable(int max_upper_hash_bits);
  ~LRUHandleTable();

  LRUHandle* Lookup(const Slice& key, uint32_t hash);
@@ -207,8 +242,8 @@ class LRUHandleTable {
  LRUHandle* Remove(const Slice& key, uint32_t hash);

  template <typename T>
-  void ApplyToAllCacheEntries(T func) {
-    for (uint32_t i = 0; i < length_; i++) {
+  void ApplyToEntriesRange(T func, uint32_t index_begin, uint32_t index_end) {
+    for (uint32_t i = index_begin; i < index_end; i++) {
      LRUHandle* h = list_[i];
      while (h != nullptr) {
        auto n = h->next_hash;
@@ -219,6 +254,8 @@ class LRUHandleTable {
    }
  }

+  int GetLengthBits() const { return length_bits_; }
+
 private:
  // Return a pointer to slot that points to a cache entry that
  // matches key/hash.  If there is no such cache entry, return a
@@ -227,11 +264,19 @@ class LRUHandleTable {

  void Resize();

+  // Number of hash bits (upper because lower bits used for sharding)
+  // used for table index. Length == 1 << length_bits_
+  int length_bits_;
+
  // The table consists of an array of buckets where each bucket is
  // a linked list of cache entries that hash into the bucket.
-  LRUHandle** list_;
-  uint32_t length_;
+  std::unique_ptr<LRUHandle*[]> list_;
+
+  // Number of elements currently in the table
  uint32_t elems_;
+
+  // Set from max_upper_hash_bits (see constructor)
+  const int max_length_bits_;
 };

 // A single shard of sharded cache.
@@ -240,7 +285,8 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
  LRUCacheShard(size_t capacity, bool strict_capacity_limit,
                double high_pri_pool_ratio, bool use_adaptive_mutex,
                CacheMetadataChargePolicy metadata_charge_policy,
-                const std::shared_ptr<TieredCache>& tiered_cache);
+                int max_upper_hash_bits,
+                const std::shared_ptr<SecondaryCache>& secondary_cache);
  virtual ~LRUCacheShard() override = default;

  // Separate from constructor so caller can easily make an array of LRUCache
@@ -256,23 +302,22 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {

  // Like Cache methods, but with an extra "hash" parameter.
  virtual Status Insert(const Slice& key, uint32_t hash, void* value,
-                        size_t charge,
-                        void (*deleter)(const Slice& key, void* value),
+                        size_t charge, Cache::DeleterFn deleter,
                        Cache::Handle** handle,
                        Cache::Priority priority) override {
    return Insert(key, hash, value, charge, deleter, nullptr, handle, priority);
  }
  virtual Status Insert(const Slice& key, uint32_t hash, void* value,
-                        Cache::CacheItemHelperCallback helper_cb, size_t charge,
+                        const Cache::CacheItemHelper* helper, size_t charge,
                        Cache::Handle** handle,
                        Cache::Priority priority) override {
-    return Insert(key, hash, value, charge, nullptr, helper_cb, handle,
-                  priority);
+    assert(helper);
+    return Insert(key, hash, value, charge, nullptr, helper, handle, priority);
  }
  // If helper_cb is null, the values of the following arguments don't
  // matter
  virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash,
-                                ShardedCache::CacheItemHelperCallback helper_cb,
+                                const ShardedCache::CacheItemHelper* helper,
                                const ShardedCache::CreateCallback& create_cb,
                                ShardedCache::Priority priority,
                                bool wait) override;
@@ -283,7 +328,7 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
                       bool force_erase) override {
    return Release(handle, force_erase);
  }
-  virtual bool isReady(Cache::Handle* /*handle*/) override { return true; }
+  virtual bool IsReady(Cache::Handle* /*handle*/) override;
  virtual void Wait(Cache::Handle* /*handle*/) override {}
  virtual bool Ref(Cache::Handle* handle) override;
  virtual bool Release(Cache::Handle* handle,
@@ -297,8 +342,10 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
  virtual size_t GetUsage() const override;
  virtual size_t GetPinnedUsage() const override;

-  virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
-                                      bool thread_safe) override;
+  virtual void ApplyToSomeEntries(
+      const std::function<void(const Slice& key, void* value, size_t charge,
+                               DeleterFn deleter)>& callback,
+      uint32_t average_entries_per_lock, uint32_t* state) override;

  virtual void EraseUnRefEntries() override;

@@ -314,11 +361,23 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
  double GetHighPriPoolRatio();

 private:
-  Status InsertItem(LRUHandle* item, Cache::Handle** handle);
+  friend class LRUCache;
+  // Insert an item into the hash table and, if handle is null, insert into
+  // the LRU list. Older items are evicted as necessary. If the cache is full
+  // and free_handle_on_fail is true, the item is deleted and handle is set to.
+  Status InsertItem(LRUHandle* item, Cache::Handle** handle,
+                    bool free_handle_on_fail);
  Status Insert(const Slice& key, uint32_t hash, void* value, size_t charge,
-                void (*deleter)(const Slice& key, void* value),
-                Cache::CacheItemHelperCallback helper_cb,
+                DeleterFn deleter, const Cache::CacheItemHelper* helper,
                Cache::Handle** handle, Cache::Priority priority);
+  // Promote an item looked up from the secondary cache to the LRU cache. The
+  // item is only inserted into the hash table and not the LRU list, and only
+  // if the cache is not at full capacity, as is the case during Insert.  The
+  // caller should hold a reference on the LRUHandle. When the caller releases
+  // the last reference, the item is added to the LRU list.
+  // The item is promoted to the high pri or low pri pool as specified by the
+  // caller in Lookup.
+  void Promote(LRUHandle* e);
  void LRU_Remove(LRUHandle* e);
  void LRU_Insert(LRUHandle* e);

@@ -380,7 +439,7 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
  // don't mind mutex_ invoking the non-const actions.
  mutable port::Mutex mutex_;

-  std::shared_ptr<TieredCache> tiered_cache_;
+  std::shared_ptr<SecondaryCache> secondary_cache_;
 };

 class LRUCache
@@ -395,16 +454,17 @@ class LRUCache
           bool use_adaptive_mutex = kDefaultToAdaptiveMutex,
           CacheMetadataChargePolicy metadata_charge_policy =
               kDontChargeCacheMetadata,
-           const std::shared_ptr<TieredCache>& tiered_cache = nullptr);
+           const std::shared_ptr<SecondaryCache>& secondary_cache = nullptr);
  virtual ~LRUCache();
  virtual const char* Name() const override { return "LRUCache"; }
-  virtual CacheShard* GetShard(int shard) override;
-  virtual const CacheShard* GetShard(int shard) const override;
+  virtual CacheShard* GetShard(uint32_t shard) override;
+  virtual const CacheShard* GetShard(uint32_t shard) const override;
  virtual void* Value(Handle* handle) override;
  virtual size_t GetCharge(Handle* handle) const override;
  virtual uint32_t GetHash(Handle* handle) const override;
+  virtual DeleterFn GetDeleter(Handle* handle) const override;
  virtual void DisownData() override;
-  virtual void WaitAll(std::vector<Handle*>& /*handles*/) override {}
+  virtual void WaitAll(std::vector<Handle*>& handles) override;

  //  Retrieves number of elements in LRU, for unit test purpose only
  size_t TEST_GetLRUSize();
@@ -414,6 +474,7 @@ class LRUCache
 private:
  LRUCacheShard* shards_ = nullptr;
  int num_shards_ = 0;
+  std::shared_ptr<SecondaryCache> secondary_cache_;
 };

 }  // namespace ROCKSDB_NAMESPACE
@@ -9,54 +9,69 @@

 #include "cache/sharded_cache.h"

-#include <string>
+#include <algorithm>
+#include <cstdint>
+#include <memory>

+#include "util/hash.h"
+#include "util/math.h"
 #include "util/mutexlock.h"

 namespace ROCKSDB_NAMESPACE {

+namespace {
+
+inline uint32_t HashSlice(const Slice& s) {
+  return Lower32of64(GetSliceNPHash64(s));
+}
+
+}  // namespace
+
 ShardedCache::ShardedCache(size_t capacity, int num_shard_bits,
                           bool strict_capacity_limit,
                           std::shared_ptr<MemoryAllocator> allocator)
    : Cache(std::move(allocator)),
-      num_shard_bits_(num_shard_bits),
+      shard_mask_((uint32_t{1} << num_shard_bits) - 1),
      capacity_(capacity),
      strict_capacity_limit_(strict_capacity_limit),
      last_id_(1) {}

 void ShardedCache::SetCapacity(size_t capacity) {
-  int num_shards = 1 << num_shard_bits_;
+  uint32_t num_shards = GetNumShards();
  const size_t per_shard = (capacity + (num_shards - 1)) / num_shards;
  MutexLock l(&capacity_mutex_);
-  for (int s = 0; s < num_shards; s++) {
+  for (uint32_t s = 0; s < num_shards; s++) {
    GetShard(s)->SetCapacity(per_shard);
  }
  capacity_ = capacity;
 }

 void ShardedCache::SetStrictCapacityLimit(bool strict_capacity_limit) {
-  int num_shards = 1 << num_shard_bits_;
+  uint32_t num_shards = GetNumShards();
  MutexLock l(&capacity_mutex_);
-  for (int s = 0; s < num_shards; s++) {
+  for (uint32_t s = 0; s < num_shards; s++) {
    GetShard(s)->SetStrictCapacityLimit(strict_capacity_limit);
  }
  strict_capacity_limit_ = strict_capacity_limit;
 }

 Status ShardedCache::Insert(const Slice& key, void* value, size_t charge,
-                            void (*deleter)(const Slice& key, void* value),
-                            Handle** handle, Priority priority) {
+                            DeleterFn deleter, Handle** handle,
+                            Priority priority) {
  uint32_t hash = HashSlice(key);
  return GetShard(Shard(hash))
      ->Insert(key, hash, value, charge, deleter, handle, priority);
 }

 Status ShardedCache::Insert(const Slice& key, void* value,
-                            CacheItemHelperCallback helper_cb, size_t charge,
+                            const CacheItemHelper* helper, size_t charge,
                            Handle** handle, Priority priority) {
  uint32_t hash = HashSlice(key);
+  if (!helper) {
+    return Status::InvalidArgument();
+  }
  return GetShard(Shard(hash))
-      ->Insert(key, hash, value, helper_cb, charge, handle, priority);
+      ->Insert(key, hash, value, helper, charge, handle, priority);
 }

 Cache::Handle* ShardedCache::Lookup(const Slice& key, Statistics* /*stats*/) {
@@ -65,18 +80,18 @@ Cache::Handle* ShardedCache::Lookup(const Slice& key, Statistics* /*stats*/) {
 }

 Cache::Handle* ShardedCache::Lookup(const Slice& key,
-                                    CacheItemHelperCallback helper_cb,
+                                    const CacheItemHelper* helper,
                                    const CreateCallback& create_cb,
                                    Priority priority, bool wait,
                                    Statistics* /*stats*/) {
  uint32_t hash = HashSlice(key);
  return GetShard(Shard(hash))
-      ->Lookup(key, hash, helper_cb, create_cb, priority, wait);
+      ->Lookup(key, hash, helper, create_cb, priority, wait);
 }

-bool ShardedCache::isReady(Handle* handle) {
+bool ShardedCache::IsReady(Handle* handle) {
  uint32_t hash = GetHash(handle);
-  return GetShard(Shard(hash))->isReady(handle);
+  return GetShard(Shard(hash))->IsReady(handle);
 }

 void ShardedCache::Wait(Handle* handle) {
@@ -120,9 +135,9 @@ bool ShardedCache::HasStrictCapacityLimit() const {

 size_t ShardedCache::GetUsage() const {
  // We will not lock the cache when getting the usage from shards.
-  int num_shards = 1 << num_shard_bits_;
+  uint32_t num_shards = GetNumShards();
  size_t usage = 0;
-  for (int s = 0; s < num_shards; s++) {
+  for (uint32_t s = 0; s < num_shards; s++) {
    usage += GetShard(s)->GetUsage();
  }
  return usage;
@@ -134,25 +149,42 @@ size_t ShardedCache::GetUsage(Handle* handle) const {

 size_t ShardedCache::GetPinnedUsage() const {
  // We will not lock the cache when getting the usage from shards.
-  int num_shards = 1 << num_shard_bits_;
+  uint32_t num_shards = GetNumShards();
  size_t usage = 0;
-  for (int s = 0; s < num_shards; s++) {
+  for (uint32_t s = 0; s < num_shards; s++) {
    usage += GetShard(s)->GetPinnedUsage();
  }
  return usage;
 }

-void ShardedCache::ApplyToAllCacheEntries(void (*callback)(void*, size_t),
-                                          bool thread_safe) {
-  int num_shards = 1 << num_shard_bits_;
-  for (int s = 0; s < num_shards; s++) {
-    GetShard(s)->ApplyToAllCacheEntries(callback, thread_safe);
-  }
+void ShardedCache::ApplyToAllEntries(
+    const std::function<void(const Slice& key, void* value, size_t charge,
+                             DeleterFn deleter)>& callback,
+    const ApplyToAllEntriesOptions& opts) {
+  uint32_t num_shards = GetNumShards();
+  // Iterate over part of each shard, rotating between shards, to
+  // minimize impact on latency of concurrent operations.
+  std::unique_ptr<uint32_t[]> states(new uint32_t[num_shards]{});
+
+  uint32_t aepl_in_32 = static_cast<uint32_t>(
+      std::min(size_t{UINT32_MAX}, opts.average_entries_per_lock));
+  aepl_in_32 = std::min(aepl_in_32, uint32_t{1});
+
+  bool remaining_work;
+  do {
+    remaining_work = false;
+    for (uint32_t s = 0; s < num_shards; s++) {
+      if (states[s] != UINT32_MAX) {
+        GetShard(s)->ApplyToSomeEntries(callback, aepl_in_32, &states[s]);
+        remaining_work |= states[s] != UINT32_MAX;
+      }
+    }
+  } while (remaining_work);
 }

 void ShardedCache::EraseUnRefEntries() {
-  int num_shards = 1 << num_shard_bits_;
-  for (int s = 0; s < num_shards; s++) {
+  uint32_t num_shards = GetNumShards();
+  for (uint32_t s = 0; s < num_shards; s++) {
    GetShard(s)->EraseUnRefEntries();
  }
 }
@@ -167,7 +199,8 @@ std::string ShardedCache::GetPrintableOptions() const {
    snprintf(buffer, kBufferSize, "    capacity : %" ROCKSDB_PRIszt "\n",
             capacity_);
    ret.append(buffer);
-    snprintf(buffer, kBufferSize, "    num_shard_bits : %d\n", num_shard_bits_);
+    snprintf(buffer, kBufferSize, "    num_shard_bits : %d\n",
+             GetNumShardBits());
    ret.append(buffer);
    snprintf(buffer, kBufferSize, "    strict_capacity_limit : %d\n",
             strict_capacity_limit_);
@@ -192,4 +225,8 @@ int GetDefaultCacheShardBits(size_t capacity) {
  return num_shard_bits;
 }

+int ShardedCache::GetNumShardBits() const { return BitsSetToOne(shard_mask_); }
+
+uint32_t ShardedCache::GetNumShards() const { return shard_mask_ + 1; }
+
 }  // namespace ROCKSDB_NAMESPACE
@@ -14,7 +14,6 @@

 #include "port/port.h"
 #include "rocksdb/cache.h"
-#include "util/hash.h"

 namespace ROCKSDB_NAMESPACE {

@@ -24,21 +23,21 @@ class CacheShard {
  CacheShard() = default;
  virtual ~CacheShard() = default;

+  using DeleterFn = Cache::DeleterFn;
  virtual Status Insert(const Slice& key, uint32_t hash, void* value,
-                        size_t charge,
-                        void (*deleter)(const Slice& key, void* value),
+                        size_t charge, DeleterFn deleter,
                        Cache::Handle** handle, Cache::Priority priority) = 0;
  virtual Status Insert(const Slice& key, uint32_t hash, void* value,
-                        Cache::CacheItemHelperCallback helper_cb, size_t charge,
+                        const Cache::CacheItemHelper* helper, size_t charge,
                        Cache::Handle** handle, Cache::Priority priority) = 0;
  virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash) = 0;
  virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash,
-                                Cache::CacheItemHelperCallback helper_cb,
+                                const Cache::CacheItemHelper* helper,
                                const Cache::CreateCallback& create_cb,
                                Cache::Priority priority, bool wait) = 0;
  virtual bool Release(Cache::Handle* handle, bool useful,
                       bool force_erase) = 0;
-  virtual bool isReady(Cache::Handle* handle) = 0;
+  virtual bool IsReady(Cache::Handle* handle) = 0;
  virtual void Wait(Cache::Handle* handle) = 0;
  virtual bool Ref(Cache::Handle* handle) = 0;
  virtual bool Release(Cache::Handle* handle, bool force_erase) = 0;
@@ -47,8 +46,14 @@ class CacheShard {
  virtual void SetStrictCapacityLimit(bool strict_capacity_limit) = 0;
  virtual size_t GetUsage() const = 0;
  virtual size_t GetPinnedUsage() const = 0;
-  virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
-                                      bool thread_safe) = 0;
+  // Handles iterating over roughly `average_entries_per_lock` entries, using
+  // `state` to somehow record where it last ended up. Caller initially uses
+  // *state == 0 and implementation sets *state = UINT32_MAX to indicate
+  // completion.
+  virtual void ApplyToSomeEntries(
+      const std::function<void(const Slice& key, void* value, size_t charge,
+                               DeleterFn deleter)>& callback,
+      uint32_t average_entries_per_lock, uint32_t* state) = 0;
  virtual void EraseUnRefEntries() = 0;
  virtual std::string GetPrintableOptions() const { return ""; }
  void set_metadata_charge_policy(
@@ -68,33 +73,28 @@ class ShardedCache : public Cache {
  ShardedCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit,
               std::shared_ptr<MemoryAllocator> memory_allocator = nullptr);
  virtual ~ShardedCache() = default;
-  virtual const char* Name() const override = 0;
-  virtual CacheShard* GetShard(int shard) = 0;
-  virtual const CacheShard* GetShard(int shard) const = 0;
-  virtual void* Value(Handle* handle) override = 0;
-  virtual size_t GetCharge(Handle* handle) const override = 0;
-  virtual void WaitAll(std::vector<Handle*>& handles) override = 0;
+  virtual CacheShard* GetShard(uint32_t shard) = 0;
+  virtual const CacheShard* GetShard(uint32_t shard) const = 0;

  virtual uint32_t GetHash(Handle* handle) const = 0;
-  virtual void DisownData() override = 0;

  virtual void SetCapacity(size_t capacity) override;
  virtual void SetStrictCapacityLimit(bool strict_capacity_limit) override;

  virtual Status Insert(const Slice& key, void* value, size_t charge,
-                        void (*deleter)(const Slice& key, void* value),
-                        Handle** handle, Priority priority) override;
+                        DeleterFn deleter, Handle** handle,
+                        Priority priority) override;
  virtual Status Insert(const Slice& key, void* value,
-                        CacheItemHelperCallback helper_cb, size_t chargge,
+                        const CacheItemHelper* helper, size_t chargge,
                        Handle** handle = nullptr,
                        Priority priority = Priority::LOW) override;
  virtual Handle* Lookup(const Slice& key, Statistics* stats) override;
-  virtual Handle* Lookup(const Slice& key, CacheItemHelperCallback helper_cb,
+  virtual Handle* Lookup(const Slice& key, const CacheItemHelper* helper,
                         const CreateCallback& create_cb, Priority priority,
                         bool wait, Statistics* stats = nullptr) override;
  virtual bool Release(Handle* handle, bool useful,
                       bool force_erase = false) override;
-  virtual bool isReady(Handle* handle) override;
+  virtual bool IsReady(Handle* handle) override;
  virtual void Wait(Handle* handle) override;
  virtual bool Ref(Handle* handle) override;
  virtual bool Release(Handle* handle, bool force_erase = false) override;
@@ -105,24 +105,21 @@ class ShardedCache : public Cache {
  virtual size_t GetUsage() const override;
  virtual size_t GetUsage(Handle* handle) const override;
  virtual size_t GetPinnedUsage() const override;
-  virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
-                                      bool thread_safe) override;
+  virtual void ApplyToAllEntries(
+      const std::function<void(const Slice& key, void* value, size_t charge,
+                               DeleterFn deleter)>& callback,
+      const ApplyToAllEntriesOptions& opts) override;
  virtual void EraseUnRefEntries() override;
  virtual std::string GetPrintableOptions() const override;

-  int GetNumShardBits() const { return num_shard_bits_; }
+  int GetNumShardBits() const;
+  uint32_t GetNumShards() const;
+
+ protected:
+  inline uint32_t Shard(uint32_t hash) { return hash & shard_mask_; }

 private:
-  static inline uint32_t HashSlice(const Slice& s) {
-    return static_cast<uint32_t>(GetSliceNPHash64(s));
-  }
-
-  uint32_t Shard(uint32_t hash) {
-    // Note, hash >> 32 yields hash in gcc, not the zero we expect!
-    return (num_shard_bits_ > 0) ? (hash >> (32 - num_shard_bits_)) : 0;
-  }
-
-  int num_shard_bits_;
+  const uint32_t shard_mask_;
  mutable port::Mutex capacity_mutex_;
  size_t capacity_;
  bool strict_capacity_limit_;
@@ -31,16 +31,15 @@ Status ArenaWrappedDBIter::GetProperty(std::string prop_name,
 }

 void ArenaWrappedDBIter::Init(
-    Env* env, const ReadOptions& read_options,
-    const ImmutableCFOptions& cf_options,
+    Env* env, const ReadOptions& read_options, const ImmutableOptions& ioptions,
    const MutableCFOptions& mutable_cf_options, const Version* version,
    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iteration,
    uint64_t version_number, ReadCallback* read_callback, DBImpl* db_impl,
    ColumnFamilyData* cfd, bool expose_blob_index, bool allow_refresh) {
  auto mem = arena_.AllocateAligned(sizeof(DBIter));
  db_iter_ =
-      new (mem) DBIter(env, read_options, cf_options, mutable_cf_options,
-                       cf_options.user_comparator, /* iter */ nullptr, version,
+      new (mem) DBIter(env, read_options, ioptions, mutable_cf_options,
+                       ioptions.user_comparator, /* iter */ nullptr, version,
                       sequence, true, max_sequential_skip_in_iteration,
                       read_callback, db_impl, cfd, expose_blob_index);
  sv_number_ = version_number;
@@ -88,16 +87,15 @@ Status ArenaWrappedDBIter::Refresh() {
 }

 ArenaWrappedDBIter* NewArenaWrappedDbIterator(
-    Env* env, const ReadOptions& read_options,
-    const ImmutableCFOptions& cf_options,
+    Env* env, const ReadOptions& read_options, const ImmutableOptions& ioptions,
    const MutableCFOptions& mutable_cf_options, const Version* version,
    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations,
    uint64_t version_number, ReadCallback* read_callback, DBImpl* db_impl,
    ColumnFamilyData* cfd, bool expose_blob_index, bool allow_refresh) {
  ArenaWrappedDBIter* iter = new ArenaWrappedDBIter();
-  iter->Init(env, read_options, cf_options, mutable_cf_options, version,
-             sequence, max_sequential_skip_in_iterations, version_number,
-             read_callback, db_impl, cfd, expose_blob_index, allow_refresh);
+  iter->Init(env, read_options, ioptions, mutable_cf_options, version, sequence,
+             max_sequential_skip_in_iterations, version_number, read_callback,
+             db_impl, cfd, expose_blob_index, allow_refresh);
  if (db_impl != nullptr && cfd != nullptr && allow_refresh) {
    iter->StoreRefreshInfo(db_impl, cfd, read_callback, expose_blob_index);
  }
@@ -72,7 +72,7 @@ class ArenaWrappedDBIter : public Iterator {
  Status Refresh() override;

  void Init(Env* env, const ReadOptions& read_options,
-            const ImmutableCFOptions& cf_options,
+            const ImmutableOptions& ioptions,
            const MutableCFOptions& mutable_cf_options, const Version* version,
            const SequenceNumber& sequence,
            uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
@@ -105,8 +105,7 @@ class ArenaWrappedDBIter : public Iterator {
 // `db_impl` and `cfd` are used for reneweal. If left null, renewal will not
 // be supported.
 extern ArenaWrappedDBIter* NewArenaWrappedDbIterator(
-    Env* env, const ReadOptions& read_options,
-    const ImmutableCFOptions& cf_options,
+    Env* env, const ReadOptions& read_options, const ImmutableOptions& ioptions,
    const MutableCFOptions& mutable_cf_options, const Version* version,
    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations,
    uint64_t version_number, ReadCallback* read_callback,
@@ -0,0 +1,22 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#include "db/blob/blob_fetcher.h"
+
+#include "db/version_set.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+Status BlobFetcher::FetchBlob(const Slice& user_key, const Slice& blob_index,
+                              PinnableSlice* blob_value) {
+  Status s;
+  assert(version_);
+  constexpr uint64_t* bytes_read = nullptr;
+  s = version_->GetBlob(read_options_, user_key, blob_index, blob_value,
+                        bytes_read);
+  return s;
+}
+
+}  // namespace ROCKSDB_NAMESPACE
@@ -0,0 +1,26 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include "rocksdb/options.h"
+#include "rocksdb/status.h"
+
+namespace ROCKSDB_NAMESPACE {
+class Version;
+
+class BlobFetcher {
+ public:
+  BlobFetcher(Version* version, const ReadOptions& read_options)
+      : version_(version), read_options_(read_options) {}
+
+  Status FetchBlob(const Slice& user_key, const Slice& blob_index,
+                   PinnableSlice* blob_value);
+
+ private:
+  Version* version_;
+  ReadOptions read_options_;
+};
+}  // namespace ROCKSDB_NAMESPACE
@@ -29,7 +29,7 @@ namespace ROCKSDB_NAMESPACE {

 BlobFileBuilder::BlobFileBuilder(
    VersionSet* versions, FileSystem* fs,
-    const ImmutableCFOptions* immutable_cf_options,
+    const ImmutableOptions* immutable_options,
    const MutableCFOptions* mutable_cf_options, const FileOptions* file_options,
    int job_id, uint32_t column_family_id,
    const std::string& column_family_name, Env::IOPriority io_priority,
@@ -39,14 +39,14 @@ BlobFileBuilder::BlobFileBuilder(
    std::vector<std::string>* blob_file_paths,
    std::vector<BlobFileAddition>* blob_file_additions)
    : BlobFileBuilder([versions]() { return versions->NewFileNumber(); }, fs,
-                      immutable_cf_options, mutable_cf_options, file_options,
+                      immutable_options, mutable_cf_options, file_options,
                      job_id, column_family_id, column_family_name, io_priority,
                      write_hint, io_tracer, blob_callback, blob_file_paths,
                      blob_file_additions) {}

 BlobFileBuilder::BlobFileBuilder(
    std::function<uint64_t()> file_number_generator, FileSystem* fs,
-    const ImmutableCFOptions* immutable_cf_options,
+    const ImmutableOptions* immutable_options,
    const MutableCFOptions* mutable_cf_options, const FileOptions* file_options,
    int job_id, uint32_t column_family_id,
    const std::string& column_family_name, Env::IOPriority io_priority,
@@ -57,7 +57,7 @@ BlobFileBuilder::BlobFileBuilder(
    std::vector<BlobFileAddition>* blob_file_additions)
    : file_number_generator_(std::move(file_number_generator)),
      fs_(fs),
-      immutable_cf_options_(immutable_cf_options),
+      immutable_options_(immutable_options),
      min_blob_size_(mutable_cf_options->min_blob_size),
      blob_file_size_(mutable_cf_options->blob_file_size),
      blob_compression_type_(mutable_cf_options->blob_compression_type),
@@ -75,7 +75,7 @@ BlobFileBuilder::BlobFileBuilder(
      blob_bytes_(0) {
  assert(file_number_generator_);
  assert(fs_);
-  assert(immutable_cf_options_);
+  assert(immutable_options_);
  assert(file_options_);
  assert(blob_file_paths_);
  assert(blob_file_paths_->empty());
@@ -156,10 +156,10 @@ Status BlobFileBuilder::OpenBlobFileIfNeeded() {
  assert(file_number_generator_);
  const uint64_t blob_file_number = file_number_generator_();

-  assert(immutable_cf_options_);
-  assert(!immutable_cf_options_->cf_paths.empty());
-  std::string blob_file_path = BlobFileName(
-      immutable_cf_options_->cf_paths.front().path, blob_file_number);
+  assert(immutable_options_);
+  assert(!immutable_options_->cf_paths.empty());
+  std::string blob_file_path =
+      BlobFileName(immutable_options_->cf_paths.front().path, blob_file_number);

  std::unique_ptr<FSWritableFile> file;

@@ -184,20 +184,20 @@ Status BlobFileBuilder::OpenBlobFileIfNeeded() {
  assert(file);
  file->SetIOPriority(io_priority_);
  file->SetWriteLifeTimeHint(write_hint_);
-  FileTypeSet tmp_set = immutable_cf_options_->checksum_handoff_file_types;
-  Statistics* const statistics = immutable_cf_options_->statistics;
+  FileTypeSet tmp_set = immutable_options_->checksum_handoff_file_types;
+  Statistics* const statistics = immutable_options_->stats;
  std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
      std::move(file), blob_file_paths_->back(), *file_options_,
-      immutable_cf_options_->clock, io_tracer_, statistics,
-      immutable_cf_options_->listeners,
-      immutable_cf_options_->file_checksum_gen_factory.get(),
+      immutable_options_->clock, io_tracer_, statistics,
+      immutable_options_->listeners,
+      immutable_options_->file_checksum_gen_factory.get(),
      tmp_set.Contains(FileType::kBlobFile)));

  constexpr bool do_flush = false;

  std::unique_ptr<BlobLogWriter> blob_log_writer(new BlobLogWriter(
-      std::move(file_writer), immutable_cf_options_->clock, statistics,
-      blob_file_number, immutable_cf_options_->use_fsync, do_flush));
+      std::move(file_writer), immutable_options_->clock, statistics,
+      blob_file_number, immutable_options_->use_fsync, do_flush));

  constexpr bool has_ttl = false;
  constexpr ExpirationRange expiration_range;
@@ -300,8 +300,8 @@ Status BlobFileBuilder::CloseBlobFile() {
                                     std::move(checksum_method),
                                     std::move(checksum_value));

-  assert(immutable_cf_options_);
-  ROCKS_LOG_INFO(immutable_cf_options_->info_log,
+  assert(immutable_options_);
+  ROCKS_LOG_INFO(immutable_options_->logger,
                 "[%s] [JOB %d] Generated blob file #%" PRIu64 ": %" PRIu64
                 " total blobs, %" PRIu64 " total bytes",
                 column_family_name_.c_str(), job_id_, blob_file_number,
@@ -19,7 +19,7 @@ namespace ROCKSDB_NAMESPACE {
 class VersionSet;
 class FileSystem;
 class SystemClock;
-struct ImmutableCFOptions;
+struct ImmutableOptions;
 struct MutableCFOptions;
 struct FileOptions;
 class BlobFileAddition;
@@ -32,7 +32,7 @@ class BlobFileCompletionCallback;
 class BlobFileBuilder {
 public:
  BlobFileBuilder(VersionSet* versions, FileSystem* fs,
-                  const ImmutableCFOptions* immutable_cf_options,
+                  const ImmutableOptions* immutable_options,
                  const MutableCFOptions* mutable_cf_options,
                  const FileOptions* file_options, int job_id,
                  uint32_t column_family_id,
@@ -45,8 +45,7 @@ class BlobFileBuilder {
                  std::vector<BlobFileAddition>* blob_file_additions);

  BlobFileBuilder(std::function<uint64_t()> file_number_generator,
-                  FileSystem* fs,
-                  const ImmutableCFOptions* immutable_cf_options,
+                  FileSystem* fs, const ImmutableOptions* immutable_options,
                  const MutableCFOptions* mutable_cf_options,
                  const FileOptions* file_options, int job_id,
                  uint32_t column_family_id,
@@ -78,7 +77,7 @@ class BlobFileBuilder {

  std::function<uint64_t()> file_number_generator_;
  FileSystem* fs_;
-  const ImmutableCFOptions* immutable_cf_options_;
+  const ImmutableOptions* immutable_options_;
  uint64_t min_blob_size_;
  uint64_t blob_file_size_;
  CompressionType blob_compression_type_;
@@ -129,7 +129,7 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckOneFile) {
  options.enable_blob_files = true;
  options.env = &mock_env_;

-  ImmutableCFOptions immutable_cf_options(options);
+  ImmutableOptions immutable_options(options);
  MutableCFOptions mutable_cf_options(options);

  constexpr int job_id = 1;
@@ -141,12 +141,11 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckOneFile) {
  std::vector<std::string> blob_file_paths;
  std::vector<BlobFileAddition> blob_file_additions;

-  BlobFileBuilder builder(TestFileNumberGenerator(), fs_, &immutable_cf_options,
-                          &mutable_cf_options, &file_options_, job_id,
-                          column_family_id, column_family_name, io_priority,
-                          write_hint, nullptr /*IOTracer*/,
-                          nullptr /*BlobFileCompletionCallback*/,
-                          &blob_file_paths, &blob_file_additions);
+  BlobFileBuilder builder(
+      TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
+      &file_options_, job_id, column_family_id, column_family_name, io_priority,
+      write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
+      &blob_file_paths, &blob_file_additions);

  std::vector<std::pair<std::string, std::string>> expected_key_value_pairs(
      number_of_blobs);
@@ -178,9 +177,9 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckOneFile) {

  const std::string& blob_file_path = blob_file_paths[0];

-  ASSERT_EQ(blob_file_path,
-            BlobFileName(immutable_cf_options.cf_paths.front().path,
-                         blob_file_number));
+  ASSERT_EQ(
+      blob_file_path,
+      BlobFileName(immutable_options.cf_paths.front().path, blob_file_number));

  ASSERT_EQ(blob_file_additions.size(), 1);

@@ -214,7 +213,7 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckMultipleFiles) {
  options.blob_file_size = value_size;
  options.env = &mock_env_;

-  ImmutableCFOptions immutable_cf_options(options);
+  ImmutableOptions immutable_options(options);
  MutableCFOptions mutable_cf_options(options);

  constexpr int job_id = 1;
@@ -226,12 +225,11 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckMultipleFiles) {
  std::vector<std::string> blob_file_paths;
  std::vector<BlobFileAddition> blob_file_additions;

-  BlobFileBuilder builder(TestFileNumberGenerator(), fs_, &immutable_cf_options,
-                          &mutable_cf_options, &file_options_, job_id,
-                          column_family_id, column_family_name, io_priority,
-                          write_hint, nullptr /*IOTracer*/,
-                          nullptr /*BlobFileCompletionCallback*/,
-                          &blob_file_paths, &blob_file_additions);
+  BlobFileBuilder builder(
+      TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
+      &file_options_, job_id, column_family_id, column_family_name, io_priority,
+      write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
+      &blob_file_paths, &blob_file_additions);

  std::vector<std::pair<std::string, std::string>> expected_key_value_pairs(
      number_of_blobs);
@@ -264,7 +262,7 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckMultipleFiles) {
    const uint64_t blob_file_number = i + 2;

    ASSERT_EQ(blob_file_paths[i],
-              BlobFileName(immutable_cf_options.cf_paths.front().path,
+              BlobFileName(immutable_options.cf_paths.front().path,
                           blob_file_number));

    const auto& blob_file_addition = blob_file_additions[i];
@@ -301,7 +299,7 @@ TEST_F(BlobFileBuilderTest, InlinedValues) {
  options.min_blob_size = 1024;
  options.env = &mock_env_;

-  ImmutableCFOptions immutable_cf_options(options);
+  ImmutableOptions immutable_options(options);
  MutableCFOptions mutable_cf_options(options);

  constexpr int job_id = 1;
@@ -313,12 +311,11 @@ TEST_F(BlobFileBuilderTest, InlinedValues) {
  std::vector<std::string> blob_file_paths;
  std::vector<BlobFileAddition> blob_file_additions;

-  BlobFileBuilder builder(TestFileNumberGenerator(), fs_, &immutable_cf_options,
-                          &mutable_cf_options, &file_options_, job_id,
-                          column_family_id, column_family_name, io_priority,
-                          write_hint, nullptr /*IOTracer*/,
-                          nullptr /*BlobFileCompletionCallback*/,
-                          &blob_file_paths, &blob_file_additions);
+  BlobFileBuilder builder(
+      TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
+      &file_options_, job_id, column_family_id, column_family_name, io_priority,
+      write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
+      &blob_file_paths, &blob_file_additions);

  for (size_t i = 0; i < number_of_blobs; ++i) {
    const std::string key = std::to_string(i);
@@ -355,7 +352,7 @@ TEST_F(BlobFileBuilderTest, Compression) {
  options.blob_compression_type = kSnappyCompression;
  options.env = &mock_env_;

-  ImmutableCFOptions immutable_cf_options(options);
+  ImmutableOptions immutable_options(options);
  MutableCFOptions mutable_cf_options(options);

  constexpr int job_id = 1;
@@ -367,12 +364,11 @@ TEST_F(BlobFileBuilderTest, Compression) {
  std::vector<std::string> blob_file_paths;
  std::vector<BlobFileAddition> blob_file_additions;

-  BlobFileBuilder builder(TestFileNumberGenerator(), fs_, &immutable_cf_options,
-                          &mutable_cf_options, &file_options_, job_id,
-                          column_family_id, column_family_name, io_priority,
-                          write_hint, nullptr /*IOTracer*/,
-                          nullptr /*BlobFileCompletionCallback*/,
-                          &blob_file_paths, &blob_file_additions);
+  BlobFileBuilder builder(
+      TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
+      &file_options_, job_id, column_family_id, column_family_name, io_priority,
+      write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
+      &blob_file_paths, &blob_file_additions);

  const std::string key("1");
  const std::string uncompressed_value(value_size, 'x');
@@ -391,9 +387,9 @@ TEST_F(BlobFileBuilderTest, Compression) {

  const std::string& blob_file_path = blob_file_paths[0];

-  ASSERT_EQ(blob_file_path,
-            BlobFileName(immutable_cf_options.cf_paths.front().path,
-                         blob_file_number));
+  ASSERT_EQ(
+      blob_file_path,
+      BlobFileName(immutable_options.cf_paths.front().path, blob_file_number));

  ASSERT_EQ(blob_file_additions.size(), 1);

@@ -438,7 +434,7 @@ TEST_F(BlobFileBuilderTest, CompressionError) {
  options.enable_blob_files = true;
  options.blob_compression_type = kSnappyCompression;
  options.env = &mock_env_;
-  ImmutableCFOptions immutable_cf_options(options);
+  ImmutableOptions immutable_options(options);
  MutableCFOptions mutable_cf_options(options);

  constexpr int job_id = 1;
@@ -450,12 +446,11 @@ TEST_F(BlobFileBuilderTest, CompressionError) {
  std::vector<std::string> blob_file_paths;
  std::vector<BlobFileAddition> blob_file_additions;

-  BlobFileBuilder builder(TestFileNumberGenerator(), fs_, &immutable_cf_options,
-                          &mutable_cf_options, &file_options_, job_id,
-                          column_family_id, column_family_name, io_priority,
-                          write_hint, nullptr /*IOTracer*/,
-                          nullptr /*BlobFileCompletionCallback*/,
-                          &blob_file_paths, &blob_file_additions);
+  BlobFileBuilder builder(
+      TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
+      &file_options_, job_id, column_family_id, column_family_name, io_priority,
+      write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
+      &blob_file_paths, &blob_file_additions);

  SyncPoint::GetInstance()->SetCallBack("CompressData:TamperWithReturnValue",
                                        [](void* arg) {
@@ -477,9 +472,9 @@ TEST_F(BlobFileBuilderTest, CompressionError) {
  constexpr uint64_t blob_file_number = 2;

  ASSERT_EQ(blob_file_paths.size(), 1);
-  ASSERT_EQ(blob_file_paths[0],
-            BlobFileName(immutable_cf_options.cf_paths.front().path,
-                         blob_file_number));
+  ASSERT_EQ(
+      blob_file_paths[0],
+      BlobFileName(immutable_options.cf_paths.front().path, blob_file_number));

  ASSERT_TRUE(blob_file_additions.empty());
 }
@@ -517,7 +512,7 @@ TEST_F(BlobFileBuilderTest, Checksum) {
      std::make_shared<DummyFileChecksumGenFactory>();
  options.env = &mock_env_;

-  ImmutableCFOptions immutable_cf_options(options);
+  ImmutableOptions immutable_options(options);
  MutableCFOptions mutable_cf_options(options);

  constexpr int job_id = 1;
@@ -529,12 +524,11 @@ TEST_F(BlobFileBuilderTest, Checksum) {
  std::vector<std::string> blob_file_paths;
  std::vector<BlobFileAddition> blob_file_additions;

-  BlobFileBuilder builder(TestFileNumberGenerator(), fs_, &immutable_cf_options,
-                          &mutable_cf_options, &file_options_, job_id,
-                          column_family_id, column_family_name, io_priority,
-                          write_hint, nullptr /*IOTracer*/,
-                          nullptr /*BlobFileCompletionCallback*/,
-                          &blob_file_paths, &blob_file_additions);
+  BlobFileBuilder builder(
+      TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
+      &file_options_, job_id, column_family_id, column_family_name, io_priority,
+      write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
+      &blob_file_paths, &blob_file_additions);

  const std::string key("1");
  const std::string value("deadbeef");
@@ -553,9 +547,9 @@ TEST_F(BlobFileBuilderTest, Checksum) {

  const std::string& blob_file_path = blob_file_paths[0];

-  ASSERT_EQ(blob_file_path,
-            BlobFileName(immutable_cf_options.cf_paths.front().path,
-                         blob_file_number));
+  ASSERT_EQ(
+      blob_file_path,
+      BlobFileName(immutable_options.cf_paths.front().path, blob_file_number));

  ASSERT_EQ(blob_file_additions.size(), 1);

@@ -614,7 +608,7 @@ TEST_P(BlobFileBuilderIOErrorTest, IOError) {
  options.blob_file_size = value_size;
  options.env = &mock_env_;

-  ImmutableCFOptions immutable_cf_options(options);
+  ImmutableOptions immutable_options(options);
  MutableCFOptions mutable_cf_options(options);

  constexpr int job_id = 1;
@@ -626,12 +620,11 @@ TEST_P(BlobFileBuilderIOErrorTest, IOError) {
  std::vector<std::string> blob_file_paths;
  std::vector<BlobFileAddition> blob_file_additions;

-  BlobFileBuilder builder(TestFileNumberGenerator(), fs_, &immutable_cf_options,
-                          &mutable_cf_options, &file_options_, job_id,
-                          column_family_id, column_family_name, io_priority,
-                          write_hint, nullptr /*IOTracer*/,
-                          nullptr /*BlobFileCompletionCallback*/,
-                          &blob_file_paths, &blob_file_additions);
+  BlobFileBuilder builder(
+      TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
+      &file_options_, job_id, column_family_id, column_family_name, io_priority,
+      write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
+      &blob_file_paths, &blob_file_additions);

  SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* arg) {
    Status* const s = static_cast<Status*>(arg);
@@ -658,7 +651,7 @@ TEST_P(BlobFileBuilderIOErrorTest, IOError) {

    ASSERT_EQ(blob_file_paths.size(), 1);
    ASSERT_EQ(blob_file_paths[0],
-              BlobFileName(immutable_cf_options.cf_paths.front().path,
+              BlobFileName(immutable_options.cf_paths.front().path,
                           blob_file_number));
  }

@@ -19,20 +19,20 @@
 namespace ROCKSDB_NAMESPACE {

 BlobFileCache::BlobFileCache(Cache* cache,
-                             const ImmutableCFOptions* immutable_cf_options,
+                             const ImmutableOptions* immutable_options,
                             const FileOptions* file_options,
                             uint32_t column_family_id,
                             HistogramImpl* blob_file_read_hist,
                             const std::shared_ptr<IOTracer>& io_tracer)
    : cache_(cache),
      mutex_(kNumberOfMutexStripes, kGetSliceNPHash64UnseededFnPtr),
-      immutable_cf_options_(immutable_cf_options),
+      immutable_options_(immutable_options),
      file_options_(file_options),
      column_family_id_(column_family_id),
      blob_file_read_hist_(blob_file_read_hist),
      io_tracer_(io_tracer) {
  assert(cache_);
-  assert(immutable_cf_options_);
+  assert(immutable_options_);
  assert(file_options_);
 }

@@ -63,8 +63,8 @@ Status BlobFileCache::GetBlobFileReader(
    return Status::OK();
  }

-  assert(immutable_cf_options_);
-  Statistics* const statistics = immutable_cf_options_->statistics;
+  assert(immutable_options_);
+  Statistics* const statistics = immutable_options_->stats;

  RecordTick(statistics, NO_FILE_OPENS);

@@ -73,7 +73,7 @@ Status BlobFileCache::GetBlobFileReader(
  {
    assert(file_options_);
    const Status s = BlobFileReader::Create(
-        *immutable_cf_options_, *file_options_, column_family_id_,
+        *immutable_options_, *file_options_, column_family_id_,
        blob_file_read_hist_, blob_file_number, io_tracer_, &reader);
    if (!s.ok()) {
      RecordTick(statistics, NO_FILE_ERRORS);
@@ -14,7 +14,7 @@
 namespace ROCKSDB_NAMESPACE {

 class Cache;
-struct ImmutableCFOptions;
+struct ImmutableOptions;
 struct FileOptions;
 class HistogramImpl;
 class Status;
@@ -24,7 +24,7 @@ class IOTracer;

 class BlobFileCache {
 public:
-  BlobFileCache(Cache* cache, const ImmutableCFOptions* immutable_cf_options,
+  BlobFileCache(Cache* cache, const ImmutableOptions* immutable_options,
                const FileOptions* file_options, uint32_t column_family_id,
                HistogramImpl* blob_file_read_hist,
                const std::shared_ptr<IOTracer>& io_tracer);
@@ -40,7 +40,7 @@ class BlobFileCache {
  // Note: mutex_ below is used to guard against multiple threads racing to open
  // the same file.
  Striped<port::Mutex, Slice> mutex_;
-  const ImmutableCFOptions* immutable_cf_options_;
+  const ImmutableOptions* immutable_options_;
  const FileOptions* file_options_;
  uint32_t column_family_id_;
  HistogramImpl* blob_file_read_hist_;
@@ -29,28 +29,27 @@ namespace {

 // Creates a test blob file with a single blob in it.
 void WriteBlobFile(uint32_t column_family_id,
-                   const ImmutableCFOptions& immutable_cf_options,
+                   const ImmutableOptions& immutable_options,
                   uint64_t blob_file_number) {
-  assert(!immutable_cf_options.cf_paths.empty());
+  assert(!immutable_options.cf_paths.empty());

-  const std::string blob_file_path = BlobFileName(
-      immutable_cf_options.cf_paths.front().path, blob_file_number);
+  const std::string blob_file_path =
+      BlobFileName(immutable_options.cf_paths.front().path, blob_file_number);

  std::unique_ptr<FSWritableFile> file;
-  ASSERT_OK(NewWritableFile(immutable_cf_options.fs, blob_file_path, &file,
+  ASSERT_OK(NewWritableFile(immutable_options.fs.get(), blob_file_path, &file,
                            FileOptions()));

-  std::unique_ptr<WritableFileWriter> file_writer(
-      new WritableFileWriter(std::move(file), blob_file_path, FileOptions(),
-                             immutable_cf_options.clock));
+  std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
+      std::move(file), blob_file_path, FileOptions(), immutable_options.clock));

  constexpr Statistics* statistics = nullptr;
  constexpr bool use_fsync = false;
  constexpr bool do_flush = false;

-  BlobLogWriter blob_log_writer(std::move(file_writer),
-                                immutable_cf_options.clock, statistics,
-                                blob_file_number, use_fsync, do_flush);
+  BlobLogWriter blob_log_writer(std::move(file_writer), immutable_options.clock,
+                                statistics, blob_file_number, use_fsync,
+                                do_flush);

  constexpr bool has_ttl = false;
  constexpr ExpirationRange expiration_range;
@@ -100,10 +99,10 @@ TEST_F(BlobFileCacheTest, GetBlobFileReader) {
  options.enable_blob_files = true;

  constexpr uint32_t column_family_id = 1;
-  ImmutableCFOptions immutable_cf_options(options);
+  ImmutableOptions immutable_options(options);
  constexpr uint64_t blob_file_number = 123;

-  WriteBlobFile(column_family_id, immutable_cf_options, blob_file_number);
+  WriteBlobFile(column_family_id, immutable_options, blob_file_number);

  constexpr size_t capacity = 10;
  std::shared_ptr<Cache> backing_cache = NewLRUCache(capacity);
@@ -111,7 +110,7 @@ TEST_F(BlobFileCacheTest, GetBlobFileReader) {
  FileOptions file_options;
  constexpr HistogramImpl* blob_file_read_hist = nullptr;

-  BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options,
+  BlobFileCache blob_file_cache(backing_cache.get(), &immutable_options,
                                &file_options, column_family_id,
                                blob_file_read_hist, nullptr /*IOTracer*/);

@@ -145,10 +144,10 @@ TEST_F(BlobFileCacheTest, GetBlobFileReader_Race) {
  options.enable_blob_files = true;

  constexpr uint32_t column_family_id = 1;
-  ImmutableCFOptions immutable_cf_options(options);
+  ImmutableOptions immutable_options(options);
  constexpr uint64_t blob_file_number = 123;

-  WriteBlobFile(column_family_id, immutable_cf_options, blob_file_number);
+  WriteBlobFile(column_family_id, immutable_options, blob_file_number);

  constexpr size_t capacity = 10;
  std::shared_ptr<Cache> backing_cache = NewLRUCache(capacity);
@@ -156,7 +155,7 @@ TEST_F(BlobFileCacheTest, GetBlobFileReader_Race) {
  FileOptions file_options;
  constexpr HistogramImpl* blob_file_read_hist = nullptr;

-  BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options,
+  BlobFileCache blob_file_cache(backing_cache.get(), &immutable_options,
                                &file_options, column_family_id,
                                blob_file_read_hist, nullptr /*IOTracer*/);

@@ -199,12 +198,12 @@ TEST_F(BlobFileCacheTest, GetBlobFileReader_IOError) {
  constexpr size_t capacity = 10;
  std::shared_ptr<Cache> backing_cache = NewLRUCache(capacity);

-  ImmutableCFOptions immutable_cf_options(options);
+  ImmutableOptions immutable_options(options);
  FileOptions file_options;
  constexpr uint32_t column_family_id = 1;
  constexpr HistogramImpl* blob_file_read_hist = nullptr;

-  BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options,
+  BlobFileCache blob_file_cache(backing_cache.get(), &immutable_options,
                                &file_options, column_family_id,
                                blob_file_read_hist, nullptr /*IOTracer*/);

@@ -231,10 +230,10 @@ TEST_F(BlobFileCacheTest, GetBlobFileReader_CacheFull) {
  options.enable_blob_files = true;

  constexpr uint32_t column_family_id = 1;
-  ImmutableCFOptions immutable_cf_options(options);
+  ImmutableOptions immutable_options(options);
  constexpr uint64_t blob_file_number = 123;

-  WriteBlobFile(column_family_id, immutable_cf_options, blob_file_number);
+  WriteBlobFile(column_family_id, immutable_options, blob_file_number);

  constexpr size_t capacity = 0;
  constexpr int num_shard_bits = -1;  // determined automatically
@@ -245,7 +244,7 @@ TEST_F(BlobFileCacheTest, GetBlobFileReader_CacheFull) {
  FileOptions file_options;
  constexpr HistogramImpl* blob_file_read_hist = nullptr;

-  BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options,
+  BlobFileCache blob_file_cache(backing_cache.get(), &immutable_options,
                                &file_options, column_family_id,
                                blob_file_read_hist, nullptr /*IOTracer*/);

@@ -21,10 +21,9 @@
 namespace ROCKSDB_NAMESPACE {

 Status BlobFileReader::Create(
-    const ImmutableCFOptions& immutable_cf_options,
-    const FileOptions& file_options, uint32_t column_family_id,
-    HistogramImpl* blob_file_read_hist, uint64_t blob_file_number,
-    const std::shared_ptr<IOTracer>& io_tracer,
+    const ImmutableOptions& immutable_options, const FileOptions& file_options,
+    uint32_t column_family_id, HistogramImpl* blob_file_read_hist,
+    uint64_t blob_file_number, const std::shared_ptr<IOTracer>& io_tracer,
    std::unique_ptr<BlobFileReader>* blob_file_reader) {
  assert(blob_file_reader);
  assert(!*blob_file_reader);
@@ -34,7 +33,7 @@ Status BlobFileReader::Create(

  {
    const Status s =
-        OpenFile(immutable_cf_options, file_options, blob_file_read_hist,
+        OpenFile(immutable_options, file_options, blob_file_read_hist,
                 blob_file_number, io_tracer, &file_size, &file_reader);
    if (!s.ok()) {
      return s;
@@ -67,20 +66,20 @@ Status BlobFileReader::Create(
 }

 Status BlobFileReader::OpenFile(
-    const ImmutableCFOptions& immutable_cf_options,
-    const FileOptions& file_opts, HistogramImpl* blob_file_read_hist,
-    uint64_t blob_file_number, const std::shared_ptr<IOTracer>& io_tracer,
-    uint64_t* file_size, std::unique_ptr<RandomAccessFileReader>* file_reader) {
+    const ImmutableOptions& immutable_options, const FileOptions& file_opts,
+    HistogramImpl* blob_file_read_hist, uint64_t blob_file_number,
+    const std::shared_ptr<IOTracer>& io_tracer, uint64_t* file_size,
+    std::unique_ptr<RandomAccessFileReader>* file_reader) {
  assert(file_size);
  assert(file_reader);

-  const auto& cf_paths = immutable_cf_options.cf_paths;
+  const auto& cf_paths = immutable_options.cf_paths;
  assert(!cf_paths.empty());

  const std::string blob_file_path =
      BlobFileName(cf_paths.front().path, blob_file_number);

-  FileSystem* const fs = immutable_cf_options.fs;
+  FileSystem* const fs = immutable_options.fs.get();
  assert(fs);

  constexpr IODebugContext* dbg = nullptr;
@@ -113,15 +112,15 @@ Status BlobFileReader::OpenFile(

  assert(file);

-  if (immutable_cf_options.advise_random_on_open) {
+  if (immutable_options.advise_random_on_open) {
    file->Hint(FSRandomAccessFile::kRandom);
  }

  file_reader->reset(new RandomAccessFileReader(
-      std::move(file), blob_file_path, immutable_cf_options.clock, io_tracer,
-      immutable_cf_options.statistics, BLOB_DB_BLOB_FILE_READ_MICROS,
-      blob_file_read_hist, immutable_cf_options.rate_limiter.get(),
-      immutable_cf_options.listeners));
+      std::move(file), blob_file_path, immutable_options.clock, io_tracer,
+      immutable_options.stats, BLOB_DB_BLOB_FILE_READ_MICROS,
+      blob_file_read_hist, immutable_options.rate_limiter.get(),
+      immutable_options.listeners));

  return Status::OK();
 }
@@ -15,7 +15,7 @@
 namespace ROCKSDB_NAMESPACE {

 class Status;
-struct ImmutableCFOptions;
+struct ImmutableOptions;
 struct FileOptions;
 class HistogramImpl;
 struct ReadOptions;
@@ -24,7 +24,7 @@ class PinnableSlice;

 class BlobFileReader {
 public:
-  static Status Create(const ImmutableCFOptions& immutable_cf_options,
+  static Status Create(const ImmutableOptions& immutable_options,
                       const FileOptions& file_options,
                       uint32_t column_family_id,
                       HistogramImpl* blob_file_read_hist,
@@ -46,7 +46,7 @@ class BlobFileReader {
  BlobFileReader(std::unique_ptr<RandomAccessFileReader>&& file_reader,
                 uint64_t file_size, CompressionType compression_type);

-  static Status OpenFile(const ImmutableCFOptions& immutable_cf_options,
+  static Status OpenFile(const ImmutableOptions& immutable_options,
                         const FileOptions& file_opts,
                         HistogramImpl* blob_file_read_hist,
                         uint64_t blob_file_number,
@@ -30,35 +30,34 @@ namespace {
 // Creates a test blob file with a single blob in it. Note: this method
 // makes it possible to test various corner cases by allowing the caller
 // to specify the contents of various blob file header/footer fields.
-void WriteBlobFile(const ImmutableCFOptions& immutable_cf_options,
+void WriteBlobFile(const ImmutableOptions& immutable_options,
                   uint32_t column_family_id, bool has_ttl,
                   const ExpirationRange& expiration_range_header,
                   const ExpirationRange& expiration_range_footer,
                   uint64_t blob_file_number, const Slice& key,
                   const Slice& blob, CompressionType compression_type,
                   uint64_t* blob_offset, uint64_t* blob_size) {
-  assert(!immutable_cf_options.cf_paths.empty());
+  assert(!immutable_options.cf_paths.empty());
  assert(blob_offset);
  assert(blob_size);

-  const std::string blob_file_path = BlobFileName(
-      immutable_cf_options.cf_paths.front().path, blob_file_number);
+  const std::string blob_file_path =
+      BlobFileName(immutable_options.cf_paths.front().path, blob_file_number);

  std::unique_ptr<FSWritableFile> file;
-  ASSERT_OK(NewWritableFile(immutable_cf_options.fs, blob_file_path, &file,
+  ASSERT_OK(NewWritableFile(immutable_options.fs.get(), blob_file_path, &file,
                            FileOptions()));

-  std::unique_ptr<WritableFileWriter> file_writer(
-      new WritableFileWriter(std::move(file), blob_file_path, FileOptions(),
-                             immutable_cf_options.clock));
+  std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
+      std::move(file), blob_file_path, FileOptions(), immutable_options.clock));

  constexpr Statistics* statistics = nullptr;
  constexpr bool use_fsync = false;
  constexpr bool do_flush = false;

-  BlobLogWriter blob_log_writer(std::move(file_writer),
-                                immutable_cf_options.clock, statistics,
-                                blob_file_number, use_fsync, do_flush);
+  BlobLogWriter blob_log_writer(std::move(file_writer), immutable_options.clock,
+                                statistics, blob_file_number, use_fsync,
+                                do_flush);

  BlobLogHeader header(column_family_id, compression_type, has_ttl,
                       expiration_range_header);
@@ -122,7 +121,7 @@ TEST_F(BlobFileReaderTest, CreateReaderAndGetBlob) {
      0);
  options.enable_blob_files = true;

-  ImmutableCFOptions immutable_cf_options(options);
+  ImmutableOptions immutable_options(options);

  constexpr uint32_t column_family_id = 1;
  constexpr bool has_ttl = false;
@@ -134,17 +133,17 @@ TEST_F(BlobFileReaderTest, CreateReaderAndGetBlob) {
  uint64_t blob_offset = 0;
  uint64_t blob_size = 0;

-  WriteBlobFile(immutable_cf_options, column_family_id, has_ttl,
-                expiration_range, expiration_range, blob_file_number, key, blob,
-                kNoCompression, &blob_offset, &blob_size);
+  WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range,
+                expiration_range, blob_file_number, key, blob, kNoCompression,
+                &blob_offset, &blob_size);

  constexpr HistogramImpl* blob_file_read_hist = nullptr;

  std::unique_ptr<BlobFileReader> reader;

  ASSERT_OK(BlobFileReader::Create(
-      immutable_cf_options, FileOptions(), column_family_id,
-      blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader));
+      immutable_options, FileOptions(), column_family_id, blob_file_read_hist,
+      blob_file_number, nullptr /*IOTracer*/, &reader));

  // Make sure the blob can be retrieved with and without checksum verification
  ReadOptions read_options;
@@ -262,7 +261,7 @@ TEST_F(BlobFileReaderTest, Malformed) {
      test::PerThreadDBPath(&mock_env_, "BlobFileReaderTest_Malformed"), 0);
  options.enable_blob_files = true;

-  ImmutableCFOptions immutable_cf_options(options);
+  ImmutableOptions immutable_options(options);

  constexpr uint32_t column_family_id = 1;
  constexpr uint64_t blob_file_number = 1;
@@ -271,23 +270,23 @@ TEST_F(BlobFileReaderTest, Malformed) {
    constexpr bool has_ttl = false;
    constexpr ExpirationRange expiration_range;

-    const std::string blob_file_path = BlobFileName(
-        immutable_cf_options.cf_paths.front().path, blob_file_number);
+    const std::string blob_file_path =
+        BlobFileName(immutable_options.cf_paths.front().path, blob_file_number);

    std::unique_ptr<FSWritableFile> file;
-    ASSERT_OK(NewWritableFile(immutable_cf_options.fs, blob_file_path, &file,
+    ASSERT_OK(NewWritableFile(immutable_options.fs.get(), blob_file_path, &file,
                              FileOptions()));

    std::unique_ptr<WritableFileWriter> file_writer(
        new WritableFileWriter(std::move(file), blob_file_path, FileOptions(),
-                               immutable_cf_options.clock));
+                               immutable_options.clock));

    constexpr Statistics* statistics = nullptr;
    constexpr bool use_fsync = false;
    constexpr bool do_flush = false;

    BlobLogWriter blob_log_writer(std::move(file_writer),
-                                  immutable_cf_options.clock, statistics,
+                                  immutable_options.clock, statistics,
                                  blob_file_number, use_fsync, do_flush);

    BlobLogHeader header(column_family_id, kNoCompression, has_ttl,
@@ -300,7 +299,7 @@ TEST_F(BlobFileReaderTest, Malformed) {

  std::unique_ptr<BlobFileReader> reader;

-  ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(),
+  ASSERT_TRUE(BlobFileReader::Create(immutable_options, FileOptions(),
                                     column_family_id, blob_file_read_hist,
                                     blob_file_number, nullptr /*IOTracer*/,
                                     &reader)
@@ -314,7 +313,7 @@ TEST_F(BlobFileReaderTest, TTL) {
      test::PerThreadDBPath(&mock_env_, "BlobFileReaderTest_TTL"), 0);
  options.enable_blob_files = true;

-  ImmutableCFOptions immutable_cf_options(options);
+  ImmutableOptions immutable_options(options);

  constexpr uint32_t column_family_id = 1;
  constexpr bool has_ttl = true;
@@ -326,15 +325,15 @@ TEST_F(BlobFileReaderTest, TTL) {
  uint64_t blob_offset = 0;
  uint64_t blob_size = 0;

-  WriteBlobFile(immutable_cf_options, column_family_id, has_ttl,
-                expiration_range, expiration_range, blob_file_number, key, blob,
-                kNoCompression, &blob_offset, &blob_size);
+  WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range,
+                expiration_range, blob_file_number, key, blob, kNoCompression,
+                &blob_offset, &blob_size);

  constexpr HistogramImpl* blob_file_read_hist = nullptr;

  std::unique_ptr<BlobFileReader> reader;

-  ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(),
+  ASSERT_TRUE(BlobFileReader::Create(immutable_options, FileOptions(),
                                     column_family_id, blob_file_read_hist,
                                     blob_file_number, nullptr /*IOTracer*/,
                                     &reader)
@@ -350,7 +349,7 @@ TEST_F(BlobFileReaderTest, ExpirationRangeInHeader) {
      0);
  options.enable_blob_files = true;

-  ImmutableCFOptions immutable_cf_options(options);
+  ImmutableOptions immutable_options(options);

  constexpr uint32_t column_family_id = 1;
  constexpr bool has_ttl = false;
@@ -364,7 +363,7 @@ TEST_F(BlobFileReaderTest, ExpirationRangeInHeader) {
  uint64_t blob_offset = 0;
  uint64_t blob_size = 0;

-  WriteBlobFile(immutable_cf_options, column_family_id, has_ttl,
+  WriteBlobFile(immutable_options, column_family_id, has_ttl,
                expiration_range_header, expiration_range_footer,
                blob_file_number, key, blob, kNoCompression, &blob_offset,
                &blob_size);
@@ -373,7 +372,7 @@ TEST_F(BlobFileReaderTest, ExpirationRangeInHeader) {

  std::unique_ptr<BlobFileReader> reader;

-  ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(),
+  ASSERT_TRUE(BlobFileReader::Create(immutable_options, FileOptions(),
                                     column_family_id, blob_file_read_hist,
                                     blob_file_number, nullptr /*IOTracer*/,
                                     &reader)
@@ -389,7 +388,7 @@ TEST_F(BlobFileReaderTest, ExpirationRangeInFooter) {
      0);
  options.enable_blob_files = true;

-  ImmutableCFOptions immutable_cf_options(options);
+  ImmutableOptions immutable_options(options);

  constexpr uint32_t column_family_id = 1;
  constexpr bool has_ttl = false;
@@ -403,7 +402,7 @@ TEST_F(BlobFileReaderTest, ExpirationRangeInFooter) {
  uint64_t blob_offset = 0;
  uint64_t blob_size = 0;

-  WriteBlobFile(immutable_cf_options, column_family_id, has_ttl,
+  WriteBlobFile(immutable_options, column_family_id, has_ttl,
                expiration_range_header, expiration_range_footer,
                blob_file_number, key, blob, kNoCompression, &blob_offset,
                &blob_size);
@@ -412,7 +411,7 @@ TEST_F(BlobFileReaderTest, ExpirationRangeInFooter) {

  std::unique_ptr<BlobFileReader> reader;

-  ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(),
+  ASSERT_TRUE(BlobFileReader::Create(immutable_options, FileOptions(),
                                     column_family_id, blob_file_read_hist,
                                     blob_file_number, nullptr /*IOTracer*/,
                                     &reader)
@@ -428,7 +427,7 @@ TEST_F(BlobFileReaderTest, IncorrectColumnFamily) {
      0);
  options.enable_blob_files = true;

-  ImmutableCFOptions immutable_cf_options(options);
+  ImmutableOptions immutable_options(options);

  constexpr uint32_t column_family_id = 1;
  constexpr bool has_ttl = false;
@@ -440,9 +439,9 @@ TEST_F(BlobFileReaderTest, IncorrectColumnFamily) {
  uint64_t blob_offset = 0;
  uint64_t blob_size = 0;

-  WriteBlobFile(immutable_cf_options, column_family_id, has_ttl,
-                expiration_range, expiration_range, blob_file_number, key, blob,
-                kNoCompression, &blob_offset, &blob_size);
+  WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range,
+                expiration_range, blob_file_number, key, blob, kNoCompression,
+                &blob_offset, &blob_size);

  constexpr HistogramImpl* blob_file_read_hist = nullptr;

@@ -450,7 +449,7 @@ TEST_F(BlobFileReaderTest, IncorrectColumnFamily) {

  constexpr uint32_t incorrect_column_family_id = 2;

-  ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(),
+  ASSERT_TRUE(BlobFileReader::Create(immutable_options, FileOptions(),
                                     incorrect_column_family_id,
                                     blob_file_read_hist, blob_file_number,
                                     nullptr /*IOTracer*/, &reader)
@@ -464,7 +463,7 @@ TEST_F(BlobFileReaderTest, BlobCRCError) {
      test::PerThreadDBPath(&mock_env_, "BlobFileReaderTest_BlobCRCError"), 0);
  options.enable_blob_files = true;

-  ImmutableCFOptions immutable_cf_options(options);
+  ImmutableOptions immutable_options(options);

  constexpr uint32_t column_family_id = 1;
  constexpr bool has_ttl = false;
@@ -476,17 +475,17 @@ TEST_F(BlobFileReaderTest, BlobCRCError) {
  uint64_t blob_offset = 0;
  uint64_t blob_size = 0;

-  WriteBlobFile(immutable_cf_options, column_family_id, has_ttl,
-                expiration_range, expiration_range, blob_file_number, key, blob,
-                kNoCompression, &blob_offset, &blob_size);
+  WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range,
+                expiration_range, blob_file_number, key, blob, kNoCompression,
+                &blob_offset, &blob_size);

  constexpr HistogramImpl* blob_file_read_hist = nullptr;

  std::unique_ptr<BlobFileReader> reader;

  ASSERT_OK(BlobFileReader::Create(
-      immutable_cf_options, FileOptions(), column_family_id,
-      blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader));
+      immutable_options, FileOptions(), column_family_id, blob_file_read_hist,
+      blob_file_number, nullptr /*IOTracer*/, &reader));

  SyncPoint::GetInstance()->SetCallBack(
      "BlobFileReader::VerifyBlob:CheckBlobCRC", [](void* arg) {
@@ -522,7 +521,7 @@ TEST_F(BlobFileReaderTest, Compression) {
      test::PerThreadDBPath(&mock_env_, "BlobFileReaderTest_Compression"), 0);
  options.enable_blob_files = true;

-  ImmutableCFOptions immutable_cf_options(options);
+  ImmutableOptions immutable_options(options);

  constexpr uint32_t column_family_id = 1;
  constexpr bool has_ttl = false;
@@ -534,8 +533,8 @@ TEST_F(BlobFileReaderTest, Compression) {
  uint64_t blob_offset = 0;
  uint64_t blob_size = 0;

-  WriteBlobFile(immutable_cf_options, column_family_id, has_ttl,
-                expiration_range, expiration_range, blob_file_number, key, blob,
+  WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range,
+                expiration_range, blob_file_number, key, blob,
                kSnappyCompression, &blob_offset, &blob_size);

  constexpr HistogramImpl* blob_file_read_hist = nullptr;
@@ -543,8 +542,8 @@ TEST_F(BlobFileReaderTest, Compression) {
  std::unique_ptr<BlobFileReader> reader;

  ASSERT_OK(BlobFileReader::Create(
-      immutable_cf_options, FileOptions(), column_family_id,
-      blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader));
+      immutable_options, FileOptions(), column_family_id, blob_file_read_hist,
+      blob_file_number, nullptr /*IOTracer*/, &reader));

  // Make sure the blob can be retrieved with and without checksum verification
  ReadOptions read_options;
@@ -590,7 +589,7 @@ TEST_F(BlobFileReaderTest, UncompressionError) {
      0);
  options.enable_blob_files = true;

-  ImmutableCFOptions immutable_cf_options(options);
+  ImmutableOptions immutable_options(options);

  constexpr uint32_t column_family_id = 1;
  constexpr bool has_ttl = false;
@@ -602,8 +601,8 @@ TEST_F(BlobFileReaderTest, UncompressionError) {
  uint64_t blob_offset = 0;
  uint64_t blob_size = 0;

-  WriteBlobFile(immutable_cf_options, column_family_id, has_ttl,
-                expiration_range, expiration_range, blob_file_number, key, blob,
+  WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range,
+                expiration_range, blob_file_number, key, blob,
                kSnappyCompression, &blob_offset, &blob_size);

  constexpr HistogramImpl* blob_file_read_hist = nullptr;
@@ -611,8 +610,8 @@ TEST_F(BlobFileReaderTest, UncompressionError) {
  std::unique_ptr<BlobFileReader> reader;

  ASSERT_OK(BlobFileReader::Create(
-      immutable_cf_options, FileOptions(), column_family_id,
-      blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader));
+      immutable_options, FileOptions(), column_family_id, blob_file_read_hist,
+      blob_file_number, nullptr /*IOTracer*/, &reader));

  SyncPoint::GetInstance()->SetCallBack(
      "BlobFileReader::UncompressBlobIfNeeded:TamperWithResult", [](void* arg) {
@@ -671,7 +670,7 @@ TEST_P(BlobFileReaderIOErrorTest, IOError) {
      0);
  options.enable_blob_files = true;

-  ImmutableCFOptions immutable_cf_options(options);
+  ImmutableOptions immutable_options(options);

  constexpr uint32_t column_family_id = 1;
  constexpr bool has_ttl = false;
@@ -683,9 +682,9 @@ TEST_P(BlobFileReaderIOErrorTest, IOError) {
  uint64_t blob_offset = 0;
  uint64_t blob_size = 0;

-  WriteBlobFile(immutable_cf_options, column_family_id, has_ttl,
-                expiration_range, expiration_range, blob_file_number, key, blob,
-                kNoCompression, &blob_offset, &blob_size);
+  WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range,
+                expiration_range, blob_file_number, key, blob, kNoCompression,
+                &blob_offset, &blob_size);

  SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* /* arg */) {
    fault_injection_env_.SetFilesystemActive(false,
@@ -698,8 +697,8 @@ TEST_P(BlobFileReaderIOErrorTest, IOError) {
  std::unique_ptr<BlobFileReader> reader;

  const Status s = BlobFileReader::Create(
-      immutable_cf_options, FileOptions(), column_family_id,
-      blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader);
+      immutable_options, FileOptions(), column_family_id, blob_file_read_hist,
+      blob_file_number, nullptr /*IOTracer*/, &reader);

  const bool fail_during_create =
      (sync_point_ != "BlobFileReader::GetBlob:ReadFromFile");
@@ -749,7 +748,7 @@ TEST_P(BlobFileReaderDecodingErrorTest, DecodingError) {
      0);
  options.enable_blob_files = true;

-  ImmutableCFOptions immutable_cf_options(options);
+  ImmutableOptions immutable_options(options);

  constexpr uint32_t column_family_id = 1;
  constexpr bool has_ttl = false;
@@ -761,9 +760,9 @@ TEST_P(BlobFileReaderDecodingErrorTest, DecodingError) {
  uint64_t blob_offset = 0;
  uint64_t blob_size = 0;

-  WriteBlobFile(immutable_cf_options, column_family_id, has_ttl,
-                expiration_range, expiration_range, blob_file_number, key, blob,
-                kNoCompression, &blob_offset, &blob_size);
+  WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range,
+                expiration_range, blob_file_number, key, blob, kNoCompression,
+                &blob_offset, &blob_size);

  SyncPoint::GetInstance()->SetCallBack(sync_point_, [](void* arg) {
    Slice* const slice = static_cast<Slice*>(arg);
@@ -780,8 +779,8 @@ TEST_P(BlobFileReaderDecodingErrorTest, DecodingError) {
  std::unique_ptr<BlobFileReader> reader;

  const Status s = BlobFileReader::Create(
-      immutable_cf_options, FileOptions(), column_family_id,
-      blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader);
+      immutable_options, FileOptions(), column_family_id, blob_file_read_hist,
+      blob_file_number, nullptr /*IOTracer*/, &reader);

  const bool fail_during_create =
      sync_point_ != "BlobFileReader::GetBlob:TamperWithResult";
@@ -310,6 +310,63 @@ TEST_F(DBBlobBasicTest, BestEffortsRecovery_MissingNewestBlobFile) {
  ASSERT_EQ("value" + std::to_string(kNumTableFiles - 2), value);
 }

+TEST_F(DBBlobBasicTest, GetMergeBlobWithPut) {
+  Options options = GetDefaultOptions();
+  options.merge_operator = MergeOperators::CreateStringAppendOperator();
+  options.enable_blob_files = true;
+  options.min_blob_size = 0;
+
+  Reopen(options);
+
+  ASSERT_OK(Put("Key1", "v1"));
+  ASSERT_OK(Flush());
+  ASSERT_OK(Merge("Key1", "v2"));
+  ASSERT_OK(Flush());
+  ASSERT_OK(Merge("Key1", "v3"));
+  ASSERT_OK(Flush());
+
+  std::string value;
+  ASSERT_OK(db_->Get(ReadOptions(), "Key1", &value));
+  ASSERT_EQ(Get("Key1"), "v1,v2,v3");
+}
+
+TEST_F(DBBlobBasicTest, MultiGetMergeBlobWithPut) {
+  constexpr size_t num_keys = 3;
+
+  Options options = GetDefaultOptions();
+  options.merge_operator = MergeOperators::CreateStringAppendOperator();
+  options.enable_blob_files = true;
+  options.min_blob_size = 0;
+
+  Reopen(options);
+
+  ASSERT_OK(Put("Key0", "v0_0"));
+  ASSERT_OK(Put("Key1", "v1_0"));
+  ASSERT_OK(Put("Key2", "v2_0"));
+  ASSERT_OK(Flush());
+  ASSERT_OK(Merge("Key0", "v0_1"));
+  ASSERT_OK(Merge("Key1", "v1_1"));
+  ASSERT_OK(Flush());
+  ASSERT_OK(Merge("Key0", "v0_2"));
+  ASSERT_OK(Flush());
+
+  std::array<Slice, num_keys> keys{{"Key0", "Key1", "Key2"}};
+  std::array<PinnableSlice, num_keys> values;
+  std::array<Status, num_keys> statuses;
+
+  db_->MultiGet(ReadOptions(), db_->DefaultColumnFamily(), num_keys, &keys[0],
+                &values[0], &statuses[0]);
+
+  ASSERT_OK(statuses[0]);
+  ASSERT_EQ(values[0], "v0_0,v0_1,v0_2");
+
+  ASSERT_OK(statuses[1]);
+  ASSERT_EQ(values[1], "v1_0,v1_1");
+
+  ASSERT_OK(statuses[2]);
+  ASSERT_EQ(values[2], "v2_0");
+}
+
 class DBBlobBasicIOErrorTest : public DBBlobBasicTest,
                               public testing::WithParamInterface<std::string> {
 protected:
@@ -399,7 +399,7 @@ TEST_F(DBBlobIndexTest, Iterate) {
           create_normal_iterator);
    verify(9, Status::kOk, get_value(10, 0), get_value(8, 0),
           create_normal_iterator);
-    verify(11, Status::kNotSupported, "", "", create_normal_iterator);
+    verify(11, Status::kCorruption, "", "", create_normal_iterator);
    verify(13, Status::kOk,
           get_value(13, 2) + "," + get_value(13, 1) + "," + get_value(13, 0),
           get_value(13, 2) + "," + get_value(13, 1) + "," + get_value(13, 0),
@@ -418,7 +418,11 @@ TEST_F(DBBlobIndexTest, Iterate) {
           create_blob_iterator, check_is_blob(false));
    verify(9, Status::kOk, get_value(10, 0), get_value(8, 0),
           create_blob_iterator, check_is_blob(false));
-    verify(11, Status::kNotSupported, "", "", create_blob_iterator);
+    if (tier <= kImmutableMemtables) {
+      verify(11, Status::kNotSupported, "", "", create_blob_iterator);
+    } else {
+      verify(11, Status::kCorruption, "", "", create_blob_iterator);
+    }
    verify(13, Status::kOk,
           get_value(13, 2) + "," + get_value(13, 1) + "," + get_value(13, 0),
           get_value(13, 2) + "," + get_value(13, 1) + "," + get_value(13, 0),
@@ -440,7 +444,11 @@ TEST_F(DBBlobIndexTest, Iterate) {
           create_blob_iterator, check_is_blob(false));
    verify(9, Status::kOk, get_value(10, 0), get_value(8, 0),
           create_blob_iterator, check_is_blob(false));
-    verify(11, Status::kNotSupported, "", "", create_blob_iterator);
+    if (tier <= kImmutableMemtables) {
+      verify(11, Status::kNotSupported, "", "", create_blob_iterator);
+    } else {
+      verify(11, Status::kCorruption, "", "", create_blob_iterator);
+    }
    verify(13, Status::kOk,
           get_value(13, 2) + "," + get_value(13, 1) + "," + get_value(13, 0),
           get_value(13, 2) + "," + get_value(13, 1) + "," + get_value(13, 0),
@@ -455,6 +463,106 @@ TEST_F(DBBlobIndexTest, Iterate) {
  }
 }

+TEST_F(DBBlobIndexTest, IntegratedBlobIterate) {
+  const std::vector<std::vector<std::string>> data = {
+      /*00*/ {"Put"},
+      /*01*/ {"Put", "Merge", "Merge", "Merge"},
+      /*02*/ {"Put"}};
+
+  auto get_key = [](size_t index) { return ("key" + std::to_string(index)); };
+
+  auto get_value = [&](size_t index, size_t version) {
+    return get_key(index) + "_value" + ToString(version);
+  };
+
+  auto check_iterator = [&](Iterator* iterator, Status expected_status,
+                            const Slice& expected_value) {
+    ASSERT_EQ(expected_status, iterator->status());
+    if (expected_status.ok()) {
+      ASSERT_TRUE(iterator->Valid());
+      ASSERT_EQ(expected_value, iterator->value());
+    } else {
+      ASSERT_FALSE(iterator->Valid());
+    }
+  };
+
+  auto verify = [&](size_t index, Status expected_status,
+                    const Slice& expected_value) {
+    // Seek
+    {
+      Iterator* iterator = db_->NewIterator(ReadOptions());
+      std::unique_ptr<Iterator> iterator_guard(iterator);
+      ASSERT_OK(iterator->status());
+      ASSERT_OK(iterator->Refresh());
+      iterator->Seek(get_key(index));
+      check_iterator(iterator, expected_status, expected_value);
+    }
+    // Next
+    {
+      Iterator* iterator = db_->NewIterator(ReadOptions());
+      std::unique_ptr<Iterator> iterator_guard(iterator);
+      ASSERT_OK(iterator->Refresh());
+      iterator->Seek(get_key(index - 1));
+      ASSERT_TRUE(iterator->Valid());
+      ASSERT_OK(iterator->status());
+      iterator->Next();
+      check_iterator(iterator, expected_status, expected_value);
+    }
+    // SeekForPrev
+    {
+      Iterator* iterator = db_->NewIterator(ReadOptions());
+      std::unique_ptr<Iterator> iterator_guard(iterator);
+      ASSERT_OK(iterator->status());
+      ASSERT_OK(iterator->Refresh());
+      iterator->SeekForPrev(get_key(index));
+      check_iterator(iterator, expected_status, expected_value);
+    }
+    // Prev
+    {
+      Iterator* iterator = db_->NewIterator(ReadOptions());
+      std::unique_ptr<Iterator> iterator_guard(iterator);
+      iterator->Seek(get_key(index + 1));
+      ASSERT_TRUE(iterator->Valid());
+      ASSERT_OK(iterator->status());
+      iterator->Prev();
+      check_iterator(iterator, expected_status, expected_value);
+    }
+  };
+
+  Options options = GetTestOptions();
+  options.enable_blob_files = true;
+  options.min_blob_size = 0;
+
+  DestroyAndReopen(options);
+
+  // fill data
+  for (size_t i = 0; i < data.size(); i++) {
+    for (size_t j = 0; j < data[i].size(); j++) {
+      std::string key = get_key(i);
+      std::string value = get_value(i, j);
+      if (data[i][j] == "Put") {
+        ASSERT_OK(Put(key, value));
+        ASSERT_OK(Flush());
+      } else if (data[i][j] == "Merge") {
+        ASSERT_OK(Merge(key, value));
+        ASSERT_OK(Flush());
+      }
+    }
+  }
+
+  std::string expected_value = get_value(1, 0) + "," + get_value(1, 1) + "," +
+                               get_value(1, 2) + "," + get_value(1, 3);
+  Status expected_status;
+  verify(1, expected_status, expected_value);
+
+#ifndef ROCKSDB_LITE
+  // Test DBIter::FindValueForCurrentKeyUsingSeek flow.
+  ASSERT_OK(dbfull()->SetOptions(cfh(),
+                                 {{"max_sequential_skip_in_iterations", "0"}}));
+  verify(1, expected_status, expected_value);
+#endif  // !ROCKSDB_LITE
+}
+
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -45,62 +45,41 @@ namespace ROCKSDB_NAMESPACE {

 class TableFactory;

-TableBuilder* NewTableBuilder(
-    const ImmutableCFOptions& ioptions, const MutableCFOptions& moptions,
-    const InternalKeyComparator& internal_comparator,
-    const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
-        int_tbl_prop_collector_factories,
-    uint32_t column_family_id, const std::string& column_family_name,
-    WritableFileWriter* file, const CompressionType compression_type,
-    const CompressionOptions& compression_opts, int level,
-    const bool skip_filters, const uint64_t creation_time,
-    const uint64_t oldest_key_time, const uint64_t target_file_size,
-    const uint64_t file_creation_time, const std::string& db_id,
-    const std::string& db_session_id) {
-  assert((column_family_id ==
+TableBuilder* NewTableBuilder(const TableBuilderOptions& tboptions,
+                              WritableFileWriter* file) {
+  assert((tboptions.column_family_id ==
          TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
-         column_family_name.empty());
-  return ioptions.table_factory->NewTableBuilder(
-      TableBuilderOptions(ioptions, moptions, internal_comparator,
-                          int_tbl_prop_collector_factories, compression_type,
-                          compression_opts, skip_filters, column_family_name,
-                          level, creation_time, oldest_key_time,
-                          target_file_size, file_creation_time, db_id,
-                          db_session_id),
-      column_family_id, file);
+         tboptions.column_family_name.empty());
+  return tboptions.ioptions.table_factory->NewTableBuilder(tboptions, file);
 }

 Status BuildTable(
    const std::string& dbname, VersionSet* versions,
-    const ImmutableDBOptions& db_options, const ImmutableCFOptions& ioptions,
-    const MutableCFOptions& mutable_cf_options, const FileOptions& file_options,
-    TableCache* table_cache, InternalIterator* iter,
+    const ImmutableDBOptions& db_options, const TableBuilderOptions& tboptions,
+    const FileOptions& file_options, TableCache* table_cache,
+    InternalIterator* iter,
    std::vector<std::unique_ptr<FragmentedRangeTombstoneIterator>>
        range_del_iters,
    FileMetaData* meta, std::vector<BlobFileAddition>* blob_file_additions,
-    const InternalKeyComparator& internal_comparator,
-    const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
-        int_tbl_prop_collector_factories,
-    uint32_t column_family_id, const std::string& column_family_name,
    std::vector<SequenceNumber> snapshots,
    SequenceNumber earliest_write_conflict_snapshot,
-    SnapshotChecker* snapshot_checker, const CompressionType compression,
-    const CompressionOptions& compression_opts, bool paranoid_file_checks,
-    InternalStats* internal_stats, TableFileCreationReason reason,
-    IOStatus* io_status, const std::shared_ptr<IOTracer>& io_tracer,
-    EventLogger* event_logger, int job_id, const Env::IOPriority io_priority,
-    TableProperties* table_properties, int level, const uint64_t creation_time,
-    const uint64_t oldest_key_time, Env::WriteLifeTimeHint write_hint,
-    const uint64_t file_creation_time, const std::string& db_id,
-    const std::string& db_session_id, const std::string* full_history_ts_low,
-    BlobFileCompletionCallback* blob_callback) {
-  assert((column_family_id ==
+    SnapshotChecker* snapshot_checker, bool paranoid_file_checks,
+    InternalStats* internal_stats, IOStatus* io_status,
+    const std::shared_ptr<IOTracer>& io_tracer, EventLogger* event_logger,
+    int job_id, const Env::IOPriority io_priority,
+    TableProperties* table_properties, Env::WriteLifeTimeHint write_hint,
+    const std::string* full_history_ts_low,
+    BlobFileCompletionCallback* blob_callback, uint64_t* num_input_entries,
+    uint64_t* memtable_payload_bytes, uint64_t* memtable_garbage_bytes) {
+  assert((tboptions.column_family_id ==
          TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
-         column_family_name.empty());
+         tboptions.column_family_name.empty());
+  auto& mutable_cf_options = tboptions.moptions;
+  auto& ioptions = tboptions.ioptions;
  // Reports the IOStats for flush for every following bytes.
  const size_t kReportFlushIOStatsEvery = 1048576;
  OutputValidator output_validator(
-      internal_comparator,
+      tboptions.internal_comparator,
      /*enable_order_check=*/
      mutable_cf_options.check_flush_compaction_key_order,
      /*enable_hash=*/paranoid_file_checks);
@@ -108,8 +87,15 @@ Status BuildTable(
  meta->fd.file_size = 0;
  iter->SeekToFirst();
  std::unique_ptr<CompactionRangeDelAggregator> range_del_agg(
-      new CompactionRangeDelAggregator(&internal_comparator, snapshots));
+      new CompactionRangeDelAggregator(&tboptions.internal_comparator,
+                                       snapshots));
+  uint64_t num_unfragmented_tombstones = 0;
+  uint64_t total_tombstone_payload_bytes = 0;
  for (auto& range_del_iter : range_del_iters) {
+    num_unfragmented_tombstones +=
+        range_del_iter->num_unfragmented_tombstones();
+    total_tombstone_payload_bytes +=
+        range_del_iter->total_tombstone_payload_bytes();
    range_del_agg->AddTombstones(std::move(range_del_iter));
  }

@@ -119,8 +105,9 @@ Status BuildTable(
  std::string file_checksum = kUnknownFileChecksum;
  std::string file_checksum_func_name = kUnknownFileChecksumFuncName;
 #ifndef ROCKSDB_LITE
-  EventHelpers::NotifyTableFileCreationStarted(
-      ioptions.listeners, dbname, column_family_name, fname, job_id, reason);
+  EventHelpers::NotifyTableFileCreationStarted(ioptions.listeners, dbname,
+                                               tboptions.column_family_name,
+                                               fname, job_id, tboptions.reason);
 #endif  // !ROCKSDB_LITE
  Env* env = db_options.env;
  assert(env);
@@ -129,6 +116,26 @@ Status BuildTable(

  TableProperties tp;
  if (iter->Valid() || !range_del_agg->IsEmpty()) {
+    std::unique_ptr<CompactionFilter> compaction_filter;
+    if (ioptions.compaction_filter_factory != nullptr &&
+        ioptions.compaction_filter_factory->ShouldFilterTableFileCreation(
+            tboptions.reason)) {
+      CompactionFilter::Context context;
+      context.is_full_compaction = false;
+      context.is_manual_compaction = false;
+      context.column_family_id = tboptions.column_family_id;
+      context.reason = tboptions.reason;
+      compaction_filter =
+          ioptions.compaction_filter_factory->CreateCompactionFilter(context);
+      if (compaction_filter != nullptr &&
+          !compaction_filter->IgnoreSnapshots()) {
+        s.PermitUncheckedError();
+        return Status::NotSupported(
+            "CompactionFilter::IgnoreSnapshots() = false is not supported "
+            "anymore.");
+      }
+    }
+
    TableBuilder* builder;
    std::unique_ptr<WritableFileWriter> file_writer;
    {
@@ -145,9 +152,10 @@ Status BuildTable(
      }
      if (!s.ok()) {
        EventHelpers::LogAndNotifyTableFileCreationFinished(
-            event_logger, ioptions.listeners, dbname, column_family_name, fname,
-            job_id, meta->fd, kInvalidBlobFileNumber, tp, reason, s,
-            file_checksum, file_checksum_func_name);
+            event_logger, ioptions.listeners, dbname,
+            tboptions.column_family_name, fname, job_id, meta->fd,
+            kInvalidBlobFileNumber, tp, tboptions.reason, s, file_checksum,
+            file_checksum_func_name);
        return s;
      }
      FileTypeSet tmp_set = ioptions.checksum_handoff_file_types;
@@ -155,43 +163,40 @@ Status BuildTable(
      file->SetWriteLifeTimeHint(write_hint);
      file_writer.reset(new WritableFileWriter(
          std::move(file), fname, file_options, ioptions.clock, io_tracer,
-          ioptions.statistics, ioptions.listeners,
+          ioptions.stats, ioptions.listeners,
          ioptions.file_checksum_gen_factory.get(),
          tmp_set.Contains(FileType::kTableFile)));

-      builder = NewTableBuilder(
-          ioptions, mutable_cf_options, internal_comparator,
-          int_tbl_prop_collector_factories, column_family_id,
-          column_family_name, file_writer.get(), compression, compression_opts,
-          level, false /* skip_filters */, creation_time, oldest_key_time,
-          0 /*target_file_size*/, file_creation_time, db_id, db_session_id);
+      builder = NewTableBuilder(tboptions, file_writer.get());
    }

-    MergeHelper merge(env, internal_comparator.user_comparator(),
-                      ioptions.merge_operator.get(), nullptr, ioptions.info_log,
-                      true /* internal key corruption is not ok */,
-                      snapshots.empty() ? 0 : snapshots.back(),
-                      snapshot_checker);
+    MergeHelper merge(
+        env, tboptions.internal_comparator.user_comparator(),
+        ioptions.merge_operator.get(), compaction_filter.get(), ioptions.logger,
+        true /* internal key corruption is not ok */,
+        snapshots.empty() ? 0 : snapshots.back(), snapshot_checker);

    std::unique_ptr<BlobFileBuilder> blob_file_builder(
        (mutable_cf_options.enable_blob_files && blob_file_additions)
            ? new BlobFileBuilder(versions, fs, &ioptions, &mutable_cf_options,
-                                  &file_options, job_id, column_family_id,
-                                  column_family_name, io_priority, write_hint,
-                                  io_tracer, blob_callback, &blob_file_paths,
-                                  blob_file_additions)
+                                  &file_options, job_id,
+                                  tboptions.column_family_id,
+                                  tboptions.column_family_name, io_priority,
+                                  write_hint, io_tracer, blob_callback,
+                                  &blob_file_paths, blob_file_additions)
            : nullptr);

    CompactionIterator c_iter(
-        iter, internal_comparator.user_comparator(), &merge, kMaxSequenceNumber,
-        &snapshots, earliest_write_conflict_snapshot, snapshot_checker, env,
-        ShouldReportDetailedTime(env, ioptions.statistics),
+        iter, tboptions.internal_comparator.user_comparator(), &merge,
+        kMaxSequenceNumber, &snapshots, earliest_write_conflict_snapshot,
+        snapshot_checker, env, ShouldReportDetailedTime(env, ioptions.stats),
        true /* internal key corruption is not ok */, range_del_agg.get(),
        blob_file_builder.get(), ioptions.allow_data_in_errors,
-        /*compaction=*/nullptr,
-        /*compaction_filter=*/nullptr, /*shutting_down=*/nullptr,
+        /*compaction=*/nullptr, compaction_filter.get(),
+        /*shutting_down=*/nullptr,
        /*preserve_deletes_seqnum=*/0, /*manual_compaction_paused=*/nullptr,
-        db_options.info_log, full_history_ts_low);
+        /*manual_compaction_canceled=*/nullptr, db_options.info_log,
+        full_history_ts_low);

    c_iter.SeekToFirst();
    for (; c_iter.Valid(); c_iter.Next()) {
@@ -227,12 +232,17 @@ Status BuildTable(
        auto kv = tombstone.Serialize();
        builder->Add(kv.first.Encode(), kv.second);
        meta->UpdateBoundariesForRange(kv.first, tombstone.SerializeEndKey(),
-                                       tombstone.seq_, internal_comparator);
+                                       tombstone.seq_,
+                                       tboptions.internal_comparator);
      }
    }

    TEST_SYNC_POINT("BuildTable:BeforeFinishBuildTable");
    const bool empty = builder->IsEmpty();
+    if (num_input_entries != nullptr) {
+      *num_input_entries =
+          c_iter.num_input_entry_scanned() + num_unfragmented_tombstones;
+    }
    if (!s.ok() || empty) {
      builder->Abandon();
    } else {
@@ -248,6 +258,25 @@ Status BuildTable(
      meta->marked_for_compaction = builder->NeedCompact();
      assert(meta->fd.GetFileSize() > 0);
      tp = builder->GetTableProperties(); // refresh now that builder is finished
+      if (memtable_payload_bytes != nullptr &&
+          memtable_garbage_bytes != nullptr) {
+        const CompactionIterationStats& ci_stats = c_iter.iter_stats();
+        uint64_t total_payload_bytes = ci_stats.total_input_raw_key_bytes +
+                                       ci_stats.total_input_raw_value_bytes +
+                                       total_tombstone_payload_bytes;
+        uint64_t total_payload_bytes_written =
+            (tp.raw_key_size + tp.raw_value_size);
+        // Prevent underflow, which may still happen at this point
+        // since we only support inserts, deletes, and deleteRanges.
+        if (total_payload_bytes_written <= total_payload_bytes) {
+          *memtable_payload_bytes = total_payload_bytes;
+          *memtable_garbage_bytes =
+              total_payload_bytes - total_payload_bytes_written;
+        } else {
+          *memtable_payload_bytes = 0;
+          *memtable_garbage_bytes = 0;
+        }
+      }
      if (table_properties) {
        *table_properties = tp;
      }
@@ -257,7 +286,7 @@ Status BuildTable(
    // Finish and check for file errors
    TEST_SYNC_POINT("BuildTable:BeforeSyncTable");
    if (s.ok() && !empty) {
-      StopWatch sw(ioptions.clock, ioptions.statistics, TABLE_SYNC_MICROS);
+      StopWatch sw(ioptions.clock, ioptions.stats, TABLE_SYNC_MICROS);
      *io_status = file_writer->Sync(ioptions.use_fsync);
    }
    TEST_SYNC_POINT("BuildTable:BeforeCloseTableFile");
@@ -296,20 +325,20 @@ Status BuildTable(
      // to cache it here for further user reads
      ReadOptions read_options;
      std::unique_ptr<InternalIterator> it(table_cache->NewIterator(
-          read_options, file_options, internal_comparator, *meta,
+          read_options, file_options, tboptions.internal_comparator, *meta,
          nullptr /* range_del_agg */,
          mutable_cf_options.prefix_extractor.get(), nullptr,
          (internal_stats == nullptr) ? nullptr
                                      : internal_stats->GetFileReadHist(0),
          TableReaderCaller::kFlush, /*arena=*/nullptr,
-          /*skip_filter=*/false, level,
+          /*skip_filter=*/false, tboptions.level_at_creation,
          MaxFileSizeForL0MetaPin(mutable_cf_options),
          /*smallest_compaction_key=*/nullptr,
          /*largest_compaction_key*/ nullptr,
          /*allow_unprepared_value*/ false));
      s = it->status();
      if (s.ok() && paranoid_file_checks) {
-        OutputValidator file_validator(internal_comparator,
+        OutputValidator file_validator(tboptions.internal_comparator,
                                       /*enable_order_check=*/true,
                                       /*enable_hash=*/true);
        for (it->SeekToFirst(); it->Valid(); it->Next()) {
@@ -354,9 +383,9 @@ Status BuildTable(
  }
  // Output to event logger and fire events.
  EventHelpers::LogAndNotifyTableFileCreationFinished(
-      event_logger, ioptions.listeners, dbname, column_family_name, fname,
-      job_id, meta->fd, meta->oldest_blob_file_number, tp, reason, s,
-      file_checksum, file_checksum_func_name);
+      event_logger, ioptions.listeners, dbname, tboptions.column_family_name,
+      fname, job_id, meta->fd, meta->oldest_blob_file_number, tp,
+      tboptions.reason, s, file_checksum, file_checksum_func_name);

  return s;
 }
@@ -24,37 +24,20 @@

 namespace ROCKSDB_NAMESPACE {

-struct Options;
 struct FileMetaData;

 class VersionSet;
-class Env;
-struct EnvOptions;
 class BlobFileAddition;
-class Iterator;
 class SnapshotChecker;
 class TableCache;
-class VersionEdit;
 class TableBuilder;
 class WritableFileWriter;
 class InternalStats;
 class BlobFileCompletionCallback;

-// @param column_family_name Name of the column family that is also identified
-//    by column_family_id, or empty string if unknown. It must outlive the
-//    TableBuilder returned by this function.
-TableBuilder* NewTableBuilder(
-    const ImmutableCFOptions& options, const MutableCFOptions& moptions,
-    const InternalKeyComparator& internal_comparator,
-    const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
-        int_tbl_prop_collector_factories,
-    uint32_t column_family_id, const std::string& column_family_name,
-    WritableFileWriter* file, const CompressionType compression_type,
-    const CompressionOptions& compression_opts, int level,
-    const bool skip_filters = false, const uint64_t creation_time = 0,
-    const uint64_t oldest_key_time = 0, const uint64_t target_file_size = 0,
-    const uint64_t file_creation_time = 0, const std::string& db_id = "",
-    const std::string& db_session_id = "");
+// Convenience function for NewTableBuilder on the embedded table_factory.
+TableBuilder* NewTableBuilder(const TableBuilderOptions& tboptions,
+                              WritableFileWriter* file);

 // Build a Table file from the contents of *iter.  The generated file
 // will be named according to number specified in meta. On success, the rest of
@@ -66,30 +49,25 @@ TableBuilder* NewTableBuilder(
 //    by column_family_id, or empty string if unknown.
 extern Status BuildTable(
    const std::string& dbname, VersionSet* versions,
-    const ImmutableDBOptions& db_options, const ImmutableCFOptions& options,
-    const MutableCFOptions& mutable_cf_options, const FileOptions& file_options,
-    TableCache* table_cache, InternalIterator* iter,
+    const ImmutableDBOptions& db_options, const TableBuilderOptions& tboptions,
+    const FileOptions& file_options, TableCache* table_cache,
+    InternalIterator* iter,
    std::vector<std::unique_ptr<FragmentedRangeTombstoneIterator>>
        range_del_iters,
    FileMetaData* meta, std::vector<BlobFileAddition>* blob_file_additions,
-    const InternalKeyComparator& internal_comparator,
-    const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
-        int_tbl_prop_collector_factories,
-    uint32_t column_family_id, const std::string& column_family_name,
    std::vector<SequenceNumber> snapshots,
    SequenceNumber earliest_write_conflict_snapshot,
-    SnapshotChecker* snapshot_checker, const CompressionType compression,
-    const CompressionOptions& compression_opts, bool paranoid_file_checks,
-    InternalStats* internal_stats, TableFileCreationReason reason,
-    IOStatus* io_status, const std::shared_ptr<IOTracer>& io_tracer,
+    SnapshotChecker* snapshot_checker, bool paranoid_file_checks,
+    InternalStats* internal_stats, IOStatus* io_status,
+    const std::shared_ptr<IOTracer>& io_tracer,
    EventLogger* event_logger = nullptr, int job_id = 0,
    const Env::IOPriority io_priority = Env::IO_HIGH,
-    TableProperties* table_properties = nullptr, int level = -1,
-    const uint64_t creation_time = 0, const uint64_t oldest_key_time = 0,
+    TableProperties* table_properties = nullptr,
    Env::WriteLifeTimeHint write_hint = Env::WLTH_NOT_SET,
-    const uint64_t file_creation_time = 0, const std::string& db_id = "",
-    const std::string& db_session_id = "",
    const std::string* full_history_ts_low = nullptr,
-    BlobFileCompletionCallback* blob_callback = nullptr);
+    BlobFileCompletionCallback* blob_callback = nullptr,
+    uint64_t* num_input_entries = nullptr,
+    uint64_t* memtable_payload_bytes = nullptr,
+    uint64_t* memtable_garbage_bytes = nullptr);

 }  // namespace ROCKSDB_NAMESPACE
@@ -40,6 +40,7 @@
 #include "rocksdb/utilities/db_ttl.h"
 #include "rocksdb/utilities/memory_util.h"
 #include "rocksdb/utilities/optimistic_transaction_db.h"
+#include "rocksdb/utilities/table_properties_collectors.h"
 #include "rocksdb/utilities/transaction.h"
 #include "rocksdb/utilities/transaction_db.h"
 #include "rocksdb/utilities/write_batch_with_index.h"
@@ -84,6 +85,7 @@ using ROCKSDB_NAMESPACE::MemoryAllocator;
 using ROCKSDB_NAMESPACE::MemoryUtil;
 using ROCKSDB_NAMESPACE::MergeOperator;
 using ROCKSDB_NAMESPACE::NewBloomFilterPolicy;
+using ROCKSDB_NAMESPACE::NewCompactOnDeletionCollectorFactory;
 using ROCKSDB_NAMESPACE::NewGenericRateLimiter;
 using ROCKSDB_NAMESPACE::NewLRUCache;
 using ROCKSDB_NAMESPACE::OptimisticTransactionDB;
@@ -104,6 +106,7 @@ using ROCKSDB_NAMESPACE::SliceTransform;
 using ROCKSDB_NAMESPACE::Snapshot;
 using ROCKSDB_NAMESPACE::SstFileWriter;
 using ROCKSDB_NAMESPACE::Status;
+using ROCKSDB_NAMESPACE::TablePropertiesCollectorFactory;
 using ROCKSDB_NAMESPACE::Transaction;
 using ROCKSDB_NAMESPACE::TransactionDB;
 using ROCKSDB_NAMESPACE::TransactionDBOptions;
@@ -1509,6 +1512,10 @@ void rocksdb_flush_cf(
  SaveError(errptr, db->rep->Flush(options->rep, column_family->rep));
 }

+void rocksdb_flush_wal(rocksdb_t* db, unsigned char sync, char** errptr) {
+  SaveError(errptr, db->rep->FlushWAL(sync));
+}
+
 void rocksdb_disable_file_deletions(
    rocksdb_t* db,
    char** errptr) {
@@ -2857,11 +2864,31 @@ void rocksdb_options_set_compression_options_zstd_max_train_bytes(
  opt->rep.compression_opts.zstd_max_train_bytes = zstd_max_train_bytes;
 }

+int rocksdb_options_get_compression_options_zstd_max_train_bytes(
+    rocksdb_options_t* opt) {
+  return opt->rep.compression_opts.zstd_max_train_bytes;
+}
+
+void rocksdb_options_set_compression_options_parallel_threads(
+    rocksdb_options_t* opt, int value) {
+  opt->rep.compression_opts.parallel_threads = value;
+}
+
+int rocksdb_options_get_compression_options_parallel_threads(
+    rocksdb_options_t* opt) {
+  return opt->rep.compression_opts.parallel_threads;
+}
+
 void rocksdb_options_set_compression_options_max_dict_buffer_bytes(
    rocksdb_options_t* opt, uint64_t max_dict_buffer_bytes) {
  opt->rep.compression_opts.max_dict_buffer_bytes = max_dict_buffer_bytes;
 }

+uint64_t rocksdb_options_get_compression_options_max_dict_buffer_bytes(
+    rocksdb_options_t* opt) {
+  return opt->rep.compression_opts.max_dict_buffer_bytes;
+}
+
 void rocksdb_options_set_prefix_extractor(
    rocksdb_options_t* opt, rocksdb_slicetransform_t* prefix_extractor) {
  opt->rep.prefix_extractor.reset(prefix_extractor);
@@ -3476,6 +3503,15 @@ unsigned char rocksdb_options_get_atomic_flush(rocksdb_options_t* opt) {
  return opt->rep.atomic_flush;
 }

+void rocksdb_options_set_manual_wal_flush(rocksdb_options_t* opt,
+                                          unsigned char manual_wal_flush) {
+  opt->rep.manual_wal_flush = manual_wal_flush;
+}
+
+unsigned char rocksdb_options_get_manual_wal_flush(rocksdb_options_t* opt) {
+  return opt->rep.manual_wal_flush;
+}
+
 rocksdb_ratelimiter_t* rocksdb_ratelimiter_create(
    int64_t rate_bytes_per_sec,
    int64_t refill_period_us,
@@ -3497,6 +3533,14 @@ void rocksdb_options_set_row_cache(rocksdb_options_t* opt, rocksdb_cache_t* cach
  }
 }

+void rocksdb_options_add_compact_on_deletion_collector_factory(
+    rocksdb_options_t* opt, size_t window_size, size_t num_dels_trigger) {
+  std::shared_ptr<ROCKSDB_NAMESPACE::TablePropertiesCollectorFactory>
+      compact_on_del =
+          NewCompactOnDeletionCollectorFactory(window_size, num_dels_trigger);
+  opt->rep.table_properties_collector_factories.emplace_back(compact_on_del);
+}
+
 void rocksdb_set_perf_level(int v) {
  PerfLevel level = static_cast<PerfLevel>(v);
  SetPerfLevel(level);
@@ -517,6 +517,9 @@ int main(int argc, char** argv) {
  coptions = rocksdb_compactoptions_create();
  rocksdb_compactoptions_set_exclusive_manual_compaction(coptions, 1);

+  rocksdb_options_add_compact_on_deletion_collector_factory(options, 10000,
+                                                            10001);
+
  StartPhase("destroy");
  rocksdb_destroy_db(options, dbname, &err);
  Free(&err);
@@ -1276,6 +1279,9 @@ int main(int argc, char** argv) {
    CheckPinGetCF(db, roptions, handles[1], "box", "c");
    rocksdb_writebatch_destroy(wb);

+    rocksdb_flush_wal(db, 1, &err);
+    CheckNoError(err);
+
    const char* keys[3] = { "box", "box", "barfooxx" };
    const rocksdb_column_family_handle_t* get_handles[3] = { handles[0], handles[1], handles[1] };
    const size_t keys_sizes[3] = { 3, 3, 8 };
@@ -1759,6 +1765,9 @@ int main(int argc, char** argv) {
    rocksdb_options_set_atomic_flush(o, 1);
    CheckCondition(1 == rocksdb_options_get_atomic_flush(o));

+    rocksdb_options_set_manual_wal_flush(o, 1);
+    CheckCondition(1 == rocksdb_options_get_manual_wal_flush(o));
+
    /* Blob Options */
    rocksdb_options_set_enable_blob_files(o, 1);
    CheckCondition(1 == rocksdb_options_get_enable_blob_files(o));
@@ -2530,6 +2539,28 @@ int main(int argc, char** argv) {
    rocksdb_backupable_db_options_destroy(bdo);
  }

+  StartPhase("compression_options");
+  {
+    rocksdb_options_t* co;
+    co = rocksdb_options_create();
+
+    rocksdb_options_set_compression_options_zstd_max_train_bytes(co, 100);
+    CheckCondition(
+        100 ==
+        rocksdb_options_get_compression_options_zstd_max_train_bytes(co));
+
+    rocksdb_options_set_compression_options_parallel_threads(co, 2);
+    CheckCondition(
+        2 == rocksdb_options_get_compression_options_parallel_threads(co));
+
+    rocksdb_options_set_compression_options_max_dict_buffer_bytes(co, 200);
+    CheckCondition(
+        200 ==
+        rocksdb_options_get_compression_options_max_dict_buffer_bytes(co));
+
+    rocksdb_options_destroy(co);
+  }
+
  StartPhase("iterate_upper_bound");
  {
    // Create new empty database
@@ -2919,7 +2950,6 @@ int main(int argc, char** argv) {
  rocksdb_readoptions_destroy(roptions);
  rocksdb_writeoptions_destroy(woptions);
  rocksdb_compactoptions_destroy(coptions);
-  rocksdb_cache_disown_data(cache);
  rocksdb_cache_destroy(cache);
  rocksdb_comparator_destroy(cmp);
  rocksdb_dbpath_destroy(dbpath);
@@ -109,8 +109,9 @@ const Comparator* ColumnFamilyHandleImpl::GetComparator() const {

 void GetIntTblPropCollectorFactory(
    const ImmutableCFOptions& ioptions,
-    std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
-        int_tbl_prop_collector_factories) {
+    IntTblPropCollectorFactories* int_tbl_prop_collector_factories) {
+  assert(int_tbl_prop_collector_factories);
+
  auto& collector_factories = ioptions.table_properties_collector_factories;
  for (size_t i = 0; i < ioptions.table_properties_collector_factories.size();
       ++i) {
@@ -214,7 +215,8 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
  // if user sets arena_block_size, we trust user to use this value. Otherwise,
  // calculate a proper value from writer_buffer_size;
  if (result.arena_block_size <= 0) {
-    result.arena_block_size = result.write_buffer_size / 8;
+    result.arena_block_size =
+        std::min(size_t{1024 * 1024}, result.write_buffer_size / 8);

    // Align up to 4k
    const size_t align = 4 * 1024;
@@ -283,7 +285,7 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
  }

  if (result.level0_file_num_compaction_trigger == 0) {
-    ROCKS_LOG_WARN(db_options.info_log.get(),
+    ROCKS_LOG_WARN(db_options.logger,
                   "level0_file_num_compaction_trigger cannot be 0");
    result.level0_file_num_compaction_trigger = 1;
  }
@@ -292,7 +294,7 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
          result.level0_slowdown_writes_trigger ||
      result.level0_slowdown_writes_trigger <
          result.level0_file_num_compaction_trigger) {
-    ROCKS_LOG_WARN(db_options.info_log.get(),
+    ROCKS_LOG_WARN(db_options.logger,
                   "This condition must be satisfied: "
                   "level0_stop_writes_trigger(%d) >= "
                   "level0_slowdown_writes_trigger(%d) >= "
@@ -309,7 +311,7 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
        result.level0_slowdown_writes_trigger) {
      result.level0_stop_writes_trigger = result.level0_slowdown_writes_trigger;
    }
-    ROCKS_LOG_WARN(db_options.info_log.get(),
+    ROCKS_LOG_WARN(db_options.logger,
                   "Adjust the value to "
                   "level0_stop_writes_trigger(%d)"
                   "level0_slowdown_writes_trigger(%d)"
@@ -502,7 +504,8 @@ ColumnFamilyData::ColumnFamilyData(
    const ColumnFamilyOptions& cf_options, const ImmutableDBOptions& db_options,
    const FileOptions& file_options, ColumnFamilySet* column_family_set,
    BlockCacheTracer* const block_cache_tracer,
-    const std::shared_ptr<IOTracer>& io_tracer)
+    const std::shared_ptr<IOTracer>& io_tracer,
+    const std::string& db_session_id)
    : id_(id),
      name_(name),
      dummy_versions_(_dummy_versions),
@@ -545,7 +548,7 @@ ColumnFamilyData::ColumnFamilyData(
      db_paths_registered_ = true;
    } else {
      ROCKS_LOG_ERROR(
-          ioptions_.info_log,
+          ioptions_.logger,
          "Failed to register data paths of column family (id: %d, name: %s)",
          id_, name_.c_str());
    }
@@ -560,7 +563,8 @@ ColumnFamilyData::ColumnFamilyData(
    internal_stats_.reset(
        new InternalStats(ioptions_.num_levels, ioptions_.clock, this));
    table_cache_.reset(new TableCache(ioptions_, file_options, _table_cache,
-                                      block_cache_tracer, io_tracer));
+                                      block_cache_tracer, io_tracer,
+                                      db_session_id));
    blob_file_cache_.reset(
        new BlobFileCache(_table_cache, ioptions(), soptions(), id_,
                          internal_stats_->GetBlobFileReadHist(), io_tracer));
@@ -578,13 +582,13 @@ ColumnFamilyData::ColumnFamilyData(
    } else if (ioptions_.compaction_style == kCompactionStyleNone) {
      compaction_picker_.reset(new NullCompactionPicker(
          ioptions_, &internal_comparator_));
-      ROCKS_LOG_WARN(ioptions_.info_log,
+      ROCKS_LOG_WARN(ioptions_.logger,
                     "Column family %s does not use any background compaction. "
                     "Compactions can only be done via CompactFiles\n",
                     GetName().c_str());
 #endif  // !ROCKSDB_LITE
    } else {
-      ROCKS_LOG_ERROR(ioptions_.info_log,
+      ROCKS_LOG_ERROR(ioptions_.logger,
                      "Unable to recognize the specified compaction style %d. "
                      "Column family %s will use kCompactionStyleLevel.\n",
                      ioptions_.compaction_style, GetName().c_str());
@@ -593,12 +597,12 @@ ColumnFamilyData::ColumnFamilyData(
    }

    if (column_family_set_->NumberOfColumnFamilies() < 10) {
-      ROCKS_LOG_INFO(ioptions_.info_log,
+      ROCKS_LOG_INFO(ioptions_.logger,
                     "--------------- Options for column family [%s]:\n",
                     name.c_str());
-      initial_cf_options_.Dump(ioptions_.info_log);
+      initial_cf_options_.Dump(ioptions_.logger);
    } else {
-      ROCKS_LOG_INFO(ioptions_.info_log, "\t(skipping printing options)\n");
+      ROCKS_LOG_INFO(ioptions_.logger, "\t(skipping printing options)\n");
    }
  }

@@ -654,7 +658,7 @@ ColumnFamilyData::~ColumnFamilyData() {
    Status s = ioptions_.env->UnregisterDbPaths(GetDbPaths());
    if (!s.ok()) {
      ROCKS_LOG_ERROR(
-          ioptions_.info_log,
+          ioptions_.logger,
          "Failed to unregister data paths of column family (id: %d, name: %s)",
          id_, name_.c_str());
    }
@@ -893,7 +897,7 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
      write_controller_token_ = write_controller->GetStopToken();
      internal_stats_->AddCFStats(InternalStats::MEMTABLE_LIMIT_STOPS, 1);
      ROCKS_LOG_WARN(
-          ioptions_.info_log,
+          ioptions_.logger,
          "[%s] Stopping writes because we have %d immutable memtables "
          "(waiting for flush), max_write_buffer_number is set to %d",
          name_.c_str(), imm()->NumNotFlushed(),
@@ -906,7 +910,7 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
        internal_stats_->AddCFStats(
            InternalStats::LOCKED_L0_FILE_COUNT_LIMIT_STOPS, 1);
      }
-      ROCKS_LOG_WARN(ioptions_.info_log,
+      ROCKS_LOG_WARN(ioptions_.logger,
                     "[%s] Stopping writes because we have %d level-0 files",
                     name_.c_str(), vstorage->l0_delay_trigger_count());
    } else if (write_stall_condition == WriteStallCondition::kStopped &&
@@ -915,7 +919,7 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
      internal_stats_->AddCFStats(
          InternalStats::PENDING_COMPACTION_BYTES_LIMIT_STOPS, 1);
      ROCKS_LOG_WARN(
-          ioptions_.info_log,
+          ioptions_.logger,
          "[%s] Stopping writes because of estimated pending compaction "
          "bytes %" PRIu64,
          name_.c_str(), compaction_needed_bytes);
@@ -927,7 +931,7 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
                     mutable_cf_options.disable_auto_compactions);
      internal_stats_->AddCFStats(InternalStats::MEMTABLE_LIMIT_SLOWDOWNS, 1);
      ROCKS_LOG_WARN(
-          ioptions_.info_log,
+          ioptions_.logger,
          "[%s] Stalling writes because we have %d immutable memtables "
          "(waiting for flush), max_write_buffer_number is set to %d "
          "rate %" PRIu64,
@@ -949,7 +953,7 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
        internal_stats_->AddCFStats(
            InternalStats::LOCKED_L0_FILE_COUNT_LIMIT_SLOWDOWNS, 1);
      }
-      ROCKS_LOG_WARN(ioptions_.info_log,
+      ROCKS_LOG_WARN(ioptions_.logger,
                     "[%s] Stalling writes because we have %d level-0 files "
                     "rate %" PRIu64,
                     name_.c_str(), vstorage->l0_delay_trigger_count(),
@@ -974,7 +978,7 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
      internal_stats_->AddCFStats(
          InternalStats::PENDING_COMPACTION_BYTES_LIMIT_SLOWDOWNS, 1);
      ROCKS_LOG_WARN(
-          ioptions_.info_log,
+          ioptions_.logger,
          "[%s] Stalling writes because of estimated pending compaction "
          "bytes %" PRIu64 " rate %" PRIu64,
          name_.c_str(), vstorage->estimated_compaction_needed_bytes(),
@@ -988,7 +992,7 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
        write_controller_token_ =
            write_controller->GetCompactionPressureToken();
        ROCKS_LOG_INFO(
-            ioptions_.info_log,
+            ioptions_.logger,
            "[%s] Increasing compaction threads because we have %d level-0 "
            "files ",
            name_.c_str(), vstorage->l0_delay_trigger_count());
@@ -1002,7 +1006,7 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
            write_controller->GetCompactionPressureToken();
        if (mutable_cf_options.soft_pending_compaction_bytes_limit > 0) {
          ROCKS_LOG_INFO(
-              ioptions_.info_log,
+              ioptions_.logger,
              "[%s] Increasing compaction threads because of estimated pending "
              "compaction "
              "bytes %" PRIu64,
@@ -1203,11 +1207,11 @@ SuperVersion* ColumnFamilyData::GetThreadLocalSuperVersion(DBImpl* db) {
  SuperVersion* sv = static_cast<SuperVersion*>(ptr);
  if (sv == SuperVersion::kSVObsolete ||
      sv->version_number != super_version_number_.load()) {
-    RecordTick(ioptions_.statistics, NUMBER_SUPERVERSION_ACQUIRES);
+    RecordTick(ioptions_.stats, NUMBER_SUPERVERSION_ACQUIRES);
    SuperVersion* sv_to_delete = nullptr;

    if (sv && sv->Unref()) {
-      RecordTick(ioptions_.statistics, NUMBER_SUPERVERSION_CLEANUPS);
+      RecordTick(ioptions_.stats, NUMBER_SUPERVERSION_CLEANUPS);
      db->mutex()->Lock();
      // NOTE: underlying resources held by superversion (sst files) might
      // not be released until the next background job.
@@ -1417,7 +1421,8 @@ Status ColumnFamilyData::AddDirectories(

    if (existing_dir == created_dirs->end()) {
      std::unique_ptr<FSDirectory> path_directory;
-      s = DBImpl::CreateAndNewDirectory(ioptions_.fs, p.path, &path_directory);
+      s = DBImpl::CreateAndNewDirectory(ioptions_.fs.get(), p.path,
+                                        &path_directory);
      if (!s.ok()) {
        return s;
      }
@@ -1448,12 +1453,13 @@ ColumnFamilySet::ColumnFamilySet(const std::string& dbname,
                                 WriteBufferManager* _write_buffer_manager,
                                 WriteController* _write_controller,
                                 BlockCacheTracer* const block_cache_tracer,
-                                 const std::shared_ptr<IOTracer>& io_tracer)
+                                 const std::shared_ptr<IOTracer>& io_tracer,
+                                 const std::string& db_session_id)
    : max_column_family_(0),
      dummy_cfd_(new ColumnFamilyData(
          ColumnFamilyData::kDummyColumnFamilyDataId, "", nullptr, nullptr,
          nullptr, ColumnFamilyOptions(), *db_options, file_options, nullptr,
-          block_cache_tracer, io_tracer)),
+          block_cache_tracer, io_tracer, db_session_id)),
      default_cfd_cache_(nullptr),
      db_name_(dbname),
      db_options_(db_options),
@@ -1462,7 +1468,8 @@ ColumnFamilySet::ColumnFamilySet(const std::string& dbname,
      write_buffer_manager_(_write_buffer_manager),
      write_controller_(_write_controller),
      block_cache_tracer_(block_cache_tracer),
-      io_tracer_(io_tracer) {
+      io_tracer_(io_tracer),
+      db_session_id_(db_session_id) {
  // initialize linked list
  dummy_cfd_->prev_ = dummy_cfd_;
  dummy_cfd_->next_ = dummy_cfd_;
@@ -1528,7 +1535,8 @@ ColumnFamilyData* ColumnFamilySet::CreateColumnFamily(
  assert(column_families_.find(name) == column_families_.end());
  ColumnFamilyData* new_cfd = new ColumnFamilyData(
      id, name, dummy_versions, table_cache_, write_buffer_manager_, options,
-      *db_options_, file_options_, this, block_cache_tracer_, io_tracer_);
+      *db_options_, file_options_, this, block_cache_tracer_, io_tracer_,
+      db_session_id_);
  column_families_.insert({name, id});
  column_family_data_.insert({id, new_cfd});
  max_column_family_ = std::max(max_column_family_, id);
@@ -258,8 +258,7 @@ extern ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
 // one too.
 extern void GetIntTblPropCollectorFactory(
    const ImmutableCFOptions& ioptions,
-    std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
-        int_tbl_prop_collector_factories);
+    IntTblPropCollectorFactories* int_tbl_prop_collector_factories);

 class ColumnFamilySet;

@@ -316,7 +315,7 @@ class ColumnFamilyData {
  FlushReason GetFlushReason() const { return flush_reason_; }
  // thread-safe
  const FileOptions* soptions() const;
-  const ImmutableCFOptions* ioptions() const { return &ioptions_; }
+  const ImmutableOptions* ioptions() const { return &ioptions_; }
  // REQUIRES: DB mutex held
  // This returns the MutableCFOptions used by current SuperVersion
  // You should use this API to reference MutableCFOptions most of the time.
@@ -428,8 +427,7 @@ class ColumnFamilyData {
    return internal_comparator_;
  }

-  const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
-  int_tbl_prop_collector_factories() const {
+  const IntTblPropCollectorFactories* int_tbl_prop_collector_factories() const {
    return &int_tbl_prop_collector_factories_;
  }

@@ -534,7 +532,8 @@ class ColumnFamilyData {
                   const FileOptions& file_options,
                   ColumnFamilySet* column_family_set,
                   BlockCacheTracer* const block_cache_tracer,
-                   const std::shared_ptr<IOTracer>& io_tracer);
+                   const std::shared_ptr<IOTracer>& io_tracer,
+                   const std::string& db_session_id);

  std::vector<std::string> GetDbPaths() const;

@@ -548,11 +547,10 @@ class ColumnFamilyData {
  std::atomic<bool> dropped_;  // true if client dropped it

  const InternalKeyComparator internal_comparator_;
-  std::vector<std::unique_ptr<IntTblPropCollectorFactory>>
-      int_tbl_prop_collector_factories_;
+  IntTblPropCollectorFactories int_tbl_prop_collector_factories_;

  const ColumnFamilyOptions initial_cf_options_;
-  const ImmutableCFOptions ioptions_;
+  const ImmutableOptions ioptions_;
  MutableCFOptions mutable_cf_options_;

  const bool is_delete_range_supported_;
@@ -671,7 +669,8 @@ class ColumnFamilySet {
                  WriteBufferManager* _write_buffer_manager,
                  WriteController* _write_controller,
                  BlockCacheTracer* const block_cache_tracer,
-                  const std::shared_ptr<IOTracer>& io_tracer);
+                  const std::shared_ptr<IOTracer>& io_tracer,
+                  const std::string& db_session_id);
  ~ColumnFamilySet();

  ColumnFamilyData* GetDefault() const;
@@ -736,6 +735,7 @@ class ColumnFamilySet {
  WriteController* write_controller_;
  BlockCacheTracer* const block_cache_tracer_;
  std::shared_ptr<IOTracer> io_tracer_;
+  std::string db_session_id_;
 };

 // We use ColumnFamilyMemTablesImpl to provide WriteBatch a way to access
@@ -56,16 +56,8 @@ class ColumnFamilyTestBase : public testing::Test {
 public:
  explicit ColumnFamilyTestBase(uint32_t format) : rnd_(139), format_(format) {
    Env* base_env = Env::Default();
-#ifndef ROCKSDB_LITE
-    const char* test_env_uri = getenv("TEST_ENV_URI");
-    if (test_env_uri) {
-      Env* test_env = nullptr;
-      Status s = Env::LoadEnv(test_env_uri, &test_env, &env_guard_);
-      base_env = test_env;
-      EXPECT_OK(s);
-      EXPECT_NE(Env::Default(), base_env);
-    }
-#endif  // !ROCKSDB_LITE
+    EXPECT_OK(
+        test::CreateEnvFromSystem(ConfigOptions(), &base_env, &env_guard_));
    EXPECT_NE(nullptr, base_env);
    env_ = new EnvCounter(base_env);
    env_->skip_fsync_ = true;
@@ -2286,6 +2278,8 @@ TEST_P(ColumnFamilyTest, SanitizeOptions) {
              // not a multiple of 4k, round up 4k
              expected_arena_block_size += 4 * 1024;
            }
+            expected_arena_block_size =
+                std::min(size_t{1024 * 1024}, expected_arena_block_size);
            ASSERT_EQ(expected_arena_block_size, result.arena_block_size);
          }
        }
@@ -0,0 +1,275 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include <cassert>
+
+#include "table/internal_iterator.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+// An internal iterator that wraps another one and ensures that any keys
+// returned are strictly within a range [start, end). If the underlying
+// iterator has already performed the bounds checking, it relies on that result;
+// otherwise, it performs the necessary key comparisons itself. Both bounds
+// are optional.
+class ClippingIterator : public InternalIterator {
+ public:
+  ClippingIterator(InternalIterator* iter, const Slice* start, const Slice* end,
+                   const Comparator* cmp)
+      : iter_(iter), start_(start), end_(end), cmp_(cmp), valid_(false) {
+    assert(iter_);
+    assert(cmp_);
+    assert(!start_ || !end_ || cmp_->Compare(*start_, *end_) <= 0);
+
+    UpdateAndEnforceBounds();
+  }
+
+  bool Valid() const override { return valid_; }
+
+  void SeekToFirst() override {
+    if (start_) {
+      iter_->Seek(*start_);
+    } else {
+      iter_->SeekToFirst();
+    }
+
+    UpdateAndEnforceUpperBound();
+  }
+
+  void SeekToLast() override {
+    if (end_) {
+      iter_->SeekForPrev(*end_);
+
+      // Upper bound is exclusive, so we need a key which is strictly smaller
+      if (iter_->Valid() && cmp_->Compare(iter_->key(), *end_) == 0) {
+        iter_->Prev();
+      }
+    } else {
+      iter_->SeekToLast();
+    }
+
+    UpdateAndEnforceLowerBound();
+  }
+
+  void Seek(const Slice& target) override {
+    if (start_ && cmp_->Compare(target, *start_) < 0) {
+      iter_->Seek(*start_);
+      UpdateAndEnforceUpperBound();
+      return;
+    }
+
+    if (end_ && cmp_->Compare(target, *end_) >= 0) {
+      valid_ = false;
+      return;
+    }
+
+    iter_->Seek(target);
+    UpdateAndEnforceUpperBound();
+  }
+
+  void SeekForPrev(const Slice& target) override {
+    if (start_ && cmp_->Compare(target, *start_) < 0) {
+      valid_ = false;
+      return;
+    }
+
+    if (end_ && cmp_->Compare(target, *end_) >= 0) {
+      iter_->SeekForPrev(*end_);
+
+      // Upper bound is exclusive, so we need a key which is strictly smaller
+      if (iter_->Valid() && cmp_->Compare(iter_->key(), *end_) == 0) {
+        iter_->Prev();
+      }
+
+      UpdateAndEnforceLowerBound();
+      return;
+    }
+
+    iter_->SeekForPrev(target);
+    UpdateAndEnforceLowerBound();
+  }
+
+  void Next() override {
+    assert(valid_);
+    iter_->Next();
+    UpdateAndEnforceUpperBound();
+  }
+
+  bool NextAndGetResult(IterateResult* result) override {
+    assert(valid_);
+    assert(result);
+
+    IterateResult res;
+    valid_ = iter_->NextAndGetResult(&res);
+
+    if (!valid_) {
+      return false;
+    }
+
+    if (end_) {
+      EnforceUpperBoundImpl(res.bound_check_result);
+
+      if (!valid_) {
+        return false;
+      }
+    }
+
+    res.bound_check_result = IterBoundCheck::kInbound;
+    *result = res;
+
+    return true;
+  }
+
+  void Prev() override {
+    assert(valid_);
+    iter_->Prev();
+    UpdateAndEnforceLowerBound();
+  }
+
+  Slice key() const override {
+    assert(valid_);
+    return iter_->key();
+  }
+
+  Slice user_key() const override {
+    assert(valid_);
+    return iter_->user_key();
+  }
+
+  Slice value() const override {
+    assert(valid_);
+    return iter_->value();
+  }
+
+  Status status() const override { return iter_->status(); }
+
+  bool PrepareValue() override {
+    assert(valid_);
+
+    if (iter_->PrepareValue()) {
+      return true;
+    }
+
+    assert(!iter_->Valid());
+    valid_ = false;
+    return false;
+  }
+
+  bool MayBeOutOfLowerBound() override {
+    assert(valid_);
+    return false;
+  }
+
+  IterBoundCheck UpperBoundCheckResult() override {
+    assert(valid_);
+    return IterBoundCheck::kInbound;
+  }
+
+  void SetPinnedItersMgr(PinnedIteratorsManager* pinned_iters_mgr) override {
+    iter_->SetPinnedItersMgr(pinned_iters_mgr);
+  }
+
+  bool IsKeyPinned() const override {
+    assert(valid_);
+    return iter_->IsKeyPinned();
+  }
+
+  bool IsValuePinned() const override {
+    assert(valid_);
+    return iter_->IsValuePinned();
+  }
+
+  Status GetProperty(std::string prop_name, std::string* prop) override {
+    return iter_->GetProperty(prop_name, prop);
+  }
+
+ private:
+  void UpdateValid() {
+    assert(!iter_->Valid() || iter_->status().ok());
+
+    valid_ = iter_->Valid();
+  }
+
+  void EnforceUpperBoundImpl(IterBoundCheck bound_check_result) {
+    if (bound_check_result == IterBoundCheck::kInbound) {
+      return;
+    }
+
+    if (bound_check_result == IterBoundCheck::kOutOfBound) {
+      valid_ = false;
+      return;
+    }
+
+    assert(bound_check_result == IterBoundCheck::kUnknown);
+
+    if (cmp_->Compare(key(), *end_) >= 0) {
+      valid_ = false;
+    }
+  }
+
+  void EnforceUpperBound() {
+    if (!valid_) {
+      return;
+    }
+
+    if (!end_) {
+      return;
+    }
+
+    EnforceUpperBoundImpl(iter_->UpperBoundCheckResult());
+  }
+
+  void EnforceLowerBound() {
+    if (!valid_) {
+      return;
+    }
+
+    if (!start_) {
+      return;
+    }
+
+    if (!iter_->MayBeOutOfLowerBound()) {
+      return;
+    }
+
+    if (cmp_->Compare(key(), *start_) < 0) {
+      valid_ = false;
+    }
+  }
+
+  void AssertBounds() {
+    assert(!valid_ || !start_ || cmp_->Compare(key(), *start_) >= 0);
+    assert(!valid_ || !end_ || cmp_->Compare(key(), *end_) < 0);
+  }
+
+  void UpdateAndEnforceBounds() {
+    UpdateValid();
+    EnforceUpperBound();
+    EnforceLowerBound();
+    AssertBounds();
+  }
+
+  void UpdateAndEnforceUpperBound() {
+    UpdateValid();
+    EnforceUpperBound();
+    AssertBounds();
+  }
+
+  void UpdateAndEnforceLowerBound() {
+    UpdateValid();
+    EnforceLowerBound();
+    AssertBounds();
+  }
+
+  InternalIterator* iter_;
+  const Slice* start_;
+  const Slice* end_;
+  const Comparator* cmp_;
+  bool valid_;
+};
+
+}  // namespace ROCKSDB_NAMESPACE
@@ -0,0 +1,256 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#include "db/compaction/clipping_iterator.h"
+
+#include <algorithm>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "rocksdb/comparator.h"
+#include "test_util/testharness.h"
+#include "test_util/testutil.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+// A vector iterator which does its own bounds checking. This is for testing the
+// optimizations in the clipping iterator where we bypass the bounds checking if
+// the input iterator has already performed it.
+class BoundsCheckingVectorIterator : public test::VectorIterator {
+ public:
+  BoundsCheckingVectorIterator(const std::vector<std::string>& keys,
+                               const std::vector<std::string>& values,
+                               const Slice* start, const Slice* end,
+                               const Comparator* cmp)
+      : VectorIterator(keys, values), start_(start), end_(end), cmp_(cmp) {
+    assert(cmp_);
+  }
+
+  bool NextAndGetResult(IterateResult* result) override {
+    assert(Valid());
+    assert(result);
+
+    Next();
+
+    if (!Valid()) {
+      return false;
+    }
+
+    result->key = key();
+    result->bound_check_result = UpperBoundCheckResult();
+    result->value_prepared = true;
+
+    return true;
+  }
+
+  bool MayBeOutOfLowerBound() override {
+    assert(Valid());
+
+    if (!start_) {
+      return false;
+    }
+
+    return cmp_->Compare(key(), *start_) < 0;
+  }
+
+  IterBoundCheck UpperBoundCheckResult() override {
+    assert(Valid());
+
+    if (!end_) {
+      return IterBoundCheck::kInbound;
+    }
+
+    return cmp_->Compare(key(), *end_) >= 0 ? IterBoundCheck::kOutOfBound
+                                            : IterBoundCheck::kInbound;
+  }
+
+ private:
+  const Slice* start_;
+  const Slice* end_;
+  const Comparator* cmp_;
+};
+
+class ClippingIteratorTest
+    : public ::testing::Test,
+      public ::testing::WithParamInterface<std::tuple<bool, size_t, size_t>> {};
+
+TEST_P(ClippingIteratorTest, Clip) {
+  const std::vector<std::string> keys{"key0", "key1", "key2", "key3", "key4",
+                                      "key5", "key6", "key7", "key8", "key9"};
+  const std::vector<std::string> values{
+      "unused0", "value1",  "value2",  "value3",  "unused4",
+      "unused5", "unused6", "unused7", "unused8", "unused9"};
+
+  assert(keys.size() == values.size());
+
+  // Note: the input always contains key1, key2, and key3; however, the clipping
+  // window is based on the test parameters: its left edge is a value in the
+  // range [0, 4], and its size is a value in the range [0, 5]
+  const std::vector<std::string> input_keys{keys[1], keys[2], keys[3]};
+  const std::vector<std::string> input_values{values[1], values[2], values[3]};
+
+  const bool use_bounds_checking_vec_it = std::get<0>(GetParam());
+
+  const size_t clip_start_idx = std::get<1>(GetParam());
+  const size_t clip_window_size = std::get<2>(GetParam());
+  const size_t clip_end_idx = clip_start_idx + clip_window_size;
+
+  const Slice start(keys[clip_start_idx]);
+  const Slice end(keys[clip_end_idx]);
+
+  std::unique_ptr<InternalIterator> input(
+      use_bounds_checking_vec_it
+          ? new BoundsCheckingVectorIterator(input_keys, input_values, &start,
+                                             &end, BytewiseComparator())
+          : new test::VectorIterator(input_keys, input_values));
+
+  ClippingIterator clip(input.get(), &start, &end, BytewiseComparator());
+
+  // The range the clipping iterator should return values from. This is
+  // essentially the intersection of the input range [1, 4) and the clipping
+  // window [clip_start_idx, clip_end_idx)
+  const size_t data_start_idx =
+      std::max(clip_start_idx, static_cast<size_t>(1));
+  const size_t data_end_idx = std::min(clip_end_idx, static_cast<size_t>(4));
+
+  // Range is empty; all Seeks should fail
+  if (data_start_idx >= data_end_idx) {
+    clip.SeekToFirst();
+    ASSERT_FALSE(clip.Valid());
+
+    clip.SeekToLast();
+    ASSERT_FALSE(clip.Valid());
+
+    for (size_t i = 0; i < keys.size(); ++i) {
+      clip.Seek(keys[i]);
+      ASSERT_FALSE(clip.Valid());
+
+      clip.SeekForPrev(keys[i]);
+      ASSERT_FALSE(clip.Valid());
+    }
+
+    return;
+  }
+
+  // Range is non-empty; call SeekToFirst and iterate forward
+  clip.SeekToFirst();
+  ASSERT_TRUE(clip.Valid());
+  ASSERT_EQ(clip.key(), keys[data_start_idx]);
+  ASSERT_EQ(clip.value(), values[data_start_idx]);
+  ASSERT_FALSE(clip.MayBeOutOfLowerBound());
+  ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound);
+
+  for (size_t i = data_start_idx + 1; i < data_end_idx; ++i) {
+    clip.Next();
+    ASSERT_TRUE(clip.Valid());
+    ASSERT_EQ(clip.key(), keys[i]);
+    ASSERT_EQ(clip.value(), values[i]);
+    ASSERT_FALSE(clip.MayBeOutOfLowerBound());
+    ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound);
+  }
+
+  clip.Next();
+  ASSERT_FALSE(clip.Valid());
+
+  // Do it again using NextAndGetResult
+  clip.SeekToFirst();
+  ASSERT_TRUE(clip.Valid());
+  ASSERT_EQ(clip.key(), keys[data_start_idx]);
+  ASSERT_EQ(clip.value(), values[data_start_idx]);
+  ASSERT_FALSE(clip.MayBeOutOfLowerBound());
+  ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound);
+
+  for (size_t i = data_start_idx + 1; i < data_end_idx; ++i) {
+    IterateResult result;
+    ASSERT_TRUE(clip.NextAndGetResult(&result));
+    ASSERT_EQ(result.key, keys[i]);
+    ASSERT_EQ(result.bound_check_result, IterBoundCheck::kInbound);
+    ASSERT_TRUE(clip.Valid());
+    ASSERT_EQ(clip.key(), keys[i]);
+    ASSERT_EQ(clip.value(), values[i]);
+    ASSERT_FALSE(clip.MayBeOutOfLowerBound());
+    ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound);
+  }
+
+  IterateResult result;
+  ASSERT_FALSE(clip.NextAndGetResult(&result));
+  ASSERT_FALSE(clip.Valid());
+
+  // Call SeekToLast and iterate backward
+  clip.SeekToLast();
+  ASSERT_TRUE(clip.Valid());
+  ASSERT_EQ(clip.key(), keys[data_end_idx - 1]);
+  ASSERT_EQ(clip.value(), values[data_end_idx - 1]);
+  ASSERT_FALSE(clip.MayBeOutOfLowerBound());
+  ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound);
+
+  for (size_t i = data_end_idx - 2; i >= data_start_idx; --i) {
+    clip.Prev();
+    ASSERT_TRUE(clip.Valid());
+    ASSERT_EQ(clip.key(), keys[i]);
+    ASSERT_EQ(clip.value(), values[i]);
+    ASSERT_FALSE(clip.MayBeOutOfLowerBound());
+    ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound);
+  }
+
+  clip.Prev();
+  ASSERT_FALSE(clip.Valid());
+
+  // Call Seek/SeekForPrev for all keys; Seek should return the smallest key
+  // which is >= the target; SeekForPrev should return the largest key which is
+  // <= the target
+  for (size_t i = 0; i < keys.size(); ++i) {
+    clip.Seek(keys[i]);
+
+    if (i < data_start_idx) {
+      ASSERT_TRUE(clip.Valid());
+      ASSERT_EQ(clip.key(), keys[data_start_idx]);
+      ASSERT_EQ(clip.value(), values[data_start_idx]);
+      ASSERT_FALSE(clip.MayBeOutOfLowerBound());
+      ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound);
+    } else if (i < data_end_idx) {
+      ASSERT_TRUE(clip.Valid());
+      ASSERT_EQ(clip.key(), keys[i]);
+      ASSERT_EQ(clip.value(), values[i]);
+      ASSERT_FALSE(clip.MayBeOutOfLowerBound());
+      ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound);
+    } else {
+      ASSERT_FALSE(clip.Valid());
+    }
+
+    clip.SeekForPrev(keys[i]);
+
+    if (i < data_start_idx) {
+      ASSERT_FALSE(clip.Valid());
+    } else if (i < data_end_idx) {
+      ASSERT_TRUE(clip.Valid());
+      ASSERT_EQ(clip.key(), keys[i]);
+      ASSERT_EQ(clip.value(), values[i]);
+      ASSERT_FALSE(clip.MayBeOutOfLowerBound());
+      ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound);
+    } else {
+      ASSERT_TRUE(clip.Valid());
+      ASSERT_EQ(clip.key(), keys[data_end_idx - 1]);
+      ASSERT_EQ(clip.value(), values[data_end_idx - 1]);
+      ASSERT_FALSE(clip.MayBeOutOfLowerBound());
+      ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound);
+    }
+  }
+}
+
+INSTANTIATE_TEST_CASE_P(
+    ClippingIteratorTest, ClippingIteratorTest,
+    ::testing::Combine(
+        ::testing::Bool(),
+        ::testing::Range(static_cast<size_t>(0), static_cast<size_t>(5)),
+        ::testing::Range(static_cast<size_t>(0), static_cast<size_t>(6))));
+
+}  // namespace ROCKSDB_NAMESPACE
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
@@ -204,27 +204,24 @@ bool Compaction::IsFullCompaction(
  return num_files_in_compaction == total_num_files;
 }

-Compaction::Compaction(VersionStorageInfo* vstorage,
-                       const ImmutableCFOptions& _immutable_cf_options,
-                       const MutableCFOptions& _mutable_cf_options,
-                       const MutableDBOptions& _mutable_db_options,
-                       std::vector<CompactionInputFiles> _inputs,
-                       int _output_level, uint64_t _target_file_size,
-                       uint64_t _max_compaction_bytes, uint32_t _output_path_id,
-                       CompressionType _compression,
-                       CompressionOptions _compression_opts,
-                       uint32_t _max_subcompactions,
-                       std::vector<FileMetaData*> _grandparents,
-                       bool _manual_compaction, double _score,
-                       bool _deletion_compaction,
-                       CompactionReason _compaction_reason)
+Compaction::Compaction(
+    VersionStorageInfo* vstorage, const ImmutableOptions& _immutable_options,
+    const MutableCFOptions& _mutable_cf_options,
+    const MutableDBOptions& _mutable_db_options,
+    std::vector<CompactionInputFiles> _inputs, int _output_level,
+    uint64_t _target_file_size, uint64_t _max_compaction_bytes,
+    uint32_t _output_path_id, CompressionType _compression,
+    CompressionOptions _compression_opts, uint32_t _max_subcompactions,
+    std::vector<FileMetaData*> _grandparents, bool _manual_compaction,
+    double _score, bool _deletion_compaction,
+    CompactionReason _compaction_reason)
    : input_vstorage_(vstorage),
      start_level_(_inputs[0].level),
      output_level_(_output_level),
      max_output_file_size_(_target_file_size),
      max_compaction_bytes_(_max_compaction_bytes),
      max_subcompactions_(_max_subcompactions),
-      immutable_cf_options_(_immutable_cf_options),
+      immutable_options_(_immutable_options),
      mutable_cf_options_(_mutable_cf_options),
      input_version_(nullptr),
      number_levels_(vstorage->num_levels()),
@@ -278,7 +275,7 @@ Compaction::~Compaction() {

 bool Compaction::InputCompressionMatchesOutput() const {
  int base_level = input_vstorage_->base_level();
-  bool matches = (GetCompressionType(immutable_cf_options_, input_vstorage_,
+  bool matches = (GetCompressionType(immutable_options_, input_vstorage_,
                                     mutable_cf_options_, start_level_,
                                     base_level) == output_compression_);
  if (matches) {
@@ -303,8 +300,8 @@ bool Compaction::IsTrivialMove() const {
  }

  if (is_manual_compaction_ &&
-      (immutable_cf_options_.compaction_filter != nullptr ||
-       immutable_cf_options_.compaction_filter_factory != nullptr)) {
+      (immutable_options_.compaction_filter != nullptr ||
+       immutable_options_.compaction_filter_factory != nullptr)) {
    // This is a manual compaction and we have a compaction filter that should
    // be executed, we cannot do a trivial move
    return false;
@@ -512,7 +509,7 @@ uint64_t Compaction::OutputFilePreallocationSize() const {
  }

  if (max_output_file_size_ != port::kMaxUint64 &&
-      (immutable_cf_options_.compaction_style == kCompactionStyleLevel ||
+      (immutable_options_.compaction_style == kCompactionStyleLevel ||
       output_level() > 0)) {
    preallocation_size = std::min(max_output_file_size_, preallocation_size);
  }
@@ -529,16 +526,23 @@ std::unique_ptr<CompactionFilter> Compaction::CreateCompactionFilter() const {
    return nullptr;
  }

+  if (!cfd_->ioptions()
+           ->compaction_filter_factory->ShouldFilterTableFileCreation(
+               TableFileCreationReason::kCompaction)) {
+    return nullptr;
+  }
+
  CompactionFilter::Context context;
  context.is_full_compaction = is_full_compaction_;
  context.is_manual_compaction = is_manual_compaction_;
  context.column_family_id = cfd_->GetID();
+  context.reason = TableFileCreationReason::kCompaction;
  return cfd_->ioptions()->compaction_filter_factory->CreateCompactionFilter(
      context);
 }

 std::unique_ptr<SstPartitioner> Compaction::CreateSstPartitioner() const {
-  if (!immutable_cf_options_.sst_partitioner_factory) {
+  if (!immutable_options_.sst_partitioner_factory) {
    return nullptr;
  }

@@ -548,8 +552,7 @@ std::unique_ptr<SstPartitioner> Compaction::CreateSstPartitioner() const {
  context.output_level = output_level_;
  context.smallest_user_key = smallest_user_key_;
  context.largest_user_key = largest_user_key_;
-  return immutable_cf_options_.sst_partitioner_factory->CreatePartitioner(
-      context);
+  return immutable_options_.sst_partitioner_factory->CreatePartitioner(context);
 }

 bool Compaction::IsOutputLevelEmpty() const {
@@ -560,6 +563,14 @@ bool Compaction::ShouldFormSubcompactions() const {
  if (max_subcompactions_ <= 1 || cfd_ == nullptr) {
    return false;
  }
+
+  // Note: the subcompaction boundary picking logic does not currently guarantee
+  // that all user keys that differ only by timestamp get processed by the same
+  // subcompaction.
+  if (cfd_->user_comparator()->timestamp_size() > 0) {
+    return false;
+  }
+
  if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
    return (start_level_ == 0 || is_manual_compaction_) && output_level_ > 0 &&
           !IsOutputLevelEmpty();
@@ -70,7 +70,7 @@ class CompactionFilter;
 class Compaction {
 public:
  Compaction(VersionStorageInfo* input_version,
-             const ImmutableCFOptions& immutable_cf_options,
+             const ImmutableOptions& immutable_options,
             const MutableCFOptions& mutable_cf_options,
             const MutableDBOptions& mutable_db_options,
             std::vector<CompactionInputFiles> inputs, int output_level,
@@ -162,7 +162,7 @@ class Compaction {
  CompressionType output_compression() const { return output_compression_; }

  // What compression options for output
-  CompressionOptions output_compression_opts() const {
+  const CompressionOptions& output_compression_opts() const {
    return output_compression_opts_;
  }

@@ -223,10 +223,10 @@ class Compaction {
  // How many total levels are there?
  int number_levels() const { return number_levels_; }

-  // Return the ImmutableCFOptions that should be used throughout the compaction
+  // Return the ImmutableOptions that should be used throughout the compaction
  // procedure
-  const ImmutableCFOptions* immutable_cf_options() const {
-    return &immutable_cf_options_;
+  const ImmutableOptions* immutable_options() const {
+    return &immutable_options_;
  }

  // Return the MutableCFOptions that should be used throughout the compaction
@@ -330,7 +330,7 @@ class Compaction {
  uint64_t max_output_file_size_;
  uint64_t max_compaction_bytes_;
  uint32_t max_subcompactions_;
-  const ImmutableCFOptions immutable_cf_options_;
+  const ImmutableOptions immutable_options_;
  const MutableCFOptions mutable_cf_options_;
  Version* input_version_;
  VersionEdit edit_;
@@ -33,7 +33,6 @@
   (snapshot_checker_ == nullptr || LIKELY(IsInEarliestSnapshot(seq))))

 namespace ROCKSDB_NAMESPACE {
-
 CompactionIterator::CompactionIterator(
    InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
    SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
@@ -46,6 +45,7 @@ CompactionIterator::CompactionIterator(
    const std::atomic<bool>* shutting_down,
    const SequenceNumber preserve_deletes_seqnum,
    const std::atomic<int>* manual_compaction_paused,
+    const std::atomic<bool>* manual_compaction_canceled,
    const std::shared_ptr<Logger> info_log,
    const std::string* full_history_ts_low)
    : CompactionIterator(
@@ -56,7 +56,8 @@ CompactionIterator::CompactionIterator(
          std::unique_ptr<CompactionProxy>(
              compaction ? new RealCompaction(compaction) : nullptr),
          compaction_filter, shutting_down, preserve_deletes_seqnum,
-          manual_compaction_paused, info_log, full_history_ts_low) {}
+          manual_compaction_paused, manual_compaction_canceled, info_log,
+          full_history_ts_low) {}

 CompactionIterator::CompactionIterator(
    InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
@@ -71,9 +72,13 @@ CompactionIterator::CompactionIterator(
    const std::atomic<bool>* shutting_down,
    const SequenceNumber preserve_deletes_seqnum,
    const std::atomic<int>* manual_compaction_paused,
+    const std::atomic<bool>* manual_compaction_canceled,
    const std::shared_ptr<Logger> info_log,
    const std::string* full_history_ts_low)
-    : input_(input),
+    : input_(
+          input, cmp,
+          compaction ==
+              nullptr),  // Now only need to count number of entries in flush.
      cmp_(cmp),
      merge_helper_(merge_helper),
      snapshots_(snapshots),
@@ -89,6 +94,7 @@ CompactionIterator::CompactionIterator(
      compaction_filter_(compaction_filter),
      shutting_down_(shutting_down),
      manual_compaction_paused_(manual_compaction_paused),
+      manual_compaction_canceled_(manual_compaction_canceled),
      preserve_deletes_seqnum_(preserve_deletes_seqnum),
      info_log_(info_log),
      allow_data_in_errors_(allow_data_in_errors),
@@ -100,8 +106,8 @@ CompactionIterator::CompactionIterator(
      blob_garbage_collection_cutoff_file_number_(
          ComputeBlobGarbageCollectionCutoffFileNumber(compaction_.get())),
      current_key_committed_(false),
-      cmp_with_history_ts_low_(0) {
-  assert(compaction_filter_ == nullptr || compaction_ != nullptr);
+      cmp_with_history_ts_low_(0),
+      level_(compaction_ == nullptr ? 0 : compaction_->level()) {
  assert(snapshots_ != nullptr);
  bottommost_level_ = compaction_ == nullptr
                          ? false
@@ -130,13 +136,13 @@ CompactionIterator::CompactionIterator(
  assert(timestamp_size_ == 0 || !full_history_ts_low_ ||
         timestamp_size_ == full_history_ts_low_->size());
 #endif
-  input_->SetPinnedItersMgr(&pinned_iters_mgr_);
+  input_.SetPinnedItersMgr(&pinned_iters_mgr_);
  TEST_SYNC_POINT_CALLBACK("CompactionIterator:AfterInit", compaction_.get());
 }

 CompactionIterator::~CompactionIterator() {
  // input_ Iterator lifetime is longer than pinned_iters_mgr_ lifetime
-  input_->SetPinnedItersMgr(nullptr);
+  input_.SetPinnedItersMgr(nullptr);
 }

 void CompactionIterator::ResetRecordCounts() {
@@ -189,7 +195,7 @@ void CompactionIterator::Next() {
    // Only advance the input iterator if there is no merge output and the
    // iterator is not already at the next record.
    if (!at_next_) {
-      input_->Next();
+      AdvanceInputIter();
    }
    NextFromInput();
  }
@@ -232,7 +238,7 @@ bool CompactionIterator::InvokeFilterIfNeeded(bool* need_skip,
    if (kTypeBlobIndex == ikey_.type) {
      blob_value_.Reset();
      filter = compaction_filter_->FilterBlobByKey(
-          compaction_->level(), filter_key, &compaction_filter_value_,
+          level_, filter_key, &compaction_filter_value_,
          compaction_filter_skip_until_.rep());
      if (CompactionFilter::Decision::kUndetermined == filter &&
          !compaction_filter_->IsStackedBlobDbInternalCompactionFilter()) {
@@ -251,6 +257,12 @@ bool CompactionIterator::InvokeFilterIfNeeded(bool* need_skip,
          valid_ = false;
          return false;
        }
+        if (compaction_ == nullptr) {
+          status_ =
+              Status::Corruption("Unexpected blob index outside of compaction");
+          valid_ = false;
+          return false;
+        }
        const Version* const version = compaction_->input_version();
        assert(version);

@@ -271,7 +283,7 @@ bool CompactionIterator::InvokeFilterIfNeeded(bool* need_skip,
    }
    if (CompactionFilter::Decision::kUndetermined == filter) {
      filter = compaction_filter_->FilterV2(
-          compaction_->level(), filter_key, value_type,
+          level_, filter_key, value_type,
          blob_value_.empty() ? value_ : blob_value_, &compaction_filter_value_,
          compaction_filter_skip_until_.rep());
    }
@@ -350,10 +362,10 @@ void CompactionIterator::NextFromInput() {
  at_next_ = false;
  valid_ = false;

-  while (!valid_ && input_->Valid() && !IsPausingManualCompaction() &&
+  while (!valid_ && input_.Valid() && !IsPausingManualCompaction() &&
         !IsShuttingDown()) {
-    key_ = input_->key();
-    value_ = input_->value();
+    key_ = input_.key();
+    value_ = input_.value();
    iter_stats_.num_input_records++;

    Status pik_status = ParseInternalKey(key_, &ikey_, allow_data_in_errors_);
@@ -553,12 +565,12 @@ void CompactionIterator::NextFromInput() {
      // The easiest way to process a SingleDelete during iteration is to peek
      // ahead at the next key.
      ParsedInternalKey next_ikey;
-      input_->Next();
+      AdvanceInputIter();

      // Check whether the next key exists, is not corrupt, and is the same key
      // as the single delete.
-      if (input_->Valid() &&
-          ParseInternalKey(input_->key(), &next_ikey, allow_data_in_errors_)
+      if (input_.Valid() &&
+          ParseInternalKey(input_.key(), &next_ikey, allow_data_in_errors_)
              .ok() &&
          cmp_->Equal(ikey_.user_key, next_ikey.user_key)) {
        // Check whether the next key belongs to the same snapshot as the
@@ -572,7 +584,7 @@ void CompactionIterator::NextFromInput() {
            // to handle the second SingleDelete

            // First SingleDelete has been skipped since we already called
-            // input_->Next().
+            // input_.Next().
            ++iter_stats_.num_record_drop_obsolete;
            ++iter_stats_.num_single_del_mismatch;
          } else if (has_outputted_key_ ||
@@ -594,9 +606,9 @@ void CompactionIterator::NextFromInput() {

            ++iter_stats_.num_record_drop_hidden;
            ++iter_stats_.num_record_drop_obsolete;
-            // Already called input_->Next() once.  Call it a second time to
+            // Already called input_.Next() once.  Call it a second time to
            // skip past the second key.
-            input_->Next();
+            AdvanceInputIter();
          } else {
            // Found a matching value, but we cannot drop both keys since
            // there is an earlier snapshot and we need to leave behind a record
@@ -664,7 +676,7 @@ void CompactionIterator::NextFromInput() {
      }

      ++iter_stats_.num_record_drop_hidden;  // rule (A)
-      input_->Next();
+      AdvanceInputIter();
    } else if (compaction_ != nullptr &&
               (ikey_.type == kTypeDeletion ||
                (ikey_.type == kTypeDeletionWithTimestamp &&
@@ -700,7 +712,7 @@ void CompactionIterator::NextFromInput() {
      if (!bottommost_level_) {
        ++iter_stats_.num_optimized_del_drop_obsolete;
      }
-      input_->Next();
+      AdvanceInputIter();
    } else if ((ikey_.type == kTypeDeletion ||
                (ikey_.type == kTypeDeletionWithTimestamp &&
                 cmp_with_history_ts_low_ < 0)) &&
@@ -711,7 +723,7 @@ void CompactionIterator::NextFromInput() {
      assert(!compaction_ || compaction_->KeyNotExistsBeyondOutputLevel(
                                 ikey_.user_key, &level_ptrs_));
      ParsedInternalKey next_ikey;
-      input_->Next();
+      AdvanceInputIter();
      // Skip over all versions of this key that happen to occur in the same
      // snapshot range as the delete.
      //
@@ -719,18 +731,18 @@ void CompactionIterator::NextFromInput() {
      // considered to have a different user key unless the timestamp is older
      // than *full_history_ts_low_.
      while (!IsPausingManualCompaction() && !IsShuttingDown() &&
-             input_->Valid() &&
-             (ParseInternalKey(input_->key(), &next_ikey, allow_data_in_errors_)
+             input_.Valid() &&
+             (ParseInternalKey(input_.key(), &next_ikey, allow_data_in_errors_)
                  .ok()) &&
             cmp_->EqualWithoutTimestamp(ikey_.user_key, next_ikey.user_key) &&
             (prev_snapshot == 0 ||
              DEFINITELY_NOT_IN_SNAPSHOT(next_ikey.sequence, prev_snapshot))) {
-        input_->Next();
+        AdvanceInputIter();
      }
      // If you find you still need to output a row with this key, we need to output the
      // delete too
-      if (input_->Valid() &&
-          (ParseInternalKey(input_->key(), &next_ikey, allow_data_in_errors_)
+      if (input_.Valid() &&
+          (ParseInternalKey(input_.key(), &next_ikey, allow_data_in_errors_)
               .ok()) &&
          cmp_->EqualWithoutTimestamp(ikey_.user_key, next_ikey.user_key)) {
        valid_ = true;
@@ -749,7 +761,7 @@ void CompactionIterator::NextFromInput() {
      // We encapsulate the merge related state machine in a different
      // object to minimize change to the existing flow.
      Status s =
-          merge_helper_->MergeUntil(input_, range_del_agg_, prev_snapshot,
+          merge_helper_->MergeUntil(&input_, range_del_agg_, prev_snapshot,
                                    bottommost_level_, allow_data_in_errors_);
      merge_out_iter_.SeekToFirst();

@@ -793,14 +805,14 @@ void CompactionIterator::NextFromInput() {
      if (should_delete) {
        ++iter_stats_.num_record_drop_hidden;
        ++iter_stats_.num_record_drop_range_del;
-        input_->Next();
+        AdvanceInputIter();
      } else {
        valid_ = true;
      }
    }

    if (need_skip) {
-      input_->Seek(skip_until);
+      SkipUntil(skip_until);
    }
  }

@@ -24,6 +24,51 @@ namespace ROCKSDB_NAMESPACE {

 class BlobFileBuilder;

+// A wrapper of internal iterator whose purpose is to count how
+// many entries there are in the iterator.
+class SequenceIterWrapper : public InternalIterator {
+ public:
+  SequenceIterWrapper(InternalIterator* iter, const Comparator* cmp,
+                      bool need_count_entries)
+      : icmp_(cmp, /*named=*/false),
+        inner_iter_(iter),
+        need_count_entries_(need_count_entries) {}
+  bool Valid() const override { return inner_iter_->Valid(); }
+  Status status() const override { return inner_iter_->status(); }
+  void Next() override {
+    num_itered_++;
+    inner_iter_->Next();
+  }
+  void Seek(const Slice& target) override {
+    if (!need_count_entries_) {
+      inner_iter_->Seek(target);
+    } else {
+      // For flush cases, we need to count total number of entries, so we
+      // do Next() rather than Seek().
+      while (inner_iter_->Valid() &&
+             icmp_.Compare(inner_iter_->key(), target) < 0) {
+        Next();
+      }
+    }
+  }
+  Slice key() const override { return inner_iter_->key(); }
+  Slice value() const override { return inner_iter_->value(); }
+
+  // Unused InternalIterator methods
+  void SeekToFirst() override { assert(false); }
+  void Prev() override { assert(false); }
+  void SeekForPrev(const Slice& /* target */) override { assert(false); }
+  void SeekToLast() override { assert(false); }
+
+  uint64_t num_itered() const { return num_itered_; }
+
+ private:
+  InternalKeyComparator icmp_;
+  InternalIterator* inner_iter_;  // not owned
+  uint64_t num_itered_ = 0;
+  bool need_count_entries_;
+};
+
 class CompactionIterator {
 public:
  // A wrapper around Compaction. Has a much smaller interface, only what
@@ -59,7 +104,7 @@ class CompactionIterator {
    explicit RealCompaction(const Compaction* compaction)
        : compaction_(compaction) {
      assert(compaction_);
-      assert(compaction_->immutable_cf_options());
+      assert(compaction_->immutable_options());
      assert(compaction_->mutable_cf_options());
    }

@@ -81,11 +126,11 @@ class CompactionIterator {
    }

    bool allow_ingest_behind() const override {
-      return compaction_->immutable_cf_options()->allow_ingest_behind;
+      return compaction_->immutable_options()->allow_ingest_behind;
    }

    bool preserve_deletes() const override {
-      return compaction_->immutable_cf_options()->preserve_deletes;
+      return compaction_->immutable_options()->preserve_deletes;
    }

    bool enable_blob_garbage_collection() const override {
@@ -105,40 +150,40 @@ class CompactionIterator {
    const Compaction* compaction_;
  };

-  CompactionIterator(InternalIterator* input, const Comparator* cmp,
-                     MergeHelper* merge_helper, SequenceNumber last_sequence,
-                     std::vector<SequenceNumber>* snapshots,
-                     SequenceNumber earliest_write_conflict_snapshot,
-                     const SnapshotChecker* snapshot_checker, Env* env,
-                     bool report_detailed_time, bool expect_valid_internal_key,
-                     CompactionRangeDelAggregator* range_del_agg,
-                     BlobFileBuilder* blob_file_builder,
-                     bool allow_data_in_errors,
-                     const Compaction* compaction = nullptr,
-                     const CompactionFilter* compaction_filter = nullptr,
-                     const std::atomic<bool>* shutting_down = nullptr,
-                     const SequenceNumber preserve_deletes_seqnum = 0,
-                     const std::atomic<int>* manual_compaction_paused = nullptr,
-                     const std::shared_ptr<Logger> info_log = nullptr,
-                     const std::string* full_history_ts_low = nullptr);
+  CompactionIterator(
+      InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
+      SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
+      SequenceNumber earliest_write_conflict_snapshot,
+      const SnapshotChecker* snapshot_checker, Env* env,
+      bool report_detailed_time, bool expect_valid_internal_key,
+      CompactionRangeDelAggregator* range_del_agg,
+      BlobFileBuilder* blob_file_builder, bool allow_data_in_errors,
+      const Compaction* compaction = nullptr,
+      const CompactionFilter* compaction_filter = nullptr,
+      const std::atomic<bool>* shutting_down = nullptr,
+      const SequenceNumber preserve_deletes_seqnum = 0,
+      const std::atomic<int>* manual_compaction_paused = nullptr,
+      const std::atomic<bool>* manual_compaction_canceled = nullptr,
+      const std::shared_ptr<Logger> info_log = nullptr,
+      const std::string* full_history_ts_low = nullptr);

  // Constructor with custom CompactionProxy, used for tests.
-  CompactionIterator(InternalIterator* input, const Comparator* cmp,
-                     MergeHelper* merge_helper, SequenceNumber last_sequence,
-                     std::vector<SequenceNumber>* snapshots,
-                     SequenceNumber earliest_write_conflict_snapshot,
-                     const SnapshotChecker* snapshot_checker, Env* env,
-                     bool report_detailed_time, bool expect_valid_internal_key,
-                     CompactionRangeDelAggregator* range_del_agg,
-                     BlobFileBuilder* blob_file_builder,
-                     bool allow_data_in_errors,
-                     std::unique_ptr<CompactionProxy> compaction,
-                     const CompactionFilter* compaction_filter = nullptr,
-                     const std::atomic<bool>* shutting_down = nullptr,
-                     const SequenceNumber preserve_deletes_seqnum = 0,
-                     const std::atomic<int>* manual_compaction_paused = nullptr,
-                     const std::shared_ptr<Logger> info_log = nullptr,
-                     const std::string* full_history_ts_low = nullptr);
+  CompactionIterator(
+      InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
+      SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
+      SequenceNumber earliest_write_conflict_snapshot,
+      const SnapshotChecker* snapshot_checker, Env* env,
+      bool report_detailed_time, bool expect_valid_internal_key,
+      CompactionRangeDelAggregator* range_del_agg,
+      BlobFileBuilder* blob_file_builder, bool allow_data_in_errors,
+      std::unique_ptr<CompactionProxy> compaction,
+      const CompactionFilter* compaction_filter = nullptr,
+      const std::atomic<bool>* shutting_down = nullptr,
+      const SequenceNumber preserve_deletes_seqnum = 0,
+      const std::atomic<int>* manual_compaction_paused = nullptr,
+      const std::atomic<bool>* manual_compaction_canceled = nullptr,
+      const std::shared_ptr<Logger> info_log = nullptr,
+      const std::string* full_history_ts_low = nullptr);

  ~CompactionIterator();

@@ -162,6 +207,7 @@ class CompactionIterator {
  bool Valid() const { return valid_; }
  const Slice& user_key() const { return current_user_key_; }
  const CompactionIterationStats& iter_stats() const { return iter_stats_; }
+  uint64_t num_input_entry_scanned() const { return input_.num_itered(); }

 private:
  // Processes the input stream to find the next output
@@ -234,7 +280,7 @@ class CompactionIterator {
  static uint64_t ComputeBlobGarbageCollectionCutoffFileNumber(
      const CompactionProxy* compaction);

-  InternalIterator* input_;
+  SequenceIterWrapper input_;
  const Comparator* cmp_;
  MergeHelper* merge_helper_;
  const std::vector<SequenceNumber>* snapshots_;
@@ -257,6 +303,7 @@ class CompactionIterator {
  const CompactionFilter* compaction_filter_;
  const std::atomic<bool>* shutting_down_;
  const std::atomic<int>* manual_compaction_paused_;
+  const std::atomic<bool>* manual_compaction_canceled_;
  const SequenceNumber preserve_deletes_seqnum_;
  bool bottommost_level_;
  bool valid_ = false;
@@ -340,6 +387,12 @@ class CompactionIterator {
  // Saved result of ucmp->CompareTimestamp(current_ts_, *full_history_ts_low_)
  int cmp_with_history_ts_low_;

+  const int level_;
+
+  void AdvanceInputIter() { input_.Next(); }
+
+  void SkipUntil(const Slice& skip_until) { input_.Seek(skip_until); }
+
  bool IsShuttingDown() {
    // This is a best-effort facility, so memory_order_relaxed is sufficient.
    return shutting_down_ && shutting_down_->load(std::memory_order_relaxed);
@@ -347,8 +400,10 @@ class CompactionIterator {

  bool IsPausingManualCompaction() {
    // This is a best-effort facility, so memory_order_relaxed is sufficient.
-    return manual_compaction_paused_ &&
-           manual_compaction_paused_->load(std::memory_order_relaxed) > 0;
+    return (manual_compaction_paused_ &&
+            manual_compaction_paused_->load(std::memory_order_relaxed) > 0) ||
+           (manual_compaction_canceled_ &&
+            manual_compaction_canceled_->load(std::memory_order_relaxed));
  }
 };
 }  // namespace ROCKSDB_NAMESPACE
@@ -282,7 +282,8 @@ class CompactionIteratorTest : public testing::TestWithParam<bool> {
        range_del_agg_.get(), nullptr /* blob_file_builder */,
        true /*allow_data_in_errors*/, std::move(compaction), filter,
        &shutting_down_, /*preserve_deletes_seqnum=*/0,
-        /*manual_compaction_paused=*/nullptr, /*info_log=*/nullptr,
+        /*manual_compaction_paused=*/nullptr,
+        /*manual_compaction_canceled=*/nullptr, /*info_log=*/nullptr,
        full_history_ts_low));
  }

@@ -66,6 +66,7 @@ class CompactionJob {
 public:
  CompactionJob(
      int job_id, Compaction* compaction, const ImmutableDBOptions& db_options,
+      const MutableDBOptions& mutable_db_options,
      const FileOptions& file_options, VersionSet* versions,
      const std::atomic<bool>* shutting_down,
      const SequenceNumber preserve_deletes_seqnum, LogBuffer* log_buffer,
@@ -80,6 +81,7 @@ class CompactionJob {
      const std::string& dbname, CompactionJobStats* compaction_job_stats,
      Env::Priority thread_pri, const std::shared_ptr<IOTracer>& io_tracer,
      const std::atomic<int>* manual_compaction_paused = nullptr,
+      const std::atomic<bool>* manual_compaction_canceled = nullptr,
      const std::string& db_id = "", const std::string& db_session_id = "",
      std::string full_history_ts_low = "",
      BlobFileCompletionCallback* blob_callback = nullptr);
@@ -125,6 +127,7 @@ class CompactionJob {
  CompactionState* compact_;
  InternalStats::CompactionStats compaction_stats_;
  const ImmutableDBOptions& db_options_;
+  const MutableDBOptions mutable_db_options_copy_;
  LogBuffer* log_buffer_;
  FSDirectory* output_directory_;
  Statistics* stats_;
@@ -143,6 +146,9 @@ class CompactionJob {
  // consecutive groups such that each group has a similar size.
  void GenSubcompactionBoundaries();

+  void ProcessKeyValueCompactionWithCompactionService(
+      SubcompactionState* sub_compact);
+
  // update the thread status for starting a compaction.
  void ReportStartedCompaction(Compaction* compaction);
  void AllocateCompactionOutputFileNumbers();
@@ -180,6 +186,7 @@ class CompactionJob {
  VersionSet* versions_;
  const std::atomic<bool>* shutting_down_;
  const std::atomic<int>* manual_compaction_paused_;
+  const std::atomic<bool>* manual_compaction_canceled_;
  const SequenceNumber preserve_deletes_seqnum_;
  FSDirectory* db_directory_;
  FSDirectory* blob_output_directory_;
@@ -235,9 +242,23 @@ struct CompactionServiceInput {
  int output_level;

  // information for subcompaction
-  Slice* begin = nullptr;
-  Slice* end = nullptr;
+  bool has_begin = false;
+  std::string begin;
+  bool has_end = false;
+  std::string end;
  uint64_t approx_size = 0;
+
+  // serialization interface to read and write the object
+  static Status Read(const std::string& data_str, CompactionServiceInput* obj);
+  Status Write(std::string* output);
+
+  // Initialize a dummy ColumnFamilyDescriptor
+  CompactionServiceInput() : column_family("", ColumnFamilyOptions()) {}
+
+#ifndef NDEBUG
+  bool TEST_Equals(CompactionServiceInput* other);
+  bool TEST_Equals(CompactionServiceInput* other, std::string* mismatch);
+#endif  // NDEBUG
 };

 // CompactionServiceOutputFile is the metadata for the output SST file
@@ -273,6 +294,7 @@ struct CompactionServiceOutputFile {
 // instance, with these information, the primary db instance with write
 // permission is able to install the result to the DB.
 struct CompactionServiceResult {
+  Status status;
  std::vector<CompactionServiceOutputFile> output_files;
  int output_level;

@@ -285,6 +307,15 @@ struct CompactionServiceResult {
  uint64_t bytes_read;
  uint64_t bytes_written;
  CompactionJobStats stats;
+
+  // serialization interface to read and write the object
+  static Status Read(const std::string& data_str, CompactionServiceResult* obj);
+  Status Write(std::string* output);
+
+#ifndef NDEBUG
+  bool TEST_Equals(CompactionServiceResult* other);
+  bool TEST_Equals(CompactionServiceResult* other, std::string* mismatch);
+#endif  // NDEBUG
 };

 // CompactionServiceCompactionJob is an read-only compaction job, it takes
@@ -294,6 +325,7 @@ class CompactionServiceCompactionJob : private CompactionJob {
 public:
  CompactionServiceCompactionJob(
      int job_id, Compaction* compaction, const ImmutableDBOptions& db_options,
+      const MutableDBOptions& mutable_db_options,
      const FileOptions& file_options, VersionSet* versions,
      const std::atomic<bool>* shutting_down, LogBuffer* log_buffer,
      FSDirectory* output_directory, Statistics* stats,
@@ -82,10 +82,11 @@ class CompactionJobTestBase : public testing::Test {
        mutable_db_options_(),
        table_cache_(NewLRUCache(50000, 16)),
        write_buffer_manager_(db_options_.db_write_buffer_size),
-        versions_(new VersionSet(
-            dbname_, &db_options_, env_options_, table_cache_.get(),
-            &write_buffer_manager_, &write_controller_,
-            /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr)),
+        versions_(new VersionSet(dbname_, &db_options_, env_options_,
+                                 table_cache_.get(), &write_buffer_manager_,
+                                 &write_controller_,
+                                 /*block_cache_tracer=*/nullptr,
+                                 /*io_tracer=*/nullptr, /*db_session_id*/ "")),
        shutting_down_(false),
        preserve_deletes_seqnum_(0),
        mock_table_factory_(new mock::MockTableFactory()),
@@ -269,7 +270,8 @@ class CompactionJobTestBase : public testing::Test {
    versions_.reset(
        new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(),
                       &write_buffer_manager_, &write_controller_,
-                       /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr));
+                       /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr,
+                       /*db_session_id*/ ""));
    compaction_job_stats_.Reset();
    ASSERT_OK(SetIdentityFile(env_, dbname_));

@@ -344,13 +346,14 @@ class CompactionJobTestBase : public testing::Test {
    ASSERT_TRUE(full_history_ts_low_.empty() ||
                ucmp_->timestamp_size() == full_history_ts_low_.size());
    CompactionJob compaction_job(
-        0, &compaction, db_options_, env_options_, versions_.get(),
-        &shutting_down_, preserve_deletes_seqnum_, &log_buffer, nullptr,
-        nullptr, nullptr, nullptr, &mutex_, &error_handler_, snapshots,
+        0, &compaction, db_options_, mutable_db_options_, env_options_,
+        versions_.get(), &shutting_down_, preserve_deletes_seqnum_, &log_buffer,
+        nullptr, nullptr, nullptr, nullptr, &mutex_, &error_handler_, snapshots,
        earliest_write_conflict_snapshot, snapshot_checker, table_cache_,
        &event_logger, false, false, dbname_, &compaction_job_stats_,
        Env::Priority::USER, nullptr /* IOTracer */,
-        /*manual_compaction_paused=*/nullptr, /*db_id=*/"",
+        /*manual_compaction_paused=*/nullptr,
+        /*manual_compaction_canceled=*/nullptr, /*db_id=*/"",
        /*db_session_id=*/"", full_history_ts_low_);
    VerifyInitializationOfCompactionJobStats(compaction_job_stats_);

@@ -1096,6 +1099,185 @@ TEST_F(CompactionJobTest, OldestBlobFileNumber) {
                /* expected_oldest_blob_file_number */ 19);
 }

+TEST_F(CompactionJobTest, InputSerialization) {
+  // Setup a random CompactionServiceInput
+  CompactionServiceInput input;
+  const int kStrMaxLen = 1000;
+  Random rnd(static_cast<uint32_t>(time(nullptr)));
+  Random64 rnd64(time(nullptr));
+  input.column_family.name = rnd.RandomString(rnd.Uniform(kStrMaxLen));
+  input.column_family.options.comparator = ReverseBytewiseComparator();
+  input.column_family.options.max_bytes_for_level_base =
+      rnd64.Uniform(UINT64_MAX);
+  input.column_family.options.disable_auto_compactions = rnd.OneIn(2);
+  input.column_family.options.compression = kZSTD;
+  input.column_family.options.compression_opts.level = 4;
+  input.db_options.max_background_flushes = 10;
+  input.db_options.paranoid_checks = rnd.OneIn(2);
+  input.db_options.statistics = CreateDBStatistics();
+  input.db_options.env = env_;
+  while (!rnd.OneIn(10)) {
+    input.snapshots.emplace_back(rnd64.Uniform(UINT64_MAX));
+  }
+  while (!rnd.OneIn(10)) {
+    input.input_files.emplace_back(rnd.RandomString(rnd.Uniform(kStrMaxLen)));
+  }
+  input.output_level = 4;
+  input.has_begin = rnd.OneIn(2);
+  if (input.has_begin) {
+    input.begin = rnd.RandomBinaryString(rnd.Uniform(kStrMaxLen));
+  }
+  input.has_end = rnd.OneIn(2);
+  if (input.has_end) {
+    input.end = rnd.RandomBinaryString(rnd.Uniform(kStrMaxLen));
+  }
+  input.approx_size = rnd64.Uniform(UINT64_MAX);
+
+  std::string output;
+  ASSERT_OK(input.Write(&output));
+
+  // Test deserialization
+  CompactionServiceInput deserialized1;
+  ASSERT_OK(CompactionServiceInput::Read(output, &deserialized1));
+  ASSERT_TRUE(deserialized1.TEST_Equals(&input));
+
+  // Test mismatch
+  deserialized1.db_options.max_background_flushes += 10;
+  std::string mismatch;
+  ASSERT_FALSE(deserialized1.TEST_Equals(&input, &mismatch));
+  ASSERT_EQ(mismatch, "db_options.max_background_flushes");
+
+  // Test unknown field
+  CompactionServiceInput deserialized2;
+  output.clear();
+  ASSERT_OK(input.Write(&output));
+  output.append("new_field=123;");
+
+  ASSERT_OK(CompactionServiceInput::Read(output, &deserialized2));
+  ASSERT_TRUE(deserialized2.TEST_Equals(&input));
+
+  // Test missing field
+  CompactionServiceInput deserialized3;
+  deserialized3.output_level = 0;
+  std::string to_remove = "output_level=4;";
+  size_t pos = output.find(to_remove);
+  ASSERT_TRUE(pos != std::string::npos);
+  output.erase(pos, to_remove.length());
+  ASSERT_OK(CompactionServiceInput::Read(output, &deserialized3));
+  mismatch.clear();
+  ASSERT_FALSE(deserialized3.TEST_Equals(&input, &mismatch));
+  ASSERT_EQ(mismatch, "output_level");
+
+  // manually set the value back, should match the original structure
+  deserialized3.output_level = 4;
+  ASSERT_TRUE(deserialized3.TEST_Equals(&input));
+
+  // Test invalid version
+  output.clear();
+  ASSERT_OK(input.Write(&output));
+
+  uint32_t data_version = DecodeFixed32(output.data());
+  const size_t kDataVersionSize = sizeof(data_version);
+  ASSERT_EQ(data_version,
+            1U);  // Update once the default data version is changed
+  char buf[kDataVersionSize];
+  EncodeFixed32(buf, data_version + 10);  // make sure it's not valid
+  output.replace(0, kDataVersionSize, buf, kDataVersionSize);
+  Status s = CompactionServiceInput::Read(output, &deserialized3);
+  ASSERT_TRUE(s.IsNotSupported());
+}
+
+TEST_F(CompactionJobTest, ResultSerialization) {
+  // Setup a random CompactionServiceResult
+  CompactionServiceResult result;
+  const int kStrMaxLen = 1000;
+  Random rnd(static_cast<uint32_t>(time(nullptr)));
+  Random64 rnd64(time(nullptr));
+  std::vector<Status> status_list = {
+      Status::OK(),
+      Status::InvalidArgument("invalid option"),
+      Status::Aborted("failed to run"),
+      Status::NotSupported("not supported option"),
+  };
+  result.status =
+      status_list.at(rnd.Uniform(static_cast<int>(status_list.size())));
+  while (!rnd.OneIn(10)) {
+    result.output_files.emplace_back(
+        rnd.RandomString(rnd.Uniform(kStrMaxLen)), rnd64.Uniform(UINT64_MAX),
+        rnd64.Uniform(UINT64_MAX),
+        rnd.RandomBinaryString(rnd.Uniform(kStrMaxLen)),
+        rnd.RandomBinaryString(rnd.Uniform(kStrMaxLen)),
+        rnd64.Uniform(UINT64_MAX), rnd64.Uniform(UINT64_MAX),
+        rnd64.Uniform(UINT64_MAX), rnd.OneIn(2));
+  }
+  result.output_level = rnd.Uniform(10);
+  result.output_path = rnd.RandomString(rnd.Uniform(kStrMaxLen));
+  result.num_output_records = rnd64.Uniform(UINT64_MAX);
+  result.total_bytes = rnd64.Uniform(UINT64_MAX);
+  result.bytes_read = 123;
+  result.bytes_written = rnd64.Uniform(UINT64_MAX);
+  result.stats.elapsed_micros = rnd64.Uniform(UINT64_MAX);
+  result.stats.num_output_files = rnd.Uniform(1000);
+  result.stats.is_full_compaction = rnd.OneIn(2);
+  result.stats.num_single_del_mismatch = rnd64.Uniform(UINT64_MAX);
+  result.stats.num_input_files = 9;
+
+  std::string output;
+  ASSERT_OK(result.Write(&output));
+
+  // Test deserialization
+  CompactionServiceResult deserialized1;
+  ASSERT_OK(CompactionServiceResult::Read(output, &deserialized1));
+  ASSERT_TRUE(deserialized1.TEST_Equals(&result));
+
+  // Test mismatch
+  deserialized1.stats.num_input_files += 10;
+  std::string mismatch;
+  ASSERT_FALSE(deserialized1.TEST_Equals(&result, &mismatch));
+  ASSERT_EQ(mismatch, "stats.num_input_files");
+
+  // Test unknown field
+  CompactionServiceResult deserialized2;
+  output.clear();
+  ASSERT_OK(result.Write(&output));
+  output.append("new_field=123;");
+
+  ASSERT_OK(CompactionServiceResult::Read(output, &deserialized2));
+  ASSERT_TRUE(deserialized2.TEST_Equals(&result));
+
+  // Test missing field
+  CompactionServiceResult deserialized3;
+  deserialized3.bytes_read = 0;
+  std::string to_remove = "bytes_read=123;";
+  size_t pos = output.find(to_remove);
+  ASSERT_TRUE(pos != std::string::npos);
+  output.erase(pos, to_remove.length());
+  ASSERT_OK(CompactionServiceResult::Read(output, &deserialized3));
+  mismatch.clear();
+  ASSERT_FALSE(deserialized3.TEST_Equals(&result, &mismatch));
+  ASSERT_EQ(mismatch, "bytes_read");
+
+  deserialized3.bytes_read = 123;
+  ASSERT_TRUE(deserialized3.TEST_Equals(&result));
+
+  // Test invalid version
+  output.clear();
+  ASSERT_OK(result.Write(&output));
+
+  uint32_t data_version = DecodeFixed32(output.data());
+  const size_t kDataVersionSize = sizeof(data_version);
+  ASSERT_EQ(data_version,
+            1U);  // Update once the default data version is changed
+  char buf[kDataVersionSize];
+  EncodeFixed32(buf, data_version + 10);  // make sure it's not valid
+  output.replace(0, kDataVersionSize, buf, kDataVersionSize);
+  Status s = CompactionServiceResult::Read(output, &deserialized3);
+  ASSERT_TRUE(s.IsNotSupported());
+  for (const auto& item : status_list) {
+    item.PermitUncheckedError();
+  }
+}
+
 class CompactionJobTimestampTest : public CompactionJobTestBase {
 public:
  CompactionJobTimestampTest()
@@ -148,7 +148,7 @@ CompressionOptions GetCompressionOptions(const MutableCFOptions& cf_options,
  return cf_options.compression_opts;
 }

-CompactionPicker::CompactionPicker(const ImmutableCFOptions& ioptions,
+CompactionPicker::CompactionPicker(const ImmutableOptions& ioptions,
                                   const InternalKeyComparator* icmp)
    : ioptions_(ioptions), icmp_(icmp) {}

@@ -530,7 +530,7 @@ bool CompactionPicker::SetupOtherInputs(
      }
    }
    if (expand_inputs) {
-      ROCKS_LOG_INFO(ioptions_.info_log,
+      ROCKS_LOG_INFO(ioptions_.logger,
                     "[%s] Expanding@%d %" ROCKSDB_PRIszt "+%" ROCKSDB_PRIszt
                     "(%" PRIu64 "+%" PRIu64 " bytes) to %" ROCKSDB_PRIszt
                     "+%" ROCKSDB_PRIszt " (%" PRIu64 "+%" PRIu64 " bytes)\n",
@@ -670,17 +670,41 @@ Compaction* CompactionPicker::CompactRange(
  // two files overlap.
  if (input_level > 0) {
    const uint64_t limit = mutable_cf_options.max_compaction_bytes;
-    uint64_t total = 0;
+    uint64_t input_level_total = 0;
+    int hint_index = -1;
+    InternalKey* smallest = nullptr;
+    InternalKey* largest = nullptr;
    for (size_t i = 0; i + 1 < inputs.size(); ++i) {
+      if (!smallest) {
+        smallest = &inputs[i]->smallest;
+      }
+      largest = &inputs[i]->largest;
+
      uint64_t s = inputs[i]->compensated_file_size;
-      total += s;
-      if (total >= limit) {
+      uint64_t output_level_total = 0;
+      if (output_level < vstorage->num_non_empty_levels()) {
+        std::vector<FileMetaData*> files;
+        vstorage->GetOverlappingInputsRangeBinarySearch(
+            output_level, smallest, largest, &files, hint_index, &hint_index);
+        for (const auto& file : files) {
+          output_level_total += file->compensated_file_size;
+        }
+      }
+
+      input_level_total += s;
+
+      if (input_level_total + output_level_total >= limit) {
        covering_the_whole_range = false;
+        // still include the current file, so the compaction could be larger
+        // than max_compaction_bytes, which is also to make sure the compaction
+        // can make progress even `max_compaction_bytes` is small (e.g. smaller
+        // than an SST file).
        inputs.files.resize(i + 1);
        break;
      }
    }
  }
+
  assert(compact_range_options.target_path_id <
         static_cast<uint32_t>(ioptions_.cf_paths.size()));

@@ -46,7 +46,7 @@ struct CompactionInputFiles;
 // compaction style specific logic for them.
 class CompactionPicker {
 public:
-  CompactionPicker(const ImmutableCFOptions& ioptions,
+  CompactionPicker(const ImmutableOptions& ioptions,
                   const InternalKeyComparator* icmp);
  virtual ~CompactionPicker();

@@ -218,7 +218,7 @@ class CompactionPicker {
  }

 protected:
-  const ImmutableCFOptions& ioptions_;
+  const ImmutableOptions& ioptions_;

 // A helper function to SanitizeCompactionInputFiles() that
 // sanitizes "input_files" by adding necessary files.
@@ -244,7 +244,7 @@ class CompactionPicker {
 // compaction.
 class NullCompactionPicker : public CompactionPicker {
 public:
-  NullCompactionPicker(const ImmutableCFOptions& ioptions,
+  NullCompactionPicker(const ImmutableOptions& ioptions,
                       const InternalKeyComparator* icmp)
      : CompactionPicker(ioptions, icmp) {}
  virtual ~NullCompactionPicker() {}
@@ -244,7 +244,7 @@ Compaction* FIFOCompactionPicker::CompactRange(
  assert(input_level == 0);
  assert(output_level == 0);
  *compaction_end = nullptr;
-  LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, ioptions_.info_log);
+  LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, ioptions_.logger);
  Compaction* c = PickCompaction(cf_name, mutable_cf_options,
                                 mutable_db_options, vstorage, &log_buffer);
  log_buffer.FlushBufferToLog();
@@ -15,7 +15,7 @@
 namespace ROCKSDB_NAMESPACE {
 class FIFOCompactionPicker : public CompactionPicker {
 public:
-  FIFOCompactionPicker(const ImmutableCFOptions& ioptions,
+  FIFOCompactionPicker(const ImmutableOptions& ioptions,
                       const InternalKeyComparator* icmp)
      : CompactionPicker(ioptions, icmp) {}

@@ -49,7 +49,7 @@ class LevelCompactionBuilder {
                         CompactionPicker* compaction_picker,
                         LogBuffer* log_buffer,
                         const MutableCFOptions& mutable_cf_options,
-                         const ImmutableCFOptions& ioptions,
+                         const ImmutableOptions& ioptions,
                         const MutableDBOptions& mutable_db_options)
      : cf_name_(cf_name),
        vstorage_(vstorage),
@@ -121,7 +121,7 @@ class LevelCompactionBuilder {
  CompactionReason compaction_reason_ = CompactionReason::kUnknown;

  const MutableCFOptions& mutable_cf_options_;
-  const ImmutableCFOptions& ioptions_;
+  const ImmutableOptions& ioptions_;
  const MutableDBOptions& mutable_db_options_;
  // Pick a path ID to place a newly generated file, with its level
  static uint32_t GetPathId(const ImmutableCFOptions& ioptions,
@@ -17,7 +17,7 @@ namespace ROCKSDB_NAMESPACE {
 // for description of Leveled compaction.
 class LevelCompactionPicker : public CompactionPicker {
 public:
-  LevelCompactionPicker(const ImmutableCFOptions& ioptions,
+  LevelCompactionPicker(const ImmutableOptions& ioptions,
                        const InternalKeyComparator* icmp)
      : CompactionPicker(ioptions, icmp) {}
  virtual Compaction* PickCompaction(
@@ -30,7 +30,7 @@ class CompactionPickerTest : public testing::Test {
  const Comparator* ucmp_;
  InternalKeyComparator icmp_;
  Options options_;
-  ImmutableCFOptions ioptions_;
+  ImmutableOptions ioptions_;
  MutableCFOptions mutable_cf_options_;
  MutableDBOptions mutable_db_options_;
  LevelCompactionPicker level_compaction_picker;
@@ -32,7 +32,7 @@ namespace {
 class UniversalCompactionBuilder {
 public:
  UniversalCompactionBuilder(
-      const ImmutableCFOptions& ioptions, const InternalKeyComparator* icmp,
+      const ImmutableOptions& ioptions, const InternalKeyComparator* icmp,
      const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
      const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage,
      UniversalCompactionPicker* picker, LogBuffer* log_buffer)
@@ -108,7 +108,7 @@ class UniversalCompactionBuilder {
  // overlapping.
  bool IsInputFilesNonOverlapping(Compaction* c);

-  const ImmutableCFOptions& ioptions_;
+  const ImmutableOptions& ioptions_;
  const InternalKeyComparator* icmp_;
  double score_;
  std::vector<SortedRun> sorted_runs_;
@@ -486,7 +486,7 @@ Compaction* UniversalCompactionBuilder::PickCompaction() {
  }
 #endif
  // update statistics
-  RecordInHistogram(ioptions_.statistics, NUM_FILES_IN_SINGLE_COMPACTION,
+  RecordInHistogram(ioptions_.stats, NUM_FILES_IN_SINGLE_COMPACTION,
                    c->inputs(0)->size());

  picker_->RegisterCompaction(c);
@@ -15,7 +15,7 @@
 namespace ROCKSDB_NAMESPACE {
 class UniversalCompactionPicker : public CompactionPicker {
 public:
-  UniversalCompactionPicker(const ImmutableCFOptions& ioptions,
+  UniversalCompactionPicker(const ImmutableOptions& ioptions,
                            const InternalKeyComparator* icmp)
      : CompactionPicker(ioptions, icmp) {}
  virtual Compaction* PickCompaction(
@@ -0,0 +1,458 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#ifndef ROCKSDB_LITE
+
+#include "db/db_test_util.h"
+#include "port/stack_trace.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+class MyTestCompactionService : public CompactionService {
+ public:
+  MyTestCompactionService(const std::string& db_path,
+                          std::shared_ptr<FileSystem> fs, Options& options)
+      : db_path_(db_path), fs_(fs), options_(options) {}
+
+  static const char* kClassName() { return "MyTestCompactionService"; }
+
+  const char* Name() const override { return kClassName(); }
+
+  CompactionServiceJobStatus Start(const std::string& compaction_service_input,
+                                   int job_id) override {
+    InstrumentedMutexLock l(&mutex_);
+    jobs_.emplace(job_id, compaction_service_input);
+    CompactionServiceJobStatus s = CompactionServiceJobStatus::kSuccess;
+    TEST_SYNC_POINT_CALLBACK("MyTestCompactionService::Start::End", &s);
+    return s;
+  }
+
+  CompactionServiceJobStatus WaitForComplete(
+      int job_id, std::string* compaction_service_result) override {
+    std::string compaction_input;
+    {
+      InstrumentedMutexLock l(&mutex_);
+      auto i = jobs_.find(job_id);
+      if (i == jobs_.end()) {
+        return CompactionServiceJobStatus::kFailure;
+      }
+      compaction_input = std::move(i->second);
+      jobs_.erase(i);
+    }
+
+    CompactionServiceOptionsOverride options_override;
+    options_override.env = options_.env;
+    options_override.file_checksum_gen_factory =
+        options_.file_checksum_gen_factory;
+    options_override.comparator = options_.comparator;
+    options_override.merge_operator = options_.merge_operator;
+    options_override.compaction_filter = options_.compaction_filter;
+    options_override.compaction_filter_factory =
+        options_.compaction_filter_factory;
+    options_override.prefix_extractor = options_.prefix_extractor;
+    options_override.table_factory = options_.table_factory;
+    options_override.sst_partitioner_factory = options_.sst_partitioner_factory;
+
+    Status s = DB::OpenAndCompact(
+        db_path_, db_path_ + "/" + ROCKSDB_NAMESPACE::ToString(job_id),
+        compaction_input, compaction_service_result, options_override);
+    TEST_SYNC_POINT_CALLBACK("MyTestCompactionService::WaitForComplete::End",
+                             compaction_service_result);
+    compaction_num_.fetch_add(1);
+    if (s.ok()) {
+      return CompactionServiceJobStatus::kSuccess;
+    } else {
+      return CompactionServiceJobStatus::kFailure;
+    }
+  }
+
+  int GetCompactionNum() { return compaction_num_.load(); }
+
+ private:
+  InstrumentedMutex mutex_;
+  std::atomic_int compaction_num_{0};
+  std::map<int, std::string> jobs_;
+  const std::string db_path_;
+  std::shared_ptr<FileSystem> fs_;
+  Options options_;
+};
+
+class CompactionServiceTest : public DBTestBase {
+ public:
+  explicit CompactionServiceTest()
+      : DBTestBase("compaction_service_test", true) {}
+
+ protected:
+  void GenerateTestData() {
+    // Generate 20 files @ L2
+    for (int i = 0; i < 20; i++) {
+      for (int j = 0; j < 10; j++) {
+        int key_id = i * 10 + j;
+        ASSERT_OK(Put(Key(key_id), "value" + ToString(key_id)));
+      }
+      ASSERT_OK(Flush());
+    }
+    MoveFilesToLevel(2);
+
+    // Generate 10 files @ L1 overlap with all 20 files @ L2
+    for (int i = 0; i < 10; i++) {
+      for (int j = 0; j < 10; j++) {
+        int key_id = i * 20 + j * 2;
+        ASSERT_OK(Put(Key(key_id), "value_new" + ToString(key_id)));
+      }
+      ASSERT_OK(Flush());
+    }
+    MoveFilesToLevel(1);
+    ASSERT_EQ(FilesPerLevel(), "0,10,20");
+  }
+
+  void VerifyTestData() {
+    for (int i = 0; i < 200; i++) {
+      auto result = Get(Key(i));
+      if (i % 2) {
+        ASSERT_EQ(result, "value" + ToString(i));
+      } else {
+        ASSERT_EQ(result, "value_new" + ToString(i));
+      }
+    }
+  }
+};
+
+TEST_F(CompactionServiceTest, BasicCompactions) {
+  Options options = CurrentOptions();
+  options.env = env_;
+  options.compaction_service = std::make_shared<MyTestCompactionService>(
+      dbname_, env_->GetFileSystem(), options);
+
+  DestroyAndReopen(options);
+
+  for (int i = 0; i < 20; i++) {
+    for (int j = 0; j < 10; j++) {
+      int key_id = i * 10 + j;
+      ASSERT_OK(Put(Key(key_id), "value" + ToString(key_id)));
+    }
+    ASSERT_OK(Flush());
+  }
+
+  for (int i = 0; i < 10; i++) {
+    for (int j = 0; j < 10; j++) {
+      int key_id = i * 20 + j * 2;
+      ASSERT_OK(Put(Key(key_id), "value_new" + ToString(key_id)));
+    }
+    ASSERT_OK(Flush());
+  }
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+
+  // verify result
+  for (int i = 0; i < 200; i++) {
+    auto result = Get(Key(i));
+    if (i % 2) {
+      ASSERT_EQ(result, "value" + ToString(i));
+    } else {
+      ASSERT_EQ(result, "value_new" + ToString(i));
+    }
+  }
+  auto my_cs =
+      dynamic_cast<MyTestCompactionService*>(options.compaction_service.get());
+  ASSERT_GE(my_cs->GetCompactionNum(), 1);
+
+  // Test failed compaction
+  SyncPoint::GetInstance()->SetCallBack(
+      "DBImplSecondary::CompactWithoutInstallation::End", [&](void* status) {
+        // override job status
+        Status* s = static_cast<Status*>(status);
+        *s = Status::Aborted("MyTestCompactionService failed to compact!");
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  Status s;
+  for (int i = 0; i < 10; i++) {
+    for (int j = 0; j < 10; j++) {
+      int key_id = i * 20 + j * 2;
+      s = Put(Key(key_id), "value_new" + ToString(key_id));
+      if (s.IsAborted()) {
+        break;
+      }
+    }
+    if (s.IsAborted()) {
+      break;
+    }
+    s = Flush();
+    if (s.IsAborted()) {
+      break;
+    }
+    s = dbfull()->TEST_WaitForCompact();
+    if (s.IsAborted()) {
+      break;
+    }
+  }
+  ASSERT_TRUE(s.IsAborted());
+}
+
+TEST_F(CompactionServiceTest, ManualCompaction) {
+  Options options = CurrentOptions();
+  options.env = env_;
+  options.disable_auto_compactions = true;
+  options.compaction_service = std::make_shared<MyTestCompactionService>(
+      dbname_, env_->GetFileSystem(), options);
+  DestroyAndReopen(options);
+  GenerateTestData();
+
+  auto my_cs =
+      dynamic_cast<MyTestCompactionService*>(options.compaction_service.get());
+
+  std::string start_str = Key(15);
+  std::string end_str = Key(45);
+  Slice start(start_str);
+  Slice end(end_str);
+  uint64_t comp_num = my_cs->GetCompactionNum();
+  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &start, &end));
+  ASSERT_GE(my_cs->GetCompactionNum(), comp_num + 1);
+  VerifyTestData();
+
+  start_str = Key(120);
+  start = start_str;
+  comp_num = my_cs->GetCompactionNum();
+  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &start, nullptr));
+  ASSERT_GE(my_cs->GetCompactionNum(), comp_num + 1);
+  VerifyTestData();
+
+  end_str = Key(92);
+  end = end_str;
+  comp_num = my_cs->GetCompactionNum();
+  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, &end));
+  ASSERT_GE(my_cs->GetCompactionNum(), comp_num + 1);
+  VerifyTestData();
+
+  comp_num = my_cs->GetCompactionNum();
+  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+  ASSERT_GE(my_cs->GetCompactionNum(), comp_num + 1);
+  VerifyTestData();
+}
+
+TEST_F(CompactionServiceTest, FailedToStart) {
+  Options options = CurrentOptions();
+  options.env = env_;
+  options.disable_auto_compactions = true;
+  options.compaction_service = std::make_shared<MyTestCompactionService>(
+      dbname_, env_->GetFileSystem(), options);
+  DestroyAndReopen(options);
+  GenerateTestData();
+
+  SyncPoint::GetInstance()->SetCallBack(
+      "MyTestCompactionService::Start::End", [&](void* status) {
+        // override job status
+        auto s = static_cast<CompactionServiceJobStatus*>(status);
+        *s = CompactionServiceJobStatus::kFailure;
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  std::string start_str = Key(15);
+  std::string end_str = Key(45);
+  Slice start(start_str);
+  Slice end(end_str);
+  Status s = db_->CompactRange(CompactRangeOptions(), &start, &end);
+  ASSERT_TRUE(s.IsIncomplete());
+}
+
+TEST_F(CompactionServiceTest, InvalidResult) {
+  Options options = CurrentOptions();
+  options.env = env_;
+  options.disable_auto_compactions = true;
+  options.compaction_service = std::make_shared<MyTestCompactionService>(
+      dbname_, env_->GetFileSystem(), options);
+  DestroyAndReopen(options);
+  GenerateTestData();
+
+  SyncPoint::GetInstance()->SetCallBack(
+      "MyTestCompactionService::WaitForComplete::End", [&](void* result) {
+        // override job status
+        auto result_str = static_cast<std::string*>(result);
+        *result_str = "Invalid Str";
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  std::string start_str = Key(15);
+  std::string end_str = Key(45);
+  Slice start(start_str);
+  Slice end(end_str);
+  Status s = db_->CompactRange(CompactRangeOptions(), &start, &end);
+  ASSERT_FALSE(s.ok());
+}
+
+// TODO: support sub-compaction
+TEST_F(CompactionServiceTest, DISABLED_SubCompaction) {
+  Options options = CurrentOptions();
+  options.env = env_;
+  options.max_subcompactions = 10;
+  options.target_file_size_base = 1 << 10;  // 1KB
+  options.disable_auto_compactions = true;
+  options.compaction_service = std::make_shared<MyTestCompactionService>(
+      dbname_, env_->GetFileSystem(), options);
+
+  DestroyAndReopen(options);
+  GenerateTestData();
+
+  auto cro = CompactRangeOptions();
+  cro.max_subcompactions = 10;
+  db_->CompactRange(cro, nullptr, nullptr);
+}
+
+class PartialDeleteCompactionFilter : public CompactionFilter {
+ public:
+  CompactionFilter::Decision FilterV2(
+      int /*level*/, const Slice& key, ValueType /*value_type*/,
+      const Slice& /*existing_value*/, std::string* /*new_value*/,
+      std::string* /*skip_until*/) const override {
+    int i = std::stoi(key.ToString().substr(3));
+    if (i > 5 && i <= 105) {
+      return CompactionFilter::Decision::kRemove;
+    }
+    return CompactionFilter::Decision::kKeep;
+  }
+
+  const char* Name() const override { return "PartialDeleteCompactionFilter"; }
+};
+
+TEST_F(CompactionServiceTest, CompactionFilter) {
+  Options options = CurrentOptions();
+  options.env = env_;
+  auto delete_comp_filter = PartialDeleteCompactionFilter();
+  options.compaction_filter = &delete_comp_filter;
+  options.compaction_service = std::make_shared<MyTestCompactionService>(
+      dbname_, env_->GetFileSystem(), options);
+
+  DestroyAndReopen(options);
+
+  for (int i = 0; i < 20; i++) {
+    for (int j = 0; j < 10; j++) {
+      int key_id = i * 10 + j;
+      ASSERT_OK(Put(Key(key_id), "value" + ToString(key_id)));
+    }
+    ASSERT_OK(Flush());
+  }
+
+  for (int i = 0; i < 10; i++) {
+    for (int j = 0; j < 10; j++) {
+      int key_id = i * 20 + j * 2;
+      ASSERT_OK(Put(Key(key_id), "value_new" + ToString(key_id)));
+    }
+    ASSERT_OK(Flush());
+  }
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+
+  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+
+  // verify result
+  for (int i = 0; i < 200; i++) {
+    auto result = Get(Key(i));
+    if (i > 5 && i <= 105) {
+      ASSERT_EQ(result, "NOT_FOUND");
+    } else if (i % 2) {
+      ASSERT_EQ(result, "value" + ToString(i));
+    } else {
+      ASSERT_EQ(result, "value_new" + ToString(i));
+    }
+  }
+  auto my_cs =
+      dynamic_cast<MyTestCompactionService*>(options.compaction_service.get());
+  ASSERT_GE(my_cs->GetCompactionNum(), 1);
+}
+
+TEST_F(CompactionServiceTest, Snapshot) {
+  Options options = CurrentOptions();
+  options.env = env_;
+  options.compaction_service = std::make_shared<MyTestCompactionService>(
+      dbname_, env_->GetFileSystem(), options);
+
+  DestroyAndReopen(options);
+
+  ASSERT_OK(Put(Key(1), "value1"));
+  ASSERT_OK(Put(Key(2), "value1"));
+  const Snapshot* s1 = db_->GetSnapshot();
+  ASSERT_OK(Flush());
+
+  ASSERT_OK(Put(Key(1), "value2"));
+  ASSERT_OK(Put(Key(3), "value2"));
+  ASSERT_OK(Flush());
+
+  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+  auto my_cs =
+      dynamic_cast<MyTestCompactionService*>(options.compaction_service.get());
+  ASSERT_GE(my_cs->GetCompactionNum(), 1);
+  ASSERT_EQ("value1", Get(Key(1), s1));
+  ASSERT_EQ("value2", Get(Key(1)));
+  db_->ReleaseSnapshot(s1);
+}
+
+TEST_F(CompactionServiceTest, ConcurrentCompaction) {
+  Options options = CurrentOptions();
+  options.level0_file_num_compaction_trigger = 100;
+  options.env = env_;
+  options.compaction_service = std::make_shared<MyTestCompactionService>(
+      dbname_, env_->GetFileSystem(), options);
+  options.max_background_jobs = 20;
+
+  DestroyAndReopen(options);
+  GenerateTestData();
+
+  ColumnFamilyMetaData meta;
+  db_->GetColumnFamilyMetaData(&meta);
+
+  std::vector<std::thread> threads;
+  for (const auto& file : meta.levels[1].files) {
+    threads.push_back(std::thread([&]() {
+      std::string fname = file.db_path + "/" + file.name;
+      ASSERT_OK(db_->CompactFiles(CompactionOptions(), {fname}, 2));
+    }));
+  }
+
+  for (auto& thread : threads) {
+    thread.join();
+  }
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+
+  // verify result
+  for (int i = 0; i < 200; i++) {
+    auto result = Get(Key(i));
+    if (i % 2) {
+      ASSERT_EQ(result, "value" + ToString(i));
+    } else {
+      ASSERT_EQ(result, "value_new" + ToString(i));
+    }
+  }
+  auto my_cs =
+      dynamic_cast<MyTestCompactionService*>(options.compaction_service.get());
+  ASSERT_EQ(my_cs->GetCompactionNum(), 10);
+  ASSERT_EQ(FilesPerLevel(), "0,0,10");
+}
+
+}  // namespace ROCKSDB_NAMESPACE
+
+#ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
+extern "C" {
+void RegisterCustomObjects(int argc, char** argv);
+}
+#else
+void RegisterCustomObjects(int /*argc*/, char** /*argv*/) {}
+#endif  // !ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
+
+int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
+  ::testing::InitGoogleTest(&argc, argv);
+  RegisterCustomObjects(argc, argv);
+  return RUN_ALL_TESTS();
+}
+
+#else
+#include <stdio.h>
+
+int main(int /*argc*/, char** /*argv*/) {
+  fprintf(stderr,
+          "SKIPPED as CompactionService is not supported in ROCKSDB_LITE\n");
+  return 0;
+}
+
+#endif  // ROCKSDB_LITE
@@ -44,7 +44,7 @@ Status VerifySstFileChecksum(const Options& options,
  std::unique_ptr<FSRandomAccessFile> file;
  uint64_t file_size;
  InternalKeyComparator internal_comparator(options.comparator);
-  ImmutableCFOptions ioptions(options);
+  ImmutableOptions ioptions(options);

  Status s = ioptions.fs->NewRandomAccessFile(file_path,
                                              FileOptions(env_options),
@@ -55,14 +55,9 @@ class CorruptionTest : public testing::Test {
    // bug in recovery code. Keep it 4 for now to make the test passes.
    tiny_cache_ = NewLRUCache(100, 4);
    Env* base_env = Env::Default();
-#ifndef ROCKSDB_LITE
-    const char* test_env_uri = getenv("TEST_ENV_URI");
-    if (test_env_uri) {
-      Status s = Env::LoadEnv(test_env_uri, &base_env, &env_guard_);
-      EXPECT_OK(s);
-      EXPECT_NE(Env::Default(), base_env);
-    }
-#endif  //! ROCKSDB_LITE
+    EXPECT_OK(
+        test::CreateEnvFromSystem(ConfigOptions(), &base_env, &env_guard_));
+    EXPECT_NE(base_env, nullptr);
    env_ = new test::ErrorEnv(base_env);
    options_.wal_recovery_mode = WALRecoveryMode::kTolerateCorruptedTailRecords;
    options_.env = env_;
@@ -551,7 +546,7 @@ TEST_F(CorruptionTest, RangeDeletionCorrupted) {
  BlockHandle range_del_handle;
  ASSERT_OK(FindMetaBlock(
      file_reader.get(), file_size, kBlockBasedTableMagicNumber,
-      ImmutableCFOptions(options_), kRangeDelBlock, &range_del_handle));
+      ImmutableOptions(options_), kRangeDelBlock, &range_del_handle));

  ASSERT_OK(TryReopen());
  ASSERT_OK(test::CorruptFile(env_, filename,
@@ -7,10 +7,14 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 #include <cstdlib>
+#include <memory>

+#include "cache/cache_entry_roles.h"
 #include "cache/lru_cache.h"
+#include "db/column_family.h"
 #include "db/db_test_util.h"
 #include "port/stack_trace.h"
+#include "rocksdb/table.h"
 #include "util/compression.h"
 #include "util/random.h"

@@ -147,6 +151,19 @@ class DBBlockCacheTest : public DBTestBase {
    compressed_insert_count_ = new_insert_count;
    compressed_failure_count_ = new_failure_count;
  }
+
+#ifndef ROCKSDB_LITE
+  const std::array<size_t, kNumCacheEntryRoles> GetCacheEntryRoleCountsBg() {
+    // Verify in cache entry role stats
+    ColumnFamilyHandleImpl* cfh =
+        static_cast<ColumnFamilyHandleImpl*>(dbfull()->DefaultColumnFamily());
+    InternalStats* internal_stats_ptr = cfh->cfd()->internal_stats();
+    InternalStats::CacheEntryRoleStats stats;
+    internal_stats_ptr->TEST_GetCacheEntryRoleStats(&stats,
+                                                    /*foreground=*/false);
+    return stats.entry_counts;
+  }
+#endif  // ROCKSDB_LITE
 };

 TEST_F(DBBlockCacheTest, IteratorBlockCacheUsage) {
@@ -156,7 +173,13 @@ TEST_F(DBBlockCacheTest, IteratorBlockCacheUsage) {
  auto options = GetOptions(table_options);
  InitTable(options);

-  std::shared_ptr<Cache> cache = NewLRUCache(0, 0, false);
+  LRUCacheOptions co;
+  co.capacity = 0;
+  co.num_shard_bits = 0;
+  co.strict_capacity_limit = false;
+  // Needed not to count entry stats collector
+  co.metadata_charge_policy = kDontChargeCacheMetadata;
+  std::shared_ptr<Cache> cache = NewLRUCache(co);
  table_options.block_cache = cache;
  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
  Reopen(options);
@@ -180,7 +203,13 @@ TEST_F(DBBlockCacheTest, TestWithoutCompressedBlockCache) {
  auto options = GetOptions(table_options);
  InitTable(options);

-  std::shared_ptr<Cache> cache = NewLRUCache(0, 0, false);
+  LRUCacheOptions co;
+  co.capacity = 0;
+  co.num_shard_bits = 0;
+  co.strict_capacity_limit = false;
+  // Needed not to count entry stats collector
+  co.metadata_charge_policy = kDontChargeCacheMetadata;
+  std::shared_ptr<Cache> cache = NewLRUCache(co);
  table_options.block_cache = cache;
  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
  Reopen(options);
@@ -228,34 +257,54 @@ TEST_F(DBBlockCacheTest, TestWithoutCompressedBlockCache) {

 #ifdef SNAPPY
 TEST_F(DBBlockCacheTest, TestWithCompressedBlockCache) {
-  ReadOptions read_options;
-  auto table_options = GetTableOptions();
-  auto options = GetOptions(table_options);
-  options.compression = CompressionType::kSnappyCompression;
-  InitTable(options);
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();

-  std::shared_ptr<Cache> cache = NewLRUCache(0, 0, false);
+  BlockBasedTableOptions table_options;
+  table_options.no_block_cache = true;
+  table_options.block_cache_compressed = nullptr;
+  table_options.block_size = 1;
+  table_options.filter_policy.reset(NewBloomFilterPolicy(20));
+  table_options.cache_index_and_filter_blocks = false;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  options.compression = CompressionType::kSnappyCompression;
+
+  DestroyAndReopen(options);
+
+  std::string value(kValueSize, 'a');
+  for (size_t i = 0; i < kNumBlocks; i++) {
+    ASSERT_OK(Put(ToString(i), value));
+    ASSERT_OK(Flush());
+  }
+
+  ReadOptions read_options;
  std::shared_ptr<Cache> compressed_cache = NewLRUCache(1 << 25, 0, false);
+  LRUCacheOptions co;
+  co.capacity = 0;
+  co.num_shard_bits = 0;
+  co.strict_capacity_limit = false;
+  // Needed not to count entry stats collector
+  co.metadata_charge_policy = kDontChargeCacheMetadata;
+  std::shared_ptr<Cache> cache = NewLRUCache(co);
  table_options.block_cache = cache;
+  table_options.no_block_cache = false;
  table_options.block_cache_compressed = compressed_cache;
+  table_options.max_auto_readahead_size = 0;
+  table_options.cache_index_and_filter_blocks = false;
  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
  Reopen(options);
  RecordCacheCounters(options);

-  std::vector<std::unique_ptr<Iterator>> iterators(kNumBlocks - 1);
-  Iterator* iter = nullptr;
-
  // Load blocks into cache.
-  for (size_t i = 0; i + 1 < kNumBlocks; i++) {
-    iter = db_->NewIterator(read_options);
-    iter->Seek(ToString(i));
-    ASSERT_OK(iter->status());
+  for (size_t i = 0; i < kNumBlocks - 1; i++) {
+    ASSERT_EQ(value, Get(ToString(i)));
    CheckCacheCounters(options, 1, 0, 1, 0);
    CheckCompressedCacheCounters(options, 1, 0, 1, 0);
-    iterators[i].reset(iter);
  }
+
  size_t usage = cache->GetUsage();
-  ASSERT_LT(0, usage);
+  ASSERT_EQ(0, usage);
  ASSERT_EQ(usage, cache->GetPinnedUsage());
  size_t compressed_usage = compressed_cache->GetUsage();
  ASSERT_LT(0, compressed_usage);
@@ -267,24 +316,21 @@ TEST_F(DBBlockCacheTest, TestWithCompressedBlockCache) {
  cache->SetCapacity(usage);
  cache->SetStrictCapacityLimit(true);
  ASSERT_EQ(usage, cache->GetPinnedUsage());
-  iter = db_->NewIterator(read_options);
-  iter->Seek(ToString(kNumBlocks - 1));
-  ASSERT_TRUE(iter->status().IsIncomplete());
-  CheckCacheCounters(options, 1, 0, 0, 1);
+
+  // Load last key block.
+  ASSERT_EQ("Result incomplete: Insert failed due to LRU cache being full.",
+            Get(ToString(kNumBlocks - 1)));
+  // Failure won't record the miss counter.
+  CheckCacheCounters(options, 0, 0, 0, 1);
  CheckCompressedCacheCounters(options, 1, 0, 1, 0);
-  delete iter;
-  iter = nullptr;

  // Clear strict capacity limit flag. This time we shall hit compressed block
-  // cache.
+  // cache and load into block cache.
  cache->SetStrictCapacityLimit(false);
-  iter = db_->NewIterator(read_options);
-  iter->Seek(ToString(kNumBlocks - 1));
-  ASSERT_OK(iter->status());
+  // Load last key block.
+  ASSERT_EQ(value, Get(ToString(kNumBlocks - 1)));
  CheckCacheCounters(options, 1, 0, 1, 0);
  CheckCompressedCacheCounters(options, 0, 1, 0, 0);
-  delete iter;
-  iter = nullptr;
 }
 #endif  // SNAPPY

@@ -432,6 +478,33 @@ TEST_F(DBBlockCacheTest, IndexAndFilterBlocksStats) {
  //           filter_bytes_insert);
 }

+#if (defined OS_LINUX || defined OS_WIN)
+TEST_F(DBBlockCacheTest, WarmCacheWithDataBlocksDuringFlush) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
+
+  BlockBasedTableOptions table_options;
+  table_options.block_cache = NewLRUCache(1 << 25, 0, false);
+  table_options.cache_index_and_filter_blocks = false;
+  table_options.prepopulate_block_cache =
+      BlockBasedTableOptions::PrepopulateBlockCache::kFlushOnly;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  DestroyAndReopen(options);
+
+  std::string value(kValueSize, 'a');
+  for (size_t i = 1; i <= kNumBlocks; i++) {
+    ASSERT_OK(Put(ToString(i), value));
+    ASSERT_OK(Flush());
+    ASSERT_EQ(i, options.statistics->getTickerCount(BLOCK_CACHE_DATA_ADD));
+
+    ASSERT_EQ(value, Get(ToString(i)));
+    ASSERT_EQ(0, options.statistics->getTickerCount(BLOCK_CACHE_DATA_MISS));
+    ASSERT_EQ(i, options.statistics->getTickerCount(BLOCK_CACHE_DATA_HIT));
+  }
+}
+#endif
+
 namespace {

 // A mock cache wraps LRUCache, and record how many entries have been
@@ -447,15 +520,17 @@ class MockCache : public LRUCache {
  }

  using ShardedCache::Insert;
-  Status Insert(const Slice& key, void* value, size_t charge,
-                void (*deleter)(const Slice& key, void* value), Handle** handle,
-                Priority priority) override {
+
+  Status Insert(const Slice& key, void* value,
+                const Cache::CacheItemHelper* helper_cb, size_t charge,
+                Handle** handle, Priority priority) override {
+    DeleterFn delete_cb = helper_cb->del_cb;
    if (priority == Priority::LOW) {
      low_pri_insert_count++;
    } else {
      high_pri_insert_count++;
    }
-    return LRUCache::Insert(key, value, charge, deleter, handle, priority);
+    return LRUCache::Insert(key, value, charge, delete_cb, handle, priority);
  }
 };

@@ -889,6 +964,252 @@ TEST_F(DBBlockCacheTest, CacheCompressionDict) {
  }
 }

+static void ClearCache(Cache* cache) {
+  auto roles = CopyCacheDeleterRoleMap();
+  std::deque<std::string> keys;
+  Cache::ApplyToAllEntriesOptions opts;
+  auto callback = [&](const Slice& key, void* /*value*/, size_t /*charge*/,
+                      Cache::DeleterFn deleter) {
+    if (roles.find(deleter) == roles.end()) {
+      // Keep the stats collector
+      return;
+    }
+    keys.push_back(key.ToString());
+  };
+  cache->ApplyToAllEntries(callback, opts);
+  for (auto& k : keys) {
+    cache->Erase(k);
+  }
+}
+
+TEST_F(DBBlockCacheTest, CacheEntryRoleStats) {
+  const size_t capacity = size_t{1} << 25;
+  int iterations_tested = 0;
+  for (bool partition : {false, true}) {
+    for (std::shared_ptr<Cache> cache :
+         {NewLRUCache(capacity), NewClockCache(capacity)}) {
+      if (!cache) {
+        // Skip clock cache when not supported
+        continue;
+      }
+      ++iterations_tested;
+
+      Options options = CurrentOptions();
+      SetTimeElapseOnlySleepOnReopen(&options);
+      options.create_if_missing = true;
+      options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
+      options.max_open_files = 13;
+      options.table_cache_numshardbits = 0;
+      // If this wakes up, it could interfere with test
+      options.stats_dump_period_sec = 0;
+
+      BlockBasedTableOptions table_options;
+      table_options.block_cache = cache;
+      table_options.cache_index_and_filter_blocks = true;
+      table_options.filter_policy.reset(NewBloomFilterPolicy(50));
+      if (partition) {
+        table_options.index_type = BlockBasedTableOptions::kTwoLevelIndexSearch;
+        table_options.partition_filters = true;
+      }
+      table_options.metadata_cache_options.top_level_index_pinning =
+          PinningTier::kNone;
+      table_options.metadata_cache_options.partition_pinning =
+          PinningTier::kNone;
+      table_options.metadata_cache_options.unpartitioned_pinning =
+          PinningTier::kNone;
+      options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+      DestroyAndReopen(options);
+
+      // Create a new table.
+      ASSERT_OK(Put("foo", "value"));
+      ASSERT_OK(Put("bar", "value"));
+      ASSERT_OK(Flush());
+
+      ASSERT_OK(Put("zfoo", "value"));
+      ASSERT_OK(Put("zbar", "value"));
+      ASSERT_OK(Flush());
+
+      ASSERT_EQ(2, NumTableFilesAtLevel(0));
+
+      // Fresh cache
+      ClearCache(cache.get());
+
+      std::array<size_t, kNumCacheEntryRoles> expected{};
+      // For CacheEntryStatsCollector
+      expected[static_cast<size_t>(CacheEntryRole::kMisc)] = 1;
+      EXPECT_EQ(expected, GetCacheEntryRoleCountsBg());
+
+      std::array<size_t, kNumCacheEntryRoles> prev_expected = expected;
+
+      // First access only filters
+      ASSERT_EQ("NOT_FOUND", Get("different from any key added"));
+      expected[static_cast<size_t>(CacheEntryRole::kFilterBlock)] += 2;
+      if (partition) {
+        expected[static_cast<size_t>(CacheEntryRole::kFilterMetaBlock)] += 2;
+      }
+      // Within some time window, we will get cached entry stats
+      EXPECT_EQ(prev_expected, GetCacheEntryRoleCountsBg());
+      // Not enough to force a miss
+      env_->MockSleepForSeconds(45);
+      EXPECT_EQ(prev_expected, GetCacheEntryRoleCountsBg());
+      // Enough to force a miss
+      env_->MockSleepForSeconds(601);
+      EXPECT_EQ(expected, GetCacheEntryRoleCountsBg());
+
+      // Now access index and data block
+      ASSERT_EQ("value", Get("foo"));
+      expected[static_cast<size_t>(CacheEntryRole::kIndexBlock)]++;
+      if (partition) {
+        // top-level
+        expected[static_cast<size_t>(CacheEntryRole::kIndexBlock)]++;
+      }
+      expected[static_cast<size_t>(CacheEntryRole::kDataBlock)]++;
+      // Enough to force a miss
+      env_->MockSleepForSeconds(601);
+      // But inject a simulated long scan so that we need a longer
+      // interval to force a miss next time.
+      SyncPoint::GetInstance()->SetCallBack(
+          "CacheEntryStatsCollector::GetStats:AfterApplyToAllEntries",
+          [this](void*) {
+            // To spend no more than 0.2% of time scanning, we would need
+            // interval of at least 10000s
+            env_->MockSleepForSeconds(20);
+          });
+      SyncPoint::GetInstance()->EnableProcessing();
+      EXPECT_EQ(expected, GetCacheEntryRoleCountsBg());
+      prev_expected = expected;
+      SyncPoint::GetInstance()->DisableProcessing();
+      SyncPoint::GetInstance()->ClearAllCallBacks();
+
+      // The same for other file
+      ASSERT_EQ("value", Get("zfoo"));
+      expected[static_cast<size_t>(CacheEntryRole::kIndexBlock)]++;
+      if (partition) {
+        // top-level
+        expected[static_cast<size_t>(CacheEntryRole::kIndexBlock)]++;
+      }
+      expected[static_cast<size_t>(CacheEntryRole::kDataBlock)]++;
+      // Because of the simulated long scan, this is not enough to force
+      // a miss
+      env_->MockSleepForSeconds(601);
+      EXPECT_EQ(prev_expected, GetCacheEntryRoleCountsBg());
+      // But this is enough
+      env_->MockSleepForSeconds(10000);
+      EXPECT_EQ(expected, GetCacheEntryRoleCountsBg());
+      prev_expected = expected;
+
+      // Also check the GetProperty interface
+      std::map<std::string, std::string> values;
+      ASSERT_TRUE(
+          db_->GetMapProperty(DB::Properties::kBlockCacheEntryStats, &values));
+
+      EXPECT_EQ(
+          ToString(expected[static_cast<size_t>(CacheEntryRole::kIndexBlock)]),
+          values["count.index-block"]);
+      EXPECT_EQ(
+          ToString(expected[static_cast<size_t>(CacheEntryRole::kDataBlock)]),
+          values["count.data-block"]);
+      EXPECT_EQ(
+          ToString(expected[static_cast<size_t>(CacheEntryRole::kFilterBlock)]),
+          values["count.filter-block"]);
+      EXPECT_EQ(
+          ToString(
+              prev_expected[static_cast<size_t>(CacheEntryRole::kWriteBuffer)]),
+          values["count.write-buffer"]);
+      EXPECT_EQ(ToString(expected[static_cast<size_t>(CacheEntryRole::kMisc)]),
+                values["count.misc"]);
+
+      // Add one for kWriteBuffer
+      {
+        WriteBufferManager wbm(size_t{1} << 20, cache);
+        wbm.ReserveMem(1024);
+        expected[static_cast<size_t>(CacheEntryRole::kWriteBuffer)]++;
+        // Now we check that the GetProperty interface is more agressive about
+        // re-scanning stats, but not totally aggressive.
+        // Within some time window, we will get cached entry stats
+        env_->MockSleepForSeconds(1);
+        EXPECT_EQ(ToString(prev_expected[static_cast<size_t>(
+                      CacheEntryRole::kWriteBuffer)]),
+                  values["count.write-buffer"]);
+        // Not enough for a "background" miss but enough for a "foreground" miss
+        env_->MockSleepForSeconds(45);
+
+        ASSERT_TRUE(db_->GetMapProperty(DB::Properties::kBlockCacheEntryStats,
+                                        &values));
+        EXPECT_EQ(
+            ToString(
+                expected[static_cast<size_t>(CacheEntryRole::kWriteBuffer)]),
+            values["count.write-buffer"]);
+      }
+      prev_expected = expected;
+
+      // With collector pinned in cache, we should be able to hit
+      // even if the cache is full
+      ClearCache(cache.get());
+      Cache::Handle* h = nullptr;
+      ASSERT_OK(cache->Insert("Fill-it-up", nullptr, capacity + 1,
+                              GetNoopDeleterForRole<CacheEntryRole::kMisc>(),
+                              &h, Cache::Priority::HIGH));
+      ASSERT_GT(cache->GetUsage(), cache->GetCapacity());
+      expected = {};
+      // For CacheEntryStatsCollector
+      expected[static_cast<size_t>(CacheEntryRole::kMisc)] = 1;
+      // For Fill-it-up
+      expected[static_cast<size_t>(CacheEntryRole::kMisc)]++;
+      // Still able to hit on saved stats
+      EXPECT_EQ(prev_expected, GetCacheEntryRoleCountsBg());
+      // Enough to force a miss
+      env_->MockSleepForSeconds(1000);
+      EXPECT_EQ(expected, GetCacheEntryRoleCountsBg());
+
+      cache->Release(h);
+
+      // Now we test that the DB mutex is not held during scans, for the ways
+      // we know how to (possibly) trigger them. Without a better good way to
+      // check this, we simply inject an acquire & release of the DB mutex
+      // deep in the stat collection code. If we were already holding the
+      // mutex, that is UB that would at least be found by TSAN.
+      int scan_count = 0;
+      SyncPoint::GetInstance()->SetCallBack(
+          "CacheEntryStatsCollector::GetStats:AfterApplyToAllEntries",
+          [this, &scan_count](void*) {
+            dbfull()->TEST_LockMutex();
+            dbfull()->TEST_UnlockMutex();
+            ++scan_count;
+          });
+      SyncPoint::GetInstance()->EnableProcessing();
+
+      // Different things that might trigger a scan, with mock sleeps to
+      // force a miss.
+      env_->MockSleepForSeconds(10000);
+      dbfull()->DumpStats();
+      ASSERT_EQ(scan_count, 1);
+
+      env_->MockSleepForSeconds(10000);
+      ASSERT_TRUE(
+          db_->GetMapProperty(DB::Properties::kBlockCacheEntryStats, &values));
+      ASSERT_EQ(scan_count, 2);
+
+      env_->MockSleepForSeconds(10000);
+      std::string value_str;
+      ASSERT_TRUE(
+          db_->GetProperty(DB::Properties::kBlockCacheEntryStats, &value_str));
+      ASSERT_EQ(scan_count, 3);
+
+      env_->MockSleepForSeconds(10000);
+      ASSERT_TRUE(db_->GetProperty(DB::Properties::kCFStats, &value_str));
+      // To match historical speed, querying this property no longer triggers
+      // a scan, even if results are old. But periodic dump stats should keep
+      // things reasonably updated.
+      ASSERT_EQ(scan_count, /*unchanged*/ 3);
+
+      SyncPoint::GetInstance()->DisableProcessing();
+      SyncPoint::GetInstance()->ClearAllCallBacks();
+    }
+    EXPECT_GE(iterations_tested, 1);
+  }
+}
+
 #endif  // ROCKSDB_LITE

 class DBBlockCachePinningTest
@@ -15,6 +15,7 @@
 #include "port/stack_trace.h"
 #include "rocksdb/perf_context.h"
 #include "table/block_based/filter_policy_internal.h"
+#include "util/string_util.h"

 namespace ROCKSDB_NAMESPACE {

@@ -506,6 +507,21 @@ TEST_P(DBBloomFilterTestWithParam, BloomFilter) {
      ASSERT_LE(reads, 3 * N / 100);
    }

+#ifndef ROCKSDB_LITE
+    // Sanity check some table properties
+    std::map<std::string, std::string> props;
+    ASSERT_TRUE(db_->GetMapProperty(
+        handles_[1], DB::Properties::kAggregatedTableProperties, &props));
+    uint64_t nkeys = N + N / 100;
+    uint64_t filter_size = ParseUint64(props["filter_size"]);
+    EXPECT_LE(filter_size,
+              (partition_filters_ ? 12 : 11) * nkeys / /*bits / byte*/ 8);
+    EXPECT_GE(filter_size, 10 * nkeys / /*bits / byte*/ 8);
+
+    uint64_t num_filter_entries = ParseUint64(props["num_filter_entries"]);
+    EXPECT_EQ(num_filter_entries, nkeys);
+#endif  // ROCKSDB_LITE
+
    env_->delay_sstable_sync_.store(false, std::memory_order_release);
    Close();
  } while (ChangeCompactOptions());
@@ -774,6 +790,14 @@ class LevelAndStyleCustomFilterPolicy : public FilterPolicy {
  const std::unique_ptr<const FilterPolicy> policy_otherwise_;
 };

+static std::map<TableFileCreationReason, std::string>
+    table_file_creation_reason_to_string{
+        {TableFileCreationReason::kCompaction, "kCompaction"},
+        {TableFileCreationReason::kFlush, "kFlush"},
+        {TableFileCreationReason::kMisc, "kMisc"},
+        {TableFileCreationReason::kRecovery, "kRecovery"},
+    };
+
 class TestingContextCustomFilterPolicy
    : public LevelAndStyleCustomFilterPolicy {
 public:
@@ -786,11 +810,17 @@ class TestingContextCustomFilterPolicy
      const FilterBuildingContext& context) const override {
    test_report_ += "cf=";
    test_report_ += context.column_family_name;
-    test_report_ += ",cs=";
+    test_report_ += ",s=";
    test_report_ +=
        OptionsHelper::compaction_style_to_string[context.compaction_style];
-    test_report_ += ",lv=";
-    test_report_ += std::to_string(context.level_at_creation);
+    test_report_ += ",n=";
+    test_report_ += ToString(context.num_levels);
+    test_report_ += ",l=";
+    test_report_ += ToString(context.level_at_creation);
+    test_report_ += ",b=";
+    test_report_ += ToString(int{context.is_bottommost});
+    test_report_ += ",r=";
+    test_report_ += table_file_creation_reason_to_string[context.reason];
    test_report_ += "\n";

    return LevelAndStyleCustomFilterPolicy::GetBuilderWithContext(context);
@@ -808,15 +838,16 @@ class TestingContextCustomFilterPolicy
 }  // namespace

 TEST_F(DBBloomFilterTest, ContextCustomFilterPolicy) {
+  auto policy = std::make_shared<TestingContextCustomFilterPolicy>(15, 8, 5);
+  Options options;
  for (bool fifo : {true, false}) {
-    Options options = CurrentOptions();
+    options = CurrentOptions();
    options.max_open_files = fifo ? -1 : options.max_open_files;
    options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
    options.compaction_style =
        fifo ? kCompactionStyleFIFO : kCompactionStyleLevel;

    BlockBasedTableOptions table_options;
-    auto policy = std::make_shared<TestingContextCustomFilterPolicy>(15, 8, 5);
    table_options.filter_policy = policy;
    table_options.format_version = 5;
    options.table_factory.reset(NewBlockBasedTableFactory(table_options));
@@ -832,16 +863,16 @@ TEST_F(DBBloomFilterTest, ContextCustomFilterPolicy) {
    ASSERT_OK(Put(1, Key(maxKey + 55555), Key(maxKey + 55555)));
    Flush(1);
    EXPECT_EQ(policy->DumpTestReport(),
-              fifo ? "cf=abe,cs=kCompactionStyleFIFO,lv=0\n"
-                   : "cf=bob,cs=kCompactionStyleLevel,lv=0\n");
+              fifo ? "cf=abe,s=kCompactionStyleFIFO,n=1,l=0,b=0,r=kFlush\n"
+                   : "cf=bob,s=kCompactionStyleLevel,n=7,l=0,b=0,r=kFlush\n");

    for (int i = maxKey / 2; i < maxKey; i++) {
      ASSERT_OK(Put(1, Key(i), Key(i)));
    }
    Flush(1);
    EXPECT_EQ(policy->DumpTestReport(),
-              fifo ? "cf=abe,cs=kCompactionStyleFIFO,lv=0\n"
-                   : "cf=bob,cs=kCompactionStyleLevel,lv=0\n");
+              fifo ? "cf=abe,s=kCompactionStyleFIFO,n=1,l=0,b=0,r=kFlush\n"
+                   : "cf=bob,s=kCompactionStyleLevel,n=7,l=0,b=0,r=kFlush\n");

    // Check that they can be found
    for (int i = 0; i < maxKey; i++) {
@@ -869,7 +900,7 @@ TEST_F(DBBloomFilterTest, ContextCustomFilterPolicy) {
      ASSERT_OK(db_->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
                                  nullptr));
      EXPECT_EQ(policy->DumpTestReport(),
-                "cf=bob,cs=kCompactionStyleLevel,lv=1\n");
+                "cf=bob,s=kCompactionStyleLevel,n=7,l=1,b=1,r=kCompaction\n");

      // Check that we now have one filter, about 9.2% FP rate (5 bits per key)
      for (int i = 0; i < maxKey; i++) {
@@ -881,6 +912,20 @@ TEST_F(DBBloomFilterTest, ContextCustomFilterPolicy) {
        EXPECT_GE(useful_count, maxKey * 0.90);
        EXPECT_LE(useful_count, maxKey * 0.91);
      }
+    } else {
+#ifndef ROCKSDB_LITE
+      // Also try external SST file
+      {
+        std::string file_path = dbname_ + "/external.sst";
+        SstFileWriter sst_file_writer(EnvOptions(), options, handles_[1]);
+        ASSERT_OK(sst_file_writer.Open(file_path));
+        ASSERT_OK(sst_file_writer.Put("key", "value"));
+        ASSERT_OK(sst_file_writer.Finish());
+      }
+      // Note: kCompactionStyleLevel is default, ignored if num_levels == -1
+      EXPECT_EQ(policy->DumpTestReport(),
+                "cf=abe,s=kCompactionStyleLevel,n=-1,l=-1,b=0,r=kMisc\n");
+#endif
    }

    // Destroy
@@ -82,6 +82,11 @@ class DeleteFilter : public CompactionFilter {
    return true;
  }

+  bool FilterMergeOperand(int /*level*/, const Slice& /*key*/,
+                          const Slice& /*operand*/) const override {
+    return true;
+  }
+
  const char* Name() const override { return "DeleteFilter"; }
 };

@@ -190,18 +195,36 @@ class KeepFilterFactory : public CompactionFilterFactory {
  bool compaction_filter_created_;
 };

+// This filter factory is configured with a `TableFileCreationReason`. Only
+// table files created for that reason will undergo filtering. This
+// configurability makes it useful to tests for filtering non-compaction table
+// files, such as "CompactionFilterFlush" and "CompactionFilterRecovery".
 class DeleteFilterFactory : public CompactionFilterFactory {
 public:
+  explicit DeleteFilterFactory(TableFileCreationReason reason)
+      : reason_(reason) {}
+
  std::unique_ptr<CompactionFilter> CreateCompactionFilter(
      const CompactionFilter::Context& context) override {
-    if (context.is_manual_compaction) {
-      return std::unique_ptr<CompactionFilter>(new DeleteFilter());
-    } else {
+    EXPECT_EQ(reason_, context.reason);
+    if (context.reason == TableFileCreationReason::kCompaction &&
+        !context.is_manual_compaction) {
+      // Table files created by automatic compaction do not undergo filtering.
+      // Presumably some tests rely on this.
      return std::unique_ptr<CompactionFilter>(nullptr);
    }
+    return std::unique_ptr<CompactionFilter>(new DeleteFilter());
+  }
+
+  bool ShouldFilterTableFileCreation(
+      TableFileCreationReason reason) const override {
+    return reason_ == reason;
  }

  const char* Name() const override { return "DeleteFilterFactory"; }
+
+ private:
+  const TableFileCreationReason reason_;
 };

 // Delete Filter Factory which ignores snapshots
@@ -349,7 +372,8 @@ TEST_F(DBTestCompactionFilter, CompactionFilter) {

  // create a new database with the compaction
  // filter in such a way that it deletes all keys
-  options.compaction_filter_factory = std::make_shared<DeleteFilterFactory>();
+  options.compaction_filter_factory = std::make_shared<DeleteFilterFactory>(
+      TableFileCreationReason::kCompaction);
  options.create_if_missing = true;
  DestroyAndReopen(options);
  CreateAndReopenWithCF({"pikachu"}, options);
@@ -421,7 +445,8 @@ TEST_F(DBTestCompactionFilter, CompactionFilter) {
 // entries in VersionEdit, but none of the 'AddFile's.
 TEST_F(DBTestCompactionFilter, CompactionFilterDeletesAll) {
  Options options = CurrentOptions();
-  options.compaction_filter_factory = std::make_shared<DeleteFilterFactory>();
+  options.compaction_filter_factory = std::make_shared<DeleteFilterFactory>(
+      TableFileCreationReason::kCompaction);
  options.disable_auto_compactions = true;
  options.create_if_missing = true;
  DestroyAndReopen(options);
@@ -450,6 +475,64 @@ TEST_F(DBTestCompactionFilter, CompactionFilterDeletesAll) {
 }
 #endif  // ROCKSDB_LITE

+TEST_F(DBTestCompactionFilter, CompactionFilterFlush) {
+  // Tests a `CompactionFilterFactory` that filters when table file is created
+  // by flush.
+  Options options = CurrentOptions();
+  options.compaction_filter_factory =
+      std::make_shared<DeleteFilterFactory>(TableFileCreationReason::kFlush);
+  options.merge_operator = MergeOperators::CreateStringAppendOperator();
+  Reopen(options);
+
+  // Puts and Merges are purged in flush.
+  ASSERT_OK(Put("a", "v"));
+  ASSERT_OK(Merge("b", "v"));
+  ASSERT_OK(Flush());
+  ASSERT_EQ("NOT_FOUND", Get("a"));
+  ASSERT_EQ("NOT_FOUND", Get("b"));
+
+  // However, Puts and Merges are preserved by recovery.
+  ASSERT_OK(Put("a", "v"));
+  ASSERT_OK(Merge("b", "v"));
+  Reopen(options);
+  ASSERT_EQ("v", Get("a"));
+  ASSERT_EQ("v", Get("b"));
+
+  // Likewise, compaction does not apply filtering.
+  ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+  ASSERT_EQ("v", Get("a"));
+  ASSERT_EQ("v", Get("b"));
+}
+
+TEST_F(DBTestCompactionFilter, CompactionFilterRecovery) {
+  // Tests a `CompactionFilterFactory` that filters when table file is created
+  // by recovery.
+  Options options = CurrentOptions();
+  options.compaction_filter_factory =
+      std::make_shared<DeleteFilterFactory>(TableFileCreationReason::kRecovery);
+  options.merge_operator = MergeOperators::CreateStringAppendOperator();
+  Reopen(options);
+
+  // Puts and Merges are purged in recovery.
+  ASSERT_OK(Put("a", "v"));
+  ASSERT_OK(Merge("b", "v"));
+  Reopen(options);
+  ASSERT_EQ("NOT_FOUND", Get("a"));
+  ASSERT_EQ("NOT_FOUND", Get("b"));
+
+  // However, Puts and Merges are preserved by flush.
+  ASSERT_OK(Put("a", "v"));
+  ASSERT_OK(Merge("b", "v"));
+  ASSERT_OK(Flush());
+  ASSERT_EQ("v", Get("a"));
+  ASSERT_EQ("v", Get("b"));
+
+  // Likewise, compaction does not apply filtering.
+  ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+  ASSERT_EQ("v", Get("a"));
+  ASSERT_EQ("v", Get("b"));
+}
+
 TEST_P(DBTestCompactionFilterWithCompactParam,
       CompactionFilterWithValueChange) {
  Options options = CurrentOptions();
@@ -842,6 +925,49 @@ TEST_F(DBTestCompactionFilter, IgnoreSnapshotsFalse) {
  delete options.compaction_filter;
 }

+class TestNotSupportedFilterFactory : public CompactionFilterFactory {
+ public:
+  explicit TestNotSupportedFilterFactory(TableFileCreationReason reason)
+      : reason_(reason) {}
+
+  bool ShouldFilterTableFileCreation(
+      TableFileCreationReason reason) const override {
+    return reason_ == reason;
+  }
+
+  std::unique_ptr<CompactionFilter> CreateCompactionFilter(
+      const CompactionFilter::Context& /* context */) override {
+    return std::unique_ptr<CompactionFilter>(new TestNotSupportedFilter());
+  }
+
+  const char* Name() const override { return "TestNotSupportedFilterFactory"; }
+
+ private:
+  const TableFileCreationReason reason_;
+};
+
+TEST_F(DBTestCompactionFilter, IgnoreSnapshotsFalseDuringFlush) {
+  Options options = CurrentOptions();
+  options.compaction_filter_factory =
+      std::make_shared<TestNotSupportedFilterFactory>(
+          TableFileCreationReason::kFlush);
+  Reopen(options);
+
+  ASSERT_OK(Put("a", "v10"));
+  ASSERT_TRUE(Flush().IsNotSupported());
+}
+
+TEST_F(DBTestCompactionFilter, IgnoreSnapshotsFalseRecovery) {
+  Options options = CurrentOptions();
+  options.compaction_filter_factory =
+      std::make_shared<TestNotSupportedFilterFactory>(
+          TableFileCreationReason::kRecovery);
+  Reopen(options);
+
+  ASSERT_OK(Put("a", "v10"));
+  ASSERT_TRUE(TryReopen(options).IsNotSupported());
+}
+
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -503,6 +503,10 @@ TEST_F(DBCompactionTest, TestTableReaderForCompaction) {
  options.new_table_reader_for_compaction_inputs = true;
  options.max_open_files = 20;
  options.level0_file_num_compaction_trigger = 3;
+  // Avoid many shards with small max_open_files, where as little as
+  // two table insertions could lead to an LRU eviction, depending on
+  // hash values.
+  options.table_cache_numshardbits = 2;
  DestroyAndReopen(options);
  Random rnd(301);

@@ -3596,6 +3600,41 @@ TEST_F(DBCompactionTest, CompactFilesOverlapInL0Bug) {
  ASSERT_EQ("new_val", Get(Key(0)));
 }

+TEST_F(DBCompactionTest, DeleteFilesInRangeConflictWithCompaction) {
+  Options options = CurrentOptions();
+  DestroyAndReopen(options);
+  const Snapshot* snapshot = nullptr;
+  const int kMaxKey = 10;
+
+  for (int i = 0; i < kMaxKey; i++) {
+    ASSERT_OK(Put(Key(i), Key(i)));
+    ASSERT_OK(Delete(Key(i)));
+    if (!snapshot) {
+      snapshot = db_->GetSnapshot();
+    }
+  }
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(1);
+  ASSERT_OK(Put(Key(kMaxKey), Key(kMaxKey)));
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+  // test DeleteFilesInRange() deletes the files already picked for compaction
+  SyncPoint::GetInstance()->LoadDependency(
+      {{"VersionSet::LogAndApply:WriteManifestStart",
+        "BackgroundCallCompaction:0"},
+       {"DBImpl::BackgroundCompaction:Finish",
+        "VersionSet::LogAndApply:WriteManifestDone"}});
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  // release snapshot which mark bottommost file for compaction
+  db_->ReleaseSnapshot(snapshot);
+  std::string begin_string = Key(0);
+  std::string end_string = Key(kMaxKey + 1);
+  Slice begin(begin_string);
+  Slice end(end_string);
+  ASSERT_OK(DeleteFilesInRange(db_, db_->DefaultColumnFamily(), &begin, &end));
+  SyncPoint::GetInstance()->DisableProcessing();
+}
+
 TEST_F(DBCompactionTest, CompactBottomLevelFilesWithDeletions) {
  // bottom-level files may contain deletions due to snapshots protecting the
  // deleted keys. Once the snapshot is released, we should see files with many
@@ -5149,6 +5188,94 @@ TEST_F(DBCompactionTest, ManualCompactionBottomLevelOptimized) {
  ASSERT_EQ(num, 0);
 }

+TEST_F(DBCompactionTest, ManualCompactionMax) {
+  uint64_t l1_avg_size = 0, l2_avg_size = 0;
+  auto generate_sst_func = [&]() {
+    Random rnd(301);
+    for (auto i = 0; i < 100; i++) {
+      for (auto j = 0; j < 10; j++) {
+        ASSERT_OK(Put(Key(i * 10 + j), rnd.RandomString(1024)));
+      }
+      ASSERT_OK(Flush());
+    }
+    MoveFilesToLevel(2);
+
+    for (auto i = 0; i < 10; i++) {
+      for (auto j = 0; j < 10; j++) {
+        ASSERT_OK(Put(Key(i * 100 + j * 10), rnd.RandomString(1024)));
+      }
+      ASSERT_OK(Flush());
+    }
+    MoveFilesToLevel(1);
+
+    std::vector<std::vector<FileMetaData>> level_to_files;
+    dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(),
+                                    &level_to_files);
+
+    uint64_t total = 0;
+    for (const auto& file : level_to_files[1]) {
+      total += file.compensated_file_size;
+    }
+    l1_avg_size = total / level_to_files[1].size();
+
+    total = 0;
+    for (const auto& file : level_to_files[2]) {
+      total += file.compensated_file_size;
+    }
+    l2_avg_size = total / level_to_files[2].size();
+  };
+
+  std::atomic_int num_compactions(0);
+  SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::BGWorkCompaction", [&](void* /*arg*/) { ++num_compactions; });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  Options opts = CurrentOptions();
+  opts.disable_auto_compactions = true;
+
+  // with default setting (1.6G by default), it should cover all files in 1
+  // compaction
+  DestroyAndReopen(opts);
+  generate_sst_func();
+  num_compactions.store(0);
+  CompactRangeOptions cro;
+  ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
+  ASSERT_TRUE(num_compactions.load() == 1);
+
+  // split the compaction to 5
+  int num_split = 5;
+  DestroyAndReopen(opts);
+  generate_sst_func();
+  uint64_t total_size = (l1_avg_size * 10) + (l2_avg_size * 100);
+  opts.max_compaction_bytes = total_size / num_split;
+  Reopen(opts);
+  num_compactions.store(0);
+  ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
+  ASSERT_TRUE(num_compactions.load() == num_split);
+
+  // very small max_compaction_bytes, it should still move forward
+  opts.max_compaction_bytes = l1_avg_size / 2;
+  DestroyAndReopen(opts);
+  generate_sst_func();
+  num_compactions.store(0);
+  ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
+  ASSERT_TRUE(num_compactions.load() > 10);
+
+  // dynamically set the option
+  num_split = 2;
+  opts.max_compaction_bytes = 0;
+  DestroyAndReopen(opts);
+  generate_sst_func();
+  total_size = (l1_avg_size * 10) + (l2_avg_size * 100);
+  Status s = db_->SetOptions(
+      {{"max_compaction_bytes", std::to_string(total_size / num_split)}});
+  ASSERT_OK(s);
+
+  num_compactions.store(0);
+  ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
+  ASSERT_TRUE(num_compactions.load() == num_split);
+}
+
 TEST_F(DBCompactionTest, CompactionDuringShutdown) {
  Options opts = CurrentOptions();
  opts.level0_file_num_compaction_trigger = 2;
@@ -98,7 +98,14 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,

  ret.emplace_back(CurrentFileName(""));
  ret.emplace_back(DescriptorFileName("", versions_->manifest_file_number()));
-  ret.emplace_back(OptionsFileName("", versions_->options_file_number()));
+  // The OPTIONS file number is zero in read-write mode when OPTIONS file
+  // writing failed and the DB was configured with
+  // `fail_if_options_file_error == false`. In read-only mode the OPTIONS file
+  // number is zero when no OPTIONS file exist at all. In those cases we do not
+  // record any OPTIONS file in the live file list.
+  if (versions_->options_file_number() != 0) {
+    ret.emplace_back(OptionsFileName("", versions_->options_file_number()));
+  }

  // find length of manifest file while holding the mutex lock
  *manifest_file_size = versions_->manifest_file_size();
@@ -285,6 +285,379 @@ TEST_F(DBFlushTest, ScheduleOnlyOneBgThread) {
  SyncPoint::GetInstance()->ClearAllCallBacks();
 }

+// The following 3 tests are designed for testing garbage statistics at flush
+// time.
+//
+// ======= General Information ======= (from GitHub Wiki).
+// There are three scenarios where memtable flush can be triggered:
+//
+// 1 - Memtable size exceeds ColumnFamilyOptions::write_buffer_size
+//     after a write.
+// 2 - Total memtable size across all column families exceeds
+// DBOptions::db_write_buffer_size,
+//     or DBOptions::write_buffer_manager signals a flush. In this scenario
+//     the largest memtable will be flushed.
+// 3 - Total WAL file size exceeds DBOptions::max_total_wal_size.
+//     In this scenario the memtable with the oldest data will be flushed,
+//     in order to allow the WAL file with data from this memtable to be
+//     purged.
+//
+// As a result, a memtable can be flushed before it is full. This is one
+// reason the generated SST file can be smaller than the corresponding
+// memtable. Compression is another factor to make SST file smaller than
+// corresponding memtable, since data in memtable is uncompressed.
+
+TEST_F(DBFlushTest, StatisticsGarbageBasic) {
+  Options options = CurrentOptions();
+
+  // The following options are used to enforce several values that
+  // may already exist as default values to make this test resilient
+  // to default value updates in the future.
+  options.statistics = CreateDBStatistics();
+
+  // Record all statistics.
+  options.statistics->set_stats_level(StatsLevel::kAll);
+
+  // create the DB if it's not already present
+  options.create_if_missing = true;
+
+  // Useful for now as we are trying to compare uncompressed data savings on
+  // flush().
+  options.compression = kNoCompression;
+
+  // Prevent memtable in place updates. Should already be disabled
+  // (from Wiki:
+  //  In place updates can be enabled by toggling on the bool
+  //  inplace_update_support flag. However, this flag is by default set to
+  //  false
+  //  because this thread-safe in-place update support is not compatible
+  //  with concurrent memtable writes. Note that the bool
+  //  allow_concurrent_memtable_write is set to true by default )
+  options.inplace_update_support = false;
+  options.allow_concurrent_memtable_write = true;
+
+  // Enforce size of a single MemTable to 64MB (64MB = 67108864 bytes).
+  options.write_buffer_size = 64 << 20;
+
+  ASSERT_OK(TryReopen(options));
+
+  // Put multiple times the same key-values.
+  // The encoded length of a db entry in the memtable is
+  // defined in db/memtable.cc (MemTable::Add) as the variable:
+  // encoded_len=  VarintLength(internal_key_size)  --> =
+  // log_256(internal_key).
+  // Min # of bytes
+  //                                                       necessary to
+  //                                                       store
+  //                                                       internal_key_size.
+  //             + internal_key_size                --> = actual key string,
+  //             (size key_size: w/o term null char)
+  //                                                      + 8 bytes for
+  //                                                      fixed uint64 "seq
+  //                                                      number
+  // +
+  //                                                      insertion type"
+  //             + VarintLength(val_size)           --> = min # of bytes to
+  //             store val_size
+  //             + val_size                         --> = actual value
+  //             string
+  // For example, in our situation, "key1" : size 4, "value1" : size 6
+  // (the terminating null characters are not copied over to the memtable).
+  // And therefore encoded_len = 1 + (4+8) + 1 + 6 = 20 bytes per entry.
+  // However in terms of raw data contained in the memtable, and written
+  // over to the SSTable, we only count internal_key_size and val_size,
+  // because this is the only raw chunk of bytes that contains everything
+  // necessary to reconstruct a user entry: sequence number, insertion type,
+  // key, and value.
+
+  // To test the relevance of our Memtable garbage statistics,
+  // namely MEMTABLE_PAYLOAD_BYTES_AT_FLUSH and MEMTABLE_GARBAGE_BYTES_AT_FLUSH,
+  // we insert K-V pairs with 3 distinct keys (of length 4),
+  // and random values of arbitrary length RAND_VALUES_LENGTH,
+  // and we repeat this step NUM_REPEAT times total.
+  // At the end, we insert 3 final K-V pairs with the same 3 keys
+  // and known values (these will be the final values, of length 6).
+  // I chose NUM_REPEAT=2,000 such that no automatic flush is
+  // triggered (the number of bytes in the memtable is therefore
+  // well below any meaningful heuristic for a memtable of size 64MB).
+  // As a result, since each K-V pair is inserted as a payload
+  // of N meaningful bytes (sequence number, insertion type,
+  // key, and value = 8 + 4 + RAND_VALUE_LENGTH),
+  // MEMTABLE_GARBAGE_BYTES_AT_FLUSH should be equal to 2,000 * N bytes
+  // and MEMTABLE_PAYLAOD_BYTES_AT_FLUSH = MEMTABLE_GARBAGE_BYTES_AT_FLUSH +
+  // (3*(8 + 4 + 6)) bytes. For RAND_VALUE_LENGTH = 172 (arbitrary value), we
+  // expect:
+  //      N = 8 + 4 + 172 = 184 bytes
+  //      MEMTABLE_GARBAGE_BYTES_AT_FLUSH = 2,000 * 184 = 368,000 bytes.
+  //      MEMTABLE_PAYLOAD_BYTES_AT_FLUSH = 368,000 + 3*18 = 368,054 bytes.
+
+  const size_t NUM_REPEAT = 2000;
+  const size_t RAND_VALUES_LENGTH = 172;
+  const std::string KEY1 = "key1";
+  const std::string KEY2 = "key2";
+  const std::string KEY3 = "key3";
+  const std::string VALUE1 = "value1";
+  const std::string VALUE2 = "value2";
+  const std::string VALUE3 = "value3";
+  uint64_t EXPECTED_MEMTABLE_PAYLOAD_BYTES_AT_FLUSH = 0;
+  uint64_t EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH = 0;
+
+  Random rnd(301);
+  // Insertion of of K-V pairs, multiple times.
+  for (size_t i = 0; i < NUM_REPEAT; i++) {
+    // Create value strings of arbitrary length RAND_VALUES_LENGTH bytes.
+    std::string p_v1 = rnd.RandomString(RAND_VALUES_LENGTH);
+    std::string p_v2 = rnd.RandomString(RAND_VALUES_LENGTH);
+    std::string p_v3 = rnd.RandomString(RAND_VALUES_LENGTH);
+    ASSERT_OK(Put(KEY1, p_v1));
+    ASSERT_OK(Put(KEY2, p_v2));
+    ASSERT_OK(Put(KEY3, p_v3));
+    EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH +=
+        KEY1.size() + p_v1.size() + sizeof(uint64_t);
+    EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH +=
+        KEY2.size() + p_v2.size() + sizeof(uint64_t);
+    EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH +=
+        KEY3.size() + p_v3.size() + sizeof(uint64_t);
+  }
+
+  // The memtable data bytes includes the "garbage"
+  // bytes along with the useful payload.
+  EXPECTED_MEMTABLE_PAYLOAD_BYTES_AT_FLUSH =
+      EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH;
+
+  ASSERT_OK(Put(KEY1, VALUE1));
+  ASSERT_OK(Put(KEY2, VALUE2));
+  ASSERT_OK(Put(KEY3, VALUE3));
+
+  // Add useful payload to the memtable data bytes:
+  EXPECTED_MEMTABLE_PAYLOAD_BYTES_AT_FLUSH +=
+      KEY1.size() + VALUE1.size() + KEY2.size() + VALUE2.size() + KEY3.size() +
+      VALUE3.size() + 3 * sizeof(uint64_t);
+
+  // We assert that the last K-V pairs have been successfully inserted,
+  // and that the valid values are VALUE1, VALUE2, VALUE3.
+  PinnableSlice value;
+  ASSERT_OK(Get(KEY1, &value));
+  ASSERT_EQ(value.ToString(), VALUE1);
+  ASSERT_OK(Get(KEY2, &value));
+  ASSERT_EQ(value.ToString(), VALUE2);
+  ASSERT_OK(Get(KEY3, &value));
+  ASSERT_EQ(value.ToString(), VALUE3);
+
+  // Force flush to SST. Increments the statistics counter.
+  ASSERT_OK(Flush());
+
+  // Collect statistics.
+  uint64_t mem_data_bytes =
+      TestGetTickerCount(options, MEMTABLE_PAYLOAD_BYTES_AT_FLUSH);
+  uint64_t mem_garbage_bytes =
+      TestGetTickerCount(options, MEMTABLE_GARBAGE_BYTES_AT_FLUSH);
+
+  EXPECT_EQ(mem_data_bytes, EXPECTED_MEMTABLE_PAYLOAD_BYTES_AT_FLUSH);
+  EXPECT_EQ(mem_garbage_bytes, EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH);
+
+  Close();
+}
+
+TEST_F(DBFlushTest, StatisticsGarbageInsertAndDeletes) {
+  Options options = CurrentOptions();
+  options.statistics = CreateDBStatistics();
+  options.statistics->set_stats_level(StatsLevel::kAll);
+  options.create_if_missing = true;
+  options.compression = kNoCompression;
+  options.inplace_update_support = false;
+  options.allow_concurrent_memtable_write = true;
+  options.write_buffer_size = 67108864;
+
+  ASSERT_OK(TryReopen(options));
+
+  const size_t NUM_REPEAT = 2000;
+  const size_t RAND_VALUES_LENGTH = 37;
+  const std::string KEY1 = "key1";
+  const std::string KEY2 = "key2";
+  const std::string KEY3 = "key3";
+  const std::string KEY4 = "key4";
+  const std::string KEY5 = "key5";
+  const std::string KEY6 = "key6";
+
+  uint64_t EXPECTED_MEMTABLE_PAYLOAD_BYTES_AT_FLUSH = 0;
+  uint64_t EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH = 0;
+
+  WriteBatch batch;
+
+  Random rnd(301);
+  // Insertion of of K-V pairs, multiple times.
+  for (size_t i = 0; i < NUM_REPEAT; i++) {
+    // Create value strings of arbitrary length RAND_VALUES_LENGTH bytes.
+    std::string p_v1 = rnd.RandomString(RAND_VALUES_LENGTH);
+    std::string p_v2 = rnd.RandomString(RAND_VALUES_LENGTH);
+    std::string p_v3 = rnd.RandomString(RAND_VALUES_LENGTH);
+    ASSERT_OK(Put(KEY1, p_v1));
+    ASSERT_OK(Put(KEY2, p_v2));
+    ASSERT_OK(Put(KEY3, p_v3));
+    EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH +=
+        KEY1.size() + p_v1.size() + sizeof(uint64_t);
+    EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH +=
+        KEY2.size() + p_v2.size() + sizeof(uint64_t);
+    EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH +=
+        KEY3.size() + p_v3.size() + sizeof(uint64_t);
+    ASSERT_OK(Delete(KEY1));
+    ASSERT_OK(Delete(KEY2));
+    ASSERT_OK(Delete(KEY3));
+    EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH +=
+        KEY1.size() + KEY2.size() + KEY3.size() + 3 * sizeof(uint64_t);
+  }
+
+  // The memtable data bytes includes the "garbage"
+  // bytes along with the useful payload.
+  EXPECTED_MEMTABLE_PAYLOAD_BYTES_AT_FLUSH =
+      EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH;
+
+  // Note : one set of delete for KEY1, KEY2, KEY3 is written to
+  // SSTable to propagate the delete operations to K-V pairs
+  // that could have been inserted into the database during past Flush
+  // opeartions.
+  EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH -=
+      KEY1.size() + KEY2.size() + KEY3.size() + 3 * sizeof(uint64_t);
+
+  // Additional useful paylaod.
+  ASSERT_OK(Delete(KEY4));
+  ASSERT_OK(Delete(KEY5));
+  ASSERT_OK(Delete(KEY6));
+
+  // // Add useful payload to the memtable data bytes:
+  EXPECTED_MEMTABLE_PAYLOAD_BYTES_AT_FLUSH +=
+      KEY4.size() + KEY5.size() + KEY6.size() + 3 * sizeof(uint64_t);
+
+  // We assert that the K-V pairs have been successfully deleted.
+  PinnableSlice value;
+  ASSERT_NOK(Get(KEY1, &value));
+  ASSERT_NOK(Get(KEY2, &value));
+  ASSERT_NOK(Get(KEY3, &value));
+
+  // Force flush to SST. Increments the statistics counter.
+  ASSERT_OK(Flush());
+
+  // Collect statistics.
+  uint64_t mem_data_bytes =
+      TestGetTickerCount(options, MEMTABLE_PAYLOAD_BYTES_AT_FLUSH);
+  uint64_t mem_garbage_bytes =
+      TestGetTickerCount(options, MEMTABLE_GARBAGE_BYTES_AT_FLUSH);
+
+  EXPECT_EQ(mem_data_bytes, EXPECTED_MEMTABLE_PAYLOAD_BYTES_AT_FLUSH);
+  EXPECT_EQ(mem_garbage_bytes, EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH);
+
+  Close();
+}
+
+TEST_F(DBFlushTest, StatisticsGarbageRangeDeletes) {
+  Options options = CurrentOptions();
+  options.statistics = CreateDBStatistics();
+  options.statistics->set_stats_level(StatsLevel::kAll);
+  options.create_if_missing = true;
+  options.compression = kNoCompression;
+  options.inplace_update_support = false;
+  options.allow_concurrent_memtable_write = true;
+  options.write_buffer_size = 67108864;
+
+  ASSERT_OK(TryReopen(options));
+
+  const size_t NUM_REPEAT = 1000;
+  const size_t RAND_VALUES_LENGTH = 42;
+  const std::string KEY1 = "key1";
+  const std::string KEY2 = "key2";
+  const std::string KEY3 = "key3";
+  const std::string KEY4 = "key4";
+  const std::string KEY5 = "key5";
+  const std::string KEY6 = "key6";
+  const std::string VALUE3 = "value3";
+
+  uint64_t EXPECTED_MEMTABLE_PAYLOAD_BYTES_AT_FLUSH = 0;
+  uint64_t EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH = 0;
+
+  Random rnd(301);
+  // Insertion of of K-V pairs, multiple times.
+  // Also insert DeleteRange
+  for (size_t i = 0; i < NUM_REPEAT; i++) {
+    // Create value strings of arbitrary length RAND_VALUES_LENGTH bytes.
+    std::string p_v1 = rnd.RandomString(RAND_VALUES_LENGTH);
+    std::string p_v2 = rnd.RandomString(RAND_VALUES_LENGTH);
+    std::string p_v3 = rnd.RandomString(RAND_VALUES_LENGTH);
+    ASSERT_OK(Put(KEY1, p_v1));
+    ASSERT_OK(Put(KEY2, p_v2));
+    ASSERT_OK(Put(KEY3, p_v3));
+    EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH +=
+        KEY1.size() + p_v1.size() + sizeof(uint64_t);
+    EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH +=
+        KEY2.size() + p_v2.size() + sizeof(uint64_t);
+    EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH +=
+        KEY3.size() + p_v3.size() + sizeof(uint64_t);
+    ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), KEY1,
+                               KEY2));
+    // Note: DeleteRange have an exclusive upper bound, e.g. here: [KEY2,KEY3)
+    // is deleted.
+    ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), KEY2,
+                               KEY3));
+    // Delete ranges are stored as a regular K-V pair, with key=STARTKEY,
+    // value=ENDKEY.
+    EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH +=
+        (KEY1.size() + KEY2.size() + sizeof(uint64_t)) +
+        (KEY2.size() + KEY3.size() + sizeof(uint64_t));
+  }
+
+  // The memtable data bytes includes the "garbage"
+  // bytes along with the useful payload.
+  EXPECTED_MEMTABLE_PAYLOAD_BYTES_AT_FLUSH =
+      EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH;
+
+  // Note : one set of deleteRange for (KEY1, KEY2) and (KEY2, KEY3) is written
+  // to SSTable to propagate the deleteRange operations to K-V pairs that could
+  // have been inserted into the database during past Flush opeartions.
+  EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH -=
+      (KEY1.size() + KEY2.size() + sizeof(uint64_t)) +
+      (KEY2.size() + KEY3.size() + sizeof(uint64_t));
+
+  // Overwrite KEY3 with known value (VALUE3)
+  // Note that during the whole time KEY3 has never been deleted
+  // by the RangeDeletes.
+  ASSERT_OK(Put(KEY3, VALUE3));
+  EXPECTED_MEMTABLE_PAYLOAD_BYTES_AT_FLUSH +=
+      KEY3.size() + VALUE3.size() + sizeof(uint64_t);
+
+  // Additional useful paylaod.
+  ASSERT_OK(
+      db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), KEY4, KEY5));
+  ASSERT_OK(
+      db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), KEY5, KEY6));
+
+  // Add useful payload to the memtable data bytes:
+  EXPECTED_MEMTABLE_PAYLOAD_BYTES_AT_FLUSH +=
+      (KEY4.size() + KEY5.size() + sizeof(uint64_t)) +
+      (KEY5.size() + KEY6.size() + sizeof(uint64_t));
+
+  // We assert that the K-V pairs have been successfully deleted.
+  PinnableSlice value;
+  ASSERT_NOK(Get(KEY1, &value));
+  ASSERT_NOK(Get(KEY2, &value));
+  // And that KEY3's value is correct.
+  ASSERT_OK(Get(KEY3, &value));
+  ASSERT_EQ(value, VALUE3);
+
+  // Force flush to SST. Increments the statistics counter.
+  ASSERT_OK(Flush());
+
+  // Collect statistics.
+  uint64_t mem_data_bytes =
+      TestGetTickerCount(options, MEMTABLE_PAYLOAD_BYTES_AT_FLUSH);
+  uint64_t mem_garbage_bytes =
+      TestGetTickerCount(options, MEMTABLE_GARBAGE_BYTES_AT_FLUSH);
+
+  EXPECT_EQ(mem_data_bytes, EXPECTED_MEMTABLE_PAYLOAD_BYTES_AT_FLUSH);
+  EXPECT_EQ(mem_garbage_bytes, EXPECTED_MEMTABLE_GARBAGE_BYTES_AT_FLUSH);
+
+  Close();
+}
+
 TEST_P(DBFlushDirectIOTest, DirectIO) {
  Options options;
  options.create_if_missing = true;
@@ -63,6 +63,7 @@
 #include "memtable/hash_linklist_rep.h"
 #include "memtable/hash_skiplist_rep.h"
 #include "monitoring/in_memory_stats_history.h"
+#include "monitoring/instrumented_mutex.h"
 #include "monitoring/iostats_context_imp.h"
 #include "monitoring/perf_context_imp.h"
 #include "monitoring/persistent_stats_history.h"
@@ -156,7 +157,7 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname,
      immutable_db_options_(initial_db_options_),
      fs_(immutable_db_options_.fs, io_tracer_),
      mutable_db_options_(initial_db_options_),
-      stats_(immutable_db_options_.statistics.get()),
+      stats_(immutable_db_options_.stats),
      mutex_(stats_, immutable_db_options_.clock, DB_MUTEX_WAIT_MICROS,
             immutable_db_options_.use_adaptive_mutex),
      default_cf_handle_(nullptr),
@@ -251,16 +252,17 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname,
  co.num_shard_bits = immutable_db_options_.table_cache_numshardbits;
  co.metadata_charge_policy = kDontChargeCacheMetadata;
  table_cache_ = NewLRUCache(co);
+  SetDbSessionId();
+  assert(!db_session_id_.empty());

  versions_.reset(new VersionSet(dbname_, &immutable_db_options_, file_options_,
                                 table_cache_.get(), write_buffer_manager_,
                                 &write_controller_, &block_cache_tracer_,
-                                 io_tracer_));
+                                 io_tracer_, db_session_id_));
  column_family_memtables_.reset(
      new ColumnFamilyMemTablesImpl(versions_->GetColumnFamilySet()));

  DumpRocksDBBuildVersion(immutable_db_options_.info_log.get());
-  SetDbSessionId();
  DumpDBFileSummary(immutable_db_options_, dbname_, db_session_id_);
  immutable_db_options_.Dump(immutable_db_options_.info_log.get());
  mutable_db_options_.Dump(immutable_db_options_.info_log.get());
@@ -706,7 +708,7 @@ const Status DBImpl::CreateArchivalDirectory() {
 }

 void DBImpl::PrintStatistics() {
-  auto dbstats = immutable_db_options_.statistics.get();
+  auto dbstats = immutable_db_options_.stats;
  if (dbstats) {
    ROCKS_LOG_INFO(immutable_db_options_.info_log, "STATISTICS:\n %s",
                   dbstats->ToString().c_str());
@@ -767,7 +769,7 @@ void DBImpl::PersistStats() {
  uint64_t now_seconds =
      immutable_db_options_.clock->NowMicros() / kMicrosInSecond;

-  Statistics* statistics = immutable_db_options_.statistics.get();
+  Statistics* statistics = immutable_db_options_.stats;
  if (!statistics) {
    return;
  }
@@ -906,32 +908,50 @@ Status DBImpl::GetStatsHistory(
 void DBImpl::DumpStats() {
  TEST_SYNC_POINT("DBImpl::DumpStats:1");
 #ifndef ROCKSDB_LITE
-  const DBPropertyInfo* cf_property_info =
-      GetPropertyInfo(DB::Properties::kCFStats);
-  assert(cf_property_info != nullptr);
-  const DBPropertyInfo* db_property_info =
-      GetPropertyInfo(DB::Properties::kDBStats);
-  assert(db_property_info != nullptr);
-
  std::string stats;
  if (shutdown_initiated_) {
    return;
  }
+
  TEST_SYNC_POINT("DBImpl::DumpStats:StartRunning");
  {
    InstrumentedMutexLock l(&mutex_);
-    default_cf_internal_stats_->GetStringProperty(
-        *db_property_info, DB::Properties::kDBStats, &stats);
    for (auto cfd : *versions_->GetColumnFamilySet()) {
      if (cfd->initialized()) {
-        cfd->internal_stats()->GetStringProperty(
-            *cf_property_info, DB::Properties::kCFStatsNoFileHistogram, &stats);
+        // Release DB mutex for gathering cache entry stats. Pass over all
+        // column families for this first so that other stats are dumped
+        // near-atomically.
+        InstrumentedMutexUnlock u(&mutex_);
+        cfd->internal_stats()->CollectCacheEntryStats(/*foreground=*/false);
      }
    }
+
+    const std::string* property = &DB::Properties::kDBStats;
+    const DBPropertyInfo* property_info = GetPropertyInfo(*property);
+    assert(property_info != nullptr);
+    assert(!property_info->need_out_of_mutex);
+    default_cf_internal_stats_->GetStringProperty(*property_info, *property,
+                                                  &stats);
+
+    property = &DB::Properties::kCFStatsNoFileHistogram;
+    property_info = GetPropertyInfo(*property);
+    assert(property_info != nullptr);
+    assert(!property_info->need_out_of_mutex);
    for (auto cfd : *versions_->GetColumnFamilySet()) {
      if (cfd->initialized()) {
-        cfd->internal_stats()->GetStringProperty(
-            *cf_property_info, DB::Properties::kCFFileHistogram, &stats);
+        cfd->internal_stats()->GetStringProperty(*property_info, *property,
+                                                 &stats);
+      }
+    }
+
+    property = &DB::Properties::kCFFileHistogram;
+    property_info = GetPropertyInfo(*property);
+    assert(property_info != nullptr);
+    assert(!property_info->need_out_of_mutex);
+    for (auto cfd : *versions_->GetColumnFamilySet()) {
+      if (cfd->initialized()) {
+        cfd->internal_stats()->GetStringProperty(*property_info, *property,
+                                                 &stats);
      }
    }
  }
@@ -1875,6 +1895,16 @@ std::vector<Status> DBImpl::MultiGet(
  }
 #endif  // NDEBUG

+  if (tracer_) {
+    // TODO: This mutex should be removed later, to improve performance when
+    // tracing is enabled.
+    InstrumentedMutexLock lock(&trace_mutex_);
+    if (tracer_) {
+      // TODO: maybe handle the tracing status?
+      tracer_->MultiGet(column_family, keys).PermitUncheckedError();
+    }
+  }
+
  SequenceNumber consistent_seqnum;

  std::unordered_map<uint32_t, MultiGetColumnFamilyData> multiget_cf_data(
@@ -2186,6 +2216,16 @@ void DBImpl::MultiGet(const ReadOptions& read_options, const size_t num_keys,
  }
 #endif  // NDEBUG

+  if (tracer_) {
+    // TODO: This mutex should be removed later, to improve performance when
+    // tracing is enabled.
+    InstrumentedMutexLock lock(&trace_mutex_);
+    if (tracer_) {
+      // TODO: maybe handle the tracing status?
+      tracer_->MultiGet(num_keys, column_families, keys).PermitUncheckedError();
+    }
+  }
+
  autovector<KeyContext, MultiGetContext::MAX_BATCH_SIZE> key_context;
  autovector<KeyContext*, MultiGetContext::MAX_BATCH_SIZE> sorted_keys;
  sorted_keys.resize(num_keys);
@@ -2348,6 +2388,15 @@ void DBImpl::MultiGet(const ReadOptions& read_options,
                      const Slice* keys, PinnableSlice* values,
                      std::string* timestamps, Status* statuses,
                      const bool sorted_input) {
+  if (tracer_) {
+    // TODO: This mutex should be removed later, to improve performance when
+    // tracing is enabled.
+    InstrumentedMutexLock lock(&trace_mutex_);
+    if (tracer_) {
+      // TODO: maybe handle the tracing status?
+      tracer_->MultiGet(num_keys, column_family, keys).PermitUncheckedError();
+    }
+  }
  autovector<KeyContext, MultiGetContext::MAX_BATCH_SIZE> key_context;
  autovector<KeyContext*, MultiGetContext::MAX_BATCH_SIZE> sorted_keys;
  sorted_keys.resize(num_keys);
@@ -3197,16 +3246,21 @@ bool DBImpl::GetProperty(ColumnFamilyHandle* column_family,
    }
    return ret_value;
  } else if (property_info->handle_string) {
-    InstrumentedMutexLock l(&mutex_);
-    return cfd->internal_stats()->GetStringProperty(*property_info, property,
-                                                    value);
-  } else if (property_info->handle_string_dbimpl) {
-    std::string tmp_value;
-    bool ret_value = (this->*(property_info->handle_string_dbimpl))(&tmp_value);
-    if (ret_value) {
-      *value = tmp_value;
+    if (property_info->need_out_of_mutex) {
+      return cfd->internal_stats()->GetStringProperty(*property_info, property,
+                                                      value);
+    } else {
+      InstrumentedMutexLock l(&mutex_);
+      return cfd->internal_stats()->GetStringProperty(*property_info, property,
+                                                      value);
+    }
+  } else if (property_info->handle_string_dbimpl) {
+    if (property_info->need_out_of_mutex) {
+      return (this->*(property_info->handle_string_dbimpl))(value);
+    } else {
+      InstrumentedMutexLock l(&mutex_);
+      return (this->*(property_info->handle_string_dbimpl))(value);
    }
-    return ret_value;
  }
  // Shouldn't reach here since exactly one of handle_string and handle_int
  // should be non-nullptr.
@@ -3224,9 +3278,14 @@ bool DBImpl::GetMapProperty(ColumnFamilyHandle* column_family,
  if (property_info == nullptr) {
    return false;
  } else if (property_info->handle_map) {
-    InstrumentedMutexLock l(&mutex_);
-    return cfd->internal_stats()->GetMapProperty(*property_info, property,
-                                                 value);
+    if (property_info->need_out_of_mutex) {
+      return cfd->internal_stats()->GetMapProperty(*property_info, property,
+                                                   value);
+    } else {
+      InstrumentedMutexLock l(&mutex_);
+      return cfd->internal_stats()->GetMapProperty(*property_info, property,
+                                                   value);
+    }
  }
  // If we reach this point it means that handle_map is not provided for the
  // requested property
@@ -3277,7 +3336,7 @@ bool DBImpl::GetIntPropertyInternal(ColumnFamilyData* cfd,

 bool DBImpl::GetPropertyHandleOptionsStatistics(std::string* value) {
  assert(value != nullptr);
-  Statistics* statistics = immutable_db_options_.statistics.get();
+  Statistics* statistics = immutable_db_options_.stats;
  if (!statistics) {
    return false;
  }
@@ -3675,6 +3734,8 @@ Status DBImpl::DeleteFilesInRanges(ColumnFamilyHandle* column_family,
          deleted_files.insert(level_file);
          level_file->being_compacted = true;
        }
+        vstorage->ComputeCompactionScore(*cfd->ioptions(),
+                                         *cfd->GetLatestMutableCFOptions());
      }
    }
    if (edit.GetDeletedFiles().empty()) {
@@ -1454,15 +1454,16 @@ class DBImpl : public DB {
    uint32_t output_path_id;
    Status status;
    bool done;
-    bool in_progress;            // compaction request being processed?
-    bool incomplete;             // only part of requested range compacted
-    bool exclusive;              // current behavior of only one manual
-    bool disallow_trivial_move;  // Force actual compaction to run
-    const InternalKey* begin;    // nullptr means beginning of key range
-    const InternalKey* end;      // nullptr means end of key range
-    InternalKey* manual_end;     // how far we are compacting
-    InternalKey tmp_storage;     // Used to keep track of compaction progress
-    InternalKey tmp_storage1;    // Used to keep track of compaction progress
+    bool in_progress;             // compaction request being processed?
+    bool incomplete;              // only part of requested range compacted
+    bool exclusive;               // current behavior of only one manual
+    bool disallow_trivial_move;   // Force actual compaction to run
+    const InternalKey* begin;     // nullptr means beginning of key range
+    const InternalKey* end;       // nullptr means end of key range
+    InternalKey* manual_end;      // how far we are compacting
+    InternalKey tmp_storage;      // Used to keep track of compaction progress
+    InternalKey tmp_storage1;     // Used to keep track of compaction progress
+    std::atomic<bool>* canceled;  // Compaction canceled by the user?
  };
  struct PrepickedCompaction {
    // background compaction takes ownership of `compaction`.
@@ -807,6 +807,10 @@ Status DBImpl::CompactRange(const CompactRangeOptions& options,
    return Status::Incomplete(Status::SubCode::kManualCompactionPaused);
  }

+  if (options.canceled && options.canceled->load(std::memory_order_acquire)) {
+    return Status::Incomplete(Status::SubCode::kManualCompactionPaused);
+  }
+
  const Comparator* const ucmp = column_family->GetComparator();
  assert(ucmp);
  size_t ts_sz = ucmp->timestamp_size();
@@ -1092,7 +1096,7 @@ Status DBImpl::CompactFiles(const CompactionOptions& compact_options,
  assert(cfd);

  Status s;
-  JobContext job_context(0, true);
+  JobContext job_context(next_job_id_.fetch_add(1), true);
  LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL,
                       immutable_db_options_.info_log.get());

@@ -1243,7 +1247,7 @@ Status DBImpl::CompactFilesImpl(
  assert(is_snapshot_supported_ || snapshots_.empty());
  CompactionJobStats compaction_job_stats;
  CompactionJob compaction_job(
-      job_context->job_id, c.get(), immutable_db_options_,
+      job_context->job_id, c.get(), immutable_db_options_, mutable_db_options_,
      file_options_for_compaction_, versions_.get(), &shutting_down_,
      preserve_deletes_seqnum_.load(), log_buffer, directories_.GetDbDir(),
      GetDataDir(c->column_family_data(), c->output_path_id()),
@@ -1253,7 +1257,7 @@ Status DBImpl::CompactFilesImpl(
      c->mutable_cf_options()->paranoid_file_checks,
      c->mutable_cf_options()->report_bg_io_stats, dbname_,
      &compaction_job_stats, Env::Priority::USER, io_tracer_,
-      &manual_compaction_paused_, db_id_, db_session_id_,
+      &manual_compaction_paused_, nullptr, db_id_, db_session_id_,
      c->column_family_data()->GetFullHistoryTsLow());

  // Creating a compaction influences the compaction score because the score
@@ -1328,7 +1332,7 @@ Status DBImpl::CompactFilesImpl(
  if (output_file_names != nullptr) {
    for (const auto& newf : c->edit()->GetNewFiles()) {
      (*output_file_names)
-          .push_back(TableFileName(c->immutable_cf_options()->cf_paths,
+          .push_back(TableFileName(c->immutable_options()->cf_paths,
                                   newf.second.fd.GetNumber(),
                                   newf.second.fd.GetPathId()));
    }
@@ -1426,10 +1430,13 @@ void DBImpl::NotifyOnCompactionCompleted(
  if (shutting_down_.load(std::memory_order_acquire)) {
    return;
  }
+  // TODO: Should disabling manual compaction squash compaction completed
+  //   notifications that aren't the result of a shutdown?
  if (c->is_manual_compaction() &&
      manual_compaction_paused_.load(std::memory_order_acquire) > 0) {
    return;
  }
+
  Version* current = cfd->current();
  current->Ref();
  // release lock while notifying events
@@ -1654,6 +1661,7 @@ Status DBImpl::RunManualCompaction(
  manual.incomplete = false;
  manual.exclusive = exclusive;
  manual.disallow_trivial_move = disallow_trivial_move;
+  manual.canceled = compact_range_options.canceled;
  // For universal compaction, we enforce every manual compaction to compact
  // all files.
  if (begin == nullptr ||
@@ -2759,6 +2767,14 @@ void DBImpl::BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction,

    // See if there's more work to be done
    MaybeScheduleFlushOrCompaction();
+
+    if (prepicked_compaction != nullptr &&
+        prepicked_compaction->task_token != nullptr) {
+      // Releasing task tokens affects the DB state, so must be done before we
+      // potentially signal the DB close process to proceed below.
+      prepicked_compaction->task_token->ReleaseOnce();
+    }
+
    if (made_progress ||
        (bg_compaction_scheduled_ == 0 &&
         bg_bottom_compaction_scheduled_ == 0) ||
@@ -2811,6 +2827,9 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
    } else if (is_manual &&
               manual_compaction_paused_.load(std::memory_order_acquire) > 0) {
      status = Status::Incomplete(Status::SubCode::kManualCompactionPaused);
+    } else if (is_manual && manual_compaction->canceled &&
+               manual_compaction->canceled->load(std::memory_order_acquire)) {
+      status = Status::Incomplete(Status::SubCode::kManualCompactionPaused);
    }
  } else {
    status = error_handler_.GetBGError();
@@ -2938,7 +2957,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
          c->column_family_data()
              ->current()
              ->storage_info()
-              ->ComputeCompactionScore(*(c->immutable_cf_options()),
+              ->ComputeCompactionScore(*(c->immutable_options()),
                                       *(c->mutable_cf_options()));
          AddToCompactionQueue(cfd);
          ++unscheduled_compactions_;
@@ -3122,8 +3141,9 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
    assert(is_snapshot_supported_ || snapshots_.empty());
    CompactionJob compaction_job(
        job_context->job_id, c.get(), immutable_db_options_,
-        file_options_for_compaction_, versions_.get(), &shutting_down_,
-        preserve_deletes_seqnum_.load(), log_buffer, directories_.GetDbDir(),
+        mutable_db_options_, file_options_for_compaction_, versions_.get(),
+        &shutting_down_, preserve_deletes_seqnum_.load(), log_buffer,
+        directories_.GetDbDir(),
        GetDataDir(c->column_family_data(), c->output_path_id()),
        GetDataDir(c->column_family_data(), 0), stats_, &mutex_,
        &error_handler_, snapshot_seqs, earliest_write_conflict_snapshot,
@@ -3131,7 +3151,8 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
        c->mutable_cf_options()->paranoid_file_checks,
        c->mutable_cf_options()->report_bg_io_stats, dbname_,
        &compaction_job_stats, thread_pri, io_tracer_,
-        is_manual ? &manual_compaction_paused_ : nullptr, db_id_,
+        is_manual ? &manual_compaction_paused_ : nullptr,
+        is_manual ? manual_compaction->canceled : nullptr, db_id_,
        db_session_id_, c->column_family_data()->GetFullHistoryTsLow());
    compaction_job.Prepare();

@@ -3211,7 +3232,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
      c->column_family_data()
          ->current()
          ->storage_info()
-          ->ComputeCompactionScore(*(c->immutable_cf_options()),
+          ->ComputeCompactionScore(*(c->immutable_options()),
                                   *(c->mutable_cf_options()));
      if (!cfd->queued_for_compaction()) {
        AddToCompactionQueue(cfd);
@@ -3352,7 +3373,7 @@ bool DBImpl::MCOverlap(ManualCompactionState* m, ManualCompactionState* m1) {
  if (m->cfd != m1->cfd) {
    return false;
  }
-  return true;
+  return false;
 }

 #ifndef ROCKSDB_LITE
@@ -3376,7 +3397,7 @@ void DBImpl::BuildCompactionJobInfo(
    for (const auto fmd : *c->inputs(i)) {
      const FileDescriptor& desc = fmd->fd;
      const uint64_t file_number = desc.GetNumber();
-      auto fn = TableFileName(c->immutable_cf_options()->cf_paths, file_number,
+      auto fn = TableFileName(c->immutable_options()->cf_paths, file_number,
                              desc.GetPathId());
      compaction_job_info->input_files.push_back(fn);
      compaction_job_info->input_file_infos.push_back(CompactionFileInfo{
@@ -3395,7 +3416,7 @@ void DBImpl::BuildCompactionJobInfo(
    const FileDescriptor& desc = meta.fd;
    const uint64_t file_number = desc.GetNumber();
    compaction_job_info->output_files.push_back(TableFileName(
-        c->immutable_cf_options()->cf_paths, file_number, desc.GetPathId()));
+        c->immutable_options()->cf_paths, file_number, desc.GetPathId()));
    compaction_job_info->output_file_infos.push_back(CompactionFileInfo{
        newf.first, file_number, meta.oldest_blob_file_number});
  }
@@ -1259,7 +1259,8 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& wal_numbers,
      // If there's no data in the WAL, or we flushed all the data, still
      // truncate the log file. If the process goes into a crash loop before
      // the file is deleted, the preallocated space will never get freed.
-      GetLogSizeAndMaybeTruncate(wal_numbers.back(), true, nullptr)
+      const bool truncate = !read_only;
+      GetLogSizeAndMaybeTruncate(wal_numbers.back(), truncate, nullptr)
          .PermitUncheckedError();
    }
  }
@@ -1292,7 +1293,7 @@ Status DBImpl::GetLogSizeAndMaybeTruncate(uint64_t wal_number, bool truncate,
      truncate_status = last_log->Close(IOOptions(), nullptr);
    }
    // Not a critical error if fail to truncate.
-    if (!truncate_status.ok()) {
+    if (!truncate_status.ok() && !truncate_status.IsNotSupported()) {
      ROCKS_LOG_WARN(immutable_db_options_.info_log,
                     "Failed to truncate log #%" PRIu64 ": %s", wal_number,
                     truncate_status.ToString().c_str());
@@ -1393,21 +1394,23 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
      }

      IOStatus io_s;
-      s = BuildTable(
-          dbname_, versions_.get(), immutable_db_options_, *cfd->ioptions(),
-          mutable_cf_options, file_options_for_compaction_, cfd->table_cache(),
-          iter.get(), std::move(range_del_iters), &meta, &blob_file_additions,
-          cfd->internal_comparator(), cfd->int_tbl_prop_collector_factories(),
-          cfd->GetID(), cfd->GetName(), snapshot_seqs,
-          earliest_write_conflict_snapshot, snapshot_checker,
+      TableBuilderOptions tboptions(
+          *cfd->ioptions(), mutable_cf_options, cfd->internal_comparator(),
+          cfd->int_tbl_prop_collector_factories(),
          GetCompressionFlush(*cfd->ioptions(), mutable_cf_options),
-          mutable_cf_options.compression_opts, paranoid_file_checks,
-          cfd->internal_stats(), TableFileCreationReason::kRecovery, &io_s,
-          io_tracer_, &event_logger_, job_id, Env::IO_HIGH,
-          nullptr /* table_properties */, 0 /* level */, current_time,
-          0 /* oldest_key_time */, write_hint, 0 /* file_creation_time */,
-          db_id_, db_session_id_, nullptr /*full_history_ts_low*/,
-          &blob_callback_);
+          mutable_cf_options.compression_opts, cfd->GetID(), cfd->GetName(),
+          0 /* level */, false /* is_bottommost */,
+          TableFileCreationReason::kRecovery, current_time,
+          0 /* oldest_key_time */, 0 /* file_creation_time */, db_id_,
+          db_session_id_, 0 /* target_file_size */, meta.fd.GetNumber());
+      s = BuildTable(
+          dbname_, versions_.get(), immutable_db_options_, tboptions,
+          file_options_for_compaction_, cfd->table_cache(), iter.get(),
+          std::move(range_del_iters), &meta, &blob_file_additions,
+          snapshot_seqs, earliest_write_conflict_snapshot, snapshot_checker,
+          paranoid_file_checks, cfd->internal_stats(), &io_s, io_tracer_,
+          &event_logger_, job_id, Env::IO_HIGH, nullptr /* table_properties */,
+          write_hint, nullptr /*full_history_ts_low*/, &blob_callback_);
      LogFlush(immutable_db_options_.info_log);
      ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
                      "[%s] [WriteLevel0TableForRecovery]"
@@ -11,6 +11,7 @@
 #include "db/merge_context.h"
 #include "logging/auto_roll_logger.h"
 #include "monitoring/perf_context_imp.h"
+#include "rocksdb/configurable.h"
 #include "util/cast_util.h"

 namespace ROCKSDB_NAMESPACE {
@@ -726,11 +727,11 @@ Status DBImplSecondary::CompactWithoutInstallation(
  const int job_id = next_job_id_.fetch_add(1);

  CompactionServiceCompactionJob compaction_job(
-      job_id, c.get(), immutable_db_options_, file_options_for_compaction_,
-      versions_.get(), &shutting_down_, &log_buffer, output_dir.get(), stats_,
-      &mutex_, &error_handler_, input.snapshots, table_cache_, &event_logger_,
-      dbname_, io_tracer_, db_id_, db_session_id_, secondary_path_, input,
-      result);
+      job_id, c.get(), immutable_db_options_, mutable_db_options_,
+      file_options_for_compaction_, versions_.get(), &shutting_down_,
+      &log_buffer, output_dir.get(), stats_, &mutex_, &error_handler_,
+      input.snapshots, table_cache_, &event_logger_, dbname_, io_tracer_,
+      db_id_, db_session_id_, secondary_path_, input, result);

  mutex_.Unlock();
  s = compaction_job.Run();
@@ -742,6 +743,79 @@ Status DBImplSecondary::CompactWithoutInstallation(
  c->ReleaseCompactionFiles(s);
  c.reset();

+  TEST_SYNC_POINT_CALLBACK("DBImplSecondary::CompactWithoutInstallation::End",
+                           &s);
+  result->status = s;
+  return s;
+}
+
+Status DB::OpenAndCompact(
+    const std::string& name, const std::string& output_directory,
+    const std::string& input, std::string* result,
+    const CompactionServiceOptionsOverride& override_options) {
+  CompactionServiceInput compaction_input;
+  Status s = CompactionServiceInput::Read(input, &compaction_input);
+  if (!s.ok()) {
+    return s;
+  }
+
+  compaction_input.db_options.max_open_files = -1;
+  compaction_input.db_options.compaction_service = nullptr;
+  if (compaction_input.db_options.statistics) {
+    compaction_input.db_options.statistics.reset();
+  }
+  compaction_input.db_options.env = override_options.env;
+  compaction_input.db_options.file_checksum_gen_factory =
+      override_options.file_checksum_gen_factory;
+  compaction_input.column_family.options.comparator =
+      override_options.comparator;
+  compaction_input.column_family.options.merge_operator =
+      override_options.merge_operator;
+  compaction_input.column_family.options.compaction_filter =
+      override_options.compaction_filter;
+  compaction_input.column_family.options.compaction_filter_factory =
+      override_options.compaction_filter_factory;
+  compaction_input.column_family.options.prefix_extractor =
+      override_options.prefix_extractor;
+  compaction_input.column_family.options.table_factory =
+      override_options.table_factory;
+  compaction_input.column_family.options.sst_partitioner_factory =
+      override_options.sst_partitioner_factory;
+
+  std::vector<ColumnFamilyDescriptor> column_families;
+  column_families.push_back(compaction_input.column_family);
+  // TODO: we have to open default CF, because of an implementation limitation,
+  // currently we just use the same CF option from input, which is not collect
+  // and open may fail.
+  if (compaction_input.column_family.name != kDefaultColumnFamilyName) {
+    column_families.emplace_back(kDefaultColumnFamilyName,
+                                 compaction_input.column_family.options);
+  }
+
+  DB* db;
+  std::vector<ColumnFamilyHandle*> handles;
+
+  s = DB::OpenAsSecondary(compaction_input.db_options, name, output_directory,
+                          column_families, &handles, &db);
+  if (!s.ok()) {
+    return s;
+  }
+
+  CompactionServiceResult compaction_result;
+  DBImplSecondary* db_secondary = static_cast_with_check<DBImplSecondary>(db);
+  assert(handles.size() > 0);
+  s = db_secondary->CompactWithoutInstallation(handles[0], compaction_input,
+                                               &compaction_result);
+
+  Status serialization_status = compaction_result.Write(result);
+
+  for (auto& handle : handles) {
+    delete handle;
+  }
+  delete db;
+  if (s.ok()) {
+    return serialization_status;
+  }
  return s;
 }

@@ -160,8 +160,8 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
    RecordTick(stats_, WRITE_WITH_WAL);
  }

-  StopWatch write_sw(immutable_db_options_.clock,
-                     immutable_db_options_.statistics.get(), DB_WRITE);
+  StopWatch write_sw(immutable_db_options_.clock, immutable_db_options_.stats,
+                     DB_WRITE);

  write_thread_.JoinBatchGroup(&w);
  if (w.state == WriteThread::STATE_PARALLEL_MEMTABLE_WRITER) {
@@ -251,6 +251,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
      write_thread_.EnterAsBatchGroupLeader(&w, &write_group);

  IOStatus io_s;
+  Status pre_release_cb_status;
  if (status.ok()) {
    // Rules for when we can update the memtable concurrently
    // 1. supported by memtable
@@ -361,7 +362,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
              writer->sequence, disable_memtable, writer->log_used, index++,
              pre_release_callback_cnt);
          if (!ws.ok()) {
-            status = ws;
+            status = pre_release_cb_status = ws;
            break;
          }
        }
@@ -414,10 +415,13 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,

  if (!w.CallbackFailed()) {
    if (!io_s.ok()) {
+      assert(pre_release_cb_status.ok());
      IOStatusCheck(io_s);
    } else {
-      WriteStatusCheck(status);
+      WriteStatusCheck(pre_release_cb_status);
    }
+  } else {
+    assert(io_s.ok() && pre_release_cb_status.ok());
  }

  if (need_log_sync) {
@@ -466,8 +470,8 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
                                  uint64_t* log_used, uint64_t log_ref,
                                  bool disable_memtable, uint64_t* seq_used) {
  PERF_TIMER_GUARD(write_pre_and_post_process_time);
-  StopWatch write_sw(immutable_db_options_.clock,
-                     immutable_db_options_.statistics.get(), DB_WRITE);
+  StopWatch write_sw(immutable_db_options_.clock, immutable_db_options_.stats,
+                     DB_WRITE);

  WriteContext write_context;

@@ -623,8 +627,8 @@ Status DBImpl::UnorderedWriteMemtable(const WriteOptions& write_options,
                                      SequenceNumber seq,
                                      const size_t sub_batch_cnt) {
  PERF_TIMER_GUARD(write_pre_and_post_process_time);
-  StopWatch write_sw(immutable_db_options_.clock,
-                     immutable_db_options_.statistics.get(), DB_WRITE);
+  StopWatch write_sw(immutable_db_options_.clock, immutable_db_options_.stats,
+                     DB_WRITE);

  WriteThread::Writer w(write_options, my_batch, callback, log_ref,
                        false /*disable_memtable*/);
@@ -679,8 +683,8 @@ Status DBImpl::WriteImplWALOnly(
  WriteThread::Writer w(write_options, my_batch, callback, log_ref,
                        disable_memtable, sub_batch_cnt, pre_release_callback);
  RecordTick(stats_, WRITE_WITH_WAL);
-  StopWatch write_sw(immutable_db_options_.clock,
-                     immutable_db_options_.statistics.get(), DB_WRITE);
+  StopWatch write_sw(immutable_db_options_.clock, immutable_db_options_.stats,
+                     DB_WRITE);

  write_thread->JoinBatchGroup(&w);
  assert(w.state != WriteThread::STATE_PARALLEL_MEMTABLE_WRITER);
@@ -36,7 +36,7 @@
 namespace ROCKSDB_NAMESPACE {

 DBIter::DBIter(Env* _env, const ReadOptions& read_options,
-               const ImmutableCFOptions& cf_options,
+               const ImmutableOptions& ioptions,
               const MutableCFOptions& mutable_cf_options,
               const Comparator* cmp, InternalIterator* iter,
               const Version* version, SequenceNumber s, bool arena_mode,
@@ -45,15 +45,15 @@ DBIter::DBIter(Env* _env, const ReadOptions& read_options,
               ColumnFamilyData* cfd, bool expose_blob_index)
    : prefix_extractor_(mutable_cf_options.prefix_extractor.get()),
      env_(_env),
-      clock_(cf_options.clock),
-      logger_(cf_options.info_log),
+      clock_(ioptions.clock),
+      logger_(ioptions.logger),
      user_comparator_(cmp),
-      merge_operator_(cf_options.merge_operator.get()),
+      merge_operator_(ioptions.merge_operator.get()),
      iter_(iter),
      version_(version),
      read_callback_(read_callback),
      sequence_(s),
-      statistics_(cf_options.statistics),
+      statistics_(ioptions.stats),
      max_skip_(max_sequential_skip_in_iterations),
      max_skippable_internal_keys_(read_options.max_skippable_internal_keys),
      num_internal_keys_skipped_(0),
@@ -75,7 +75,7 @@ DBIter::DBIter(Env* _env, const ReadOptions& read_options,
      expose_blob_index_(expose_blob_index),
      is_blob_(false),
      arena_mode_(arena_mode),
-      range_del_agg_(&cf_options.internal_comparator, s),
+      range_del_agg_(&ioptions.internal_comparator, s),
      db_impl_(db_impl),
      cfd_(cfd),
      start_seqnum_(read_options.iter_start_seqnum),
@@ -576,12 +576,8 @@ bool DBIter::MergeValuesNewToOld() {
      // hit a put, merge the put value with operands and store the
      // final result in saved_value_. We are done!
      const Slice val = iter_.value();
-      Status s = MergeHelper::TimedFullMerge(
-          merge_operator_, ikey.user_key, &val, merge_context_.GetOperands(),
-          &saved_value_, logger_, statistics_, clock_, &pinned_value_, true);
+      Status s = Merge(&val, ikey.user_key);
      if (!s.ok()) {
-        valid_ = false;
-        status_ = s;
        return false;
      }
      // iter_ is positioned after put
@@ -598,9 +594,31 @@ bool DBIter::MergeValuesNewToOld() {
          iter_.value(), iter_.iter()->IsValuePinned() /* operand_pinned */);
      PERF_COUNTER_ADD(internal_merge_count, 1);
    } else if (kTypeBlobIndex == ikey.type) {
-      status_ = Status::NotSupported("BlobDB does not support merge operator.");
-      valid_ = false;
-      return false;
+      if (expose_blob_index_) {
+        status_ =
+            Status::NotSupported("BlobDB does not support merge operator.");
+        valid_ = false;
+        return false;
+      }
+      // hit a put, merge the put value with operands and store the
+      // final result in saved_value_. We are done!
+      if (!SetBlobValueIfNeeded(ikey.user_key, iter_.value())) {
+        return false;
+      }
+      valid_ = true;
+      const Slice blob_value = value();
+      Status s = Merge(&blob_value, ikey.user_key);
+      if (!s.ok()) {
+        return false;
+      }
+      is_blob_ = false;
+      // iter_ is positioned after put
+      iter_.Next();
+      if (!iter_.status().ok()) {
+        valid_ = false;
+        return false;
+      }
+      return true;
    } else {
      valid_ = false;
      status_ = Status::Corruption(
@@ -619,16 +637,10 @@ bool DBIter::MergeValuesNewToOld() {
  // a deletion marker.
  // feed null as the existing value to the merge operator, such that
  // client can differentiate this scenario and do things accordingly.
-  Status s = MergeHelper::TimedFullMerge(
-      merge_operator_, saved_key_.GetUserKey(), nullptr,
-      merge_context_.GetOperands(), &saved_value_, logger_, statistics_, clock_,
-      &pinned_value_, true);
+  Status s = Merge(nullptr, saved_key_.GetUserKey());
  if (!s.ok()) {
-    valid_ = false;
-    status_ = s;
    return false;
  }
-
  assert(status_.ok());
  return true;
 }
@@ -931,21 +943,36 @@ bool DBIter::FindValueForCurrentKey() {
      if (last_not_merge_type == kTypeDeletion ||
          last_not_merge_type == kTypeSingleDeletion ||
          last_not_merge_type == kTypeRangeDeletion) {
-        s = MergeHelper::TimedFullMerge(
-            merge_operator_, saved_key_.GetUserKey(), nullptr,
-            merge_context_.GetOperands(), &saved_value_, logger_, statistics_,
-            clock_, &pinned_value_, true);
+        s = Merge(nullptr, saved_key_.GetUserKey());
+        if (!s.ok()) {
+          return false;
+        }
+        return true;
      } else if (last_not_merge_type == kTypeBlobIndex) {
-        status_ =
-            Status::NotSupported("BlobDB does not support merge operator.");
-        valid_ = false;
-        return false;
+        if (expose_blob_index_) {
+          status_ =
+              Status::NotSupported("BlobDB does not support merge operator.");
+          valid_ = false;
+          return false;
+        }
+        if (!SetBlobValueIfNeeded(saved_key_.GetUserKey(), pinned_value_)) {
+          return false;
+        }
+        valid_ = true;
+        const Slice blob_value = value();
+        s = Merge(&blob_value, saved_key_.GetUserKey());
+        if (!s.ok()) {
+          return false;
+        }
+        is_blob_ = false;
+        return true;
      } else {
        assert(last_not_merge_type == kTypeValue);
-        s = MergeHelper::TimedFullMerge(
-            merge_operator_, saved_key_.GetUserKey(), &pinned_value_,
-            merge_context_.GetOperands(), &saved_value_, logger_, statistics_,
-            clock_, &pinned_value_, true);
+        s = Merge(&pinned_value_, saved_key_.GetUserKey());
+        if (!s.ok()) {
+          return false;
+        }
+        return true;
      }
      break;
    case kTypeValue:
@@ -955,7 +982,6 @@ bool DBIter::FindValueForCurrentKey() {
      if (!SetBlobValueIfNeeded(saved_key_.GetUserKey(), pinned_value_)) {
        return false;
      }
-
      break;
    default:
      valid_ = false;
@@ -1095,25 +1121,33 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {

    if (ikey.type == kTypeValue) {
      const Slice val = iter_.value();
-      Status s = MergeHelper::TimedFullMerge(
-          merge_operator_, saved_key_.GetUserKey(), &val,
-          merge_context_.GetOperands(), &saved_value_, logger_, statistics_,
-          clock_, &pinned_value_, true);
+      Status s = Merge(&val, saved_key_.GetUserKey());
      if (!s.ok()) {
-        valid_ = false;
-        status_ = s;
        return false;
      }
-      valid_ = true;
      return true;
    } else if (ikey.type == kTypeMerge) {
      merge_context_.PushOperand(
          iter_.value(), iter_.iter()->IsValuePinned() /* operand_pinned */);
      PERF_COUNTER_ADD(internal_merge_count, 1);
    } else if (ikey.type == kTypeBlobIndex) {
-      status_ = Status::NotSupported("BlobDB does not support merge operator.");
-      valid_ = false;
-      return false;
+      if (expose_blob_index_) {
+        status_ =
+            Status::NotSupported("BlobDB does not support merge operator.");
+        valid_ = false;
+        return false;
+      }
+      if (!SetBlobValueIfNeeded(ikey.user_key, iter_.value())) {
+        return false;
+      }
+      valid_ = true;
+      const Slice blob_value = value();
+      Status s = Merge(&blob_value, saved_key_.GetUserKey());
+      if (!s.ok()) {
+        return false;
+      }
+      is_blob_ = false;
+      return true;
    } else {
      valid_ = false;
      status_ = Status::Corruption(
@@ -1123,13 +1157,8 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
    }
  }

-  Status s = MergeHelper::TimedFullMerge(
-      merge_operator_, saved_key_.GetUserKey(), nullptr,
-      merge_context_.GetOperands(), &saved_value_, logger_, statistics_, clock_,
-      &pinned_value_, true);
+  Status s = Merge(nullptr, saved_key_.GetUserKey());
  if (!s.ok()) {
-    valid_ = false;
-    status_ = s;
    return false;
  }

@@ -1152,6 +1181,19 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
  return true;
 }

+Status DBIter::Merge(const Slice* val, const Slice& user_key) {
+  Status s = MergeHelper::TimedFullMerge(
+      merge_operator_, user_key, val, merge_context_.GetOperands(),
+      &saved_value_, logger_, statistics_, clock_, &pinned_value_, true);
+  if (!s.ok()) {
+    valid_ = false;
+    status_ = s;
+    return s;
+  }
+  valid_ = true;
+  return s;
+}
+
 // Move backwards until the key smaller than saved_key_.
 // Changes valid_ only if return value is false.
 bool DBIter::FindUserKeyBeforeSavedKey() {
@@ -1536,7 +1578,7 @@ void DBIter::SeekToLast() {
 }

 Iterator* NewDBIterator(Env* env, const ReadOptions& read_options,
-                        const ImmutableCFOptions& cf_options,
+                        const ImmutableOptions& ioptions,
                        const MutableCFOptions& mutable_cf_options,
                        const Comparator* user_key_comparator,
                        InternalIterator* internal_iter, const Version* version,
@@ -1545,7 +1587,7 @@ Iterator* NewDBIterator(Env* env, const ReadOptions& read_options,
                        ReadCallback* read_callback, DBImpl* db_impl,
                        ColumnFamilyData* cfd, bool expose_blob_index) {
  DBIter* db_iter =
-      new DBIter(env, read_options, cf_options, mutable_cf_options,
+      new DBIter(env, read_options, ioptions, mutable_cf_options,
                 user_key_comparator, internal_iter, version, sequence, false,
                 max_sequential_skip_in_iterations, read_callback, db_impl, cfd,
                 expose_blob_index);
@@ -114,7 +114,7 @@ class DBIter final : public Iterator {
  };

  DBIter(Env* _env, const ReadOptions& read_options,
-         const ImmutableCFOptions& cf_options,
+         const ImmutableOptions& ioptions,
         const MutableCFOptions& mutable_cf_options, const Comparator* cmp,
         InternalIterator* iter, const Version* version, SequenceNumber s,
         bool arena_mode, uint64_t max_sequential_skip_in_iterations,
@@ -299,6 +299,8 @@ class DBIter final : public Iterator {
  // index when using the integrated BlobDB implementation.
  bool SetBlobValueIfNeeded(const Slice& user_key, const Slice& blob_index);

+  Status Merge(const Slice* val, const Slice& user_key);
+
  const SliceTransform* prefix_extractor_;
  Env* const env_;
  SystemClock* clock_;
@@ -383,8 +385,7 @@ class DBIter final : public Iterator {
 // "*internal_iter") that were live at the specified `sequence` number
 // into appropriate user keys.
 extern Iterator* NewDBIterator(
-    Env* env, const ReadOptions& read_options,
-    const ImmutableCFOptions& cf_options,
+    Env* env, const ReadOptions& read_options, const ImmutableOptions& ioptions,
    const MutableCFOptions& mutable_cf_options,
    const Comparator* user_key_comparator, InternalIterator* internal_iter,
    const Version* version, const SequenceNumber& sequence,
@@ -511,7 +511,7 @@ TEST_F(DBIteratorStressTest, StressTest) {
                      target_hidden_fraction;
                  internal_iter->trace = trace;
                  db_iter.reset(NewDBIterator(
-                      env_, ropt, ImmutableCFOptions(options),
+                      env_, ropt, ImmutableOptions(options),
                      MutableCFOptions(options), BytewiseComparator(),
                      internal_iter, nullptr /* version */, sequence,
                      options.max_sequential_skip_in_iterations,
@@ -237,7 +237,7 @@ class DBIteratorTest : public testing::Test {

 TEST_F(DBIteratorTest, DBIteratorPrevNext) {
  Options options;
-  ImmutableCFOptions cf_options = ImmutableCFOptions(options);
+  ImmutableOptions ioptions = ImmutableOptions(options);
  MutableCFOptions mutable_cf_options = MutableCFOptions(options);
  {
    TestIterator* internal_iter = new TestIterator(BytewiseComparator());
@@ -252,7 +252,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    ReadOptions ro;
    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 10 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -286,7 +286,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    ReadOptions ro;
    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 10 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -314,7 +314,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
    ro.iterate_upper_bound = &prefix;

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 10 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -348,7 +348,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
    ro.iterate_upper_bound = &prefix;

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 10 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -385,7 +385,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
    ro.iterate_upper_bound = &prefix;

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 10 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -417,7 +417,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
    ro.iterate_upper_bound = &prefix;

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 7 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -456,7 +456,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
    ro.iterate_upper_bound = &prefix;

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 4 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -483,7 +483,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
    ro.iterate_upper_bound = &prefix;

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 10 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -508,7 +508,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
    ro.iterate_upper_bound = &prefix;

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 10 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -545,7 +545,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
    ro.iterate_upper_bound = &prefix;

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 7 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -576,7 +576,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    ReadOptions ro;
    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 10 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -621,7 +621,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    ReadOptions ro;
    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 2 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -654,7 +654,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    ReadOptions ro;
    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 10 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -677,7 +677,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

 TEST_F(DBIteratorTest, DBIteratorEmpty) {
  Options options;
-  ImmutableCFOptions cf_options = ImmutableCFOptions(options);
+  ImmutableOptions ioptions = ImmutableOptions(options);
  MutableCFOptions mutable_cf_options = MutableCFOptions(options);
  ReadOptions ro;

@@ -686,7 +686,7 @@ TEST_F(DBIteratorTest, DBIteratorEmpty) {
    internal_iter->Finish();

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 0 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -700,7 +700,7 @@ TEST_F(DBIteratorTest, DBIteratorEmpty) {
    internal_iter->Finish();

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 0 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -725,7 +725,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkipCountSkips) {
  internal_iter->Finish();

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
-      env_, ro, ImmutableCFOptions(options), MutableCFOptions(options),
+      env_, ro, ImmutableOptions(options), MutableCFOptions(options),
      BytewiseComparator(), internal_iter, nullptr /* version */,
      2 /* sequence */, options.max_sequential_skip_in_iterations,
      nullptr /* read_callback */));
@@ -757,7 +757,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
  ReadOptions ro;
  Options options;
  options.merge_operator = MergeOperators::CreateFromStringId("stringappend");
-  ImmutableCFOptions cf_options = ImmutableCFOptions(options);
+  ImmutableOptions ioptions = ImmutableOptions(options);
  MutableCFOptions mutable_cf_options = MutableCFOptions(options);

  {
@@ -772,7 +772,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {

      options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
      std::unique_ptr<Iterator> db_iter(NewDBIterator(
-          env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+          env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
          internal_iter, nullptr /* version */, i + 2 /* sequence */,
          options.max_sequential_skip_in_iterations,
          nullptr /* read_callback */));
@@ -810,7 +810,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
      internal_iter->Finish();

      std::unique_ptr<Iterator> db_iter(NewDBIterator(
-          env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+          env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
          internal_iter, nullptr /* version */, i + 2 /* sequence */,
          options.max_sequential_skip_in_iterations,
          nullptr /* read_callback */));
@@ -841,7 +841,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
      internal_iter->Finish();

      std::unique_ptr<Iterator> db_iter(NewDBIterator(
-          env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+          env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
          internal_iter, nullptr /* version */, 202 /* sequence */,
          options.max_sequential_skip_in_iterations,
          nullptr /* read_callback */));
@@ -876,7 +876,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
      internal_iter->AddPut("c", "200");
      internal_iter->Finish();
      std::unique_ptr<Iterator> db_iter(NewDBIterator(
-          env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+          env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
          internal_iter, nullptr /* version */, i /* sequence */,
          options.max_sequential_skip_in_iterations,
          nullptr /* read_callback */));
@@ -896,7 +896,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
    internal_iter->AddPut("c", "200");
    internal_iter->Finish();
    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 200 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -934,7 +934,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
      internal_iter->Finish();

      std::unique_ptr<Iterator> db_iter(NewDBIterator(
-          env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+          env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
          internal_iter, nullptr /* version */, i + 2 /* sequence */,
          options.max_sequential_skip_in_iterations,
          nullptr /* read_callback */));
@@ -971,7 +971,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
      internal_iter->Finish();

      std::unique_ptr<Iterator> db_iter(NewDBIterator(
-          env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+          env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
          internal_iter, nullptr /* version */, i + 2 /* sequence */,
          options.max_sequential_skip_in_iterations,
          nullptr /* read_callback */));
@@ -1004,7 +1004,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {

 TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
  Options options;
-  ImmutableCFOptions cf_options = ImmutableCFOptions(options);
+  ImmutableOptions ioptions = ImmutableOptions(options);
  MutableCFOptions mutable_cf_options = MutableCFOptions(options);
  ReadOptions ro;

@@ -1023,7 +1023,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {

    ro.max_skippable_internal_keys = 0;
    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 10 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -1071,7 +1071,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {

    ro.max_skippable_internal_keys = 2;
    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 10 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -1117,7 +1117,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {

    ro.max_skippable_internal_keys = 2;
    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 10 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -1157,7 +1157,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {

    ro.max_skippable_internal_keys = 2;
    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 10 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -1194,7 +1194,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {

    ro.max_skippable_internal_keys = 2;
    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 10 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -1226,7 +1226,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {

    ro.max_skippable_internal_keys = 2;
    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 10 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -1265,7 +1265,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {

    ro.max_skippable_internal_keys = 2;
    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 10 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -1304,7 +1304,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {

      ro.max_skippable_internal_keys = i;
      std::unique_ptr<Iterator> db_iter(NewDBIterator(
-          env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+          env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
          internal_iter, nullptr /* version */, 2 * i + 1 /* sequence */,
          options.max_sequential_skip_in_iterations,
          nullptr /* read_callback */));
@@ -1359,7 +1359,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
      options.max_sequential_skip_in_iterations = 1000;
      ro.max_skippable_internal_keys = i;
      std::unique_ptr<Iterator> db_iter(NewDBIterator(
-          env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+          env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
          internal_iter, nullptr /* version */, 2 * i + 1 /* sequence */,
          options.max_sequential_skip_in_iterations,
          nullptr /* read_callback */));
@@ -1399,7 +1399,7 @@ TEST_F(DBIteratorTest, DBIterator1) {
  internal_iter->Finish();

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
-      env_, ro, ImmutableCFOptions(options), MutableCFOptions(options),
+      env_, ro, ImmutableOptions(options), MutableCFOptions(options),
      BytewiseComparator(), internal_iter, nullptr /* version */,
      1 /* sequence */, options.max_sequential_skip_in_iterations,
      nullptr /* read_callback */));
@@ -1428,7 +1428,7 @@ TEST_F(DBIteratorTest, DBIterator2) {
  internal_iter->Finish();

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
-      env_, ro, ImmutableCFOptions(options), MutableCFOptions(options),
+      env_, ro, ImmutableOptions(options), MutableCFOptions(options),
      BytewiseComparator(), internal_iter, nullptr /* version */,
      0 /* sequence */, options.max_sequential_skip_in_iterations,
      nullptr /* read_callback */));
@@ -1454,7 +1454,7 @@ TEST_F(DBIteratorTest, DBIterator3) {
  internal_iter->Finish();

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
-      env_, ro, ImmutableCFOptions(options), MutableCFOptions(options),
+      env_, ro, ImmutableOptions(options), MutableCFOptions(options),
      BytewiseComparator(), internal_iter, nullptr /* version */,
      2 /* sequence */, options.max_sequential_skip_in_iterations,
      nullptr /* read_callback */));
@@ -1480,7 +1480,7 @@ TEST_F(DBIteratorTest, DBIterator4) {
  internal_iter->Finish();

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
-      env_, ro, ImmutableCFOptions(options), MutableCFOptions(options),
+      env_, ro, ImmutableOptions(options), MutableCFOptions(options),
      BytewiseComparator(), internal_iter, nullptr /* version */,
      4 /* sequence */, options.max_sequential_skip_in_iterations,
      nullptr /* read_callback */));
@@ -1500,7 +1500,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
  ReadOptions ro;
  Options options;
  options.merge_operator = MergeOperators::CreateFromStringId("stringappend");
-  ImmutableCFOptions cf_options = ImmutableCFOptions(options);
+  ImmutableOptions ioptions = ImmutableOptions(options);
  MutableCFOptions mutable_cf_options = MutableCFOptions(options);

  {
@@ -1515,7 +1515,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
    internal_iter->Finish();

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 0 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -1539,7 +1539,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
    internal_iter->Finish();

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 1 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -1563,7 +1563,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
    internal_iter->Finish();

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 2 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -1587,7 +1587,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
    internal_iter->Finish();

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 3 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -1611,7 +1611,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
    internal_iter->Finish();

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 4 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -1635,7 +1635,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
    internal_iter->Finish();

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 5 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -1659,7 +1659,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
    internal_iter->Finish();

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 6 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -1681,7 +1681,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
    internal_iter->AddPut("b", "val_b");
    internal_iter->Finish();
    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 10 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -1698,7 +1698,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
  ReadOptions ro;
  Options options;
  options.merge_operator = MergeOperators::CreateFromStringId("stringappend");
-  ImmutableCFOptions cf_options = ImmutableCFOptions(options);
+  ImmutableOptions ioptions = ImmutableOptions(options);
  MutableCFOptions mutable_cf_options = MutableCFOptions(options);

  {
@@ -1713,7 +1713,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
    internal_iter->Finish();

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 0 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -1737,7 +1737,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
    internal_iter->Finish();

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 1 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -1761,7 +1761,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
    internal_iter->Finish();

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 2 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -1785,7 +1785,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
    internal_iter->Finish();

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 3 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -1805,7 +1805,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
    internal_iter->Finish();

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 4 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -1829,7 +1829,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
    internal_iter->Finish();

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 5 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -1853,7 +1853,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
    internal_iter->Finish();

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 6 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -1870,7 +1870,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
  ReadOptions ro;
  Options options;
  options.merge_operator = MergeOperators::CreateFromStringId("stringappend");
-  ImmutableCFOptions cf_options = ImmutableCFOptions(options);
+  ImmutableOptions ioptions = ImmutableOptions(options);
  MutableCFOptions mutable_cf_options = MutableCFOptions(options);

  {
@@ -1897,7 +1897,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
    internal_iter->Finish();

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 0 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -1933,7 +1933,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
    internal_iter->Finish();

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 2 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -1975,7 +1975,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
    internal_iter->Finish();

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 4 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -2017,7 +2017,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
    internal_iter->Finish();

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 5 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -2064,7 +2064,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
    internal_iter->Finish();

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 6 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -2112,7 +2112,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
    internal_iter->Finish();

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 7 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -2154,7 +2154,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
    internal_iter->Finish();

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 9 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -2202,7 +2202,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
    internal_iter->Finish();

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 13 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -2251,7 +2251,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
    internal_iter->Finish();

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, cf_options, mutable_cf_options, BytewiseComparator(),
+        env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
        internal_iter, nullptr /* version */, 14 /* sequence */,
        options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -2283,7 +2283,7 @@ TEST_F(DBIteratorTest, DBIterator8) {
  internal_iter->Finish();

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
-      env_, ro, ImmutableCFOptions(options), MutableCFOptions(options),
+      env_, ro, ImmutableOptions(options), MutableCFOptions(options),
      BytewiseComparator(), internal_iter, nullptr /* version */,
      10 /* sequence */, options.max_sequential_skip_in_iterations,
      nullptr /* read_callback */));
@@ -2315,7 +2315,7 @@ TEST_F(DBIteratorTest, DBIterator9) {
    internal_iter->Finish();

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, ImmutableCFOptions(options), MutableCFOptions(options),
+        env_, ro, ImmutableOptions(options), MutableCFOptions(options),
        BytewiseComparator(), internal_iter, nullptr /* version */,
        10 /* sequence */, options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -2383,7 +2383,7 @@ TEST_F(DBIteratorTest, DBIterator10) {
  internal_iter->Finish();

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
-      env_, ro, ImmutableCFOptions(options), MutableCFOptions(options),
+      env_, ro, ImmutableOptions(options), MutableCFOptions(options),
      BytewiseComparator(), internal_iter, nullptr /* version */,
      10 /* sequence */, options.max_sequential_skip_in_iterations,
      nullptr /* read_callback */));
@@ -2424,7 +2424,7 @@ TEST_F(DBIteratorTest, SeekToLastOccurrenceSeq0) {
  internal_iter->Finish();

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
-      env_, ro, ImmutableCFOptions(options), MutableCFOptions(options),
+      env_, ro, ImmutableOptions(options), MutableCFOptions(options),
      BytewiseComparator(), internal_iter, nullptr /* version */,
      10 /* sequence */, 0 /* force seek */, nullptr /* read_callback */));
  db_iter->SeekToFirst();
@@ -2453,7 +2453,7 @@ TEST_F(DBIteratorTest, DBIterator11) {
  internal_iter->Finish();

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
-      env_, ro, ImmutableCFOptions(options), MutableCFOptions(options),
+      env_, ro, ImmutableOptions(options), MutableCFOptions(options),
      BytewiseComparator(), internal_iter, nullptr /* version */,
      1 /* sequence */, options.max_sequential_skip_in_iterations,
      nullptr /* read_callback */));
@@ -2481,7 +2481,7 @@ TEST_F(DBIteratorTest, DBIterator12) {
  internal_iter->Finish();

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
-      env_, ro, ImmutableCFOptions(options), MutableCFOptions(options),
+      env_, ro, ImmutableOptions(options), MutableCFOptions(options),
      BytewiseComparator(), internal_iter, nullptr /* version */,
      10 /* sequence */, 0 /* force seek */, nullptr /* read_callback */));
  db_iter->SeekToLast();
@@ -2519,7 +2519,7 @@ TEST_F(DBIteratorTest, DBIterator13) {
  internal_iter->Finish();

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
-      env_, ro, ImmutableCFOptions(options), MutableCFOptions(options),
+      env_, ro, ImmutableOptions(options), MutableCFOptions(options),
      BytewiseComparator(), internal_iter, nullptr /* version */,
      2 /* sequence */, 3 /* max_sequential_skip_in_iterations */,
      nullptr /* read_callback */));
@@ -2549,7 +2549,7 @@ TEST_F(DBIteratorTest, DBIterator14) {
  internal_iter->Finish();

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
-      env_, ro, ImmutableCFOptions(options), MutableCFOptions(options),
+      env_, ro, ImmutableOptions(options), MutableCFOptions(options),
      BytewiseComparator(), internal_iter, nullptr /* version */,
      4 /* sequence */, 1 /* max_sequential_skip_in_iterations */,
      nullptr /* read_callback */));
@@ -2578,7 +2578,7 @@ TEST_F(DBIteratorTest, DBIteratorTestDifferentialSnapshots) {
    internal_iter->Finish();

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, ImmutableCFOptions(options), MutableCFOptions(options),
+        env_, ro, ImmutableOptions(options), MutableCFOptions(options),
        BytewiseComparator(), internal_iter, nullptr /* version */,
        13 /* sequence */, options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -2614,7 +2614,7 @@ TEST_F(DBIteratorTest, DBIteratorTestDifferentialSnapshots) {
    internal_iter->Finish();

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, ImmutableCFOptions(options), MutableCFOptions(options),
+        env_, ro, ImmutableOptions(options), MutableCFOptions(options),
        BytewiseComparator(), internal_iter, nullptr /* version */,
        13 /* sequence */, options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -2665,7 +2665,7 @@ class DBIterWithMergeIterTest : public testing::Test {
        NewMergingIterator(&icomp_, &child_iters[0], 2u);

    db_iter_.reset(NewDBIterator(
-        env_, ro_, ImmutableCFOptions(options_), MutableCFOptions(options_),
+        env_, ro_, ImmutableOptions(options_), MutableCFOptions(options_),
        BytewiseComparator(), merge_iter, nullptr /* version */,
        8 /* read data earlier than seqId 8 */,
        3 /* max iterators before reseek */, nullptr /* read_callback */));
@@ -3105,7 +3105,7 @@ TEST_F(DBIteratorTest, SeekPrefixTombstones) {

  ro.prefix_same_as_start = true;
  std::unique_ptr<Iterator> db_iter(NewDBIterator(
-      env_, ro, ImmutableCFOptions(options), MutableCFOptions(options),
+      env_, ro, ImmutableOptions(options), MutableCFOptions(options),
      BytewiseComparator(), internal_iter, nullptr /* version */,
      10 /* sequence */, options.max_sequential_skip_in_iterations,
      nullptr /* read_callback */));
@@ -3142,7 +3142,7 @@ TEST_F(DBIteratorTest, SeekToFirstLowerBound) {
    ro.iterate_lower_bound = &lower_bound;
    Options options;
    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ro, ImmutableCFOptions(options), MutableCFOptions(options),
+        env_, ro, ImmutableOptions(options), MutableCFOptions(options),
        BytewiseComparator(), internal_iter, nullptr /* version */,
        10 /* sequence */, options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */));
@@ -3182,7 +3182,7 @@ TEST_F(DBIteratorTest, PrevLowerBound) {
  ro.iterate_lower_bound = &lower_bound;
  Options options;
  std::unique_ptr<Iterator> db_iter(NewDBIterator(
-      env_, ro, ImmutableCFOptions(options), MutableCFOptions(options),
+      env_, ro, ImmutableOptions(options), MutableCFOptions(options),
      BytewiseComparator(), internal_iter, nullptr /* version */,
      10 /* sequence */, options.max_sequential_skip_in_iterations,
      nullptr /* read_callback */));
@@ -3211,7 +3211,7 @@ TEST_F(DBIteratorTest, SeekLessLowerBound) {
  ro.iterate_lower_bound = &lower_bound;
  Options options;
  std::unique_ptr<Iterator> db_iter(NewDBIterator(
-      env_, ro, ImmutableCFOptions(options), MutableCFOptions(options),
+      env_, ro, ImmutableOptions(options), MutableCFOptions(options),
      BytewiseComparator(), internal_iter, nullptr /* version */,
      10 /* sequence */, options.max_sequential_skip_in_iterations,
      nullptr /* read_callback */));
@@ -3237,10 +3237,10 @@ TEST_F(DBIteratorTest, ReverseToForwardWithDisappearingKeys) {
  internal_iter->Finish();

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
-      env_, ReadOptions(), ImmutableCFOptions(options),
-      MutableCFOptions(options), BytewiseComparator(), internal_iter,
-      nullptr /* version */, 10 /* sequence */,
-      options.max_sequential_skip_in_iterations, nullptr /* read_callback */));
+      env_, ReadOptions(), ImmutableOptions(options), MutableCFOptions(options),
+      BytewiseComparator(), internal_iter, nullptr /* version */,
+      10 /* sequence */, options.max_sequential_skip_in_iterations,
+      nullptr /* read_callback */));

  db_iter->SeekForPrev("a");
  ASSERT_TRUE(db_iter->Valid());
@@ -139,7 +139,7 @@ TEST_F(DBMemTableTest, DuplicateSeq) {
  InternalKeyComparator cmp(BytewiseComparator());
  auto factory = std::make_shared<SkipListFactory>();
  options.memtable_factory = factory;
-  ImmutableCFOptions ioptions(options);
+  ImmutableOptions ioptions(options);
  WriteBufferManager wb(options.db_write_buffer_size);
  MemTable* mem = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb,
                               kMaxSequenceNumber, 0 /* column_family_id */);
@@ -184,7 +184,7 @@ TEST_F(DBMemTableTest, DuplicateSeq) {
  // Test with InsertWithHint
  options.memtable_insert_with_hint_prefix_extractor.reset(
      new TestPrefixExtractor());  // which uses _ to extract the prefix
-  ioptions = ImmutableCFOptions(options);
+  ioptions = ImmutableOptions(options);
  mem = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb,
                     kMaxSequenceNumber, 0 /* column_family_id */);
  // Insert a duplicate key with _ in it
@@ -197,7 +197,7 @@ TEST_F(DBMemTableTest, DuplicateSeq) {

  // Test when InsertConcurrently will be invoked
  options.allow_concurrent_memtable_write = true;
-  ioptions = ImmutableCFOptions(options);
+  ioptions = ImmutableOptions(options);
  mem = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb,
                     kMaxSequenceNumber, 0 /* column_family_id */);
  MemTablePostProcessInfo post_process_info;
@@ -224,7 +224,7 @@ TEST_F(DBMemTableTest, ConcurrentMergeWrite) {
  auto factory = std::make_shared<SkipListFactory>();
  options.memtable_factory = factory;
  options.allow_concurrent_memtable_write = true;
-  ImmutableCFOptions ioptions(options);
+  ImmutableOptions ioptions(options);
  WriteBufferManager wb(options.db_write_buffer_size);
  MemTable* mem = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb,
                               kMaxSequenceNumber, 0 /* column_family_id */);
@@ -19,6 +19,28 @@

 namespace ROCKSDB_NAMESPACE {

+namespace {
+class LimitedStringAppendMergeOp : public StringAppendTESTOperator {
+ public:
+  LimitedStringAppendMergeOp(int limit, char delim)
+      : StringAppendTESTOperator(delim), limit_(limit) {}
+
+  const char* Name() const override {
+    return "DBMergeOperatorTest::LimitedStringAppendMergeOp";
+  }
+
+  bool ShouldMerge(const std::vector<Slice>& operands) const override {
+    if (operands.size() > 0 && limit_ > 0 && operands.size() >= limit_) {
+      return true;
+    }
+    return false;
+  }
+
+ private:
+  size_t limit_ = 0;
+};
+}  // namespace
+
 class DBMergeOperandTest : public DBTestBase {
 public:
  DBMergeOperandTest()
@@ -26,26 +48,6 @@ class DBMergeOperandTest : public DBTestBase {
 };

 TEST_F(DBMergeOperandTest, GetMergeOperandsBasic) {
-  class LimitedStringAppendMergeOp : public StringAppendTESTOperator {
-   public:
-    LimitedStringAppendMergeOp(int limit, char delim)
-        : StringAppendTESTOperator(delim), limit_(limit) {}
-
-    const char* Name() const override {
-      return "DBMergeOperatorTest::LimitedStringAppendMergeOp";
-    }
-
-    bool ShouldMerge(const std::vector<Slice>& operands) const override {
-      if (operands.size() > 0 && limit_ > 0 && operands.size() >= limit_) {
-        return true;
-      }
-      return false;
-    }
-
-   private:
-    size_t limit_ = 0;
-  };
-
  Options options;
  options.create_if_missing = true;
  // Use only the latest two merge operands.
@@ -214,7 +216,8 @@ TEST_F(DBMergeOperandTest, GetMergeOperandsBasic) {
  ASSERT_EQ(values[2], "dc");
  ASSERT_EQ(values[3], "ed");

-  // First 3 k5 values are in SST and next 4 k5 values are in Immutable Memtable
+  // First 3 k5 values are in SST and next 4 k5 values are in Immutable
+  // Memtable
  ASSERT_OK(Merge("k5", "who"));
  ASSERT_OK(Merge("k5", "am"));
  ASSERT_OK(Merge("k5", "i"));
@@ -232,6 +235,93 @@ TEST_F(DBMergeOperandTest, GetMergeOperandsBasic) {
  ASSERT_EQ(values[2], "am");
 }

+TEST_F(DBMergeOperandTest, BlobDBGetMergeOperandsBasic) {
+  Options options;
+  options.create_if_missing = true;
+  options.enable_blob_files = true;
+  options.min_blob_size = 0;
+  // Use only the latest two merge operands.
+  options.merge_operator = std::make_shared<LimitedStringAppendMergeOp>(2, ',');
+  options.env = env_;
+  Reopen(options);
+  int num_records = 4;
+  int number_of_operands = 0;
+  std::vector<PinnableSlice> values(num_records);
+  GetMergeOperandsOptions merge_operands_info;
+  merge_operands_info.expected_max_number_of_operands = num_records;
+
+  // All k1 values are in memtable.
+  ASSERT_OK(Put("k1", "x"));
+  ASSERT_OK(Merge("k1", "b"));
+  ASSERT_OK(Merge("k1", "c"));
+  ASSERT_OK(Merge("k1", "d"));
+  ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(),
+                                  "k1", values.data(), &merge_operands_info,
+                                  &number_of_operands));
+  ASSERT_EQ(values[0], "x");
+  ASSERT_EQ(values[1], "b");
+  ASSERT_EQ(values[2], "c");
+  ASSERT_EQ(values[3], "d");
+
+  // expected_max_number_of_operands is less than number of merge operands so
+  // status should be Incomplete.
+  merge_operands_info.expected_max_number_of_operands = num_records - 1;
+  Status status = db_->GetMergeOperands(
+      ReadOptions(), db_->DefaultColumnFamily(), "k1", values.data(),
+      &merge_operands_info, &number_of_operands);
+  ASSERT_EQ(status.IsIncomplete(), true);
+  merge_operands_info.expected_max_number_of_operands = num_records;
+
+  // All k2 values are flushed to L0 into a single file.
+  ASSERT_OK(Put("k2", "q"));
+  ASSERT_OK(Merge("k2", "w"));
+  ASSERT_OK(Merge("k2", "e"));
+  ASSERT_OK(Merge("k2", "r"));
+  ASSERT_OK(Flush());
+  ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(),
+                                  "k2", values.data(), &merge_operands_info,
+                                  &number_of_operands));
+  ASSERT_EQ(values[0], "q,w,e,r");
+
+  // Do some compaction that will make the following tests more predictable
+  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+
+  // All k3 values are flushed and are in different files.
+  ASSERT_OK(Put("k3", "ab"));
+  ASSERT_OK(Flush());
+  ASSERT_OK(Merge("k3", "bc"));
+  ASSERT_OK(Flush());
+  ASSERT_OK(Merge("k3", "cd"));
+  ASSERT_OK(Flush());
+  ASSERT_OK(Merge("k3", "de"));
+  ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(),
+                                  "k3", values.data(), &merge_operands_info,
+                                  &number_of_operands));
+  ASSERT_EQ(values[0], "ab");
+  ASSERT_EQ(values[1], "bc");
+  ASSERT_EQ(values[2], "cd");
+  ASSERT_EQ(values[3], "de");
+
+  // All K4 values are in different levels
+  ASSERT_OK(Put("k4", "ba"));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(4);
+  ASSERT_OK(Merge("k4", "cb"));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(3);
+  ASSERT_OK(Merge("k4", "dc"));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(1);
+  ASSERT_OK(Merge("k4", "ed"));
+  ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(),
+                                  "k4", values.data(), &merge_operands_info,
+                                  &number_of_operands));
+  ASSERT_EQ(values[0], "ba");
+  ASSERT_EQ(values[1], "cb");
+  ASSERT_EQ(values[2], "dc");
+  ASSERT_EQ(values[3], "ed");
+}
+
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -34,7 +34,7 @@ class DBOptionsTest : public DBTestBase {
      const DBOptions& options) {
    std::string options_str;
    std::unordered_map<std::string, std::string> mutable_map;
-    ConfigOptions config_options;
+    ConfigOptions config_options(options);
    config_options.delimiter = "; ";

    EXPECT_OK(GetStringFromMutableDBOptions(
@@ -1685,10 +1685,11 @@ TEST_F(DBRangeDelTest, OverlappedKeys) {
                                        true /* disallow_trivial_move */));
  ASSERT_EQ(3, NumTableFilesAtLevel(1));

-  std::vector<std::vector<FileMetaData>> files;
  ASSERT_OK(dbfull()->TEST_CompactRange(1, nullptr, nullptr, nullptr,
                                        true /* disallow_trivial_move */));
-  ASSERT_EQ(1, NumTableFilesAtLevel(2));
+  ASSERT_EQ(
+      3, NumTableFilesAtLevel(
+             2));  // L1->L2 compaction size is limited to max_compaction_bytes
  ASSERT_EQ(0, NumTableFilesAtLevel(1));
 }

@@ -187,6 +187,7 @@ TEST_F(DBSecondaryTest, SimpleInternalCompaction) {
  ASSERT_EQ(result.output_path, this->secondary_path_);
  ASSERT_EQ(result.num_output_records, 2);
  ASSERT_GT(result.bytes_written, 0);
+  ASSERT_OK(result.status);
 }

 TEST_F(DBSecondaryTest, InternalCompactionMultiLevels) {
@@ -235,6 +236,7 @@ TEST_F(DBSecondaryTest, InternalCompactionMultiLevels) {
  CompactionServiceResult result;
  ASSERT_OK(db_secondary_full()->TEST_CompactWithoutInstallation(cfh, input1,
                                                                 &result));
+  ASSERT_OK(result.status);

  // pick 2 files on level 1 for compaction, which has 6 overlap files on L2
  CompactionServiceInput input2;
@@ -247,6 +249,7 @@ TEST_F(DBSecondaryTest, InternalCompactionMultiLevels) {
  input2.output_level = 2;
  ASSERT_OK(db_secondary_full()->TEST_CompactWithoutInstallation(cfh, input2,
                                                                 &result));
+  ASSERT_OK(result.status);

  CloseSecondary();

@@ -259,6 +262,7 @@ TEST_F(DBSecondaryTest, InternalCompactionMultiLevels) {
  Status s = db_secondary_full()->TEST_CompactWithoutInstallation(cfh, input2,
                                                                  &result);
  ASSERT_TRUE(s.IsInvalidArgument());
+  ASSERT_OK(result.status);

  // TODO: L0 -> L1 compaction should success, currently version is not built
  // if files is missing.
@@ -304,6 +308,7 @@ TEST_F(DBSecondaryTest, InternalCompactionCompactedFiles) {
  Status s =
      db_secondary_full()->TEST_CompactWithoutInstallation(cfh, input, &result);
  ASSERT_TRUE(s.IsInvalidArgument());
+  ASSERT_OK(result.status);
 }

 TEST_F(DBSecondaryTest, InternalCompactionMissingFiles) {
@@ -340,11 +345,13 @@ TEST_F(DBSecondaryTest, InternalCompactionMissingFiles) {
  Status s =
      db_secondary_full()->TEST_CompactWithoutInstallation(cfh, input, &result);
  ASSERT_TRUE(s.IsInvalidArgument());
+  ASSERT_OK(result.status);

  input.input_files.erase(input.input_files.begin());

  ASSERT_OK(db_secondary_full()->TEST_CompactWithoutInstallation(cfh, input,
                                                                 &result));
+  ASSERT_OK(result.status);
 }

 TEST_F(DBSecondaryTest, OpenAsSecondary) {
@@ -5314,41 +5314,45 @@ TEST_F(DBTest, DynamicMiscOptions) {
 #endif  // ROCKSDB_LITE

 TEST_F(DBTest, L0L1L2AndUpHitCounter) {
-  Options options = CurrentOptions();
-  options.write_buffer_size = 32 * 1024;
-  options.target_file_size_base = 32 * 1024;
-  options.level0_file_num_compaction_trigger = 2;
-  options.level0_slowdown_writes_trigger = 2;
-  options.level0_stop_writes_trigger = 4;
-  options.max_bytes_for_level_base = 64 * 1024;
-  options.max_write_buffer_number = 2;
-  options.max_background_compactions = 8;
-  options.max_background_flushes = 8;
-  options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
-  CreateAndReopenWithCF({"mypikachu"}, options);
+  const int kNumLevels = 3;
+  const int kNumKeysPerLevel = 10000;
+  const int kNumKeysPerDb = kNumLevels * kNumKeysPerLevel;

-  int numkeys = 20000;
-  for (int i = 0; i < numkeys; i++) {
-    ASSERT_OK(Put(1, Key(i), "val"));
+  Options options = CurrentOptions();
+  options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
+  Reopen(options);
+
+  // After the below loop there will be one file on each of L0, L1, and L2.
+  int key = 0;
+  for (int output_level = kNumLevels - 1; output_level >= 0; --output_level) {
+    for (int i = 0; i < kNumKeysPerLevel; ++i) {
+      ASSERT_OK(Put(Key(key), "val"));
+      key++;
+    }
+    ASSERT_OK(Flush());
+    for (int input_level = 0; input_level < output_level; ++input_level) {
+      // `TEST_CompactRange(input_level, ...)` compacts from `input_level` to
+      // `input_level + 1`.
+      ASSERT_OK(dbfull()->TEST_CompactRange(input_level, nullptr, nullptr));
+    }
  }
+  assert(key == kNumKeysPerDb);
+
  ASSERT_EQ(0, TestGetTickerCount(options, GET_HIT_L0));
  ASSERT_EQ(0, TestGetTickerCount(options, GET_HIT_L1));
  ASSERT_EQ(0, TestGetTickerCount(options, GET_HIT_L2_AND_UP));

-  ASSERT_OK(Flush(1));
-  dbfull()->TEST_WaitForCompact();
-
-  for (int i = 0; i < numkeys; i++) {
-    ASSERT_EQ(Get(1, Key(i)), "val");
+  for (int i = 0; i < kNumKeysPerDb; i++) {
+    ASSERT_EQ(Get(Key(i)), "val");
  }

-  ASSERT_GT(TestGetTickerCount(options, GET_HIT_L0), 100);
-  ASSERT_GT(TestGetTickerCount(options, GET_HIT_L1), 100);
-  ASSERT_GT(TestGetTickerCount(options, GET_HIT_L2_AND_UP), 100);
+  ASSERT_EQ(kNumKeysPerLevel, TestGetTickerCount(options, GET_HIT_L0));
+  ASSERT_EQ(kNumKeysPerLevel, TestGetTickerCount(options, GET_HIT_L1));
+  ASSERT_EQ(kNumKeysPerLevel, TestGetTickerCount(options, GET_HIT_L2_AND_UP));

-  ASSERT_EQ(numkeys, TestGetTickerCount(options, GET_HIT_L0) +
-                         TestGetTickerCount(options, GET_HIT_L1) +
-                         TestGetTickerCount(options, GET_HIT_L2_AND_UP));
+  ASSERT_EQ(kNumKeysPerDb, TestGetTickerCount(options, GET_HIT_L0) +
+                               TestGetTickerCount(options, GET_HIT_L1) +
+                               TestGetTickerCount(options, GET_HIT_L2_AND_UP));
 }

 TEST_F(DBTest, EncodeDecompressedBlockSizeTest) {
@@ -9,6 +9,7 @@
 #include <atomic>
 #include <cstdlib>
 #include <functional>
+#include <memory>

 #include "db/db_test_util.h"
 #include "db/read_callback.h"
@@ -3201,6 +3202,180 @@ TEST_F(DBTest2, PausingManualCompaction4) {
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }

+TEST_F(DBTest2, CancelManualCompaction1) {
+  CompactRangeOptions compact_options;
+  auto canceledPtr =
+      std::unique_ptr<std::atomic<bool>>(new std::atomic<bool>{true});
+  compact_options.canceled = canceledPtr.get();
+
+  Options options = CurrentOptions();
+  options.disable_auto_compactions = true;
+  options.num_levels = 7;
+
+  Random rnd(301);
+  auto generate_files = [&]() {
+    for (int i = 0; i < options.num_levels; i++) {
+      for (int j = 0; j < options.num_levels - i + 1; j++) {
+        for (int k = 0; k < 1000; k++) {
+          ASSERT_OK(Put(Key(k + j * 1000), rnd.RandomString(50)));
+        }
+        Flush();
+      }
+
+      for (int l = 1; l < options.num_levels - i; l++) {
+        MoveFilesToLevel(l);
+      }
+    }
+  };
+
+  DestroyAndReopen(options);
+  generate_files();
+#ifndef ROCKSDB_LITE
+  ASSERT_EQ("2,3,4,5,6,7,8", FilesPerLevel());
+#endif  // !ROCKSDB_LITE
+
+  int run_manual_compactions = 0;
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "CompactionJob::Run():PausingManualCompaction:1",
+      [&](void* /*arg*/) { run_manual_compactions++; });
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  // Setup a callback to disable compactions after a couple of levels are
+  // compacted
+  int compactions_run = 0;
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::RunManualCompaction()::1",
+      [&](void* /*arg*/) { ++compactions_run; });
+
+  dbfull()->CompactRange(compact_options, nullptr, nullptr);
+  dbfull()->TEST_WaitForCompact(true);
+
+  // Since compactions are disabled, we shouldn't start compacting.
+  // E.g. we should call the compaction function exactly one time.
+  ASSERT_EQ(compactions_run, 0);
+  ASSERT_EQ(run_manual_compactions, 0);
+#ifndef ROCKSDB_LITE
+  ASSERT_EQ("2,3,4,5,6,7,8", FilesPerLevel());
+#endif  // !ROCKSDB_LITE
+
+  compactions_run = 0;
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearCallBack(
+      "DBImpl::RunManualCompaction()::1");
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::RunManualCompaction()::1", [&](void* /*arg*/) {
+        ++compactions_run;
+        // After 3 compactions disable
+        if (compactions_run == 3) {
+          compact_options.canceled->store(true, std::memory_order_release);
+        }
+      });
+
+  compact_options.canceled->store(false, std::memory_order_release);
+  dbfull()->CompactRange(compact_options, nullptr, nullptr);
+  dbfull()->TEST_WaitForCompact(true);
+
+  ASSERT_EQ(compactions_run, 3);
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearCallBack(
+      "DBImpl::RunManualCompaction()::1");
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearCallBack(
+      "CompactionJob::Run():PausingManualCompaction:1");
+
+  // Compactions should work again if we re-enable them..
+  compact_options.canceled->store(false, std::memory_order_relaxed);
+  dbfull()->CompactRange(compact_options, nullptr, nullptr);
+  dbfull()->TEST_WaitForCompact(true);
+#ifndef ROCKSDB_LITE
+  ASSERT_EQ("0,0,0,0,0,0,2", FilesPerLevel());
+#endif  // !ROCKSDB_LITE
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+}
+
+TEST_F(DBTest2, CancelManualCompaction2) {
+  CompactRangeOptions compact_options;
+  auto canceledPtr =
+      std::unique_ptr<std::atomic<bool>>(new std::atomic<bool>{true});
+  compact_options.canceled = canceledPtr.get();
+  compact_options.max_subcompactions = 1;
+
+  Options options = CurrentOptions();
+  options.disable_auto_compactions = true;
+  options.num_levels = 7;
+
+  Random rnd(301);
+  auto generate_files = [&]() {
+    for (int i = 0; i < options.num_levels; i++) {
+      for (int j = 0; j < options.num_levels - i + 1; j++) {
+        for (int k = 0; k < 1000; k++) {
+          ASSERT_OK(Put(Key(k + j * 1000), rnd.RandomString(50)));
+        }
+        Flush();
+      }
+
+      for (int l = 1; l < options.num_levels - i; l++) {
+        MoveFilesToLevel(l);
+      }
+    }
+  };
+
+  DestroyAndReopen(options);
+  generate_files();
+#ifndef ROCKSDB_LITE
+  ASSERT_EQ("2,3,4,5,6,7,8", FilesPerLevel());
+#endif  // !ROCKSDB_LITE
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  int compactions_run = 0;
+  std::atomic<int> kv_compactions{0};
+  int compactions_stopped_at = 0;
+  int kv_compactions_stopped_at = 0;
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::RunManualCompaction()::1", [&](void* /*arg*/) {
+        ++compactions_run;
+        // After 3 compactions disable
+      });
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "CompactionIterator:ProcessKV", [&](void* /*arg*/) {
+        int kv_compactions_run =
+            kv_compactions.fetch_add(1, std::memory_order_release);
+        if (kv_compactions_run == 5) {
+          compact_options.canceled->store(true, std::memory_order_release);
+          kv_compactions_stopped_at = kv_compactions_run;
+          compactions_stopped_at = compactions_run;
+        }
+      });
+
+  compact_options.canceled->store(false, std::memory_order_release);
+  dbfull()->CompactRange(compact_options, nullptr, nullptr);
+  dbfull()->TEST_WaitForCompact(true);
+
+  // NOTE: as we set compact_options.max_subcompacitons = 1, and store true to
+  // the canceled variable from the single compacting thread (via callback),
+  // this value is deterministically kv_compactions_stopped_at + 1.
+  ASSERT_EQ(kv_compactions, kv_compactions_stopped_at + 1);
+  ASSERT_EQ(compactions_run, compactions_stopped_at);
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearCallBack(
+      "CompactionIterator::ProcessKV");
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearCallBack(
+      "DBImpl::RunManualCompaction()::1");
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearCallBack(
+      "CompactionJob::Run():PausingManualCompaction:1");
+
+  // Compactions should work again if we re-enable them..
+  compact_options.canceled->store(false, std::memory_order_relaxed);
+  dbfull()->CompactRange(compact_options, nullptr, nullptr);
+  dbfull()->TEST_WaitForCompact(true);
+#ifndef ROCKSDB_LITE
+  ASSERT_EQ("0,0,0,0,0,0,2", FilesPerLevel());
+#endif  // !ROCKSDB_LITE
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+}
+
 TEST_F(DBTest2, OptimizeForPointLookup) {
  Options options = CurrentOptions();
  Close();
@@ -4988,7 +5163,7 @@ TEST_F(DBTest2, SameSmallestInSameLevel) {
  ASSERT_OK(Put("key", "2"));
  ASSERT_OK(db_->Merge(WriteOptions(), "key", "3"));
  ASSERT_OK(db_->Merge(WriteOptions(), "key", "4"));
-  Flush();
+  ASSERT_OK(Flush());
  CompactRangeOptions cro;
  cro.change_level = true;
  cro.target_level = 2;
@@ -5531,6 +5706,28 @@ TEST_P(RenameCurrentTest, Compaction) {
  ASSERT_EQ("NOT_FOUND", Get("foo"));
  ASSERT_EQ("d_value", Get("d"));
 }
+
+TEST_F(DBTest2, BottommostTemperature) {
+  Options options = CurrentOptions();
+  options.bottommost_temperature = Temperature::kWarm;
+  options.level0_file_num_compaction_trigger = 2;
+  Reopen(options);
+
+  ASSERT_OK(Put("foo", "bar"));
+  ASSERT_OK(Put("bar", "bar"));
+  ASSERT_OK(Flush());
+  ASSERT_OK(Put("foo", "bar"));
+  ASSERT_OK(Put("bar", "bar"));
+  ASSERT_OK(Flush());
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+
+  Reopen(options);
+
+  ColumnFamilyMetaData metadata;
+  db_->GetColumnFamilyMetaData(&metadata);
+  ASSERT_EQ(1, metadata.file_count);
+  ASSERT_EQ(Temperature::kWarm, metadata.levels[1].files[0].temperature);
+}
 #endif  // ROCKSDB_LITE

 // WAL recovery mode is WALRecoveryMode::kPointInTimeRecovery.
@@ -5586,7 +5783,6 @@ TEST_F(DBTest2, PointInTimeRecoveryWithSyncFailureInCFCreation) {
  options.wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery;
  ReopenWithColumnFamilies({"default", "test1", "test2"}, options);
 }
-
 }  // namespace ROCKSDB_NAMESPACE

 #ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
@@ -59,16 +59,8 @@ SpecialEnv::SpecialEnv(Env* base, bool time_elapse_only_sleep)
 DBTestBase::DBTestBase(const std::string path, bool env_do_fsync)
    : mem_env_(nullptr), encrypted_env_(nullptr), option_config_(kDefault) {
  Env* base_env = Env::Default();
-#ifndef ROCKSDB_LITE
-  const char* test_env_uri = getenv("TEST_ENV_URI");
-  if (test_env_uri) {
-    Env* test_env = nullptr;
-    Status s = Env::LoadEnv(test_env_uri, &test_env, &env_guard_);
-    base_env = test_env;
-    EXPECT_OK(s);
-    EXPECT_NE(Env::Default(), base_env);
-  }
-#endif  // !ROCKSDB_LITE
+  ConfigOptions config_options;
+  EXPECT_OK(test::CreateEnvFromSystem(config_options, &base_env, &env_guard_));
  EXPECT_NE(nullptr, base_env);
  if (getenv("MEM_ENV")) {
    mem_env_ = new MockEnv(base_env);
@@ -77,7 +69,7 @@ DBTestBase::DBTestBase(const std::string path, bool env_do_fsync)
  if (getenv("ENCRYPTED_ENV")) {
    std::shared_ptr<EncryptionProvider> provider;
    Status s = EncryptionProvider::CreateFromString(
-        ConfigOptions(), std::string("test://") + getenv("ENCRYPTED_ENV"),
+        config_options, std::string("test://") + getenv("ENCRYPTED_ENV"),
        &provider);
    encrypted_env_ = NewEncryptedEnv(mem_env_ ? mem_env_ : base_env, provider);
  }
@@ -303,6 +303,9 @@ class SpecialEnv : public EnvWrapper {
      Status Allocate(uint64_t offset, uint64_t len) override {
        return base_->Allocate(offset, len);
      }
+      size_t GetUniqueId(char* id, size_t max_size) const override {
+        return base_->GetUniqueId(id, max_size);
+      }
    };
    class ManifestFile : public WritableFile {
     public:
@@ -875,11 +878,22 @@ class CacheWrapper : public Cache {
    return target_->GetCharge(handle);
  }

+  DeleterFn GetDeleter(Handle* handle) const override {
+    return target_->GetDeleter(handle);
+  }
+
  void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
                              bool thread_safe) override {
    target_->ApplyToAllCacheEntries(callback, thread_safe);
  }

+  void ApplyToAllEntries(
+      const std::function<void(const Slice& key, void* value, size_t charge,
+                               DeleterFn deleter)>& callback,
+      const ApplyToAllEntriesOptions& opts) override {
+    target_->ApplyToAllEntries(callback, opts);
+  }
+
  void EraseUnRefEntries() override { target_->EraseUnRefEntries(); }

 protected:
@@ -1262,10 +1262,11 @@ class RecoveryTestHelper {
    std::unique_ptr<WalManager> wal_manager;
    WriteController write_controller;

-    versions.reset(new VersionSet(
-        test->dbname_, &db_options, file_options, table_cache.get(),
-        &write_buffer_manager, &write_controller,
-        /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr));
+    versions.reset(new VersionSet(test->dbname_, &db_options, file_options,
+                                  table_cache.get(), &write_buffer_manager,
+                                  &write_controller,
+                                  /*block_cache_tracer=*/nullptr,
+                                  /*io_tracer=*/nullptr, /*db_session_id*/ ""));

    wal_manager.reset(
        new WalManager(db_options, file_options, /*io_tracer=*/nullptr));
@@ -2001,6 +2002,61 @@ TEST_F(DBWALTest, TruncateLastLogAfterRecoverWALEmpty) {
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
 }
+
+TEST_F(DBWALTest, ReadOnlyRecoveryNoTruncate) {
+  constexpr size_t kKB = 1024;
+  Options options = CurrentOptions();
+  options.env = env_;
+  options.avoid_flush_during_recovery = true;
+  if (mem_env_) {
+    ROCKSDB_GTEST_SKIP("Test requires non-mem environment");
+    return;
+  }
+  if (!IsFallocateSupported()) {
+    return;
+  }
+
+  // create DB and close with file truncate disabled
+  std::atomic_bool enable_truncate{false};
+
+  SyncPoint::GetInstance()->SetCallBack(
+      "PosixWritableFile::Close", [&](void* arg) {
+        if (!enable_truncate) {
+          *(reinterpret_cast<size_t*>(arg)) = 0;
+        }
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  DestroyAndReopen(options);
+  size_t preallocated_size =
+      dbfull()->TEST_GetWalPreallocateBlockSize(options.write_buffer_size);
+  ASSERT_OK(Put("foo", "v1"));
+  VectorLogPtr log_files_before;
+  ASSERT_OK(dbfull()->GetSortedWalFiles(log_files_before));
+  ASSERT_EQ(1, log_files_before.size());
+  auto& file_before = log_files_before[0];
+  ASSERT_LT(file_before->SizeFileBytes(), 1 * kKB);
+  // The log file has preallocated space.
+  auto db_size = GetAllocatedFileSize(dbname_ + file_before->PathName());
+  ASSERT_GE(db_size, preallocated_size);
+  Close();
+
+  // enable truncate and open DB as readonly, the file should not be truncated
+  // and DB size is not changed.
+  enable_truncate = true;
+  ASSERT_OK(ReadOnlyReopen(options));
+  VectorLogPtr log_files_after;
+  ASSERT_OK(dbfull()->GetSortedWalFiles(log_files_after));
+  ASSERT_EQ(1, log_files_after.size());
+  ASSERT_LT(log_files_after[0]->SizeFileBytes(), 1 * kKB);
+  ASSERT_EQ(log_files_after[0]->PathName(), file_before->PathName());
+  // The preallocated space should NOT be truncated.
+  // the DB size is almost the same.
+  ASSERT_NEAR(GetAllocatedFileSize(dbname_ + file_before->PathName()), db_size,
+              db_size / 100);
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+}
 #endif  // ROCKSDB_FALLOCATE_PRESENT
 #endif  // ROCKSDB_PLATFORM_POSIX

@@ -418,15 +418,18 @@ const Status& ErrorHandler::SetBGError(const IOStatus& bg_io_err,
                                          &bg_err, db_mutex_, &auto_recovery);
    recover_context_ = context;
    return bg_error_;
-  } else if (bg_io_err.GetScope() ==
-                 IOStatus::IOErrorScope::kIOErrorScopeFile ||
-             bg_io_err.GetRetryable()) {
+  } else if (bg_io_err.subcode() != IOStatus::SubCode::kNoSpace &&
+             (bg_io_err.GetScope() ==
+                  IOStatus::IOErrorScope::kIOErrorScopeFile ||
+              bg_io_err.GetRetryable())) {
    // Second, check if the error is a retryable IO error (file scope IO error
    // is also treated as retryable IO error in RocksDB write path). if it is
    // retryable error and its severity is higher than bg_error_, overwrite the
    // bg_error_ with new error. In current stage, for retryable IO error of
    // compaction, treat it as soft error. In other cases, treat the retryable
-    // IO error as hard error.
+    // IO error as hard error. Note that, all the NoSpace error should be
+    // handled by the SstFileManager::StartErrorRecovery(). Therefore, no matter
+    // it is retryable or file scope, this logic will be bypassed.
    bool auto_recovery = false;
    EventHelpers::NotifyOnBackgroundError(db_options_.listeners, reason,
                                          &new_bg_io_err, db_mutex_,
@@ -193,6 +193,53 @@ TEST_F(DBErrorHandlingFSTest, FLushWriteError) {
  Destroy(options);
 }

+// All the NoSpace IOError will be handled as the regular BG Error no matter the
+// retryable flag is set of not. So the auto resume for retryable IO Error will
+// not be triggered. Also, it is mapped as hard error.
+TEST_F(DBErrorHandlingFSTest, FLushWriteNoSpaceError) {
+  std::shared_ptr<ErrorHandlerFSListener> listener(
+      new ErrorHandlerFSListener());
+  Options options = GetDefaultOptions();
+  options.env = fault_env_.get();
+  options.create_if_missing = true;
+  options.listeners.emplace_back(listener);
+  options.max_bgerror_resume_count = 2;
+  options.bgerror_resume_retry_interval = 100000;  // 0.1 second
+  options.statistics = CreateDBStatistics();
+  Status s;
+
+  listener->EnableAutoRecovery(false);
+  DestroyAndReopen(options);
+
+  IOStatus error_msg = IOStatus::NoSpace("Retryable IO Error");
+  error_msg.SetRetryable(true);
+
+  ASSERT_OK(Put(Key(1), "val1"));
+  SyncPoint::GetInstance()->SetCallBack(
+      "BuildTable:BeforeFinishBuildTable",
+      [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
+  SyncPoint::GetInstance()->EnableProcessing();
+  s = Flush();
+  ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
+  SyncPoint::GetInstance()->DisableProcessing();
+  fault_fs_->SetFilesystemActive(true);
+  s = dbfull()->Resume();
+  ASSERT_OK(s);
+  ASSERT_EQ(1, options.statistics->getAndResetTickerCount(
+                   ERROR_HANDLER_BG_ERROR_COUNT));
+  ASSERT_EQ(1, options.statistics->getAndResetTickerCount(
+                   ERROR_HANDLER_BG_IO_ERROR_COUNT));
+  ASSERT_EQ(0, options.statistics->getAndResetTickerCount(
+                   ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT));
+  ASSERT_EQ(0, options.statistics->getAndResetTickerCount(
+                   ERROR_HANDLER_AUTORESUME_COUNT));
+  ASSERT_EQ(0, options.statistics->getAndResetTickerCount(
+                   ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT));
+  ASSERT_EQ(0, options.statistics->getAndResetTickerCount(
+                   ERROR_HANDLER_AUTORESUME_SUCCESS_COUNT));
+  Destroy(options);
+}
+
 TEST_F(DBErrorHandlingFSTest, FLushWriteRetryableError) {
  std::shared_ptr<ErrorHandlerFSListener> listener(
      new ErrorHandlerFSListener());
@@ -108,6 +108,7 @@ void EventHelpers::LogAndNotifyTableFileCreationFinished(
                            table_properties.num_entries)
              << "num_data_blocks" << table_properties.num_data_blocks
              << "num_entries" << table_properties.num_entries
+              << "num_filter_entries" << table_properties.num_filter_entries
              << "num_deletions" << table_properties.num_deletions
              << "num_merge_operands" << table_properties.num_merge_operands
              << "num_range_deletions" << table_properties.num_range_deletions
@@ -1136,6 +1136,41 @@ TEST_F(ExternalSSTFileBasicTest, SyncFailure) {
  }
 }

+TEST_F(ExternalSSTFileBasicTest, ReopenNotSupported) {
+  Options options;
+  options.create_if_missing = true;
+  options.env = env_;
+
+  SyncPoint::GetInstance()->SetCallBack(
+      "ExternalSstFileIngestionJob::Prepare:Reopen", [&](void* arg) {
+        Status* s = static_cast<Status*>(arg);
+        *s = Status::NotSupported();
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  DestroyAndReopen(options);
+
+  Options sst_file_writer_options;
+  sst_file_writer_options.env = env_;
+  std::unique_ptr<SstFileWriter> sst_file_writer(
+      new SstFileWriter(EnvOptions(), sst_file_writer_options));
+  std::string file_name =
+      sst_files_dir_ + "reopen_not_supported_test_" + ".sst";
+  ASSERT_OK(sst_file_writer->Open(file_name));
+  ASSERT_OK(sst_file_writer->Put("bar", "v2"));
+  ASSERT_OK(sst_file_writer->Finish());
+
+  IngestExternalFileOptions ingest_opt;
+  ingest_opt.move_files = true;
+  const Snapshot* snapshot = db_->GetSnapshot();
+  ASSERT_OK(db_->IngestExternalFile({file_name}, ingest_opt));
+  db_->ReleaseSnapshot(snapshot);
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+  Destroy(options);
+}
+
 TEST_F(ExternalSSTFileBasicTest, VerifyChecksumReadahead) {
  Options options;
  options.create_if_missing = true;
@@ -109,17 +109,26 @@ Status ExternalSstFileIngestionJob::Prepare(
        // directory before ingest the file. For integrity of RocksDB we need
        // to sync the file.
        std::unique_ptr<FSWritableFile> file_to_sync;
-        status = fs_->ReopenWritableFile(path_inside_db, env_options_,
-                                         &file_to_sync, nullptr);
-        if (status.ok()) {
-          TEST_SYNC_POINT(
-              "ExternalSstFileIngestionJob::BeforeSyncIngestedFile");
-          status = SyncIngestedFile(file_to_sync.get());
-          TEST_SYNC_POINT("ExternalSstFileIngestionJob::AfterSyncIngestedFile");
-          if (!status.ok()) {
-            ROCKS_LOG_WARN(db_options_.info_log,
-                           "Failed to sync ingested file %s: %s",
-                           path_inside_db.c_str(), status.ToString().c_str());
+        Status s = fs_->ReopenWritableFile(path_inside_db, env_options_,
+                                           &file_to_sync, nullptr);
+        TEST_SYNC_POINT_CALLBACK("ExternalSstFileIngestionJob::Prepare:Reopen",
+                                 &s);
+        // Some file systems (especially remote/distributed) don't support
+        // reopening a file for writing and don't require reopening and
+        // syncing the file. Ignore the NotSupported error in that case.
+        if (!s.IsNotSupported()) {
+          status = s;
+          if (status.ok()) {
+            TEST_SYNC_POINT(
+                "ExternalSstFileIngestionJob::BeforeSyncIngestedFile");
+            status = SyncIngestedFile(file_to_sync.get());
+            TEST_SYNC_POINT(
+                "ExternalSstFileIngestionJob::AfterSyncIngestedFile");
+            if (!status.ok()) {
+              ROCKS_LOG_WARN(db_options_.info_log,
+                             "Failed to sync ingested file %s: %s",
+                             path_inside_db.c_str(), status.ToString().c_str());
+            }
          }
        }
      } else if (status.IsNotSupported() &&
@@ -1291,8 +1291,11 @@ TEST_F(ExternalSSTFileTest, PickedLevelBug) {
  // We have 2 overlapping files in L0
  EXPECT_EQ(FilesPerLevel(), "2");

+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
-      {{"DBImpl::AddFile:MutexLock", "ExternalSSTFileTest::PickedLevelBug:0"},
+      {{"DBImpl::IngestExternalFile:AfterIncIngestFileCounter",
+        "ExternalSSTFileTest::PickedLevelBug:0"},
       {"ExternalSSTFileTest::PickedLevelBug:1", "DBImpl::AddFile:MutexUnlock"},
       {"ExternalSSTFileTest::PickedLevelBug:2",
        "DBImpl::RunManualCompaction:0"},
@@ -1333,10 +1336,14 @@ TEST_F(ExternalSSTFileTest, PickedLevelBug) {
    // wait for 2 seconds to give a chance for compactions to run during
    // this period, and then make sure that no compactions where able to run
    env_->SleepForMicroseconds(1000000 * 2);
-    ASSERT_FALSE(bg_compact_started.load());
+    bool bg_compact_started_tmp = bg_compact_started.load();

    // Hold AddFile from finishing writing the MANIFEST
    TEST_SYNC_POINT("ExternalSSTFileTest::PickedLevelBug:1");
+
+    // check the status at the end, so even if the ASSERT fails the threads
+    // could be joined and return.
+    ASSERT_FALSE(bg_compact_started_tmp);
  }

  ASSERT_OK(bg_addfile_status);
@@ -166,7 +166,6 @@ void FlushJob::RecordFlushIOStats() {
      ThreadStatus::FLUSH_BYTES_WRITTEN, IOSTATS(bytes_written));
  IOSTATS_RESET(bytes_written);
 }
-
 void FlushJob::PickMemTable() {
  db_mutex_->AssertHeld();
  assert(!pick_memtable_called);
@@ -403,25 +402,49 @@ Status FlushJob::WriteLevel0Table() {
                                   ? current_time
                                   : meta_.oldest_ancester_time;

+      uint64_t num_input_entries = 0;
+      uint64_t memtable_payload_bytes = 0;
+      uint64_t memtable_garbage_bytes = 0;
      IOStatus io_s;
      const std::string* const full_history_ts_low =
          (full_history_ts_low_.empty()) ? nullptr : &full_history_ts_low_;
+      TableBuilderOptions tboptions(
+          *cfd_->ioptions(), mutable_cf_options_, cfd_->internal_comparator(),
+          cfd_->int_tbl_prop_collector_factories(), output_compression_,
+          mutable_cf_options_.compression_opts, cfd_->GetID(), cfd_->GetName(),
+          0 /* level */, false /* is_bottommost */,
+          TableFileCreationReason::kFlush, creation_time, oldest_key_time,
+          current_time, db_id_, db_session_id_, 0 /* target_file_size */,
+          meta_.fd.GetNumber());
      s = BuildTable(
-          dbname_, versions_, db_options_, *cfd_->ioptions(),
-          mutable_cf_options_, file_options_, cfd_->table_cache(), iter.get(),
-          std::move(range_del_iters), &meta_, &blob_file_additions,
-          cfd_->internal_comparator(), cfd_->int_tbl_prop_collector_factories(),
-          cfd_->GetID(), cfd_->GetName(), existing_snapshots_,
+          dbname_, versions_, db_options_, tboptions, file_options_,
+          cfd_->table_cache(), iter.get(), std::move(range_del_iters), &meta_,
+          &blob_file_additions, existing_snapshots_,
          earliest_write_conflict_snapshot_, snapshot_checker_,
-          output_compression_, mutable_cf_options_.compression_opts,
          mutable_cf_options_.paranoid_file_checks, cfd_->internal_stats(),
-          TableFileCreationReason::kFlush, &io_s, io_tracer_, event_logger_,
-          job_context_->job_id, Env::IO_HIGH, &table_properties_, 0 /* level */,
-          creation_time, oldest_key_time, write_hint, current_time, db_id_,
-          db_session_id_, full_history_ts_low, blob_callback_);
+          &io_s, io_tracer_, event_logger_, job_context_->job_id, Env::IO_HIGH,
+          &table_properties_, write_hint, full_history_ts_low, blob_callback_,
+          &num_input_entries, &memtable_payload_bytes, &memtable_garbage_bytes);
      if (!io_s.ok()) {
        io_status_ = io_s;
      }
+      if (num_input_entries != total_num_entries && s.ok()) {
+        std::string msg = "Expected " + ToString(total_num_entries) +
+                          " entries in memtables, but read " +
+                          ToString(num_input_entries);
+        ROCKS_LOG_WARN(db_options_.info_log, "[%s] [JOB %d] Level-0 flush %s",
+                       cfd_->GetName().c_str(), job_context_->job_id,
+                       msg.c_str());
+        if (db_options_.flush_verify_memtable_count) {
+          s = Status::Corruption(msg);
+        }
+      }
+      if (tboptions.reason == TableFileCreationReason::kFlush) {
+        RecordTick(stats_, MEMTABLE_PAYLOAD_BYTES_AT_FLUSH,
+                   memtable_payload_bytes);
+        RecordTick(stats_, MEMTABLE_GARBAGE_BYTES_AT_FLUSH,
+                   memtable_garbage_bytes);
+      }
      LogFlush(db_options_.info_log);
    }
    ROCKS_LOG_INFO(db_options_.info_log,