Update HISTORY.md and version for 8.10.2

Update ZLib to 1.3.1 (#12358 )
Summary: pdillinger This fixes the RocksJava build, is also needed in the 8.10.fb and 8.11.fb branches please? Pull Request resolved: https://github.com/facebook/rocksdb/pull/12358 Reviewed By: jaykorean Differential Revision: D53859743 Pulled By: pdillinger fbshipit-source-id: b8417fccfee931591805f9aecdfae7c086fee708
2026-11-07 11:32:34 -05:00 · 2024-02-16 12:09:07 -08:00 · 2024-02-16 12:08:11 -08:00 · 2024-01-16 15:54:18 -08:00 · 2024-01-16 15:47:09 -08:00 · 2023-12-15 13:01:14 -08:00
575 changed files with 31164 additions and 6349 deletions
@@ -16,7 +16,8 @@ commands:
      - run:
          name: Install JDK 8 on macos
          command: |
-            brew install --cask adoptopenjdk/openjdk/adoptopenjdk8
+            HOMEBREW_NO_AUTO_UPDATE=1 brew tap bell-sw/liberica
+            HOMEBREW_NO_AUTO_UPDATE=1 brew install --cask liberica-jdk8

  increase-max-open-files-on-macos:
    steps:
@@ -53,6 +54,7 @@ commands:
          command: |
            echo "Installing CMake..."
            choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' -y
+            choco install liberica8jdk -y
            mkdir $Env:THIRDPARTY_HOME
            cd $Env:THIRDPARTY_HOME
            echo "Building Snappy dependency..."
@@ -66,9 +68,10 @@ commands:
      - run:
          name: "Build RocksDB"
          command: |
+            $env:Path = $env:JAVA_HOME + ";" + $env:Path
            mkdir build
            cd build
-            & $Env:CMAKE_BIN -G "$Env:CMAKE_GENERATOR" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DJNI=1 ..
+            & $Env:CMAKE_BIN -G "$Env:CMAKE_GENERATOR" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE="$Env:CMAKE_PORTABLE" -DSNAPPY=1 -DJNI=1 ..
            cd ..
            echo "Building with VS version: $Env:CMAKE_GENERATOR"
            msbuild.exe build/rocksdb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
@@ -77,6 +80,11 @@ commands:
          shell: powershell.exe
          command: |
            build_tools\run_ci_db_test.ps1 -SuiteRun arena_test,db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16
+      - run:
+          name: "Test RocksJava"
+          command: |
+            cd build\java
+            & $Env:CTEST_BIN -C Debug -j 16
  pre-steps-macos:
      steps:
        - pre-steps
@@ -104,6 +112,15 @@ commands:
          path: /tmp/core_dumps
          when: on_fail

+  post-pmd-steps:
+    steps:
+      - store_artifacts:
+          path: /home/circleci/project/java/target/pmd.xml
+          when: on_fail
+      - store_artifacts:
+          path: /home/circleci/project/java/target/site
+          when: on_fail
+
  upgrade-cmake:
    steps:
      - run:
@@ -126,6 +143,13 @@ commands:
          command: |
            HOMEBREW_NO_AUTO_UPDATE=1 brew install gflags

+  install-maven:
+    steps:
+      - run:
+          name: Install maven
+          command: |
+            sudo apt-get update -y && sudo apt-get install -y maven
+
  setup-folly:
    steps:
      - run:
@@ -205,25 +229,28 @@ executors:
      #  $ docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -it zjay437/rocksdb:0.5 bash
      # option `--cap-add=SYS_PTRACE --security-opt seccomp=unconfined` is used to enable gdb to attach an existing process
      - image: zjay437/rocksdb:0.6
+  linux-java-docker:
+    docker:
+      - image: evolvedbinary/rocksjava:centos6_x64-be

 jobs:
  build-macos:
    macos:
-      xcode: 12.5.1
-    resource_class: large
+      xcode: 14.3.1
+    resource_class: macos.m1.medium.gen1
    environment:
      ROCKSDB_DISABLE_JEMALLOC: 1 # jemalloc cause env_test hang, disable it for now
    steps:
      - increase-max-open-files-on-macos
      - install-gflags-on-macos
      - pre-steps-macos
-      - run: ulimit -S -n `ulimit -H -n` && OPT=-DCIRCLECI make V=1 J=32 -j32 all
+      - run: ulimit -S -n `ulimit -H -n` && OPT=-DCIRCLECI make V=1 J=16 -j16 all
      - post-steps

  build-macos-cmake:
    macos:
-      xcode: 12.5.1
-    resource_class: large
+      xcode: 14.3.1
+    resource_class: macos.m1.medium.gen1
    parameters:
      run_even_tests:
        description: run even or odd tests, used to split tests to 2 groups
@@ -239,20 +266,20 @@ jobs:
          command: ulimit -S -n `ulimit -H -n` && mkdir build && cd build && cmake -DWITH_GFLAGS=1 ..
      - run:
          name: "Build tests"
-          command: cd build && make V=1 -j32
+          command: cd build && make V=1 -j16
      - when:
          condition: << parameters.run_even_tests >>
          steps:
            - run:
                name: "Run even tests"
-                command: ulimit -S -n `ulimit -H -n` && cd build && ctest -j32 -I 0,,2
+                command: ulimit -S -n `ulimit -H -n` && cd build && ctest -j16 -I 0,,2
      - when:
          condition:
            not: << parameters.run_even_tests >>
          steps:
            - run:
                name: "Run odd tests"
-                command: ulimit -S -n `ulimit -H -n` && cd build && ctest -j32 -I 1,,2
+                command: ulimit -S -n `ulimit -H -n` && cd build && ctest -j16 -I 1,,2
      - post-steps

  build-linux:
@@ -541,6 +568,24 @@ jobs:
          no_output_timeout: 100m
      - post-steps

+  build-windows-vs2022-avx2:
+    executor:
+      name: win/server-2022
+      size: 2xlarge
+    environment:
+      THIRDPARTY_HOME: C:/Users/circleci/thirdparty
+      CMAKE_HOME: C:/Program Files/CMake
+      CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe
+      CTEST_BIN: C:/Program Files/CMake/bin/ctest.exe
+      JAVA_HOME: C:/Program Files/BellSoft/LibericaJDK-8
+      SNAPPY_HOME: C:/Users/circleci/thirdparty/snappy-1.1.8
+      SNAPPY_INCLUDE: C:/Users/circleci/thirdparty/snappy-1.1.8;C:/Users/circleci/thirdparty/snappy-1.1.8/build
+      SNAPPY_LIB_DEBUG: C:/Users/circleci/thirdparty/snappy-1.1.8/build/Debug/snappy.lib
+      CMAKE_GENERATOR: Visual Studio 17 2022
+      CMAKE_PORTABLE: AVX2
+    steps:
+      - windows-build-steps
+
  build-windows-vs2022:
    executor:
      name: win/server-2022
@@ -549,10 +594,13 @@ jobs:
      THIRDPARTY_HOME: C:/Users/circleci/thirdparty
      CMAKE_HOME: C:/Program Files/CMake
      CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe
+      CTEST_BIN: C:/Program Files/CMake/bin/ctest.exe
+      JAVA_HOME: C:/Program Files/BellSoft/LibericaJDK-8
      SNAPPY_HOME: C:/Users/circleci/thirdparty/snappy-1.1.8
      SNAPPY_INCLUDE: C:/Users/circleci/thirdparty/snappy-1.1.8;C:/Users/circleci/thirdparty/snappy-1.1.8/build
      SNAPPY_LIB_DEBUG: C:/Users/circleci/thirdparty/snappy-1.1.8/build/Debug/snappy.lib
      CMAKE_GENERATOR: Visual Studio 17 2022
+      CMAKE_PORTABLE: 1
    steps:
      - windows-build-steps

@@ -564,10 +612,13 @@ jobs:
      THIRDPARTY_HOME: C:/Users/circleci/thirdparty
      CMAKE_HOME: C:/Program Files/CMake
      CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe
+      CTEST_BIN: C:/Program Files/CMake/bin/ctest.exe
+      JAVA_HOME: C:/Program Files/BellSoft/LibericaJDK-8
      SNAPPY_HOME: C:/Users/circleci/thirdparty/snappy-1.1.8
      SNAPPY_INCLUDE: C:/Users/circleci/thirdparty/snappy-1.1.8;C:/Users/circleci/thirdparty/snappy-1.1.8/build
      SNAPPY_LIB_DEBUG: C:/Users/circleci/thirdparty/snappy-1.1.8/build/Debug/snappy.lib
      CMAKE_GENERATOR: Visual Studio 16 2019
+      CMAKE_PORTABLE: 1
    steps:
      - windows-build-steps

@@ -588,8 +639,29 @@ jobs:
          command: make V=1 J=8 -j8 jtest
      - post-steps

+  build-linux-java-pmd:
+    machine:
+      image: ubuntu-2004:202111-02
+    resource_class: large
+    environment:
+      JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+    steps:
+      - install-maven
+      - pre-steps
+      - run:
+          name: "Set Java Environment"
+          command: |
+            echo "JAVA_HOME=${JAVA_HOME}"
+            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
+            which java && java -version
+            which javac && javac -version
+      - run:
+          name: "PMD RocksDBJava"
+          command: make V=1 J=8 -j8 jpmd
+      - post-pmd-steps
+
  build-linux-java-static:
-    executor: linux-docker
+    executor: linux-java-docker
    resource_class: large
    steps:
      - pre-steps
@@ -602,15 +674,15 @@ jobs:
            which javac && javac -version
      - run:
          name: "Build RocksDBJava Static Library"
-          command: make V=1 J=8 -j8 rocksdbjavastatic
+          command: scl enable devtoolset-7 'make V=1 J=8 -j8 rocksdbjavastatic'
      - post-steps

  build-macos-java:
    macos:
-      xcode: 12.5.1
-    resource_class: large
+      xcode: 14.3.1
+    resource_class: macos.m1.medium.gen1
    environment:
-      JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
+      JAVA_HOME: /Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home
      ROCKSDB_DISABLE_JEMALLOC: 1 # jemalloc causes java 8 crash
    steps:
      - increase-max-open-files-on-macos
@@ -632,10 +704,10 @@ jobs:

  build-macos-java-static:
    macos:
-      xcode: 12.5.1
-    resource_class: large
+      xcode: 14.3.1
+    resource_class: macos.m1.medium.gen1
    environment:
-      JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
+      JAVA_HOME: /Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home
    steps:
      - increase-max-open-files-on-macos
      - install-gflags-on-macos
@@ -657,10 +729,10 @@ jobs:

  build-macos-java-static-universal:
    macos:
-      xcode: 12.5.1
-    resource_class: large
+      xcode: 14.3.1
+    resource_class: macos.m1.medium.gen1
    environment:
-      JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
+      JAVA_HOME: /Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home
    steps:
      - increase-max-open-files-on-macos
      - install-gflags-on-macos
@@ -842,7 +914,6 @@ workflows:
      - build-linux-mini-crashtest
  jobs-windows:
    jobs:
-      - build-windows-vs2022
      - build-windows-vs2019
      - build-cmake-mingw
  jobs-java:
@@ -852,6 +923,7 @@ workflows:
      - build-macos-java
      - build-macos-java-static
      - build-macos-java-static-universal
+      - build-linux-java-pmd
  jobs-macos:
    jobs:
      - build-macos
@@ -890,3 +962,5 @@ workflows:
      - build-linux-non-shm
      - build-linux-clang-13-asan-ubsan-with-folly
      - build-linux-valgrind
+      - build-windows-vs2022-avx2
+      - build-windows-vs2022
@@ -85,6 +85,7 @@ fbcode/
 fbcode
 buckifier/*.pyc
 buckifier/__pycache__
+.arcconfig

 compile_commands.json
 clang-format-diff.py
@@ -254,7 +254,7 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "loongarch64")
 endif(CMAKE_SYSTEM_PROCESSOR MATCHES "loongarch64")

 set(PORTABLE 0 CACHE STRING "Minimum CPU arch to support, or 0 = current CPU, 1 = baseline CPU")
-if(PORTABLE STREQUAL 1)
+if(PORTABLE MATCHES "1|ON|YES|TRUE|Y")
  # Usually nothing to do; compiler default is typically the most general
  if(NOT MSVC)
    if(CMAKE_SYSTEM_PROCESSOR MATCHES "^s390x")
@@ -264,14 +264,7 @@ if(PORTABLE STREQUAL 1)
      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=loongarch64")
    endif()
  endif()
-elseif(PORTABLE MATCHES [^0]+)
-  # Name of a CPU arch spec or feature set to require
-  if(MSVC)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:${PORTABLE}")
-  else()
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=${PORTABLE}")
-  endif()
-else()
+elseif(PORTABLE MATCHES "0|OFF|NO|FALSE|N")
  if(MSVC)
    # NOTE: No auto-detection of current CPU, but instead assume some useful
    # level of optimization is supported
@@ -285,6 +278,13 @@ else()
      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
    endif()
  endif()
+else()
+  # Name of a CPU arch spec or feature set to require
+  if(MSVC)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:${PORTABLE}")
+  else()
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=${PORTABLE}")
+  endif()
 endif()

 include(CheckCXXSourceCompiles)
@@ -632,6 +632,7 @@ set(SOURCES
        cache/secondary_cache.cc
        cache/secondary_cache_adapter.cc
        cache/sharded_cache.cc
+        cache/tiered_secondary_cache.cc
        db/arena_wrapped_db_iter.cc
        db/blob/blob_contents.cc
        db/blob/blob_fetcher.cc
@@ -714,6 +715,7 @@ set(SOURCES
        db/wal_manager.cc
        db/wide/wide_column_serialization.cc
        db/wide/wide_columns.cc
+        db/wide/wide_columns_helper.cc
        db/write_batch.cc
        db/write_batch_base.cc
        db/write_controller.cc
@@ -770,6 +772,7 @@ set(SOURCES
        options/configurable.cc
        options/customizable.cc
        options/db_options.cc
+        options/offpeak_time_info.cc
        options/options.cc
        options/options_helper.cc
        options/options_parser.cc
@@ -1103,11 +1106,15 @@ set(BUILD_VERSION_CC ${CMAKE_BINARY_DIR}/build_version.cc)
 configure_file(util/build_version.cc.in ${BUILD_VERSION_CC} @ONLY)

 add_library(${ROCKSDB_STATIC_LIB} STATIC ${SOURCES} ${BUILD_VERSION_CC})
+target_include_directories(${ROCKSDB_STATIC_LIB} PUBLIC
+  $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>)
 target_link_libraries(${ROCKSDB_STATIC_LIB} PRIVATE
  ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})

 if(ROCKSDB_BUILD_SHARED)
  add_library(${ROCKSDB_SHARED_LIB} SHARED ${SOURCES} ${BUILD_VERSION_CC})
+  target_include_directories(${ROCKSDB_SHARED_LIB} PUBLIC
+    $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>)
  target_link_libraries(${ROCKSDB_SHARED_LIB} PRIVATE
    ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})

@@ -1262,6 +1269,7 @@ if(WITH_TESTS)
        cache/cache_test.cc
        cache/compressed_secondary_cache_test.cc
        cache/lru_cache_test.cc
+        cache/tiered_secondary_cache_test.cc
        db/blob/blob_counting_iterator_test.cc
        db/blob/blob_file_addition_test.cc
        db/blob/blob_file_builder_test.cc
@@ -1357,6 +1365,7 @@ if(WITH_TESTS)
        db/wal_edit_test.cc
        db/wide/db_wide_basic_test.cc
        db/wide/wide_column_serialization_test.cc
+        db/wide/wide_columns_helper_test.cc
        db/write_batch_test.cc
        db/write_callback_test.cc
        db/write_controller_test.cc
@@ -1592,6 +1601,3 @@ option(WITH_BENCHMARK "build benchmark tests" OFF)
 if(WITH_BENCHMARK)
  add_subdirectory(${PROJECT_SOURCE_DIR}/microbench/)
 endif()
-
-target_include_directories(${PROJECT_NAME} PUBLIC
-  $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>)
@@ -1,6 +1,169 @@
 # Rocksdb Change Log
 > NOTE: Entries for next release do not go here. Follow instructions in `unreleased_history/README.txt`

+## 8.10.2 (02/16/2024)
+* Update zlib to 1.3.1 for Java builds
+
+## 8.10.1 (01/16/2024)
+### Bug Fixes
+* Fix bug in auto_readahead_size that combined with IndexType::kBinarySearchWithFirstKey + fails or iterator lands at a wrong key
+
+## 8.10.0 (12/15/2023)
+### New Features
+* Provide support for async_io to trim readahead_size by doing block cache lookup
+* Added initial wide-column support in `WriteBatchWithIndex`. This includes the `PutEntity` API and support for wide columns in the existing read APIs (`GetFromBatch`, `GetFromBatchAndDB`, `MultiGetFromBatchAndDB`, and `BaseDeltaIterator`).
+
+### Public API Changes
+* Custom implementations of `TablePropertiesCollectorFactory` may now return a `nullptr` collector to decline processing a file, reducing callback overheads in such cases.
+
+### Behavior Changes
+* Make ReadOptions.auto_readahead_size default true which does prefetching optimizations for forward scans if iterate_upper_bound and block_cache is also specified.
+* Compactions can be scheduled in parallel in an additional scenario: high compaction debt relative to the data size
+* HyperClockCache now has built-in protection against excessive CPU consumption under the extreme stress condition of no (or very few) evictable cache entries, which can slightly increase memory usage such conditions. New option `HyperClockCacheOptions::eviction_effort_cap` controls the space-time trade-off of the response. The default should be generally well-balanced, with no measurable affect on normal operation.
+
+### Bug Fixes
+* Fix a corner case with auto_readahead_size where Prev Operation returns NOT SUPPORTED error when scans direction is changed from forward to backward.
+* Avoid destroying the periodic task scheduler's default timer in order to prevent static destruction order issues.
+* Fix double counting of BYTES_WRITTEN ticker when doing writes with transactions.
+* Fix a WRITE_STALL counter that was reporting wrong value in few cases.
+* A lookup by MultiGet in a TieredCache that goes to the local flash cache and finishes with very low latency, i.e before the subsequent call to WaitAll, is ignored, resulting in a false negative and a memory leak.
+
+### Performance Improvements
+* Java API extensions to improve consistency and completeness of APIs
+1 Extended `RocksDB.get([ColumnFamilyHandle columnFamilyHandle,] ReadOptions opt, ByteBuffer key, ByteBuffer value)` which now accepts indirect buffer parameters as well as direct buffer parameters
+2 Extended `RocksDB.put( [ColumnFamilyHandle columnFamilyHandle,] WriteOptions writeOpts, final ByteBuffer key, final ByteBuffer value)` which now accepts indirect buffer parameters as well as direct buffer parameters
+3 Added `RocksDB.merge([ColumnFamilyHandle columnFamilyHandle,] WriteOptions writeOptions, ByteBuffer key, ByteBuffer value)` methods with the same parameter options as `put(...)` - direct and indirect buffers are supported
+4 Added `RocksIterator.key( byte[] key [, int offset, int len])` methods which retrieve the iterator key into the supplied buffer
+5 Added `RocksIterator.value( byte[] value [, int offset, int len])` methods which retrieve the iterator value into the supplied buffer
+6 Deprecated `get(final ColumnFamilyHandle columnFamilyHandle, final ReadOptions readOptions, byte[])` in favour of `get(final ReadOptions readOptions, final ColumnFamilyHandle columnFamilyHandle, byte[])` which has consistent parameter ordering with other methods in the same class
+7 Added `Transaction.get( ReadOptions opt, [ColumnFamilyHandle columnFamilyHandle, ] byte[] key, byte[] value)` methods which retrieve the requested value into the supplied buffer
+8 Added `Transaction.get( ReadOptions opt, [ColumnFamilyHandle columnFamilyHandle, ] ByteBuffer key, ByteBuffer value)` methods which retrieve the requested value into the supplied buffer
+9 Added `Transaction.getForUpdate( ReadOptions readOptions, [ColumnFamilyHandle columnFamilyHandle, ] byte[] key, byte[] value, boolean exclusive [, boolean doValidate])` methods which retrieve the requested value into the supplied buffer
+10 Added `Transaction.getForUpdate( ReadOptions readOptions, [ColumnFamilyHandle columnFamilyHandle, ] ByteBuffer key, ByteBuffer value, boolean exclusive [, boolean doValidate])` methods which retrieve the requested value into the supplied buffer
+11 Added `Transaction.getIterator()` method as a convenience which defaults the `ReadOptions` value supplied to existing `Transaction.iterator()` methods. This mirrors the existing `RocksDB.iterator()` method.
+12 Added `Transaction.put([ColumnFamilyHandle columnFamilyHandle, ]  ByteBuffer key, ByteBuffer value [, boolean assumeTracked])` methods which supply the key, and the value to be written in a `ByteBuffer` parameter
+13 Added `Transaction.merge([ColumnFamilyHandle columnFamilyHandle, ] ByteBuffer key, ByteBuffer value [,  boolean assumeTracked])` methods which supply the key, and the value to be written/merged in a `ByteBuffer` parameter
+14 Added `Transaction.mergeUntracked([ColumnFamilyHandle columnFamilyHandle, ] ByteBuffer key, ByteBuffer value)` methods which supply the key, and the value to be written/merged in a `ByteBuffer` parameter
+
+
+## 8.9.0 (11/17/2023)
+### New Features
+* Add GetEntity() and PutEntity() API implementation for Attribute Group support. Through the use of Column Families, AttributeGroup enables users to logically group wide-column entities.
+
+### Public API Changes
+* Added rocksdb_ratelimiter_create_auto_tuned API to create an auto-tuned GenericRateLimiter.
+* Added clipColumnFamily() to the Java API to clip the entries in the CF according to the range [begin_key, end_key).
+* Make the `EnableFileDeletion` API not default to force enabling. For users that rely on this default behavior and still
+want to continue to use force enabling, they need to explicitly pass a `true` to `EnableFileDeletion`.
+* Add new Cache APIs GetSecondaryCacheCapacity() and GetSecondaryCachePinnedUsage() to return the configured capacity, and cache reservation charged to the secondary cache.
+
+### Behavior Changes
+* During off-peak hours defined by `daily_offpeak_time_utc`, the compaction picker will select a larger number of files for periodic compaction. This selection will include files that are projected to expire by the next off-peak start time, ensuring that these files are not chosen for periodic compaction outside of off-peak hours.
+* If an error occurs when writing to a trace file after `DB::StartTrace()`, the subsequent trace writes are skipped to avoid writing to a file that has previously seen error. In this case, `DB::EndTrace()` will also return a non-ok status with info about the error occured previously in its status message.
+* Deleting stale files upon recovery are delegated to SstFileManger if available so they can be rate limited.
+* Make RocksDB only call `TablePropertiesCollector::Finish()` once.
+* When `WAL_ttl_seconds > 0`, we now process archived WALs for deletion at least every `WAL_ttl_seconds / 2` seconds. Previously it could be less frequent in case of small `WAL_ttl_seconds` values when size-based expiration (`WAL_size_limit_MB > 0 `) was simultaneously enabled.
+
+### Bug Fixes
+* Fixed a crash or assertion failure bug in experimental new HyperClockCache variant, especially when running with a SecondaryCache.
+* Fix a race between flush error recovery and db destruction that can lead to db crashing.
+* Fixed some bugs in the index builder/reader path for user-defined timestamps in Memtable only feature.
+
+## 8.8.0 (10/23/2023)
+### New Features
+* Introduce AttributeGroup by adding the first AttributeGroup support API, MultiGetEntity(). Through the use of Column Families, AttributeGroup enables users to logically group wide-column entities. More APIs to support AttributeGroup will come soon, including GetEntity, PutEntity, and others.
+* Added new tickers `rocksdb.fifo.{max.size|ttl}.compactions` to count FIFO compactions that drop files for different reasons
+* Add an experimental offpeak duration awareness by setting `DBOptions::daily_offpeak_time_utc` in "HH:mm-HH:mm" format. This information will be used for resource optimization in the future
+* Users can now change the max bytes granted in a single refill period (i.e, burst) during runtime by `SetSingleBurstBytes()` for RocksDB rate limiter
+
+### Public API Changes
+* The default value of `DBOptions::fail_if_options_file_error` changed from `false` to `true`. Operations that set in-memory options (e.g., `DB::Open*()`, `DB::SetOptions()`, `DB::CreateColumnFamily*()`, and `DB::DropColumnFamily()`) but fail to persist the change will now return a non-OK `Status` by default.
+
+### Behavior Changes
+* For non direct IO, eliminate the file system prefetching attempt for compaction read when `Options::compaction_readahead_size` is 0
+* During a write stop, writes now block on in-progress recovery attempts
+
+### Bug Fixes
+* Fix a bug in auto_readahead_size where first_internal_key of index blocks wasn't copied properly resulting in corruption error when first_internal_key was used for comparison.
+* Fixed a bug where compaction read under non direct IO still falls back to RocksDB internal prefetching after file system's prefetching returns non-OK status other than `Status::NotSupported()`
+* Add bounds check in WBWIIteratorImpl and make BaseDeltaIterator, WriteUnpreparedTxn and WritePreparedTxn respect the upper bound and lower bound in ReadOption. See 11680.
+* Fixed the handling of wide-column base values in the `max_successive_merges` logic.
+* Fixed a rare race bug involving a concurrent combination of Create/DropColumnFamily and/or Set(DB)Options that could lead to inconsistency between (a) the DB's reported options state, (b) the DB options in effect, and (c) the latest persisted OPTIONS file.
+* Fixed a possible underflow when computing the compressed secondary cache share of memory reservations while updating the compressed secondary to total block cache ratio.
+
+### Performance Improvements
+* Improved the I/O efficiency of DB::Open a new DB with `create_missing_column_families=true` and many column families.
+
+## 8.7.0 (09/22/2023)
+### New Features
+* Added an experimental new "automatic" variant of HyperClockCache that does not require a prior estimate of the average size of cache entries. This variant is activated when HyperClockCacheOptions::estimated\_entry\_charge = 0 and has essentially the same concurrency benefits as the existing HyperClockCache.
+* Add a new statistic `COMPACTION_CPU_TOTAL_TIME` that records cumulative compaction cpu time. This ticker is updated regularly while a compaction is running.
+* Add `GetEntity()` API for ReadOnly DB and Secondary DB.
+* Add a new iterator API `Iterator::Refresh(const Snapshot *)` that allows iterator to be refreshed while using the input snapshot to read.
+* Added a new read option `merge_operand_count_threshold`. When the number of merge operands applied during a successful point lookup exceeds this threshold, the query will return a special OK status with a new subcode `kMergeOperandThresholdExceeded`. Applications might use this signal to take action to reduce the number of merge operands for the affected key(s), for example by running a compaction.
+* For `NewRibbonFilterPolicy()`, made the `bloom_before_level` option mutable through the Configurable interface and the SetOptions API, allowing dynamic switching between all-Bloom and all-Ribbon configurations, and configurations in between. See comments on `NewRibbonFilterPolicy()`
+* RocksDB now allows the block cache to be stacked on top of a compressed secondary cache and a non-volatile secondary cache, thus creating a three-tier cache. To set it up, use the `NewTieredCache()` API in rocksdb/cache.h..
+* Added a new wide-column aware full merge API called `FullMergeV3` to `MergeOperator`. `FullMergeV3` supports wide columns both as base value and merge result, which enables the application to perform more general transformations during merges. For backward compatibility, the default implementation implements the earlier logic of applying the merge operation to the default column of any wide-column entities. Specifically, if there is no base value or the base value is a plain key-value, the default implementation falls back to `FullMergeV2`. If the base value is a wide-column entity, the default implementation invokes `FullMergeV2` to perform the merge on the default column, and leaves any other columns unchanged.
+* Add wide column support to ldb commands (scan, dump, idump, dump_wal) and sst_dump tool's scan command
+
+### Public API Changes
+* Expose more information about input files used in table creation (if any) in `CompactionFilter::Context`. See `CompactionFilter::Context::input_start_level`,`CompactionFilter::Context::input_table_properties` for more.
+* `Options::compaction_readahead_size` 's default value is changed from 0 to 2MB.
+* When using LZ4 compression, the `acceleration` parameter is configurable by setting the negated value in `CompressionOptions::level`. For example, `CompressionOptions::level=-10` will set `acceleration=10`
+* The `NewTieredCache` API has been changed to take the total cache capacity (inclusive of both the primary and the compressed secondary cache) and the ratio of total capacity to allocate to the compressed cache. These are specified in `TieredCacheOptions`. Any capacity specified in `LRUCacheOptions`, `HyperClockCacheOptions` and `CompressedSecondaryCacheOptions` is ignored. A new API, `UpdateTieredCache` is provided to dynamically update the total capacity, ratio of compressed cache, and admission policy.
+* The `NewTieredVolatileCache()` API in rocksdb/cache.h has been renamed to `NewTieredCache()`.
+
+### Behavior Changes
+* Compaction read performance will regress when `Options::compaction_readahead_size` is explicitly set to 0
+* Universal size amp compaction will conditionally exclude some of the newest L0 files when selecting input with a small negative impact to size amp. This is to prevent a large number of L0 files from being locked by a size amp compaction, potentially leading to write stop with a few more flushes.
+* Change ldb scan command delimiter from ':' to '==>'.
+
+### Bug Fixes
+* Fix a bug where if there is an error reading from offset 0 of a file from L1+ and that the file is not the first file in the sorted run, data can be lost in compaction and read/scan can return incorrect results.
+* Fix a bug where iterator may return incorrect result for DeleteRange() users if there was an error reading from a file.
+* Fix a bug with atomic_flush=true that can cause DB to stuck after a flush fails (#11872).
+* Fix a bug where RocksDB (with atomic_flush=false) can delete output SST files of pending flushes when a previous concurrent flush fails (#11865). This can result in DB entering read-only state with error message like `IO error: No such file or directory: While open a file for random read: /tmp/rocksdbtest-501/db_flush_test_87732_4230653031040984171/000013.sst`.
+* Fix an assertion fault during seek with async_io when readahead trimming is enabled.
+* When the compressed secondary cache capacity is reduced to 0, it should be completely disabled. Before this fix, inserts and lookups would still go to the backing `LRUCache` before returning, thus incurring locking overhead. With this fix, inserts and lookups are no-ops and do not add any overhead.
+* Updating the tiered cache (cache allocated using NewTieredCache()) by calling SetCapacity() on it was not working properly. The initial creation would set the primary cache capacity to the combined primary and compressed secondary cache capacity. But SetCapacity() would just set the primary cache capacity. With this fix, the user always specifies the total budget and compressed secondary cache ratio on creation. Subsequently, SetCapacity() will distribute the new capacity across the two caches by the same ratio.
+* Fixed a bug in `MultiGet` for cleaning up SuperVersion acquired with locking db mutex.
+* Fix a bug where row cache can falsely return kNotFound even though row cache entry is hit.
+* Fixed a race condition in `GenericRateLimiter` that could cause it to stop granting requests
+* Fix a bug (Issue #10257) where DB can hang after write stall since no compaction is scheduled (#11764).
+* Add a fix for async_io where during seek, when reading a block for seeking a target key in a file without any readahead, the iterator aligned the read on a page boundary and reading more than necessary. This increased the storage read bandwidth usage.
+* Fix an issue in sst dump tool to handle bounds specified for data with user-defined timestamps.
+* When auto_readahead_size is enabled, update readahead upper bound during readahead trimming when reseek changes iterate_upper_bound dynamically.
+* Fixed a bug where `rocksdb.file.read.verify.file.checksums.micros` is not populated
+
+### Performance Improvements
+* Added additional improvements in tuning readahead_size during Scans when auto_readahead_size is enabled. However it's not supported with Iterator::Prev operation and will return NotSupported error.
+* During async_io, the Seek happens in 2 phases. Phase 1 starts an asynchronous read on a block cache miss, and phase 2 waits for it to complete and finishes the seek. In both phases, it tries to lookup the block cache for the data block first before looking in the prefetch buffer. It's optimized by doing the block cache lookup only in the first phase that would save some CPU.
+
+## 8.6.0 (08/18/2023)
+### New Features
+* Added enhanced data integrity checking on SST files with new format_version=6. Performance impact is very small or negligible. Previously if SST data was misplaced or re-arranged by the storage layer, it could pass block checksum with higher than 1 in 4 billion probability. With format_version=6, block checksums depend on what file they are in and location within the file. This way, misplaced SST data is no more likely to pass checksum verification than randomly corrupted data. Also in format_version=6, SST footers are checksum-protected.
+* Add a new feature to trim readahead_size during scans upto upper_bound when iterate_upper_bound is specified. It's enabled through ReadOptions.auto_readahead_size. Users must also specify ReadOptions.iterate_upper_bound.
+* RocksDB will compare the number of input keys to the number of keys processed after each compaction. Compaction will fail and report Corruption status if the verification fails. Option `compaction_verify_record_count` is introduced for this purpose and is enabled by default.
+* Add a CF option `bottommost_file_compaction_delay` to allow specifying the delay of bottommost level single-file compactions.
+* Add support to allow enabling / disabling user-defined timestamps feature for an existing column family in combination with the in-Memtable only feature.
+* Implement a new admission policy for the compressed secondary cache that admits blocks evicted from the primary cache with the hit bit set. This policy can be specified in TieredVolatileCacheOptions by setting the newly added adm_policy option.
+* Add a column family option `memtable_max_range_deletions` that limits the number of range deletions in a memtable. RocksDB will try to do an automatic flush after the limit is reached. (#11358)
+* Add PutEntity API in sst_file_writer
+* Add `timeout` in microsecond option to `WaitForCompactOptions` to allow timely termination of prolonged waiting in scenarios like recurring recoverable errors, such as out-of-space situations and continuous write streams that sustain ongoing flush and compactions
+* New statistics `rocksdb.file.read.{get|multiget|db.iterator|verify.checksum|verify.file.checksums}.micros` measure read time of block-based SST tables or blob files during db open, `Get()`, `MultiGet()`, using db iterator, `VerifyFileChecksums()` and `VerifyChecksum()`. They require stats level greater than `StatsLevel::kExceptDetailedTimers`.
+* Add close_db option to `WaitForCompactOptions` to call Close() after waiting is done.
+* Add a new compression option `CompressionOptions::checksum` for enabling ZSTD's checksum feature to detect corruption during decompression.
+
+### Public API Changes
+* Mark `Options::access_hint_on_compaction_start` related APIs as deprecated. See #11631 for alternative behavior.
+
+### Behavior Changes
+* Statistics `rocksdb.sst.read.micros` now includes time spent on multi read and async read into the file
+* For Universal Compaction users, periodic compaction (option `periodic_compaction_seconds`) will be set to 30 days by default if block based table is used.
+
+### Bug Fixes
+* Fix a bug in FileTTLBooster that can cause users with a large number of levels (more than 65) to see errors like "runtime error: shift exponent .. is too large.." (#11673).
+
 ## 8.5.0 (07/21/2023)
 ### Public API Changes
 * Removed recently added APIs `GeneralCache` and `MakeSharedGeneralCache()` as our plan changed to stop exposing a general-purpose cache interface. The old forms of these APIs, `Cache` and `NewLRUCache()`, are still available, although general-purpose caching support will be dropped eventually.
@@ -17,7 +17,7 @@ There are few options when compiling RocksDB:
 * `make check` will compile and run all the unit tests. `make check` will compile RocksDB in debug mode.

 * `make all` will compile our static library, and all our tools and unit tests. Our tools
-depend on gflags. You will need to have gflags installed to run `make all`. This will compile RocksDB in debug mode. Don't
+depend on gflags 2.2.0 or newer. You will need to have gflags installed to run `make all`. This will compile RocksDB in debug mode. Don't
 use binaries compiled by `make all` in production.

 * By default the binary we produce is optimized for the CPU you're compiling on
@@ -77,7 +77,7 @@ most processors made since roughly 2013.

              git clone https://github.com/gflags/gflags.git
              cd gflags
-              git checkout v2.0
+              git checkout v2.2.0
              ./configure && make && sudo make install

      **Notice**: Once installed, please add the include path for gflags to your `CPATH` environment variable and the
@@ -539,7 +539,7 @@ endif

 ifdef USE_CLANG
 	# Used by some teams in Facebook
-	WARNING_FLAGS += -Wshift-sign-overflow -Wambiguous-reversed-operator
+	WARNING_FLAGS += -Wshift-sign-overflow -Wambiguous-reversed-operator -Wimplicit-fallthrough
 endif

 ifeq ($(PLATFORM), OS_OPENBSD)
@@ -994,7 +994,7 @@ endif

 .PHONY: check_0
 check_0:
-	printf '%s\n' ''						\
+	@printf '%s\n' ''						\
 	  'To monitor subtest <duration,pass/fail,name>,'		\
 	  '  run "make watch-log" in a separate window' '';		\
 	{ \
@@ -1016,7 +1016,7 @@ valgrind-exclude-regexp = InlineSkipTest.ConcurrentInsert|TransactionStressTest.
 .PHONY: valgrind_check_0
 valgrind_check_0: test_log_prefix := valgrind_
 valgrind_check_0:
-	printf '%s\n' ''						\
+	@printf '%s\n' ''						\
 	  'To monitor subtest <duration,pass/fail,name>,'		\
 	  '  run "make watch-log" in a separate window' '';		\
 	{								\
@@ -1885,6 +1885,9 @@ compressed_secondary_cache_test: $(OBJ_DIR)/cache/compressed_secondary_cache_tes
 lru_cache_test: $(OBJ_DIR)/cache/lru_cache_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)

+tiered_secondary_cache_test: $(OBJ_DIR)/cache/tiered_secondary_cache_test.o $(TEST_LIBRARY) $(LIBRARY)
+	$(AM_LINK)
+
 range_del_aggregator_test: $(OBJ_DIR)/db/range_del_aggregator_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)

@@ -1984,6 +1987,9 @@ cache_reservation_manager_test: $(OBJ_DIR)/cache/cache_reservation_manager_test.
 wide_column_serialization_test: $(OBJ_DIR)/db/wide/wide_column_serialization_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)

+wide_columns_helper_test: $(OBJ_DIR)/db/wide/wide_columns_helper_test.o $(TEST_LIBRARY) $(LIBRARY)
+	$(AM_LINK)
+
 #-------------------------------------------------
 # make install related stuff
 PREFIX ?= /usr/local
@@ -2054,7 +2060,7 @@ JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/linux
 ifeq ($(PLATFORM), OS_SOLARIS)
 	ARCH := $(shell isainfo -b)
 else ifeq ($(PLATFORM), OS_OPENBSD)
-	ifneq (,$(filter amd64 ppc64 ppc64le s390x arm64 aarch64 sparc64 loongarch64, $(MACHINE)))
+	ifneq (,$(filter amd64 ppc64 ppc64le s390x arm64 aarch64 riscv64 sparc64 loongarch64, $(MACHINE)))
 		ARCH := 64
 	else
 		ARCH := 32
@@ -2075,7 +2081,7 @@ ifneq ($(origin JNI_LIBC), undefined)
 endif

 ifeq (,$(ROCKSDBJNILIB))
-ifneq (,$(filter ppc% s390x arm64 aarch64 sparc64 loongarch64, $(MACHINE)))
+ifneq (,$(filter ppc% s390x arm64 aarch64 riscv64 sparc64 loongarch64, $(MACHINE)))
 	ROCKSDBJNILIB = librocksdbjni-linux-$(MACHINE)$(JNI_LIBC_POSTFIX).so
 else
 	ROCKSDBJNILIB = librocksdbjni-linux$(ARCH)$(JNI_LIBC_POSTFIX).so
@@ -2088,8 +2094,8 @@ ROCKSDB_JAVADOCS_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-javadoc.jar
 ROCKSDB_SOURCES_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-sources.jar
 SHA256_CMD = sha256sum

-ZLIB_VER ?= 1.2.13
-ZLIB_SHA256 ?= b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30
+ZLIB_VER ?= 1.3.1
+ZLIB_SHA256 ?= 9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23
 ZLIB_DOWNLOAD_BASE ?= http://zlib.net
 BZIP2_VER ?= 1.0.8
 BZIP2_SHA256 ?= ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269
@@ -2097,11 +2103,11 @@ BZIP2_DOWNLOAD_BASE ?= http://sourceware.org/pub/bzip2
 SNAPPY_VER ?= 1.1.8
 SNAPPY_SHA256 ?= 16b677f07832a612b0836178db7f374e414f94657c138e6993cbfc5dcc58651f
 SNAPPY_DOWNLOAD_BASE ?= https://github.com/google/snappy/archive
-LZ4_VER ?= 1.9.3
-LZ4_SHA256 ?= 030644df4611007ff7dc962d981f390361e6c97a34e5cbc393ddfbe019ffe2c1
+LZ4_VER ?= 1.9.4
+LZ4_SHA256 ?= 0b0e3aa07c8c063ddf40b082bdf7e37a1562bda40a0ff5272957f3e987e0e54b
 LZ4_DOWNLOAD_BASE ?= https://github.com/lz4/lz4/archive
-ZSTD_VER ?= 1.4.9
-ZSTD_SHA256 ?= acf714d98e3db7b876e5b540cbf6dee298f60eb3c0723104f6d3f065cd60d6a8
+ZSTD_VER ?= 1.5.5
+ZSTD_SHA256 ?= 98e9c3d949d1b924e28e01eccb7deed865eefebf25c2f21c702e5cd5b63b85e1
 ZSTD_DOWNLOAD_BASE ?= https://github.com/facebook/zstd/archive
 CURL_SSL_OPTS ?= --tlsv1

@@ -2340,6 +2346,10 @@ rocksdbjavastaticdockers390x:
 	mkdir -p java/target
 	docker run --rm --name rocksdb_linux_s390x-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:ubuntu18_s390x-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh

+rocksdbjavastaticdockerriscv64:
+	mkdir -p java/target
+	docker run --rm --name rocksdb_linux_riscv64-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:ubuntu20_riscv64-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh
+
 rocksdbjavastaticdockerx86musl:
 	mkdir -p java/target
 	docker run --rm --name rocksdb_linux_x86-musl-be --platform linux/386 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:alpine3_x86-be /rocksdb-host/java/crossbuild/docker-build-linux-alpine.sh
@@ -2413,6 +2423,9 @@ jtest_run:
 jtest: rocksdbjava
 	cd java;$(MAKE) sample test

+jpmd: rocksdbjava rocksdbjavageneratepom
+	cd java;$(MAKE) pmd
+
 jdb_bench:
 	cd java;$(MAKE) db_bench;

@@ -5,4 +5,5 @@ This is the list of all known third-party plugins for RocksDB. If something is m
 * [ZenFS](https://github.com/westerndigitalcorporation/zenfs): a file system for zoned block devices
 * [RADOS](https://github.com/riversand963/rocksdb-rados-env): an Env used for interacting with RADOS. Migrated from RocksDB main repo.
 * [PMEM](https://github.com/pmem/pmem-rocksdb-plugin): a collection of plugins to enable Persistent Memory on RocksDB.
-* [IPPCP](https://github.com/intel/ippcp-plugin-rocksdb): a plugin to enable encryption on RocksDB based on Intel optimized open source IPP-Crypto library.
+* [IPPCP](https://github.com/intel/ippcp-plugin-rocksdb): a plugin to enable encryption on RocksDB based on Intel optimized open source IPP-Crypto library.
+* [encfs](https://github.com/pegasus-kv/encfs): a plugin to enable encryption on RocksDB based on OpenSSL library.
@@ -3,8 +3,6 @@
 # --> DO NOT EDIT MANUALLY <--
 # This file is a Facebook-specific integration for buck builds, so can
 # only be validated by Facebook employees.
-#
-# @noautodeps @nocodemods
 load("//rocks/buckifier:defs.bzl", "cpp_library_wrapper","rocks_cpp_library_wrapper","cpp_binary_wrapper","cpp_unittest_wrapper","fancy_bench_wrapper","add_c_test_wrapper")


@@ -21,6 +19,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
        "cache/secondary_cache.cc",
        "cache/secondary_cache_adapter.cc",
        "cache/sharded_cache.cc",
+        "cache/tiered_secondary_cache.cc",
        "db/arena_wrapped_db_iter.cc",
        "db/blob/blob_contents.cc",
        "db/blob/blob_fetcher.cc",
@@ -103,6 +102,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
        "db/wal_manager.cc",
        "db/wide/wide_column_serialization.cc",
        "db/wide/wide_columns.cc",
+        "db/wide/wide_columns_helper.cc",
        "db/write_batch.cc",
        "db/write_batch_base.cc",
        "db/write_controller.cc",
@@ -163,6 +163,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
        "options/configurable.cc",
        "options/customizable.cc",
        "options/db_options.cc",
+        "options/offpeak_time_info.cc",
        "options/options.cc",
        "options/options_helper.cc",
        "options/options_parser.cc",
@@ -393,6 +394,7 @@ rocks_cpp_library_wrapper(name="rocksdb_stress_lib", srcs=[
        "db_stress_tool/db_stress_stat.cc",
        "db_stress_tool/db_stress_test_base.cc",
        "db_stress_tool/db_stress_tool.cc",
+        "db_stress_tool/db_stress_wide_merge_operator.cc",
        "db_stress_tool/expected_state.cc",
        "db_stress_tool/expected_value.cc",
        "db_stress_tool/multi_ops_txns_stress.cc",
@@ -405,6 +407,8 @@ rocks_cpp_library_wrapper(name="rocksdb_stress_lib", srcs=[

 cpp_binary_wrapper(name="db_stress", srcs=["db_stress_tool/db_stress.cc"], deps=[":rocksdb_stress_lib"], extra_preprocessor_flags=[], extra_bench_libs=False)

+cpp_binary_wrapper(name="cache_bench", srcs=["cache/cache_bench.cc"], deps=[":rocksdb_cache_bench_tools_lib"], extra_preprocessor_flags=[], extra_bench_libs=False)
+
 cpp_binary_wrapper(name="ribbon_bench", srcs=["microbench/ribbon_bench.cc"], deps=[], extra_preprocessor_flags=[], extra_bench_libs=True)

 cpp_binary_wrapper(name="db_basic_bench", srcs=["microbench/db_basic_bench.cc"], deps=[], extra_preprocessor_flags=[], extra_bench_libs=True)
@@ -5474,6 +5478,12 @@ cpp_unittest_wrapper(name="tiered_compaction_test",
            extra_compiler_flags=[])


+cpp_unittest_wrapper(name="tiered_secondary_cache_test",
+            srcs=["cache/tiered_secondary_cache_test.cc"],
+            deps=[":rocksdb_test_lib"],
+            extra_compiler_flags=[])
+
+
 cpp_unittest_wrapper(name="timer_queue_test",
            srcs=["util/timer_queue_test.cc"],
            deps=[":rocksdb_test_lib"],
@@ -5552,6 +5562,12 @@ cpp_unittest_wrapper(name="wide_column_serialization_test",
            extra_compiler_flags=[])


+cpp_unittest_wrapper(name="wide_columns_helper_test",
+            srcs=["db/wide/wide_columns_helper_test.cc"],
+            deps=[":rocksdb_test_lib"],
+            extra_compiler_flags=[])
+
+
 cpp_unittest_wrapper(name="work_queue_test",
            srcs=["util/work_queue_test.cc"],
            deps=[":rocksdb_test_lib"],
@@ -5605,3 +5621,5 @@ cpp_unittest_wrapper(name="write_unprepared_transaction_test",
            deps=[":rocksdb_test_lib"],
            extra_compiler_flags=[])

+
+export_file(name = "tools/db_crashtest.py")
@@ -152,6 +152,9 @@ LzLabs is using RocksDB as a storage engine in their multi-database distributed
 ## ArangoDB
 [ArangoDB](https://www.arangodb.com/) is a native multi-model database with flexible data models for documents, graphs, and key-values, for building high performance applications using a convenient SQL-like query language or JavaScript extensions. It uses RocksDB as its storage engine.

+## Qdrant
+[Qdrant](https://qdrant.tech/) is an open source vector database, it [uses](https://qdrant.tech/documentation/concepts/storage/) RocksDB as its persistent storage.
+
 ## Milvus
 [Milvus](https://milvus.io/) is an open source vector database for unstructured data. It uses RocksDB not only as one of the supported kv storage engines, but also as a message queue.

@@ -161,5 +164,9 @@ LzLabs is using RocksDB as a storage engine in their multi-database distributed
 ## Solana Labs
 [Solana](https://github.com/solana-labs/solana) is a fast, secure, scalable, and decentralized blockchain.  It uses RocksDB as the underlying storage for its ledger store.

+## Apache Kvrocks
+
+[Apache Kvrocks](https://github.com/apache/kvrocks) is an open-source distributed key-value NoSQL database built on top of RocksDB. It serves as a cost-saving and capacity-increasing alternative drop-in replacement for Redis.
+
 ## Others
 More databases using RocksDB can be found at [dbdb.io](https://dbdb.io/browse?embeds=rocksdb).
@@ -197,6 +197,10 @@ def generate_targets(repo_path, deps_map):
    TARGETS.add_binary(
        "db_stress", ["db_stress_tool/db_stress.cc"], [":rocksdb_stress_lib"]
    )
+    # cache_bench binary
+    TARGETS.add_binary(
+        "cache_bench", ["cache/cache_bench.cc"], [":rocksdb_cache_bench_tools_lib"]
+    )
    # bench binaries
    for src in src_mk.get("MICROBENCH_SOURCES", []):
        name = src.rsplit("/", 1)[1].split(".")[0] if "/" in src else src.split(".")[0]
@@ -299,6 +303,7 @@ def generate_targets(repo_path, deps_map):
                    deps=json.dumps(deps["extra_deps"] + [":rocksdb_test_lib"]),
                    extra_compiler_flags=json.dumps(deps["extra_compiler_flags"]),
                )
+    TARGETS.export_file("tools/db_crashtest.py")

    print(ColorString.info("Generated TARGETS Summary:"))
    print(ColorString.info("- %d libs" % TARGETS.total_lib))
@@ -23,7 +23,7 @@ def pretty_list(lst, indent=8):
    return res


-class TARGETSBuilder(object):
+class TARGETSBuilder:
    def __init__(self, path, extra_argv):
        self.path = path
        header = targets_cfg.rocksdb_target_header_template.format(
@@ -148,3 +148,9 @@ add_c_test_wrapper()
                ).encode("utf-8")
            )
        self.total_test = self.total_test + 1
+
+    def export_file(self, name):
+        with open(self.path, "a") as targets_file:
+            targets_file.write(
+                targets_cfg.export_file_template.format(name=name)
+            )
@@ -6,8 +6,6 @@ rocksdb_target_header_template = """# This file \100generated by:
 # --> DO NOT EDIT MANUALLY <--
 # This file is a Facebook-specific integration for buck builds, so can
 # only be validated by Facebook employees.
-#
-# @noautodeps @nocodemods
 load("//rocks/buckifier:defs.bzl", "cpp_library_wrapper","rocks_cpp_library_wrapper","cpp_binary_wrapper","cpp_unittest_wrapper","fancy_bench_wrapper","add_c_test_wrapper")

 """
@@ -39,3 +37,7 @@ fancy_bench_template = """
 fancy_bench_wrapper(suite_name="{name}", binary_to_bench_to_metric_list_map={bench_config}, slow={slow}, expected_runtime={expected_runtime}, sl_iterations={sl_iterations}, regression_threshold={regression_threshold})

 """
+
+export_file_template = """
+export_file(name = "{name}")
+"""
@@ -14,7 +14,7 @@ import sys
 import time


-class ColorString(object):
+class ColorString:
    """Generate colorful strings on terminal"""

    HEADER = "\033[95m"
@@ -647,8 +647,10 @@ if [ "$PORTABLE" == "" ] || [ "$PORTABLE" == 0 ]; then
    fi
    COMMON_FLAGS="$COMMON_FLAGS"
  elif test -n "`echo $TARGET_ARCHITECTURE | grep ^riscv64`"; then
-    RISC_ISA=$(cat /proc/cpuinfo | grep isa | head -1 | cut --delimiter=: -f 2 | cut -b 2-)
-    COMMON_FLAGS="$COMMON_FLAGS -march=${RISC_ISA}"
+    RISC_ISA=$(cat /proc/cpuinfo | grep -E '^isa\s*:' | head -1 | cut --delimiter=: -f 2 | cut -b 2-)
+    if [ -n "${RISCV_ISA}" ]; then
+      COMMON_FLAGS="$COMMON_FLAGS -march=${RISC_ISA}"
+    fi
  elif [ "$TARGET_OS" == "IOS" ]; then
    COMMON_FLAGS="$COMMON_FLAGS"
  else
@@ -660,8 +662,7 @@ else
    if test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then
      COMMON_FLAGS="$COMMON_FLAGS -march=z196 "
    elif test -n "`echo $TARGET_ARCHITECTURE | grep ^riscv64`"; then
-      RISC_ISA=$(cat /proc/cpuinfo | grep isa | head -1 | cut --delimiter=: -f 2 | cut -b 2-)
-      COMMON_FLAGS="$COMMON_FLAGS -march=${RISC_ISA}"
+      COMMON_FLAGS="$COMMON_FLAGS -march=rv64gc"
    elif test "$USE_SSE"; then
      # USE_SSE is DEPRECATED
      # This is a rough approximation of the old USE_SSE behavior
@@ -674,7 +675,7 @@ else
  fi

  if [[ "${PLATFORM}" == "OS_MACOSX" ]]; then
-    # For portability compile for macOS 10.14 or newer
+    # For portability compile for macOS 10.14 (2018) or newer
    COMMON_FLAGS="$COMMON_FLAGS -mmacosx-version-min=10.14"
    PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -mmacosx-version-min=10.14"
    # -mmacosx-version-min must come first here.
@@ -15,7 +15,7 @@ import re
 import sys


-class ErrorParserBase(object):
+class ErrorParserBase:
    def parse_error(self, line):
        """Parses a line of test output. If it contains an error, returns a
        formatted message describing the error; otherwise, returns None.
@@ -66,6 +66,41 @@ static std::unordered_map<std::string, OptionTypeInfo>
          OptionTypeFlags::kMutable}},
 };

+namespace {
+static void NoopDelete(Cache::ObjectPtr /*obj*/,
+                       MemoryAllocator* /*allocator*/) {
+  assert(false);
+}
+
+static size_t SliceSize(Cache::ObjectPtr obj) {
+  return static_cast<Slice*>(obj)->size();
+}
+
+static Status SliceSaveTo(Cache::ObjectPtr from_obj, size_t from_offset,
+                          size_t length, char* out) {
+  const Slice& slice = *static_cast<Slice*>(from_obj);
+  std::memcpy(out, slice.data() + from_offset, length);
+  return Status::OK();
+}
+
+static Status NoopCreate(const Slice& /*data*/, CompressionType /*type*/,
+                         CacheTier /*source*/, Cache::CreateContext* /*ctx*/,
+                         MemoryAllocator* /*allocator*/,
+                         Cache::ObjectPtr* /*out_obj*/,
+                         size_t* /*out_charge*/) {
+  assert(false);
+  return Status::NotSupported();
+}
+
+static Cache::CacheItemHelper kBasicCacheItemHelper(CacheEntryRole::kMisc,
+                                                    &NoopDelete);
+}  // namespace
+
+const Cache::CacheItemHelper kSliceCacheItemHelper{
+    CacheEntryRole::kMisc, &NoopDelete, &SliceSize,
+    &SliceSaveTo,          &NoopCreate, &kBasicCacheItemHelper,
+};
+
 Status SecondaryCache::CreateFromString(
    const ConfigOptions& config_options, const std::string& value,
    std::shared_ptr<SecondaryCache>* result) {
@@ -3,7 +3,6 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).

-#include "cache_key.h"
 #ifdef GFLAGS
 #include <cinttypes>
 #include <cstddef>
@@ -13,9 +12,12 @@
 #include <set>
 #include <sstream>

+#include "cache/cache_key.h"
+#include "cache/sharded_cache.h"
 #include "db/db_impl/db_impl.h"
 #include "monitoring/histogram.h"
 #include "port/port.h"
+#include "port/stack_trace.h"
 #include "rocksdb/advanced_cache.h"
 #include "rocksdb/convenience.h"
 #include "rocksdb/db.h"
@@ -44,7 +46,12 @@ static constexpr uint64_t GiB = MiB << 10;
 DEFINE_uint32(threads, 16, "Number of concurrent threads to run.");
 DEFINE_uint64(cache_size, 1 * GiB,
              "Number of bytes to use as a cache of uncompressed data.");
-DEFINE_uint32(num_shard_bits, 6, "shard_bits.");
+DEFINE_int32(num_shard_bits, -1,
+             "ShardedCacheOptions::shard_bits. Default = auto");
+DEFINE_int32(
+    eviction_effort_cap,
+    ROCKSDB_NAMESPACE::HyperClockCacheOptions(1, 1).eviction_effort_cap,
+    "HyperClockCacheOptions::eviction_effort_cap");

 DEFINE_double(resident_ratio, 0.25,
              "Ratio of keys fitting in cache to keyspace.");
@@ -54,14 +61,28 @@ DEFINE_uint32(value_bytes_estimate, 0,
              "If > 0, overrides estimated_entry_charge or "
              "min_avg_entry_charge depending on cache_type.");

+DEFINE_int32(
+    degenerate_hash_bits, 0,
+    "With HCC, fix this many hash bits to increase table hash collisions");
 DEFINE_uint32(skew, 5, "Degree of skew in key selection. 0 = no skew");
 DEFINE_bool(populate_cache, true, "Populate cache before operations");

-DEFINE_uint32(lookup_insert_percent, 87,
+DEFINE_double(pinned_ratio, 0.25,
+              "Keep roughly this portion of entries pinned in cache.");
+DEFINE_double(
+    vary_capacity_ratio, 0.0,
+    "If greater than 0.0, will periodically vary the capacity between this "
+    "ratio less than full size and full size. If vary_capacity_ratio + "
+    "pinned_ratio is close to or exceeds 1.0, the cache might thrash.");
+
+DEFINE_uint32(lookup_insert_percent, 82,
              "Ratio of lookup (+ insert on not found) to total workload "
              "(expressed as a percentage)");
 DEFINE_uint32(insert_percent, 2,
              "Ratio of insert to total workload (expressed as a percentage)");
+DEFINE_uint32(blind_insert_percent, 5,
+              "Ratio of insert without keeping handle to total workload "
+              "(expressed as a percentage)");
 DEFINE_uint32(lookup_percent, 10,
              "Ratio of lookup to total workload (expressed as a percentage)");
 DEFINE_uint32(erase_percent, 1,
@@ -76,6 +97,8 @@ DEFINE_uint32(
 DEFINE_uint32(gather_stats_entries_per_lock, 256,
              "For Cache::ApplyToAllEntries");

+DEFINE_uint32(usleep, 0, "Sleep up to this many microseconds after each op.");
+
 DEFINE_bool(lean, false,
            "If true, no additional computation is performed besides cache "
            "operations.");
@@ -93,10 +116,20 @@ DEFINE_uint32(seed, 0, "Hashing/random seed to use. 0 = choose at random");

 DEFINE_string(secondary_cache_uri, "",
              "Full URI for creating a custom secondary cache object");
-static class std::shared_ptr<ROCKSDB_NAMESPACE::SecondaryCache> secondary_cache;

 DEFINE_string(cache_type, "lru_cache", "Type of block cache.");

+DEFINE_bool(use_jemalloc_no_dump_allocator, false,
+            "Whether to use JemallocNoDumpAllocator");
+
+DEFINE_uint32(jemalloc_no_dump_allocator_num_arenas,
+              ROCKSDB_NAMESPACE::JemallocAllocatorOptions().num_arenas,
+              "JemallocNodumpAllocator::num_arenas");
+
+DEFINE_bool(jemalloc_no_dump_allocator_limit_tcache_size,
+            ROCKSDB_NAMESPACE::JemallocAllocatorOptions().limit_tcache_size,
+            "JemallocNodumpAllocator::limit_tcache_size");
+
 // ## BEGIN stress_cache_key sub-tool options ##
 // See class StressCacheKey below.
 DEFINE_bool(stress_cache_key, false,
@@ -181,16 +214,19 @@ class SharedState {

  bool Started() const { return start_; }

-  void AddLookupStats(uint64_t hits, uint64_t misses) {
+  void AddLookupStats(uint64_t hits, uint64_t misses, size_t pinned_count) {
    MutexLock l(&mu_);
    lookup_count_ += hits + misses;
    lookup_hits_ += hits;
+    pinned_count_ += pinned_count;
  }

  double GetLookupHitRatio() const {
    return 1.0 * lookup_hits_ / lookup_count_;
  }

+  size_t GetPinnedCount() const { return pinned_count_; }
+
 private:
  port::Mutex mu_;
  port::CondVar cv_;
@@ -202,6 +238,7 @@ class SharedState {
  uint64_t num_done_ = 0;
  uint64_t lookup_count_ = 0;
  uint64_t lookup_hits_ = 0;
+  size_t pinned_count_ = 0;
 };

 // Per-thread state for concurrent executions of the same benchmark.
@@ -226,6 +263,20 @@ struct KeyGen {
      raw = std::min(raw, rnd.Next());
    }
    uint64_t key = FastRange64(raw, max_key);
+    if (FLAGS_degenerate_hash_bits) {
+      uint64_t key_hash =
+          Hash64(reinterpret_cast<const char*>(&key), sizeof(key));
+      // HCC uses the high 64 bits and a lower bit mask for starting probe
+      // location, so we fix hash bits starting at the bottom of that word.
+      auto hi_hash = uint64_t{0x9e3779b97f4a7c13U} ^
+                     (key_hash << 1 << (FLAGS_degenerate_hash_bits - 1));
+      uint64_t un_hi, un_lo;
+      BijectiveUnhash2x64(hi_hash, key_hash, &un_hi, &un_lo);
+      un_lo ^= BitwiseAnd(FLAGS_seed, INT32_MAX);
+      EncodeFixed64(key_data, un_lo);
+      EncodeFixed64(key_data + 8, un_hi);
+      return Slice(key_data, kCacheKeySize);
+    }
    // Variable size and alignment
    size_t off = key % 8;
    key_data[0] = char{42};
@@ -239,8 +290,8 @@ struct KeyGen {
  }
 };

-Cache::ObjectPtr createValue(Random64& rnd) {
-  char* rv = new char[FLAGS_value_bytes];
+Cache::ObjectPtr createValue(Random64& rnd, MemoryAllocator* alloc) {
+  char* rv = AllocateBlock(FLAGS_value_bytes, alloc).release();
  // Fill with some filler data, and take some CPU time
  for (uint32_t i = 0; i < FLAGS_value_bytes; i += 8) {
    EncodeFixed64(rv + i, rnd.Next());
@@ -257,7 +308,8 @@ Status SaveToFn(Cache::ObjectPtr from_obj, size_t /*from_offset*/,
  return Status::OK();
 }

-Status CreateFn(const Slice& data, Cache::CreateContext* /*context*/,
+Status CreateFn(const Slice& data, CompressionType /*type*/,
+                CacheTier /*source*/, Cache::CreateContext* /*context*/,
                MemoryAllocator* /*allocator*/, Cache::ObjectPtr* out_obj,
                size_t* out_charge) {
  *out_obj = new char[data.size()];
@@ -266,8 +318,8 @@ Status CreateFn(const Slice& data, Cache::CreateContext* /*context*/,
  return Status::OK();
 };

-void DeleteFn(Cache::ObjectPtr value, MemoryAllocator* /*alloc*/) {
-  delete[] static_cast<char*>(value);
+void DeleteFn(Cache::ObjectPtr value, MemoryAllocator* alloc) {
+  CustomDeleter{alloc}(static_cast<char*>(value));
 }

 Cache::CacheItemHelper helper1_wos(CacheEntryRole::kDataBlock, DeleteFn);
@@ -279,6 +331,28 @@ Cache::CacheItemHelper helper2(CacheEntryRole::kIndexBlock, DeleteFn, SizeFn,
 Cache::CacheItemHelper helper3_wos(CacheEntryRole::kFilterBlock, DeleteFn);
 Cache::CacheItemHelper helper3(CacheEntryRole::kFilterBlock, DeleteFn, SizeFn,
                               SaveToFn, CreateFn, &helper3_wos);
+
+void ConfigureSecondaryCache(ShardedCacheOptions& opts) {
+  if (!FLAGS_secondary_cache_uri.empty()) {
+    std::shared_ptr<SecondaryCache> secondary_cache;
+    Status s = SecondaryCache::CreateFromString(
+        ConfigOptions(), FLAGS_secondary_cache_uri, &secondary_cache);
+    if (secondary_cache == nullptr) {
+      fprintf(stderr,
+              "No secondary cache registered matching string: %s status=%s\n",
+              FLAGS_secondary_cache_uri.c_str(), s.ToString().c_str());
+      exit(1);
+    }
+    opts.secondary_cache = secondary_cache;
+  }
+}
+
+ShardedCacheBase* AsShardedCache(Cache* c) {
+  if (!FLAGS_secondary_cache_uri.empty()) {
+    c = static_cast_with_check<CacheWrapper>(c)->GetTarget().get();
+  }
+  return static_cast_with_check<ShardedCacheBase>(c);
+}
 }  // namespace

 class CacheBench {
@@ -293,7 +367,9 @@ class CacheBench {
                                 FLAGS_lookup_insert_percent),
        insert_threshold_(lookup_insert_threshold_ +
                          kHundredthUint64 * FLAGS_insert_percent),
-        lookup_threshold_(insert_threshold_ +
+        blind_insert_threshold_(insert_threshold_ +
+                                kHundredthUint64 * FLAGS_blind_insert_percent),
+        lookup_threshold_(blind_insert_threshold_ +
                          kHundredthUint64 * FLAGS_lookup_percent),
        erase_threshold_(lookup_threshold_ +
                         kHundredthUint64 * FLAGS_erase_percent) {
@@ -302,6 +378,15 @@ class CacheBench {
      exit(1);
    }

+    std::shared_ptr<MemoryAllocator> allocator;
+    if (FLAGS_use_jemalloc_no_dump_allocator) {
+      JemallocAllocatorOptions opts;
+      opts.num_arenas = FLAGS_jemalloc_no_dump_allocator_num_arenas;
+      opts.limit_tcache_size =
+          FLAGS_jemalloc_no_dump_allocator_limit_tcache_size;
+      Status s = NewJemallocNodumpAllocator(opts, &allocator);
+      assert(s.ok());
+    }
    if (FLAGS_cache_type == "clock_cache") {
      fprintf(stderr, "Old clock cache implementation has been removed.\n");
      exit(1);
@@ -309,6 +394,8 @@ class CacheBench {
      HyperClockCacheOptions opts(
          FLAGS_cache_size, /*estimated_entry_charge=*/0, FLAGS_num_shard_bits);
      opts.hash_seed = BitwiseAnd(FLAGS_seed, INT32_MAX);
+      opts.memory_allocator = allocator;
+      opts.eviction_effort_cap = FLAGS_eviction_effort_cap;
      if (FLAGS_cache_type == "fixed_hyper_clock_cache" ||
          FLAGS_cache_type == "hyper_clock_cache") {
        opts.estimated_entry_charge = FLAGS_value_bytes_estimate > 0
@@ -319,31 +406,21 @@ class CacheBench {
          opts.min_avg_entry_charge = FLAGS_value_bytes_estimate;
        }
      } else {
-        fprintf(stderr, "Cache type not supported.");
+        fprintf(stderr, "Cache type not supported.\n");
        exit(1);
      }
+      ConfigureSecondaryCache(opts);
      cache_ = opts.MakeSharedCache();
    } else if (FLAGS_cache_type == "lru_cache") {
      LRUCacheOptions opts(FLAGS_cache_size, FLAGS_num_shard_bits,
                           false /* strict_capacity_limit */,
                           0.5 /* high_pri_pool_ratio */);
      opts.hash_seed = BitwiseAnd(FLAGS_seed, INT32_MAX);
-      if (!FLAGS_secondary_cache_uri.empty()) {
-        Status s = SecondaryCache::CreateFromString(
-            ConfigOptions(), FLAGS_secondary_cache_uri, &secondary_cache);
-        if (secondary_cache == nullptr) {
-          fprintf(
-              stderr,
-              "No secondary cache registered matching string: %s status=%s\n",
-              FLAGS_secondary_cache_uri.c_str(), s.ToString().c_str());
-          exit(1);
-        }
-        opts.secondary_cache = secondary_cache;
-      }
-
+      opts.memory_allocator = allocator;
+      ConfigureSecondaryCache(opts);
      cache_ = NewLRUCache(opts);
    } else {
-      fprintf(stderr, "Cache type not supported.");
+      fprintf(stderr, "Cache type not supported.\n");
      exit(1);
    }
  }
@@ -373,7 +450,8 @@ class CacheBench {
      keys_since_last_not_found = 0;

      Status s =
-          cache_->Insert(key, createValue(rnd), &helper1, FLAGS_value_bytes);
+          cache_->Insert(key, createValue(rnd, cache_->memory_allocator()),
+                         &helper1, FLAGS_value_bytes);
      assert(s.ok());

      handle = cache_->Lookup(key);
@@ -459,6 +537,8 @@ class CacheBench {
    size_t slot = cache_->GetTableAddressCount();
    printf("Final load factor: %g (%zu / %zu)\n", 1.0 * occ / slot, occ, slot);

+    printf("Final pinned count: %zu\n", shared.GetPinnedCount());
+
    if (FLAGS_histograms) {
      printf("\nOperation latency (ns):\n");
      HistogramImpl combined;
@@ -490,6 +570,7 @@ class CacheBench {
  // Cumulative thresholds in the space of a random uint64_t
  const uint64_t lookup_insert_threshold_;
  const uint64_t insert_threshold_;
+  const uint64_t blind_insert_threshold_;
  const uint64_t lookup_threshold_;
  const uint64_t erase_threshold_;

@@ -605,28 +686,44 @@ class CacheBench {
    uint64_t lookup_misses = 0;
    uint64_t lookup_hits = 0;
    // To hold handles for a non-trivial amount of time
-    Cache::Handle* handle = nullptr;
+    std::deque<Cache::Handle*> pinned;
+    size_t total_pin_count = static_cast<size_t>(
+        (FLAGS_cache_size * FLAGS_pinned_ratio) / FLAGS_value_bytes + 0.999999);
+    // For this thread. Some round up, some round down, as appropriate
+    size_t pin_count = (total_pin_count + thread->tid) / FLAGS_threads;
+
    KeyGen gen;
    const auto clock = SystemClock::Default().get();
    uint64_t start_time = clock->NowMicros();
    StopWatchNano timer(clock);
+    auto system_clock = SystemClock::Default();
+    size_t steps_to_next_capacity_change = 0;

    for (uint64_t i = 0; i < FLAGS_ops_per_thread; i++) {
      Slice key = gen.GetRand(thread->rnd, max_key_, FLAGS_skew);
      uint64_t random_op = thread->rnd.Next();

+      if (FLAGS_vary_capacity_ratio > 0.0 && thread->tid == 0) {
+        if (steps_to_next_capacity_change == 0) {
+          double cut_ratio = static_cast<double>(thread->rnd.Next()) /
+                             static_cast<double>(UINT64_MAX) *
+                             FLAGS_vary_capacity_ratio;
+          cache_->SetCapacity(FLAGS_cache_size * (1.0 - cut_ratio));
+          steps_to_next_capacity_change =
+              static_cast<size_t>(FLAGS_ops_per_thread / 100);
+        } else {
+          --steps_to_next_capacity_change;
+        }
+      }
+
      if (FLAGS_histograms) {
        timer.Start();
      }

      if (random_op < lookup_insert_threshold_) {
-        if (handle) {
-          cache_->Release(handle);
-          handle = nullptr;
-        }
        // do lookup
-        handle = cache_->Lookup(key, &helper2, /*context*/ nullptr,
-                                Cache::Priority::LOW);
+        auto handle = cache_->Lookup(key, &helper2, /*context*/ nullptr,
+                                     Cache::Priority::LOW);
        if (handle) {
          ++lookup_hits;
          if (!FLAGS_lean) {
@@ -634,30 +731,31 @@ class CacheBench {
            result += NPHash64(static_cast<char*>(cache_->Value(handle)),
                               FLAGS_value_bytes);
          }
+          pinned.push_back(handle);
        } else {
          ++lookup_misses;
          // do insert
-          Status s = cache_->Insert(key, createValue(thread->rnd), &helper2,
-                                    FLAGS_value_bytes, &handle);
+          Status s = cache_->Insert(
+              key, createValue(thread->rnd, cache_->memory_allocator()),
+              &helper2, FLAGS_value_bytes, &pinned.emplace_back());
          assert(s.ok());
        }
      } else if (random_op < insert_threshold_) {
-        if (handle) {
-          cache_->Release(handle);
-          handle = nullptr;
-        }
        // do insert
-        Status s = cache_->Insert(key, createValue(thread->rnd), &helper3,
-                                  FLAGS_value_bytes, &handle);
+        Status s = cache_->Insert(
+            key, createValue(thread->rnd, cache_->memory_allocator()), &helper3,
+            FLAGS_value_bytes, &pinned.emplace_back());
+        assert(s.ok());
+      } else if (random_op < blind_insert_threshold_) {
+        // insert without keeping a handle
+        Status s = cache_->Insert(
+            key, createValue(thread->rnd, cache_->memory_allocator()), &helper3,
+            FLAGS_value_bytes);
        assert(s.ok());
      } else if (random_op < lookup_threshold_) {
-        if (handle) {
-          cache_->Release(handle);
-          handle = nullptr;
-        }
        // do lookup
-        handle = cache_->Lookup(key, &helper2, /*context*/ nullptr,
-                                Cache::Priority::LOW);
+        auto handle = cache_->Lookup(key, &helper2, /*context*/ nullptr,
+                                     Cache::Priority::LOW);
        if (handle) {
          ++lookup_hits;
          if (!FLAGS_lean) {
@@ -665,6 +763,7 @@ class CacheBench {
            result += NPHash64(static_cast<char*>(cache_->Value(handle)),
                               FLAGS_value_bytes);
          }
+          pinned.push_back(handle);
        } else {
          ++lookup_misses;
        }
@@ -678,13 +777,24 @@ class CacheBench {
      if (FLAGS_histograms) {
        thread->latency_ns_hist.Add(timer.ElapsedNanos());
      }
-      thread->shared->AddLookupStats(lookup_hits, lookup_misses);
+      if (FLAGS_usleep > 0) {
+        unsigned us =
+            static_cast<unsigned>(thread->rnd.Uniform(FLAGS_usleep + 1));
+        if (us > 0) {
+          system_clock->SleepForMicroseconds(us);
+        }
+      }
+      while (pinned.size() > pin_count) {
+        cache_->Release(pinned.front());
+        pinned.pop_front();
+      }
    }
    if (FLAGS_early_exit) {
      MutexLock l(thread->shared->GetMutex());
      exit(0);
    }
-    if (handle) {
+    thread->shared->AddLookupStats(lookup_hits, lookup_misses, pinned.size());
+    for (auto handle : pinned) {
      cache_->Release(handle);
      handle = nullptr;
    }
@@ -712,7 +822,8 @@ class CacheBench {
    printf("Ops per thread      : %" PRIu64 "\n", FLAGS_ops_per_thread);
    printf("Cache size          : %s\n",
           BytesToHumanString(FLAGS_cache_size).c_str());
-    printf("Num shard bits      : %u\n", FLAGS_num_shard_bits);
+    printf("Num shard bits      : %d\n",
+           AsShardedCache(cache_.get())->GetNumShardBits());
    printf("Max key             : %" PRIu64 "\n", max_key_);
    printf("Resident ratio      : %g\n", FLAGS_resident_ratio);
    printf("Skew degree         : %u\n", FLAGS_skew);
@@ -1032,6 +1143,7 @@ class StressCacheKey {
 };

 int cache_bench_tool(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ParseCommandLineFlags(&argc, &argv, true);

  if (FLAGS_stress_cache_key) {
@@ -25,7 +25,8 @@ Status WarmInCache(Cache* cache, const Slice& key, const Slice& saved,
  assert(helper->create_cb);
  Cache::ObjectPtr value;
  size_t charge;
-  Status st = helper->create_cb(saved, create_context,
+  Status st = helper->create_cb(saved, CompressionType::kNoCompression,
+                                CacheTier::kVolatileTier, create_context,
                                cache->memory_allocator(), &value, &charge);
  if (st.ok()) {
    st =
@@ -273,9 +273,10 @@ class ConcurrentCacheReservationManager
    std::size_t total_mem_used = cache_res_mgr_->GetTotalMemoryUsed();
    Status s;
    if (!increase) {
-      assert(total_mem_used >= memory_used_delta);
-      s = cache_res_mgr_->UpdateCacheReservation(total_mem_used -
-                                                 memory_used_delta);
+      s = cache_res_mgr_->UpdateCacheReservation(
+          (total_mem_used > memory_used_delta)
+              ? (total_mem_used - memory_used_delta)
+              : 0);
    } else {
      s = cache_res_mgr_->UpdateCacheReservation(total_mem_used +
                                                 memory_used_delta);
@@ -175,7 +175,7 @@ class LRUCacheTest : public CacheTest {};
 TEST_P(CacheTest, UsageTest) {
  // cache is std::shared_ptr and will be automatically cleaned up.
  const size_t kCapacity = 100000;
-  auto cache = NewCache(kCapacity, 8, false, kDontChargeCacheMetadata);
+  auto cache = NewCache(kCapacity, 6, false, kDontChargeCacheMetadata);
  auto precise_cache = NewCache(kCapacity, 0, false, kFullChargeCacheMetadata);
  ASSERT_EQ(0, cache->GetUsage());
  size_t baseline_meta_usage = precise_cache->GetUsage();
@@ -193,9 +193,13 @@ TEST_P(CacheTest, UsageTest) {
    ASSERT_OK(precise_cache->Insert(key, value, &kDumbHelper, kv_size));
    usage += kv_size;
    ASSERT_EQ(usage, cache->GetUsage());
-    if (IsHyperClock()) {
+    if (GetParam() == kFixedHyperClock) {
      ASSERT_EQ(baseline_meta_usage + usage, precise_cache->GetUsage());
    } else {
+      // AutoHyperClockCache meta usage grows in proportion to lifetime
+      // max number of entries. LRUCache in proportion to resident number of
+      // entries, though there is an untracked component proportional to
+      // lifetime max number of entries.
      ASSERT_LT(usage, precise_cache->GetUsage());
    }
  }
@@ -203,7 +207,11 @@ TEST_P(CacheTest, UsageTest) {
  cache->EraseUnRefEntries();
  precise_cache->EraseUnRefEntries();
  ASSERT_EQ(0, cache->GetUsage());
-  ASSERT_EQ(baseline_meta_usage, precise_cache->GetUsage());
+  if (GetParam() != kAutoHyperClock) {
+    // NOTE: AutoHyperClockCache meta usage grows in proportion to lifetime
+    // max number of entries.
+    ASSERT_EQ(baseline_meta_usage, precise_cache->GetUsage());
+  }

  // make sure the cache will be overloaded
  for (size_t i = 1; i < kCapacity; ++i) {
@@ -318,7 +326,11 @@ TEST_P(CacheTest, PinnedUsageTest) {
  cache->EraseUnRefEntries();
  precise_cache->EraseUnRefEntries();
  ASSERT_EQ(0, cache->GetUsage());
-  ASSERT_EQ(baseline_meta_usage, precise_cache->GetUsage());
+  if (GetParam() != kAutoHyperClock) {
+    // NOTE: AutoHyperClockCache meta usage grows in proportion to lifetime
+    // max number of entries.
+    ASSERT_EQ(baseline_meta_usage, precise_cache->GetUsage());
+  }
 }

 TEST_P(CacheTest, HitAndMiss) {
@@ -19,8 +19,10 @@ ChargedCache::ChargedCache(std::shared_ptr<Cache> cache,

 Status ChargedCache::Insert(const Slice& key, ObjectPtr obj,
                            const CacheItemHelper* helper, size_t charge,
-                            Handle** handle, Priority priority) {
-  Status s = target_->Insert(key, obj, helper, charge, handle, priority);
+                            Handle** handle, Priority priority,
+                            const Slice& compressed_val, CompressionType type) {
+  Status s = target_->Insert(key, obj, helper, charge, handle, priority,
+                             compressed_val, type);
  if (s.ok()) {
    // Insert may cause the cache entry eviction if the cache is full. So we
    // directly call the reservation manager to update the total memory used
@@ -22,9 +22,11 @@ class ChargedCache : public CacheWrapper {
  ChargedCache(std::shared_ptr<Cache> cache,
               std::shared_ptr<Cache> block_cache);

-  Status Insert(const Slice& key, ObjectPtr obj, const CacheItemHelper* helper,
-                size_t charge, Handle** handle = nullptr,
-                Priority priority = Priority::LOW) override;
+  Status Insert(
+      const Slice& key, ObjectPtr obj, const CacheItemHelper* helper,
+      size_t charge, Handle** handle = nullptr,
+      Priority priority = Priority::LOW, const Slice& compressed_val = Slice(),
+      CompressionType type = CompressionType::kNoCompression) override;

  Cache::Handle* Lookup(const Slice& key, const CacheItemHelper* helper,
                        CreateContext* create_context,
@@ -11,6 +11,7 @@

 #include <array>
 #include <atomic>
+#include <climits>
 #include <cstddef>
 #include <cstdint>
 #include <memory>
@@ -20,9 +21,11 @@
 #include "cache/sharded_cache.h"
 #include "port/lang.h"
 #include "port/malloc.h"
+#include "port/mmap.h"
 #include "port/port.h"
 #include "rocksdb/cache.h"
 #include "rocksdb/secondary_cache.h"
+#include "util/atomic.h"
 #include "util/autovector.h"
 #include "util/math.h"

@@ -39,24 +42,31 @@ class ClockCacheTest;
 //
 // Benefits
 // --------
-// * Fully lock free (no waits or spins) for efficiency under high concurrency
+// * Lock/wait free (no waits or spins) for efficiency under high concurrency
+//   * Fixed version (estimated_entry_charge > 0) is fully lock/wait free
+//   * Automatic version (estimated_entry_charge = 0) has rare waits among
+//     certain insertion or erase operations that involve the same very small
+//     set of entries.
 // * Optimized for hot path reads. For concurrency control, most Lookup() and
 // essentially all Release() are a single atomic add operation.
-// * Eviction on insertion is fully parallel and lock-free.
+// * Eviction on insertion is fully parallel.
 // * Uses a generalized + aging variant of CLOCK eviction that might outperform
 // LRU in some cases. (For background, see
 // https://en.wikipedia.org/wiki/Page_replacement_algorithm)
 //
 // Costs
 // -----
-// * Hash table is not resizable (for lock-free efficiency) so capacity is not
-// dynamically changeable. Rely on an estimated average value (block) size for
+// * FixedHyperClockCache (estimated_entry_charge > 0) - Hash table is not
+// resizable (for lock-free efficiency) so capacity is not dynamically
+// changeable. Rely on an estimated average value (block) size for
 // space+time efficiency. (See estimated_entry_charge option details.)
+// EXPERIMENTAL - This limitation is fixed in AutoHyperClockCache, activated
+// with estimated_entry_charge == 0.
 // * Insert usually does not (but might) overwrite a previous entry associated
-// with a cache key. This is OK for RocksDB uses of Cache.
+// with a cache key. This is OK for RocksDB uses of Cache, though it does mess
+// up our REDUNDANT block cache insertion statistics.
 // * Only supports keys of exactly 16 bytes, which is what RocksDB uses for
-// block cache (not row cache or table cache).
-// * SecondaryCache is not supported.
+// block cache (but not row cache or table cache).
 // * Cache priorities are less aggressively enforced. Unlike LRUCache, enough
 // transient LOW or BOTTOM priority items can evict HIGH priority entries that
 // are not referenced recently (or often) enough.
@@ -139,7 +149,8 @@ class ClockCacheTest;
 // * Empty - slot is not in use and unowned. All other metadata and data is
 // in an undefined state.
 // * Construction - slot is exclusively owned by one thread, the thread
-// successfully entering this state, for populating or freeing data.
+// successfully entering this state, for populating or freeing data
+// (de-construction, same state marker).
 // * Shareable (group) - slot holds an entry with counted references for
 // pinning and reading, including
 //   * Visible - slot holds an entry that can be returned by Lookup
@@ -187,15 +198,19 @@ class ClockCacheTest;
 // know from our "redundant" stats that overwrites are very rare for the block
 // cache, so we should not spend much to make them effective.
 //
-// So instead we Insert as soon as we find an empty slot in the probing
-// sequence without seeing an existing (visible) entry for the same key. This
-// way we only insert if we can improve the probing performance, and we don't
-// need to probe beyond our insert position, assuming we are willing to let
-// the previous entry for the same key die of old age (eventual eviction from
-// not being used). We can reach a similar state with concurrent insertions,
-// where one will pass over the other while it is "under construction."
-// This temporary duplication is acceptable for RocksDB block cache because
-// we know redundant insertion is rare.
+// FixedHyperClockCache: Instead we Insert as soon as we find an empty slot in
+// the probing sequence without seeing an existing (visible) entry for the same
+// key. This way we only insert if we can improve the probing performance, and
+// we don't need to probe beyond our insert position, assuming we are willing
+// to let the previous entry for the same key die of old age (eventual eviction
+// from not being used). We can reach a similar state with concurrent
+// insertions, where one will pass over the other while it is "under
+// construction." This temporary duplication is acceptable for RocksDB block
+// cache because we know redundant insertion is rare.
+// AutoHyperClockCache: Similar, except we only notice and return an existing
+// match if it is found in the search for a suitable empty slot (starting with
+// the same slot as the head pointer), not by following the existing chain of
+// entries. Insertions are always made to the head of the chain.
 //
 // Another problem to solve is what to return to the caller when we find an
 // existing entry whose probing position we cannot improve on, or when the
@@ -322,7 +337,6 @@ struct ClockHandle : public ClockHandleBasicData {
  // For setting the hit bit
  static constexpr uint8_t kHitBitShift = 2U * kCounterNumBits;
  static constexpr uint64_t kHitBitMask = uint64_t{1} << kHitBitShift;
-  ;

  // For reading or updating the state marker in meta word
  static constexpr uint8_t kStateShift = kHitBitShift + 1;
@@ -356,11 +370,19 @@ struct ClockHandle : public ClockHandleBasicData {
  // TODO: make these coundown values tuning parameters for eviction?

  // See above. Mutable for read reference counting.
-  mutable std::atomic<uint64_t> meta{};
+  mutable AcqRelAtomic<uint64_t> meta{};
 };  // struct ClockHandle

 class BaseClockTable {
 public:
+  struct BaseOpts {
+    explicit BaseOpts(int _eviction_effort_cap)
+        : eviction_effort_cap(_eviction_effort_cap) {}
+    explicit BaseOpts(const HyperClockCacheOptions& opts)
+        : BaseOpts(opts.eviction_effort_cap) {}
+    int eviction_effort_cap;
+  };
+
  BaseClockTable(CacheMetadataChargePolicy metadata_charge_policy,
                 MemoryAllocator* allocator,
                 const Cache::EvictionCallback* eviction_callback,
@@ -373,34 +395,37 @@ class BaseClockTable {
  template <class Table>
  typename Table::HandleImpl* CreateStandalone(ClockHandleBasicData& proto,
                                               size_t capacity,
-                                               bool strict_capacity_limit,
+                                               uint32_t eec_and_scl,
                                               bool allow_uncharged);

  template <class Table>
  Status Insert(const ClockHandleBasicData& proto,
                typename Table::HandleImpl** handle, Cache::Priority priority,
-                size_t capacity, bool strict_capacity_limit);
+                size_t capacity, uint32_t eec_and_scl);

  void Ref(ClockHandle& handle);

-  size_t GetOccupancy() const {
-    return occupancy_.load(std::memory_order_relaxed);
-  }
+  size_t GetOccupancy() const { return occupancy_.LoadRelaxed(); }

-  size_t GetUsage() const { return usage_.load(std::memory_order_relaxed); }
+  size_t GetUsage() const { return usage_.LoadRelaxed(); }

-  size_t GetStandaloneUsage() const {
-    return standalone_usage_.load(std::memory_order_relaxed);
-  }
+  size_t GetStandaloneUsage() const { return standalone_usage_.LoadRelaxed(); }

  uint32_t GetHashSeed() const { return hash_seed_; }

+  uint64_t GetYieldCount() const { return yield_count_.LoadRelaxed(); }
+
+  uint64_t GetEvictionEffortExceededCount() const {
+    return eviction_effort_exceeded_count_.LoadRelaxed();
+  }
+
  struct EvictionData {
    size_t freed_charge = 0;
    size_t freed_count = 0;
+    size_t seen_pinned_count = 0;
  };

-  void TrackAndReleaseEvictedEntry(ClockHandle* h, EvictionData* data);
+  void TrackAndReleaseEvictedEntry(ClockHandle* h);

 #ifndef NDEBUG
  // Acquire N references
@@ -425,6 +450,7 @@ class BaseClockTable {
  template <class Table>
  Status ChargeUsageMaybeEvictStrict(size_t total_charge, size_t capacity,
                                     bool need_evict_for_occupancy,
+                                     uint32_t eviction_effort_cap,
                                     typename Table::InsertState& state);

  // Helper for updating `usage_` for new entry with given `total_charge`
@@ -438,6 +464,7 @@ class BaseClockTable {
  template <class Table>
  bool ChargeUsageMaybeEvictNonStrict(size_t total_charge, size_t capacity,
                                      bool need_evict_for_occupancy,
+                                      uint32_t eviction_effort_cap,
                                      typename Table::InsertState& state);

 protected:  // data
@@ -446,18 +473,29 @@ class BaseClockTable {
  // operations in ClockCacheShard.

  // Clock algorithm sweep pointer.
-  std::atomic<uint64_t> clock_pointer_{};
+  // (Relaxed: only needs to be consistent with itself.)
+  RelaxedAtomic<uint64_t> clock_pointer_{};
+
+  // Counter for number of times we yield to wait on another thread.
+  // It is normal for this to occur rarely in normal operation.
+  // (Relaxed: a simple stat counter.)
+  RelaxedAtomic<uint64_t> yield_count_{};
+
+  // Counter for number of times eviction effort cap is exceeded.
+  // It is normal for this to occur rarely in normal operation.
+  // (Relaxed: a simple stat counter.)
+  RelaxedAtomic<uint64_t> eviction_effort_exceeded_count_{};

  // TODO: is this separation needed if we don't do background evictions?
  ALIGN_AS(CACHE_LINE_SIZE)
  // Number of elements in the table.
-  std::atomic<size_t> occupancy_{};
+  AcqRelAtomic<size_t> occupancy_{};

  // Memory usage by entries tracked by the cache (including standalone)
-  std::atomic<size_t> usage_{};
+  AcqRelAtomic<size_t> usage_{};

  // Part of usage by standalone entries (not in table)
-  std::atomic<size_t> standalone_usage_{};
+  AcqRelAtomic<size_t> standalone_usage_{};

  ALIGN_AS(CACHE_LINE_SIZE)
  const CacheMetadataChargePolicy metadata_charge_policy_;
@@ -472,6 +510,10 @@ class BaseClockTable {
  const uint32_t& hash_seed_;
 };

+// Hash table for cache entries with size determined at creation time.
+// Uses open addressing and double hashing. Since entries cannot be moved,
+// the "displacements" count ensures probing sequences find entries even when
+// entries earlier in the probing sequence have been removed.
 class FixedHyperClockTable : public BaseClockTable {
 public:
  // Target size to be exactly a common cache line size (see static_assert in
@@ -479,7 +521,11 @@ class FixedHyperClockTable : public BaseClockTable {
  struct ALIGN_AS(64U) HandleImpl : public ClockHandle {
    // The number of elements that hash to this slot or a lower one, but wind
    // up in this slot or a higher one.
-    std::atomic<uint32_t> displacements{};
+    // (Relaxed: within a Cache op, does not need consistency with entries
+    // inserted/removed during that op. For example, a Lookup() that
+    // happens-after an Insert() will see an appropriate displacements value
+    // for the entry to be in a published state.)
+    RelaxedAtomic<uint32_t> displacements{};

    // Whether this is a "deteched" handle that is independently allocated
    // with `new` (so must be deleted with `delete`).
@@ -493,17 +539,19 @@ class FixedHyperClockTable : public BaseClockTable {
    inline void SetStandalone() { standalone = true; }
  };  // struct HandleImpl

-  struct Opts {
-    explicit Opts(size_t _estimated_value_size)
-        : estimated_value_size(_estimated_value_size) {}
-    explicit Opts(const HyperClockCacheOptions& opts) {
+  struct Opts : public BaseOpts {
+    explicit Opts(size_t _estimated_value_size, int _eviction_effort_cap)
+        : BaseOpts(_eviction_effort_cap),
+          estimated_value_size(_estimated_value_size) {}
+    explicit Opts(const HyperClockCacheOptions& opts)
+        : BaseOpts(opts.eviction_effort_cap) {
      assert(opts.estimated_entry_charge > 0);
      estimated_value_size = opts.estimated_entry_charge;
    }
    size_t estimated_value_size;
  };

-  FixedHyperClockTable(size_t capacity, bool strict_capacity_limit,
+  FixedHyperClockTable(size_t capacity,
                       CacheMetadataChargePolicy metadata_charge_policy,
                       MemoryAllocator* allocator,
                       const Cache::EvictionCallback* eviction_callback,
@@ -525,7 +573,8 @@ class FixedHyperClockTable : public BaseClockTable {
  // Runs the clock eviction algorithm trying to reclaim at least
  // requested_charge. Returns how much is evicted, which could be less
  // if it appears impossible to evict the requested amount without blocking.
-  void Evict(size_t requested_charge, InsertState& state, EvictionData* data);
+  void Evict(size_t requested_charge, InsertState& state, EvictionData* data,
+             uint32_t eviction_effort_cap);

  HandleImpl* Lookup(const UniqueId64x2& hashed_key);

@@ -626,11 +675,320 @@ class FixedHyperClockTable : public BaseClockTable {
  const std::unique_ptr<HandleImpl[]> array_;
 };  // class FixedHyperClockTable

-// Placeholder for future automatic table variant
-// For now, just use FixedHyperClockTable.
-class AutoHyperClockTable : public FixedHyperClockTable {
+// Hash table for cache entries that resizes automatically based on occupancy.
+// However, it depends on a contiguous memory region to grow into
+// incrementally, using linear hashing, so uses an anonymous mmap so that
+// only the used portion of the memory region is mapped to physical memory
+// (part of RSS).
+//
+// This table implementation uses the same "low-level protocol" for managing
+// the contens of an entry slot as FixedHyperClockTable does, captured in the
+// ClockHandle struct. The provides most of the essential data safety, but
+// AutoHyperClockTable is another "high-level protocol" for organizing entries
+// into a hash table, with automatic resizing.
+//
+// This implementation is not fully wait-free but we can call it "essentially
+// wait-free," and here's why. First, like FixedHyperClockCache, there is no
+// locking nor other forms of waiting at the cache or shard level. Also like
+// FixedHCC there is essentially an entry-level read-write lock implemented
+// with atomics, but our relaxed atomicity/consistency guarantees (e.g.
+// duplicate inserts are possible) mean we do not need to wait for entry
+// locking. Lookups, non-erasing Releases, and non-evicting non-growing Inserts
+// are all fully wait-free. Of course, these waits are not dependent on any
+// external factors such as I/O.
+//
+// For operations that remove entries from a chain or grow the table by
+// splitting a chain, there is a chain-level locking mechanism that we call a
+// "rewrite" lock, and the only waits are for these locks. On average, each
+// chain lock is relevant to < 2 entries each. (The average would be less than
+// one entry each, but we do not lock when there's no entry to remove or
+// migrate.) And a given thread can only hold two such chain locks at a time,
+// more typically just one. So in that sense alone, the waiting that does exist
+// is very localized.
+//
+// If we look closer at the operations utilizing that locking mechanism, we
+// can see why it's "essentially wait-free."
+// * Grow operations to increase the size of the table: each operation splits
+// an existing chain into two, and chains for splitting are chosen in table
+// order. Grow operations are fully parallel except for the chain locking, but
+// for one Grow operation to wait on another, it has to be feeding into the
+// other, which means the table has doubled in size already from other Grow
+// operations without the original one finishing. So Grow operations are very
+// low latency (unlike LRUCache doubling the table size in one operation) and
+// very parallelizeable. (We use some tricks to break up dependencies in
+// updating metadata on the usable size of the table.) And obviously Grow
+// operations are very rare after the initial population of the table.
+// * Evict operations (part of many Inserts): clock updates and evictions
+// sweep through the structure in table order, so like Grow operations,
+// parallel Evict can only wait on each other if an Evict has lingered (slept)
+// long enough that the clock pointer has wrapped around the entire structure.
+// * Random erasures (Erase, Release with erase_if_last_ref, etc.): these
+// operations are rare and not really considered performance critical.
+// Currently they're mostly used for removing placeholder cache entries, e.g.
+// for memory tracking, though that could use standalone entries instead to
+// avoid potential contention in table operations. It's possible that future
+// enhancements could pro-actively remove cache entries from obsolete files,
+// but that's not yet implemented.
+class AutoHyperClockTable : public BaseClockTable {
 public:
-  using FixedHyperClockTable::FixedHyperClockTable;
+  // Target size to be exactly a common cache line size (see static_assert in
+  // clock_cache.cc)
+  struct ALIGN_AS(64U) HandleImpl : public ClockHandle {
+    // To orgainize AutoHyperClockTable entries into a hash table while
+    // allowing the table size to grow without existing entries being moved,
+    // a version of chaining is used. Rather than being heap allocated (and
+    // incurring overheads to ensure memory safety) entries must go into
+    // Handles ("slots") in the pre-allocated array. To improve CPU cache
+    // locality, the chain head pointers are interleved with the entries;
+    // specifically, a Handle contains
+    // * A head pointer for a chain of entries with this "home" location.
+    // * A ClockHandle, for an entry that may or may not be in the chain
+    // starting from that head (but for performance ideally is on that
+    // chain).
+    // * A next pointer for the continuation of the chain containing this
+    // entry.
+    //
+    // The pointers are not raw pointers, but are indices into the array,
+    // and are decorated in two ways to help detect and recover from
+    // relevant concurrent modifications during Lookup, so that Lookup is
+    // fully wait-free:
+    // * Each "with_shift" pointer contains a shift count that indicates
+    // how many hash bits were used in chosing the home address for the
+    // chain--specifically the next entry in the chain.
+    // * The end of a chain is given a special "end" marker and refers back
+    // to the head of the chain.
+    //
+    // Why do we need shift on each pointer? To make Lookup wait-free, we need
+    // to be able to query a chain without missing anything, and preferably
+    // avoid synchronously double-checking the length_info. Without the shifts,
+    // there is a risk that we start down a chain and while paused on an entry
+    // that goes to a new home, we then follow the rest of the
+    // partially-migrated chain to see the shared ending with the old home, but
+    // for a time were following the chain for the new home, missing some
+    // entries for the old home.
+    //
+    // Why do we need the end of the chain to loop back? If Lookup pauses
+    // at an "under construction" entry, and sees that "next" is null after
+    // waking up, we need something to tell whether the "under construction"
+    // entry was freed and reused for another chain. Otherwise, we could
+    // miss entries still on the original chain due in the presence of a
+    // concurrent modification. Until an entry is fully erased from a chain,
+    // it is normal to see "under construction" entries on the chain, and it
+    // is not safe to read their hashed key without either a read reference
+    // on the entry or a rewrite lock on the chain.
+
+    // Marker in a "with_shift" head pointer for some thread owning writes
+    // to the chain structure (except for inserts), but only if not an
+    // "end" pointer. Also called the "rewrite lock."
+    static constexpr uint64_t kHeadLocked = uint64_t{1} << 7;
+
+    // Marker in a "with_shift" pointer for the end of a chain. Must also
+    // point back to the head of the chain (with end marker removed).
+    // Also includes the "locked" bit so that attempting to lock an empty
+    // chain has no effect (not needed, as the lock is only needed for
+    // removals).
+    static constexpr uint64_t kNextEndFlags = (uint64_t{1} << 6) | kHeadLocked;
+
+    static inline bool IsEnd(uint64_t next_with_shift) {
+      // Assuming certain values never used, suffices to check this one bit
+      constexpr auto kCheckBit = kNextEndFlags ^ kHeadLocked;
+      return next_with_shift & kCheckBit;
+    }
+
+    // Bottom bits to right shift away to get an array index from a
+    // "with_shift" pointer.
+    static constexpr int kNextShift = 8;
+
+    // A bit mask for the "shift" associated with each "with_shift" pointer.
+    // Always bottommost bits.
+    static constexpr int kShiftMask = 63;
+
+    // A marker for head_next_with_shift that indicates this HandleImpl is
+    // heap allocated (standalone) rather than in the table.
+    static constexpr uint64_t kStandaloneMarker = UINT64_MAX;
+
+    // A marker for head_next_with_shift indicating the head is not yet part
+    // of the usable table, or for chain_next_with_shift indicating that the
+    // entry is not present or is not yet part of a chain (must not be
+    // "shareable" state).
+    static constexpr uint64_t kUnusedMarker = 0;
+
+    // See above. The head pointer is logically independent of the rest of
+    // the entry, including the chain next pointer.
+    AcqRelAtomic<uint64_t> head_next_with_shift{kUnusedMarker};
+    AcqRelAtomic<uint64_t> chain_next_with_shift{kUnusedMarker};
+
+    // For supporting CreateStandalone and some fallback cases.
+    inline bool IsStandalone() const {
+      return head_next_with_shift.Load() == kStandaloneMarker;
+    }
+
+    inline void SetStandalone() {
+      head_next_with_shift.Store(kStandaloneMarker);
+    }
+  };  // struct HandleImpl
+
+  struct Opts : public BaseOpts {
+    explicit Opts(size_t _min_avg_value_size, int _eviction_effort_cap)
+        : BaseOpts(_eviction_effort_cap),
+          min_avg_value_size(_min_avg_value_size) {}
+
+    explicit Opts(const HyperClockCacheOptions& opts)
+        : BaseOpts(opts.eviction_effort_cap) {
+      assert(opts.estimated_entry_charge == 0);
+      min_avg_value_size = opts.min_avg_entry_charge;
+    }
+    size_t min_avg_value_size;
+  };
+
+  AutoHyperClockTable(size_t capacity,
+                      CacheMetadataChargePolicy metadata_charge_policy,
+                      MemoryAllocator* allocator,
+                      const Cache::EvictionCallback* eviction_callback,
+                      const uint32_t* hash_seed, const Opts& opts);
+  ~AutoHyperClockTable();
+
+  // For BaseClockTable::Insert
+  struct InsertState {
+    uint64_t saved_length_info = 0;
+    size_t likely_empty_slot = 0;
+  };
+
+  void StartInsert(InsertState& state);
+
+  // Does initial check for whether there's hash table room for another
+  // inserted entry, possibly growing if needed. Returns true iff (after
+  // the call) there is room for the proposed number of entries.
+  bool GrowIfNeeded(size_t new_occupancy, InsertState& state);
+
+  HandleImpl* DoInsert(const ClockHandleBasicData& proto,
+                       uint64_t initial_countdown, bool take_ref,
+                       InsertState& state);
+
+  // Runs the clock eviction algorithm trying to reclaim at least
+  // requested_charge. Returns how much is evicted, which could be less
+  // if it appears impossible to evict the requested amount without blocking.
+  void Evict(size_t requested_charge, InsertState& state, EvictionData* data,
+             uint32_t eviction_effort_cap);
+
+  HandleImpl* Lookup(const UniqueId64x2& hashed_key);
+
+  bool Release(HandleImpl* handle, bool useful, bool erase_if_last_ref);
+
+  void Erase(const UniqueId64x2& hashed_key);
+
+  void EraseUnRefEntries();
+
+  size_t GetTableSize() const;
+
+  size_t GetOccupancyLimit() const;
+
+  const HandleImpl* HandlePtr(size_t idx) const { return &array_[idx]; }
+
+#ifndef NDEBUG
+  size_t& TEST_MutableOccupancyLimit() {
+    return *reinterpret_cast<size_t*>(&occupancy_limit_);
+  }
+
+  // Release N references
+  void TEST_ReleaseN(HandleImpl* handle, size_t n);
+#endif
+
+  // Maximum ratio of number of occupied slots to number of usable slots. The
+  // actual load factor should float pretty close to this number, which should
+  // be a nice space/time trade-off, though large swings in WriteBufferManager
+  // memory could lead to low (but very much safe) load factors (only after
+  // seeing high load factors). Linear hashing along with (modified) linear
+  // probing to find an available slot increases potential risks of high
+  // load factors, so are disallowed.
+  static constexpr double kMaxLoadFactor = 0.60;
+
+ private:  // functions
+  // Returns true iff increased usable length. Due to load factor
+  // considerations, GrowIfNeeded might call this more than once to make room
+  // for one more entry.
+  bool Grow(InsertState& state);
+
+  // Operational details of splitting a chain into two for Grow().
+  void SplitForGrow(size_t grow_home, size_t old_home, int old_shift);
+
+  // Takes an "under construction" entry and ensures it is no longer connected
+  // to its home chain (in preparaion for completing erasure and freeing the
+  // slot). Note that previous operations might have already noticed it being
+  // "under (de)construction" and removed it from its chain.
+  void Remove(HandleImpl* h);
+
+  // Try to take ownership of an entry and erase+remove it from the table.
+  // Returns true if successful. Could fail if
+  // * There are other references to the entry
+  // * Some other thread has exclusive ownership or has freed it.
+  bool TryEraseHandle(HandleImpl* h, bool holding_ref, bool mark_invisible);
+
+  // Calculates the appropriate maximum table size, for creating the memory
+  // mapping.
+  static size_t CalcMaxUsableLength(
+      size_t capacity, size_t min_avg_value_size,
+      CacheMetadataChargePolicy metadata_charge_policy);
+
+  // Shared helper function that implements removing entries from a chain
+  // with proper handling to ensure all existing data is seen even in the
+  // presence of concurrent insertions, etc. (See implementation.)
+  template <class OpData>
+  void PurgeImpl(OpData* op_data, size_t home = SIZE_MAX,
+                 EvictionData* data = nullptr);
+
+  // An RAII wrapper for locking a chain of entries for removals. See
+  // implementation.
+  class ChainRewriteLock;
+
+  // Helper function for PurgeImpl while holding a ChainRewriteLock. See
+  // implementation.
+  template <class OpData>
+  void PurgeImplLocked(OpData* op_data, ChainRewriteLock& rewrite_lock,
+                       size_t home, EvictionData* data);
+
+  // Update length_info_ as much as possible without waiting, given a known
+  // usable (ready for inserts and lookups) grow_home. (Previous grow_homes
+  // might not be usable yet, but we can check if they are by looking at
+  // the corresponding old home.)
+  void CatchUpLengthInfoNoWait(size_t known_usable_grow_home);
+
+ private:  // data
+  // mmaped area holding handles
+  const TypedMemMapping<HandleImpl> array_;
+
+  // Metadata for table size under linear hashing.
+  //
+  // Lowest 8 bits are the minimum number of lowest hash bits to use
+  // ("min shift"). The upper 56 bits are a threshold. If that minumum number
+  // of bits taken from a hash value is < this threshold, then one more bit of
+  // hash value is taken and used.
+  //
+  // Other mechanisms (shift amounts on pointers) ensure complete availability
+  // of data already in the table even if a reader only sees a completely
+  // out-of-date version of this value. In the worst case, it could take
+  // log time to find the correct chain, but normally this value enables
+  // readers to find the correct chain on the first try.
+  //
+  // To maximize parallelization of Grow() operations, this field is only
+  // updated opportunistically after Grow() operations and in DoInsert() where
+  // it is found to be out-of-date. See CatchUpLengthInfoNoWait().
+  AcqRelAtomic<uint64_t> length_info_;
+
+  // An already-computed version of the usable length times the max load
+  // factor. Could be slightly out of date but GrowIfNeeded()/Grow() handle
+  // that internally.
+  // (Relaxed: allowed to lag behind length_info_ by a little)
+  RelaxedAtomic<size_t> occupancy_limit_;
+
+  // The next index to use from array_ upon the next Grow(). Might be ahead of
+  // length_info_.
+  // (Relaxed: self-contained source of truth for next grow home)
+  RelaxedAtomic<size_t> grow_frontier_;
+
+  // See explanation in AutoHyperClockTable::Evict
+  // (Relaxed: allowed to lag behind clock_pointer_ and length_info_ state)
+  RelaxedAtomic<size_t> clock_pointer_mask_;
 };  // class AutoHyperClockTable

 // A single shard of sharded cache.
@@ -746,10 +1104,13 @@ class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShardBase {
  Table table_;

  // Maximum total charge of all elements stored in the table.
-  std::atomic<size_t> capacity_;
+  // (Relaxed: eventual consistency/update is OK)
+  RelaxedAtomic<size_t> capacity_;

-  // Whether to reject insertion if cache reaches its full capacity.
-  std::atomic<bool> strict_capacity_limit_;
+  // Encodes eviction_effort_cap (bottom 31 bits) and strict_capacity_limit
+  // (top bit). See HyperClockCacheOptions::eviction_effort_cap etc.
+  // (Relaxed: eventual consistency/update is OK)
+  RelaxedAtomic<uint32_t> eec_and_scl_;
 };  // class ClockCacheShard

 template <class Table>
@@ -785,7 +1146,6 @@ class FixedHyperClockCache
      const std::shared_ptr<Logger>& /*info_log*/) const override;
 };  // class FixedHyperClockCache

-// Placeholder for future automatic HCC variant
 class AutoHyperClockCache
 #ifdef NDEBUG
    final
@@ -795,6 +1155,9 @@ class AutoHyperClockCache
  using BaseHyperClockCache::BaseHyperClockCache;

  const char* Name() const override { return "AutoHyperClockCache"; }
+
+  void ReportProblems(
+      const std::shared_ptr<Logger>& /*info_log*/) const override;
 };  // class AutoHyperClockCache

 }  // namespace clock_cache
@@ -11,6 +11,7 @@

 #include "memory/memory_allocator_impl.h"
 #include "monitoring/perf_context_imp.h"
+#include "util/coding.h"
 #include "util/compression.h"
 #include "util/string_util.h"

@@ -22,17 +23,24 @@ CompressedSecondaryCache::CompressedSecondaryCache(
      cache_options_(opts),
      cache_res_mgr_(std::make_shared<ConcurrentCacheReservationManager>(
          std::make_shared<CacheReservationManagerImpl<CacheEntryRole::kMisc>>(
-              cache_))) {}
+              cache_))),
+      disable_cache_(opts.capacity == 0) {}

-CompressedSecondaryCache::~CompressedSecondaryCache() {
-  assert(cache_res_mgr_->GetTotalReservedCacheSize() == 0);
-}
+CompressedSecondaryCache::~CompressedSecondaryCache() {}

 std::unique_ptr<SecondaryCacheResultHandle> CompressedSecondaryCache::Lookup(
    const Slice& key, const Cache::CacheItemHelper* helper,
    Cache::CreateContext* create_context, bool /*wait*/, bool advise_erase,
-    bool& kept_in_sec_cache) {
+    Statistics* stats, bool& kept_in_sec_cache) {
  assert(helper);
+  // This is a minor optimization. Its ok to skip it in TSAN in order to
+  // avoid a false positive.
+#ifndef __SANITIZE_THREAD__
+  if (disable_cache_) {
+    return nullptr;
+  }
+#endif
+
  std::unique_ptr<SecondaryCacheResultHandle> handle;
  kept_in_sec_cache = false;
  Cache::Handle* lru_handle = cache_->Lookup(key);
@@ -43,46 +51,72 @@ std::unique_ptr<SecondaryCacheResultHandle> CompressedSecondaryCache::Lookup(
  void* handle_value = cache_->Value(lru_handle);
  if (handle_value == nullptr) {
    cache_->Release(lru_handle, /*erase_if_last_ref=*/false);
+    RecordTick(stats, COMPRESSED_SECONDARY_CACHE_DUMMY_HITS);
    return nullptr;
  }

  CacheAllocationPtr* ptr{nullptr};
  CacheAllocationPtr merged_value;
  size_t handle_value_charge{0};
+  const char* data_ptr = nullptr;
+  CacheTier source = CacheTier::kVolatileCompressedTier;
+  CompressionType type = cache_options_.compression_type;
  if (cache_options_.enable_custom_split_merge) {
    CacheValueChunk* value_chunk_ptr =
        reinterpret_cast<CacheValueChunk*>(handle_value);
    merged_value = MergeChunksIntoValue(value_chunk_ptr, handle_value_charge);
    ptr = &merged_value;
+    data_ptr = ptr->get();
  } else {
+    uint32_t type_32 = static_cast<uint32_t>(type);
+    uint32_t source_32 = static_cast<uint32_t>(source);
    ptr = reinterpret_cast<CacheAllocationPtr*>(handle_value);
    handle_value_charge = cache_->GetCharge(lru_handle);
+    data_ptr = ptr->get();
+    data_ptr = GetVarint32Ptr(data_ptr, data_ptr + 1,
+                              static_cast<uint32_t*>(&type_32));
+    type = static_cast<CompressionType>(type_32);
+    data_ptr = GetVarint32Ptr(data_ptr, data_ptr + 1,
+                              static_cast<uint32_t*>(&source_32));
+    source = static_cast<CacheTier>(source_32);
+    handle_value_charge -= (data_ptr - ptr->get());
  }
  MemoryAllocator* allocator = cache_options_.memory_allocator.get();

  Status s;
  Cache::ObjectPtr value{nullptr};
  size_t charge{0};
-  if (cache_options_.compression_type == kNoCompression ||
-      cache_options_.do_not_compress_roles.Contains(helper->role)) {
-    s = helper->create_cb(Slice(ptr->get(), handle_value_charge),
-                          create_context, allocator, &value, &charge);
-  } else {
-    UncompressionContext uncompression_context(cache_options_.compression_type);
-    UncompressionInfo uncompression_info(uncompression_context,
-                                         UncompressionDict::GetEmptyDict(),
-                                         cache_options_.compression_type);
+  if (source == CacheTier::kVolatileCompressedTier) {
+    if (cache_options_.compression_type == kNoCompression ||
+        cache_options_.do_not_compress_roles.Contains(helper->role)) {
+      s = helper->create_cb(Slice(data_ptr, handle_value_charge),
+                            kNoCompression, CacheTier::kVolatileTier,
+                            create_context, allocator, &value, &charge);
+    } else {
+      UncompressionContext uncompression_context(
+          cache_options_.compression_type);
+      UncompressionInfo uncompression_info(uncompression_context,
+                                           UncompressionDict::GetEmptyDict(),
+                                           cache_options_.compression_type);

-    size_t uncompressed_size{0};
-    CacheAllocationPtr uncompressed = UncompressData(
-        uncompression_info, (char*)ptr->get(), handle_value_charge,
-        &uncompressed_size, cache_options_.compress_format_version, allocator);
+      size_t uncompressed_size{0};
+      CacheAllocationPtr uncompressed =
+          UncompressData(uncompression_info, (char*)data_ptr,
+                         handle_value_charge, &uncompressed_size,
+                         cache_options_.compress_format_version, allocator);

-    if (!uncompressed) {
-      cache_->Release(lru_handle, /*erase_if_last_ref=*/true);
-      return nullptr;
+      if (!uncompressed) {
+        cache_->Release(lru_handle, /*erase_if_last_ref=*/true);
+        return nullptr;
+      }
+      s = helper->create_cb(Slice(uncompressed.get(), uncompressed_size),
+                            kNoCompression, CacheTier::kVolatileTier,
+                            create_context, allocator, &value, &charge);
    }
-    s = helper->create_cb(Slice(uncompressed.get(), uncompressed_size),
+  } else {
+    // The item was not compressed by us. Let the helper create_cb
+    // uncompress it
+    s = helper->create_cb(Slice(data_ptr, handle_value_charge), type, source,
                          create_context, allocator, &value, &charge);
  }

@@ -104,44 +138,60 @@ std::unique_ptr<SecondaryCacheResultHandle> CompressedSecondaryCache::Lookup(
    cache_->Release(lru_handle, /*erase_if_last_ref=*/false);
  }
  handle.reset(new CompressedSecondaryCacheResultHandle(value, charge));
+  RecordTick(stats, COMPRESSED_SECONDARY_CACHE_HITS);
  return handle;
 }

-Status CompressedSecondaryCache::Insert(const Slice& key,
-                                        Cache::ObjectPtr value,
-                                        const Cache::CacheItemHelper* helper,
-                                        bool force_insert) {
-  if (value == nullptr) {
-    return Status::InvalidArgument();
+bool CompressedSecondaryCache::MaybeInsertDummy(const Slice& key) {
+  auto internal_helper = GetHelper(cache_options_.enable_custom_split_merge);
+  Cache::Handle* lru_handle = cache_->Lookup(key);
+  if (lru_handle == nullptr) {
+    PERF_COUNTER_ADD(compressed_sec_cache_insert_dummy_count, 1);
+    // Insert a dummy handle if the handle is evicted for the first time.
+    cache_->Insert(key, /*obj=*/nullptr, internal_helper, /*charge=*/0)
+        .PermitUncheckedError();
+    return true;
+  } else {
+    cache_->Release(lru_handle, /*erase_if_last_ref=*/false);
+  }
+
+  return false;
+}
+
+Status CompressedSecondaryCache::InsertInternal(
+    const Slice& key, Cache::ObjectPtr value,
+    const Cache::CacheItemHelper* helper, CompressionType type,
+    CacheTier source) {
+  if (source != CacheTier::kVolatileCompressedTier &&
+      cache_options_.enable_custom_split_merge) {
+    // We don't support custom split/merge for the tiered case
+    return Status::OK();
  }

  auto internal_helper = GetHelper(cache_options_.enable_custom_split_merge);
-  if (!force_insert) {
-    Cache::Handle* lru_handle = cache_->Lookup(key);
-    if (lru_handle == nullptr) {
-      PERF_COUNTER_ADD(compressed_sec_cache_insert_dummy_count, 1);
-      // Insert a dummy handle if the handle is evicted for the first time.
-      return cache_->Insert(key, /*obj=*/nullptr, internal_helper,
-                            /*charge=*/0);
-    } else {
-      cache_->Release(lru_handle, /*erase_if_last_ref=*/false);
-    }
-  }
+  char header[10];
+  char* payload = header;
+  payload = EncodeVarint32(payload, static_cast<uint32_t>(type));
+  payload = EncodeVarint32(payload, static_cast<uint32_t>(source));

-  size_t size = (*helper->size_cb)(value);
+  size_t header_size = payload - header;
+  size_t data_size = (*helper->size_cb)(value);
+  size_t total_size = data_size + header_size;
  CacheAllocationPtr ptr =
-      AllocateBlock(size, cache_options_.memory_allocator.get());
+      AllocateBlock(total_size, cache_options_.memory_allocator.get());
+  char* data_ptr = ptr.get() + header_size;

-  Status s = (*helper->saveto_cb)(value, 0, size, ptr.get());
+  Status s = (*helper->saveto_cb)(value, 0, data_size, data_ptr);
  if (!s.ok()) {
    return s;
  }
-  Slice val(ptr.get(), size);
+  Slice val(data_ptr, data_size);

  std::string compressed_val;
  if (cache_options_.compression_type != kNoCompression &&
+      type == kNoCompression &&
      !cache_options_.do_not_compress_roles.Contains(helper->role)) {
-    PERF_COUNTER_ADD(compressed_sec_cache_uncompressed_bytes, size);
+    PERF_COUNTER_ADD(compressed_sec_cache_uncompressed_bytes, data_size);
    CompressionOptions compression_opts;
    CompressionContext compression_context(cache_options_.compression_type,
                                           compression_opts);
@@ -159,12 +209,14 @@ Status CompressedSecondaryCache::Insert(const Slice& key,
    }

    val = Slice(compressed_val);
-    size = compressed_val.size();
-    PERF_COUNTER_ADD(compressed_sec_cache_compressed_bytes, size);
+    data_size = compressed_val.size();
+    total_size = header_size + data_size;
+    PERF_COUNTER_ADD(compressed_sec_cache_compressed_bytes, data_size);

    if (!cache_options_.enable_custom_split_merge) {
-      ptr = AllocateBlock(size, cache_options_.memory_allocator.get());
-      memcpy(ptr.get(), compressed_val.data(), size);
+      ptr = AllocateBlock(total_size, cache_options_.memory_allocator.get());
+      data_ptr = ptr.get() + header_size;
+      memcpy(data_ptr, compressed_val.data(), data_size);
    }
  }

@@ -175,17 +227,52 @@ Status CompressedSecondaryCache::Insert(const Slice& key,
        SplitValueIntoChunks(val, cache_options_.compression_type, charge);
    return cache_->Insert(key, value_chunks_head, internal_helper, charge);
  } else {
+    std::memcpy(ptr.get(), header, header_size);
    CacheAllocationPtr* buf = new CacheAllocationPtr(std::move(ptr));
-    return cache_->Insert(key, buf, internal_helper, size);
+    return cache_->Insert(key, buf, internal_helper, total_size);
  }
 }

+Status CompressedSecondaryCache::Insert(const Slice& key,
+                                        Cache::ObjectPtr value,
+                                        const Cache::CacheItemHelper* helper,
+                                        bool force_insert) {
+  if (value == nullptr) {
+    return Status::InvalidArgument();
+  }
+
+  if (!force_insert && MaybeInsertDummy(key)) {
+    return Status::OK();
+  }
+
+  return InsertInternal(key, value, helper, kNoCompression,
+                        CacheTier::kVolatileCompressedTier);
+}
+
+Status CompressedSecondaryCache::InsertSaved(
+    const Slice& key, const Slice& saved, CompressionType type = kNoCompression,
+    CacheTier source = CacheTier::kVolatileTier) {
+  if (type == kNoCompression) {
+    return Status::OK();
+  }
+
+  auto slice_helper = &kSliceCacheItemHelper;
+  if (MaybeInsertDummy(key)) {
+    return Status::OK();
+  }
+
+  return InsertInternal(
+      key, static_cast<Cache::ObjectPtr>(const_cast<Slice*>(&saved)),
+      slice_helper, type, source);
+}
+
 void CompressedSecondaryCache::Erase(const Slice& key) { cache_->Erase(key); }

 Status CompressedSecondaryCache::SetCapacity(size_t capacity) {
  MutexLock l(&capacity_mutex_);
  cache_options_.capacity = capacity;
  cache_->SetCapacity(capacity);
+  disable_cache_ = capacity == 0;
  return Status::OK();
 }

@@ -80,10 +80,13 @@ class CompressedSecondaryCache : public SecondaryCache {
                const Cache::CacheItemHelper* helper,
                bool force_insert) override;

+  Status InsertSaved(const Slice& key, const Slice& saved, CompressionType type,
+                     CacheTier source) override;
+
  std::unique_ptr<SecondaryCacheResultHandle> Lookup(
      const Slice& key, const Cache::CacheItemHelper* helper,
      Cache::CreateContext* create_context, bool /*wait*/, bool advise_erase,
-      bool& kept_in_sec_cache) override;
+      Statistics* stats, bool& kept_in_sec_cache) override;

  bool SupportForceErase() const override { return true; }

@@ -130,12 +133,19 @@ class CompressedSecondaryCache : public SecondaryCache {
  CacheAllocationPtr MergeChunksIntoValue(const void* chunks_head,
                                          size_t& charge);

+  bool MaybeInsertDummy(const Slice& key);
+
+  Status InsertInternal(const Slice& key, Cache::ObjectPtr value,
+                        const Cache::CacheItemHelper* helper,
+                        CompressionType type, CacheTier source);
+
  // TODO: clean up to use cleaner interfaces in typed_cache.h
  const Cache::CacheItemHelper* GetHelper(bool enable_custom_split_merge) const;
  std::shared_ptr<Cache> cache_;
  CompressedSecondaryCacheOptions cache_options_;
  mutable port::Mutex capacity_mutex_;
  std::shared_ptr<ConcurrentCacheReservationManager> cache_res_mgr_;
+  bool disable_cache_;
 };

 }  // namespace ROCKSDB_NAMESPACE
@@ -44,7 +44,7 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
    // Lookup an non-existent key.
    std::unique_ptr<SecondaryCacheResultHandle> handle0 =
        sec_cache->Lookup(key0, GetHelper(), this, true, /*advise_erase=*/true,
-                          kept_in_sec_cache);
+                          /*stats=*/nullptr, kept_in_sec_cache);
    ASSERT_EQ(handle0, nullptr);

    Random rnd(301);
@@ -59,7 +59,7 @@ class CompressedSecondaryCacheTestBase : public testing::Test,

    std::unique_ptr<SecondaryCacheResultHandle> handle1_1 =
        sec_cache->Lookup(key1, GetHelper(), this, true, /*advise_erase=*/false,
-                          kept_in_sec_cache);
+                          /*stats=*/nullptr, kept_in_sec_cache);
    ASSERT_EQ(handle1_1, nullptr);

    // Insert and Lookup the item k1 for the second time and advise erasing it.
@@ -68,7 +68,7 @@ class CompressedSecondaryCacheTestBase : public testing::Test,

    std::unique_ptr<SecondaryCacheResultHandle> handle1_2 =
        sec_cache->Lookup(key1, GetHelper(), this, true, /*advise_erase=*/true,
-                          kept_in_sec_cache);
+                          /*stats=*/nullptr, kept_in_sec_cache);
    ASSERT_NE(handle1_2, nullptr);
    ASSERT_FALSE(kept_in_sec_cache);
    if (sec_cache_is_compressed) {
@@ -89,7 +89,7 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
    // Lookup the item k1 again.
    std::unique_ptr<SecondaryCacheResultHandle> handle1_3 =
        sec_cache->Lookup(key1, GetHelper(), this, true, /*advise_erase=*/true,
-                          kept_in_sec_cache);
+                          /*stats=*/nullptr, kept_in_sec_cache);
    ASSERT_EQ(handle1_3, nullptr);

    // Insert and Lookup the item k2.
@@ -99,7 +99,7 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
    ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 2);
    std::unique_ptr<SecondaryCacheResultHandle> handle2_1 =
        sec_cache->Lookup(key2, GetHelper(), this, true, /*advise_erase=*/false,
-                          kept_in_sec_cache);
+                          /*stats=*/nullptr, kept_in_sec_cache);
    ASSERT_EQ(handle2_1, nullptr);

    ASSERT_OK(sec_cache->Insert(key2, &item2, GetHelper(), false));
@@ -115,7 +115,7 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
    }
    std::unique_ptr<SecondaryCacheResultHandle> handle2_2 =
        sec_cache->Lookup(key2, GetHelper(), this, true, /*advise_erase=*/false,
-                          kept_in_sec_cache);
+                          /*stats=*/nullptr, kept_in_sec_cache);
    ASSERT_NE(handle2_2, nullptr);
    std::unique_ptr<TestItem> val2 =
        std::unique_ptr<TestItem>(static_cast<TestItem*>(handle2_2->Value()));
@@ -196,14 +196,14 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
    bool kept_in_sec_cache{false};
    std::unique_ptr<SecondaryCacheResultHandle> handle1 =
        sec_cache->Lookup(key1, GetHelper(), this, true, /*advise_erase=*/false,
-                          kept_in_sec_cache);
+                          /*stats=*/nullptr, kept_in_sec_cache);
    ASSERT_EQ(handle1, nullptr);

    // Insert k2 and k1 is evicted.
    ASSERT_OK(sec_cache->Insert(key2, &item2, GetHelper(), false));
    std::unique_ptr<SecondaryCacheResultHandle> handle2 =
        sec_cache->Lookup(key2, GetHelper(), this, true, /*advise_erase=*/false,
-                          kept_in_sec_cache);
+                          /*stats=*/nullptr, kept_in_sec_cache);
    ASSERT_NE(handle2, nullptr);
    std::unique_ptr<TestItem> val2 =
        std::unique_ptr<TestItem>(static_cast<TestItem*>(handle2->Value()));
@@ -215,14 +215,14 @@ class CompressedSecondaryCacheTestBase : public testing::Test,

    std::unique_ptr<SecondaryCacheResultHandle> handle1_1 =
        sec_cache->Lookup(key1, GetHelper(), this, true, /*advise_erase=*/false,
-                          kept_in_sec_cache);
+                          /*stats=*/nullptr, kept_in_sec_cache);
    ASSERT_EQ(handle1_1, nullptr);

    // Create Fails.
    SetFailCreate(true);
    std::unique_ptr<SecondaryCacheResultHandle> handle2_1 =
        sec_cache->Lookup(key2, GetHelper(), this, true, /*advise_erase=*/true,
-                          kept_in_sec_cache);
+                          /*stats=*/nullptr, kept_in_sec_cache);
    ASSERT_EQ(handle2_1, nullptr);

    // Save Fails.
@@ -912,9 +912,9 @@ TEST_P(CompressedSecondaryCacheTestWithCompressionParam, EntryRoles) {
    ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 1U);

    bool kept_in_sec_cache{true};
-    std::unique_ptr<SecondaryCacheResultHandle> handle =
-        sec_cache->Lookup(ith_key, GetHelper(role), this, true,
-                          /*advise_erase=*/true, kept_in_sec_cache);
+    std::unique_ptr<SecondaryCacheResultHandle> handle = sec_cache->Lookup(
+        ith_key, GetHelper(role), this, true,
+        /*advise_erase=*/true, /*stats=*/nullptr, kept_in_sec_cache);
    ASSERT_NE(handle, nullptr);

    // Lookup returns the right data
@@ -989,11 +989,13 @@ class CompressedSecCacheTestWithTiered
  CompressedSecCacheTestWithTiered() {
    LRUCacheOptions lru_opts;
    HyperClockCacheOptions hcc_opts(
-        /*_capacity=*/70 << 20,
+        /*_capacity=*/0,
        /*_estimated_entry_charge=*/256 << 10,
        /*_num_shard_bits=*/0);
-    TieredVolatileCacheOptions opts;
-    lru_opts.capacity = 70 << 20;
+    // eviction_effort_cap setting simply to avoid churn in existing test
+    hcc_opts.eviction_effort_cap = 100;
+    TieredCacheOptions opts;
+    lru_opts.capacity = 0;
    lru_opts.num_shard_bits = 0;
    lru_opts.high_pri_pool_ratio = 0;
    opts.cache_type = std::get<0>(GetParam());
@@ -1004,9 +1006,11 @@ class CompressedSecCacheTestWithTiered
    }
    opts.adm_policy = std::get<1>(GetParam());
    ;
-    opts.comp_cache_opts.capacity = 30 << 20;
+    opts.comp_cache_opts.capacity = 0;
    opts.comp_cache_opts.num_shard_bits = 0;
-    cache_ = NewTieredVolatileCache(opts);
+    opts.total_capacity = 100 << 20;
+    opts.compressed_secondary_ratio = 0.3;
+    cache_ = NewTieredCache(opts);
    cache_res_mgr_ =
        std::make_shared<CacheReservationManagerImpl<CacheEntryRole::kMisc>>(
            cache_);
@@ -1023,7 +1027,7 @@ class CompressedSecCacheTestWithTiered
 protected:
  CacheReservationManager* cache_res_mgr() { return cache_res_mgr_.get(); }

-  Cache* GetTieredCache() { return cache_.get(); }
+  std::shared_ptr<Cache> GetTieredCache() { return cache_; }

  Cache* GetCache() {
    return static_cast_with_check<CacheWithSecondaryAdapter, Cache>(
@@ -1110,7 +1114,7 @@ TEST_P(CompressedSecCacheTestWithTiered, AdmissionPolicy) {
    return;
  }

-  Cache* tiered_cache = GetTieredCache();
+  Cache* tiered_cache = GetTieredCache().get();
  Cache* cache = GetCache();
  std::vector<CacheKey> keys;
  std::vector<std::string> vals;
@@ -1165,6 +1169,199 @@ TEST_P(CompressedSecCacheTestWithTiered, AdmissionPolicy) {
  ASSERT_EQ(handle1, nullptr);
 }

+TEST_P(CompressedSecCacheTestWithTiered, DynamicUpdate) {
+  CompressedSecondaryCache* sec_cache =
+      reinterpret_cast<CompressedSecondaryCache*>(GetSecondaryCache());
+  std::shared_ptr<Cache> tiered_cache = GetTieredCache();
+
+  // Use EXPECT_PRED3 instead of EXPECT_NEAR to void too many size_t to
+  // double explicit casts
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (30 << 20),
+               GetPercent(30 << 20, 1));
+  size_t sec_capacity;
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (30 << 20));
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, 130 << 20));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (39 << 20),
+               GetPercent(39 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (39 << 20));
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, 70 << 20));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (21 << 20),
+               GetPercent(21 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (21 << 20));
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, 100 << 20));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (30 << 20),
+               GetPercent(30 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (30 << 20));
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 0.4));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (40 << 20),
+               GetPercent(40 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (40 << 20));
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 0.2));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (20 << 20),
+               GetPercent(20 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (20 << 20));
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 1.0));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (100 << 20),
+               GetPercent(100 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, 100 << 20);
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 0.0));
+  // Only check usage for LRU cache. HCC shows a 64KB usage for some reason
+  if (std::get<0>(GetParam()) == PrimaryCacheType::kCacheTypeLRU) {
+    ASSERT_EQ(GetCache()->GetUsage(), 0);
+  }
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, 0);
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 0.3));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (30 << 20),
+               GetPercent(30 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (30 << 20));
+}
+
+TEST_P(CompressedSecCacheTestWithTiered, DynamicUpdateWithReservation) {
+  CompressedSecondaryCache* sec_cache =
+      reinterpret_cast<CompressedSecondaryCache*>(GetSecondaryCache());
+  std::shared_ptr<Cache> tiered_cache = GetTieredCache();
+
+  ASSERT_OK(cache_res_mgr()->UpdateCacheReservation(10 << 20));
+  // Use EXPECT_PRED3 instead of EXPECT_NEAR to void too many size_t to
+  // double explicit casts
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (37 << 20),
+               GetPercent(37 << 20, 1));
+  EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (3 << 20),
+               GetPercent(3 << 20, 1));
+  size_t sec_capacity;
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (30 << 20));
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, 70 << 20));
+  // Only check usage for LRU cache. HCC is slightly off for some reason
+  if (std::get<0>(GetParam()) == PrimaryCacheType::kCacheTypeLRU) {
+    EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (28 << 20),
+                 GetPercent(28 << 20, 1));
+  }
+  EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (3 << 20),
+               GetPercent(3 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (21 << 20));
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, 130 << 20));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (46 << 20),
+               GetPercent(46 << 20, 1));
+  EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (3 << 20),
+               GetPercent(3 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (39 << 20));
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, 100 << 20));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (37 << 20),
+               GetPercent(37 << 20, 1));
+  EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (3 << 20),
+               GetPercent(3 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (30 << 20));
+
+  ASSERT_OK(tiered_cache->GetSecondaryCacheCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, 30 << 20);
+  size_t sec_usage;
+  ASSERT_OK(tiered_cache->GetSecondaryCachePinnedUsage(sec_usage));
+  EXPECT_PRED3(CacheUsageWithinBounds, sec_usage, 3 << 20,
+               GetPercent(3 << 20, 1));
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 0.39));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (45 << 20),
+               GetPercent(45 << 20, 1));
+  EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (4 << 20),
+               GetPercent(4 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (39 << 20));
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 0.2));
+  // Only check usage for LRU cache. HCC is slightly off for some reason
+  if (std::get<0>(GetParam()) == PrimaryCacheType::kCacheTypeLRU) {
+    EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (28 << 20),
+                 GetPercent(28 << 20, 1));
+  }
+  EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (2 << 20),
+               GetPercent(2 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (20 << 20));
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 1.0));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (100 << 20),
+               GetPercent(100 << 20, 1));
+  EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (10 << 20),
+               GetPercent(10 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, 100 << 20);
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 0.0));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (10 << 20),
+               GetPercent(10 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, 0);
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 0.3));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (37 << 20),
+               GetPercent(37 << 20, 1));
+  EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (3 << 20),
+               GetPercent(3 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, 30 << 20);
+
+  ASSERT_OK(cache_res_mgr()->UpdateCacheReservation(0));
+}
+
+TEST_P(CompressedSecCacheTestWithTiered, ReservationOverCapacity) {
+  CompressedSecondaryCache* sec_cache =
+      reinterpret_cast<CompressedSecondaryCache*>(GetSecondaryCache());
+  std::shared_ptr<Cache> tiered_cache = GetTieredCache();
+
+  ASSERT_OK(cache_res_mgr()->UpdateCacheReservation(110 << 20));
+  // Use EXPECT_PRED3 instead of EXPECT_NEAR to void too many size_t to
+  // double explicit casts
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (110 << 20),
+               GetPercent(110 << 20, 1));
+  EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (30 << 20),
+               GetPercent(30 << 20, 1));
+  size_t sec_capacity;
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (30 << 20));
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 0.39));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (110 << 20),
+               GetPercent(110 << 20, 1));
+  EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (39 << 20),
+               GetPercent(39 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (39 << 20));
+
+  ASSERT_OK(cache_res_mgr()->UpdateCacheReservation(90 << 20));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (94 << 20),
+               GetPercent(94 << 20, 1));
+  EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (35 << 20),
+               GetPercent(35 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (39 << 20));
+
+  ASSERT_OK(cache_res_mgr()->UpdateCacheReservation(0));
+}
+
 INSTANTIATE_TEST_CASE_P(
    CompressedSecCacheTests, CompressedSecCacheTestWithTiered,
    ::testing::Values(
@@ -389,12 +389,13 @@ class ClockCacheTest : public testing::Test {
    }
  }

-  void NewShard(size_t capacity, bool strict_capacity_limit = true) {
+  void NewShard(size_t capacity, bool strict_capacity_limit = true,
+                int eviction_effort_cap = 30) {
    DeleteShard();
    shard_ =
        reinterpret_cast<Shard*>(port::cacheline_aligned_alloc(sizeof(Shard)));

-    TableOpts opts{1 /*value_size*/};
+    TableOpts opts{1 /*value_size*/, eviction_effort_cap};
    new (shard_)
        Shard(capacity, strict_capacity_limit, kDontChargeCacheMetadata,
              /*allocator*/ nullptr, &eviction_callback_, &hash_seed_, opts);
@@ -445,12 +446,20 @@ class ClockCacheTest : public testing::Test {
    return Slice(reinterpret_cast<const char*>(&hashed_key), 16U);
  }

+  // A bad hash function for testing / stressing collision handling
  static inline UniqueId64x2 TestHashedKey(char key) {
    // For testing hash near-collision behavior, put the variance in
    // hashed_key in bits that are unlikely to be used as hash bits.
    return {(static_cast<uint64_t>(key) << 56) + 1234U, 5678U};
  }

+  // A reasonable hash function, for testing "typical behavior" etc.
+  template <typename T>
+  static inline UniqueId64x2 CheapHash(T i) {
+    return {static_cast<uint64_t>(i) * uint64_t{0x85EBCA77C2B2AE63},
+            static_cast<uint64_t>(i) * uint64_t{0xC2B2AE3D27D4EB4F}};
+  }
+
  Shard* shard_ = nullptr;

 private:
@@ -534,7 +543,7 @@ TYPED_TEST(ClockCacheTest, Limits) {
    // (Cleverly using mostly zero-charge entries, but some non-zero to
    // verify usage tracking on detached entries.)
    {
-      size_t n = shard.GetTableAddressCount() + 1;
+      size_t n = kCapacity * 5 + 1;
      std::unique_ptr<HandleImpl* []> ha { new HandleImpl* [n] {} };
      Status s;
      for (size_t i = 0; i < n && s.ok(); ++i) {
@@ -560,6 +569,8 @@ TYPED_TEST(ClockCacheTest, Limits) {
        EXPECT_OK(s);
      }

+      EXPECT_EQ(shard.GetOccupancyCount(), shard.GetOccupancyLimit());
+
      // Regardless, we didn't allow table to actually get full
      EXPECT_LT(shard.GetOccupancyCount(), shard.GetTableAddressCount());

@@ -681,6 +692,53 @@ TYPED_TEST(ClockCacheTest, ClockEvictionTest) {
  }
 }

+TYPED_TEST(ClockCacheTest, ClockEvictionEffortCapTest) {
+  using HandleImpl = typename ClockCacheTest<TypeParam>::Shard::HandleImpl;
+  for (bool strict_capacity_limit : {true, false}) {
+    SCOPED_TRACE("strict_capacity_limit = " +
+                 std::to_string(strict_capacity_limit));
+    for (int eec : {-42, 0, 1, 10, 100, 1000}) {
+      SCOPED_TRACE("eviction_effort_cap = " + std::to_string(eec));
+      constexpr size_t kCapacity = 1000;
+      // Start with much larger capacity to ensure that we can go way over
+      // capacity without reaching table occupancy limit.
+      this->NewShard(3 * kCapacity, strict_capacity_limit, eec);
+      auto& shard = *this->shard_;
+      shard.SetCapacity(kCapacity);
+
+      // Nearly fill the cache with pinned entries, then add a bunch of
+      // non-pinned entries. eviction_effort_cap should affect how many
+      // evictable entries are present beyond the cache capacity, despite
+      // being evictable.
+      constexpr size_t kCount = kCapacity - 1;
+      std::unique_ptr<HandleImpl* []> ha { new HandleImpl* [kCount] {} };
+      for (size_t i = 0; i < 2 * kCount; ++i) {
+        UniqueId64x2 hkey = this->CheapHash(i);
+        ASSERT_OK(shard.Insert(
+            this->TestKey(hkey), hkey, nullptr /*value*/, &kNoopCacheItemHelper,
+            1 /*charge*/, i < kCount ? &ha[i] : nullptr, Cache::Priority::LOW));
+      }
+
+      if (strict_capacity_limit) {
+        // If strict_capacity_limit is enabled, the cache will never exceed its
+        // capacity
+        EXPECT_EQ(shard.GetOccupancyCount(), kCapacity);
+      } else {
+        // Rough inverse relationship between cap and possible memory
+        // explosion, which shows up as increased table occupancy count.
+        int effective_eec = std::max(int{1}, eec) + 1;
+        EXPECT_NEAR(shard.GetOccupancyCount() * 1.0,
+                    kCount * (1 + 1.4 / effective_eec),
+                    kCount * (0.6 / effective_eec) + 1.0);
+      }
+
+      for (size_t i = 0; i < kCount; ++i) {
+        shard.Release(ha[i]);
+      }
+    }
+  }
+}
+
 namespace {
 struct DeleteCounter {
  int deleted = 0;
@@ -981,13 +1039,14 @@ class TestSecondaryCache : public SecondaryCache {

  using ResultMap = std::unordered_map<std::string, ResultType>;

-  explicit TestSecondaryCache(size_t capacity)
+  explicit TestSecondaryCache(size_t capacity, bool insert_saved = false)
      : cache_(NewLRUCache(capacity, 0, false, 0.5 /* high_pri_pool_ratio */,
                           nullptr, kDefaultToAdaptiveMutex,
                           kDontChargeCacheMetadata)),
        num_inserts_(0),
        num_lookups_(0),
-        inject_failure_(false) {}
+        inject_failure_(false),
+        insert_saved_(insert_saved) {}

  const char* Name() const override { return "TestSecondaryCache"; }

@@ -1018,10 +1077,22 @@ class TestSecondaryCache : public SecondaryCache {
    return cache_.Insert(key, buf, size);
  }

+  Status InsertSaved(const Slice& key, const Slice& saved,
+                     CompressionType /*type*/ = kNoCompression,
+                     CacheTier /*source*/ = CacheTier::kVolatileTier) override {
+    if (insert_saved_) {
+      return Insert(key, const_cast<Slice*>(&saved), &kSliceCacheItemHelper,
+                    /*force_insert=*/true);
+    } else {
+      return Status::OK();
+    }
+  }
+
  std::unique_ptr<SecondaryCacheResultHandle> Lookup(
      const Slice& key, const Cache::CacheItemHelper* helper,
      Cache::CreateContext* create_context, bool /*wait*/,
-      bool /*advise_erase*/, bool& kept_in_sec_cache) override {
+      bool /*advise_erase*/, Statistics* /*stats*/,
+      bool& kept_in_sec_cache) override {
    std::string key_str = key.ToString();
    TEST_SYNC_POINT_CALLBACK("TestSecondaryCache::Lookup", &key_str);

@@ -1046,7 +1117,8 @@ class TestSecondaryCache : public SecondaryCache {
        char* ptr = cache_.Value(handle);
        size_t size = DecodeFixed64(ptr);
        ptr += sizeof(uint64_t);
-        s = helper->create_cb(Slice(ptr, size), create_context,
+        s = helper->create_cb(Slice(ptr, size), kNoCompression,
+                              CacheTier::kVolatileTier, create_context,
                              /*alloc*/ nullptr, &value, &charge);
      }
      if (s.ok()) {
@@ -1135,6 +1207,7 @@ class TestSecondaryCache : public SecondaryCache {
  uint32_t num_inserts_;
  uint32_t num_lookups_;
  bool inject_failure_;
+  bool insert_saved_;
  std::string ckey_prefix_;
  ResultMap result_map_;
 };
@@ -1165,7 +1238,7 @@ INSTANTIATE_TEST_CASE_P(DBSecondaryCacheTest, DBSecondaryCacheTest,

 TEST_P(BasicSecondaryCacheTest, BasicTest) {
  std::shared_ptr<TestSecondaryCache> secondary_cache =
-      std::make_shared<TestSecondaryCache>(4096);
+      std::make_shared<TestSecondaryCache>(4096, true);
  std::shared_ptr<Cache> cache =
      NewCache(1024 /* capacity */, 0 /* num_shard_bits */,
               false /* strict_capacity_limit */, secondary_cache);
@@ -1222,7 +1295,7 @@ TEST_P(BasicSecondaryCacheTest, BasicTest) {

 TEST_P(BasicSecondaryCacheTest, StatsTest) {
  std::shared_ptr<TestSecondaryCache> secondary_cache =
-      std::make_shared<TestSecondaryCache>(4096);
+      std::make_shared<TestSecondaryCache>(4096, true);
  std::shared_ptr<Cache> cache =
      NewCache(1024 /* capacity */, 0 /* num_shard_bits */,
               false /* strict_capacity_limit */, secondary_cache);
@@ -1276,7 +1349,7 @@ TEST_P(BasicSecondaryCacheTest, StatsTest) {

 TEST_P(BasicSecondaryCacheTest, BasicFailTest) {
  std::shared_ptr<TestSecondaryCache> secondary_cache =
-      std::make_shared<TestSecondaryCache>(2048);
+      std::make_shared<TestSecondaryCache>(2048, true);
  std::shared_ptr<Cache> cache =
      NewCache(1024 /* capacity */, 0 /* num_shard_bits */,
               false /* strict_capacity_limit */, secondary_cache);
@@ -1318,7 +1391,7 @@ TEST_P(BasicSecondaryCacheTest, BasicFailTest) {

 TEST_P(BasicSecondaryCacheTest, SaveFailTest) {
  std::shared_ptr<TestSecondaryCache> secondary_cache =
-      std::make_shared<TestSecondaryCache>(2048);
+      std::make_shared<TestSecondaryCache>(2048, true);
  std::shared_ptr<Cache> cache =
      NewCache(1024 /* capacity */, 0 /* num_shard_bits */,
               false /* strict_capacity_limit */, secondary_cache);
@@ -1359,7 +1432,7 @@ TEST_P(BasicSecondaryCacheTest, SaveFailTest) {

 TEST_P(BasicSecondaryCacheTest, CreateFailTest) {
  std::shared_ptr<TestSecondaryCache> secondary_cache =
-      std::make_shared<TestSecondaryCache>(2048);
+      std::make_shared<TestSecondaryCache>(2048, true);
  std::shared_ptr<Cache> cache =
      NewCache(1024 /* capacity */, 0 /* num_shard_bits */,
               false /* strict_capacity_limit */, secondary_cache);
@@ -1400,7 +1473,7 @@ TEST_P(BasicSecondaryCacheTest, CreateFailTest) {
 TEST_P(BasicSecondaryCacheTest, FullCapacityTest) {
  for (bool strict_capacity_limit : {false, true}) {
    std::shared_ptr<TestSecondaryCache> secondary_cache =
-        std::make_shared<TestSecondaryCache>(2048);
+        std::make_shared<TestSecondaryCache>(2048, true);
    std::shared_ptr<Cache> cache =
        NewCache(1024 /* capacity */, 0 /* num_shard_bits */,
                 strict_capacity_limit, secondary_cache);
@@ -2019,8 +2092,9 @@ class CacheWithStats : public CacheWrapper {

  Status Insert(const Slice& key, Cache::ObjectPtr value,
                const CacheItemHelper* helper, size_t charge,
-                Handle** handle = nullptr,
-                Priority priority = Priority::LOW) override {
+                Handle** handle = nullptr, Priority priority = Priority::LOW,
+                const Slice& /*compressed*/ = Slice(),
+                CompressionType /*type*/ = kNoCompression) override {
    insert_count_++;
    return target_->Insert(key, value, helper, charge, handle, priority);
  }
@@ -2113,7 +2187,7 @@ TEST_P(DBSecondaryCacheTest, LRUCacheDumpLoadBasic) {
  // we have a new cache it is empty, then, before we do the Get, we do the
  // dumpload
  std::shared_ptr<TestSecondaryCache> secondary_cache =
-      std::make_shared<TestSecondaryCache>(2048 * 1024);
+      std::make_shared<TestSecondaryCache>(2048 * 1024, true);
  // This time with secondary cache
  base_cache = NewCache(1024 * 1024 /* capacity */, 0 /* num_shard_bits */,
                        false /* strict_capacity_limit */, secondary_cache);
@@ -2269,7 +2343,7 @@ TEST_P(DBSecondaryCacheTest, LRUCacheDumpLoadWithFilter) {
  // we have a new cache it is empty, then, before we do the Get, we do the
  // dumpload
  std::shared_ptr<TestSecondaryCache> secondary_cache =
-      std::make_shared<TestSecondaryCache>(2048 * 1024);
+      std::make_shared<TestSecondaryCache>(2048 * 1024, true);
  // This time with secondary_cache
  base_cache = NewCache(1024 * 1024 /* capacity */, 0 /* num_shard_bits */,
                        false /* strict_capacity_limit */, secondary_cache);
@@ -9,37 +9,4 @@

 namespace ROCKSDB_NAMESPACE {

-namespace {
-
-void NoopDelete(Cache::ObjectPtr, MemoryAllocator*) {}
-
-size_t SliceSize(Cache::ObjectPtr obj) {
-  return static_cast<Slice*>(obj)->size();
-}
-
-Status SliceSaveTo(Cache::ObjectPtr from_obj, size_t from_offset, size_t length,
-                   char* out) {
-  const Slice& slice = *static_cast<Slice*>(from_obj);
-  std::memcpy(out, slice.data() + from_offset, length);
-  return Status::OK();
-}
-
-Status FailCreate(const Slice&, Cache::CreateContext*, MemoryAllocator*,
-                  Cache::ObjectPtr*, size_t*) {
-  return Status::NotSupported("Only for dumping data into SecondaryCache");
-}
-
-}  // namespace
-
-Status SecondaryCache::InsertSaved(const Slice& key, const Slice& saved) {
-  static Cache::CacheItemHelper helper_no_secondary{CacheEntryRole::kMisc,
-                                                    &NoopDelete};
-  static Cache::CacheItemHelper helper{
-      CacheEntryRole::kMisc, &NoopDelete, &SliceSize,
-      &SliceSaveTo,          &FailCreate, &helper_no_secondary};
-  // NOTE: depends on Insert() being synchronous, not keeping pointer `&saved`
-  return Insert(key, const_cast<Slice*>(&saved), &helper,
-                /*force_insert=*/true);
-}
-
 }  // namespace ROCKSDB_NAMESPACE
@@ -5,7 +5,11 @@

 #include "cache/secondary_cache_adapter.h"

+#include <atomic>
+
+#include "cache/tiered_secondary_cache.h"
 #include "monitoring/perf_context_imp.h"
+#include "test_util/sync_point.h"
 #include "util/cast_util.h"

 namespace ROCKSDB_NAMESPACE {
@@ -17,6 +21,7 @@ struct Dummy {
 };
 const Dummy kDummy{};
 Cache::ObjectPtr const kDummyObj = const_cast<Dummy*>(&kDummy);
+const char* kTieredCacheName = "TieredCache";
 }  // namespace

 // When CacheWithSecondaryAdapter is constructed with the distribute_cache_res
@@ -78,7 +83,10 @@ CacheWithSecondaryAdapter::CacheWithSecondaryAdapter(
    : CacheWrapper(std::move(target)),
      secondary_cache_(std::move(secondary_cache)),
      adm_policy_(adm_policy),
-      distribute_cache_res_(distribute_cache_res) {
+      distribute_cache_res_(distribute_cache_res),
+      placeholder_usage_(0),
+      reserved_usage_(0),
+      sec_reserved_(0) {
  target_->SetEvictionCallback(
      [this](const Slice& key, Handle* handle, bool was_hit) {
        return EvictionHandler(key, handle, was_hit);
@@ -111,7 +119,9 @@ CacheWithSecondaryAdapter::~CacheWithSecondaryAdapter() {
    size_t sec_capacity = 0;
    Status s = secondary_cache_->GetCapacity(sec_capacity);
    assert(s.ok());
-    assert(pri_cache_res_->GetTotalReservedCacheSize() == sec_capacity);
+    assert(placeholder_usage_ == 0);
+    assert(reserved_usage_ == 0);
+    assert(pri_cache_res_->GetTotalMemoryUsed() == sec_capacity);
  }
 #endif  // NDEBUG
 }
@@ -119,7 +129,8 @@ CacheWithSecondaryAdapter::~CacheWithSecondaryAdapter() {
 bool CacheWithSecondaryAdapter::EvictionHandler(const Slice& key,
                                                Handle* handle, bool was_hit) {
  auto helper = GetCacheItemHelper(handle);
-  if (helper->IsSecondaryCacheCompatible()) {
+  if (helper->IsSecondaryCacheCompatible() &&
+      adm_policy_ != TieredAdmissionPolicy::kAdmPolicyThreeQueue) {
    auto obj = target_->Value(handle);
    // Ignore dummy entry
    if (obj != kDummyObj) {
@@ -225,14 +236,42 @@ Cache::Handle* CacheWithSecondaryAdapter::Promote(
 Status CacheWithSecondaryAdapter::Insert(const Slice& key, ObjectPtr value,
                                         const CacheItemHelper* helper,
                                         size_t charge, Handle** handle,
-                                         Priority priority) {
+                                         Priority priority,
+                                         const Slice& compressed_value,
+                                         CompressionType type) {
  Status s = target_->Insert(key, value, helper, charge, handle, priority);
-  if (s.ok() && value == nullptr && distribute_cache_res_) {
-    size_t sec_charge = static_cast<size_t>(charge * (sec_cache_res_ratio_));
-    s = secondary_cache_->Deflate(sec_charge);
-    assert(s.ok());
-    s = pri_cache_res_->UpdateCacheReservation(sec_charge, /*increase=*/false);
-    assert(s.ok());
+  if (s.ok() && value == nullptr && distribute_cache_res_ && handle) {
+    charge = target_->GetCharge(*handle);
+
+    MutexLock l(&cache_res_mutex_);
+    placeholder_usage_ += charge;
+    // Check if total placeholder reservation is more than the overall
+    // cache capacity. If it is, then we don't try to charge the
+    // secondary cache because we don't want to overcharge it (beyond
+    // its capacity).
+    // In order to make this a bit more lightweight, we also check if
+    // the difference between placeholder_usage_ and reserved_usage_ is
+    // atleast kReservationChunkSize and avoid any adjustments if not.
+    if ((placeholder_usage_ <= target_->GetCapacity()) &&
+        ((placeholder_usage_ - reserved_usage_) >= kReservationChunkSize)) {
+      reserved_usage_ = placeholder_usage_ & ~(kReservationChunkSize - 1);
+      size_t new_sec_reserved =
+          static_cast<size_t>(reserved_usage_ * sec_cache_res_ratio_);
+      size_t sec_charge = new_sec_reserved - sec_reserved_;
+      s = secondary_cache_->Deflate(sec_charge);
+      assert(s.ok());
+      s = pri_cache_res_->UpdateCacheReservation(sec_charge,
+                                                 /*increase=*/false);
+      assert(s.ok());
+      sec_reserved_ += sec_charge;
+    }
+  }
+  // Warm up the secondary cache with the compressed block. The secondary
+  // cache may choose to ignore it based on the admission policy.
+  if (value != nullptr && !compressed_value.empty() &&
+      adm_policy_ == TieredAdmissionPolicy::kAdmPolicyThreeQueue) {
+    Status status = secondary_cache_->InsertSaved(key, compressed_value, type);
+    assert(status.ok() || status.IsNotSupported());
  }

  return s;
@@ -255,7 +294,8 @@ Cache::Handle* CacheWithSecondaryAdapter::Lookup(const Slice& key,
    bool kept_in_sec_cache = false;
    std::unique_ptr<SecondaryCacheResultHandle> secondary_handle =
        secondary_cache_->Lookup(key, helper, create_context, /*wait*/ true,
-                                 found_dummy_entry, /*out*/ kept_in_sec_cache);
+                                 found_dummy_entry, stats,
+                                 /*out*/ kept_in_sec_cache);
    if (secondary_handle) {
      result = Promote(std::move(secondary_handle), key, helper, priority,
                       stats, found_dummy_entry, kept_in_sec_cache);
@@ -270,11 +310,27 @@ bool CacheWithSecondaryAdapter::Release(Handle* handle,
    ObjectPtr v = target_->Value(handle);
    if (v == nullptr && distribute_cache_res_) {
      size_t charge = target_->GetCharge(handle);
-      size_t sec_charge = static_cast<size_t>(charge * (sec_cache_res_ratio_));
-      Status s = secondary_cache_->Inflate(sec_charge);
-      assert(s.ok());
-      s = pri_cache_res_->UpdateCacheReservation(sec_charge, /*increase=*/true);
-      assert(s.ok());
+
+      MutexLock l(&cache_res_mutex_);
+      placeholder_usage_ -= charge;
+      // Check if total placeholder reservation is more than the overall
+      // cache capacity. If it is, then we do nothing as reserved_usage_ must
+      // be already maxed out
+      if ((placeholder_usage_ <= target_->GetCapacity()) &&
+          (placeholder_usage_ < reserved_usage_)) {
+        // Adjust reserved_usage_ in chunks of kReservationChunkSize, so
+        // we don't hit this slow path too often.
+        reserved_usage_ = placeholder_usage_ & ~(kReservationChunkSize - 1);
+        size_t new_sec_reserved =
+            static_cast<size_t>(reserved_usage_ * sec_cache_res_ratio_);
+        size_t sec_charge = sec_reserved_ - new_sec_reserved;
+        Status s = secondary_cache_->Inflate(sec_charge);
+        assert(s.ok());
+        s = pri_cache_res_->UpdateCacheReservation(sec_charge,
+                                                   /*increase=*/true);
+        assert(s.ok());
+        sec_reserved_ -= sec_charge;
+      }
    }
  }
  return target_->Release(handle, erase_if_last_ref);
@@ -293,10 +349,10 @@ void CacheWithSecondaryAdapter::StartAsyncLookupOnMySecondary(
  assert(async_handle.result_handle == nullptr);

  std::unique_ptr<SecondaryCacheResultHandle> secondary_handle =
-      secondary_cache_->Lookup(async_handle.key, async_handle.helper,
-                               async_handle.create_context, /*wait*/ false,
-                               async_handle.found_dummy_entry,
-                               /*out*/ async_handle.kept_in_sec_cache);
+      secondary_cache_->Lookup(
+          async_handle.key, async_handle.helper, async_handle.create_context,
+          /*wait*/ false, async_handle.found_dummy_entry, async_handle.stats,
+          /*out*/ async_handle.kept_in_sec_cache);
  if (secondary_handle) {
    // TODO with stacked secondaries: Check & process if already ready?
    async_handle.pending_handle = secondary_handle.release();
@@ -406,19 +462,220 @@ std::string CacheWithSecondaryAdapter::GetPrintableOptions() const {
 }

 const char* CacheWithSecondaryAdapter::Name() const {
-  // To the user, at least for now, configure the underlying cache with
-  // a secondary cache. So we pretend to be that cache
-  return target_->Name();
+  if (distribute_cache_res_) {
+    return kTieredCacheName;
+  } else {
+    // To the user, at least for now, configure the underlying cache with
+    // a secondary cache. So we pretend to be that cache
+    return target_->Name();
+  }
 }

-std::shared_ptr<Cache> NewTieredVolatileCache(
-    TieredVolatileCacheOptions& opts) {
-  if (!opts.cache_opts) {
+// Update the total cache capacity. If we're distributing cache reservations
+// to both primary and secondary, then update the pri_cache_res_reservation
+// as well. At the moment, we don't have a good way of handling the case
+// where the new capacity < total cache reservations.
+void CacheWithSecondaryAdapter::SetCapacity(size_t capacity) {
+  size_t sec_capacity = static_cast<size_t>(
+      capacity * (distribute_cache_res_ ? sec_cache_res_ratio_ : 0.0));
+  size_t old_sec_capacity = 0;
+
+  if (distribute_cache_res_) {
+    MutexLock m(&cache_res_mutex_);
+
+    Status s = secondary_cache_->GetCapacity(old_sec_capacity);
+    if (!s.ok()) {
+      return;
+    }
+    if (old_sec_capacity > sec_capacity) {
+      // We're shrinking the cache. We do things in the following order to
+      // avoid a temporary spike in usage over the configured capacity -
+      // 1. Lower the secondary cache capacity
+      // 2. Credit an equal amount (by decreasing pri_cache_res_) to the
+      //    primary cache
+      // 3. Decrease the primary cache capacity to the total budget
+      s = secondary_cache_->SetCapacity(sec_capacity);
+      if (s.ok()) {
+        if (placeholder_usage_ > capacity) {
+          // Adjust reserved_usage_ down
+          reserved_usage_ = capacity & ~(kReservationChunkSize - 1);
+        }
+        size_t new_sec_reserved =
+            static_cast<size_t>(reserved_usage_ * sec_cache_res_ratio_);
+        s = pri_cache_res_->UpdateCacheReservation(
+            (old_sec_capacity - sec_capacity) -
+                (sec_reserved_ - new_sec_reserved),
+            /*increase=*/false);
+        sec_reserved_ = new_sec_reserved;
+        assert(s.ok());
+        target_->SetCapacity(capacity);
+      }
+    } else {
+      // We're expanding the cache. Do it in the following order to avoid
+      // unnecessary evictions -
+      // 1. Increase the primary cache capacity to total budget
+      // 2. Reserve additional memory in primary on behalf of secondary (by
+      //    increasing pri_cache_res_ reservation)
+      // 3. Increase secondary cache capacity
+      target_->SetCapacity(capacity);
+      s = pri_cache_res_->UpdateCacheReservation(
+          sec_capacity - old_sec_capacity,
+          /*increase=*/true);
+      assert(s.ok());
+      s = secondary_cache_->SetCapacity(sec_capacity);
+      assert(s.ok());
+    }
+  } else {
+    // No cache reservation distribution. Just set the primary cache capacity.
+    target_->SetCapacity(capacity);
+  }
+}
+
+Status CacheWithSecondaryAdapter::GetSecondaryCacheCapacity(
+    size_t& size) const {
+  return secondary_cache_->GetCapacity(size);
+}
+
+Status CacheWithSecondaryAdapter::GetSecondaryCachePinnedUsage(
+    size_t& size) const {
+  Status s;
+  if (distribute_cache_res_) {
+    MutexLock m(&cache_res_mutex_);
+    size_t capacity = 0;
+    s = secondary_cache_->GetCapacity(capacity);
+    if (s.ok()) {
+      size = capacity - pri_cache_res_->GetTotalMemoryUsed();
+    } else {
+      size = 0;
+    }
+  } else {
+    size = 0;
+  }
+  return s;
+}
+
+// Update the secondary/primary allocation ratio (remember, the primary
+// capacity is the total memory budget when distribute_cache_res_ is true).
+// When the ratio changes, we may accumulate some error in the calculations
+// for secondary cache inflate/deflate and pri_cache_res_ reservations.
+// This is due to the rounding of the reservation amount.
+//
+// We rely on the current pri_cache_res_ total memory used to estimate the
+// new secondary cache reservation after the ratio change. For this reason,
+// once the ratio is lowered to 0.0 (effectively disabling the secondary
+// cache and pri_cache_res_ total mem used going down to 0), we cannot
+// increase the ratio and re-enable it, We might remove this limitation
+// in the future.
+Status CacheWithSecondaryAdapter::UpdateCacheReservationRatio(
+    double compressed_secondary_ratio) {
+  if (!distribute_cache_res_) {
+    return Status::NotSupported();
+  }
+
+  MutexLock m(&cache_res_mutex_);
+  size_t pri_capacity = target_->GetCapacity();
+  size_t sec_capacity =
+      static_cast<size_t>(pri_capacity * compressed_secondary_ratio);
+  size_t old_sec_capacity;
+  Status s = secondary_cache_->GetCapacity(old_sec_capacity);
+  if (!s.ok()) {
+    return s;
+  }
+
+  // Calculate the new secondary cache reservation
+  // reserved_usage_ will never be > the cache capacity, so we don't
+  // have to worry about adjusting it here.
+  sec_cache_res_ratio_ = compressed_secondary_ratio;
+  size_t new_sec_reserved =
+      static_cast<size_t>(reserved_usage_ * sec_cache_res_ratio_);
+  if (sec_capacity > old_sec_capacity) {
+    // We're increasing the ratio, thus ending up with a larger secondary
+    // cache and a smaller usable primary cache capacity. Similar to
+    // SetCapacity(), we try to avoid a temporary increase in total usage
+    // beyond the configured capacity -
+    // 1. A higher secondary cache ratio means it gets a higher share of
+    //    cache reservations. So first account for that by deflating the
+    //    secondary cache
+    // 2. Increase pri_cache_res_ reservation to reflect the new secondary
+    //    cache utilization (increase in capacity - increase in share of cache
+    //    reservation)
+    // 3. Increase secondary cache capacity
+    s = secondary_cache_->Deflate(new_sec_reserved - sec_reserved_);
+    assert(s.ok());
+    s = pri_cache_res_->UpdateCacheReservation(
+        (sec_capacity - old_sec_capacity) - (new_sec_reserved - sec_reserved_),
+        /*increase=*/true);
+    assert(s.ok());
+    sec_reserved_ = new_sec_reserved;
+    s = secondary_cache_->SetCapacity(sec_capacity);
+    assert(s.ok());
+  } else {
+    // We're shrinking the ratio. Try to avoid unnecessary evictions -
+    // 1. Lower the secondary cache capacity
+    // 2. Decrease pri_cache_res_ reservation to relect lower secondary
+    //    cache utilization (decrease in capacity - decrease in share of cache
+    //    reservations)
+    // 3. Inflate the secondary cache to give it back the reduction in its
+    //    share of cache reservations
+    s = secondary_cache_->SetCapacity(sec_capacity);
+    if (s.ok()) {
+      s = pri_cache_res_->UpdateCacheReservation(
+          (old_sec_capacity - sec_capacity) -
+              (sec_reserved_ - new_sec_reserved),
+          /*increase=*/false);
+      assert(s.ok());
+      s = secondary_cache_->Inflate(sec_reserved_ - new_sec_reserved);
+      assert(s.ok());
+      sec_reserved_ = new_sec_reserved;
+    }
+  }
+
+  return s;
+}
+
+Status CacheWithSecondaryAdapter::UpdateAdmissionPolicy(
+    TieredAdmissionPolicy adm_policy) {
+  adm_policy_ = adm_policy;
+  return Status::OK();
+}
+
+std::shared_ptr<Cache> NewTieredCache(const TieredCacheOptions& _opts) {
+  if (!_opts.cache_opts) {
    return nullptr;
  }

-  if (opts.adm_policy >= TieredAdmissionPolicy::kAdmPolicyMax) {
-    return nullptr;
+  TieredCacheOptions opts = _opts;
+  {
+    bool valid_adm_policy = true;
+
+    switch (_opts.adm_policy) {
+      case TieredAdmissionPolicy::kAdmPolicyAuto:
+        // Select an appropriate default policy
+        if (opts.adm_policy == TieredAdmissionPolicy::kAdmPolicyAuto) {
+          if (opts.nvm_sec_cache) {
+            opts.adm_policy = TieredAdmissionPolicy::kAdmPolicyThreeQueue;
+          } else {
+            opts.adm_policy = TieredAdmissionPolicy::kAdmPolicyPlaceholder;
+          }
+        }
+        break;
+      case TieredAdmissionPolicy::kAdmPolicyPlaceholder:
+      case TieredAdmissionPolicy::kAdmPolicyAllowCacheHits:
+        if (opts.nvm_sec_cache) {
+          valid_adm_policy = false;
+        }
+        break;
+      case TieredAdmissionPolicy::kAdmPolicyThreeQueue:
+        if (!opts.nvm_sec_cache) {
+          valid_adm_policy = false;
+        }
+        break;
+      default:
+        valid_adm_policy = false;
+    }
+    if (!valid_adm_policy) {
+      return nullptr;
+    }
  }

  std::shared_ptr<Cache> cache;
@@ -426,21 +683,58 @@ std::shared_ptr<Cache> NewTieredVolatileCache(
    LRUCacheOptions cache_opts =
        *(static_cast_with_check<LRUCacheOptions, ShardedCacheOptions>(
            opts.cache_opts));
-    cache_opts.capacity += opts.comp_cache_opts.capacity;
+    cache_opts.capacity = opts.total_capacity;
+    cache_opts.secondary_cache = nullptr;
    cache = cache_opts.MakeSharedCache();
  } else if (opts.cache_type == PrimaryCacheType::kCacheTypeHCC) {
    HyperClockCacheOptions cache_opts =
        *(static_cast_with_check<HyperClockCacheOptions, ShardedCacheOptions>(
            opts.cache_opts));
-    cache_opts.capacity += opts.comp_cache_opts.capacity;
+    cache_opts.capacity = opts.total_capacity;
+    cache_opts.secondary_cache = nullptr;
    cache = cache_opts.MakeSharedCache();
  } else {
    return nullptr;
  }
  std::shared_ptr<SecondaryCache> sec_cache;
+  opts.comp_cache_opts.capacity = static_cast<size_t>(
+      opts.total_capacity * opts.compressed_secondary_ratio);
  sec_cache = NewCompressedSecondaryCache(opts.comp_cache_opts);

+  if (opts.nvm_sec_cache) {
+    if (opts.adm_policy == TieredAdmissionPolicy::kAdmPolicyThreeQueue) {
+      sec_cache = std::make_shared<TieredSecondaryCache>(
+          sec_cache, opts.nvm_sec_cache,
+          TieredAdmissionPolicy::kAdmPolicyThreeQueue);
+    } else {
+      return nullptr;
+    }
+  }
+
  return std::make_shared<CacheWithSecondaryAdapter>(
      cache, sec_cache, opts.adm_policy, /*distribute_cache_res=*/true);
 }
+
+Status UpdateTieredCache(const std::shared_ptr<Cache>& cache,
+                         int64_t total_capacity,
+                         double compressed_secondary_ratio,
+                         TieredAdmissionPolicy adm_policy) {
+  if (!cache || strcmp(cache->Name(), kTieredCacheName)) {
+    return Status::InvalidArgument();
+  }
+  CacheWithSecondaryAdapter* tiered_cache =
+      static_cast<CacheWithSecondaryAdapter*>(cache.get());
+
+  Status s;
+  if (total_capacity > 0) {
+    tiered_cache->SetCapacity(total_capacity);
+  }
+  if (compressed_secondary_ratio >= 0.0 && compressed_secondary_ratio <= 1.0) {
+    s = tiered_cache->UpdateCacheReservationRatio(compressed_secondary_ratio);
+  }
+  if (adm_policy < TieredAdmissionPolicy::kAdmPolicyMax) {
+    s = tiered_cache->UpdateAdmissionPolicy(adm_policy);
+  }
+  return s;
+}
 }  // namespace ROCKSDB_NAMESPACE
@@ -20,10 +20,12 @@ class CacheWithSecondaryAdapter : public CacheWrapper {

  ~CacheWithSecondaryAdapter() override;

-  Status Insert(const Slice& key, ObjectPtr value,
-                const CacheItemHelper* helper, size_t charge,
-                Handle** handle = nullptr,
-                Priority priority = Priority::LOW) override;
+  Status Insert(
+      const Slice& key, ObjectPtr value, const CacheItemHelper* helper,
+      size_t charge, Handle** handle = nullptr,
+      Priority priority = Priority::LOW,
+      const Slice& compressed_value = Slice(),
+      CompressionType type = CompressionType::kNoCompression) override;

  Handle* Lookup(const Slice& key, const CacheItemHelper* helper,
                 CreateContext* create_context,
@@ -43,11 +45,23 @@ class CacheWithSecondaryAdapter : public CacheWrapper {

  const char* Name() const override;

+  void SetCapacity(size_t capacity) override;
+
+  Status GetSecondaryCacheCapacity(size_t& size) const override;
+
+  Status GetSecondaryCachePinnedUsage(size_t& size) const override;
+
+  Status UpdateCacheReservationRatio(double ratio);
+
+  Status UpdateAdmissionPolicy(TieredAdmissionPolicy adm_policy);
+
  Cache* TEST_GetCache() { return target_.get(); }

  SecondaryCache* TEST_GetSecondaryCache() { return secondary_cache_.get(); }

 private:
+  static constexpr size_t kReservationChunkSize = 1 << 20;
+
  bool EvictionHandler(const Slice& key, Handle* handle, bool was_hit);

  void StartAsyncLookupOnMySecondary(AsyncLookupHandle& async_handle);
@@ -73,6 +87,17 @@ class CacheWithSecondaryAdapter : public CacheWrapper {
  // Fraction of a cache memory reservation to be assigned to the secondary
  // cache
  double sec_cache_res_ratio_;
+  // Mutex for use when managing cache memory reservations. Should not be used
+  // for other purposes, as it may risk causing deadlocks.
+  mutable port::Mutex cache_res_mutex_;
+  // Total memory reserved by placeholder entriesin the cache
+  size_t placeholder_usage_;
+  // Total placeholoder memory charged to both the primary and secondary
+  // caches. Will be <= placeholder_usage_.
+  size_t reserved_usage_;
+  // Amount of memory reserved in the secondary cache. This should be
+  // reserved_usage_ * sec_cache_res_ratio_ in steady state.
+  size_t sec_reserved_;
 };

 }  // namespace ROCKSDB_NAMESPACE
@@ -83,6 +83,16 @@ size_t ShardedCacheBase::GetCapacity() const {
  return capacity_;
 }

+Status ShardedCacheBase::GetSecondaryCacheCapacity(size_t& size) const {
+  size = 0;
+  return Status::OK();
+}
+
+Status ShardedCacheBase::GetSecondaryCachePinnedUsage(size_t& size) const {
+  size = 0;
+  return Status::OK();
+}
+
 bool ShardedCacheBase::HasStrictCapacityLimit() const {
  MutexLock l(&config_mutex_);
  return strict_capacity_limit_;
@@ -99,6 +99,8 @@ class ShardedCacheBase : public Cache {

  bool HasStrictCapacityLimit() const override;
  size_t GetCapacity() const override;
+  Status GetSecondaryCacheCapacity(size_t& size) const override;
+  Status GetSecondaryCachePinnedUsage(size_t& size) const override;

  using Cache::GetUsage;
  size_t GetUsage(Handle* handle) const override;
@@ -170,9 +172,12 @@ class ShardedCache : public ShardedCacheBase {
        [s_c_l](CacheShard* cs) { cs->SetStrictCapacityLimit(s_c_l); });
  }

-  Status Insert(const Slice& key, ObjectPtr obj, const CacheItemHelper* helper,
-                size_t charge, Handle** handle = nullptr,
-                Priority priority = Priority::LOW) override {
+  Status Insert(
+      const Slice& key, ObjectPtr obj, const CacheItemHelper* helper,
+      size_t charge, Handle** handle = nullptr,
+      Priority priority = Priority::LOW,
+      const Slice& /*compressed_value*/ = Slice(),
+      CompressionType /*type*/ = CompressionType::kNoCompression) override {
    assert(helper);
    HashVal hash = CacheShard::ComputeHash(key, hash_seed_);
    auto h_out = reinterpret_cast<HandleImpl**>(handle);
@@ -0,0 +1,125 @@
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#include "cache/tiered_secondary_cache.h"
+
+#include "monitoring/statistics_impl.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+// Creation callback for use in the lookup path. It calls the upper layer
+// create_cb to create the object, and optionally calls the compressed
+// secondary cache InsertSaved to save the compressed block. If
+// advise_erase is set, it means the primary cache wants the block to be
+// erased in the secondary cache, so we skip calling InsertSaved.
+//
+// For the time being, we assume that all blocks in the nvm tier belong to
+// the primary block cache (i.e CacheTier::kVolatileTier). That can be changed
+// if we implement demotion from the compressed secondary cache to the nvm
+// cache in the future.
+Status TieredSecondaryCache::MaybeInsertAndCreate(
+    const Slice& data, CompressionType type, CacheTier source,
+    Cache::CreateContext* ctx, MemoryAllocator* allocator,
+    Cache::ObjectPtr* out_obj, size_t* out_charge) {
+  TieredSecondaryCache::CreateContext* context =
+      static_cast<TieredSecondaryCache::CreateContext*>(ctx);
+  assert(source == CacheTier::kVolatileTier);
+  if (!context->advise_erase && type != kNoCompression) {
+    // Attempt to insert into compressed secondary cache
+    // TODO: Don't hardcode the source
+    context->comp_sec_cache->InsertSaved(*context->key, data, type, source)
+        .PermitUncheckedError();
+    RecordTick(context->stats, COMPRESSED_SECONDARY_CACHE_PROMOTIONS);
+  } else {
+    RecordTick(context->stats, COMPRESSED_SECONDARY_CACHE_PROMOTION_SKIPS);
+  }
+  // Primary cache will accept the object, so call its helper to create
+  // the object
+  return context->helper->create_cb(data, type, source, context->inner_ctx,
+                                    allocator, out_obj, out_charge);
+}
+
+// The lookup first looks up in the compressed secondary cache. If its a miss,
+// then the nvm cache lookup is called. The cache item helper and create
+// context are wrapped in order to intercept the creation callback to make
+// the decision on promoting to the compressed secondary cache.
+std::unique_ptr<SecondaryCacheResultHandle> TieredSecondaryCache::Lookup(
+    const Slice& key, const Cache::CacheItemHelper* helper,
+    Cache::CreateContext* create_context, bool wait, bool advise_erase,
+    Statistics* stats, bool& kept_in_sec_cache) {
+  bool dummy = false;
+  std::unique_ptr<SecondaryCacheResultHandle> result =
+      target()->Lookup(key, helper, create_context, wait, advise_erase, stats,
+                       /*kept_in_sec_cache=*/dummy);
+  // We never want the item to spill back into the secondary cache
+  kept_in_sec_cache = true;
+  if (result) {
+    assert(result->IsReady());
+    return result;
+  }
+
+  // If wait is true, then we can be a bit more efficient and avoid a memory
+  // allocation for the CReateContext.
+  const Cache::CacheItemHelper* outer_helper =
+      TieredSecondaryCache::GetHelper();
+  if (wait) {
+    TieredSecondaryCache::CreateContext ctx;
+    ctx.key = &key;
+    ctx.advise_erase = advise_erase;
+    ctx.helper = helper;
+    ctx.inner_ctx = create_context;
+    ctx.comp_sec_cache = target();
+    ctx.stats = stats;
+
+    return nvm_sec_cache_->Lookup(key, outer_helper, &ctx, wait, advise_erase,
+                                  stats, kept_in_sec_cache);
+  }
+
+  // If wait is false, i.e its an async lookup, we have to allocate a result
+  // handle for tracking purposes. Embed the CreateContext inside the handle
+  // so we need only allocate memory once instead of twice.
+  std::unique_ptr<ResultHandle> handle(new ResultHandle());
+  handle->ctx()->key = &key;
+  handle->ctx()->advise_erase = advise_erase;
+  handle->ctx()->helper = helper;
+  handle->ctx()->inner_ctx = create_context;
+  handle->ctx()->comp_sec_cache = target();
+  handle->ctx()->stats = stats;
+  handle->SetInnerHandle(
+      nvm_sec_cache_->Lookup(key, outer_helper, handle->ctx(), wait,
+                             advise_erase, stats, kept_in_sec_cache));
+  if (!handle->inner_handle()) {
+    handle.reset();
+  } else {
+    result.reset(handle.release());
+  }
+
+  return result;
+}
+
+// Call the nvm cache WaitAll to complete the lookups
+void TieredSecondaryCache::WaitAll(
+    std::vector<SecondaryCacheResultHandle*> handles) {
+  std::vector<SecondaryCacheResultHandle*> nvm_handles;
+  std::vector<ResultHandle*> my_handles;
+  nvm_handles.reserve(handles.size());
+  for (auto handle : handles) {
+    // The handle could belong to the compressed secondary cache. Skip if
+    // that's the case.
+    if (handle->IsReady()) {
+      continue;
+    }
+    ResultHandle* hdl = static_cast<ResultHandle*>(handle);
+    nvm_handles.push_back(hdl->inner_handle());
+    my_handles.push_back(hdl);
+  }
+  nvm_sec_cache_->WaitAll(nvm_handles);
+  for (auto handle : my_handles) {
+    assert(handle->inner_handle()->IsReady());
+    handle->Complete();
+  }
+}
+
+}  // namespace ROCKSDB_NAMESPACE
@@ -0,0 +1,160 @@
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include "rocksdb/cache.h"
+#include "rocksdb/secondary_cache.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+// A SecondaryCache that implements stacking of a compressed secondary cache
+// and a non-volatile (local flash) cache. It implements an admission
+// policy of warming the bottommost tier (local flash) with compressed
+// blocks from the SST on misses, and on hits in the bottommost tier,
+// promoting to the compressed and/or primary block cache. The admission
+// policies of the primary block cache and compressed secondary cache remain
+// unchanged - promote on second access. There is no demotion ofablocks
+// evicted from a tier. They are just discarded.
+//
+// In order to properly handle compressed blocks directly read from SSTs, and
+// to allow writeback of blocks compressed by the compressed secondary
+// cache in the future, we make use of the compression type and source
+// cache tier arguments in InsertSaved.
+class TieredSecondaryCache : public SecondaryCacheWrapper {
+ public:
+  TieredSecondaryCache(std::shared_ptr<SecondaryCache> comp_sec_cache,
+                       std::shared_ptr<SecondaryCache> nvm_sec_cache,
+                       TieredAdmissionPolicy adm_policy)
+      : SecondaryCacheWrapper(comp_sec_cache), nvm_sec_cache_(nvm_sec_cache) {
+#ifndef NDEBUG
+    assert(adm_policy == TieredAdmissionPolicy::kAdmPolicyThreeQueue);
+#else
+    (void)adm_policy;
+#endif
+  }
+
+  ~TieredSecondaryCache() override {}
+
+  const char* Name() const override { return "TieredSecondaryCache"; }
+
+  // This is a no-op as we currently don't allow demotion (i.e
+  // insertion by the upper layer) of evicted blocks.
+  virtual Status Insert(const Slice& /*key*/, Cache::ObjectPtr /*obj*/,
+                        const Cache::CacheItemHelper* /*helper*/,
+                        bool /*force_insert*/) override {
+    return Status::OK();
+  }
+
+  // Warm up the nvm tier directly
+  virtual Status InsertSaved(
+      const Slice& key, const Slice& saved,
+      CompressionType type = CompressionType::kNoCompression,
+      CacheTier source = CacheTier::kVolatileTier) override {
+    return nvm_sec_cache_->InsertSaved(key, saved, type, source);
+  }
+
+  virtual std::unique_ptr<SecondaryCacheResultHandle> Lookup(
+      const Slice& key, const Cache::CacheItemHelper* helper,
+      Cache::CreateContext* create_context, bool wait, bool advise_erase,
+      Statistics* stats, bool& kept_in_sec_cache) override;
+
+  virtual void WaitAll(
+      std::vector<SecondaryCacheResultHandle*> handles) override;
+
+ private:
+  struct CreateContext : public Cache::CreateContext {
+    const Slice* key;
+    bool advise_erase;
+    const Cache::CacheItemHelper* helper;
+    Cache::CreateContext* inner_ctx;
+    std::shared_ptr<SecondaryCacheResultHandle> inner_handle;
+    SecondaryCache* comp_sec_cache;
+    Statistics* stats;
+  };
+
+  class ResultHandle : public SecondaryCacheResultHandle {
+   public:
+    ~ResultHandle() override {}
+
+    bool IsReady() override {
+      if (inner_handle_ && inner_handle_->IsReady()) {
+        Complete();
+      }
+      return ready_;
+    }
+
+    void Wait() override {
+      inner_handle_->Wait();
+      Complete();
+    }
+
+    size_t Size() override { return size_; }
+
+    Cache::ObjectPtr Value() override { return value_; }
+
+    void Complete() {
+      size_ = inner_handle_->Size();
+      value_ = inner_handle_->Value();
+      inner_handle_.reset();
+      ready_ = true;
+    }
+
+    void SetInnerHandle(std::unique_ptr<SecondaryCacheResultHandle>&& handle) {
+      inner_handle_ = std::move(handle);
+    }
+
+    void SetSize(size_t size) { size_ = size; }
+
+    void SetValue(Cache::ObjectPtr val) { value_ = val; }
+
+    CreateContext* ctx() { return &ctx_; }
+
+    SecondaryCacheResultHandle* inner_handle() { return inner_handle_.get(); }
+
+   private:
+    std::unique_ptr<SecondaryCacheResultHandle> inner_handle_;
+    CreateContext ctx_;
+    size_t size_;
+    Cache::ObjectPtr value_;
+    bool ready_ = false;
+  };
+
+  static void NoopDelete(Cache::ObjectPtr /*obj*/,
+                         MemoryAllocator* /*allocator*/) {
+    assert(false);
+  }
+  static size_t ZeroSize(Cache::ObjectPtr /*obj*/) {
+    assert(false);
+    return 0;
+  }
+  static Status NoopSaveTo(Cache::ObjectPtr /*from_obj*/,
+                           size_t /*from_offset*/, size_t /*length*/,
+                           char* /*out_buf*/) {
+    assert(false);
+    return Status::OK();
+  }
+  static Status MaybeInsertAndCreate(const Slice& data, CompressionType type,
+                                     CacheTier source,
+                                     Cache::CreateContext* ctx,
+                                     MemoryAllocator* allocator,
+                                     Cache::ObjectPtr* out_obj,
+                                     size_t* out_charge);
+
+  static const Cache::CacheItemHelper* GetHelper() {
+    const static Cache::CacheItemHelper basic_helper(CacheEntryRole::kMisc,
+                                                     &NoopDelete);
+    const static Cache::CacheItemHelper maybe_insert_and_create_helper{
+        CacheEntryRole::kMisc, &NoopDelete,           &ZeroSize,
+        &NoopSaveTo,           &MaybeInsertAndCreate, &basic_helper,
+    };
+    return &maybe_insert_and_create_helper;
+  }
+
+  std::shared_ptr<SecondaryCache> comp_sec_cache_;
+  std::shared_ptr<SecondaryCache> nvm_sec_cache_;
+};
+
+}  // namespace ROCKSDB_NAMESPACE
@@ -0,0 +1,831 @@
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+//
+#include "cache/compressed_secondary_cache.h"
+#include "cache/secondary_cache_adapter.h"
+#include "db/db_test_util.h"
+#include "rocksdb/cache.h"
+#include "rocksdb/secondary_cache.h"
+#include "typed_cache.h"
+#include "util/random.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+class TestSecondaryCache : public SecondaryCache {
+ public:
+  explicit TestSecondaryCache(size_t capacity, bool ready_before_wait)
+      : cache_(NewLRUCache(capacity, 0, false, 0.5 /* high_pri_pool_ratio */,
+                           nullptr, kDefaultToAdaptiveMutex,
+                           kDontChargeCacheMetadata)),
+        ready_before_wait_(ready_before_wait),
+        num_insert_saved_(0),
+        num_hits_(0),
+        num_misses_(0) {}
+
+  const char* Name() const override { return "TestSecondaryCache"; }
+
+  Status Insert(const Slice& /*key*/, Cache::ObjectPtr /*value*/,
+                const Cache::CacheItemHelper* /*helper*/,
+                bool /*force_insert*/) override {
+    assert(false);
+    return Status::NotSupported();
+  }
+
+  Status InsertSaved(const Slice& key, const Slice& saved,
+                     CompressionType type = kNoCompression,
+                     CacheTier source = CacheTier::kVolatileTier) override {
+    CheckCacheKeyCommonPrefix(key);
+    size_t size;
+    char* buf;
+    Status s;
+
+    num_insert_saved_++;
+    size = saved.size();
+    buf = new char[size + sizeof(uint64_t) + 2 * sizeof(uint16_t)];
+    EncodeFixed64(buf, size);
+    buf += sizeof(uint64_t);
+    EncodeFixed16(buf, type);
+    buf += sizeof(uint16_t);
+    EncodeFixed16(buf, (uint16_t)source);
+    buf += sizeof(uint16_t);
+    memcpy(buf, saved.data(), size);
+    buf -= sizeof(uint64_t) + 2 * sizeof(uint16_t);
+    if (!s.ok()) {
+      delete[] buf;
+      return s;
+    }
+    return cache_.Insert(key, buf, size);
+  }
+
+  std::unique_ptr<SecondaryCacheResultHandle> Lookup(
+      const Slice& key, const Cache::CacheItemHelper* helper,
+      Cache::CreateContext* create_context, bool wait, bool /*advise_erase*/,
+      Statistics* /*stats*/, bool& kept_in_sec_cache) override {
+    std::string key_str = key.ToString();
+    TEST_SYNC_POINT_CALLBACK("TestSecondaryCache::Lookup", &key_str);
+
+    std::unique_ptr<SecondaryCacheResultHandle> secondary_handle;
+    kept_in_sec_cache = false;
+
+    TypedHandle* handle = cache_.Lookup(key);
+    if (handle) {
+      num_hits_++;
+      Cache::ObjectPtr value = nullptr;
+      size_t charge = 0;
+      Status s;
+      char* ptr = cache_.Value(handle);
+      CompressionType type;
+      CacheTier source;
+      size_t size = DecodeFixed64(ptr);
+      ptr += sizeof(uint64_t);
+      type = static_cast<CompressionType>(DecodeFixed16(ptr));
+      ptr += sizeof(uint16_t);
+      source = static_cast<CacheTier>(DecodeFixed16(ptr));
+      assert(source == CacheTier::kVolatileTier);
+      ptr += sizeof(uint16_t);
+      s = helper->create_cb(Slice(ptr, size), type, source, create_context,
+                            /*alloc*/ nullptr, &value, &charge);
+      if (s.ok()) {
+        secondary_handle.reset(new TestSecondaryCacheResultHandle(
+            cache_.get(), handle, value, charge,
+            /*ready=*/wait || ready_before_wait_));
+        kept_in_sec_cache = true;
+      } else {
+        cache_.Release(handle);
+      }
+    } else {
+      num_misses_++;
+    }
+    return secondary_handle;
+  }
+
+  bool SupportForceErase() const override { return false; }
+
+  void Erase(const Slice& /*key*/) override {}
+
+  void WaitAll(std::vector<SecondaryCacheResultHandle*> handles) override {
+    for (SecondaryCacheResultHandle* handle : handles) {
+      TestSecondaryCacheResultHandle* sec_handle =
+          static_cast<TestSecondaryCacheResultHandle*>(handle);
+      EXPECT_FALSE(sec_handle->IsReady());
+      sec_handle->SetReady();
+    }
+  }
+
+  std::string GetPrintableOptions() const override { return ""; }
+
+  uint32_t num_insert_saved() { return num_insert_saved_; }
+
+  uint32_t num_hits() { return num_hits_; }
+
+  uint32_t num_misses() { return num_misses_; }
+
+  void CheckCacheKeyCommonPrefix(const Slice& key) {
+    Slice current_prefix(key.data(), OffsetableCacheKey::kCommonPrefixSize);
+    if (ckey_prefix_.empty()) {
+      ckey_prefix_ = current_prefix.ToString();
+    } else {
+      EXPECT_EQ(ckey_prefix_, current_prefix.ToString());
+    }
+  }
+
+ private:
+  class TestSecondaryCacheResultHandle : public SecondaryCacheResultHandle {
+   public:
+    TestSecondaryCacheResultHandle(Cache* cache, Cache::Handle* handle,
+                                   Cache::ObjectPtr value, size_t size,
+                                   bool ready)
+        : cache_(cache),
+          handle_(handle),
+          value_(value),
+          size_(size),
+          is_ready_(ready) {}
+
+    ~TestSecondaryCacheResultHandle() override { cache_->Release(handle_); }
+
+    bool IsReady() override { return is_ready_; }
+
+    void Wait() override {}
+
+    Cache::ObjectPtr Value() override {
+      assert(is_ready_);
+      return value_;
+    }
+
+    size_t Size() override { return Value() ? size_ : 0; }
+
+    void SetReady() { is_ready_ = true; }
+
+   private:
+    Cache* cache_;
+    Cache::Handle* handle_;
+    Cache::ObjectPtr value_;
+    size_t size_;
+    bool is_ready_;
+  };
+
+  using SharedCache =
+      BasicTypedSharedCacheInterface<char[], CacheEntryRole::kMisc>;
+  using TypedHandle = SharedCache::TypedHandle;
+  SharedCache cache_;
+  bool ready_before_wait_;
+  uint32_t num_insert_saved_;
+  uint32_t num_hits_;
+  uint32_t num_misses_;
+  std::string ckey_prefix_;
+};
+
+class DBTieredSecondaryCacheTest : public DBTestBase {
+ public:
+  DBTieredSecondaryCacheTest()
+      : DBTestBase("db_tiered_secondary_cache_test", /*env_do_fsync=*/true) {}
+
+  std::shared_ptr<Cache> NewCache(
+      size_t pri_capacity, size_t compressed_capacity, size_t nvm_capacity,
+      TieredAdmissionPolicy adm_policy = TieredAdmissionPolicy::kAdmPolicyAuto,
+      bool ready_before_wait = false) {
+    LRUCacheOptions lru_opts;
+    TieredCacheOptions opts;
+    lru_opts.capacity = 0;
+    lru_opts.num_shard_bits = 0;
+    lru_opts.high_pri_pool_ratio = 0;
+    opts.cache_opts = &lru_opts;
+    opts.cache_type = PrimaryCacheType::kCacheTypeLRU;
+    opts.comp_cache_opts.capacity = 0;
+    opts.comp_cache_opts.num_shard_bits = 0;
+    opts.total_capacity = pri_capacity + compressed_capacity;
+    opts.compressed_secondary_ratio = compressed_secondary_ratio_ =
+        (double)compressed_capacity / opts.total_capacity;
+    if (nvm_capacity > 0) {
+      nvm_sec_cache_.reset(
+          new TestSecondaryCache(nvm_capacity, ready_before_wait));
+      opts.nvm_sec_cache = nvm_sec_cache_;
+    }
+    opts.adm_policy = adm_policy;
+    cache_ = NewTieredCache(opts);
+    assert(cache_ != nullptr);
+
+    return cache_;
+  }
+
+  void ClearPrimaryCache() {
+    ASSERT_EQ(UpdateTieredCache(cache_, -1, 1.0), Status::OK());
+    ASSERT_EQ(UpdateTieredCache(cache_, -1, compressed_secondary_ratio_),
+              Status::OK());
+  }
+
+  TestSecondaryCache* nvm_sec_cache() { return nvm_sec_cache_.get(); }
+
+  CompressedSecondaryCache* compressed_secondary_cache() {
+    return static_cast<CompressedSecondaryCache*>(
+        static_cast<CacheWithSecondaryAdapter*>(cache_.get())
+            ->TEST_GetSecondaryCache());
+  }
+
+ private:
+  std::shared_ptr<Cache> cache_;
+  std::shared_ptr<TestSecondaryCache> nvm_sec_cache_;
+  double compressed_secondary_ratio_;
+};
+
+// In this test, the block size is set to 4096. Each value is 1007 bytes, so
+// each data block contains exactly 4 KV pairs. Metadata blocks are not
+// cached, so we can accurately estimate the cache usage.
+TEST_F(DBTieredSecondaryCacheTest, BasicTest) {
+  if (!LZ4_Supported()) {
+    ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
+    return;
+  }
+
+  BlockBasedTableOptions table_options;
+  // We want a block cache of size 5KB, and a compressed secondary cache of
+  // size 5KB. However, we specify a block cache size of 256KB here in order
+  // to take into account the cache reservation in the block cache on
+  // behalf of the compressed cache. The unit of cache reservation is 256KB.
+  // The effective block cache capacity will be calculated as 256 + 5 = 261KB,
+  // and 256KB will be reserved for the compressed cache, leaving 5KB for
+  // the primary block cache. We only have to worry about this here because
+  // the cache size is so small.
+  table_options.block_cache = NewCache(256 * 1024, 5 * 1024, 256 * 1024);
+  table_options.block_size = 4 * 1024;
+  table_options.cache_index_and_filter_blocks = false;
+  Options options = GetDefaultOptions();
+  options.create_if_missing = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+
+  // Disable paranoid_file_checks so that flush will not read back the newly
+  // written file
+  options.paranoid_file_checks = false;
+  DestroyAndReopen(options);
+  Random rnd(301);
+  const int N = 256;
+  for (int i = 0; i < N; i++) {
+    std::string p_v;
+    test::CompressibleString(&rnd, 0.5, 1007, &p_v);
+    ASSERT_OK(Put(Key(i), p_v));
+  }
+
+  ASSERT_OK(Flush());
+
+  // The first 2 Gets, for keys 0 and 5, will load the corresponding data
+  // blocks as they will be cache misses. The nvm secondary cache will be
+  // warmed up with the compressed blocks
+  std::string v = Get(Key(0));
+  ASSERT_EQ(1007, v.size());
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 1u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 1u);
+
+  v = Get(Key(5));
+  ASSERT_EQ(1007, v.size());
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 2u);
+
+  // At this point, the nvm cache is warmed up with the data blocks for 0
+  // and 5. The next Get will lookup the block in nvm and will be a hit.
+  // It will be created as a standalone entry in memory, and a placeholder
+  // will be inserted in the primary and compressed caches.
+  v = Get(Key(0));
+  ASSERT_EQ(1007, v.size());
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 1u);
+
+  // For this Get, the primary and compressed only have placeholders for
+  // the required data block. So we will lookup the nvm cache and find the
+  // block there. This time, the block will be promoted to the primary
+  // block cache. No promotion to the compressed secondary cache happens,
+  // and it will retain the placeholder.
+  v = Get(Key(0));
+  ASSERT_EQ(1007, v.size());
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 2u);
+
+  // This Get will find the data block in the primary cache.
+  v = Get(Key(0));
+  ASSERT_EQ(1007, v.size());
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 2u);
+
+  // We repeat the sequence for key 5. This will end up evicting the block
+  // for 0 from the in-memory cache.
+  v = Get(Key(5));
+  ASSERT_EQ(1007, v.size());
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 3u);
+
+  v = Get(Key(5));
+  ASSERT_EQ(1007, v.size());
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 4u);
+
+  v = Get(Key(5));
+  ASSERT_EQ(1007, v.size());
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 4u);
+
+  // This Get for key 0 will find the data block in nvm. Since the compressed
+  // cache still has the placeholder, the block (compressed) will be
+  // admitted. It is theh inserted into the primary as a standalone entry.
+  v = Get(Key(0));
+  ASSERT_EQ(1007, v.size());
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 5u);
+
+  // This Get for key 0 will find the data block in the compressed secondary
+  // cache.
+  v = Get(Key(0));
+  ASSERT_EQ(1007, v.size());
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 5u);
+
+  Destroy(options);
+}
+
+// This test is very similar to BasicTest, except it calls MultiGet rather
+// than Get, in order to exercise the async lookup and WaitAll path.
+TEST_F(DBTieredSecondaryCacheTest, BasicMultiGetTest) {
+  if (!LZ4_Supported()) {
+    ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
+    return;
+  }
+
+  BlockBasedTableOptions table_options;
+  table_options.block_cache = NewCache(260 * 1024, 10 * 1024, 256 * 1024);
+  table_options.block_size = 4 * 1024;
+  table_options.cache_index_and_filter_blocks = false;
+  Options options = GetDefaultOptions();
+  options.create_if_missing = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+
+  options.paranoid_file_checks = false;
+  DestroyAndReopen(options);
+  Random rnd(301);
+  const int N = 256;
+  for (int i = 0; i < N; i++) {
+    std::string p_v;
+    test::CompressibleString(&rnd, 0.5, 1007, &p_v);
+    ASSERT_OK(Put(Key(i), p_v));
+  }
+
+  ASSERT_OK(Flush());
+
+  std::vector<std::string> keys;
+  std::vector<std::string> values;
+
+  keys.push_back(Key(0));
+  keys.push_back(Key(4));
+  keys.push_back(Key(8));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 3u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 3u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 0u);
+
+  keys.clear();
+  values.clear();
+  keys.push_back(Key(12));
+  keys.push_back(Key(16));
+  keys.push_back(Key(20));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 0u);
+
+  keys.clear();
+  values.clear();
+  keys.push_back(Key(0));
+  keys.push_back(Key(4));
+  keys.push_back(Key(8));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 3u);
+
+  keys.clear();
+  values.clear();
+  keys.push_back(Key(0));
+  keys.push_back(Key(4));
+  keys.push_back(Key(8));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 6u);
+
+  keys.clear();
+  values.clear();
+  keys.push_back(Key(0));
+  keys.push_back(Key(4));
+  keys.push_back(Key(8));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 6u);
+
+  keys.clear();
+  values.clear();
+  keys.push_back(Key(12));
+  keys.push_back(Key(16));
+  keys.push_back(Key(20));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 9u);
+
+  keys.clear();
+  values.clear();
+  keys.push_back(Key(12));
+  keys.push_back(Key(16));
+  keys.push_back(Key(20));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 12u);
+
+  keys.clear();
+  values.clear();
+  keys.push_back(Key(12));
+  keys.push_back(Key(16));
+  keys.push_back(Key(20));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 12u);
+
+  Destroy(options);
+}
+
+TEST_F(DBTieredSecondaryCacheTest, WaitAllTest) {
+  if (!LZ4_Supported()) {
+    ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
+    return;
+  }
+
+  BlockBasedTableOptions table_options;
+  table_options.block_cache = NewCache(250 * 1024, 20 * 1024, 256 * 1024);
+  table_options.block_size = 4 * 1024;
+  table_options.cache_index_and_filter_blocks = false;
+  Options options = GetDefaultOptions();
+  options.create_if_missing = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+
+  options.paranoid_file_checks = false;
+  DestroyAndReopen(options);
+  Random rnd(301);
+  const int N = 256;
+  for (int i = 0; i < N; i++) {
+    std::string p_v;
+    test::CompressibleString(&rnd, 0.5, 1007, &p_v);
+    ASSERT_OK(Put(Key(i), p_v));
+  }
+
+  ASSERT_OK(Flush());
+
+  std::vector<std::string> keys;
+  std::vector<std::string> values;
+
+  keys.push_back(Key(0));
+  keys.push_back(Key(4));
+  keys.push_back(Key(8));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 3u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 3u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 0u);
+
+  keys.clear();
+  values.clear();
+  keys.push_back(Key(12));
+  keys.push_back(Key(16));
+  keys.push_back(Key(20));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 0u);
+
+  // Insert placeholders for 4 in primary and compressed
+  std::string val = Get(Key(4));
+
+  // Force placeholder 4 out of primary
+  keys.clear();
+  values.clear();
+  keys.push_back(Key(24));
+  keys.push_back(Key(28));
+  keys.push_back(Key(32));
+  keys.push_back(Key(36));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 10u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 10u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 1u);
+
+  // Now read 4 again. This will create a placeholder in primary, and insert
+  // in compressed secondary since it already has a placeholder
+  val = Get(Key(4));
+
+  // Now read 0, 4 and 8. While 4 is already in the compressed secondary
+  // cache, 0 and 8 will be read asynchronously from the nvm tier. The
+  // WaitAll will be called for all 3 blocks.
+  keys.clear();
+  values.clear();
+  keys.push_back(Key(0));
+  keys.push_back(Key(4));
+  keys.push_back(Key(8));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 10u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 10u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 4u);
+
+  Destroy(options);
+}
+
+TEST_F(DBTieredSecondaryCacheTest, ReadyBeforeWaitAllTest) {
+  if (!LZ4_Supported()) {
+    ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
+    return;
+  }
+
+  BlockBasedTableOptions table_options;
+  table_options.block_cache = NewCache(250 * 1024, 20 * 1024, 256 * 1024,
+                                       TieredAdmissionPolicy::kAdmPolicyAuto,
+                                       /*ready_before_wait=*/true);
+  table_options.block_size = 4 * 1024;
+  table_options.cache_index_and_filter_blocks = false;
+  Options options = GetDefaultOptions();
+  options.create_if_missing = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  options.statistics = CreateDBStatistics();
+
+  options.paranoid_file_checks = false;
+  DestroyAndReopen(options);
+  Random rnd(301);
+  const int N = 256;
+  for (int i = 0; i < N; i++) {
+    std::string p_v;
+    test::CompressibleString(&rnd, 0.5, 1007, &p_v);
+    ASSERT_OK(Put(Key(i), p_v));
+  }
+
+  ASSERT_OK(Flush());
+
+  std::vector<std::string> keys;
+  std::vector<std::string> values;
+
+  keys.push_back(Key(0));
+  keys.push_back(Key(4));
+  keys.push_back(Key(8));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 3u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 3u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 0u);
+  ASSERT_EQ(options.statistics->getTickerCount(BLOCK_CACHE_MISS), 3u);
+
+  keys.clear();
+  values.clear();
+  keys.push_back(Key(12));
+  keys.push_back(Key(16));
+  keys.push_back(Key(20));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 0u);
+  ASSERT_EQ(options.statistics->getTickerCount(BLOCK_CACHE_MISS), 6u);
+
+  keys.clear();
+  values.clear();
+  keys.push_back(Key(0));
+  keys.push_back(Key(4));
+  keys.push_back(Key(8));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 3u);
+  ASSERT_EQ(options.statistics->getTickerCount(BLOCK_CACHE_MISS), 6u);
+
+  ClearPrimaryCache();
+
+  keys.clear();
+  values.clear();
+  keys.push_back(Key(0));
+  keys.push_back(Key(32));
+  keys.push_back(Key(36));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 8u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 8u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 4u);
+  ASSERT_EQ(options.statistics->getTickerCount(BLOCK_CACHE_MISS), 8u);
+
+  keys.clear();
+  values.clear();
+  keys.push_back(Key(0));
+  keys.push_back(Key(32));
+  keys.push_back(Key(36));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 8u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 8u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 4u);
+  ASSERT_EQ(options.statistics->getTickerCount(BLOCK_CACHE_MISS), 8u);
+
+  Destroy(options);
+}
+
+// This test is for iteration. It iterates through a set of keys in two
+// passes. First pass loads the compressed blocks into the nvm tier, and
+// the second pass should hit all of those blocks.
+TEST_F(DBTieredSecondaryCacheTest, IterateTest) {
+  if (!LZ4_Supported()) {
+    ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
+    return;
+  }
+
+  BlockBasedTableOptions table_options;
+  table_options.block_cache = NewCache(250 * 1024, 10 * 1024, 256 * 1024);
+  table_options.block_size = 4 * 1024;
+  table_options.cache_index_and_filter_blocks = false;
+  Options options = GetDefaultOptions();
+  options.create_if_missing = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+
+  options.paranoid_file_checks = false;
+  DestroyAndReopen(options);
+  Random rnd(301);
+  const int N = 256;
+  for (int i = 0; i < N; i++) {
+    std::string p_v;
+    test::CompressibleString(&rnd, 0.5, 1007, &p_v);
+    ASSERT_OK(Put(Key(i), p_v));
+  }
+
+  ASSERT_OK(Flush());
+
+  ReadOptions ro;
+  ro.readahead_size = 256 * 1024;
+  auto iter = dbfull()->NewIterator(ro);
+  iter->SeekToFirst();
+  for (int i = 0; i < 31; ++i) {
+    ASSERT_EQ(Key(i), iter->key().ToString());
+    ASSERT_EQ(1007, iter->value().size());
+    iter->Next();
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 8u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 8u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 0u);
+  delete iter;
+
+  iter = dbfull()->NewIterator(ro);
+  iter->SeekToFirst();
+  for (int i = 0; i < 31; ++i) {
+    ASSERT_EQ(Key(i), iter->key().ToString());
+    ASSERT_EQ(1007, iter->value().size());
+    iter->Next();
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 8u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 8u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 8u);
+  delete iter;
+
+  Destroy(options);
+}
+
+class DBTieredAdmPolicyTest
+    : public DBTieredSecondaryCacheTest,
+      public testing::WithParamInterface<TieredAdmissionPolicy> {};
+
+TEST_P(DBTieredAdmPolicyTest, CompressedOnlyTest) {
+  if (!LZ4_Supported()) {
+    ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
+    return;
+  }
+
+  BlockBasedTableOptions table_options;
+  // We want a block cache of size 10KB, and a compressed secondary cache of
+  // size 10KB. However, we specify a block cache size of 256KB here in order
+  // to take into account the cache reservation in the block cache on
+  // behalf of the compressed cache. The unit of cache reservation is 256KB.
+  // The effective block cache capacity will be calculated as 256 + 10 = 266KB,
+  // and 256KB will be reserved for the compressed cache, leaving 10KB for
+  // the primary block cache. We only have to worry about this here because
+  // the cache size is so small.
+  table_options.block_cache = NewCache(256 * 1024, 10 * 1024, 0, GetParam());
+  table_options.block_size = 4 * 1024;
+  table_options.cache_index_and_filter_blocks = false;
+  Options options = GetDefaultOptions();
+  options.create_if_missing = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+
+  size_t comp_cache_usage = compressed_secondary_cache()->TEST_GetUsage();
+  // Disable paranoid_file_checks so that flush will not read back the newly
+  // written file
+  options.paranoid_file_checks = false;
+  DestroyAndReopen(options);
+  Random rnd(301);
+  const int N = 256;
+  for (int i = 0; i < N; i++) {
+    std::string p_v;
+    test::CompressibleString(&rnd, 0.5, 1007, &p_v);
+    ASSERT_OK(Put(Key(i), p_v));
+  }
+
+  ASSERT_OK(Flush());
+
+  // The first 2 Gets, for keys 0 and 5, will load the corresponding data
+  // blocks as they will be cache misses. Since this is a 2-tier cache (
+  // primary and compressed), no warm-up should happen with the compressed
+  // blocks.
+  std::string v = Get(Key(0));
+  ASSERT_EQ(1007, v.size());
+
+  v = Get(Key(5));
+  ASSERT_EQ(1007, v.size());
+
+  ASSERT_EQ(compressed_secondary_cache()->TEST_GetUsage(), comp_cache_usage);
+
+  Destroy(options);
+}
+
+INSTANTIATE_TEST_CASE_P(
+    DBTieredAdmPolicyTest, DBTieredAdmPolicyTest,
+    ::testing::Values(TieredAdmissionPolicy::kAdmPolicyAuto,
+                      TieredAdmissionPolicy::kAdmPolicyPlaceholder,
+                      TieredAdmissionPolicy::kAdmPolicyAllowCacheHits));
+
+}  // namespace ROCKSDB_NAMESPACE
+
+int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
@@ -234,15 +234,19 @@ class FullTypedCacheHelperFns : public BasicTypedCacheHelperFns<TValue> {
    return Status::OK();
  }

-  static Status Create(const Slice& data, CreateContext* context,
+  static Status Create(const Slice& data, CompressionType type,
+                       CacheTier source, CreateContext* context,
                       MemoryAllocator* allocator, ObjectPtr* out_obj,
                       size_t* out_charge) {
    std::unique_ptr<TValue> value = nullptr;
+    if (source != CacheTier::kVolatileTier) {
+      return Status::InvalidArgument();
+    }
    if constexpr (sizeof(TCreateContext) > 0) {
      TCreateContext* tcontext = static_cast<TCreateContext*>(context);
-      tcontext->Create(&value, out_charge, data, allocator);
+      tcontext->Create(&value, out_charge, data, type, allocator);
    } else {
-      TCreateContext::Create(&value, out_charge, data, allocator);
+      TCreateContext::Create(&value, out_charge, data, type, allocator);
    }
    *out_obj = UpCastValue(value.release());
    return Status::OK();
@@ -301,13 +305,15 @@ class FullTypedCacheInterface
  inline Status InsertFull(
      const Slice& key, TValuePtr value, size_t charge,
      TypedHandle** handle = nullptr, Priority priority = Priority::LOW,
-      CacheTier lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier) {
+      CacheTier lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier,
+      const Slice& compressed = Slice(),
+      CompressionType type = CompressionType::kNoCompression) {
    auto untyped_handle = reinterpret_cast<Handle**>(handle);
-    auto helper = lowest_used_cache_tier == CacheTier::kNonVolatileBlockTier
+    auto helper = lowest_used_cache_tier > CacheTier::kVolatileTier
                      ? GetFullHelper()
                      : GetBasicHelper();
    return this->cache_->Insert(key, UpCastValue(value), helper, charge,
-                                untyped_handle, priority);
+                                untyped_handle, priority, compressed, type);
  }

  // Like SecondaryCache::InsertSaved, with SecondaryCache compatibility
@@ -319,9 +325,9 @@ class FullTypedCacheInterface
      size_t* out_charge = nullptr) {
    ObjectPtr value;
    size_t charge;
-    Status st = GetFullHelper()->create_cb(data, create_context,
-                                           this->cache_->memory_allocator(),
-                                           &value, &charge);
+    Status st = GetFullHelper()->create_cb(
+        data, kNoCompression, CacheTier::kVolatileTier, create_context,
+        this->cache_->memory_allocator(), &value, &charge);
    if (out_charge) {
      *out_charge = charge;
    }
@@ -340,7 +346,7 @@ class FullTypedCacheInterface
      const Slice& key, TCreateContext* create_context = nullptr,
      Priority priority = Priority::LOW, Statistics* stats = nullptr,
      CacheTier lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier) {
-    if (lowest_used_cache_tier == CacheTier::kNonVolatileBlockTier) {
+    if (lowest_used_cache_tier > CacheTier::kVolatileTier) {
      return reinterpret_cast<TypedHandle*>(this->cache_->Lookup(
          key, GetFullHelper(), create_context, priority, stats));
    } else {
@@ -352,7 +358,7 @@ class FullTypedCacheInterface
  inline void StartAsyncLookupFull(
      TypedAsyncLookupHandle& async_handle,
      CacheTier lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier) {
-    if (lowest_used_cache_tier == CacheTier::kNonVolatileBlockTier) {
+    if (lowest_used_cache_tier > CacheTier::kVolatileTier) {
      async_handle.helper = GetFullHelper();
      this->cache_->StartAsyncLookup(async_handle);
    } else {
@@ -19,6 +19,14 @@

 namespace ROCKSDB_NAMESPACE {

+inline static SequenceNumber GetSeqNum(const DBImpl* db, const Snapshot* s) {
+  if (s) {
+    return s->GetSequenceNumber();
+  } else {
+    return db->GetLatestSequenceNumber();
+  }
+}
+
 Status ArenaWrappedDBIter::GetProperty(std::string prop_name,
                                       std::string* prop) {
  if (prop_name == "rocksdb.iterator.super-version-number") {
@@ -54,7 +62,9 @@ void ArenaWrappedDBIter::Init(
  }
 }

-Status ArenaWrappedDBIter::Refresh() {
+Status ArenaWrappedDBIter::Refresh() { return Refresh(nullptr); }
+
+Status ArenaWrappedDBIter::Refresh(const Snapshot* snapshot) {
  if (cfd_ == nullptr || db_impl_ == nullptr || !allow_refresh_) {
    return Status::NotSupported("Creating renew iterator is not allowed.");
  }
@@ -63,6 +73,10 @@ Status ArenaWrappedDBIter::Refresh() {
  // correct behavior. Will be corrected automatically when we take a snapshot
  // here for the case of WritePreparedTxnDB.
  uint64_t cur_sv_number = cfd_->GetSuperVersionNumber();
+  // If we recreate a new internal iterator below (NewInternalIterator()),
+  // we will pass in read_options_. We need to make sure it
+  // has the right snapshot.
+  read_options_.snapshot = snapshot;
  TEST_SYNC_POINT("ArenaWrappedDBIter::Refresh:1");
  TEST_SYNC_POINT("ArenaWrappedDBIter::Refresh:2");
  auto reinit_internal_iter = [&]() {
@@ -72,18 +86,19 @@ Status ArenaWrappedDBIter::Refresh() {
    new (&arena_) Arena();

    SuperVersion* sv = cfd_->GetReferencedSuperVersion(db_impl_);
-    SequenceNumber latest_seq = db_impl_->GetLatestSequenceNumber();
+    assert(sv->version_number >= cur_sv_number);
+    SequenceNumber read_seq = GetSeqNum(db_impl_, snapshot);
    if (read_callback_) {
-      read_callback_->Refresh(latest_seq);
+      read_callback_->Refresh(read_seq);
    }
    Init(env, read_options_, *(cfd_->ioptions()), sv->mutable_cf_options,
-         sv->current, latest_seq,
+         sv->current, read_seq,
         sv->mutable_cf_options.max_sequential_skip_in_iterations,
-         cur_sv_number, read_callback_, db_impl_, cfd_, expose_blob_index_,
+         sv->version_number, read_callback_, db_impl_, cfd_, expose_blob_index_,
         allow_refresh_);

    InternalIterator* internal_iter = db_impl_->NewInternalIterator(
-        read_options_, cfd_, sv, &arena_, latest_seq,
+        read_options_, cfd_, sv, &arena_, read_seq,
        /* allow_unprepared_value */ true, /* db_iter */ this);
    SetIterUnderDBIter(internal_iter);
  };
@@ -92,13 +107,13 @@ Status ArenaWrappedDBIter::Refresh() {
      reinit_internal_iter();
      break;
    } else {
-      SequenceNumber latest_seq = db_impl_->GetLatestSequenceNumber();
+      SequenceNumber read_seq = GetSeqNum(db_impl_, snapshot);
      // Refresh range-tombstones in MemTable
      if (!read_options_.ignore_range_deletions) {
        SuperVersion* sv = cfd_->GetThreadLocalSuperVersion(db_impl_);
        TEST_SYNC_POINT_CALLBACK("ArenaWrappedDBIter::Refresh:SV", nullptr);
        auto t = sv->mem->NewRangeTombstoneIterator(
-            read_options_, latest_seq, false /* immutable_memtable */);
+            read_options_, read_seq, false /* immutable_memtable */);
        if (!t || t->empty()) {
          // If memtable_range_tombstone_iter_ points to a non-empty tombstone
          // iterator, then it means sv->mem is not the memtable that
@@ -128,9 +143,6 @@ Status ArenaWrappedDBIter::Refresh() {
        }
        db_impl_->ReturnAndCleanupSuperVersion(cfd_, sv);
      }
-      // Refresh latest sequence number
-      db_iter_->set_sequence(latest_seq);
-      db_iter_->set_valid(false);
      // Check again if the latest super version number is changed
      uint64_t latest_sv_number = cfd_->GetSuperVersionNumber();
      if (latest_sv_number != cur_sv_number) {
@@ -139,6 +151,8 @@ Status ArenaWrappedDBIter::Refresh() {
        cur_sv_number = latest_sv_number;
        continue;
      }
+      db_iter_->set_sequence(read_seq);
+      db_iter_->set_valid(false);
      break;
    }
  }
@@ -80,6 +80,7 @@ class ArenaWrappedDBIter : public Iterator {
  Status GetProperty(std::string prop_name, std::string* prop) override;

  Status Refresh() override;
+  Status Refresh(const Snapshot*) override;

  void Init(Env* env, const ReadOptions& read_options,
            const ImmutableOptions& ioptions,
@@ -46,7 +46,8 @@ class BlobContents {
 class BlobContentsCreator : public Cache::CreateContext {
 public:
  static void Create(std::unique_ptr<BlobContents>* out, size_t* out_charge,
-                     const Slice& contents, MemoryAllocator* alloc) {
+                     const Slice& contents, CompressionType /*type*/,
+                     MemoryAllocator* alloc) {
    auto raw = new BlobContents(AllocateAndCopyBlock(contents, alloc),
                                contents.size());
    out->reset(raw);
@@ -585,7 +585,8 @@ Status BlobFileReader::UncompressBlobIfNeeded(
  assert(result);

  if (compression_type == kNoCompression) {
-    BlobContentsCreator::Create(result, nullptr, value_slice, allocator);
+    BlobContentsCreator::Create(result, nullptr, value_slice, kNoCompression,
+                                allocator);
    return Status::OK();
  }

@@ -1220,7 +1220,7 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) {
      auto sec_handle0 = secondary_cache->Lookup(
          key0, BlobSource::SharedCacheInterface::GetFullHelper(),
          /*context*/ nullptr, true,
-          /*advise_erase=*/true, kept_in_sec_cache);
+          /*advise_erase=*/true, /*stats=*/nullptr, kept_in_sec_cache);
      ASSERT_FALSE(kept_in_sec_cache);
      ASSERT_NE(sec_handle0, nullptr);
      ASSERT_TRUE(sec_handle0->IsReady());
@@ -1248,7 +1248,7 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) {
      auto sec_handle1 = secondary_cache->Lookup(
          key1, BlobSource::SharedCacheInterface::GetFullHelper(),
          /*context*/ nullptr, true,
-          /*advise_erase=*/true, kept_in_sec_cache);
+          /*advise_erase=*/true, /*stats=*/nullptr, kept_in_sec_cache);
      ASSERT_FALSE(kept_in_sec_cache);
      ASSERT_EQ(sec_handle1, nullptr);

@@ -168,6 +168,7 @@ TEST_F(DBBlobBasicTest, IterateBlobsFromCache) {
      ASSERT_EQ(iter->value().ToString(), blobs[i]);
      ++i;
    }
+    ASSERT_OK(iter->status());
    ASSERT_EQ(i, num_blobs);
    ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD), 0);
  }
@@ -203,6 +204,7 @@ TEST_F(DBBlobBasicTest, IterateBlobsFromCache) {
      ASSERT_EQ(iter->value().ToString(), blobs[i]);
      ++i;
    }
+    ASSERT_OK(iter->status());
    ASSERT_EQ(i, num_blobs);
    ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD),
              num_blobs);
@@ -224,6 +226,7 @@ TEST_F(DBBlobBasicTest, IterateBlobsFromCache) {
      ASSERT_EQ(iter->value().ToString(), blobs[i]);
      ++i;
    }
+    ASSERT_OK(iter->status());
    ASSERT_EQ(i, num_blobs);
    ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD), 0);
  }
@@ -2123,6 +2126,7 @@ TEST_F(DBBlobWithTimestampTest, IterateBlobs) {
                       /*key_is_internal*/ false);
      iter->Prev();
    }
+    ASSERT_OK(iter->status());
  }

  // Backward iteration, then reverse to forward.
@@ -2169,6 +2173,7 @@ TEST_F(DBBlobWithTimestampTest, IterateBlobs) {
        iter->Next();
      }
    }
+    ASSERT_OK(iter->status());
  }

  // Backward iterating multiple versions of the same key, get in this order:
@@ -2187,6 +2192,7 @@ TEST_F(DBBlobWithTimestampTest, IterateBlobs) {
        iter->Prev();
      }
    }
+    ASSERT_OK(iter->status());
  }

  int upper_bound_idx = num_blobs - 2;
@@ -2209,6 +2215,7 @@ TEST_F(DBBlobWithTimestampTest, IterateBlobs) {
        iter->Next();
      }
    }
+    ASSERT_OK(iter->status());
  }

  // Backward iteration with upper and lower bound.
@@ -2224,6 +2231,7 @@ TEST_F(DBBlobWithTimestampTest, IterateBlobs) {
        iter->Prev();
      }
    }
+    ASSERT_OK(iter->status());
  }
 }

@@ -96,9 +96,13 @@ class DBBlobIndexTest : public DBTestBase {
  }

  ArenaWrappedDBIter* GetBlobIterator() {
-    return dbfull()->NewIteratorImpl(
-        ReadOptions(), cfd(), dbfull()->GetLatestSequenceNumber(),
-        nullptr /*read_callback*/, true /*expose_blob_index*/);
+    ColumnFamilyData* column_family = cfd();
+    DBImpl* db_impl = dbfull();
+    return db_impl->NewIteratorImpl(
+        ReadOptions(), column_family,
+        column_family->GetReferencedSuperVersion(db_impl),
+        db_impl->GetLatestSequenceNumber(), nullptr /*read_callback*/,
+        true /*expose_blob_index*/);
  }

  Options GetTestOptions() {
@@ -294,12 +294,12 @@ Status BuildTable(
    if (!s.ok() || empty) {
      builder->Abandon();
    } else {
-      std::string seqno_time_mapping_str;
+      std::string seqno_to_time_mapping_str;
      seqno_to_time_mapping.Encode(
-          seqno_time_mapping_str, meta->fd.smallest_seqno,
+          seqno_to_time_mapping_str, meta->fd.smallest_seqno,
          meta->fd.largest_seqno, meta->file_creation_time);
      builder->SetSeqnoTimeTableProperties(
-          seqno_time_mapping_str,
+          seqno_to_time_mapping_str,
          ioptions.compaction_style == CompactionStyle::kCompactionStyleFIFO
              ? meta->file_creation_time
              : meta->oldest_ancester_time);
@@ -45,6 +45,7 @@
 #include "rocksdb/utilities/transaction_db.h"
 #include "rocksdb/utilities/write_batch_with_index.h"
 #include "rocksdb/write_batch.h"
+#include "rocksdb/write_buffer_manager.h"
 #include "utilities/merge_operators.h"

 using ROCKSDB_NAMESPACE::BackupEngine;
@@ -120,10 +121,12 @@ using ROCKSDB_NAMESPACE::TransactionDB;
 using ROCKSDB_NAMESPACE::TransactionDBOptions;
 using ROCKSDB_NAMESPACE::TransactionLogIterator;
 using ROCKSDB_NAMESPACE::TransactionOptions;
+using ROCKSDB_NAMESPACE::WaitForCompactOptions;
 using ROCKSDB_NAMESPACE::WALRecoveryMode;
 using ROCKSDB_NAMESPACE::WritableFile;
 using ROCKSDB_NAMESPACE::WriteBatch;
 using ROCKSDB_NAMESPACE::WriteBatchWithIndex;
+using ROCKSDB_NAMESPACE::WriteBufferManager;
 using ROCKSDB_NAMESPACE::WriteOptions;

 using std::unordered_set;
@@ -218,6 +221,9 @@ struct rocksdb_memory_allocator_t {
 struct rocksdb_cache_t {
  std::shared_ptr<Cache> rep;
 };
+struct rocksdb_write_buffer_manager_t {
+  std::shared_ptr<WriteBufferManager> rep;
+};
 struct rocksdb_livefiles_t {
  std::vector<LiveFileMetaData> rep;
 };
@@ -275,6 +281,9 @@ struct rocksdb_optimistictransactiondb_t {
 struct rocksdb_optimistictransaction_options_t {
  OptimisticTransactionOptions rep;
 };
+struct rocksdb_wait_for_compact_options_t {
+  WaitForCompactOptions rep;
+};

 struct rocksdb_compactionfiltercontext_t {
  CompactionFilter::Context rep;
@@ -2900,6 +2909,16 @@ void rocksdb_options_set_db_paths(rocksdb_options_t* opt,
  opt->rep.db_paths = db_paths;
 }

+void rocksdb_options_set_cf_paths(rocksdb_options_t* opt,
+                                  const rocksdb_dbpath_t** dbpath_values,
+                                  size_t num_paths) {
+  std::vector<DbPath> cf_paths(num_paths);
+  for (size_t i = 0; i < num_paths; ++i) {
+    cf_paths[i] = dbpath_values[i]->rep;
+  }
+  opt->rep.cf_paths = cf_paths;
+}
+
 void rocksdb_options_set_env(rocksdb_options_t* opt, rocksdb_env_t* env) {
  opt->rep.env = (env ? env->rep : nullptr);
 }
@@ -2931,6 +2950,11 @@ void rocksdb_options_set_write_buffer_size(rocksdb_options_t* opt, size_t s) {
  opt->rep.write_buffer_size = s;
 }

+void rocksdb_options_set_write_buffer_manager(
+    rocksdb_options_t* opt, rocksdb_write_buffer_manager_t* wbm) {
+  opt->rep.write_buffer_manager = wbm->rep;
+}
+
 size_t rocksdb_options_get_write_buffer_size(rocksdb_options_t* opt) {
  return opt->rep.write_buffer_size;
 }
@@ -3025,6 +3049,16 @@ void rocksdb_options_set_max_bytes_for_level_multiplier_additional(
  }
 }

+void rocksdb_options_set_periodic_compaction_seconds(rocksdb_options_t* opt,
+                                                     uint64_t seconds) {
+  opt->rep.periodic_compaction_seconds = seconds;
+}
+
+uint64_t rocksdb_options_get_periodic_compaction_seconds(
+    rocksdb_options_t* opt) {
+  return opt->rep.periodic_compaction_seconds;
+}
+
 void rocksdb_options_enable_statistics(rocksdb_options_t* opt) {
  opt->rep.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
 }
@@ -3953,6 +3987,16 @@ rocksdb_ratelimiter_t* rocksdb_ratelimiter_create(int64_t rate_bytes_per_sec,
  return rate_limiter;
 }

+rocksdb_ratelimiter_t* rocksdb_ratelimiter_create_auto_tuned(
+    int64_t rate_bytes_per_sec, int64_t refill_period_us, int32_t fairness) {
+  rocksdb_ratelimiter_t* rate_limiter = new rocksdb_ratelimiter_t;
+  rate_limiter->rep.reset(NewGenericRateLimiter(rate_bytes_per_sec,
+                                                refill_period_us, fairness,
+                                                RateLimiter::Mode::kWritesOnly,
+                                                true));  // auto_tuned
+  return rate_limiter;
+}
+
 void rocksdb_ratelimiter_destroy(rocksdb_ratelimiter_t* limiter) {
  delete limiter;
 }
@@ -4585,6 +4629,11 @@ void rocksdb_readoptions_set_iter_start_ts(rocksdb_readoptions_t* opt,
  }
 }

+void rocksdb_readoptions_set_auto_readahead_size(rocksdb_readoptions_t* opt,
+                                                 unsigned char v) {
+  opt->rep.auto_readahead_size = v;
+}
+
 rocksdb_writeoptions_t* rocksdb_writeoptions_create() {
  return new rocksdb_writeoptions_t;
 }
@@ -4857,6 +4906,60 @@ size_t rocksdb_cache_get_occupancy_count(const rocksdb_cache_t* cache) {
  return cache->rep->GetOccupancyCount();
 }

+rocksdb_write_buffer_manager_t* rocksdb_write_buffer_manager_create(
+    size_t buffer_size, bool allow_stall) {
+  rocksdb_write_buffer_manager_t* wbm = new rocksdb_write_buffer_manager_t;
+  wbm->rep.reset(new WriteBufferManager(buffer_size, {}, allow_stall));
+  return wbm;
+}
+
+rocksdb_write_buffer_manager_t* rocksdb_write_buffer_manager_create_with_cache(
+    size_t buffer_size, const rocksdb_cache_t* cache, bool allow_stall) {
+  rocksdb_write_buffer_manager_t* wbm = new rocksdb_write_buffer_manager_t;
+  wbm->rep.reset(new WriteBufferManager(buffer_size, cache->rep, allow_stall));
+  return wbm;
+}
+
+void rocksdb_write_buffer_manager_destroy(rocksdb_write_buffer_manager_t* wbm) {
+  delete wbm;
+}
+
+bool rocksdb_write_buffer_manager_enabled(rocksdb_write_buffer_manager_t* wbm) {
+  return wbm->rep->enabled();
+}
+
+bool rocksdb_write_buffer_manager_cost_to_cache(
+    rocksdb_write_buffer_manager_t* wbm) {
+  return wbm->rep->cost_to_cache();
+}
+
+size_t rocksdb_write_buffer_manager_memory_usage(
+    rocksdb_write_buffer_manager_t* wbm) {
+  return wbm->rep->memory_usage();
+}
+
+size_t rocksdb_write_buffer_manager_mutable_memtable_memory_usage(
+    rocksdb_write_buffer_manager_t* wbm) {
+  return wbm->rep->mutable_memtable_memory_usage();
+}
+
+size_t rocksdb_write_buffer_manager_dummy_entries_in_cache_usage(
+    rocksdb_write_buffer_manager_t* wbm) {
+  return wbm->rep->dummy_entries_in_cache_usage();
+}
+size_t rocksdb_write_buffer_manager_buffer_size(
+    rocksdb_write_buffer_manager_t* wbm) {
+  return wbm->rep->buffer_size();
+}
+void rocksdb_write_buffer_manager_set_buffer_size(
+    rocksdb_write_buffer_manager_t* wbm, size_t new_size) {
+  wbm->rep->SetBufferSize(new_size);
+}
+ROCKSDB_LIBRARY_API void rocksdb_write_buffer_manager_set_allow_stall(
+    rocksdb_write_buffer_manager_t* wbm, bool new_allow_stall) {
+  wbm->rep->SetAllowStall(new_allow_stall);
+}
+
 rocksdb_dbpath_t* rocksdb_dbpath_create(const char* path,
                                        uint64_t target_size) {
  rocksdb_dbpath_t* result = new rocksdb_dbpath_t;
@@ -5455,6 +5558,11 @@ char* rocksdb_sst_file_metadata_get_relative_filename(
  return strdup(file_meta->rep->relative_filename.c_str());
 }

+char* rocksdb_sst_file_metadata_get_directory(
+    rocksdb_sst_file_metadata_t* file_meta) {
+  return strdup(file_meta->rep->directory.c_str());
+}
+
 uint64_t rocksdb_sst_file_metadata_get_size(
    rocksdb_sst_file_metadata_t* file_meta) {
  return file_meta->rep->size;
@@ -6719,4 +6827,59 @@ double rocksdb_statistics_histogram_data_get_min(
  return data->rep.min;
 }

+void rocksdb_wait_for_compact(rocksdb_t* db,
+                              rocksdb_wait_for_compact_options_t* options,
+                              char** errptr) {
+  SaveError(errptr, db->rep->WaitForCompact(options->rep));
+}
+
+rocksdb_wait_for_compact_options_t* rocksdb_wait_for_compact_options_create() {
+  return new rocksdb_wait_for_compact_options_t;
+}
+
+void rocksdb_wait_for_compact_options_destroy(
+    rocksdb_wait_for_compact_options_t* opt) {
+  delete opt;
+}
+
+void rocksdb_wait_for_compact_options_set_abort_on_pause(
+    rocksdb_wait_for_compact_options_t* opt, unsigned char v) {
+  opt->rep.abort_on_pause = v;
+}
+
+unsigned char rocksdb_wait_for_compact_options_get_abort_on_pause(
+    rocksdb_wait_for_compact_options_t* opt) {
+  return opt->rep.abort_on_pause;
+}
+
+void rocksdb_wait_for_compact_options_set_flush(
+    rocksdb_wait_for_compact_options_t* opt, unsigned char v) {
+  opt->rep.flush = v;
+}
+
+unsigned char rocksdb_wait_for_compact_options_get_flush(
+    rocksdb_wait_for_compact_options_t* opt) {
+  return opt->rep.flush;
+}
+
+void rocksdb_wait_for_compact_options_set_close_db(
+    rocksdb_wait_for_compact_options_t* opt, unsigned char v) {
+  opt->rep.close_db = v;
+}
+
+unsigned char rocksdb_wait_for_compact_options_get_close_db(
+    rocksdb_wait_for_compact_options_t* opt) {
+  return opt->rep.close_db;
+}
+
+void rocksdb_wait_for_compact_options_set_timeout(
+    rocksdb_wait_for_compact_options_t* opt, uint64_t microseconds) {
+  opt->rep.timeout = std::chrono::microseconds(microseconds);
+}
+
+uint64_t rocksdb_wait_for_compact_options_get_timeout(
+    rocksdb_wait_for_compact_options_t* opt) {
+  return opt->rep.timeout.count();
+}
+
 }  // end extern "C"
@@ -375,6 +375,11 @@ static rocksdb_t* CheckCompaction(rocksdb_t* db, rocksdb_options_t* options,

  // Force compaction
  rocksdb_compact_range(db, NULL, 0, NULL, 0);
+  rocksdb_wait_for_compact_options_t* wco;
+  wco = rocksdb_wait_for_compact_options_create();
+  rocksdb_wait_for_compact(db, wco, &err);
+  CheckNoError(err);
+  rocksdb_wait_for_compact_options_destroy(wco);
  // should have filtered bar, but not foo
  CheckGet(db, roptions, "foo", "foovalue");
  CheckGet(db, roptions, "bar", NULL);
@@ -708,6 +713,11 @@ int main(int argc, char** argv) {
  rocksdb_options_set_ratelimiter(options, rate_limiter);
  rocksdb_ratelimiter_destroy(rate_limiter);

+  rate_limiter =
+      rocksdb_ratelimiter_create_auto_tuned(1000 * 1024 * 1024, 100 * 1000, 10);
+  rocksdb_options_set_ratelimiter(options, rate_limiter);
+  rocksdb_ratelimiter_destroy(rate_limiter);
+
  roptions = rocksdb_readoptions_create();
  rocksdb_readoptions_set_verify_checksums(roptions, 1);
  rocksdb_readoptions_set_fill_cache(roptions, 1);
@@ -1469,10 +1479,20 @@ int main(int argc, char** argv) {
    CheckCondition(cflen == 2);
    rocksdb_list_column_families_destroy(column_fams, cflen);

-    rocksdb_options_t* cf_options = rocksdb_options_create();
+    rocksdb_options_t* cf_options_1 = rocksdb_options_create();
+    rocksdb_options_t* cf_options_2 = rocksdb_options_create();
+
+    // use dbpathname2 as the cf_path for "cf1"
+    rocksdb_dbpath_t* dbpath2;
+    char dbpathname2[200];
+    snprintf(dbpathname2, sizeof(dbpathname2), "%s/rocksdb_c_test-%d-dbpath2",
+             GetTempDir(), ((int)geteuid()));
+    dbpath2 = rocksdb_dbpath_create(dbpathname2, 1024 * 1024);
+    const rocksdb_dbpath_t* cf_paths[1] = {dbpath2};
+    rocksdb_options_set_cf_paths(cf_options_2, cf_paths, 1);

    const char* cf_names[2] = {"default", "cf1"};
-    const rocksdb_options_t* cf_opts[2] = {cf_options, cf_options};
+    const rocksdb_options_t* cf_opts[2] = {cf_options_1, cf_options_2};
    rocksdb_column_family_handle_t* handles[2];

    LoadAndCheckLatestOptions(dbname, env, false, cache, NULL, 2, cf_names,
@@ -1500,6 +1520,37 @@ int main(int argc, char** argv) {
    rocksdb_flushoptions_t* flush_options = rocksdb_flushoptions_create();
    rocksdb_flushoptions_set_wait(flush_options, 1);
    rocksdb_flush_cf(db, flush_options, handles[1], &err);
+
+    // make sure all files in "cf1" are under the specified cf path
+    {
+      rocksdb_column_family_metadata_t* cf_meta =
+          rocksdb_get_column_family_metadata_cf(db, handles[1]);
+      size_t cf_file_count = rocksdb_column_family_metadata_get_size(cf_meta);
+      assert(cf_file_count > 0);
+      size_t level_count =
+          rocksdb_column_family_metadata_get_level_count(cf_meta);
+      assert(level_count > 0);
+      for (size_t l = 0; l < level_count; ++l) {
+        rocksdb_level_metadata_t* level_meta =
+            rocksdb_column_family_metadata_get_level_metadata(cf_meta, l);
+        assert(level_meta);
+
+        size_t file_count = rocksdb_level_metadata_get_file_count(level_meta);
+        for (size_t f = 0; f < file_count; ++f) {
+          rocksdb_sst_file_metadata_t* file_meta =
+              rocksdb_level_metadata_get_sst_file_metadata(level_meta, f);
+          assert(file_meta);
+          char* file_path = rocksdb_sst_file_metadata_get_directory(file_meta);
+          assert(strcmp(file_path, dbpathname2) == 0);
+          Free(&file_path);
+          rocksdb_sst_file_metadata_destroy(file_meta);
+        }
+        rocksdb_level_metadata_destroy(level_meta);
+      }
+
+      rocksdb_column_family_metadata_destroy(cf_meta);
+    }
+
    CheckNoError(err) rocksdb_flushoptions_destroy(flush_options);

    CheckGetCF(db, roptions, handles[1], "foo", "hello");
@@ -1663,7 +1714,9 @@ int main(int argc, char** argv) {
    }
    rocksdb_destroy_db(options, dbname, &err);
    rocksdb_options_destroy(db_options);
-    rocksdb_options_destroy(cf_options);
+    rocksdb_options_destroy(cf_options_1);
+    rocksdb_options_destroy(cf_options_2);
+    rocksdb_dbpath_destroy(dbpath2);
  }

  StartPhase("prefix");
@@ -1868,6 +1921,10 @@ int main(int argc, char** argv) {
    CheckCondition(2.0 ==
                   rocksdb_options_get_max_bytes_for_level_multiplier(o));

+    rocksdb_options_set_periodic_compaction_seconds(o, 100000);
+    CheckCondition(100000 ==
+                   rocksdb_options_get_periodic_compaction_seconds(o));
+
    rocksdb_options_set_skip_stats_update_on_db_open(o, 1);
    CheckCondition(1 == rocksdb_options_get_skip_stats_update_on_db_open(o));

@@ -2298,6 +2355,12 @@ int main(int argc, char** argv) {
    CheckCondition(2.0 ==
                   rocksdb_options_get_max_bytes_for_level_multiplier(o));

+    rocksdb_options_set_periodic_compaction_seconds(copy, 8000);
+    CheckCondition(8000 ==
+                   rocksdb_options_get_periodic_compaction_seconds(copy));
+    CheckCondition(100000 ==
+                   rocksdb_options_get_periodic_compaction_seconds(o));
+
    rocksdb_options_set_skip_stats_update_on_db_open(copy, 0);
    CheckCondition(0 == rocksdb_options_get_skip_stats_update_on_db_open(copy));
    CheckCondition(1 == rocksdb_options_get_skip_stats_update_on_db_open(o));
@@ -3687,6 +3750,61 @@ int main(int argc, char** argv) {
    rocksdb_statistics_histogram_data_destroy(hist);
  }

+  StartPhase("wait_for_compact_options");
+  {
+    rocksdb_wait_for_compact_options_t* wco;
+    wco = rocksdb_wait_for_compact_options_create();
+
+    rocksdb_wait_for_compact_options_set_abort_on_pause(wco, 1);
+    CheckCondition(1 ==
+                   rocksdb_wait_for_compact_options_get_abort_on_pause(wco));
+
+    rocksdb_wait_for_compact_options_set_flush(wco, 1);
+    CheckCondition(1 == rocksdb_wait_for_compact_options_get_flush(wco));
+
+    rocksdb_wait_for_compact_options_set_close_db(wco, 1);
+    CheckCondition(1 == rocksdb_wait_for_compact_options_get_close_db(wco));
+
+    rocksdb_wait_for_compact_options_set_timeout(wco, 342);
+    CheckCondition(342 == rocksdb_wait_for_compact_options_get_timeout(wco));
+
+    rocksdb_wait_for_compact_options_destroy(wco);
+  }
+  StartPhase("wait_for_compact");
+  {
+    rocksdb_wait_for_compact_options_t* wco;
+    wco = rocksdb_wait_for_compact_options_create();
+    rocksdb_wait_for_compact_options_set_flush(wco, 1);
+
+    rocksdb_wait_for_compact(db, wco, &err);
+    CheckNoError(err);
+    rocksdb_wait_for_compact_options_destroy(wco);
+  }
+
+  StartPhase("write_buffer_manager");
+  {
+    rocksdb_cache_t* lru;
+    lru = rocksdb_cache_create_lru(100);
+
+    rocksdb_write_buffer_manager_t* write_buffer_manager;
+    write_buffer_manager =
+        rocksdb_write_buffer_manager_create_with_cache(200, lru, false);
+
+    CheckCondition(true ==
+                   rocksdb_write_buffer_manager_enabled(write_buffer_manager));
+    CheckCondition(true == rocksdb_write_buffer_manager_cost_to_cache(
+                               write_buffer_manager));
+    CheckCondition(
+        200 == rocksdb_write_buffer_manager_buffer_size(write_buffer_manager));
+
+    rocksdb_write_buffer_manager_set_buffer_size(write_buffer_manager, 300);
+    CheckCondition(
+        300 == rocksdb_write_buffer_manager_buffer_size(write_buffer_manager));
+
+    rocksdb_write_buffer_manager_destroy(write_buffer_manager);
+    rocksdb_cache_destroy(lru);
+  }
+
  StartPhase("cancel_all_background_work");
  rocksdb_cancel_all_background_work(db, 1);

@@ -476,6 +476,7 @@ void SuperVersion::Init(ColumnFamilyData* new_cfd, MemTable* new_mem,
  mem = new_mem;
  imm = new_imm;
  current = new_current;
+  full_history_ts_low = cfd->GetFullHistoryTsLow();
  cfd->Ref();
  mem->Ref();
  imm->Ref();
@@ -832,8 +833,8 @@ std::unique_ptr<WriteControllerToken> SetupDelay(
  return write_controller->GetDelayToken(write_rate);
 }

-int GetL0ThresholdSpeedupCompaction(int level0_file_num_compaction_trigger,
-                                    int level0_slowdown_writes_trigger) {
+int GetL0FileCountForCompactionSpeedup(int level0_file_num_compaction_trigger,
+                                       int level0_slowdown_writes_trigger) {
  // SanitizeOptions() ensures it.
  assert(level0_file_num_compaction_trigger <= level0_slowdown_writes_trigger);

@@ -863,6 +864,36 @@ int GetL0ThresholdSpeedupCompaction(int level0_file_num_compaction_trigger,
    return static_cast<int>(res);
  }
 }
+
+uint64_t GetPendingCompactionBytesForCompactionSpeedup(
+    const MutableCFOptions& mutable_cf_options,
+    const VersionStorageInfo* vstorage) {
+  // Compaction debt relatively large compared to the stable (bottommost) data
+  // size indicates compaction fell behind.
+  const uint64_t kBottommostSizeMultiplier = 8;
+  // Meaningful progress toward the slowdown trigger is another good indication.
+  const uint64_t kSlowdownTriggerDivisor = 4;
+
+  uint64_t bottommost_files_size = 0;
+  for (const auto& level_and_file : vstorage->BottommostFiles()) {
+    bottommost_files_size += level_and_file.second->fd.GetFileSize();
+  }
+
+  // Slowdown trigger might be zero but that means compaction speedup should
+  // always happen (undocumented/historical), so no special treatment is needed.
+  uint64_t slowdown_threshold =
+      mutable_cf_options.soft_pending_compaction_bytes_limit /
+      kSlowdownTriggerDivisor;
+
+  // Size of zero, however, should not be used to decide to speedup compaction.
+  if (bottommost_files_size == 0) {
+    return slowdown_threshold;
+  }
+
+  uint64_t size_threshold =
+      MultiplyCheckOverflow(bottommost_files_size, kBottommostSizeMultiplier);
+  return std::min(size_threshold, slowdown_threshold);
+}
 }  // anonymous namespace

 std::pair<WriteStallCondition, WriteStallCause>
@@ -1018,7 +1049,7 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
    } else {
      assert(write_stall_condition == WriteStallCondition::kNormal);
      if (vstorage->l0_delay_trigger_count() >=
-          GetL0ThresholdSpeedupCompaction(
+          GetL0FileCountForCompactionSpeedup(
              mutable_cf_options.level0_file_num_compaction_trigger,
              mutable_cf_options.level0_slowdown_writes_trigger)) {
        write_controller_token_ =
@@ -1028,22 +1059,22 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
            "[%s] Increasing compaction threads because we have %d level-0 "
            "files ",
            name_.c_str(), vstorage->l0_delay_trigger_count());
-      } else if (vstorage->estimated_compaction_needed_bytes() >=
-                 mutable_cf_options.soft_pending_compaction_bytes_limit / 4) {
-        // Increase compaction threads if bytes needed for compaction exceeds
-        // 1/4 of threshold for slowing down.
+      } else if (mutable_cf_options.soft_pending_compaction_bytes_limit == 0) {
        // If soft pending compaction byte limit is not set, always speed up
        // compaction.
        write_controller_token_ =
            write_controller->GetCompactionPressureToken();
-        if (mutable_cf_options.soft_pending_compaction_bytes_limit > 0) {
-          ROCKS_LOG_INFO(
-              ioptions_.logger,
-              "[%s] Increasing compaction threads because of estimated pending "
-              "compaction "
-              "bytes %" PRIu64,
-              name_.c_str(), vstorage->estimated_compaction_needed_bytes());
-        }
+      } else if (vstorage->estimated_compaction_needed_bytes() >=
+                 GetPendingCompactionBytesForCompactionSpeedup(
+                     mutable_cf_options, vstorage)) {
+        write_controller_token_ =
+            write_controller->GetCompactionPressureToken();
+        ROCKS_LOG_INFO(
+            ioptions_.logger,
+            "[%s] Increasing compaction threads because of estimated pending "
+            "compaction "
+            "bytes %" PRIu64,
+            name_.c_str(), vstorage->estimated_compaction_needed_bytes());
      } else {
        write_controller_token_.reset();
      }
@@ -1118,7 +1149,7 @@ Compaction* ColumnFamilyData::PickCompaction(
      GetName(), mutable_options, mutable_db_options, current_->storage_info(),
      log_buffer);
  if (result != nullptr) {
-    result->SetInputVersion(current_);
+    result->FinalizeInputInfo(current_);
  }
  return result;
 }
@@ -1202,7 +1233,7 @@ Compaction* ColumnFamilyData::CompactRange(
      compact_range_options, begin, end, compaction_end, conflict,
      max_file_num_to_ignore, trim_ts);
  if (result != nullptr) {
-    result->SetInputVersion(current_);
+    result->FinalizeInputInfo(current_);
  }
  TEST_SYNC_POINT("ColumnFamilyData::CompactRange:Return");
  return result;
@@ -1283,8 +1314,6 @@ void ColumnFamilyData::InstallSuperVersion(
  new_superversion->Init(this, mem_, imm_.current(), current_);
  SuperVersion* old_superversion = super_version_;
  super_version_ = new_superversion;
-  ++super_version_number_;
-  super_version_->version_number = super_version_number_;
  if (old_superversion == nullptr || old_superversion->current != current() ||
      old_superversion->mem != mem_ ||
      old_superversion->imm != imm_.current()) {
@@ -1319,6 +1348,8 @@ void ColumnFamilyData::InstallSuperVersion(
      sv_context->superversions_to_free.push_back(old_superversion);
    }
  }
+  ++super_version_number_;
+  super_version_->version_number = super_version_number_;
 }

 void ColumnFamilyData::ResetThreadLocalSuperVersions() {
@@ -1557,24 +1588,15 @@ bool ColumnFamilyData::ShouldPostponeFlushToRetainUDT(
  if (full_history_ts_low.empty()) {
    return false;
  }
-#ifndef NDEBUG
-  Slice last_table_newest_udt;
-#endif /* !NDEBUG */
  for (const Slice& table_newest_udt :
       imm()->GetTablesNewestUDT(max_memtable_id)) {
    assert(table_newest_udt.size() == full_history_ts_low.size());
-    assert(last_table_newest_udt.empty() ||
-           ucmp->CompareTimestamp(table_newest_udt, last_table_newest_udt) >=
-               0);
    // Checking the newest UDT contained in MemTable with ascending ID up to
-    // `max_memtable_id`. MemTable with bigger ID will have newer UDT, return
-    // immediately on finding the first MemTable that needs postponing.
+    // `max_memtable_id`. Return immediately on finding the first MemTable that
+    // needs postponing.
    if (ucmp->CompareTimestamp(table_newest_udt, full_history_ts_low) >= 0) {
      return true;
    }
-#ifndef NDEBUG
-    last_table_newest_udt = table_newest_udt;
-#endif /* !NDEBUG */
  }
  return false;
 }
@@ -1756,4 +1778,20 @@ const Comparator* GetColumnFamilyUserComparator(
  return nullptr;
 }

+const ImmutableOptions& GetImmutableOptions(ColumnFamilyHandle* column_family) {
+  assert(column_family);
+
+  ColumnFamilyHandleImpl* const handle =
+      static_cast_with_check<ColumnFamilyHandleImpl>(column_family);
+  assert(handle);
+
+  const ColumnFamilyData* const cfd = handle->cfd();
+  assert(cfd);
+
+  const ImmutableOptions* ioptions = cfd->ioptions();
+  assert(ioptions);
+
+  return *ioptions;
+}
+
 }  // namespace ROCKSDB_NAMESPACE
@@ -211,6 +211,12 @@ struct SuperVersion {
  // Version number of the current SuperVersion
  uint64_t version_number;
  WriteStallCondition write_stall_condition;
+  // Each time `full_history_ts_low` collapses history, a new SuperVersion is
+  // installed. This field tracks the effective `full_history_ts_low` for that
+  // SuperVersion, to be used by read APIs for sanity checks. This field is
+  // immutable once SuperVersion is installed. For column family that doesn't
+  // enable UDT feature, this is an empty string.
+  std::string full_history_ts_low;

  // should be called outside the mutex
  SuperVersion() = default;
@@ -871,4 +877,7 @@ extern uint32_t GetColumnFamilyID(ColumnFamilyHandle* column_family);
 extern const Comparator* GetColumnFamilyUserComparator(
    ColumnFamilyHandle* column_family);

+extern const ImmutableOptions& GetImmutableOptions(
+    ColumnFamilyHandle* column_family);
+
 }  // namespace ROCKSDB_NAMESPACE
@@ -8,6 +8,7 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.

 #include <algorithm>
+#include <atomic>
 #include <string>
 #include <thread>
 #include <vector>
@@ -27,6 +28,7 @@
 #include "test_util/testharness.h"
 #include "test_util/testutil.h"
 #include "util/coding.h"
+#include "util/defer.h"
 #include "util/string_util.h"
 #include "utilities/fault_injection_env.h"
 #include "utilities/merge_operators.h"
@@ -2169,13 +2171,57 @@ TEST_P(ColumnFamilyTest, FlushStaleColumnFamilies) {
  Close();
 }

+namespace {
+struct CountOptionsFilesFs : public FileSystemWrapper {
+  explicit CountOptionsFilesFs(const std::shared_ptr<FileSystem>& t)
+      : FileSystemWrapper(t) {}
+  const char* Name() const override { return "CountOptionsFilesFs"; }
+
+  IOStatus NewWritableFile(const std::string& f, const FileOptions& file_opts,
+                           std::unique_ptr<FSWritableFile>* r,
+                           IODebugContext* dbg) override {
+    if (f.find("OPTIONS-") != std::string::npos) {
+      options_files_created.fetch_add(1, std::memory_order_relaxed);
+    }
+    return FileSystemWrapper::NewWritableFile(f, file_opts, r, dbg);
+  }
+
+  std::atomic<int> options_files_created{};
+};
+}  // namespace
+
 TEST_P(ColumnFamilyTest, CreateMissingColumnFamilies) {
-  Status s = TryOpen({"one", "two"});
-  ASSERT_TRUE(!s.ok());
-  db_options_.create_missing_column_families = true;
-  s = TryOpen({"default", "one", "two"});
-  ASSERT_TRUE(s.ok());
+  // Can't accidentally add CFs to an existing DB
+  Open();
  Close();
+  ASSERT_FALSE(db_options_.create_missing_column_families);
+  ASSERT_NOK(TryOpen({"one", "two"}));
+
+  // Nor accidentally create in a new DB
+  Destroy();
+  db_options_.create_if_missing = true;
+  ASSERT_NOK(TryOpen({"one", "two"}));
+
+  // Only with the option (new DB case)
+  db_options_.create_missing_column_families = true;
+  // Also setup to count number of options files created (see check below)
+  auto my_fs =
+      std::make_shared<CountOptionsFilesFs>(db_options_.env->GetFileSystem());
+  auto my_env = std::make_unique<CompositeEnvWrapper>(db_options_.env, my_fs);
+  SaveAndRestore<Env*> save_restore_env(&db_options_.env, my_env.get());
+
+  ASSERT_OK(TryOpen({"default", "one", "two"}));
+  Close();
+
+  // An older version would write an updated options file for each column
+  // family created under create_missing_column_families, which would be
+  // quadratic I/O in the number of column families.
+  ASSERT_EQ(my_fs->options_files_created.load(), 1);
+
+  // Add to existing DB case
+  ASSERT_OK(TryOpen({"default", "one", "two", "three", "four"}));
+  Close();
+  ASSERT_EQ(my_fs->options_files_created.load(), 2);
 }

 TEST_P(ColumnFamilyTest, SanitizeOptions) {
@@ -2427,7 +2473,10 @@ void DropSingleColumnFamily(ColumnFamilyTest* cf_test, int cf_id,
 }
 }  // anonymous namespace

-TEST_P(ColumnFamilyTest, CreateAndDropRace) {
+// This test attempts to set up a race condition in a way that is no longer
+// possible, causing the test to hang. If DBImpl::options_mutex_ is removed
+// in the future, this test might become relevant again.
+TEST_P(ColumnFamilyTest, DISABLED_CreateAndDropRace) {
  const int kCfCount = 5;
  std::vector<ColumnFamilyOptions> cf_opts;
  std::vector<Comparator*> comparators;
@@ -2489,6 +2538,53 @@ TEST_P(ColumnFamilyTest, CreateAndDropRace) {
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
 }

+TEST_P(ColumnFamilyTest, CreateAndDropPeriodicRace) {
+  // This is a mini-stress test looking for inconsistency between the set of
+  // CFs in the DB, particularly whether any use preserve_internal_time_seconds,
+  // and whether that is accurately reflected in the periodic task setup.
+  constexpr size_t kNumThreads = 12;
+  std::vector<std::thread> threads;
+  bool last_cf_on = Random::GetTLSInstance()->OneIn(2);
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::RegisterRecordSeqnoTimeWorker:BeforePeriodicTaskType",
+      [&](void* /*arg*/) { std::this_thread::yield(); });
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  ASSERT_EQ(column_family_options_.preserve_internal_time_seconds, 0U);
+  ColumnFamilyOptions other_opts = column_family_options_;
+  ColumnFamilyOptions last_opts = column_family_options_;
+  (last_cf_on ? last_opts : other_opts).preserve_internal_time_seconds =
+      1000000;
+  Open();
+
+  for (size_t i = 0; i < kNumThreads; i++) {
+    threads.emplace_back([this, &other_opts, i]() {
+      ColumnFamilyHandle* cfh;
+      ASSERT_OK(db_->CreateColumnFamily(other_opts, std::to_string(i), &cfh));
+      ASSERT_OK(db_->DropColumnFamily(cfh));
+      ASSERT_OK(db_->DestroyColumnFamilyHandle(cfh));
+    });
+  }
+
+  ColumnFamilyHandle* last_cfh;
+  ASSERT_OK(db_->CreateColumnFamily(last_opts, "last", &last_cfh));
+
+  for (auto& t : threads) {
+    t.join();
+  }
+
+  bool task_enabled = dbfull()->TEST_GetPeriodicTaskScheduler().TEST_HasTask(
+      PeriodicTaskType::kRecordSeqnoTime);
+  ASSERT_EQ(last_cf_on, task_enabled);
+
+  ASSERT_OK(db_->DropColumnFamily(last_cfh));
+  ASSERT_OK(db_->DestroyColumnFamilyHandle(last_cfh));
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
+}
+
 TEST_P(ColumnFamilyTest, WriteStallSingleColumnFamily) {
  const uint64_t kBaseRate = 800000u;
  db_options_.delayed_write_rate = kBaseRate;
@@ -2875,6 +2971,43 @@ TEST_P(ColumnFamilyTest, CompactionSpeedupTwoColumnFamilies) {
  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
 }

+TEST_P(ColumnFamilyTest, CompactionSpeedupForCompactionDebt) {
+  db_options_.max_background_compactions = 6;
+  Open();
+  ColumnFamilyData* cfd =
+      static_cast<ColumnFamilyHandleImpl*>(db_->DefaultColumnFamily())->cfd();
+  MutableCFOptions mutable_cf_options(column_family_options_);
+  mutable_cf_options.soft_pending_compaction_bytes_limit =
+      std::numeric_limits<uint64_t>::max();
+
+  {
+    // No bottommost data, so debt ratio cannot trigger speedup.
+    VersionStorageInfo* vstorage = cfd->current()->storage_info();
+    vstorage->TEST_set_estimated_compaction_needed_bytes(1048576 /* 1MB */);
+    RecalculateWriteStallConditions(cfd, mutable_cf_options);
+    ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
+  }
+
+  // Add a tiny amount of bottommost data.
+  ASSERT_OK(db_->Put(WriteOptions(), "foo", "bar"));
+  ASSERT_OK(db_->Flush(FlushOptions()));
+
+  {
+    // 1MB debt is way bigger than bottommost data so definitely triggers
+    // speedup.
+    VersionStorageInfo* vstorage = cfd->current()->storage_info();
+    vstorage->TEST_set_estimated_compaction_needed_bytes(1048576 /* 1MB */);
+    RecalculateWriteStallConditions(cfd, mutable_cf_options);
+    ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed());
+
+    // Eight bytes is way smaller than bottommost data so definitely does not
+    // trigger speedup.
+    vstorage->TEST_set_estimated_compaction_needed_bytes(8);
+    RecalculateWriteStallConditions(cfd, mutable_cf_options);
+    ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
+  }
+}
+
 TEST_P(ColumnFamilyTest, CreateAndDestroyOptions) {
  std::unique_ptr<ColumnFamilyOptions> cfo(new ColumnFamilyOptions());
  ColumnFamilyHandle* cfh;
@@ -115,6 +115,42 @@ void Compaction::GetBoundaryKeys(
  }
 }

+void Compaction::GetBoundaryInternalKeys(
+    VersionStorageInfo* vstorage,
+    const std::vector<CompactionInputFiles>& inputs, InternalKey* smallest_key,
+    InternalKey* largest_key, int exclude_level) {
+  bool initialized = false;
+  const InternalKeyComparator* icmp = vstorage->InternalComparator();
+  for (size_t i = 0; i < inputs.size(); ++i) {
+    if (inputs[i].files.empty() || inputs[i].level == exclude_level) {
+      continue;
+    }
+    if (inputs[i].level == 0) {
+      // we need to consider all files on level 0
+      for (const auto* f : inputs[i].files) {
+        if (!initialized || icmp->Compare(f->smallest, *smallest_key) < 0) {
+          *smallest_key = f->smallest;
+        }
+        if (!initialized || icmp->Compare(f->largest, *largest_key) > 0) {
+          *largest_key = f->largest;
+        }
+        initialized = true;
+      }
+    } else {
+      // we only need to consider the first and last file
+      if (!initialized ||
+          icmp->Compare(inputs[i].files[0]->smallest, *smallest_key) < 0) {
+        *smallest_key = inputs[i].files[0]->smallest;
+      }
+      if (!initialized ||
+          icmp->Compare(inputs[i].files.back()->largest, *largest_key) > 0) {
+        *largest_key = inputs[i].files.back()->largest;
+      }
+      initialized = true;
+    }
+  }
+}
+
 std::vector<CompactionInputFiles> Compaction::PopulateWithAtomicBoundaries(
    VersionStorageInfo* vstorage, std::vector<CompactionInputFiles> inputs) {
  const Comparator* ucmp = vstorage->InternalComparator()->user_comparator();
@@ -204,32 +240,36 @@ bool Compaction::IsFullCompaction(
  return num_files_in_compaction == total_num_files;
 }

-const TablePropertiesCollection& Compaction::GetTableProperties() {
-  if (!input_table_properties_initialized_) {
-    const ReadOptions read_options(Env::IOActivity::kCompaction);
-    for (size_t i = 0; i < num_input_levels(); ++i) {
-      for (const FileMetaData* fmd : *(this->inputs(i))) {
-        std::shared_ptr<const TableProperties> tp;
-        std::string file_name =
-            TableFileName(immutable_options_.cf_paths, fmd->fd.GetNumber(),
-                          fmd->fd.GetPathId());
-        Status s = input_version_->GetTableProperties(read_options, &tp, fmd,
-                                                      &file_name);
-        if (s.ok()) {
-          table_properties_[file_name] = tp;
-        } else {
-          ROCKS_LOG_ERROR(immutable_options_.info_log,
-                          "Unable to load table properties for file %" PRIu64
-                          " --- %s\n",
-                          fmd->fd.GetNumber(), s.ToString().c_str());
-        }
+Status Compaction::InitInputTableProperties() {
+  if (!input_table_properties_.empty()) {
+    return Status::OK();
+  }
+
+  Status s;
+  const ReadOptions read_options(Env::IOActivity::kCompaction);
+  assert(input_version_);
+  for (size_t i = 0; i < num_input_levels(); ++i) {
+    for (const FileMetaData* fmd : *(this->inputs(i))) {
+      std::shared_ptr<const TableProperties> tp;
+      std::string file_name =
+          TableFileName(immutable_options_.cf_paths, fmd->fd.GetNumber(),
+                        fmd->fd.GetPathId());
+      s = input_version_->GetTableProperties(read_options, &tp, fmd,
+                                             &file_name);
+      if (s.ok()) {
+        input_table_properties_[file_name] = tp;
+      } else {
+        ROCKS_LOG_ERROR(immutable_options_.info_log,
+                        "Unable to load table properties for file %" PRIu64
+                        " --- %s\n",
+                        fmd->fd.GetNumber(), s.ToString().c_str());
+        input_table_properties_.clear();
+        return s;
      }
    }
+  }

-    input_table_properties_initialized_ = true;
-  };
-
-  return table_properties_;
+  return s;
 }

 Compaction::Compaction(
@@ -395,9 +435,14 @@ void Compaction::PopulatePenultimateLevelOutputRange() {
    }
  }

-  GetBoundaryKeys(input_vstorage_, inputs_,
-                  &penultimate_level_smallest_user_key_,
-                  &penultimate_level_largest_user_key_, exclude_level);
+  // FIXME: should make use of `penultimate_output_range_type_`.
+  // FIXME: when last level's input range does not overlap with
+  //  penultimate level, and penultimate level input is empty,
+  //  this call will not set penultimate_level_smallest_ or
+  //  penultimate_level_largest_. No keys will be compacted up.
+  GetBoundaryInternalKeys(input_vstorage_, inputs_,
+                          &penultimate_level_smallest_,
+                          &penultimate_level_largest_, exclude_level);
 }

 Compaction::~Compaction() {
@@ -422,33 +467,40 @@ bool Compaction::OverlapPenultimateLevelOutputRange(
  if (!SupportsPerKeyPlacement()) {
    return false;
  }
+
+  // See FIXME in Compaction::PopulatePenultimateLevelOutputRange().
+  // We do not compact any key up in this case.
+  if (penultimate_level_smallest_.size() == 0 ||
+      penultimate_level_largest_.size() == 0) {
+    return false;
+  }
+
  const Comparator* ucmp =
      input_vstorage_->InternalComparator()->user_comparator();

  return ucmp->CompareWithoutTimestamp(
-             smallest_key, penultimate_level_largest_user_key_) <= 0 &&
+             smallest_key, penultimate_level_largest_.user_key()) <= 0 &&
         ucmp->CompareWithoutTimestamp(
-             largest_key, penultimate_level_smallest_user_key_) >= 0;
+             largest_key, penultimate_level_smallest_.user_key()) >= 0;
 }

 // key includes timestamp if user-defined timestamp is enabled.
-bool Compaction::WithinPenultimateLevelOutputRange(const Slice& key) const {
+bool Compaction::WithinPenultimateLevelOutputRange(
+    const ParsedInternalKey& ikey) const {
  if (!SupportsPerKeyPlacement()) {
    return false;
  }

-  if (penultimate_level_smallest_user_key_.empty() ||
-      penultimate_level_largest_user_key_.empty()) {
+  if (penultimate_level_smallest_.size() == 0 ||
+      penultimate_level_largest_.size() == 0) {
    return false;
  }

-  const Comparator* ucmp =
-      input_vstorage_->InternalComparator()->user_comparator();
+  const InternalKeyComparator* icmp = input_vstorage_->InternalComparator();

-  return ucmp->CompareWithoutTimestamp(
-             key, penultimate_level_smallest_user_key_) >= 0 &&
-         ucmp->CompareWithoutTimestamp(
-             key, penultimate_level_largest_user_key_) <= 0;
+  // op_type of a key can change during compaction, e.g. Merge -> Put.
+  return icmp->CompareKeySeq(ikey, penultimate_level_smallest_.Encode()) >= 0 &&
+         icmp->CompareKeySeq(ikey, penultimate_level_largest_.Encode()) <= 0;
 }

 bool Compaction::InputCompressionMatchesOutput() const {
@@ -774,8 +826,17 @@ std::unique_ptr<CompactionFilter> Compaction::CreateCompactionFilter() const {
  CompactionFilter::Context context;
  context.is_full_compaction = is_full_compaction_;
  context.is_manual_compaction = is_manual_compaction_;
+  context.input_start_level = start_level_;
  context.column_family_id = cfd_->GetID();
  context.reason = TableFileCreationReason::kCompaction;
+  context.input_table_properties = GetInputTableProperties();
+  if (context.input_table_properties.empty()) {
+    ROCKS_LOG_WARN(
+        immutable_options_.info_log,
+        "Unable to set `input_table_properties` of `CompactionFilter::Context` "
+        "for compaction.");
+  }
+
  return cfd_->ioptions()->compaction_filter_factory->CreateCompactionFilter(
      context);
 }
@@ -289,7 +289,14 @@ class Compaction {
  // is the sum of all input file sizes.
  uint64_t OutputFilePreallocationSize() const;

-  void SetInputVersion(Version* input_version);
+  // TODO(hx235): eventually we should consider `InitInputTableProperties()`'s
+  // status and fail the compaction if needed
+  // TODO(hx235): consider making this function part of the construction so we
+  // don't forget to call it
+  void FinalizeInputInfo(Version* input_version) {
+    SetInputVersion(input_version);
+    InitInputTableProperties().PermitUncheckedError();
+  }

  struct InputLevelSummaryBuffer {
    char buffer[128];
@@ -326,30 +333,26 @@ class Compaction {
      int output_level, VersionStorageInfo* vstorage,
      const std::vector<CompactionInputFiles>& inputs);

-  // If called before a compaction finishes, will return
-  // table properties of all compaction input files.
-  // If called after a compaction finished, will return
-  // table properties of all compaction input and output files.
-  const TablePropertiesCollection& GetTableProperties();
+  const TablePropertiesCollection& GetInputTableProperties() const {
+    return input_table_properties_;
+  }

+  // TODO(hx235): consider making this function symmetric to
+  // InitInputTableProperties()
  void SetOutputTableProperties(
      const std::string& file_name,
      const std::shared_ptr<const TableProperties>& tp) {
-    table_properties_[file_name] = tp;
+    output_table_properties_[file_name] = tp;
+  }
+
+  const TablePropertiesCollection& GetOutputTableProperties() const {
+    return output_table_properties_;
  }

  Slice GetSmallestUserKey() const { return smallest_user_key_; }

  Slice GetLargestUserKey() const { return largest_user_key_; }

-  Slice GetPenultimateLevelSmallestUserKey() const {
-    return penultimate_level_smallest_user_key_;
-  }
-
-  Slice GetPenultimateLevelLargestUserKey() const {
-    return penultimate_level_largest_user_key_;
-  }
-
  PenultimateOutputRangeType GetPenultimateOutputRangeType() const {
    return penultimate_output_range_type_;
  }
@@ -372,10 +375,8 @@ class Compaction {
  // per_key_placement feature, which is safe to place the key to the
  // penultimate level. different compaction strategy has different rules.
  // If per_key_placement is not supported, always return false.
-  // TODO: currently it doesn't support moving data from the last level to the
-  //  penultimate level
  //  key includes timestamp if user-defined timestamp is enabled.
-  bool WithinPenultimateLevelOutputRange(const Slice& key) const;
+  bool WithinPenultimateLevelOutputRange(const ParsedInternalKey& ikey) const;

  CompactionReason compaction_reason() const { return compaction_reason_; }

@@ -432,6 +433,10 @@ class Compaction {
                                      const int output_level);

 private:
+  void SetInputVersion(Version* input_version);
+
+  Status InitInputTableProperties();
+
  // mark (or clear) all files that are being compacted
  void MarkFilesBeingCompacted(bool mark_as_compacted);

@@ -441,6 +446,13 @@ class Compaction {
                              Slice* smallest_key, Slice* largest_key,
                              int exclude_level = -1);

+  // get the smallest and largest internal key present in files to be compacted
+  static void GetBoundaryInternalKeys(
+      VersionStorageInfo* vstorage,
+      const std::vector<CompactionInputFiles>& inputs,
+      InternalKey* smallest_key, InternalKey* largest_key,
+      int exclude_level = -1);
+
  // populate penultimate level output range, which will be used to determine if
  // a key is safe to output to the penultimate level (details see
  // `Compaction::WithinPenultimateLevelOutputRange()`.
@@ -522,9 +534,8 @@ class Compaction {
  // Does input compression match the output compression?
  bool InputCompressionMatchesOutput() const;

-  bool input_table_properties_initialized_ = false;
-  // table properties of output files
-  TablePropertiesCollection table_properties_;
+  TablePropertiesCollection input_table_properties_;
+  TablePropertiesCollection output_table_properties_;

  // smallest user keys in compaction
  // includes timestamp if user-defined timestamp is enabled.
@@ -554,8 +565,8 @@ class Compaction {
  // Key range for penultimate level output
  // includes timestamp if user-defined timestamp is enabled.
  // penultimate_output_range_type_ shows the range type
-  Slice penultimate_level_smallest_user_key_;
-  Slice penultimate_level_largest_user_key_;
+  InternalKey penultimate_level_smallest_;
+  InternalKey penultimate_level_largest_;
  PenultimateOutputRangeType penultimate_output_range_type_ =
      PenultimateOutputRangeType::kNotSupported;
 };
@@ -14,6 +14,7 @@
 #include "db/blob/prefetch_buffer_collection.h"
 #include "db/snapshot_checker.h"
 #include "db/wide/wide_column_serialization.h"
+#include "db/wide/wide_columns_helper.h"
 #include "logging/logging.h"
 #include "port/likely.h"
 #include "rocksdb/listener.h"
@@ -423,16 +424,13 @@ bool CompactionIterator::InvokeFilterIfNeeded(bool* need_skip,
    return false;
  } else if (decision == CompactionFilter::Decision::kChangeWideColumnEntity) {
    WideColumns sorted_columns;
-
    sorted_columns.reserve(new_columns.size());
+
    for (const auto& column : new_columns) {
      sorted_columns.emplace_back(column.first, column.second);
    }

-    std::sort(sorted_columns.begin(), sorted_columns.end(),
-              [](const WideColumn& lhs, const WideColumn& rhs) {
-                return lhs.name().compare(rhs.name()) < 0;
-              });
+    WideColumnsHelper::SortColumns(sorted_columns);

    {
      const Status s = WideColumnSerialization::Serialize(
@@ -1229,7 +1227,7 @@ void CompactionIterator::DecideOutputLevel() {
    // not from this compaction.
    // TODO: add statistic for declined output_to_penultimate_level
    bool safe_to_penultimate_level =
-        compaction_->WithinPenultimateLevelOutputRange(ikey_.user_key);
+        compaction_->WithinPenultimateLevelOutputRange(ikey_);
    if (!safe_to_penultimate_level) {
      output_to_penultimate_level_ = false;
      // It could happen when disable/enable `last_level_temperature` while
@@ -1258,10 +1256,13 @@ void CompactionIterator::PrepareOutput() {
      } else if (ikey_.type == kTypeBlobIndex) {
        GarbageCollectBlobIfNeeded();
      }
-    }

-    if (compaction_ != nullptr && compaction_->SupportsPerKeyPlacement()) {
-      DecideOutputLevel();
+      // For range del sentinel, we don't use it to cut files for bottommost
+      // compaction. So it should not make a difference which output level we
+      // decide.
+      if (compaction_ != nullptr && compaction_->SupportsPerKeyPlacement()) {
+        DecideOutputLevel();
+      }
    }

    // Zeroing out the sequence number leads to better compression.
@@ -119,7 +119,8 @@ class CompactionIterator {
    virtual bool SupportsPerKeyPlacement() const = 0;

    // `key` includes timestamp if user-defined timestamp is enabled.
-    virtual bool WithinPenultimateLevelOutputRange(const Slice& key) const = 0;
+    virtual bool WithinPenultimateLevelOutputRange(
+        const ParsedInternalKey&) const = 0;
  };

  class RealCompaction : public CompactionProxy {
@@ -186,8 +187,9 @@ class CompactionIterator {
    // Check if key is within penultimate level output range, to see if it's
    // safe to output to the penultimate level for per_key_placement feature.
    // `key` includes timestamp if user-defined timestamp is enabled.
-    bool WithinPenultimateLevelOutputRange(const Slice& key) const override {
-      return compaction_->WithinPenultimateLevelOutputRange(key);
+    bool WithinPenultimateLevelOutputRange(
+        const ParsedInternalKey& ikey) const override {
+      return compaction_->WithinPenultimateLevelOutputRange(ikey);
    }

   private:
@@ -184,8 +184,9 @@ class FakeCompaction : public CompactionIterator::CompactionProxy {
    return supports_per_key_placement;
  }

-  bool WithinPenultimateLevelOutputRange(const Slice& key) const override {
-    return (!key.starts_with("unsafe_pb"));
+  bool WithinPenultimateLevelOutputRange(
+      const ParsedInternalKey& key) const override {
+    return (!key.user_key.starts_with("unsafe_pb"));
  }

  bool key_not_exists_beyond_output_level = false;
@@ -288,23 +288,23 @@ void CompactionJob::Prepare() {

  if (preserve_time_duration > 0) {
    const ReadOptions read_options(Env::IOActivity::kCompaction);
-    // setup seqno_time_mapping_
-    seqno_time_mapping_.SetMaxTimeDuration(preserve_time_duration);
+    // setup seqno_to_time_mapping_
+    seqno_to_time_mapping_.SetMaxTimeDuration(preserve_time_duration);
    for (const auto& each_level : *c->inputs()) {
      for (const auto& fmd : each_level.files) {
        std::shared_ptr<const TableProperties> tp;
        Status s =
            cfd->current()->GetTableProperties(read_options, &tp, fmd, nullptr);
        if (s.ok()) {
-          seqno_time_mapping_.Add(tp->seqno_to_time_mapping)
+          seqno_to_time_mapping_.Add(tp->seqno_to_time_mapping)
              .PermitUncheckedError();
-          seqno_time_mapping_.Add(fmd->fd.smallest_seqno,
-                                  fmd->oldest_ancester_time);
+          seqno_to_time_mapping_.Add(fmd->fd.smallest_seqno,
+                                     fmd->oldest_ancester_time);
        }
      }
    }

-    auto status = seqno_time_mapping_.Sort();
+    auto status = seqno_to_time_mapping_.Sort();
    if (!status.ok()) {
      ROCKS_LOG_WARN(db_options_.info_log,
                     "Invalid sequence number to time mapping: Status: %s",
@@ -320,13 +320,17 @@ void CompactionJob::Prepare() {
      preserve_time_min_seqno_ = 0;
      preclude_last_level_min_seqno_ = 0;
    } else {
-      seqno_time_mapping_.TruncateOldEntries(_current_time);
+      seqno_to_time_mapping_.TruncateOldEntries(_current_time);
      uint64_t preserve_time =
          static_cast<uint64_t>(_current_time) > preserve_time_duration
              ? _current_time - preserve_time_duration
              : 0;
+      // GetProximalSeqnoBeforeTime tells us the last seqno known to have been
+      // written at or before the given time. + 1 to get the minimum we should
+      // preserve without excluding anything that might have been written on or
+      // after the given time.
      preserve_time_min_seqno_ =
-          seqno_time_mapping_.GetOldestSequenceNum(preserve_time);
+          seqno_to_time_mapping_.GetProximalSeqnoBeforeTime(preserve_time) + 1;
      if (c->immutable_options()->preclude_last_level_data_seconds > 0) {
        uint64_t preclude_last_level_time =
            static_cast<uint64_t>(_current_time) >
@@ -335,7 +339,9 @@ void CompactionJob::Prepare() {
                      c->immutable_options()->preclude_last_level_data_seconds
                : 0;
        preclude_last_level_min_seqno_ =
-            seqno_time_mapping_.GetOldestSequenceNum(preclude_last_level_time);
+            seqno_to_time_mapping_.GetProximalSeqnoBeforeTime(
+                preclude_last_level_time) +
+            1;
      }
    }
  }
@@ -844,7 +850,8 @@ Status CompactionJob::Run() {
  return status;
 }

-Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) {
+Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options,
+                              bool* compaction_released) {
  assert(compact_);

  AutoThreadOperationStageUpdater stage_updater(
@@ -860,7 +867,7 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) {
                                            compaction_stats_);

  if (status.ok()) {
-    status = InstallCompactionResults(mutable_cf_options);
+    status = InstallCompactionResults(mutable_cf_options, compaction_released);
  }
  if (!versions_->io_status().ok()) {
    io_status_ = versions_->io_status();
@@ -1318,6 +1325,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
      "CompactionJob::ProcessKeyValueCompaction()::Processing",
      reinterpret_cast<void*>(
          const_cast<Compaction*>(sub_compact->compaction)));
+  uint64_t last_cpu_micros = prev_cpu_micros;
  while (status.ok() && !cfd->IsDropped() && c_iter->Valid()) {
    // Invariant: c_iter.status() is guaranteed to be OK if c_iter->Valid()
    // returns true.
@@ -1329,6 +1337,12 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
      RecordDroppedKeys(c_iter_stats, &sub_compact->compaction_job_stats);
      c_iter->ResetRecordCounts();
      RecordCompactionIOStats();
+
+      uint64_t cur_cpu_micros = db_options_.clock->CPUMicros();
+      assert(cur_cpu_micros >= last_cpu_micros);
+      RecordTick(stats_, COMPACTION_CPU_TOTAL_TIME,
+                 cur_cpu_micros - last_cpu_micros);
+      last_cpu_micros = cur_cpu_micros;
    }

    // Add current compaction_iterator key to target compaction output, if the
@@ -1436,8 +1450,11 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
    sub_compact->Current().UpdateBlobStats();
  }

+  uint64_t cur_cpu_micros = db_options_.clock->CPUMicros();
  sub_compact->compaction_job_stats.cpu_micros =
-      db_options_.clock->CPUMicros() - prev_cpu_micros;
+      cur_cpu_micros - prev_cpu_micros;
+  RecordTick(stats_, COMPACTION_CPU_TOTAL_TIME,
+             cur_cpu_micros - last_cpu_micros);

  if (measure_io_stats_) {
    sub_compact->compaction_job_stats.file_write_nanos +=
@@ -1559,7 +1576,7 @@ Status CompactionJob::FinishCompactionOutputFile(

  const uint64_t current_entries = outputs.NumEntries();

-  s = outputs.Finish(s, seqno_time_mapping_);
+  s = outputs.Finish(s, seqno_to_time_mapping_);

  if (s.ok()) {
    // With accurate smallest and largest key, we can get a slightly more
@@ -1687,7 +1704,7 @@ Status CompactionJob::FinishCompactionOutputFile(
 }

 Status CompactionJob::InstallCompactionResults(
-    const MutableCFOptions& mutable_cf_options) {
+    const MutableCFOptions& mutable_cf_options, bool* compaction_released) {
  assert(compact_);

  db_mutex_->AssertHeld();
@@ -1769,9 +1786,15 @@ Status CompactionJob::InstallCompactionResults(
    }
  }

-  return versions_->LogAndApply(compaction->column_family_data(),
-                                mutable_cf_options, read_options, edit,
-                                db_mutex_, db_directory_);
+  auto manifest_wcb = [&compaction, &compaction_released](const Status& s) {
+    compaction->ReleaseCompactionFiles(s);
+    *compaction_released = true;
+  };
+
+  return versions_->LogAndApply(
+      compaction->column_family_data(), mutable_cf_options, read_options, edit,
+      db_mutex_, db_directory_, /*new_descriptor_log=*/false,
+      /*column_family_options=*/nullptr, manifest_wcb);
 }

 void CompactionJob::RecordCompactionIOStats() {
@@ -1877,6 +1900,8 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
          sub_compact->start.has_value() ? &tmp_start : nullptr,
          sub_compact->end.has_value() ? &tmp_end : nullptr);
  if (oldest_ancester_time == std::numeric_limits<uint64_t>::max()) {
+    // TODO: fix DBSSTTest.GetTotalSstFilesSize and use
+    //  kUnknownOldestAncesterTime
    oldest_ancester_time = current_time;
  }

@@ -1921,6 +1946,7 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
      db_options_.stats, listeners, db_options_.file_checksum_gen_factory.get(),
      tmp_set.Contains(FileType::kTableFile), false));

+  // TODO(hx235): pass in the correct `oldest_key_time` instead of `0`
  TableBuilderOptions tboptions(
      *cfd->ioptions(), *(sub_compact->compaction->mutable_cf_options()),
      cfd->internal_comparator(), cfd->int_tbl_prop_collector_factories(),
@@ -1962,7 +1988,7 @@ bool CompactionJob::UpdateCompactionStats(uint64_t* num_input_range_del) {

  bool has_error = false;
  const ReadOptions read_options(Env::IOActivity::kCompaction);
-  const auto& input_table_properties = compaction->GetTableProperties();
+  const auto& input_table_properties = compaction->GetInputTableProperties();
  for (int input_level = 0;
       input_level < static_cast<int>(compaction->num_input_levels());
       ++input_level) {
@@ -186,7 +186,10 @@ class CompactionJob {

  // REQUIRED: mutex held
  // Add compaction input/output to the current version
-  Status Install(const MutableCFOptions& mutable_cf_options);
+  // Releases compaction file through Compaction::ReleaseCompactionFiles().
+  // Sets *compaction_released to true if compaction is released.
+  Status Install(const MutableCFOptions& mutable_cf_options,
+                 bool* compaction_released);

  // Return the IO status
  IOStatus io_status() const { return io_status_; }
@@ -273,7 +276,8 @@ class CompactionJob {
                                    const Slice& next_table_min_key,
                                    const Slice* comp_start_user_key,
                                    const Slice* comp_end_user_key);
-  Status InstallCompactionResults(const MutableCFOptions& mutable_cf_options);
+  Status InstallCompactionResults(const MutableCFOptions& mutable_cf_options,
+                                  bool* compaction_released);
  Status OpenCompactionOutputFile(SubcompactionState* sub_compact,
                                  CompactionOutputs& outputs);
  void UpdateCompactionJobStats(
@@ -346,7 +350,7 @@ class CompactionJob {

  // Stores the sequence number to time mapping gathered from all input files
  // it also collects the smallest_seqno -> oldest_ancester_time from the SST.
-  SeqnoToTimeMapping seqno_time_mapping_;
+  SeqnoToTimeMapping seqno_to_time_mapping_;

  // Minimal sequence number for preserving the time information. The time info
  // older than this sequence number won't be preserved after the compaction and
@@ -215,7 +215,9 @@ class CompactionJobTestBase : public testing::Test {
            dbname_, &db_options_, env_options_, table_cache_.get(),
            &write_buffer_manager_, &write_controller_,
            /*block_cache_tracer=*/nullptr,
-            /*io_tracer=*/nullptr, /*db_id*/ "", /*db_session_id*/ "")),
+            /*io_tracer=*/nullptr, /*db_id=*/"", /*db_session_id=*/"",
+            /*daily_offpeak_time_utc=*/"",
+            /*error_handler=*/nullptr)),
        shutting_down_(false),
        mock_table_factory_(new mock::MockTableFactory()),
        error_handler_(nullptr, db_options_, &mutex_),
@@ -540,11 +542,12 @@ class CompactionJobTestBase : public testing::Test {
    ASSERT_OK(s);
    db_options_.info_log = info_log;

-    versions_.reset(
-        new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(),
-                       &write_buffer_manager_, &write_controller_,
-                       /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr,
-                       /*db_id*/ "", /*db_session_id*/ ""));
+    versions_.reset(new VersionSet(
+        dbname_, &db_options_, env_options_, table_cache_.get(),
+        &write_buffer_manager_, &write_controller_,
+        /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr,
+        /*db_id=*/"", /*db_session_id=*/"", /*daily_offpeak_time_utc=*/"",
+        /*error_handler=*/nullptr));
    compaction_job_stats_.Reset();
    ASSERT_OK(SetIdentityFile(env_, dbname_));

@@ -644,7 +647,7 @@ class CompactionJobTestBase : public testing::Test {
        mutable_cf_options_.max_compaction_bytes, 0, kNoCompression,
        cfd->GetLatestMutableCFOptions()->compression_opts,
        Temperature::kUnknown, max_subcompactions, grandparents, true);
-    compaction.SetInputVersion(cfd->current());
+    compaction.FinalizeInputInfo(cfd->current());

    assert(db_options_.info_log);
    LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, db_options_.info_log.get());
@@ -674,7 +677,9 @@ class CompactionJobTestBase : public testing::Test {
    ASSERT_OK(s);
    ASSERT_OK(compaction_job.io_status());
    mutex_.Lock();
-    ASSERT_OK(compaction_job.Install(*cfd->GetLatestMutableCFOptions()));
+    bool compaction_released = false;
+    ASSERT_OK(compaction_job.Install(*cfd->GetLatestMutableCFOptions(),
+                                     &compaction_released));
    ASSERT_OK(compaction_job.io_status());
    mutex_.Unlock();
    log_buffer.FlushBufferToLog();
@@ -1522,13 +1527,15 @@ TEST_F(CompactionJobTest, VerifyPenultimateLevelOutput) {
      {files0, files1, files2, files3}, input_levels,
      /*verify_func=*/[&](Compaction& comp) {
        for (char c = 'a'; c <= 'z'; c++) {
-          std::string c_str;
-          c_str = c;
-          const Slice key(c_str);
          if (c == 'a') {
-            ASSERT_FALSE(comp.WithinPenultimateLevelOutputRange(key));
+            ParsedInternalKey pik("a", 0U, kTypeValue);
+            ASSERT_FALSE(comp.WithinPenultimateLevelOutputRange(pik));
          } else {
-            ASSERT_TRUE(comp.WithinPenultimateLevelOutputRange(key));
+            std::string c_str{c};
+            // WithinPenultimateLevelOutputRange checks internal key range.
+            // 'z' is the last key, so set seqno properly.
+            ParsedInternalKey pik(c_str, c == 'z' ? 12U : 0U, kTypeValue);
+            ASSERT_TRUE(comp.WithinPenultimateLevelOutputRange(pik));
          }
        }
      });
@@ -18,16 +18,18 @@ void CompactionOutputs::NewBuilder(const TableBuilderOptions& tboptions) {
  builder_.reset(NewTableBuilder(tboptions, file_writer_.get()));
 }

-Status CompactionOutputs::Finish(const Status& intput_status,
-                                 const SeqnoToTimeMapping& seqno_time_mapping) {
+Status CompactionOutputs::Finish(
+    const Status& intput_status,
+    const SeqnoToTimeMapping& seqno_to_time_mapping) {
  FileMetaData* meta = GetMetaData();
  assert(meta != nullptr);
  Status s = intput_status;
  if (s.ok()) {
-    std::string seqno_time_mapping_str;
-    seqno_time_mapping.Encode(seqno_time_mapping_str, meta->fd.smallest_seqno,
-                              meta->fd.largest_seqno, meta->file_creation_time);
-    builder_->SetSeqnoTimeTableProperties(seqno_time_mapping_str,
+    std::string seqno_to_time_mapping_str;
+    seqno_to_time_mapping.Encode(
+        seqno_to_time_mapping_str, meta->fd.smallest_seqno,
+        meta->fd.largest_seqno, meta->file_creation_time);
+    builder_->SetSeqnoTimeTableProperties(seqno_to_time_mapping_str,
                                          meta->oldest_ancester_time);
    s = builder_->Finish();

@@ -107,7 +107,7 @@ class CompactionOutputs {

  // Finish the current output file
  Status Finish(const Status& intput_status,
-                const SeqnoToTimeMapping& seqno_time_mapping);
+                const SeqnoToTimeMapping& seqno_to_time_mapping);

  // Update output table properties from table builder
  void UpdateTableProperties() {
@@ -17,6 +17,9 @@
 #include "logging/log_buffer.h"
 #include "logging/logging.h"
 #include "options/options_helper.h"
+#include "rocksdb/listener.h"
+#include "rocksdb/statistics.h"
+#include "rocksdb/status.h"
 #include "util/string_util.h"

 namespace ROCKSDB_NAMESPACE {
@@ -85,7 +85,8 @@ class CompactionPickerTestBase : public testing::Test {
    vstorage_.reset(new VersionStorageInfo(
        &icmp_, ucmp_, options_.num_levels, style, nullptr, false,
        EpochNumberRequirement::kMustPresent, ioptions_.clock,
-        options_.bottommost_file_compaction_delay));
+        options_.bottommost_file_compaction_delay,
+        OffpeakTimeOption(mutable_db_options_.daily_offpeak_time_utc)));
    vstorage_->PrepareForVersionAppend(ioptions_, mutable_cf_options_);
  }

@@ -95,7 +96,8 @@ class CompactionPickerTestBase : public testing::Test {
    temp_vstorage_.reset(new VersionStorageInfo(
        &icmp_, ucmp_, options_.num_levels, ioptions_.compaction_style,
        vstorage_.get(), false, EpochNumberRequirement::kMustPresent,
-        ioptions_.clock, options_.bottommost_file_compaction_delay));
+        ioptions_.clock, options_.bottommost_file_compaction_delay,
+        OffpeakTimeOption(mutable_db_options_.daily_offpeak_time_utc)));
  }

  void DeleteVersionStorage() {
@@ -994,6 +996,61 @@ TEST_F(CompactionPickerTest, UniversalIncrementalSpace5) {
  ASSERT_EQ(13, compaction->num_input_files(1));
 }

+TEST_F(CompactionPickerTest,
+       PartiallyExcludeL0ToReduceWriteStopForSizeAmpCompaction) {
+  const uint64_t kFileSize = 100000;
+  const uint64_t kL0FileCount = 30;
+  const uint64_t kLastLevelFileCount = 1;
+  const uint64_t kNumLevels = 5;
+
+  for (const uint64_t test_no_exclusion : {false, true}) {
+    const uint64_t kExpectedNumExcludedL0 =
+        test_no_exclusion ? 0 : kL0FileCount * 1 / 10;
+
+    mutable_cf_options_.level0_stop_writes_trigger = 36;
+    mutable_cf_options_.compaction_options_universal
+        .max_size_amplification_percent = 1;
+    mutable_cf_options_.compaction_options_universal.max_merge_width =
+        test_no_exclusion
+            // In universal compaction, sorted runs from non L0 levels are
+            // counted toward `level0_stop_writes_trigger`. Therefore we need to
+            // subtract the total number of sorted runs picked originally for
+            // this compaction (i.e, kL0FileCount + kLastLevelFileCount) from
+            // `level0_stop_writes_trigger` to calculate `max_merge_width` that
+            // results in no L0 exclusion for testing purpose.
+            ? mutable_cf_options_.level0_stop_writes_trigger -
+                  (kL0FileCount + kLastLevelFileCount)
+            : UINT_MAX;
+
+    UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
+    NewVersionStorage(kNumLevels, kCompactionStyleUniversal);
+
+    for (uint64_t i = 1; i <= kL0FileCount + kLastLevelFileCount; ++i) {
+      Add(i <= kL0FileCount ? 0 : kNumLevels - 1, static_cast<uint32_t>(i),
+          std::to_string((i + 100) * 1000).c_str(),
+          std::to_string((i + 100) * 1000 + 999).c_str(), kFileSize, 0, i * 100,
+          i * 100 + 99);
+    }
+
+    UpdateVersionStorageInfo();
+
+    ASSERT_TRUE(universal_compaction_picker.NeedsCompaction(vstorage_.get()));
+    std::unique_ptr<Compaction> compaction(
+        universal_compaction_picker.PickCompaction(
+            cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(),
+            &log_buffer_));
+    ASSERT_TRUE(compaction.get() != nullptr);
+    ASSERT_EQ(compaction->compaction_reason(),
+              CompactionReason::kUniversalSizeAmplification);
+    ASSERT_EQ(compaction->num_input_files(0),
+              kL0FileCount - kExpectedNumExcludedL0);
+    ASSERT_EQ(compaction->num_input_files(kNumLevels - 1), kLastLevelFileCount);
+    for (uint64_t level = 1; level <= kNumLevels - 2; level++) {
+      ASSERT_EQ(compaction->num_input_files(level), 0);
+    }
+  }
+}
+
 TEST_F(CompactionPickerTest, NeedsCompactionFIFO) {
  NewVersionStorage(1, kCompactionStyleFIFO);
  const int kFileCount =
@@ -3380,6 +3437,9 @@ TEST_F(CompactionPickerTest, UniversalSizeAmpTierCompactionNonLastLevel) {
  ioptions_.preclude_last_level_data_seconds = 1000;
  mutable_cf_options_.compaction_options_universal
      .max_size_amplification_percent = 200;
+  // To avoid any L0 file exclusion in size amp compaction intended for reducing
+  // write stop
+  mutable_cf_options_.compaction_options_universal.max_merge_width = 2;
  UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);

  NewVersionStorage(kNumLevels, kCompactionStyleUniversal);
@@ -3453,6 +3513,9 @@ TEST_F(CompactionPickerTest, UniversalSizeAmpTierCompactionNotSuport) {
  ioptions_.preclude_last_level_data_seconds = 1000;
  mutable_cf_options_.compaction_options_universal
      .max_size_amplification_percent = 200;
+  // To avoid any L0 file exclusion in size amp compaction intended for reducing
+  // write stop
+  mutable_cf_options_.compaction_options_universal.max_merge_width = 2;
  UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);

  NewVersionStorage(kNumLevels, kCompactionStyleUniversal);
@@ -9,7 +9,7 @@

 #include "db/compaction/compaction_picker_universal.h"

-#include <cinttypes>
+#include <cstdint>
 #include <limits>
 #include <queue>
 #include <string>
@@ -114,6 +114,13 @@ class UniversalCompactionBuilder {
  // because some files are being compacted.
  Compaction* PickPeriodicCompaction();

+  bool ShouldSkipLastSortedRunForSizeAmpCompaction() const {
+    assert(!sorted_runs_.empty());
+    return ioptions_.preclude_last_level_data_seconds > 0 &&
+           ioptions_.num_levels > 2 &&
+           sorted_runs_.back().level == ioptions_.num_levels - 1 &&
+           sorted_runs_.size() > 1;
+  }
  // Used in universal compaction when the allow_trivial_move
  // option is set. Checks whether there are any overlapping files
  // in the input. Returns true if the input files are non
@@ -122,6 +129,100 @@ class UniversalCompactionBuilder {

  uint64_t GetMaxOverlappingBytes() const;

+  // To conditionally exclude some of the newest L0 files
+  // from a size amp compaction. This is to prevent a large number of L0
+  // files from being locked by a size amp compaction, potentially leading to
+  // write stop with a few more flushes.
+  //
+  // Such exclusion is based on `num_l0_input_pre_exclusion`,
+  // `level0_stop_writes_trigger`, `max/min_merge_width` and the pre-exclusion
+  // compaction score. Noted that it will not make the size amp compaction of
+  // interest invalid from running as a size amp compaction as long as its
+  // pre-exclusion compaction score satisfies the condition to run.
+  //
+  // @param `num_l0_input_pre_exclusion` Number of L0 input files prior to
+  // exclusion
+  // @param `end_index` Index of the last sorted run selected as compaction
+  // input. Will not be affected by this exclusion.
+  // @param `start_index` Index of the first input sorted run prior to
+  // exclusion. Will be modified as output based on the exclusion.
+  // @param `candidate_size` Total size of all except for the last input sorted
+  // runs prior to exclusion. Will be modified as output based on the exclusion.
+  //
+  // @return Number of L0 files to exclude. `start_index` and
+  // `candidate_size` will be modified accordingly
+  std::size_t MightExcludeNewL0sToReduceWriteStop(
+      std::size_t num_l0_input_pre_exclusion, std::size_t end_index,
+      std::size_t& start_index, uint64_t& candidate_size) const {
+    if (num_l0_input_pre_exclusion == 0) {
+      return 0;
+    }
+
+    assert(start_index <= end_index && sorted_runs_.size() > end_index);
+    assert(mutable_cf_options_.level0_stop_writes_trigger > 0);
+    const std::size_t level0_stop_writes_trigger = static_cast<std::size_t>(
+        mutable_cf_options_.level0_stop_writes_trigger);
+    const std::size_t max_merge_width = static_cast<std::size_t>(
+        mutable_cf_options_.compaction_options_universal.max_merge_width);
+    const std::size_t min_merge_width = static_cast<std::size_t>(
+        mutable_cf_options_.compaction_options_universal.min_merge_width);
+    const uint64_t max_size_amplification_percent =
+        mutable_cf_options_.compaction_options_universal
+            .max_size_amplification_percent;
+    const uint64_t base_sr_size = sorted_runs_[end_index].size;
+
+    // Leave at least 1 L0 file and 2 input sorted runs after exclusion
+    const std::size_t max_num_l0_to_exclude =
+        std::min(num_l0_input_pre_exclusion - 1, end_index - start_index - 1);
+    // In universal compaction, sorted runs from non L0 levels are counted
+    // toward `level0_stop_writes_trigger`. Therefore we need to subtract the
+    // total number of sorted runs picked originally for this compaction from
+    // `level0_stop_writes_trigger` to calculate
+    // `num_extra_l0_before_write_stop`
+    const std::size_t num_extra_l0_before_write_stop =
+        level0_stop_writes_trigger -
+        std::min(level0_stop_writes_trigger, end_index - start_index + 1);
+    const std::size_t num_l0_to_exclude_for_max_merge_width =
+        std::min(max_merge_width -
+                     std::min(max_merge_width, num_extra_l0_before_write_stop),
+                 max_num_l0_to_exclude);
+    const std::size_t num_l0_to_exclude_for_min_merge_width =
+        std::min(min_merge_width -
+                     std::min(min_merge_width, num_extra_l0_before_write_stop),
+                 max_num_l0_to_exclude);
+
+    std::size_t num_l0_to_exclude = 0;
+    uint64_t candidate_size_post_exclusion = candidate_size;
+
+    for (std::size_t possible_num_l0_to_exclude =
+             num_l0_to_exclude_for_min_merge_width;
+         possible_num_l0_to_exclude <= num_l0_to_exclude_for_max_merge_width;
+         ++possible_num_l0_to_exclude) {
+      uint64_t current_candidate_size = candidate_size_post_exclusion;
+      for (std::size_t j = num_l0_to_exclude; j < possible_num_l0_to_exclude;
+           ++j) {
+        current_candidate_size -=
+            sorted_runs_.at(start_index + j).compensated_file_size;
+      }
+
+      // To ensure the compaction score before and after exclusion is similar
+      // so this exclusion will not make the size amp compaction of
+      // interest invalid from running as a size amp compaction as long as its
+      // pre-exclusion compaction score satisfies the condition to run.
+      if (current_candidate_size * 100 <
+              max_size_amplification_percent * base_sr_size ||
+          current_candidate_size < candidate_size * 9 / 10) {
+        break;
+      }
+      num_l0_to_exclude = possible_num_l0_to_exclude;
+      candidate_size_post_exclusion = current_candidate_size;
+    }
+
+    start_index += num_l0_to_exclude;
+    candidate_size = candidate_size_post_exclusion;
+    return num_l0_to_exclude;
+  }
+
  const ImmutableOptions& ioptions_;
  const InternalKeyComparator* icmp_;
  double score_;
@@ -778,85 +879,67 @@ Compaction* UniversalCompactionBuilder::PickCompactionToReduceSortedRuns(

 // Look at overall size amplification. If size amplification
 // exceeds the configured value, then do a compaction
-// of the candidate files all the way upto the earliest
-// base file (overrides configured values of file-size ratios,
-// min_merge_width and max_merge_width).
-//
+// on longest span of candidate files without conflict with other compactions
+// ending at the earliest base file (overriding configured values of file-size
+// ratios, min_merge_width and max_merge_width).
 Compaction* UniversalCompactionBuilder::PickCompactionToReduceSizeAmp() {
-  // percentage flexibility while reducing size amplification
-  uint64_t ratio = mutable_cf_options_.compaction_options_universal
-                       .max_size_amplification_percent;
-
-  unsigned int candidate_count = 0;
-  uint64_t candidate_size = 0;
-  size_t start_index = 0;
-  const SortedRun* sr = nullptr;
-
  assert(!sorted_runs_.empty());
-  if (sorted_runs_.back().being_compacted) {
+
+  const size_t end_index = ShouldSkipLastSortedRunForSizeAmpCompaction()
+                               ? sorted_runs_.size() - 2
+                               : sorted_runs_.size() - 1;
+  if (sorted_runs_[end_index].being_compacted) {
+    return nullptr;
+  }
+  const uint64_t base_sr_size = sorted_runs_[end_index].size;
+  size_t start_index = end_index;
+  uint64_t candidate_size = 0;
+  size_t num_l0_files = 0;
+
+  // Get longest span (i.e, [start_index, end_index]) of available sorted runs
+  while (start_index > 0) {
+    const SortedRun* sr = &sorted_runs_[start_index - 1];
+    if (sr->being_compacted) {
+      char file_num_buf[kFormatFileNumberBufSize];
+      sr->Dump(file_num_buf, sizeof(file_num_buf), true);
+      ROCKS_LOG_BUFFER(
+          log_buffer_,
+          "[%s] Universal: stopping at sorted run undergoing compaction: "
+          "%s[%" ROCKSDB_PRIszt "]",
+          cf_name_.c_str(), file_num_buf, start_index - 1);
+      break;
+    }
+    candidate_size += sr->compensated_file_size;
+    num_l0_files += sr->level == 0 ? 1 : 0;
+    --start_index;
+  }
+
+  if (start_index == end_index) {
    return nullptr;
  }

-  // Skip files that are already being compacted
-  for (size_t loop = 0; loop + 1 < sorted_runs_.size(); loop++) {
-    sr = &sorted_runs_[loop];
-    if (!sr->being_compacted) {
-      start_index = loop;  // Consider this as the first candidate.
-      break;
-    }
-    char file_num_buf[kFormatFileNumberBufSize];
-    sr->Dump(file_num_buf, sizeof(file_num_buf), true);
+  {
+    const size_t num_l0_to_exclude = MightExcludeNewL0sToReduceWriteStop(
+        num_l0_files, end_index, start_index, candidate_size);
    ROCKS_LOG_BUFFER(log_buffer_,
-                     "[%s] Universal: skipping %s[%d] compacted %s",
-                     cf_name_.c_str(), file_num_buf, loop,
-                     " cannot be a candidate to reduce size amp.\n");
-    sr = nullptr;
+                     "[%s] Universal: Excluding %" ROCKSDB_PRIszt
+                     " latest L0 files to reduce potential write stop "
+                     "triggered by `level0_stop_writes_trigger`",
+                     cf_name_.c_str(), num_l0_to_exclude);
  }

-  if (sr == nullptr) {
-    return nullptr;  // no candidate files
-  }
  {
    char file_num_buf[kFormatFileNumberBufSize];
-    sr->Dump(file_num_buf, sizeof(file_num_buf), true);
+    sorted_runs_[start_index].Dump(file_num_buf, sizeof(file_num_buf), true);
    ROCKS_LOG_BUFFER(
        log_buffer_,
        "[%s] Universal: First candidate %s[%" ROCKSDB_PRIszt "] %s",
        cf_name_.c_str(), file_num_buf, start_index, " to reduce size amp.\n");
  }

-  // size of the base sorted run for size amp calculation
-  uint64_t base_sr_size = sorted_runs_.back().size;
-  size_t sr_end_idx = sorted_runs_.size() - 1;
-  // If tiered compaction is enabled and the last sorted run is the last level
-  if (ioptions_.preclude_last_level_data_seconds > 0 &&
-      ioptions_.num_levels > 2 &&
-      sorted_runs_.back().level == ioptions_.num_levels - 1 &&
-      sorted_runs_.size() > 1) {
-    sr_end_idx = sorted_runs_.size() - 2;
-    base_sr_size = sorted_runs_[sr_end_idx].size;
-  }
-
-  // keep adding up all the remaining files
-  for (size_t loop = start_index; loop < sr_end_idx; loop++) {
-    sr = &sorted_runs_[loop];
-    if (sr->being_compacted) {
-      // TODO with incremental compaction is supported, we might want to
-      // schedule some incremental compactions in parallel if needed.
-      char file_num_buf[kFormatFileNumberBufSize];
-      sr->Dump(file_num_buf, sizeof(file_num_buf), true);
-      ROCKS_LOG_BUFFER(
-          log_buffer_, "[%s] Universal: Possible candidate %s[%d] %s",
-          cf_name_.c_str(), file_num_buf, start_index,
-          " is already being compacted. No size amp reduction possible.\n");
-      return nullptr;
-    }
-    candidate_size += sr->compensated_file_size;
-    candidate_count++;
-  }
-  if (candidate_count == 0) {
-    return nullptr;
-  }
+  // percentage flexibility while reducing size amplification
+  const uint64_t ratio = mutable_cf_options_.compaction_options_universal
+                             .max_size_amplification_percent;

  // size amplification = percentage of additional size
  if (candidate_size * 100 < ratio * base_sr_size) {
@@ -893,7 +976,7 @@ Compaction* UniversalCompactionBuilder::PickCompactionToReduceSizeAmp() {
    }
  }
  return PickCompactionWithSortedRunRange(
-      start_index, sr_end_idx, CompactionReason::kUniversalSizeAmplification);
+      start_index, end_index, CompactionReason::kUniversalSizeAmplification);
 }

 Compaction* UniversalCompactionBuilder::PickIncrementalForReduceSizeAmp(
@@ -1442,4 +1525,3 @@ uint64_t UniversalCompactionBuilder::GetMaxOverlappingBytes() const {
  }
 }
 }  // namespace ROCKSDB_NAMESPACE
-
@@ -1202,20 +1202,126 @@ TEST_P(TieredCompactionTest, RangeBasedTieredStorageLevel) {
  ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel());
  ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0);
  ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
-
  ASSERT_EQ(
      options.statistics->getTickerCount(COMPACTION_RANGE_DEL_DROP_OBSOLETE),
      1);
+
+  // Tests that we only compact keys up to penultimate level
+  // that are within penultimate level input's internal key range.
+  {
+    MutexLock l(&mutex);
+    hot_start = Key(0);
+    hot_end = Key(100);
+  }
+  const Snapshot* temp_snap = db_->GetSnapshot();
+  // Key(0) and Key(1) here are inserted with higher sequence number
+  // than Key(0) and Key(1) inserted above.
+  // Only Key(0) in last level will be compacted up, not Key(1).
+  ASSERT_OK(Put(Key(0), "value" + std::to_string(0)));
+  ASSERT_OK(Put(Key(1), "value" + std::to_string(100)));
+  ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
+  ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel());
+  {
+    std::vector<LiveFileMetaData> metas;
+    db_->GetLiveFilesMetaData(&metas);
+    for (const auto& f : metas) {
+      if (f.temperature == Temperature::kUnknown) {
+        // Expect Key(0), Key(0), Key(1)
+        ASSERT_EQ(f.num_entries, 3);
+        ASSERT_EQ(f.smallestkey, Key(0));
+        ASSERT_EQ(f.largestkey, Key(1));
+      } else {
+        ASSERT_EQ(f.temperature, Temperature::kCold);
+        // Key(2)-Key(49) and Key(100).
+        ASSERT_EQ(f.num_entries, 50);
+      }
+    }
+  }
+  db_->ReleaseSnapshot(temp_snap);
 }

 INSTANTIATE_TEST_CASE_P(TieredCompactionTest, TieredCompactionTest,
                        testing::Bool());

+TEST_P(TieredCompactionTest, CheckInternalKeyRange) {
+  // When compacting keys from the last level to penultimate level,
+  // output to penultimate level should be within internal key range
+  // of input files from penultimate level.
+  // Set up:
+  // L5:
+  //  File 1: DeleteRange[1, 3)@4, File 2: [3@5, 100@6]
+  // L6:
+  // File 3: [2@1, 3@2], File 4: [50@3]
+  //
+  // When File 1 and File 3 are being compacted,
+  // Key(3) cannot be compacted up, otherwise it causes
+  // inconsistency where File 3's Key(3) has a lower sequence number
+  // than File 2's Key(3).
+  const int kNumLevels = 7;
+  auto options = CurrentOptions();
+  SetColdTemperature(options);
+  options.level_compaction_dynamic_level_bytes = true;
+  options.num_levels = kNumLevels;
+  options.statistics = CreateDBStatistics();
+  options.max_subcompactions = 10;
+  options.preclude_last_level_data_seconds = 10000;
+  DestroyAndReopen(options);
+  auto cmp = options.comparator;
+
+  std::string hot_start = Key(0);
+  std::string hot_end = Key(0);
+  SyncPoint::GetInstance()->SetCallBack(
+      "CompactionIterator::PrepareOutput.context", [&](void* arg) {
+        auto context = static_cast<PerKeyPlacementContext*>(arg);
+        context->output_to_penultimate_level =
+            cmp->Compare(context->key, hot_start) >= 0 &&
+            cmp->Compare(context->key, hot_end) < 0;
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+  // File 1
+  ASSERT_OK(Put(Key(2), "val2"));
+  ASSERT_OK(Put(Key(3), "val3"));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(6);
+  // File 2
+  ASSERT_OK(Put(Key(50), "val50"));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(6);
+
+  const Snapshot* snapshot = db_->GetSnapshot();
+  hot_end = Key(100);
+  std::string start = Key(1);
+  std::string end = Key(3);
+  ASSERT_OK(
+      db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), start, end));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(5);
+  // File 3
+  ASSERT_OK(Put(Key(3), "vall"));
+  ASSERT_OK(Put(Key(100), "val100"));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(5);
+  // Try to compact keys up
+  CompactRangeOptions cro;
+  cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
+  start = Key(1);
+  end = Key(2);
+  Slice begin_slice(start);
+  Slice end_slice(end);
+  ASSERT_OK(db_->CompactRange(cro, &begin_slice, &end_slice));
+  // Without internal key range checking, we get the following error:
+  // Corruption: force_consistency_checks(DEBUG): VersionBuilder: L5 has
+  // overlapping ranges: file #18 largest key: '6B6579303030303033' seq:102,
+  // type:1 vs. file #15 smallest key: '6B6579303030303033' seq:104, type:1
+  db_->ReleaseSnapshot(snapshot);
+}
+
 class PrecludeLastLevelTest : public DBTestBase {
 public:
  PrecludeLastLevelTest()
      : DBTestBase("preclude_last_level_test", /*env_do_fsync=*/false) {
    mock_clock_ = std::make_shared<MockSystemClock>(env_->GetSystemClock());
+    mock_clock_->SetCurrentTime(kMockStartTime);
    mock_env_ = std::make_unique<CompositeEnvWrapper>(env_, mock_clock_);
  }

@@ -1223,6 +1329,10 @@ class PrecludeLastLevelTest : public DBTestBase {
  std::unique_ptr<Env> mock_env_;
  std::shared_ptr<MockSystemClock> mock_clock_;

+  // Sufficient starting time that preserve time doesn't under-flow into
+  // pre-history
+  static constexpr uint32_t kMockStartTime = 10000000;
+
  void SetUp() override {
    mock_clock_->InstallTimedWaitFixCallback();
    SyncPoint::GetInstance()->SetCallBack(
@@ -1231,7 +1341,7 @@ class PrecludeLastLevelTest : public DBTestBase {
              reinterpret_cast<PeriodicTaskScheduler*>(arg);
          periodic_task_scheduler_ptr->TEST_OverrideTimer(mock_clock_.get());
        });
-    mock_clock_->SetCurrentTime(0);
+    mock_clock_->SetCurrentTime(kMockStartTime);
  }
 };

@@ -1249,11 +1359,6 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeManualCompaction) {
  options.num_levels = kNumLevels;
  DestroyAndReopen(options);

-  // pass some time first, otherwise the first a few keys write time are going
-  // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
-  dbfull()->TEST_WaitForPeriodicTaskRun(
-      [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); });
-
  int sst_num = 0;
  // Write files that are overlap and enough to trigger compaction
  for (; sst_num < kNumTrigger; sst_num++) {
@@ -1311,11 +1416,6 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeAutoCompaction) {
  options.num_levels = kNumLevels;
  DestroyAndReopen(options);

-  // pass some time first, otherwise the first a few keys write time are going
-  // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
-  dbfull()->TEST_WaitForPeriodicTaskRun(
-      [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); });
-
  int sst_num = 0;
  // Write files that are overlap and enough to trigger compaction
  for (; sst_num < kNumTrigger; sst_num++) {
@@ -1387,11 +1487,6 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimePartial) {
  options.num_levels = kNumLevels;
  DestroyAndReopen(options);

-  // pass some time first, otherwise the first a few keys write time are going
-  // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
-  dbfull()->TEST_WaitForPeriodicTaskRun(
-      [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); });
-
  int sst_num = 0;
  // Write files that are overlap and enough to trigger compaction
  for (; sst_num < kNumTrigger; sst_num++) {
@@ -1514,11 +1609,6 @@ TEST_F(PrecludeLastLevelTest, LastLevelOnlyCompactionPartial) {
  options.num_levels = kNumLevels;
  DestroyAndReopen(options);

-  // pass some time first, otherwise the first a few keys write time are going
-  // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
-  dbfull()->TEST_WaitForPeriodicTaskRun(
-      [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); });
-
  int sst_num = 0;
  // Write files that are overlap and enough to trigger compaction
  for (; sst_num < kNumTrigger; sst_num++) {
@@ -1592,11 +1682,6 @@ TEST_P(PrecludeLastLevelTestWithParms, LastLevelOnlyCompactionNoPreclude) {
  options.num_levels = kNumLevels;
  DestroyAndReopen(options);

-  // pass some time first, otherwise the first a few keys write time are going
-  // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
-  dbfull()->TEST_WaitForPeriodicTaskRun(
-      [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); });
-
  Random rnd(301);
  int sst_num = 0;
  // Write files that are overlap and enough to trigger compaction
@@ -1906,11 +1991,6 @@ TEST_F(PrecludeLastLevelTest, PartialPenultimateLevelCompaction) {
  options.num_levels = kNumLevels;
  DestroyAndReopen(options);

-  // pass some time first, otherwise the first a few keys write time are going
-  // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
-  dbfull()->TEST_WaitForPeriodicTaskRun(
-      [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); });
-
  Random rnd(301);

  for (int i = 0; i < 300; i++) {
@@ -2017,12 +2097,6 @@ TEST_F(PrecludeLastLevelTest, RangeDelsCauseFileEndpointsToOverlap) {
  options.target_file_size_base = kFileBytes;
  DestroyAndReopen(options);

-  // pass some time first, otherwise the first a few keys write time are going
-  // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
-  dbfull()->TEST_WaitForPeriodicTaskRun([&] {
-    mock_clock_->MockSleepForSeconds(static_cast<int>(kSecondsPerKey));
-  });
-
  // Flush an L0 file with the following contents (new to old):
  //
  // Range deletions [4, 6) [7, 8) [9, 11)
@@ -2139,7 +2213,6 @@ TEST_F(PrecludeLastLevelTest, RangeDelsCauseFileEndpointsToOverlap) {
  Close();
 }

-
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -165,6 +165,7 @@ void DoRandomIteraratorTest(DB* db, std::vector<std::string> source_strings,
    AssertItersEqual(iter.get(), result_iter.get());
    is_valid = iter->Valid();
  }
+  ASSERT_OK(iter->status());
 }

 class DoubleComparator : public Comparator {
@@ -138,6 +138,7 @@ TEST_F(DBBasicTest, ReadOnlyDB) {
      ASSERT_OK(iter->status());
      ++count;
    }
+    ASSERT_OK(iter->status());
    // Always expect two keys: "foo" and "bar"
    ASSERT_EQ(count, 2);
  };
@@ -1427,10 +1428,7 @@ TEST_P(DBMultiGetTestWithParam, MultiGetMultiCFMutex) {
  int retries = 0;
  bool last_try = false;
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::MultiGet::LastTry", [&](void* /*arg*/) {
-        last_try = true;
-        ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
-      });
+      "DBImpl::MultiGet::LastTry", [&](void* /*arg*/) { last_try = true; });
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "DBImpl::MultiGet::AfterRefSV", [&](void* /*arg*/) {
        if (last_try) {
@@ -1447,8 +1445,28 @@ TEST_P(DBMultiGetTestWithParam, MultiGetMultiCFMutex) {
          }
        }
      });
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({
+      {"DBImpl::MultiGet::AfterLastTryRefSV",
+       "DBMultiGetTestWithParam::MultiGetMultiCFMutex:BeforeCreateSV"},
+      {"DBMultiGetTestWithParam::MultiGetMultiCFMutex:AfterCreateSV",
+       "DBImpl::MultiGet::BeforeLastTryUnRefSV"},
+  });
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();

+  port::Thread create_sv_thread([this]() {
+    TEST_SYNC_POINT(
+        "DBMultiGetTestWithParam::MultiGetMultiCFMutex:BeforeCreateSV");
+    // Create a new SuperVersion for each column family after last_try
+    // of MultiGet ref SuperVersion and before unref it.
+    for (int i = 0; i < 8; ++i) {
+      ASSERT_OK(Put(i, "cf" + std::to_string(i) + "_key",
+                    "cf" + std::to_string(i) + "_val_after_last_try"));
+      ASSERT_OK(Flush(i));
+    }
+    TEST_SYNC_POINT(
+        "DBMultiGetTestWithParam::MultiGetMultiCFMutex:AfterCreateSV");
+  });
+
  std::vector<int> cfs;
  std::vector<std::string> keys;
  std::vector<std::string> values;
@@ -1460,6 +1478,7 @@ TEST_P(DBMultiGetTestWithParam, MultiGetMultiCFMutex) {

  values = MultiGet(cfs, keys, nullptr, std::get<0>(GetParam()),
                    std::get<1>(GetParam()));
+  create_sv_thread.join();
  ASSERT_TRUE(last_try);
  ASSERT_EQ(values.size(), 8);
  for (unsigned int j = 0; j < values.size(); ++j) {
@@ -1473,6 +1492,7 @@ TEST_P(DBMultiGetTestWithParam, MultiGetMultiCFMutex) {
            ->cfd();
    ASSERT_NE(cfd->TEST_GetLocalSV()->Get(), SuperVersion::kSVInUse);
  }
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }

 TEST_P(DBMultiGetTestWithParam, MultiGetMultiCFSnapshot) {
@@ -3635,10 +3655,12 @@ class DBBasicTestMultiGet : public DBTestBase {

    Status Insert(const Slice& key, Cache::ObjectPtr value,
                  const CacheItemHelper* helper, size_t charge,
-                  Handle** handle = nullptr,
-                  Priority priority = Priority::LOW) override {
+                  Handle** handle = nullptr, Priority priority = Priority::LOW,
+                  const Slice& compressed = Slice(),
+                  CompressionType type = kNoCompression) override {
      num_inserts_++;
-      return target_->Insert(key, value, helper, charge, handle, priority);
+      return target_->Insert(key, value, helper, charge, handle, priority,
+                             compressed, type);
    }

    Handle* Lookup(const Slice& key, const CacheItemHelper* helper,
@@ -4393,6 +4415,8 @@ TEST_F(DBBasicTest, ManifestWriteFailure) {
  options.create_if_missing = true;
  options.disable_auto_compactions = true;
  options.env = env_;
+  options.enable_blob_files = true;
+  options.blob_file_size = 0;
  DestroyAndReopen(options);
  ASSERT_OK(Put("foo", "bar"));
  ASSERT_OK(Flush());
@@ -4413,6 +4437,11 @@ TEST_F(DBBasicTest, ManifestWriteFailure) {
  SyncPoint::GetInstance()->ClearAllCallBacks();
  SyncPoint::GetInstance()->EnableProcessing();
  Reopen(options);
+  // The IO error was a mocked one from the `AfterSyncManifest` callback. The
+  // Flush's VersionEdit actually made it into the Manifest. So these keys can
+  // be read back. Read them to check all live sst files and blob files.
+  ASSERT_EQ("bar", Get("foo"));
+  ASSERT_EQ("value", Get("key"));
 }

 TEST_F(DBBasicTest, DestroyDefaultCfHandle) {
@@ -294,7 +294,9 @@ class ReadOnlyCacheWrapper : public CacheWrapper {

  Status Insert(const Slice& /*key*/, Cache::ObjectPtr /*value*/,
                const CacheItemHelper* /*helper*/, size_t /*charge*/,
-                Handle** /*handle*/, Priority /*priority*/) override {
+                Handle** /*handle*/, Priority /*priority*/,
+                const Slice& /*compressed*/,
+                CompressionType /*type*/) override {
    return Status::NotSupported();
  }
 };
@@ -387,6 +389,7 @@ TEST_F(DBBlockCacheTest, FillCacheAndIterateDB) {
  while (iter->Valid()) {
    iter->Next();
  }
+  ASSERT_OK(iter->status());
  delete iter;
  iter = nullptr;
 }
@@ -628,13 +631,15 @@ class MockCache : public LRUCache {

  Status Insert(const Slice& key, Cache::ObjectPtr value,
                const Cache::CacheItemHelper* helper, size_t charge,
-                Handle** handle, Priority priority) override {
+                Handle** handle, Priority priority, const Slice& compressed,
+                CompressionType type) override {
    if (priority == Priority::LOW) {
      low_pri_insert_count++;
    } else {
      high_pri_insert_count++;
    }
-    return LRUCache::Insert(key, value, helper, charge, handle, priority);
+    return LRUCache::Insert(key, value, helper, charge, handle, priority,
+                            compressed, type);
  }
 };

@@ -1775,6 +1775,64 @@ TEST_F(DBBloomFilterTest, ContextCustomFilterPolicy) {
  }
 }

+TEST_F(DBBloomFilterTest, MutatingRibbonFilterPolicy) {
+  // Test that RibbonFilterPolicy has a mutable bloom_before_level fields that
+  // can be updated through SetOptions
+
+  Options options = CurrentOptions();
+  options.statistics = CreateDBStatistics();
+  auto& stats = *options.statistics;
+  BlockBasedTableOptions table_options;
+  // First config forces Bloom filter, to establish a baseline before
+  // SetOptions().
+  table_options.filter_policy.reset(NewRibbonFilterPolicy(10, INT_MAX));
+  double expected_bpk = 10.0;
+  // Other configs to try, with approx expected bits per key
+  std::vector<std::pair<std::string, double>> configs = {{"-1", 7.0},
+                                                         {"0", 10.0}};
+
+  table_options.cache_index_and_filter_blocks = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+
+  ASSERT_OK(TryReopen(options));
+
+  char v[] = "a";
+
+  for (;; ++(v[0])) {
+    const int maxKey = 8000;
+    for (int i = 0; i < maxKey; i++) {
+      ASSERT_OK(Put(Key(i), v));
+    }
+    ASSERT_OK(Flush());
+
+    for (int i = 0; i < maxKey; i++) {
+      ASSERT_EQ(Get(Key(i)), v);
+    }
+
+    uint64_t filter_bytes =
+        stats.getAndResetTickerCount(BLOCK_CACHE_FILTER_BYTES_INSERT);
+
+    EXPECT_NEAR(filter_bytes * 8.0 / maxKey, expected_bpk, 0.3);
+
+    if (configs.empty()) {
+      break;
+    }
+
+    ASSERT_OK(
+        db_->SetOptions({{"table_factory.filter_policy.bloom_before_level",
+                          configs.back().first}}));
+
+    // Ensure original object is mutated
+    std::string val;
+    ASSERT_OK(
+        table_options.filter_policy->GetOption({}, "bloom_before_level", &val));
+    ASSERT_EQ(configs.back().first, val);
+
+    expected_bpk = configs.back().second;
+    configs.pop_back();
+  }
+}
+
 class SliceTransformLimitedDomain : public SliceTransform {
  const char* Name() const override { return "SliceTransformLimitedDomain"; }

@@ -1847,6 +1905,7 @@ TEST_F(DBBloomFilterTest, PrefixExtractorWithFilter2) {
  for (iter->Seek("zzzzz_AAAA"); iter->Valid(); iter->Next()) {
    iter_res.emplace_back(iter->value().ToString());
  }
+  ASSERT_OK(iter->status());

  std::vector<std::string> expected_res = {"val1", "val2", "val3", "val4"};
  ASSERT_EQ(iter_res, expected_res);
@@ -166,9 +166,12 @@ class ChangeFilter : public CompactionFilter {
 class KeepFilterFactory : public CompactionFilterFactory {
 public:
  explicit KeepFilterFactory(bool check_context = false,
-                             bool check_context_cf_id = false)
+                             bool check_context_cf_id = false,
+                             bool check_context_input_table_properties = false)
      : check_context_(check_context),
        check_context_cf_id_(check_context_cf_id),
+        check_context_input_table_properties_(
+            check_context_input_table_properties),
        compaction_filter_created_(false) {}

  std::unique_ptr<CompactionFilter> CreateCompactionFilter(
@@ -176,6 +179,11 @@ class KeepFilterFactory : public CompactionFilterFactory {
    if (check_context_) {
      EXPECT_EQ(expect_full_compaction_.load(), context.is_full_compaction);
      EXPECT_EQ(expect_manual_compaction_.load(), context.is_manual_compaction);
+      EXPECT_EQ(expect_input_start_level_.load(), context.input_start_level);
+    }
+    if (check_context_input_table_properties_) {
+      EXPECT_TRUE(expect_input_table_properties_ ==
+                  context.input_table_properties);
    }
    if (check_context_cf_id_) {
      EXPECT_EQ(expect_cf_id_.load(), context.column_family_id);
@@ -189,9 +197,15 @@ class KeepFilterFactory : public CompactionFilterFactory {
  const char* Name() const override { return "KeepFilterFactory"; }
  bool check_context_;
  bool check_context_cf_id_;
+  // `check_context_input_table_properties_` can be true only when access to
+  // `expect_input_table_properties_` is syncronized since we can't have
+  // std::atomic<TablePropertiesCollection> unfortunately
+  bool check_context_input_table_properties_;
  std::atomic_bool expect_full_compaction_;
  std::atomic_bool expect_manual_compaction_;
  std::atomic<uint32_t> expect_cf_id_;
+  std::atomic<int> expect_input_start_level_;
+  TablePropertiesCollection expect_input_table_properties_;
  bool compaction_filter_created_;
 };

@@ -654,7 +668,9 @@ TEST_F(DBTestCompactionFilter, CompactionFilterWithMergeOperator) {
 }

 TEST_F(DBTestCompactionFilter, CompactionFilterContextManual) {
-  KeepFilterFactory* filter = new KeepFilterFactory(true, true);
+  KeepFilterFactory* filter = new KeepFilterFactory(
+      true /* check_context */, true /* check_context_cf_id */,
+      true /* check_context_input_table_properties */);

  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleUniversal;
@@ -662,8 +678,9 @@ TEST_F(DBTestCompactionFilter, CompactionFilterContextManual) {
  options.compression = kNoCompression;
  options.level0_file_num_compaction_trigger = 8;
  Reopen(options);
+  const int kNumFiles = 3;
  int num_keys_per_file = 400;
-  for (int j = 0; j < 3; j++) {
+  for (int j = 0; j < kNumFiles; j++) {
    // Write several keys.
    const std::string value(10, 'x');
    for (int i = 0; i < num_keys_per_file; i++) {
@@ -683,6 +700,11 @@ TEST_F(DBTestCompactionFilter, CompactionFilterContextManual) {
  filter->expect_manual_compaction_.store(true);
  filter->expect_full_compaction_.store(true);
  filter->expect_cf_id_.store(0);
+  filter->expect_input_start_level_.store(0);
+  ASSERT_OK(dbfull()->GetPropertiesOfAllTables(
+      &filter->expect_input_table_properties_));
+  ASSERT_TRUE(filter->expect_input_table_properties_.size() == kNumFiles);
+
  ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
  ASSERT_EQ(cfilter_count, 700);
  ASSERT_EQ(NumSortedRuns(0), 1);
@@ -783,6 +805,7 @@ TEST_F(DBTestCompactionFilter, CompactionFilterIgnoreSnapshot) {
      count++;
      iter->Next();
    }
+    ASSERT_OK(iter->status());
    ASSERT_EQ(count, 6);
    read_options.snapshot = nullptr;
    std::unique_ptr<Iterator> iter1(db_->NewIterator(read_options));
@@ -793,6 +816,7 @@ TEST_F(DBTestCompactionFilter, CompactionFilterIgnoreSnapshot) {
      count++;
      iter1->Next();
    }
+    ASSERT_OK(iter1->status());
    // We have deleted 10 keys from 40 using the compaction filter
    //  Keys 6-9 before the snapshot and 100-105 after the snapshot
    ASSERT_EQ(count, 30);
@@ -19,7 +19,7 @@
 #include "rocksdb/concurrent_task_limiter.h"
 #include "rocksdb/experimental.h"
 #include "rocksdb/sst_file_writer.h"
-#include "rocksdb/utilities/convenience.h"
+#include "test_util/mock_time_env.h"
 #include "test_util/sync_point.h"
 #include "test_util/testutil.h"
 #include "util/concurrent_task_limiter_impl.h"
@@ -2673,6 +2673,7 @@ TEST_P(DBCompactionTestWithParam, ConvertCompactionStyle) {
    keys_in_db.append(iter->key().ToString());
    keys_in_db.push_back(',');
  }
+  ASSERT_OK(iter->status());
  delete iter;

  std::string expected_keys;
@@ -3536,12 +3537,12 @@ TEST_P(DBCompactionWaitForCompactTest,
  // (has_unpersisted_data_ true) Check to make sure there's no extra L0 file
  // created from WAL. Re-opening DB won't trigger any flush or compaction

-  int compaction_finished = 0;
+  std::atomic_int compaction_finished = 0;
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "DBImpl::BackgroundCompaction:Finish",
      [&](void*) { compaction_finished++; });

-  int flush_finished = 0;
+  std::atomic_int flush_finished = 0;
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "FlushJob::End", [&](void*) { flush_finished++; });

@@ -4793,9 +4794,9 @@ TEST_F(DBCompactionTest, LevelTtlCascadingCompactions) {
      ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
          "VersionEdit::EncodeTo:VarintOldestAncesterTime", [&](void* arg) {
            if (if_restart && if_open_all_files) {
-              std::string* encoded_fieled = static_cast<std::string*>(arg);
-              *encoded_fieled = "";
-              PutVarint64(encoded_fieled, 0);
+              std::string* encoded_field = static_cast<std::string*>(arg);
+              *encoded_field = "";
+              PutVarint64(encoded_field, 0);
            }
          });
      ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
@@ -4941,9 +4942,9 @@ TEST_F(DBCompactionTest, LevelPeriodicCompaction) {
      ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
          "VersionEdit::EncodeTo:VarintFileCreationTime", [&](void* arg) {
            if (if_restart && if_open_all_files) {
-              std::string* encoded_fieled = static_cast<std::string*>(arg);
-              *encoded_fieled = "";
-              PutVarint64(encoded_fieled, 0);
+              std::string* encoded_field = static_cast<std::string*>(arg);
+              *encoded_field = "";
+              PutVarint64(encoded_field, 0);
            }
          });
      ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
@@ -5019,6 +5020,106 @@ TEST_F(DBCompactionTest, LevelPeriodicCompaction) {
  }
 }

+TEST_F(DBCompactionTest, LevelPeriodicCompactionOffpeak) {
+  // This test simply checks if offpeak adjustment works in Leveled
+  // Compactions. For testing offpeak periodic compactions in various
+  // scenarios, please refer to
+  // DBTestUniversalCompaction2::PeriodicCompactionOffpeak
+  constexpr int kNumKeysPerFile = 32;
+  constexpr int kNumLevelFiles = 2;
+  constexpr int kValueSize = 100;
+  constexpr int kSecondsPerDay = 86400;
+  constexpr int kSecondsPerHour = 3600;
+  constexpr int kSecondsPerMinute = 60;
+
+  for (bool if_restart : {false, true}) {
+    SCOPED_TRACE("if_restart=" + std::to_string(if_restart));
+    Options options = CurrentOptions();
+    options.ttl = 0;
+    options.periodic_compaction_seconds = 5 * kSecondsPerDay;  // 5 days
+    // In the case where all files are opened and doing DB restart
+    // forcing the file creation time in manifest file to be 0 to
+    // simulate the case of reading from an old version.
+    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+        "VersionEdit::EncodeTo:VarintFileCreationTime", [&](void* arg) {
+          if (if_restart) {
+            std::string* encoded_field = static_cast<std::string*>(arg);
+            *encoded_field = "";
+            PutVarint64(encoded_field, 0);
+          }
+        });
+    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+    // Just to add some extra random days to current time
+    Random rnd(test::RandomSeed());
+    int days = rnd.Uniform(100);
+
+    int periodic_compactions = 0;
+    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+        "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) {
+          Compaction* compaction = static_cast<Compaction*>(arg);
+          auto compaction_reason = compaction->compaction_reason();
+          if (compaction_reason == CompactionReason::kPeriodicCompaction) {
+            periodic_compactions++;
+          }
+        });
+
+    // Starting at 12:15AM
+    int now_hour = 0;
+    int now_minute = 15;
+    auto mock_clock = std::make_shared<MockSystemClock>(env_->GetSystemClock());
+    auto mock_env = std::make_unique<CompositeEnvWrapper>(env_, mock_clock);
+    options.env = mock_env.get();
+    mock_clock->SetCurrentTime(days * kSecondsPerDay +
+                               now_hour * kSecondsPerHour +
+                               now_minute * kSecondsPerMinute);
+    // Offpeak is set from 12:30AM to 4:30AM
+    options.daily_offpeak_time_utc = "00:30-04:30";
+    Reopen(options);
+
+    for (int i = 0; i < kNumLevelFiles; ++i) {
+      for (int j = 0; j < kNumKeysPerFile; ++j) {
+        ASSERT_OK(
+            Put(Key(i * kNumKeysPerFile + j), rnd.RandomString(kValueSize)));
+      }
+      ASSERT_OK(Flush());
+    }
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
+    ASSERT_EQ("2", FilesPerLevel());
+    ASSERT_EQ(0, periodic_compactions);
+
+    // Move clock forward by 1 hour. Now at 1:15AM Day 0. No compaction.
+    mock_clock->MockSleepForSeconds(1 * kSecondsPerHour);
+    ASSERT_OK(Put("a", "1"));
+    ASSERT_OK(Flush());
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
+    // Assert that the files stay in the same level
+    ASSERT_EQ("3", FilesPerLevel());
+    ASSERT_EQ(0, periodic_compactions);
+    MoveFilesToLevel(1);
+    ASSERT_EQ("0,3", FilesPerLevel());
+
+    // Move clock forward by 4 days and check if it triggers periodic
+    // comapaction at 1:15AM Day 4. Files created on Day 0 at 12:15AM is
+    // expected to expire before the offpeak starts next day at 12:30AM
+    mock_clock->MockSleepForSeconds(4 * kSecondsPerDay);
+    ASSERT_OK(Put("b", "2"));
+    if (if_restart) {
+      Reopen(options);
+    } else {
+      ASSERT_OK(Flush());
+    }
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
+    ASSERT_EQ("1,3", FilesPerLevel());
+    // The two old files go through the periodic compaction process
+    ASSERT_EQ(2, periodic_compactions);
+
+    Destroy(options);
+
+    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+  }
+}
+
 TEST_F(DBCompactionTest, LevelPeriodicCompactionWithOldDB) {
  // This test makes sure that periodic compactions are working with a DB
  // where file_creation_time of some files is 0.
@@ -6025,23 +6126,18 @@ TEST_P(DBCompactionDirectIOTest, DirectIO) {
  options.use_direct_io_for_flush_and_compaction = GetParam();
  options.env = MockEnv::Create(Env::Default());
  Reopen(options);
-  bool readahead = false;
  SyncPoint::GetInstance()->SetCallBack(
      "CompactionJob::OpenCompactionOutputFile", [&](void* arg) {
        bool* use_direct_writes = static_cast<bool*>(arg);
        ASSERT_EQ(*use_direct_writes,
                  options.use_direct_io_for_flush_and_compaction);
      });
-  if (options.use_direct_io_for_flush_and_compaction) {
-    SyncPoint::GetInstance()->SetCallBack(
-        "SanitizeOptions:direct_io", [&](void* /*arg*/) { readahead = true; });
-  }
  SyncPoint::GetInstance()->EnableProcessing();
  CreateAndReopenWithCF({"pikachu"}, options);
  MakeTables(3, "p", "q", 1);
  ASSERT_EQ("1,1,1", FilesPerLevel(1));
  Compact(1, "p", "q");
-  ASSERT_EQ(readahead, options.use_direct_reads);
+  ASSERT_EQ(false, options.use_direct_reads);
  ASSERT_EQ("0,0,1", FilesPerLevel(1));
  Destroy(options);
  delete options.env;
@@ -10042,6 +10138,189 @@ TEST_F(DBCompactionTest, VerifyRecordCount) {
      "processed.";
  ASSERT_TRUE(std::strstr(s.getState(), expect));
 }
+
+TEST_F(DBCompactionTest, ErrorWhenReadFileHead) {
+  // This is to test a bug that is fixed in
+  // https://github.com/facebook/rocksdb/pull/11782.
+  //
+  // Ingest error when reading from a file with offset = 0,
+  // See if compaction handles it correctly.
+  Options opts = CurrentOptions();
+  opts.num_levels = 7;
+  opts.compression = kNoCompression;
+  DestroyAndReopen(opts);
+
+  // Set up LSM
+  // L5: F1 [key0, key99], F2 [key100, key199]
+  // L6:        F3 [key50, key149]
+  Random rnd(301);
+  const int kValLen = 100;
+  for (int error_file = 1; error_file <= 3; ++error_file) {
+    for (int i = 50; i < 150; ++i) {
+      ASSERT_OK(Put(Key(i), rnd.RandomString(kValLen)));
+    }
+    ASSERT_OK(Flush());
+    MoveFilesToLevel(6);
+
+    std::vector<std::string> values;
+    for (int i = 0; i < 100; ++i) {
+      values.emplace_back(rnd.RandomString(kValLen));
+      ASSERT_OK(Put(Key(i), values.back()));
+    }
+    ASSERT_OK(Flush());
+    MoveFilesToLevel(5);
+
+    for (int i = 100; i < 200; ++i) {
+      values.emplace_back(rnd.RandomString(kValLen));
+      ASSERT_OK(Put(Key(i), values.back()));
+    }
+    ASSERT_OK(Flush());
+    MoveFilesToLevel(5);
+
+    ASSERT_EQ(2, NumTableFilesAtLevel(5));
+    ASSERT_EQ(1, NumTableFilesAtLevel(6));
+
+    std::atomic_int count = 0;
+    SyncPoint::GetInstance()->SetCallBack(
+        "RandomAccessFileReader::Read::BeforeReturn",
+        [&count, &error_file](void* pair_ptr) {
+          auto p =
+              reinterpret_cast<std::pair<std::string*, IOStatus*>*>(pair_ptr);
+          int cur = ++count;
+          if (cur == error_file) {
+            IOStatus* io_s = p->second;
+            *io_s = IOStatus::IOError();
+            io_s->SetRetryable(true);
+          }
+        });
+    SyncPoint::GetInstance()->EnableProcessing();
+
+    Status s = db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
+    // Failed compaction should not lose data.
+    PinnableSlice slice;
+    for (int i = 0; i < 200; ++i) {
+      ASSERT_OK(Get(Key(i), &slice));
+      ASSERT_EQ(slice, values[i]);
+    }
+    ASSERT_NOK(s);
+    ASSERT_TRUE(s.IsIOError());
+    s = db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
+    ASSERT_OK(s);
+    for (int i = 0; i < 200; ++i) {
+      ASSERT_OK(Get(Key(i), &slice));
+      ASSERT_EQ(slice, values[i]);
+    }
+    SyncPoint::GetInstance()->DisableProcessing();
+    DestroyAndReopen(opts);
+  }
+}
+
+TEST_F(DBCompactionTest, ReleaseCompactionDuringManifestWrite) {
+  // Tests the fix for issue #10257.
+  // Compactions are released in LogAndApply() so that picking a compaction
+  // from the new Version won't see these compactions as registered.
+  Options options = CurrentOptions();
+  options.compaction_style = kCompactionStyleLevel;
+  // Make sure we can run multiple compactions at the same time.
+  env_->SetBackgroundThreads(3, Env::Priority::LOW);
+  env_->SetBackgroundThreads(3, Env::Priority::BOTTOM);
+  options.max_background_compactions = 3;
+  options.num_levels = 4;
+  DestroyAndReopen(options);
+  Random rnd(301);
+
+  // Construct the following LSM
+  // L2:  [K1-K2]  [K10-K11]      [k100-k101]
+  // L3:  [K1]     [K10]          [k100]
+  // We will have 3 threads to run 3 manual compactions.
+  // The first thread that writes to MANIFEST will not finish
+  // until the next two threads enters LogAndApply() and form
+  // a write group.
+  // We check that compactions are all released after the first
+  // thread from the write group finishes writing to MANIFEST.
+
+  // L3
+  ASSERT_OK(Put(Key(1), rnd.RandomString(20)));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(3);
+  ASSERT_OK(Put(Key(10), rnd.RandomString(20)));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(3);
+  ASSERT_OK(Put(Key(100), rnd.RandomString(20)));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(3);
+  // L2
+  ASSERT_OK(Put(Key(100), rnd.RandomString(20)));
+  ASSERT_OK(Put(Key(101), rnd.RandomString(20)));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(2);
+  ASSERT_OK(Put(Key(1), rnd.RandomString(20)));
+  ASSERT_OK(Put(Key(2), rnd.RandomString(20)));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(2);
+  ASSERT_OK(Put(Key(10), rnd.RandomString(20)));
+  ASSERT_OK(Put(Key(11), rnd.RandomString(20)));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(2);
+
+  ASSERT_EQ(NumTableFilesAtLevel(1), 0);
+  ASSERT_EQ(NumTableFilesAtLevel(2), 3);
+  ASSERT_EQ(NumTableFilesAtLevel(3), 3);
+
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+  std::atomic_int count = 0;
+  SyncPoint::GetInstance()->SetCallBack(
+      "VersionSet::LogAndApply:BeforeWriterWaiting", [&](void*) {
+        int c = count.fetch_add(1);
+        if (c == 2) {
+          TEST_SYNC_POINT("all threads to enter LogAndApply");
+        }
+      });
+  SyncPoint::GetInstance()->LoadDependency(
+      {{"all threads to enter LogAndApply",
+        "VersionSet::LogAndApply:WriteManifestStart"}});
+  // Verify that compactions are released after writing to MANIFEST
+  std::atomic_int after_compact_count = 0;
+  SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::BackgroundCompaction:AfterCompaction", [&](void* ptr) {
+        int c = after_compact_count.fetch_add(1);
+        if (c > 0) {
+          ColumnFamilyData* cfd = (ColumnFamilyData*)(ptr);
+          ASSERT_TRUE(
+              cfd->compaction_picker()->compactions_in_progress()->empty());
+        }
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  std::vector<std::thread> threads;
+  threads.emplace_back(std::thread([&]() {
+    std::string k1_str = Key(1);
+    std::string k2_str = Key(2);
+    Slice k1 = k1_str;
+    Slice k2 = k2_str;
+    ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &k1, &k2));
+  }));
+  threads.emplace_back(std::thread([&]() {
+    std::string k10_str = Key(10);
+    std::string k11_str = Key(11);
+    Slice k10 = k10_str;
+    Slice k11 = k11_str;
+    ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &k10, &k11));
+  }));
+  std::string k100_str = Key(100);
+  std::string k101_str = Key(101);
+  Slice k100 = k100_str;
+  Slice k101 = k101_str;
+  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &k100, &k101));
+
+  for (auto& thread : threads) {
+    thread.join();
+  }
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+}
+
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -121,7 +121,7 @@ Status DBImpl::GetSortedWalFiles(VectorLogPtr& files) {

  // DisableFileDeletions / EnableFileDeletions not supported in read-only DB
  if (deletions_disabled.ok()) {
-    Status s2 = EnableFileDeletions(/*force*/ false);
+    Status s2 = EnableFileDeletions(/*force=*/false);
    assert(s2.ok());
    s2.PermitUncheckedError();
  } else {
@@ -1377,6 +1377,7 @@ TEST_F(DBFlushTest, MemPurgeDeleteAndDeleteRange) {
        ASSERT_EQ(value, NOT_FOUND);
      count++;
    }
+    ASSERT_OK(iter->status());

    // Expected count here is 3: KEY3, KEY4, KEY5.
    ASSERT_EQ(count, EXPECTED_COUNT_FORLOOP);
@@ -3193,6 +3194,279 @@ INSTANTIATE_TEST_CASE_P(DBFlushDirectIOTest, DBFlushDirectIOTest,

 INSTANTIATE_TEST_CASE_P(DBAtomicFlushTest, DBAtomicFlushTest, testing::Bool());

+TEST_F(DBFlushTest, NonAtomicFlushRollbackPendingFlushes) {
+  // Fix a bug in when atomic_flush=false.
+  // The bug can happen as follows:
+  // Start Flush0 for memtable M0 to SST0
+  // Start Flush1 for memtable M1 to SST1
+  // Flush1 returns OK, but don't install to MANIFEST and let whoever flushes
+  // M0 to take care of it
+  // Flush0 finishes with a retryable IOError
+  //   - It rollbacks M0, (incorrectly) not M1
+  //   - Deletes SST1 and SST2
+  //
+  // Auto-recovery will start Flush2 for M0, it does not pick up M1 since it
+  // thinks that M1 is flushed
+  // Flush2 writes SST3 and finishes OK, tries to install SST3 and SST2
+  // Error opening SST2 since it's already deleted
+  //
+  // The fix is to let Flush0 also rollback M1.
+  Options opts = CurrentOptions();
+  opts.atomic_flush = false;
+  opts.memtable_factory.reset(test::NewSpecialSkipListFactory(1));
+  opts.max_write_buffer_number = 64;
+  opts.max_background_flushes = 4;
+  env_->SetBackgroundThreads(4, Env::HIGH);
+  DestroyAndReopen(opts);
+  std::atomic_int flush_count = 0;
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->SetCallBack(
+      "FlushJob::WriteLevel0Table:s", [&](void* s_ptr) {
+        int c = flush_count.fetch_add(1);
+        if (c == 0) {
+          Status* s = (Status*)(s_ptr);
+          IOStatus io_error = IOStatus::IOError("injected foobar");
+          io_error.SetRetryable(true);
+          *s = io_error;
+          TEST_SYNC_POINT("Let mem1 flush start");
+          TEST_SYNC_POINT("Wait for mem1 flush to finish");
+        }
+      });
+  SyncPoint::GetInstance()->LoadDependency(
+      {{"Let mem1 flush start", "Mem1 flush starts"},
+       {"DBImpl::BGWorkFlush:done", "Wait for mem1 flush to finish"},
+       {"RecoverFromRetryableBGIOError:RecoverSuccess",
+        "Wait for error recover"}});
+  // Need first flush to wait for the second flush to finish
+  SyncPoint::GetInstance()->EnableProcessing();
+  ASSERT_OK(Put(Key(1), "val1"));
+  // trigger bg flush mem0
+  ASSERT_OK(Put(Key(2), "val2"));
+  TEST_SYNC_POINT("Mem1 flush starts");
+  // trigger bg flush mem1
+  ASSERT_OK(Put(Key(3), "val3"));
+
+  TEST_SYNC_POINT("Wait for error recover");
+  ASSERT_EQ(1, NumTableFilesAtLevel(0));
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+  SyncPoint::GetInstance()->DisableProcessing();
+}
+
+TEST_F(DBFlushTest, AbortNonAtomicFlushWhenBGError) {
+  // Fix a bug in when atomic_flush=false.
+  // The bug can happen as follows:
+  // Start Flush0 for memtable M0 to SST0
+  // Start Flush1 for memtable M1 to SST1
+  // Flush1 returns OK, but doesn't install output MANIFEST and let whoever
+  // flushes M0 to take care of it
+  // Start Flush2 for memtable M2 to SST2
+  // Flush0 finishes with a retryable IOError
+  //   - It rollbacks M0 AND M1
+  //   - Deletes SST1 and SST2
+  // Flush2 finishes, does not rollback M2,
+  //  - releases the pending file number that keeps SST2 alive
+  //  - deletes SST2
+  //
+  // Then auto-recovery starts, error opening SST2 when try to install
+  // flush result
+  //
+  // The fix is to let Flush2 rollback M2 if it finds that
+  // there is a background error.
+  Options opts = CurrentOptions();
+  opts.atomic_flush = false;
+  opts.memtable_factory.reset(test::NewSpecialSkipListFactory(1));
+  opts.max_write_buffer_number = 64;
+  opts.max_background_flushes = 4;
+  env_->SetBackgroundThreads(4, Env::HIGH);
+  DestroyAndReopen(opts);
+  std::atomic_int flush_count = 0;
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->SetCallBack(
+      "FlushJob::WriteLevel0Table:s", [&](void* s_ptr) {
+        int c = flush_count.fetch_add(1);
+        if (c == 0) {
+          Status* s = (Status*)(s_ptr);
+          IOStatus io_error = IOStatus::IOError("injected foobar");
+          io_error.SetRetryable(true);
+          *s = io_error;
+          TEST_SYNC_POINT("Let mem1 flush start");
+          TEST_SYNC_POINT("Wait for mem1 flush to finish");
+
+          TEST_SYNC_POINT("Let mem2 flush start");
+          TEST_SYNC_POINT("Wait for mem2 to start writing table");
+        }
+      });
+
+  SyncPoint::GetInstance()->SetCallBack(
+      "FlushJob::WriteLevel0Table", [&](void* mems) {
+        autovector<MemTable*>* mems_ptr = (autovector<MemTable*>*)mems;
+        if ((*mems_ptr)[0]->GetID() == 3) {
+          TEST_SYNC_POINT("Mem2 flush starts writing table");
+          TEST_SYNC_POINT("Mem2 flush waits until rollback");
+        }
+      });
+  SyncPoint::GetInstance()->LoadDependency(
+      {{"Let mem1 flush start", "Mem1 flush starts"},
+       {"DBImpl::BGWorkFlush:done", "Wait for mem1 flush to finish"},
+       {"Let mem2 flush start", "Mem2 flush starts"},
+       {"Mem2 flush starts writing table",
+        "Wait for mem2 to start writing table"},
+       {"RollbackMemtableFlush", "Mem2 flush waits until rollback"},
+       {"RecoverFromRetryableBGIOError:RecoverSuccess",
+        "Wait for error recover"}});
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  ASSERT_OK(Put(Key(1), "val1"));
+  // trigger bg flush mem0
+  ASSERT_OK(Put(Key(2), "val2"));
+  TEST_SYNC_POINT("Mem1 flush starts");
+  // trigger bg flush mem1
+  ASSERT_OK(Put(Key(3), "val3"));
+
+  TEST_SYNC_POINT("Mem2 flush starts");
+  ASSERT_OK(Put(Key(4), "val4"));
+
+  TEST_SYNC_POINT("Wait for error recover");
+  // Recovery flush writes 3 memtables together into 1 file.
+  ASSERT_EQ(1, NumTableFilesAtLevel(0));
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+  SyncPoint::GetInstance()->DisableProcessing();
+}
+
+TEST_F(DBFlushTest, NonAtomicNormalFlushAbortWhenBGError) {
+  Options opts = CurrentOptions();
+  opts.atomic_flush = false;
+  opts.memtable_factory.reset(test::NewSpecialSkipListFactory(1));
+  opts.max_write_buffer_number = 64;
+  opts.max_background_flushes = 1;
+  env_->SetBackgroundThreads(2, Env::HIGH);
+  DestroyAndReopen(opts);
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+  SyncPoint::GetInstance()->DisableProcessing();
+  std::atomic_int flush_write_table_count = 0;
+  SyncPoint::GetInstance()->SetCallBack(
+      "FlushJob::WriteLevel0Table:s", [&](void* s_ptr) {
+        int c = flush_write_table_count.fetch_add(1);
+        if (c == 0) {
+          Status* s = (Status*)(s_ptr);
+          IOStatus io_error = IOStatus::IOError("injected foobar");
+          io_error.SetRetryable(true);
+          *s = io_error;
+        }
+      });
+
+  SyncPoint::GetInstance()->EnableProcessing();
+  SyncPoint::GetInstance()->LoadDependency(
+      {{"Let error recovery start",
+        "RecoverFromRetryableBGIOError:BeforeStart"},
+       {"RecoverFromRetryableBGIOError:RecoverSuccess",
+        "Wait for error recover"}});
+
+  ASSERT_OK(Put(Key(1), "val1"));
+  // trigger bg flush0 for mem0
+  ASSERT_OK(Put(Key(2), "val2"));
+  // Not checking status since this wait can finish before flush starts.
+  dbfull()->TEST_WaitForFlushMemTable().PermitUncheckedError();
+
+  // trigger bg flush1 for mem1, should see bg error and abort
+  // before picking a memtable to flush
+  ASSERT_OK(Put(Key(3), "val3"));
+  ASSERT_NOK(dbfull()->TEST_WaitForFlushMemTable());
+  ASSERT_EQ(0, NumTableFilesAtLevel(0));
+
+  TEST_SYNC_POINT("Let error recovery start");
+  TEST_SYNC_POINT("Wait for error recover");
+  // Recovery flush writes 2 memtables together into 1 file.
+  ASSERT_EQ(1, NumTableFilesAtLevel(0));
+  // 1 for flush 0 and 1 for recovery flush
+  ASSERT_EQ(2, flush_write_table_count);
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+  SyncPoint::GetInstance()->DisableProcessing();
+}
+
+TEST_F(DBFlushTest, DBStuckAfterAtomicFlushError) {
+  // Test for a bug with atomic flush where DB can become stuck
+  // after a flush error. A repro timeline:
+  //
+  // Start Flush0 for mem0
+  // Start Flush1 for mem1
+  // Now Flush1 will wait for Flush0 to install mem0
+  // Flush0 finishes with retryable IOError, rollbacks mem0
+  // Resume starts and waits for background job to finish, i.e., Flush1
+  // Fill memtable again, trigger Flush2 for mem0
+  // Flush2 will get error status, and not rollback mem0, see code in
+  // https://github.com/facebook/rocksdb/blob/b927ba5936216861c2c35ab68f50ba4a78e65747/db/db_impl/db_impl_compaction_flush.cc#L725
+  //
+  // DB is stuck since mem0 can never be picked now
+  //
+  // The fix is to rollback mem0 in Flush2, and let Flush1 also abort upon
+  // background error besides waiting for older memtables to be installed.
+  // The recovery flush in this case should pick up all memtables
+  // and write them to a single L0 file.
+  Options opts = CurrentOptions();
+  opts.atomic_flush = true;
+  opts.memtable_factory.reset(test::NewSpecialSkipListFactory(1));
+  opts.max_write_buffer_number = 64;
+  opts.max_background_flushes = 4;
+  env_->SetBackgroundThreads(4, Env::HIGH);
+  DestroyAndReopen(opts);
+
+  std::atomic_int flush_count = 0;
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->SetCallBack(
+      "FlushJob::WriteLevel0Table:s", [&](void* s_ptr) {
+        int c = flush_count.fetch_add(1);
+        if (c == 0) {
+          Status* s = (Status*)(s_ptr);
+          IOStatus io_error = IOStatus::IOError("injected foobar");
+          io_error.SetRetryable(true);
+          *s = io_error;
+          TEST_SYNC_POINT("Let flush for mem1 start");
+          // Wait for Flush1 to start waiting to install flush result
+          TEST_SYNC_POINT("Wait for flush for mem1");
+        }
+      });
+  SyncPoint::GetInstance()->LoadDependency(
+      {{"Let flush for mem1 start", "Flush for mem1"},
+       {"DBImpl::AtomicFlushMemTablesToOutputFiles:WaitCV",
+        "Wait for flush for mem1"},
+       {"RecoverFromRetryableBGIOError:BeforeStart",
+        "Wait for resume to start"},
+       {"Recovery should continue here",
+        "RecoverFromRetryableBGIOError:BeforeStart2"},
+       {"RecoverFromRetryableBGIOError:RecoverSuccess",
+        "Wait for error recover"}});
+  SyncPoint::GetInstance()->EnableProcessing();
+  ASSERT_OK(Put(Key(1), "val1"));
+  // trigger Flush0 for mem0
+  ASSERT_OK(Put(Key(2), "val2"));
+
+  // trigger Flush1 for mem1
+  TEST_SYNC_POINT("Flush for mem1");
+  ASSERT_OK(Put(Key(3), "val3"));
+
+  // Wait until resume started to schedule another flush
+  TEST_SYNC_POINT("Wait for resume to start");
+  // This flush should not be scheduled due to bg error
+  ASSERT_OK(Put(Key(4), "val4"));
+
+  // TEST_WaitForBackgroundWork() returns background error
+  // after all background work is done.
+  ASSERT_NOK(dbfull()->TEST_WaitForBackgroundWork());
+  // Flush should abort and not writing any table
+  ASSERT_EQ(0, NumTableFilesAtLevel(0));
+
+  // Wait until this flush is done.
+  TEST_SYNC_POINT("Recovery should continue here");
+  TEST_SYNC_POINT("Wait for error recover");
+  // error recovery can schedule new flushes, but should not
+  // encounter error
+  ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork());
+  ASSERT_EQ(1, NumTableFilesAtLevel(0));
+}
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -59,12 +59,18 @@ Status CompactedDBImpl::Get(const ReadOptions& _read_options,

  assert(user_comparator_);
  if (read_options.timestamp) {
-    const Status s =
-        FailIfTsMismatchCf(DefaultColumnFamily(), *(read_options.timestamp),
-                           /*ts_for_read=*/true);
+    Status s =
+        FailIfTsMismatchCf(DefaultColumnFamily(), *(read_options.timestamp));
    if (!s.ok()) {
      return s;
    }
+    if (read_options.timestamp->size() > 0) {
+      s = FailIfReadCollapsedHistory(cfd_, cfd_->GetSuperVersion(),
+                                     *(read_options.timestamp));
+      if (!s.ok()) {
+        return s;
+      }
+    }
  } else {
    const Status s = FailIfCfHasTs(DefaultColumnFamily());
    if (!s.ok()) {
@@ -133,11 +139,17 @@ std::vector<Status> CompactedDBImpl::MultiGet(

  if (read_options.timestamp) {
    Status s =
-        FailIfTsMismatchCf(DefaultColumnFamily(), *(read_options.timestamp),
-                           /*ts_for_read=*/true);
+        FailIfTsMismatchCf(DefaultColumnFamily(), *(read_options.timestamp));
    if (!s.ok()) {
      return std::vector<Status>(num_keys, s);
    }
+    if (read_options.timestamp->size() > 0) {
+      s = FailIfReadCollapsedHistory(cfd_, cfd_->GetSuperVersion(),
+                                     *(read_options.timestamp));
+      if (!s.ok()) {
+        return std::vector<Status>(num_keys, s);
+      }
+    }
  } else {
    Status s = FailIfCfHasTs(DefaultColumnFamily());
    if (!s.ok()) {
@@ -197,6 +197,8 @@ class DBImpl : public DB {
  Status PutEntity(const WriteOptions& options,
                   ColumnFamilyHandle* column_family, const Slice& key,
                   const WideColumns& columns) override;
+  Status PutEntity(const WriteOptions& options, const Slice& key,
+                   const AttributeGroups& attribute_groups) override;

  using DB::Merge;
  Status Merge(const WriteOptions& options, ColumnFamilyHandle* column_family,
@@ -242,6 +244,8 @@ class DBImpl : public DB {
  Status GetEntity(const ReadOptions& options,
                   ColumnFamilyHandle* column_family, const Slice& key,
                   PinnableWideColumns* columns) override;
+  Status GetEntity(const ReadOptions& options, const Slice& key,
+                   PinnableAttributeGroups* result) override;

  using DB::GetMergeOperands;
  Status GetMergeOperands(const ReadOptions& options,
@@ -311,6 +315,9 @@ class DBImpl : public DB {
                      ColumnFamilyHandle** column_families, const Slice* keys,
                      PinnableWideColumns* results, Status* statuses,
                      bool sorted_input) override;
+  void MultiGetEntity(const ReadOptions& options, size_t num_keys,
+                      const Slice* keys,
+                      PinnableAttributeGroups* results) override;

  virtual Status CreateColumnFamily(const ColumnFamilyOptions& cf_options,
                                    const std::string& column_family,
@@ -643,12 +650,12 @@ class DBImpl : public DB {
  // get_impl_options.key via get_impl_options.value
  // If get_impl_options.get_value = false get merge operands associated with
  // get_impl_options.key via get_impl_options.merge_operands
-  Status GetImpl(const ReadOptions& options, const Slice& key,
-                 GetImplOptions& get_impl_options);
+  virtual Status GetImpl(const ReadOptions& options, const Slice& key,
+                         GetImplOptions& get_impl_options);

  // If `snapshot` == kMaxSequenceNumber, set a recent one inside the file.
  ArenaWrappedDBIter* NewIteratorImpl(const ReadOptions& options,
-                                      ColumnFamilyData* cfd,
+                                      ColumnFamilyData* cfd, SuperVersion* sv,
                                      SequenceNumber snapshot,
                                      ReadCallback* read_callback,
                                      bool expose_blob_index = false,
@@ -1183,6 +1190,7 @@ class DBImpl : public DB {
  size_t TEST_GetWalPreallocateBlockSize(uint64_t write_buffer_size) const;
  void TEST_WaitForPeriodicTaskRun(std::function<void()> callback) const;
  SeqnoToTimeMapping TEST_GetSeqnoToTimeMapping() const;
+  const autovector<uint64_t>& TEST_GetFilesToQuarantine() const;
  size_t TEST_EstimateInMemoryStatsHistorySize() const;

  uint64_t TEST_GetCurrentLogNumber() const {
@@ -1197,6 +1205,10 @@ class DBImpl : public DB {

  const PeriodicTaskScheduler& TEST_GetPeriodicTaskScheduler() const;

+  static Status TEST_ValidateOptions(const DBOptions& db_options) {
+    return ValidateOptions(db_options);
+  }
+
 #endif  // NDEBUG

  // persist stats to column family "_persistent_stats"
@@ -1208,8 +1220,11 @@ class DBImpl : public DB {
  // flush LOG out of application buffer
  void FlushInfoLog();

-  // record current sequence number to time mapping
-  void RecordSeqnoToTimeMapping();
+  // record current sequence number to time mapping. If
+  // populate_historical_seconds > 0 then pre-populate all the
+  // sequence numbers from [1, last] to map to [now minus
+  // populate_historical_seconds, now].
+  void RecordSeqnoToTimeMapping(uint64_t populate_historical_seconds);

  // Interface to block and signal the DB in case of stalling writes by
  // WriteBufferManager. Each DBImpl object contains ptr to WBMStallInterface.
@@ -1383,15 +1398,15 @@ class DBImpl : public DB {
    autovector<ColumnFamilyData*> cfds_;
    autovector<const MutableCFOptions*> mutable_cf_opts_;
    autovector<autovector<VersionEdit*>> edit_lists_;
-    // files_to_delete_ contains sst files
-    std::unordered_set<std::string> files_to_delete_;
+    // Stale SST files to delete found upon recovery. This stores a mapping from
+    // such a file's absolute path to its parent directory.
+    std::unordered_map<std::string, std::string> files_to_delete_;
+    bool is_new_db_ = false;
  };

-  // Except in DB::Open(), WriteOptionsFile can only be called when:
-  // Persist options to options file.
-  // If need_mutex_lock = false, the method will lock DB mutex.
-  // If need_enter_write_thread = false, the method will enter write thread.
-  Status WriteOptionsFile(bool need_mutex_lock, bool need_enter_write_thread);
+  // Persist options to options file. Must be holding options_mutex_.
+  // Will lock DB mutex if !db_mutex_already_held.
+  Status WriteOptionsFile(bool db_mutex_already_held);

  Status CompactRangeInternal(const CompactRangeOptions& options,
                              ColumnFamilyHandle* column_family,
@@ -1543,8 +1558,18 @@ class DBImpl : public DB {
  void SetDbSessionId();

  Status FailIfCfHasTs(const ColumnFamilyHandle* column_family) const;
-  Status FailIfTsMismatchCf(ColumnFamilyHandle* column_family, const Slice& ts,
-                            bool ts_for_read) const;
+  Status FailIfTsMismatchCf(ColumnFamilyHandle* column_family,
+                            const Slice& ts) const;
+
+  // Check that the read timestamp `ts` is at or above the `full_history_ts_low`
+  // timestamp in a `SuperVersion`. It's necessary to do this check after
+  // grabbing the SuperVersion. If the check passed, the referenced SuperVersion
+  // this read holds on to can ensure the read won't be affected if
+  // `full_history_ts_low` is increased concurrently, and this achieves that
+  // without explicitly locking by piggybacking the SuperVersion.
+  Status FailIfReadCollapsedHistory(const ColumnFamilyData* cfd,
+                                    const SuperVersion* sv,
+                                    const Slice& ts) const;

  // recovery_ctx stores the context about version edits and
  // LogAndApplyForRecovery persist all those edits to new Manifest after
@@ -1809,10 +1834,15 @@ class DBImpl : public DB {

  const Status CreateArchivalDirectory();

+  // Create a column family, without some of the follow-up work yet
  Status CreateColumnFamilyImpl(const ColumnFamilyOptions& cf_options,
                                const std::string& cf_name,
                                ColumnFamilyHandle** handle);

+  // Follow-up work to user creating a column family or (families)
+  Status WrapUpCreateColumnFamilies(
+      const std::vector<const ColumnFamilyOptions*>& cf_options);
+
  Status DropColumnFamilyImpl(ColumnFamilyHandle* column_family);

  // Delete any unneeded files and stale in-memory entries.
@@ -1842,7 +1872,8 @@ class DBImpl : public DB {
  void ReleaseFileNumberFromPendingOutputs(
      std::unique_ptr<std::list<uint64_t>::iterator>& v);

-  IOStatus SyncClosedLogs(JobContext* job_context, VersionEdit* synced_wals);
+  IOStatus SyncClosedLogs(JobContext* job_context, VersionEdit* synced_wals,
+                          bool error_recovery_in_prog);

  // Flush the in-memory write buffer to storage.  Switches to a new
  // log-file/memtable and writes a new descriptor iff successful. Then
@@ -1937,6 +1968,8 @@ class DBImpl : public DB {
      const autovector<ColumnFamilyData*>& provided_candidate_cfds = {},
      bool entered_write_thread = false);

+  Status RetryFlushesForErrorRecovery(FlushReason flush_reason, bool wait);
+
  // Wait until flushing this column family won't stall writes
  Status WaitUntilFlushWouldNotStallWrites(ColumnFamilyData* cfd,
                                           bool* flush_needed);
@@ -2085,6 +2118,12 @@ class DBImpl : public DB {
 #endif /* !NDEBUG */
  };

+  // In case of atomic flush, generates a `FlushRequest` for the latest atomic
+  // cuts for these `cfds`. Atomic cuts are recorded in
+  // `AssignAtomicFlushSeq()`. For each entry in `cfds`, all CFDs sharing the
+  // same latest atomic cut must also be present.
+  //
+  // REQUIRES: mutex held
  void GenerateFlushRequest(const autovector<ColumnFamilyData*>& cfds,
                            FlushReason flush_reason, FlushRequest* req);

@@ -2136,7 +2175,7 @@ class DBImpl : public DB {
  // Cancel scheduled periodic tasks
  Status CancelPeriodicTaskScheduler();

-  Status RegisterRecordSeqnoTimeWorker();
+  Status RegisterRecordSeqnoTimeWorker(bool is_new_db);

  void PrintStatistics();

@@ -2216,6 +2255,7 @@ class DBImpl : public DB {
  bool ShouldntRunManualCompaction(ManualCompactionState* m);
  bool HaveManualCompaction(ColumnFamilyData* cfd);
  bool MCOverlap(ManualCompactionState* m, ManualCompactionState* m1);
+  void UpdateDeletionCompactionStats(const std::unique_ptr<Compaction>& c);
  void BuildCompactionJobInfo(const ColumnFamilyData* cfd, Compaction* c,
                              const Status& st,
                              const CompactionJobStats& compaction_job_stats,
@@ -2312,15 +2352,18 @@ class DBImpl : public DB {
  // If callback is non-null, the callback is refreshed with the snapshot
  // sequence number
  //
-  // A return value of true indicates that the SuperVersions were obtained
-  // from the ColumnFamilyData, whereas false indicates they are thread
-  // local
+  // `sv_from_thread_local` being set to false indicates that the SuperVersion
+  // obtained from the ColumnFamilyData, whereas true indicates they are thread
+  // local.
+  // A non-OK status will be returned if for a column family that enables
+  // user-defined timestamp feature, the specified `ReadOptions.timestamp`
+  // attemps to read collapsed history.
  template <class T>
-  bool MultiCFSnapshot(
+  Status MultiCFSnapshot(
      const ReadOptions& read_options, ReadCallback* callback,
      std::function<MultiGetColumnFamilyData*(typename T::iterator&)>&
          iter_deref_func,
-      T* cf_list, SequenceNumber* snapshot);
+      T* cf_list, SequenceNumber* snapshot, bool* sv_from_thread_local);

  // The actual implementation of the batching MultiGet. The caller is expected
  // to have acquired the SuperVersion and pass in a snapshot sequence number
@@ -2331,6 +2374,11 @@ class DBImpl : public DB {
      autovector<KeyContext*, MultiGetContext::MAX_BATCH_SIZE>* sorted_keys,
      SuperVersion* sv, SequenceNumber snap_seqnum, ReadCallback* callback);

+  void MultiGetWithCallbackImpl(
+      const ReadOptions& read_options, ColumnFamilyHandle* column_family,
+      ReadCallback* callback,
+      autovector<KeyContext*, MultiGetContext::MAX_BATCH_SIZE>* sorted_keys);
+
  Status DisableFileDeletionsWithLock();

  Status IncreaseFullHistoryTsLowImpl(ColumnFamilyData* cfd,
@@ -2341,9 +2389,19 @@ class DBImpl : public DB {
  // Lock over the persistent DB state.  Non-nullptr iff successfully acquired.
  FileLock* db_lock_;

-  // In addition to mutex_, log_write_mutex_ protected writes to stats_history_
+  // Guards changes to DB and CF options to ensure consistency between
+  // * In-memory options objects
+  // * Settings in effect
+  // * Options file contents
+  // while allowing the DB mutex to be released during slow operations like
+  // persisting options file or modifying global periodic task timer.
+  // Always acquired *before* DB mutex when this one is applicable.
+  InstrumentedMutex options_mutex_;
+
+  // Guards reads and writes to in-memory stats_history_.
  InstrumentedMutex stats_history_mutex_;
-  // In addition to mutex_, log_write_mutex_ protected writes to logs_ and
+
+  // In addition to mutex_, log_write_mutex_ protects writes to logs_ and
  // logfile_number_. With two_write_queues it also protects alive_log_files_,
  // and log_empty_. Refer to the definition of each variable below for more
  // details.
@@ -2620,9 +2678,6 @@ class DBImpl : public DB {
  // initialized with startup time.
  uint64_t delete_obsolete_files_last_run_;

-  // last time stats were dumped to LOG
-  std::atomic<uint64_t> last_stats_dump_time_microsec_;
-
  // The thread that wants to switch memtable, can wait on this cv until the
  // pending writes to memtable finishes.
  std::condition_variable switch_cv_;
@@ -2733,9 +2788,9 @@ class DBImpl : public DB {
  // Pointer to WriteBufferManager stalling interface.
  std::unique_ptr<StallInterface> wbm_stall_;

-  // seqno_time_mapping_ stores the sequence number to time mapping, it's not
+  // seqno_to_time_mapping_ stores the sequence number to time mapping, it's not
  // thread safe, both read and write need db mutex hold.
-  SeqnoToTimeMapping seqno_time_mapping_;
+  SeqnoToTimeMapping seqno_to_time_mapping_;

  // Stop write token that is acquired when first LockWAL() is called.
  // Destroyed when last UnlockWAL() is called. Controlled by DB mutex.
@@ -2815,7 +2870,9 @@ static void ClipToRange(T* ptr, V minvalue, V maxvalue) {

 inline Status DBImpl::FailIfCfHasTs(
    const ColumnFamilyHandle* column_family) const {
-  column_family = column_family ? column_family : DefaultColumnFamily();
+  if (!column_family) {
+    return Status::InvalidArgument("column family handle cannot be null");
+  }
  assert(column_family);
  const Comparator* const ucmp = column_family->GetComparator();
  assert(ucmp);
@@ -2829,8 +2886,7 @@ inline Status DBImpl::FailIfCfHasTs(
 }

 inline Status DBImpl::FailIfTsMismatchCf(ColumnFamilyHandle* column_family,
-                                         const Slice& ts,
-                                         bool ts_for_read) const {
+                                         const Slice& ts) const {
  if (!column_family) {
    return Status::InvalidArgument("column family handle cannot be null");
  }
@@ -2850,20 +2906,28 @@ inline Status DBImpl::FailIfTsMismatchCf(ColumnFamilyHandle* column_family,
        << ts_sz << " given";
    return Status::InvalidArgument(oss.str());
  }
-  if (ts_for_read) {
-    auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(column_family);
-    auto cfd = cfh->cfd();
-    std::string current_ts_low = cfd->GetFullHistoryTsLow();
-    if (!current_ts_low.empty() &&
-        ucmp->CompareTimestamp(ts, current_ts_low) < 0) {
-      std::stringstream oss;
-      oss << "Read timestamp: " << ts.ToString(true)
-          << " is smaller than full_history_ts_low: "
-          << Slice(current_ts_low).ToString(true) << std::endl;
-      return Status::InvalidArgument(oss.str());
-    }
-  }
  return Status::OK();
 }

+inline Status DBImpl::FailIfReadCollapsedHistory(const ColumnFamilyData* cfd,
+                                                 const SuperVersion* sv,
+                                                 const Slice& ts) const {
+  // Reaching to this point means the timestamp size matching sanity check in
+  // `DBImpl::FailIfTsMismatchCf` already passed. So we skip that and assume
+  // column family has the same user-defined timestamp format as `ts`.
+  const Comparator* const ucmp = cfd->user_comparator();
+  assert(ucmp);
+  const std::string& full_history_ts_low = sv->full_history_ts_low;
+  assert(full_history_ts_low.empty() ||
+         full_history_ts_low.size() == ts.size());
+  if (!full_history_ts_low.empty() &&
+      ucmp->CompareTimestamp(ts, full_history_ts_low) < 0) {
+    std::stringstream oss;
+    oss << "Read timestamp: " << ts.ToString(true)
+        << " is smaller than full_history_ts_low: "
+        << Slice(full_history_ts_low).ToString(true) << std::endl;
+    return Status::InvalidArgument(oss.str());
+  }
+  return Status::OK();
+}
 }  // namespace ROCKSDB_NAMESPACE
@@ -23,6 +23,7 @@
 #include "util/cast_util.h"
 #include "util/coding.h"
 #include "util/concurrent_task_limiter_impl.h"
+#include "util/udt_util.h"

 namespace ROCKSDB_NAMESPACE {

@@ -112,7 +113,8 @@ bool DBImpl::ShouldRescheduleFlushRequestToRetainUDT(
 }

 IOStatus DBImpl::SyncClosedLogs(JobContext* job_context,
-                                VersionEdit* synced_wals) {
+                                VersionEdit* synced_wals,
+                                bool error_recovery_in_prog) {
  TEST_SYNC_POINT("DBImpl::SyncClosedLogs:Start");
  InstrumentedMutexLock l(&log_write_mutex_);
  autovector<log::Writer*, 1> logs_to_sync;
@@ -138,7 +140,7 @@ IOStatus DBImpl::SyncClosedLogs(JobContext* job_context,
      ROCKS_LOG_INFO(immutable_db_options_.info_log,
                     "[JOB %d] Syncing log #%" PRIu64, job_context->job_id,
                     log->get_log_number());
-      if (error_handler_.IsRecoveryInProgress()) {
+      if (error_recovery_in_prog) {
        log->file()->reset_seen_error();
      }
      io_s = log->file()->Sync(immutable_db_options_.use_fsync);
@@ -147,7 +149,7 @@ IOStatus DBImpl::SyncClosedLogs(JobContext* job_context,
      }

      if (immutable_db_options_.recycle_log_file_num > 0) {
-        if (error_handler_.IsRecoveryInProgress()) {
+        if (error_recovery_in_prog) {
          log->file()->reset_seen_error();
        }
        io_s = log->Close();
@@ -221,9 +223,10 @@ Status DBImpl::FlushMemTableToOutputFile(
  // `snapshot_seqs` has already been computed before this function starts.
  // Recording the max memtable ID ensures that the flush job does not flush
  // a memtable without knowing such snapshot(s).
-  uint64_t max_memtable_id = needs_to_sync_closed_wals
-                                 ? cfd->imm()->GetLatestMemTableID()
-                                 : std::numeric_limits<uint64_t>::max();
+  uint64_t max_memtable_id =
+      needs_to_sync_closed_wals
+          ? cfd->imm()->GetLatestMemTableID(false /* for_atomic_flush */)
+          : std::numeric_limits<uint64_t>::max();

  // If needs_to_sync_closed_wals is false, then the flush job will pick ALL
  // existing memtables of the column family when PickMemTable() is called
@@ -232,7 +235,7 @@ Status DBImpl::FlushMemTableToOutputFile(
  // releases and re-acquires the db mutex. In the meantime, the application
  // can still insert into the memtables and increase the db's sequence number.
  // The application can take a snapshot, hoping that the latest visible state
-  // to this snapshto is preserved. This is hard to guarantee since db mutex
+  // to this snapshot is preserved. This is hard to guarantee since db mutex
  // not held. This newly-created snapshot is not included in `snapshot_seqs`
  // and the flush job is unaware of its presence. Consequently, the flush job
  // may drop certain keys when generating the L0, causing incorrect data to be
@@ -249,7 +252,7 @@ Status DBImpl::FlushMemTableToOutputFile(
      GetCompressionFlush(*cfd->ioptions(), mutable_cf_options), stats_,
      &event_logger_, mutable_cf_options.report_bg_io_stats,
      true /* sync_output_directory */, true /* write_manifest */, thread_pri,
-      io_tracer_, seqno_time_mapping_, db_id_, db_session_id_,
+      io_tracer_, seqno_to_time_mapping_, db_id_, db_session_id_,
      cfd->GetFullHistoryTsLow(), &blob_callback_);
  FileMetaData file_meta;

@@ -260,8 +263,10 @@ Status DBImpl::FlushMemTableToOutputFile(
    // SyncClosedLogs() may unlock and re-lock the log_write_mutex multiple
    // times.
    VersionEdit synced_wals;
+    bool error_recovery_in_prog = error_handler_.IsRecoveryInProgress();
    mutex_.Unlock();
-    log_io_s = SyncClosedLogs(job_context, &synced_wals);
+    log_io_s =
+        SyncClosedLogs(job_context, &synced_wals, error_recovery_in_prog);
    mutex_.Lock();
    if (log_io_s.ok() && synced_wals.IsWalAddition()) {
      const ReadOptions read_options(Env::IOActivity::kFlush);
@@ -283,6 +288,24 @@ Status DBImpl::FlushMemTableToOutputFile(
  // If the log sync failed, we do not need to pick memtable. Otherwise,
  // num_flush_not_started_ needs to be rollback.
  TEST_SYNC_POINT("DBImpl::FlushMemTableToOutputFile:BeforePickMemtables");
+  // Exit a flush due to bg error should not set bg error again.
+  bool skip_set_bg_error = false;
+  if (s.ok() && !error_handler_.GetBGError().ok() &&
+      error_handler_.IsBGWorkStopped() &&
+      flush_reason != FlushReason::kErrorRecovery &&
+      flush_reason != FlushReason::kErrorRecoveryRetryFlush) {
+    // Error recovery in progress, should not pick memtable which excludes
+    // them from being picked up by recovery flush.
+    // This ensures that when bg error is set, no new flush can pick
+    // memtables.
+    skip_set_bg_error = true;
+    s = error_handler_.GetBGError();
+    assert(!s.ok());
+    ROCKS_LOG_BUFFER(log_buffer,
+                     "[JOB %d] Skip flush due to background error %s",
+                     job_context->job_id, s.ToString().c_str());
+  }
+
  if (s.ok()) {
    flush_job.PickMemTable();
    need_cancel = true;
@@ -303,7 +326,8 @@ Status DBImpl::FlushMemTableToOutputFile(
  // is unlocked by the current thread.
  if (s.ok()) {
    s = flush_job.Run(&logs_with_prep_tracker_, &file_meta,
-                      &switched_to_mempurge);
+                      &switched_to_mempurge, &skip_set_bg_error,
+                      &error_handler_);
    need_cancel = false;
  }

@@ -344,7 +368,8 @@ Status DBImpl::FlushMemTableToOutputFile(
    }
  }

-  if (!s.ok() && !s.IsShutdownInProgress() && !s.IsColumnFamilyDropped()) {
+  if (!s.ok() && !s.IsShutdownInProgress() && !s.IsColumnFamilyDropped() &&
+      !skip_set_bg_error) {
    if (log_io_s.ok()) {
      // Error while writing to MANIFEST.
      // In fact, versions_->io_status() can also be the result of renaming
@@ -501,7 +526,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
        GetCompressionFlush(*cfd->ioptions(), mutable_cf_options), stats_,
        &event_logger_, mutable_cf_options.report_bg_io_stats,
        false /* sync_output_directory */, false /* write_manifest */,
-        thread_pri, io_tracer_, seqno_time_mapping_, db_id_, db_session_id_,
+        thread_pri, io_tracer_, seqno_to_time_mapping_, db_id_, db_session_id_,
        cfd->GetFullHistoryTsLow(), &blob_callback_));
  }

@@ -525,8 +550,10 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
    // TODO (yanqin) investigate whether we should sync the closed logs for
    // single column family case.
    VersionEdit synced_wals;
+    bool error_recovery_in_prog = error_handler_.IsRecoveryInProgress();
    mutex_.Unlock();
-    log_io_s = SyncClosedLogs(job_context, &synced_wals);
+    log_io_s =
+        SyncClosedLogs(job_context, &synced_wals, error_recovery_in_prog);
    mutex_.Lock();
    if (log_io_s.ok() && synced_wals.IsWalAddition()) {
      const ReadOptions read_options(Env::IOActivity::kFlush);
@@ -556,6 +583,21 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
    pick_status.push_back(false);
  }

+  bool flush_for_recovery =
+      bg_flush_args[0].flush_reason_ == FlushReason::kErrorRecovery ||
+      bg_flush_args[0].flush_reason_ == FlushReason::kErrorRecoveryRetryFlush;
+  bool skip_set_bg_error = false;
+
+  if (s.ok() && !error_handler_.GetBGError().ok() &&
+      error_handler_.IsBGWorkStopped() && !flush_for_recovery) {
+    s = error_handler_.GetBGError();
+    skip_set_bg_error = true;
+    assert(!s.ok());
+    ROCKS_LOG_BUFFER(log_buffer,
+                     "[JOB %d] Skip flush due to background error %s",
+                     job_context->job_id, s.ToString().c_str());
+  }
+
  if (s.ok()) {
    for (int i = 0; i != num_cfs; ++i) {
      jobs[i]->PickMemTable();
@@ -620,7 +662,10 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
        }
      }
    }
-  } else {
+  } else if (!skip_set_bg_error) {
+    // When `skip_set_bg_error` is true, no memtable is picked so
+    // there is no need to call Cancel() or RollbackMemtableFlush().
+    //
    // Need to undo atomic flush if something went wrong, i.e. s is not OK and
    // it is not because of CF drop.
    // Have to cancel the flush jobs that have NOT executed because we need to
@@ -633,8 +678,8 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
    for (int i = 0; i != num_cfs; ++i) {
      if (exec_status[i].second.ok() && exec_status[i].first) {
        auto& mems = jobs[i]->GetMemTables();
-        cfds[i]->imm()->RollbackMemtableFlush(mems,
-                                              file_meta[i].fd.GetNumber());
+        cfds[i]->imm()->RollbackMemtableFlush(
+            mems, /*rollback_succeeding_memtables=*/false);
      }
    }
  }
@@ -676,10 +721,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
    };

    bool resuming_from_bg_err =
-        error_handler_.IsDBStopped() ||
-        (bg_flush_args[0].flush_reason_ == FlushReason::kErrorRecovery ||
-         bg_flush_args[0].flush_reason_ ==
-             FlushReason::kErrorRecoveryRetryFlush);
+        error_handler_.IsDBStopped() || flush_for_recovery;
    while ((!resuming_from_bg_err || error_handler_.GetRecoveryError().ok())) {
      std::pair<Status, bool> res = wait_to_install_func();

@@ -690,15 +732,27 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
        s = res.first;
        break;
      } else if (!res.second) {
+        // we are the oldest immutable memtable
+        break;
+      }
+      // We are not the oldest immutable memtable
+      TEST_SYNC_POINT_CALLBACK(
+          "DBImpl::AtomicFlushMemTablesToOutputFiles:WaitCV", &res);
+      //
+      // If bg work is stopped, recovery thread first calls
+      // WaitForBackgroundWork() before proceeding to flush for recovery. This
+      // flush can block WaitForBackgroundWork() while waiting for recovery
+      // flush to install result. To avoid this deadlock, we should abort here
+      // if there is background error.
+      if (!flush_for_recovery && error_handler_.IsBGWorkStopped() &&
+          !error_handler_.GetBGError().ok()) {
+        s = error_handler_.GetBGError();
+        assert(!s.ok());
        break;
      }
      atomic_flush_install_cv_.Wait();

-      resuming_from_bg_err =
-          error_handler_.IsDBStopped() ||
-          (bg_flush_args[0].flush_reason_ == FlushReason::kErrorRecovery ||
-           bg_flush_args[0].flush_reason_ ==
-               FlushReason::kErrorRecoveryRetryFlush);
+      resuming_from_bg_err = error_handler_.IsDBStopped() || flush_for_recovery;
    }

    if (!resuming_from_bg_err) {
@@ -714,6 +768,17 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
      // installation.
      s = error_handler_.GetRecoveryError();
    }
+    // Since we are not installing these memtables, need to rollback
+    // to allow future flush job to pick up these memtables.
+    if (!s.ok()) {
+      for (int i = 0; i != num_cfs; ++i) {
+        assert(exec_status[i].first);
+        assert(exec_status[i].second.ok());
+        auto& mems = jobs[i]->GetMemTables();
+        cfds[i]->imm()->RollbackMemtableFlush(
+            mems, /*rollback_succeeding_memtables=*/false);
+      }
+    }
  }

  if (s.ok()) {
@@ -817,7 +882,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(

  // Need to undo atomic flush if something went wrong, i.e. s is not OK and
  // it is not because of CF drop.
-  if (!s.ok() && !s.IsColumnFamilyDropped()) {
+  if (!s.ok() && !s.IsColumnFamilyDropped() && !skip_set_bg_error) {
    if (log_io_s.ok()) {
      // Error while writing to MANIFEST.
      // In fact, versions_->io_status() can also be the result of renaming
@@ -947,26 +1012,14 @@ Status DBImpl::CompactRange(const CompactRangeOptions& options,
                                end_without_ts, "" /*trim_ts*/);
  }

-  std::string begin_str;
-  std::string end_str;
+  std::string begin_str, end_str;
+  auto [begin, end] =
+      MaybeAddTimestampsToRange(begin_without_ts, end_without_ts, ts_sz,
+                                &begin_str, &end_str, false /*exclusive_end*/);

-  // CompactRange compact all keys: [begin, end] inclusively. Add maximum
-  // timestamp to include all `begin` keys, and add minimal timestamp to include
-  // all `end` keys.
-  if (begin_without_ts != nullptr) {
-    AppendKeyWithMaxTimestamp(&begin_str, *begin_without_ts, ts_sz);
-  }
-  if (end_without_ts != nullptr) {
-    AppendKeyWithMinTimestamp(&end_str, *end_without_ts, ts_sz);
-  }
-  Slice begin(begin_str);
-  Slice end(end_str);
-
-  Slice* begin_with_ts = begin_without_ts ? &begin : nullptr;
-  Slice* end_with_ts = end_without_ts ? &end : nullptr;
-
-  return CompactRangeInternal(options, column_family, begin_with_ts,
-                              end_with_ts, "" /*trim_ts*/);
+  return CompactRangeInternal(
+      options, column_family, begin.has_value() ? &begin.value() : nullptr,
+      end.has_value() ? &end.value() : nullptr, "" /*trim_ts*/);
 }

 Status DBImpl::IncreaseFullHistoryTsLow(ColumnFamilyHandle* column_family,
@@ -1473,7 +1526,8 @@ Status DBImpl::CompactFilesImpl(
  // without releasing the lock, so we're guaranteed a compaction can be formed.
  assert(c != nullptr);

-  c->SetInputVersion(version);
+  c->FinalizeInputInfo(version);
+
  // deletion compaction currently not allowed in CompactFiles.
  assert(!c->deletion_compaction());

@@ -1523,7 +1577,12 @@ Status DBImpl::CompactFilesImpl(
  TEST_SYNC_POINT("CompactFilesImpl:3");
  mutex_.Lock();

-  Status status = compaction_job.Install(*c->mutable_cf_options());
+  bool compaction_released = false;
+  Status status =
+      compaction_job.Install(*c->mutable_cf_options(), &compaction_released);
+  if (!compaction_released) {
+    c->ReleaseCompactionFiles(s);
+  }
  if (status.ok()) {
    assert(compaction_job.io_status().ok());
    InstallSuperVersionAndScheduleWork(c->column_family_data(),
@@ -1534,7 +1593,6 @@ Status DBImpl::CompactFilesImpl(
  // not check compaction_job.io_status() explicitly if we're not calling
  // SetBGError
  compaction_job.io_status().PermitUncheckedError();
-  c->ReleaseCompactionFiles(s);
  // Need to make sure SstFileManager does its bookkeeping
  auto sfm = static_cast<SstFileManagerImpl*>(
      immutable_db_options_.sst_file_manager.get());
@@ -2178,7 +2236,8 @@ void DBImpl::GenerateFlushRequest(const autovector<ColumnFamilyData*>& cfds,
      // cfd may be null, see DBImpl::ScheduleFlushes
      continue;
    }
-    uint64_t max_memtable_id = cfd->imm()->GetLatestMemTableID();
+    uint64_t max_memtable_id = cfd->imm()->GetLatestMemTableID(
+        immutable_db_options_.atomic_flush /* for_atomic_flush */);
    req->cfd_to_max_mem_id_to_persist.emplace(cfd, max_memtable_id);
  }
 }
@@ -2222,15 +2281,7 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd,
    }
    WaitForPendingWrites();

-    if (flush_reason != FlushReason::kErrorRecoveryRetryFlush &&
-        (!cfd->mem()->IsEmpty() || !cached_recoverable_state_empty_.load())) {
-      // Note that, when flush reason is kErrorRecoveryRetryFlush, during the
-      // auto retry resume, we want to avoid creating new small memtables.
-      // Therefore, SwitchMemtable will not be called. Also, since ResumeImpl
-      // will iterate through all the CFs and call FlushMemtable during auto
-      // retry resume, it is possible that in some CFs,
-      // cfd->imm()->NumNotFlushed() = 0. In this case, so no flush request will
-      // be created and scheduled, status::OK() will be returned.
+    if (!cfd->mem()->IsEmpty() || !cached_recoverable_state_empty_.load()) {
      s = SwitchMemtable(cfd, &context);
    }
    const uint64_t flush_memtable_id = std::numeric_limits<uint64_t>::max();
@@ -2239,10 +2290,10 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd,
          !cached_recoverable_state_empty_.load()) {
        FlushRequest req{flush_reason, {{cfd, flush_memtable_id}}};
        flush_reqs.emplace_back(std::move(req));
-        memtable_ids_to_wait.emplace_back(cfd->imm()->GetLatestMemTableID());
+        memtable_ids_to_wait.emplace_back(
+            cfd->imm()->GetLatestMemTableID(false /* for_atomic_flush */));
      }
-      if (immutable_db_options_.persist_stats_to_disk &&
-          flush_reason != FlushReason::kErrorRecoveryRetryFlush) {
+      if (immutable_db_options_.persist_stats_to_disk) {
        ColumnFamilyData* cfd_stats =
            versions_->GetColumnFamilySet()->GetColumnFamily(
                kPersistentStatsColumnFamilyName);
@@ -2268,7 +2319,8 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd,
            FlushRequest req{flush_reason, {{cfd_stats, flush_memtable_id}}};
            flush_reqs.emplace_back(std::move(req));
            memtable_ids_to_wait.emplace_back(
-                cfd_stats->imm()->GetLatestMemTableID());
+                cfd_stats->imm()->GetLatestMemTableID(
+                    false /* for_atomic_flush */));
          }
        }
      }
@@ -2319,8 +2371,7 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd,
    }
    s = WaitForFlushMemTables(
        cfds, flush_memtable_ids,
-        (flush_reason == FlushReason::kErrorRecovery ||
-         flush_reason == FlushReason::kErrorRecoveryRetryFlush));
+        flush_reason == FlushReason::kErrorRecovery /* resuming_from_bg_err */);
    InstrumentedMutexLock lock_guard(&mutex_);
    for (auto* tmp_cfd : cfds) {
      tmp_cfd->UnrefAndTryDelete();
@@ -2415,8 +2466,7 @@ Status DBImpl::AtomicFlushMemTables(
    }

    for (auto cfd : cfds) {
-      if ((cfd->mem()->IsEmpty() && cached_recoverable_state_empty_.load()) ||
-          flush_reason == FlushReason::kErrorRecoveryRetryFlush) {
+      if (cfd->mem()->IsEmpty() && cached_recoverable_state_empty_.load()) {
        continue;
      }
      cfd->Ref();
@@ -2461,8 +2511,7 @@ Status DBImpl::AtomicFlushMemTables(
    }
    s = WaitForFlushMemTables(
        cfds, flush_memtable_ids,
-        (flush_reason == FlushReason::kErrorRecovery ||
-         flush_reason == FlushReason::kErrorRecoveryRetryFlush));
+        flush_reason == FlushReason::kErrorRecovery /* resuming_from_bg_err */);
    InstrumentedMutexLock lock_guard(&mutex_);
    for (auto* cfd : cfds) {
      cfd->UnrefAndTryDelete();
@@ -2471,6 +2520,68 @@ Status DBImpl::AtomicFlushMemTables(
  return s;
 }

+Status DBImpl::RetryFlushesForErrorRecovery(FlushReason flush_reason,
+                                            bool wait) {
+  mutex_.AssertHeld();
+  assert(flush_reason == FlushReason::kErrorRecoveryRetryFlush ||
+         flush_reason == FlushReason::kCatchUpAfterErrorRecovery);
+
+  // Collect referenced CFDs.
+  autovector<ColumnFamilyData*> cfds;
+  for (ColumnFamilyData* cfd : *versions_->GetColumnFamilySet()) {
+    if (!cfd->IsDropped() && cfd->initialized() &&
+        cfd->imm()->NumNotFlushed() != 0) {
+      cfd->Ref();
+      cfd->imm()->FlushRequested();
+      cfds.push_back(cfd);
+    }
+  }
+
+  // Submit flush requests for all immutable memtables needing flush.
+  // `flush_memtable_ids` will be populated such that all immutable
+  // memtables eligible for flush are waited on before this function
+  // returns.
+  autovector<uint64_t> flush_memtable_ids;
+  if (immutable_db_options_.atomic_flush) {
+    FlushRequest flush_req;
+    GenerateFlushRequest(cfds, flush_reason, &flush_req);
+    SchedulePendingFlush(flush_req);
+    for (auto& iter : flush_req.cfd_to_max_mem_id_to_persist) {
+      flush_memtable_ids.push_back(iter.second);
+    }
+  } else {
+    for (auto cfd : cfds) {
+      flush_memtable_ids.push_back(
+          cfd->imm()->GetLatestMemTableID(false /* for_atomic_flush */));
+      // Impose no bound on the highest memtable ID flushed. There is no
+      // reason to do so outside of atomic flush.
+      FlushRequest flush_req{
+          flush_reason,
+          {{cfd,
+            std::numeric_limits<uint64_t>::max() /* max_mem_id_to_persist */}}};
+      SchedulePendingFlush(flush_req);
+    }
+  }
+  MaybeScheduleFlushOrCompaction();
+
+  Status s;
+  if (wait) {
+    mutex_.Unlock();
+    autovector<const uint64_t*> flush_memtable_id_ptrs;
+    for (auto& flush_memtable_id : flush_memtable_ids) {
+      flush_memtable_id_ptrs.push_back(&flush_memtable_id);
+    }
+    s = WaitForFlushMemTables(cfds, flush_memtable_id_ptrs,
+                              true /* resuming_from_bg_err */);
+    mutex_.Lock();
+  }
+
+  for (auto* cfd : cfds) {
+    cfd->UnrefAndTryDelete();
+  }
+  return s;
+}
+
 // Calling FlushMemTable(), whether from DB::Flush() or from Backup Engine, can
 // cause write stall, for example if one memtable is being flushed already.
 // This method tries to avoid write stall (similar to CompactRange() behavior)
@@ -2672,6 +2783,7 @@ void DBImpl::EnableManualCompaction() {

 void DBImpl::MaybeScheduleFlushOrCompaction() {
  mutex_.AssertHeld();
+  TEST_SYNC_POINT("DBImpl::MaybeScheduleFlushOrCompaction:Start");
  if (!opened_successfully_) {
    // Compaction may introduce data race to DB open
    return;
@@ -2684,6 +2796,11 @@ void DBImpl::MaybeScheduleFlushOrCompaction() {
    // There has been a hard error and this call is not part of the recovery
    // sequence. Bail out here so we don't get into an endless loop of
    // scheduling BG work which will again call this function
+    //
+    // Note that a non-recovery flush can still be scheduled if
+    // error_handler_.IsRecoveryInProgress() returns true. We rely on
+    // BackgroundCallFlush() to check flush reason and drop non-recovery
+    // flushes.
    return;
  } else if (shutting_down_.load(std::memory_order_acquire)) {
    // DB is being deleted; no more background compactions
@@ -3013,6 +3130,24 @@ Status DBImpl::BackgroundFlush(bool* made_progress, JobContext* job_context,
    // This cfd is already referenced
    FlushRequest flush_req = PopFirstFromFlushQueue();
    FlushReason flush_reason = flush_req.flush_reason;
+    if (!error_handler_.GetBGError().ok() && error_handler_.IsBGWorkStopped() &&
+        flush_reason != FlushReason::kErrorRecovery &&
+        flush_reason != FlushReason::kErrorRecoveryRetryFlush) {
+      // Stop non-recovery flush when bg work is stopped
+      // Note that we drop the flush request here.
+      // Recovery thread should schedule further flushes after bg error
+      // is cleared.
+      status = error_handler_.GetBGError();
+      assert(!status.ok());
+      ROCKS_LOG_BUFFER(log_buffer,
+                       "[JOB %d] Abort flush due to background error %s",
+                       job_context->job_id, status.ToString().c_str());
+      *reason = flush_reason;
+      for (auto item : flush_req.cfd_to_max_mem_id_to_persist) {
+        item.first->UnrefAndTryDelete();
+      }
+      return status;
+    }
    if (!immutable_db_options_.atomic_flush &&
        ShouldRescheduleFlushRequestToRetainUDT(flush_req)) {
      assert(flush_req.cfd_to_max_mem_id_to_persist.size() == 1);
@@ -3155,9 +3290,9 @@ void DBImpl::BackgroundCallFlush(Env::Priority thread_pri) {
      bg_cv_.SignalAll();  // In case a waiter can proceed despite the error
      mutex_.Unlock();
      ROCKS_LOG_ERROR(immutable_db_options_.info_log,
-                      "Waiting after background flush error: %s"
+                      "[JOB %d] Waiting after background flush error: %s"
                      "Accumulated background error counts: %" PRIu64,
-                      s.ToString().c_str(), error_cnt);
+                      job_context.job_id, s.ToString().c_str(), error_cnt);
      log_buffer.FlushBufferToLog();
      LogFlush(immutable_db_options_.info_log);
      immutable_db_options_.clock->SleepForMicroseconds(1000000);
@@ -3166,7 +3301,7 @@ void DBImpl::BackgroundCallFlush(Env::Priority thread_pri) {

    TEST_SYNC_POINT("DBImpl::BackgroundCallFlush:FlushFinish:0");
    ReleaseFileNumberFromPendingOutputs(pending_outputs_inserted_elem);
-    // There is no need to do these clean up if the flush job is rescheduled
+    // There is no need to find obsolete files if the flush job is rescheduled
    // to retain user-defined timestamps because the job doesn't get to the
    // stage of actually flushing the MemTables.
    if (!flush_rescheduled_to_retain_udt) {
@@ -3174,25 +3309,25 @@ void DBImpl::BackgroundCallFlush(Env::Priority thread_pri) {
      // have created. Thus, we force full scan in FindObsoleteFiles()
      FindObsoleteFiles(&job_context, !s.ok() && !s.IsShutdownInProgress() &&
                                          !s.IsColumnFamilyDropped());
-      // delete unnecessary files if any, this is done outside the mutex
-      if (job_context.HaveSomethingToClean() ||
-          job_context.HaveSomethingToDelete() || !log_buffer.IsEmpty()) {
-        mutex_.Unlock();
-        TEST_SYNC_POINT("DBImpl::BackgroundCallFlush:FilesFound");
-        // Have to flush the info logs before bg_flush_scheduled_--
-        // because if bg_flush_scheduled_ becomes 0 and the lock is
-        // released, the deconstructor of DB can kick in and destroy all the
-        // states of DB so info_log might not be available after that point.
-        // It also applies to access other states that DB owns.
-        log_buffer.FlushBufferToLog();
-        if (job_context.HaveSomethingToDelete()) {
-          PurgeObsoleteFiles(job_context);
-        }
-        job_context.Clean();
-        mutex_.Lock();
-      }
-      TEST_SYNC_POINT("DBImpl::BackgroundCallFlush:ContextCleanedUp");
    }
+    // delete unnecessary files if any, this is done outside the mutex
+    if (job_context.HaveSomethingToClean() ||
+        job_context.HaveSomethingToDelete() || !log_buffer.IsEmpty()) {
+      mutex_.Unlock();
+      TEST_SYNC_POINT("DBImpl::BackgroundCallFlush:FilesFound");
+      // Have to flush the info logs before bg_flush_scheduled_--
+      // because if bg_flush_scheduled_ becomes 0 and the lock is
+      // released, the deconstructor of DB can kick in and destroy all the
+      // states of DB so info_log might not be available after that point.
+      // It also applies to access other states that DB owns.
+      log_buffer.FlushBufferToLog();
+      if (job_context.HaveSomethingToDelete()) {
+        PurgeObsoleteFiles(job_context);
+      }
+      job_context.Clean();
+      mutex_.Lock();
+    }
+    TEST_SYNC_POINT("DBImpl::BackgroundCallFlush:ContextCleanedUp");

    assert(num_running_flushes_ > 0);
    num_running_flushes_--;
@@ -3399,8 +3534,6 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,

  std::unique_ptr<TaskLimiterToken> task_token;

-  // InternalKey manual_end_storage;
-  // InternalKey* manual_end = &manual_end_storage;
  bool sfm_reserved_compact_space = false;
  if (is_manual) {
    ManualCompactionState* m = manual_compaction;
@@ -3536,6 +3669,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
  }

  IOStatus io_s;
+  bool compaction_released = false;
  if (!c) {
    // Nothing to do
    ROCKS_LOG_BUFFER(log_buffer, "Compaction nothing to do");
@@ -3558,7 +3692,12 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
    }
    status = versions_->LogAndApply(
        c->column_family_data(), *c->mutable_cf_options(), read_options,
-        c->edit(), &mutex_, directories_.GetDbDir());
+        c->edit(), &mutex_, directories_.GetDbDir(),
+        /*new_descriptor_log=*/false, /*column_family_options=*/nullptr,
+        [&c, &compaction_released](const Status& s) {
+          c->ReleaseCompactionFiles(s);
+          compaction_released = true;
+        });
    io_s = versions_->io_status();
    InstallSuperVersionAndScheduleWork(c->column_family_data(),
                                       &job_context->superversion_contexts[0],
@@ -3566,6 +3705,9 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
    ROCKS_LOG_BUFFER(log_buffer, "[%s] Deleted %d files\n",
                     c->column_family_data()->GetName().c_str(),
                     c->num_input_files(0));
+    if (status.ok() && io_s.ok()) {
+      UpdateDeletionCompactionStats(c);
+    }
    *made_progress = true;
    TEST_SYNC_POINT_CALLBACK("DBImpl::BackgroundCompaction:AfterCompaction",
                             c->column_family_data());
@@ -3624,7 +3766,12 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
    }
    status = versions_->LogAndApply(
        c->column_family_data(), *c->mutable_cf_options(), read_options,
-        c->edit(), &mutex_, directories_.GetDbDir());
+        c->edit(), &mutex_, directories_.GetDbDir(),
+        /*new_descriptor_log=*/false, /*column_family_options=*/nullptr,
+        [&c, &compaction_released](const Status& s) {
+          c->ReleaseCompactionFiles(s);
+          compaction_released = true;
+        });
    io_s = versions_->io_status();
    // Use latest MutableCFOptions
    InstallSuperVersionAndScheduleWork(c->column_family_data(),
@@ -3674,6 +3821,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
    // Transfer requested token, so it doesn't need to do it again.
    ca->prepicked_compaction->task_token = std::move(task_token);
    ++bg_bottom_compaction_scheduled_;
+    assert(c == nullptr);
    env_->Schedule(&DBImpl::BGWorkBottomCompaction, ca, Env::Priority::BOTTOM,
                   this, &DBImpl::UnscheduleCompactionCallback);
  } else {
@@ -3717,8 +3865,8 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
    compaction_job.Run().PermitUncheckedError();
    TEST_SYNC_POINT("DBImpl::BackgroundCompaction:NonTrivial:AfterRun");
    mutex_.Lock();
-
-    status = compaction_job.Install(*c->mutable_cf_options());
+    status =
+        compaction_job.Install(*c->mutable_cf_options(), &compaction_released);
    io_s = compaction_job.io_status();
    if (status.ok()) {
      InstallSuperVersionAndScheduleWork(c->column_family_data(),
@@ -3737,7 +3885,23 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
  }

  if (c != nullptr) {
-    c->ReleaseCompactionFiles(status);
+    if (!compaction_released) {
+      c->ReleaseCompactionFiles(status);
+    } else {
+#ifndef NDEBUG
+      // Sanity checking that compaction files are freed.
+      for (size_t i = 0; i < c->num_input_levels(); i++) {
+        for (size_t j = 0; j < c->inputs(i)->size(); j++) {
+          assert(!c->input(i, j)->being_compacted);
+        }
+      }
+      std::unordered_set<Compaction*>* cip = c->column_family_data()
+                                                 ->compaction_picker()
+                                                 ->compactions_in_progress();
+      assert(cip->find(c.get()) == cip->end());
+#endif
+    }
+
    *made_progress = true;

    // Need to make sure SstFileManager does its bookkeeping
@@ -3922,6 +4086,27 @@ bool DBImpl::MCOverlap(ManualCompactionState* m, ManualCompactionState* m1) {
  return false;
 }

+void DBImpl::UpdateDeletionCompactionStats(
+    const std::unique_ptr<Compaction>& c) {
+  if (c == nullptr) {
+    return;
+  }
+
+  CompactionReason reason = c->compaction_reason();
+
+  switch (reason) {
+    case CompactionReason::kFIFOMaxSize:
+      RecordTick(stats_, FIFO_MAX_SIZE_COMPACTIONS);
+      break;
+    case CompactionReason::kFIFOTtl:
+      RecordTick(stats_, FIFO_TTL_COMPACTIONS);
+      break;
+    default:
+      assert(false);
+      break;
+  }
+}
+
 void DBImpl::BuildCompactionJobInfo(
    const ColumnFamilyData* cfd, Compaction* c, const Status& st,
    const CompactionJobStats& compaction_job_stats, const int job_id,
@@ -3935,7 +4120,12 @@ void DBImpl::BuildCompactionJobInfo(
  compaction_job_info->base_input_level = c->start_level();
  compaction_job_info->output_level = c->output_level();
  compaction_job_info->stats = compaction_job_stats;
-  compaction_job_info->table_properties = c->GetTableProperties();
+  const auto& input_table_properties = c->GetInputTableProperties();
+  const auto& output_table_properties = c->GetOutputTableProperties();
+  compaction_job_info->table_properties.insert(input_table_properties.begin(),
+                                               input_table_properties.end());
+  compaction_job_info->table_properties.insert(output_table_properties.begin(),
+                                               output_table_properties.end());
  compaction_job_info->compaction_reason = c->compaction_reason();
  compaction_job_info->compression = c->output_compression();

@@ -306,11 +306,16 @@ const PeriodicTaskScheduler& DBImpl::TEST_GetPeriodicTaskScheduler() const {

 SeqnoToTimeMapping DBImpl::TEST_GetSeqnoToTimeMapping() const {
  InstrumentedMutexLock l(&mutex_);
-  return seqno_time_mapping_;
+  return seqno_to_time_mapping_;
 }

+const autovector<uint64_t>& DBImpl::TEST_GetFilesToQuarantine() const {
+  InstrumentedMutexLock l(&mutex_);
+  return error_handler_.GetFilesToQuarantine();
+}

 size_t DBImpl::TEST_EstimateInMemoryStatsHistorySize() const {
+  InstrumentedMutexLock l(&const_cast<DBImpl*>(this)->stats_history_mutex_);
  return EstimateInMemoryStatsHistorySize();
 }
 }  // namespace ROCKSDB_NAMESPACE
@@ -146,6 +146,7 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
  // mutex_ cannot be released. Otherwise, we might see no min_pending_output
  // here but later find newer generated unfinalized files while scanning.
  job_context->min_pending_output = MinObsoleteSstNumberToKeep();
+  job_context->files_to_quarantine = error_handler_.GetFilesToQuarantine();

  // Get obsolete files.  This function will also update the list of
  // pending files in VersionSet().
@@ -419,6 +420,8 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
                                             state.blob_live.end());
  std::unordered_set<uint64_t> log_recycle_files_set(
      state.log_recycle_files.begin(), state.log_recycle_files.end());
+  std::unordered_set<uint64_t> quarantine_files_set(
+      state.files_to_quarantine.begin(), state.files_to_quarantine.end());

  auto candidate_files = state.full_scan_candidate_files;
  candidate_files.reserve(
@@ -457,12 +460,12 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
  std::sort(candidate_files.begin(), candidate_files.end(),
            [](const JobContext::CandidateFileInfo& lhs,
               const JobContext::CandidateFileInfo& rhs) {
-              if (lhs.file_name > rhs.file_name) {
+              if (lhs.file_name < rhs.file_name) {
                return true;
-              } else if (lhs.file_name < rhs.file_name) {
+              } else if (lhs.file_name > rhs.file_name) {
                return false;
              } else {
-                return (lhs.file_path > rhs.file_path);
+                return (lhs.file_path < rhs.file_path);
              }
            });
  candidate_files.erase(
@@ -522,6 +525,10 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
      continue;
    }

+    if (quarantine_files_set.find(number) != quarantine_files_set.end()) {
+      continue;
+    }
+
    bool keep = true;
    switch (type) {
      case kWalFile:
@@ -995,7 +1002,7 @@ Status DBImpl::DeleteUnreferencedSstFiles(RecoveryContext* recovery_ctx) {
      if (type == kTableFile && number >= next_file_number &&
          recovery_ctx->files_to_delete_.find(normalized_fpath) ==
              recovery_ctx->files_to_delete_.end()) {
-        recovery_ctx->files_to_delete_.emplace(normalized_fpath);
+        recovery_ctx->files_to_delete_.emplace(normalized_fpath, path);
      }
    }
  }
@@ -25,6 +25,7 @@
 #include "rocksdb/wal_filter.h"
 #include "test_util/sync_point.h"
 #include "util/rate_limiter_impl.h"
+#include "util/string_util.h"
 #include "util/udt_util.h"

 namespace ROCKSDB_NAMESPACE {
@@ -143,13 +144,6 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src,
    result.wal_dir = result.wal_dir.substr(0, result.wal_dir.size() - 1);
  }

-  if (result.compaction_readahead_size == 0) {
-    if (result.use_direct_reads) {
-      TEST_SYNC_POINT_CALLBACK("SanitizeOptions:direct_io", nullptr);
-    }
-    result.compaction_readahead_size = 1024 * 1024 * 2;
-  }
-
  // Force flush on DB open if 2PC is enabled, since with 2PC we have no
  // guarantee that consecutive log files have consecutive sequence id, which
  // make recovery complicated.
@@ -298,6 +292,18 @@ Status DBImpl::ValidateOptions(const DBOptions& db_options) {
        "writes in direct IO require writable_file_max_buffer_size > 0");
  }

+  if (db_options.daily_offpeak_time_utc != "") {
+    int start_time, end_time;
+    if (!TryParseTimeRangeString(db_options.daily_offpeak_time_utc, start_time,
+                                 end_time)) {
+      return Status::InvalidArgument(
+          "daily_offpeak_time_utc should be set in the format HH:mm-HH:mm "
+          "(e.g. 04:30-07:30)");
+    } else if (start_time == end_time) {
+      return Status::InvalidArgument(
+          "start_time and end_time cannot be the same");
+    }
+  }
  return Status::OK();
 }

@@ -412,7 +418,8 @@ Status DBImpl::Recover(
    uint64_t* recovered_seq, RecoveryContext* recovery_ctx) {
  mutex_.AssertHeld();

-  bool is_new_db = false;
+  bool tmp_is_new_db = false;
+  bool& is_new_db = recovery_ctx ? recovery_ctx->is_new_db_ : tmp_is_new_db;
  assert(db_lock_ == nullptr);
  std::vector<std::string> files_in_dbname;
  if (!read_only) {
@@ -865,7 +872,8 @@ Status DBImpl::PersistentStatsProcessFormatVersion() {
      if (s.ok()) {
        ColumnFamilyOptions cfo;
        OptimizeForPersistentStats(&cfo);
-        s = CreateColumnFamily(cfo, kPersistentStatsColumnFamilyName, &handle);
+        s = CreateColumnFamilyImpl(cfo, kPersistentStatsColumnFamilyName,
+                                   &handle);
      }
      if (s.ok()) {
        persist_stats_cf_handle_ = static_cast<ColumnFamilyHandleImpl*>(handle);
@@ -918,7 +926,7 @@ Status DBImpl::InitPersistStatsColumnFamily() {
    ColumnFamilyHandle* handle = nullptr;
    ColumnFamilyOptions cfo;
    OptimizeForPersistentStats(&cfo);
-    s = CreateColumnFamily(cfo, kPersistentStatsColumnFamilyName, &handle);
+    s = CreateColumnFamilyImpl(cfo, kPersistentStatsColumnFamilyName, &handle);
    persist_stats_cf_handle_ = static_cast<ColumnFamilyHandleImpl*>(handle);
    mutex_.Lock();
  }
@@ -934,8 +942,11 @@ Status DBImpl::LogAndApplyForRecovery(const RecoveryContext& recovery_ctx) {
      recovery_ctx.edit_lists_, &mutex_, directories_.GetDbDir());
  if (s.ok() && !(recovery_ctx.files_to_delete_.empty())) {
    mutex_.Unlock();
-    for (const auto& fname : recovery_ctx.files_to_delete_) {
-      s = env_->DeleteFile(fname);
+    for (const auto& stale_sst_file : recovery_ctx.files_to_delete_) {
+      s = DeleteDBFile(&immutable_db_options_, stale_sst_file.first,
+                       stale_sst_file.second,
+                       /*force_bg=*/false,
+                       /*force_fg=*/false);
      if (!s.ok()) {
        break;
      }
@@ -1240,6 +1251,13 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& wal_numbers,
      }

      SequenceNumber sequence = WriteBatchInternal::Sequence(batch_to_use);
+      if (sequence > kMaxSequenceNumber) {
+        reporter.Corruption(
+            record.size(),
+            Status::Corruption("sequence " + std::to_string(sequence) +
+                               " is too large"));
+        continue;
+      }

      if (immutable_db_options_.wal_recovery_mode ==
          WALRecoveryMode::kPointInTimeRecovery) {
@@ -1305,7 +1323,7 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& wal_numbers,
          flushed = true;

          cfd->CreateNewMemtable(*cfd->GetLatestMutableCFOptions(),
-                                 *next_sequence);
+                                 *next_sequence - 1);
        }
      }
    }
@@ -1656,7 +1674,7 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
          TableFileCreationReason::kRecovery, 0 /* oldest_key_time */,
          0 /* file_creation_time */, db_id_, db_session_id_,
          0 /* target_file_size */, meta.fd.GetNumber());
-      SeqnoToTimeMapping empty_seqno_time_mapping;
+      SeqnoToTimeMapping empty_seqno_to_time_mapping;
      Version* version = cfd->current();
      version->Ref();
      const ReadOptions read_option(Env::IOActivity::kDBOpen);
@@ -1668,7 +1686,7 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
          snapshot_seqs, earliest_write_conflict_snapshot, kMaxSequenceNumber,
          snapshot_checker, paranoid_file_checks, cfd->internal_stats(), &io_s,
          io_tracer_, BlobFileCreationReason::kRecovery,
-          empty_seqno_time_mapping, &event_logger_, job_id, Env::IO_HIGH,
+          empty_seqno_to_time_mapping, &event_logger_, job_id, Env::IO_HIGH,
          nullptr /* table_properties */, write_hint,
          nullptr /*full_history_ts_low*/, &blob_callback_, version,
          &num_input_entries);
@@ -1722,6 +1740,22 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
    for (const auto& blob : blob_file_additions) {
      edit->AddBlobFile(blob);
    }
+
+    // For UDT in memtable only feature, move up the cutoff timestamp whenever
+    // a flush happens.
+    const Comparator* ucmp = cfd->user_comparator();
+    size_t ts_sz = ucmp->timestamp_size();
+    if (ts_sz > 0 && !cfd->ioptions()->persist_user_defined_timestamps) {
+      Slice mem_newest_udt = mem->GetNewestUDT();
+      std::string full_history_ts_low = cfd->GetFullHistoryTsLow();
+      if (full_history_ts_low.empty() ||
+          ucmp->CompareTimestamp(mem_newest_udt, full_history_ts_low) >= 0) {
+        std::string new_full_history_ts_low;
+        GetFullHistoryTsLowFromU64CutoffTs(&mem_newest_udt,
+                                           &new_full_history_ts_low);
+        edit->SetFullHistoryTsLow(new_full_history_ts_low);
+      }
+    }
  }

  InternalStats::CompactionStats stats(CompactionReason::kFlush, 1);
@@ -1966,6 +2000,7 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,

  impl->wal_in_db_path_ = impl->immutable_db_options_.IsWalDirSameAsDBPath();
  RecoveryContext recovery_ctx;
+  impl->options_mutex_.Lock();
  impl->mutex_.Lock();

  // Handles create_if_missing, error_if_exists
@@ -2047,7 +2082,9 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
          // missing column family, create it
          ColumnFamilyHandle* handle = nullptr;
          impl->mutex_.Unlock();
-          s = impl->CreateColumnFamily(cf.options, cf.name, &handle);
+          // NOTE: the work normally done in WrapUpCreateColumnFamilies will
+          // be done separately below.
+          s = impl->CreateColumnFamilyImpl(cf.options, cf.name, &handle);
          impl->mutex_.Lock();
          if (s.ok()) {
            handles->push_back(handle);
@@ -2098,9 +2135,8 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
  if (s.ok()) {
    // Persist RocksDB Options before scheduling the compaction.
    // The WriteOptionsFile() will release and lock the mutex internally.
-    persist_options_status = impl->WriteOptionsFile(
-        false /*need_mutex_lock*/, false /*need_enter_write_thread*/);
-
+    persist_options_status =
+        impl->WriteOptionsFile(true /*db_mutex_already_held*/);
    *dbptr = impl;
    impl->opened_successfully_ = true;
    impl->DeleteObsoleteFiles();
@@ -2221,10 +2257,10 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
  if (s.ok()) {
    s = impl->StartPeriodicTaskScheduler();
  }
-
  if (s.ok()) {
-    s = impl->RegisterRecordSeqnoTimeWorker();
+    s = impl->RegisterRecordSeqnoTimeWorker(recovery_ctx.is_new_db_);
  }
+  impl->options_mutex_.Unlock();
  if (!s.ok()) {
    for (auto* h : *handles) {
      delete h;
@@ -29,42 +29,23 @@ DBImplReadOnly::DBImplReadOnly(const DBOptions& db_options,
 DBImplReadOnly::~DBImplReadOnly() {}

 // Implementations of the DB interface
-Status DBImplReadOnly::Get(const ReadOptions& read_options,
-                           ColumnFamilyHandle* column_family, const Slice& key,
-                           PinnableSlice* pinnable_val) {
-  return Get(read_options, column_family, key, pinnable_val,
-             /*timestamp*/ nullptr);
-}
+Status DBImplReadOnly::GetImpl(const ReadOptions& read_options,
+                               const Slice& key,
+                               GetImplOptions& get_impl_options) {
+  assert(get_impl_options.value != nullptr ||
+         get_impl_options.columns != nullptr);
+  assert(get_impl_options.column_family);

-Status DBImplReadOnly::Get(const ReadOptions& _read_options,
-                           ColumnFamilyHandle* column_family, const Slice& key,
-                           PinnableSlice* pinnable_val,
-                           std::string* timestamp) {
-  if (_read_options.io_activity != Env::IOActivity::kUnknown &&
-      _read_options.io_activity != Env::IOActivity::kGet) {
-    return Status::InvalidArgument(
-        "Can only call Get with `ReadOptions::io_activity` is "
-        "`Env::IOActivity::kUnknown` or `Env::IOActivity::kGet`");
-  }
-  ReadOptions read_options(_read_options);
-  if (read_options.io_activity == Env::IOActivity::kUnknown) {
-    read_options.io_activity = Env::IOActivity::kGet;
-  }
-  assert(pinnable_val != nullptr);
-  PERF_CPU_TIMER_GUARD(get_cpu_nanos, immutable_db_options_.clock);
-  StopWatch sw(immutable_db_options_.clock, stats_, DB_GET);
-  PERF_TIMER_GUARD(get_snapshot_time);
+  Status s;

-  assert(column_family);
  if (read_options.timestamp) {
-    const Status s =
-        FailIfTsMismatchCf(column_family, *(read_options.timestamp),
-                           /*ts_for_read=*/true);
+    s = FailIfTsMismatchCf(get_impl_options.column_family,
+                           *(read_options.timestamp));
    if (!s.ok()) {
      return s;
    }
  } else {
-    const Status s = FailIfCfHasTs(column_family);
+    s = FailIfCfHasTs(get_impl_options.column_family);
    if (!s.ok()) {
      return s;
    }
@@ -72,53 +53,80 @@ Status DBImplReadOnly::Get(const ReadOptions& _read_options,

  // Clear the timestamps for returning results so that we can distinguish
  // between tombstone or key that has never been written
-  if (timestamp) {
-    timestamp->clear();
+  if (get_impl_options.timestamp) {
+    get_impl_options.timestamp->clear();
  }

-  const Comparator* ucmp = column_family->GetComparator();
-  assert(ucmp);
-  std::string* ts = ucmp->timestamp_size() > 0 ? timestamp : nullptr;
+  PERF_CPU_TIMER_GUARD(get_cpu_nanos, immutable_db_options_.clock);
+  StopWatch sw(immutable_db_options_.clock, stats_, DB_GET);
+  PERF_TIMER_GUARD(get_snapshot_time);

-  Status s;
+  const Comparator* ucmp = get_impl_options.column_family->GetComparator();
+  assert(ucmp);
+  std::string* ts =
+      ucmp->timestamp_size() > 0 ? get_impl_options.timestamp : nullptr;
  SequenceNumber snapshot = versions_->LastSequence();
  GetWithTimestampReadCallback read_cb(snapshot);
-  auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(column_family);
+  auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(
+      get_impl_options.column_family);
  auto cfd = cfh->cfd();
  if (tracer_) {
    InstrumentedMutexLock lock(&trace_mutex_);
    if (tracer_) {
-      tracer_->Get(column_family, key);
+      tracer_->Get(get_impl_options.column_family, key);
    }
  }
+
+  // In read-only mode Get(), no super version operation is needed (i.e.
+  // GetAndRefSuperVersion and ReturnAndCleanupSuperVersion)
  SuperVersion* super_version = cfd->GetSuperVersion();
+  if (read_options.timestamp && read_options.timestamp->size() > 0) {
+    s = FailIfReadCollapsedHistory(cfd, super_version,
+                                   *(read_options.timestamp));
+    if (!s.ok()) {
+      return s;
+    }
+  }
  MergeContext merge_context;
  SequenceNumber max_covering_tombstone_seq = 0;
  LookupKey lkey(key, snapshot, read_options.timestamp);
  PERF_TIMER_STOP(get_snapshot_time);
-  if (super_version->mem->Get(lkey, pinnable_val->GetSelf(),
-                              /*columns=*/nullptr, ts, &s, &merge_context,
-                              &max_covering_tombstone_seq, read_options,
-                              false /* immutable_memtable */, &read_cb)) {
-    pinnable_val->PinSelf();
+
+  // Look up starts here
+  if (super_version->mem->Get(
+          lkey,
+          get_impl_options.value ? get_impl_options.value->GetSelf() : nullptr,
+          get_impl_options.columns, ts, &s, &merge_context,
+          &max_covering_tombstone_seq, read_options,
+          false /* immutable_memtable */, &read_cb)) {
+    if (get_impl_options.value) {
+      get_impl_options.value->PinSelf();
+    }
    RecordTick(stats_, MEMTABLE_HIT);
  } else {
    PERF_TIMER_GUARD(get_from_output_files_time);
    PinnedIteratorsManager pinned_iters_mgr;
    super_version->current->Get(
-        read_options, lkey, pinnable_val, /*columns=*/nullptr, ts, &s,
-        &merge_context, &max_covering_tombstone_seq, &pinned_iters_mgr,
+        read_options, lkey, get_impl_options.value, get_impl_options.columns,
+        ts, &s, &merge_context, &max_covering_tombstone_seq, &pinned_iters_mgr,
        /*value_found*/ nullptr,
        /*key_exists*/ nullptr, /*seq*/ nullptr, &read_cb,
        /*is_blob*/ nullptr,
        /*do_merge*/ true);
    RecordTick(stats_, MEMTABLE_MISS);
  }
-  RecordTick(stats_, NUMBER_KEYS_READ);
-  size_t size = pinnable_val->size();
-  RecordTick(stats_, BYTES_READ, size);
-  RecordInHistogram(stats_, BYTES_PER_READ, size);
-  PERF_COUNTER_ADD(get_read_bytes, size);
+  {
+    RecordTick(stats_, NUMBER_KEYS_READ);
+    size_t size = 0;
+    if (get_impl_options.value) {
+      size = get_impl_options.value->size();
+    } else if (get_impl_options.columns) {
+      size = get_impl_options.columns->serialized_size();
+    }
+    RecordTick(stats_, BYTES_READ, size);
+    RecordInHistogram(stats_, BYTES_PER_READ, size);
+    PERF_COUNTER_ADD(get_read_bytes, size);
+  }
  return s;
 }

@@ -137,8 +145,7 @@ Iterator* DBImplReadOnly::NewIterator(const ReadOptions& _read_options,
  assert(column_family);
  if (read_options.timestamp) {
    const Status s =
-        FailIfTsMismatchCf(column_family, *(read_options.timestamp),
-                           /*ts_for_read=*/true);
+        FailIfTsMismatchCf(column_family, *(read_options.timestamp));
    if (!s.ok()) {
      return NewErrorIterator(s);
    }
@@ -151,6 +158,14 @@ Iterator* DBImplReadOnly::NewIterator(const ReadOptions& _read_options,
  auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(column_family);
  auto cfd = cfh->cfd();
  SuperVersion* super_version = cfd->GetSuperVersion()->Ref();
+  if (read_options.timestamp && read_options.timestamp->size() > 0) {
+    const Status s = FailIfReadCollapsedHistory(cfd, super_version,
+                                                *(read_options.timestamp));
+    if (!s.ok()) {
+      cfd->GetSuperVersion()->Unref();
+      return NewErrorIterator(s);
+    }
+  }
  SequenceNumber latest_snapshot = versions_->LastSequence();
  SequenceNumber read_seq =
      read_options.snapshot != nullptr
@@ -177,8 +192,7 @@ Status DBImplReadOnly::NewIterators(
  if (read_options.timestamp) {
    for (auto* cf : column_families) {
      assert(cf);
-      const Status s = FailIfTsMismatchCf(cf, *(read_options.timestamp),
-                                          /*ts_for_read=*/true);
+      const Status s = FailIfTsMismatchCf(cf, *(read_options.timestamp));
      if (!s.ok()) {
        return s;
      }
@@ -206,9 +220,27 @@ Status DBImplReadOnly::NewIterators(
                ->number_
          : latest_snapshot;

+  autovector<std::tuple<ColumnFamilyData*, SuperVersion*>> cfd_to_sv;
+
+  const bool check_read_ts =
+      read_options.timestamp && read_options.timestamp->size() > 0;
  for (auto cfh : column_families) {
    auto* cfd = static_cast_with_check<ColumnFamilyHandleImpl>(cfh)->cfd();
    auto* sv = cfd->GetSuperVersion()->Ref();
+    cfd_to_sv.emplace_back(cfd, sv);
+    if (check_read_ts) {
+      const Status s =
+          FailIfReadCollapsedHistory(cfd, sv, *(read_options.timestamp));
+      if (!s.ok()) {
+        for (auto prev_entry : cfd_to_sv) {
+          std::get<1>(prev_entry)->Unref();
+        }
+        return s;
+      }
+    }
+  }
+  assert(cfd_to_sv.size() == column_families.size());
+  for (auto [cfd, sv] : cfd_to_sv) {
    auto* db_iter = NewArenaWrappedDbIterator(
        env_, read_options, *cfd->ioptions(), sv->mutable_cf_options,
        sv->current, read_seq,
@@ -24,13 +24,9 @@ class DBImplReadOnly : public DBImpl {
  virtual ~DBImplReadOnly();

  // Implementations of the DB interface
-  using DB::Get;
-  virtual Status Get(const ReadOptions& options,
-                     ColumnFamilyHandle* column_family, const Slice& key,
-                     PinnableSlice* value) override;
-  Status Get(const ReadOptions& _read_options,
-             ColumnFamilyHandle* column_family, const Slice& key,
-             PinnableSlice* value, std::string* timestamp) override;
+  using DBImpl::GetImpl;
+  Status GetImpl(const ReadOptions& options, const Slice& key,
+                 GetImplOptions& get_impl_options) override;

  // TODO: Implement ReadOnly MultiGet?

@@ -57,6 +53,10 @@ class DBImplReadOnly : public DBImpl {
                   const WideColumns& /* columns */) override {
    return Status::NotSupported("Not supported operation in read only mode.");
  }
+  Status PutEntity(const WriteOptions& /* options */, const Slice& /* key */,
+                   const AttributeGroups& /* attribute_groups */) override {
+    return Status::NotSupported("Not supported operation in read only mode.");
+  }

  using DBImpl::Merge;
  virtual Status Merge(const WriteOptions& /*options*/,
@@ -339,105 +339,93 @@ Status DBImplSecondary::RecoverLogFiles(
  return status;
 }

-// Implementation of the DB interface
-Status DBImplSecondary::Get(const ReadOptions& _read_options,
-                            ColumnFamilyHandle* column_family, const Slice& key,
-                            PinnableSlice* value) {
-  if (_read_options.io_activity != Env::IOActivity::kUnknown &&
-      _read_options.io_activity != Env::IOActivity::kGet) {
-    return Status::InvalidArgument(
-        "Can only call Get with `ReadOptions::io_activity` is "
-        "`Env::IOActivity::kUnknown` or `Env::IOActivity::kGet`");
-  }
-  ReadOptions read_options(_read_options);
-  if (read_options.io_activity == Env::IOActivity::kUnknown) {
-    read_options.io_activity = Env::IOActivity::kGet;
-  }
-  return GetImpl(read_options, column_family, key, value,
-                 /*timestamp*/ nullptr);
-}
-
-Status DBImplSecondary::Get(const ReadOptions& _read_options,
-                            ColumnFamilyHandle* column_family, const Slice& key,
-                            PinnableSlice* value, std::string* timestamp) {
-  if (_read_options.io_activity != Env::IOActivity::kUnknown &&
-      _read_options.io_activity != Env::IOActivity::kGet) {
-    return Status::InvalidArgument(
-        "Can only call Get with `ReadOptions::io_activity` is "
-        "`Env::IOActivity::kUnknown` or `Env::IOActivity::kGet`");
-  }
-  ReadOptions read_options(_read_options);
-  if (read_options.io_activity == Env::IOActivity::kUnknown) {
-    read_options.io_activity = Env::IOActivity::kGet;
-  }
-  return GetImpl(read_options, column_family, key, value, timestamp);
-}
-
 Status DBImplSecondary::GetImpl(const ReadOptions& read_options,
-                                ColumnFamilyHandle* column_family,
-                                const Slice& key, PinnableSlice* pinnable_val,
-                                std::string* timestamp) {
-  assert(pinnable_val != nullptr);
-  PERF_CPU_TIMER_GUARD(get_cpu_nanos, immutable_db_options_.clock);
-  StopWatch sw(immutable_db_options_.clock, stats_, DB_GET);
-  PERF_TIMER_GUARD(get_snapshot_time);
+                                const Slice& key,
+                                GetImplOptions& get_impl_options) {
+  assert(get_impl_options.value != nullptr ||
+         get_impl_options.columns != nullptr);
+  assert(get_impl_options.column_family);
+
+  Status s;

-  assert(column_family);
  if (read_options.timestamp) {
-    const Status s = FailIfTsMismatchCf(
-        column_family, *(read_options.timestamp), /*ts_for_read=*/true);
+    s = FailIfTsMismatchCf(get_impl_options.column_family,
+                           *(read_options.timestamp));
    if (!s.ok()) {
      return s;
    }
  } else {
-    const Status s = FailIfCfHasTs(column_family);
+    s = FailIfCfHasTs(get_impl_options.column_family);
    if (!s.ok()) {
      return s;
    }
  }

-  // Clear the timestamp for returning results so that we can distinguish
-  // between tombstone or key that has never been written later.
-  if (timestamp) {
-    timestamp->clear();
+  // Clear the timestamps for returning results so that we can distinguish
+  // between tombstone or key that has never been written
+  if (get_impl_options.timestamp) {
+    get_impl_options.timestamp->clear();
  }

-  auto cfh = static_cast<ColumnFamilyHandleImpl*>(column_family);
-  ColumnFamilyData* cfd = cfh->cfd();
+  PERF_CPU_TIMER_GUARD(get_cpu_nanos, immutable_db_options_.clock);
+  StopWatch sw(immutable_db_options_.clock, stats_, DB_GET);
+  PERF_TIMER_GUARD(get_snapshot_time);
+
+  const Comparator* ucmp = get_impl_options.column_family->GetComparator();
+  assert(ucmp);
+  std::string* ts =
+      ucmp->timestamp_size() > 0 ? get_impl_options.timestamp : nullptr;
+  SequenceNumber snapshot = versions_->LastSequence();
+  GetWithTimestampReadCallback read_cb(snapshot);
+  auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(
+      get_impl_options.column_family);
+  auto cfd = cfh->cfd();
  if (tracer_) {
    InstrumentedMutexLock lock(&trace_mutex_);
    if (tracer_) {
-      tracer_->Get(column_family, key);
+      tracer_->Get(get_impl_options.column_family, key);
    }
  }
+
  // Acquire SuperVersion
  SuperVersion* super_version = GetAndRefSuperVersion(cfd);
-  SequenceNumber snapshot = versions_->LastSequence();
-  GetWithTimestampReadCallback read_cb(snapshot);
+  if (read_options.timestamp && read_options.timestamp->size() > 0) {
+    s = FailIfReadCollapsedHistory(cfd, super_version,
+                                   *(read_options.timestamp));
+    if (!s.ok()) {
+      ReturnAndCleanupSuperVersion(cfd, super_version);
+      return s;
+    }
+  }
  MergeContext merge_context;
  SequenceNumber max_covering_tombstone_seq = 0;
-  Status s;
  LookupKey lkey(key, snapshot, read_options.timestamp);
  PERF_TIMER_STOP(get_snapshot_time);
-
  bool done = false;
-  const Comparator* ucmp = column_family->GetComparator();
-  assert(ucmp);
-  std::string* ts = ucmp->timestamp_size() > 0 ? timestamp : nullptr;
-  if (super_version->mem->Get(lkey, pinnable_val->GetSelf(),
-                              /*columns=*/nullptr, ts, &s, &merge_context,
-                              &max_covering_tombstone_seq, read_options,
-                              false /* immutable_memtable */, &read_cb)) {
+
+  // Look up starts here
+  if (super_version->mem->Get(
+          lkey,
+          get_impl_options.value ? get_impl_options.value->GetSelf() : nullptr,
+          get_impl_options.columns, ts, &s, &merge_context,
+          &max_covering_tombstone_seq, read_options,
+          false /* immutable_memtable */, &read_cb)) {
    done = true;
-    pinnable_val->PinSelf();
+    if (get_impl_options.value) {
+      get_impl_options.value->PinSelf();
+    }
    RecordTick(stats_, MEMTABLE_HIT);
  } else if ((s.ok() || s.IsMergeInProgress()) &&
             super_version->imm->Get(
-                 lkey, pinnable_val->GetSelf(), /*columns=*/nullptr, ts, &s,
-                 &merge_context, &max_covering_tombstone_seq, read_options,
-                 &read_cb)) {
+                 lkey,
+                 get_impl_options.value ? get_impl_options.value->GetSelf()
+                                        : nullptr,
+                 get_impl_options.columns, ts, &s, &merge_context,
+                 &max_covering_tombstone_seq, read_options, &read_cb)) {
    done = true;
-    pinnable_val->PinSelf();
+    if (get_impl_options.value) {
+      get_impl_options.value->PinSelf();
+    }
    RecordTick(stats_, MEMTABLE_HIT);
  }
  if (!done && !s.ok() && !s.IsMergeInProgress()) {
@@ -448,8 +436,8 @@ Status DBImplSecondary::GetImpl(const ReadOptions& read_options,
    PERF_TIMER_GUARD(get_from_output_files_time);
    PinnedIteratorsManager pinned_iters_mgr;
    super_version->current->Get(
-        read_options, lkey, pinnable_val, /*columns=*/nullptr, ts, &s,
-        &merge_context, &max_covering_tombstone_seq, &pinned_iters_mgr,
+        read_options, lkey, get_impl_options.value, get_impl_options.columns,
+        ts, &s, &merge_context, &max_covering_tombstone_seq, &pinned_iters_mgr,
        /*value_found*/ nullptr,
        /*key_exists*/ nullptr, /*seq*/ nullptr, &read_cb, /*is_blob*/ nullptr,
        /*do_merge*/ true);
@@ -459,7 +447,12 @@ Status DBImplSecondary::GetImpl(const ReadOptions& read_options,
    PERF_TIMER_GUARD(get_post_process_time);
    ReturnAndCleanupSuperVersion(cfd, super_version);
    RecordTick(stats_, NUMBER_KEYS_READ);
-    size_t size = pinnable_val->size();
+    size_t size = 0;
+    if (get_impl_options.value) {
+      size = get_impl_options.value->size();
+    } else if (get_impl_options.columns) {
+      size = get_impl_options.columns->serialized_size();
+    }
    RecordTick(stats_, BYTES_READ, size);
    RecordTimeToHistogram(stats_, BYTES_PER_READ, size);
    PERF_COUNTER_ADD(get_read_bytes, size);
@@ -491,8 +484,7 @@ Iterator* DBImplSecondary::NewIterator(const ReadOptions& _read_options,
  assert(column_family);
  if (read_options.timestamp) {
    const Status s =
-        FailIfTsMismatchCf(column_family, *(read_options.timestamp),
-                           /*ts_for_read=*/true);
+        FailIfTsMismatchCf(column_family, *(read_options.timestamp));
    if (!s.ok()) {
      return NewErrorIterator(s);
    }
@@ -516,17 +508,25 @@ Iterator* DBImplSecondary::NewIterator(const ReadOptions& _read_options,
        Status::NotSupported("snapshot not supported in secondary mode"));
  } else {
    SequenceNumber snapshot(kMaxSequenceNumber);
-    result = NewIteratorImpl(read_options, cfd, snapshot, read_callback);
+    SuperVersion* sv = cfd->GetReferencedSuperVersion(this);
+    if (read_options.timestamp && read_options.timestamp->size() > 0) {
+      const Status s =
+          FailIfReadCollapsedHistory(cfd, sv, *(read_options.timestamp));
+      if (!s.ok()) {
+        CleanupSuperVersion(sv);
+        return NewErrorIterator(s);
+      }
+    }
+    result = NewIteratorImpl(read_options, cfd, sv, snapshot, read_callback);
  }
  return result;
 }

 ArenaWrappedDBIter* DBImplSecondary::NewIteratorImpl(
    const ReadOptions& read_options, ColumnFamilyData* cfd,
-    SequenceNumber snapshot, ReadCallback* read_callback,
-    bool expose_blob_index, bool allow_refresh) {
+    SuperVersion* super_version, SequenceNumber snapshot,
+    ReadCallback* read_callback, bool expose_blob_index, bool allow_refresh) {
  assert(nullptr != cfd);
-  SuperVersion* super_version = cfd->GetReferencedSuperVersion(this);
  assert(snapshot == kMaxSequenceNumber);
  snapshot = versions_->LastSequence();
  assert(snapshot != kMaxSequenceNumber);
@@ -535,7 +535,7 @@ ArenaWrappedDBIter* DBImplSecondary::NewIteratorImpl(
      super_version->current, snapshot,
      super_version->mutable_cf_options.max_sequential_skip_in_iterations,
      super_version->version_number, read_callback, this, cfd,
-      expose_blob_index, read_options.snapshot ? false : allow_refresh);
+      expose_blob_index, allow_refresh);
  auto internal_iter = NewInternalIterator(
      db_iter->GetReadOptions(), cfd, super_version, db_iter->GetArena(),
      snapshot, /* allow_unprepared_value */ true, db_iter);
@@ -572,8 +572,7 @@ Status DBImplSecondary::NewIterators(
  if (read_options.timestamp) {
    for (auto* cf : column_families) {
      assert(cf);
-      const Status s = FailIfTsMismatchCf(cf, *(read_options.timestamp),
-                                          /*ts_for_read=*/true);
+      const Status s = FailIfTsMismatchCf(cf, *(read_options.timestamp));
      if (!s.ok()) {
        return s;
      }
@@ -597,10 +596,28 @@ Status DBImplSecondary::NewIterators(
    return Status::NotSupported("snapshot not supported in secondary mode");
  } else {
    SequenceNumber read_seq(kMaxSequenceNumber);
+    autovector<std::tuple<ColumnFamilyData*, SuperVersion*>> cfd_to_sv;
+    const bool check_read_ts =
+        read_options.timestamp && read_options.timestamp->size() > 0;
    for (auto cfh : column_families) {
      ColumnFamilyData* cfd = static_cast<ColumnFamilyHandleImpl*>(cfh)->cfd();
+      SuperVersion* sv = cfd->GetReferencedSuperVersion(this);
+      cfd_to_sv.emplace_back(cfd, sv);
+      if (check_read_ts) {
+        const Status s =
+            FailIfReadCollapsedHistory(cfd, sv, *(read_options.timestamp));
+        if (!s.ok()) {
+          for (auto prev_entry : cfd_to_sv) {
+            CleanupSuperVersion(std::get<1>(prev_entry));
+          }
+          return s;
+        }
+      }
+    }
+    assert(cfd_to_sv.size() == column_families.size());
+    for (auto [cfd, sv] : cfd_to_sv) {
      iterators->push_back(
-          NewIteratorImpl(read_options, cfd, read_seq, read_callback));
+          NewIteratorImpl(read_options, cfd, sv, read_seq, read_callback));
    }
  }
  return Status::OK();
@@ -868,7 +885,7 @@ Status DBImplSecondary::CompactWithoutInstallation(
      *mutable_cf_options, mutable_db_options_, 0));
  assert(c != nullptr);

-  c->SetInputVersion(version);
+  c->FinalizeInputInfo(version);

  // Create output directory if it's not existed yet
  std::unique_ptr<FSDirectory> output_dir;
@@ -85,8 +85,6 @@ class DBImplSecondary : public DBImpl {
                 bool error_if_data_exists_in_wals, uint64_t* = nullptr,
                 RecoveryContext* recovery_ctx = nullptr) override;

-  // Implementations of the DB interface.
-  using DB::Get;
  // Can return IOError due to files being deleted by the primary. To avoid
  // IOError in this case, application can coordinate between primary and
  // secondaries so that primary will not delete files that are currently being
@@ -96,17 +94,9 @@ class DBImplSecondary : public DBImpl {
  // workaround, the secondaries can be opened with `max_open_files=-1` so that
  // it eagerly keeps all talbe files open and is able to access the contents of
  // deleted files via prior open fd.
-  Status Get(const ReadOptions& _read_options,
-             ColumnFamilyHandle* column_family, const Slice& key,
-             PinnableSlice* value) override;
-
-  Status Get(const ReadOptions& _read_options,
-             ColumnFamilyHandle* column_family, const Slice& key,
-             PinnableSlice* value, std::string* timestamp) override;
-
-  Status GetImpl(const ReadOptions& options, ColumnFamilyHandle* column_family,
-                 const Slice& key, PinnableSlice* value,
-                 std::string* timestamp);
+  using DBImpl::GetImpl;
+  Status GetImpl(const ReadOptions& options, const Slice& key,
+                 GetImplOptions& get_impl_options) override;

  using DBImpl::NewIterator;
  // Operations on the created iterators can return IOError due to files being
@@ -122,7 +112,7 @@ class DBImplSecondary : public DBImpl {
                        ColumnFamilyHandle* column_family) override;

  ArenaWrappedDBIter* NewIteratorImpl(const ReadOptions& read_options,
-                                      ColumnFamilyData* cfd,
+                                      ColumnFamilyData* cfd, SuperVersion* sv,
                                      SequenceNumber snapshot,
                                      ReadCallback* read_callback,
                                      bool expose_blob_index = false,
@@ -146,6 +136,10 @@ class DBImplSecondary : public DBImpl {
                   const WideColumns& /* columns */) override {
    return Status::NotSupported("Not supported operation in secondary mode.");
  }
+  Status PutEntity(const WriteOptions& /* options */, const Slice& /* key */,
+                   const AttributeGroups& /* attribute_groups */) override {
+    return Status::NotSupported("Not supported operation in secondary mode.");
+  }

  using DBImpl::Merge;
  Status Merge(const WriteOptions& /*options*/,
@@ -30,7 +30,7 @@ Status DBImpl::Put(const WriteOptions& o, ColumnFamilyHandle* column_family,

 Status DBImpl::Put(const WriteOptions& o, ColumnFamilyHandle* column_family,
                   const Slice& key, const Slice& ts, const Slice& val) {
-  const Status s = FailIfTsMismatchCf(column_family, ts, /*ts_for_read=*/false);
+  const Status s = FailIfTsMismatchCf(column_family, ts);
  if (!s.ok()) {
    return s;
  }
@@ -48,6 +48,17 @@ Status DBImpl::PutEntity(const WriteOptions& options,
  return DB::PutEntity(options, column_family, key, columns);
 }

+Status DBImpl::PutEntity(const WriteOptions& options, const Slice& key,
+                         const AttributeGroups& attribute_groups) {
+  for (const AttributeGroup& ag : attribute_groups) {
+    const Status s = FailIfCfHasTs(ag.column_family());
+    if (!s.ok()) {
+      return s;
+    }
+  }
+  return DB::PutEntity(options, key, attribute_groups);
+}
+
 Status DBImpl::Merge(const WriteOptions& o, ColumnFamilyHandle* column_family,
                     const Slice& key, const Slice& val) {
  const Status s = FailIfCfHasTs(column_family);
@@ -64,7 +75,7 @@ Status DBImpl::Merge(const WriteOptions& o, ColumnFamilyHandle* column_family,

 Status DBImpl::Merge(const WriteOptions& o, ColumnFamilyHandle* column_family,
                     const Slice& key, const Slice& ts, const Slice& val) {
-  const Status s = FailIfTsMismatchCf(column_family, ts, /*ts_for_read=*/false);
+  const Status s = FailIfTsMismatchCf(column_family, ts);
  if (!s.ok()) {
    return s;
  }
@@ -83,7 +94,7 @@ Status DBImpl::Delete(const WriteOptions& write_options,
 Status DBImpl::Delete(const WriteOptions& write_options,
                      ColumnFamilyHandle* column_family, const Slice& key,
                      const Slice& ts) {
-  const Status s = FailIfTsMismatchCf(column_family, ts, /*ts_for_read=*/false);
+  const Status s = FailIfTsMismatchCf(column_family, ts);
  if (!s.ok()) {
    return s;
  }
@@ -103,7 +114,7 @@ Status DBImpl::SingleDelete(const WriteOptions& write_options,
 Status DBImpl::SingleDelete(const WriteOptions& write_options,
                            ColumnFamilyHandle* column_family, const Slice& key,
                            const Slice& ts) {
-  const Status s = FailIfTsMismatchCf(column_family, ts, /*ts_for_read=*/false);
+  const Status s = FailIfTsMismatchCf(column_family, ts);
  if (!s.ok()) {
    return s;
  }
@@ -124,7 +135,7 @@ Status DBImpl::DeleteRange(const WriteOptions& write_options,
                           ColumnFamilyHandle* column_family,
                           const Slice& begin_key, const Slice& end_key,
                           const Slice& ts) {
-  const Status s = FailIfTsMismatchCf(column_family, ts, /*ts_for_read=*/false);
+  const Status s = FailIfTsMismatchCf(column_family, ts);
  if (!s.ok()) {
    return s;
  }
@@ -403,17 +414,6 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
  IOStatus io_s;
  Status pre_release_cb_status;
  if (status.ok()) {
-    // TODO: this use of operator bool on `tracer_` can avoid unnecessary lock
-    // grabs but does not seem thread-safe.
-    if (tracer_) {
-      InstrumentedMutexLock lock(&trace_mutex_);
-      if (tracer_ && tracer_->IsWriteOrderPreserved()) {
-        for (auto* writer : write_group) {
-          // TODO: maybe handle the tracing status?
-          tracer_->Write(writer->batch).PermitUncheckedError();
-        }
-      }
-    }
    // Rules for when we can update the memtable concurrently
    // 1. supported by memtable
    // 2. Puts are not okay if inplace_update_support
@@ -437,15 +437,29 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
        valid_batches += writer->batch_cnt;
        if (writer->ShouldWriteToMemtable()) {
          total_count += WriteBatchInternal::Count(writer->batch);
+          total_byte_size = WriteBatchInternal::AppendedByteSize(
+              total_byte_size, WriteBatchInternal::ByteSize(writer->batch));
          parallel = parallel && !writer->batch->HasMerge();
        }
-        total_byte_size = WriteBatchInternal::AppendedByteSize(
-            total_byte_size, WriteBatchInternal::ByteSize(writer->batch));
        if (writer->pre_release_callback) {
          pre_release_callback_cnt++;
        }
      }
    }
+    // TODO: this use of operator bool on `tracer_` can avoid unnecessary lock
+    // grabs but does not seem thread-safe.
+    if (tracer_) {
+      InstrumentedMutexLock lock(&trace_mutex_);
+      if (tracer_ && tracer_->IsWriteOrderPreserved()) {
+        for (auto* writer : write_group) {
+          if (writer->CallbackFailed()) {
+            continue;
+          }
+          // TODO: maybe handle the tracing status?
+          tracer_->Write(writer->batch).PermitUncheckedError();
+        }
+      }
+    }
    // Note about seq_per_batch_: either disableWAL is set for the entire write
    // group or not. In either case we inc seq for each write batch with no
    // failed callback. This means that there could be a batch with
@@ -706,11 +720,11 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
          if (writer->ShouldWriteToMemtable()) {
            writer->sequence = next_sequence;
            size_t count = WriteBatchInternal::Count(writer->batch);
+            total_byte_size = WriteBatchInternal::AppendedByteSize(
+                total_byte_size, WriteBatchInternal::ByteSize(writer->batch));
            next_sequence += count;
            total_count += count;
          }
-          total_byte_size = WriteBatchInternal::AppendedByteSize(
-              total_byte_size, WriteBatchInternal::ByteSize(writer->batch));
        }
      }
      if (w.disable_wal) {
@@ -1811,11 +1825,9 @@ uint64_t DBImpl::GetMaxTotalWalSize() const {
 Status DBImpl::DelayWrite(uint64_t num_bytes, WriteThread& write_thread,
                          const WriteOptions& write_options) {
  mutex_.AssertHeld();
-  uint64_t time_delayed = 0;
+  uint64_t start_time = 0;
  bool delayed = false;
  {
-    StopWatch sw(immutable_db_options_.clock, stats_, WRITE_STALL,
-                 Histograms::HISTOGRAM_ENUM_MAX, &time_delayed);
    // To avoid parallel timed delays (bad throttling), only support them
    // on the primary write queue.
    uint64_t delay;
@@ -1827,6 +1839,8 @@ Status DBImpl::DelayWrite(uint64_t num_bytes, WriteThread& write_thread,
      delay = 0;
    }
    TEST_SYNC_POINT("DBImpl::DelayWrite:Start");
+    start_time = immutable_db_options_.clock->NowMicros();
+
    if (delay > 0) {
      if (write_options.no_slowdown) {
        return Status::Incomplete("Write stall");
@@ -1843,7 +1857,7 @@ Status DBImpl::DelayWrite(uint64_t num_bytes, WriteThread& write_thread,
      // (slightly longer because WriteController minimum delay is 1ms, in
      // case of sleep imprecision, rounding, etc.)
      const uint64_t kDelayInterval = 1001;
-      uint64_t stall_end = sw.start_time() + delay;
+      uint64_t stall_end = start_time + delay;
      while (write_controller_.NeedsDelay()) {
        if (immutable_db_options_.clock->NowMicros() >= stall_end) {
          // We already delayed this write `delay` microseconds
@@ -1858,11 +1872,11 @@ Status DBImpl::DelayWrite(uint64_t num_bytes, WriteThread& write_thread,
      write_thread.EndWriteStall();
    }

-    // Don't wait if there's a background error, even if its a soft error. We
-    // might wait here indefinitely as the background compaction may never
-    // finish successfully, resulting in the stall condition lasting
-    // indefinitely
-    while (error_handler_.GetBGError().ok() && write_controller_.IsStopped() &&
+    // Don't wait if there's a background error that is not pending recovery
+    // since recovery might never be attempted.
+    while ((error_handler_.GetBGError().ok() ||
+            error_handler_.IsRecoveryInProgress()) &&
+           write_controller_.IsStopped() &&
           !shutting_down_.load(std::memory_order_relaxed)) {
      if (write_options.no_slowdown) {
        return Status::Incomplete("Write stall");
@@ -1884,9 +1898,11 @@ Status DBImpl::DelayWrite(uint64_t num_bytes, WriteThread& write_thread,
  }
  assert(!delayed || !write_options.no_slowdown);
  if (delayed) {
+    auto time_delayed = immutable_db_options_.clock->NowMicros() - start_time;
    default_cf_internal_stats_->AddDBStats(
        InternalStats::kIntStatsWriteStallMicros, time_delayed);
    RecordTick(stats_, STALL_MICROS, time_delayed);
+    RecordInHistogram(stats_, WRITE_STALL, time_delayed);
  }

  // If DB is not in read-only mode and write_controller is not stopping
@@ -1953,9 +1969,13 @@ Status DBImpl::ThrottleLowPriWritesIfNeeded(const WriteOptions& write_options,
      // a chance to run. Now we guarantee we are still slowly making
      // progress.
      PERF_TIMER_GUARD(write_delay_time);
-      write_controller_.low_pri_rate_limiter()->Request(
-          my_batch->GetDataSize(), Env::IO_HIGH, nullptr /* stats */,
-          RateLimiter::OpType::kWrite);
+      auto data_size = my_batch->GetDataSize();
+      while (data_size > 0) {
+        size_t allowed = write_controller_.low_pri_rate_limiter()->RequestToken(
+            data_size, 0 /* alignment */, Env::IO_HIGH, nullptr /* stats */,
+            RateLimiter::OpType::kWrite);
+        data_size -= allowed;
+      }
    }
  }
  return Status::OK();
@@ -2378,6 +2398,22 @@ Status DB::PutEntity(const WriteOptions& options,
  return Write(options, &batch);
 }

+Status DB::PutEntity(const WriteOptions& options, const Slice& key,
+                     const AttributeGroups& attribute_groups) {
+  ColumnFamilyHandle* default_cf = DefaultColumnFamily();
+  assert(default_cf);
+  const Comparator* const default_cf_ucmp = default_cf->GetComparator();
+  assert(default_cf_ucmp);
+  WriteBatch batch(0 /* reserved_bytes */, 0 /* max_bytes */,
+                   options.protection_bytes_per_key,
+                   default_cf_ucmp->timestamp_size());
+  const Status s = batch.PutEntity(key, attribute_groups);
+  if (!s.ok()) {
+    return s;
+  }
+  return Write(options, &batch);
+}
+
 Status DB::Delete(const WriteOptions& opt, ColumnFamilyHandle* column_family,
                  const Slice& key) {
  WriteBatch batch(0 /* reserved_bytes */, 0 /* max_bytes */,
@@ -34,6 +34,12 @@ void DumpDBFileSummary(const ImmutableDBOptions& options,
  std::string file_info, wal_info;

  Header(options.info_log, "DB SUMMARY\n");
+  {
+    std::string hostname;
+    if (env->GetHostNameString(&hostname).ok()) {
+      Header(options.info_log, "Host name (Env):  %s\n", hostname.c_str());
+    }
+  }
  Header(options.info_log, "DB Session ID:  %s\n", session_id.c_str());

  Status s;
@@ -18,6 +18,7 @@
 #include "db/merge_helper.h"
 #include "db/pinned_iterators_manager.h"
 #include "db/wide/wide_column_serialization.h"
+#include "db/wide/wide_columns_helper.h"
 #include "file/filename.h"
 #include "logging/logging.h"
 #include "memory/arena.h"
@@ -112,6 +113,9 @@ Status DBIter::GetProperty(std::string prop_name, std::string* prop) {
  } else if (prop_name == "rocksdb.iterator.internal-key") {
    *prop = saved_key_.GetUserKey().ToString();
    return Status::OK();
+  } else if (prop_name == "rocksdb.iterator.write-time") {
+    // TODO(yuzhangyu): implement return the actual write time.
+    return Status::NotSupported("write time property is under construction");
  }
  return Status::InvalidArgument("Unidentified property.");
 }
@@ -230,14 +234,38 @@ bool DBIter::SetValueAndColumnsFromEntity(Slice slice) {
    return false;
  }

-  if (!wide_columns_.empty() &&
-      wide_columns_[0].name() == kDefaultWideColumnName) {
-    value_ = wide_columns_[0].value();
+  if (WideColumnsHelper::HasDefaultColumn(wide_columns_)) {
+    value_ = WideColumnsHelper::GetDefaultColumn(wide_columns_);
  }

  return true;
 }

+bool DBIter::SetValueAndColumnsFromMergeResult(const Status& merge_status,
+                                               ValueType result_type) {
+  if (!merge_status.ok()) {
+    valid_ = false;
+    status_ = merge_status;
+    return false;
+  }
+
+  if (result_type == kTypeWideColumnEntity) {
+    if (!SetValueAndColumnsFromEntity(saved_value_)) {
+      assert(!valid_);
+      return false;
+    }
+
+    valid_ = true;
+    return true;
+  }
+
+  assert(result_type == kTypeValue);
+  SetValueAndColumnsFromPlain(pinned_value_.data() ? pinned_value_
+                                                   : saved_value_);
+  valid_ = true;
+  return true;
+}
+
 // PRE: saved_key_ has the current user key if skipping_saved_key
 // POST: saved_key_ should have the next user key if valid_,
 //       if the current entry is a result of merge
@@ -366,9 +394,7 @@ bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key,
          case kTypeValue:
          case kTypeBlobIndex:
          case kTypeWideColumnEntity:
-            if (!iter_.PrepareValue()) {
-              assert(!iter_.status().ok());
-              valid_ = false;
+            if (!PrepareValue()) {
              return false;
            }
            if (timestamp_lb_) {
@@ -399,9 +425,7 @@ bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key,
            return true;
            break;
          case kTypeMerge:
-            if (!iter_.PrepareValue()) {
-              assert(!iter_.status().ok());
-              valid_ = false;
+            if (!PrepareValue()) {
              return false;
            }
            saved_key_.SetUserKey(
@@ -546,16 +570,14 @@ bool DBIter::MergeValuesNewToOld() {
      iter_.Next();
      break;
    }
-    if (!iter_.PrepareValue()) {
-      valid_ = false;
+    if (!PrepareValue()) {
      return false;
    }

    if (kTypeValue == ikey.type) {
      // hit a put, merge the put value with operands and store the
      // final result in saved_value_. We are done!
-      const Slice val = iter_.value();
-      if (!Merge(&val, ikey.user_key)) {
+      if (!MergeWithPlainBaseValue(iter_.value(), ikey.user_key)) {
        return false;
      }
      // iter_ is positioned after put
@@ -584,7 +606,7 @@ bool DBIter::MergeValuesNewToOld() {
        return false;
      }
      valid_ = true;
-      if (!Merge(&blob_value_, ikey.user_key)) {
+      if (!MergeWithPlainBaseValue(blob_value_, ikey.user_key)) {
        return false;
      }

@@ -598,7 +620,7 @@ bool DBIter::MergeValuesNewToOld() {
      }
      return true;
    } else if (kTypeWideColumnEntity == ikey.type) {
-      if (!MergeEntity(iter_.value(), ikey.user_key)) {
+      if (!MergeWithWideColumnBaseValue(iter_.value(), ikey.user_key)) {
        return false;
      }

@@ -628,7 +650,7 @@ bool DBIter::MergeValuesNewToOld() {
  // a deletion marker.
  // feed null as the existing value to the merge operator, such that
  // client can differentiate this scenario and do things accordingly.
-  if (!Merge(nullptr, saved_key_.GetUserKey())) {
+  if (!MergeWithNoBaseValue(saved_key_.GetUserKey())) {
    return false;
  }
  assert(status_.ok());
@@ -874,8 +896,7 @@ bool DBIter::FindValueForCurrentKey() {
      return FindValueForCurrentKeyUsingSeek();
    }

-    if (!iter_.PrepareValue()) {
-      valid_ = false;
+    if (!PrepareValue()) {
      return false;
    }

@@ -979,7 +1000,7 @@ bool DBIter::FindValueForCurrentKey() {
      if (last_not_merge_type == kTypeDeletion ||
          last_not_merge_type == kTypeSingleDeletion ||
          last_not_merge_type == kTypeDeletionWithTimestamp) {
-        if (!Merge(nullptr, saved_key_.GetUserKey())) {
+        if (!MergeWithNoBaseValue(saved_key_.GetUserKey())) {
          return false;
        }
        return true;
@@ -994,7 +1015,7 @@ bool DBIter::FindValueForCurrentKey() {
          return false;
        }
        valid_ = true;
-        if (!Merge(&blob_value_, saved_key_.GetUserKey())) {
+        if (!MergeWithPlainBaseValue(blob_value_, saved_key_.GetUserKey())) {
          return false;
        }

@@ -1002,14 +1023,15 @@ bool DBIter::FindValueForCurrentKey() {

        return true;
      } else if (last_not_merge_type == kTypeWideColumnEntity) {
-        if (!MergeEntity(pinned_value_, saved_key_.GetUserKey())) {
+        if (!MergeWithWideColumnBaseValue(pinned_value_,
+                                          saved_key_.GetUserKey())) {
          return false;
        }

        return true;
      } else {
        assert(last_not_merge_type == kTypeValue);
-        if (!Merge(&pinned_value_, saved_key_.GetUserKey())) {
+        if (!MergeWithPlainBaseValue(pinned_value_, saved_key_.GetUserKey())) {
          return false;
        }
        return true;
@@ -1113,8 +1135,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
    }
    return true;
  }
-  if (!iter_.PrepareValue()) {
-    valid_ = false;
+  if (!PrepareValue()) {
    return false;
  }
  if (timestamp_size_ > 0) {
@@ -1179,14 +1200,12 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
        ikey.type == kTypeDeletionWithTimestamp) {
      break;
    }
-    if (!iter_.PrepareValue()) {
-      valid_ = false;
+    if (!PrepareValue()) {
      return false;
    }

    if (ikey.type == kTypeValue) {
-      const Slice val = iter_.value();
-      if (!Merge(&val, saved_key_.GetUserKey())) {
+      if (!MergeWithPlainBaseValue(iter_.value(), saved_key_.GetUserKey())) {
        return false;
      }
      return true;
@@ -1205,7 +1224,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
        return false;
      }
      valid_ = true;
-      if (!Merge(&blob_value_, saved_key_.GetUserKey())) {
+      if (!MergeWithPlainBaseValue(blob_value_, saved_key_.GetUserKey())) {
        return false;
      }

@@ -1213,7 +1232,8 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {

      return true;
    } else if (ikey.type == kTypeWideColumnEntity) {
-      if (!MergeEntity(iter_.value(), saved_key_.GetUserKey())) {
+      if (!MergeWithWideColumnBaseValue(iter_.value(),
+                                        saved_key_.GetUserKey())) {
        return false;
      }

@@ -1227,7 +1247,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
    }
  }

-  if (!Merge(nullptr, saved_key_.GetUserKey())) {
+  if (!MergeWithNoBaseValue(saved_key_.GetUserKey())) {
    return false;
  }

@@ -1250,47 +1270,42 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
  return true;
 }

-bool DBIter::Merge(const Slice* val, const Slice& user_key) {
+bool DBIter::MergeWithNoBaseValue(const Slice& user_key) {
  // `op_failure_scope` (an output parameter) is not provided (set to nullptr)
  // since a failure must be propagated regardless of its value.
-  Status s = MergeHelper::TimedFullMerge(
-      merge_operator_, user_key, val, merge_context_.GetOperands(),
-      &saved_value_, logger_, statistics_, clock_, &pinned_value_,
-      /* update_num_ops_stats */ true,
-      /* op_failure_scope */ nullptr);
-  if (!s.ok()) {
-    valid_ = false;
-    status_ = s;
-    return false;
-  }
-
-  SetValueAndColumnsFromPlain(pinned_value_.data() ? pinned_value_
-                                                   : saved_value_);
-
-  valid_ = true;
-  return true;
+  ValueType result_type;
+  const Status s = MergeHelper::TimedFullMerge(
+      merge_operator_, user_key, MergeHelper::kNoBaseValue,
+      merge_context_.GetOperands(), logger_, statistics_, clock_,
+      /* update_num_ops_stats */ true, /* op_failure_scope */ nullptr,
+      &saved_value_, &pinned_value_, &result_type);
+  return SetValueAndColumnsFromMergeResult(s, result_type);
 }

-bool DBIter::MergeEntity(const Slice& entity, const Slice& user_key) {
+bool DBIter::MergeWithPlainBaseValue(const Slice& value,
+                                     const Slice& user_key) {
  // `op_failure_scope` (an output parameter) is not provided (set to nullptr)
  // since a failure must be propagated regardless of its value.
-  Status s = MergeHelper::TimedFullMergeWithEntity(
-      merge_operator_, user_key, entity, merge_context_.GetOperands(),
-      &saved_value_, logger_, statistics_, clock_,
-      /* update_num_ops_stats */ true,
-      /* op_failure_scope */ nullptr);
-  if (!s.ok()) {
-    valid_ = false;
-    status_ = s;
-    return false;
-  }
+  ValueType result_type;
+  const Status s = MergeHelper::TimedFullMerge(
+      merge_operator_, user_key, MergeHelper::kPlainBaseValue, value,
+      merge_context_.GetOperands(), logger_, statistics_, clock_,
+      /* update_num_ops_stats */ true, /* op_failure_scope */ nullptr,
+      &saved_value_, &pinned_value_, &result_type);
+  return SetValueAndColumnsFromMergeResult(s, result_type);
+}

-  if (!SetValueAndColumnsFromEntity(saved_value_)) {
-    return false;
-  }
-
-  valid_ = true;
-  return true;
+bool DBIter::MergeWithWideColumnBaseValue(const Slice& entity,
+                                          const Slice& user_key) {
+  // `op_failure_scope` (an output parameter) is not provided (set to nullptr)
+  // since a failure must be propagated regardless of its value.
+  ValueType result_type;
+  const Status s = MergeHelper::TimedFullMerge(
+      merge_operator_, user_key, MergeHelper::kWideBaseValue, entity,
+      merge_context_.GetOperands(), logger_, statistics_, clock_,
+      /* update_num_ops_stats */ true, /* op_failure_scope */ nullptr,
+      &saved_value_, &pinned_value_, &result_type);
+  return SetValueAndColumnsFromMergeResult(s, result_type);
 }

 // Move backwards until the key smaller than saved_key_.
@@ -209,6 +209,7 @@ class DBIter final : public Iterator {
    if (read_callback_) {
      read_callback_->Refresh(s);
    }
+    iter_.SetRangeDelReadSeqno(s);
  }
  void set_valid(bool v) { valid_ = v; }

@@ -312,14 +313,36 @@ class DBIter final : public Iterator {

  bool SetValueAndColumnsFromEntity(Slice slice);

+  bool SetValueAndColumnsFromMergeResult(const Status& merge_status,
+                                         ValueType result_type);
+
  void ResetValueAndColumns() {
    value_.clear();
    wide_columns_.clear();
  }

+  // The following methods perform the actual merge operation for the
+  // no base value/plain base value/wide-column base value cases.
  // If user-defined timestamp is enabled, `user_key` includes timestamp.
-  bool Merge(const Slice* val, const Slice& user_key);
-  bool MergeEntity(const Slice& entity, const Slice& user_key);
+  bool MergeWithNoBaseValue(const Slice& user_key);
+  bool MergeWithPlainBaseValue(const Slice& value, const Slice& user_key);
+  bool MergeWithWideColumnBaseValue(const Slice& entity, const Slice& user_key);
+
+  bool PrepareValue() {
+    if (!iter_.PrepareValue()) {
+      assert(!iter_.status().ok());
+      valid_ = false;
+      return false;
+    }
+    // ikey_ could change as BlockBasedTableIterator does Block cache
+    // lookup and index_iter_ could point to different block resulting
+    // in ikey_ pointing to wrong key. So ikey_ needs to be updated in
+    // case of Seek/Next calls to point to right key again.
+    if (!ParseKey(&ikey_)) {
+      return false;
+    }
+    return true;
+  }

  const SliceTransform* prefix_extractor_;
  Env* const env_;
@@ -275,6 +275,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    db_iter->Next();
    ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
  }
  // Test to check the SeekToLast() with iterate_upper_bound not set
  {
@@ -1415,6 +1416,7 @@ TEST_F(DBIteratorTest, DBIterator1) {
  ASSERT_EQ(db_iter->key().ToString(), "b");
  db_iter->Next();
  ASSERT_FALSE(db_iter->Valid());
+  ASSERT_OK(db_iter->status());
 }

 TEST_F(DBIteratorTest, DBIterator2) {
@@ -1528,6 +1530,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
    ASSERT_EQ(db_iter->value().ToString(), "merge_1");
    db_iter->Prev();
    ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
  }

  {
@@ -1552,6 +1555,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
    ASSERT_EQ(db_iter->value().ToString(), "merge_1,merge_2");
    db_iter->Prev();
    ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
  }

  {
@@ -1576,6 +1580,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
    ASSERT_EQ(db_iter->value().ToString(), "merge_1,merge_2,merge_3");
    db_iter->Prev();
    ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
  }

  {
@@ -1600,6 +1605,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
    ASSERT_EQ(db_iter->value().ToString(), "put_1");
    db_iter->Prev();
    ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
  }

  {
@@ -1624,6 +1630,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
    ASSERT_EQ(db_iter->value().ToString(), "put_1,merge_4");
    db_iter->Prev();
    ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
  }

  {
@@ -1648,6 +1655,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
    ASSERT_EQ(db_iter->value().ToString(), "put_1,merge_4,merge_5");
    db_iter->Prev();
    ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
  }

  {
@@ -1672,6 +1680,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
    ASSERT_EQ(db_iter->value().ToString(), "put_1,merge_4,merge_5,merge_6");
    db_iter->Prev();
    ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
  }

  {
@@ -1726,6 +1735,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
    ASSERT_EQ(db_iter->value().ToString(), "merge_1");
    db_iter->Prev();
    ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
  }

  {
@@ -1750,6 +1760,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
    ASSERT_EQ(db_iter->value().ToString(), "merge_1,merge_2");
    db_iter->Prev();
    ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
  }

  {
@@ -1774,6 +1785,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
    ASSERT_EQ(db_iter->value().ToString(), "merge_1,merge_2,merge_3");
    db_iter->Prev();
    ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
  }

  {
@@ -1794,6 +1806,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
        nullptr /* read_callback */));
    db_iter->SeekToLast();
    ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
  }

  {
@@ -1818,6 +1831,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
    ASSERT_EQ(db_iter->value().ToString(), "merge_4");
    db_iter->Prev();
    ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
  }

  {
@@ -1842,6 +1856,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
    ASSERT_EQ(db_iter->value().ToString(), "merge_4,merge_5");
    db_iter->Prev();
    ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
  }

  {
@@ -1866,6 +1881,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
    ASSERT_EQ(db_iter->value().ToString(), "merge_4,merge_5,merge_6");
    db_iter->Prev();
    ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
  }
 }

@@ -1910,6 +1926,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
    ASSERT_EQ(db_iter->value().ToString(), "merge_1");
    db_iter->Prev();
    ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
  }

  {
@@ -1952,6 +1969,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
    ASSERT_EQ(db_iter->value().ToString(), "merge_1");
    db_iter->Prev();
    ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
  }

  {
@@ -1994,6 +2012,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
    ASSERT_EQ(db_iter->value().ToString(), "merge_1");
    db_iter->Prev();
    ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
  }

  {
@@ -2041,6 +2060,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
    ASSERT_EQ(db_iter->value().ToString(), "merge_1");
    db_iter->Prev();
    ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
  }

  {
@@ -2089,6 +2109,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
    ASSERT_EQ(db_iter->value().ToString(), "merge_1");
    db_iter->Prev();
    ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
  }

  {
@@ -2131,6 +2152,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
    ASSERT_EQ(db_iter->value().ToString(), "merge_1");
    db_iter->Prev();
    ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
  }

  {
@@ -2179,6 +2201,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
    ASSERT_EQ(db_iter->value().ToString(), "merge_1");
    db_iter->Prev();
    ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
  }

  {
@@ -2228,6 +2251,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
    ASSERT_EQ(db_iter->value().ToString(), "merge_1");
    db_iter->Prev();
    ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
  }

  {
@@ -2271,6 +2295,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
    ASSERT_EQ(db_iter->value().ToString(), "merge_1");
    db_iter->Prev();
    ASSERT_TRUE(!db_iter->Valid());
+    ASSERT_OK(db_iter->status());
  }
 }

@@ -2440,6 +2465,7 @@ TEST_F(DBIteratorTest, SeekToLastOccurrenceSeq0) {
  ASSERT_EQ(db_iter->value().ToString(), "2");
  db_iter->Next();
  ASSERT_FALSE(db_iter->Valid());
+  ASSERT_OK(db_iter->status());
 }

 TEST_F(DBIteratorTest, DBIterator11) {
@@ -2469,6 +2495,7 @@ TEST_F(DBIteratorTest, DBIterator11) {
  ASSERT_EQ(db_iter->key().ToString(), "b");
  db_iter->Next();
  ASSERT_FALSE(db_iter->Valid());
+  ASSERT_OK(db_iter->status());
 }

 TEST_F(DBIteratorTest, DBIterator12) {
@@ -2497,6 +2524,7 @@ TEST_F(DBIteratorTest, DBIterator12) {
  ASSERT_EQ(db_iter->value().ToString(), "1");
  db_iter->Prev();
  ASSERT_FALSE(db_iter->Valid());
+  ASSERT_OK(db_iter->status());
 }

 TEST_F(DBIteratorTest, DBIterator13) {
@@ -2635,6 +2663,7 @@ TEST_F(DBIterWithMergeIterTest, InnerMergeIterator1) {
  ASSERT_EQ(db_iter_->value().ToString(), "3");
  db_iter_->Next();
  ASSERT_FALSE(db_iter_->Valid());
+  ASSERT_OK(db_iter_->status());
 }

 TEST_F(DBIterWithMergeIterTest, InnerMergeIterator2) {
@@ -107,7 +107,10 @@ class DBIteratorTest : public DBIteratorBaseTest,
      read_callbacks_.push_back(
          std::unique_ptr<DummyReadCallback>(read_callback));
    }
-    return dbfull()->NewIteratorImpl(read_options, cfd, seq, read_callback);
+    DBImpl* db_impl = dbfull();
+    SuperVersion* super_version = cfd->GetReferencedSuperVersion(db_impl);
+    return db_impl->NewIteratorImpl(read_options, cfd, super_version, seq,
+                                    read_callback);
  }

 private:
@@ -180,6 +183,7 @@ TEST_P(DBIteratorTest, NonBlockingIteration) {
      ASSERT_OK(iter->status());
      count++;
    }
+    ASSERT_OK(iter->status());
    ASSERT_EQ(count, 1);
    delete iter;

@@ -214,6 +218,7 @@ TEST_P(DBIteratorTest, NonBlockingIteration) {
      ASSERT_OK(iter->status());
      count++;
    }
+    ASSERT_OK(iter->status());
    ASSERT_EQ(count, 1);
    ASSERT_EQ(numopen, TestGetTickerCount(options, NO_FILE_OPENS));
    ASSERT_EQ(cache_added, TestGetTickerCount(options, BLOCK_CACHE_ADD));
@@ -867,6 +872,7 @@ TEST_P(DBIteratorTest, IterWithSnapshot) {
      }
    }
    db_->ReleaseSnapshot(snapshot);
+    ASSERT_OK(iter->status());
    delete iter;
  } while (ChangeOptions());
 }
@@ -1211,6 +1217,7 @@ TEST_P(DBIteratorTest, DBIteratorBoundOptimizationTest) {

    iter->Next();
    ASSERT_FALSE(iter->Valid());
+    ASSERT_OK(iter->status());
    ASSERT_EQ(upper_bound_hits, 1);
  }
 }
@@ -1335,6 +1342,7 @@ TEST_P(DBIteratorTest, IndexWithFirstKey) {

    iter->Next();
    ASSERT_FALSE(iter->Valid());
+    ASSERT_OK(iter->status());
    EXPECT_EQ(3, stats->getTickerCount(BLOCK_CACHE_DATA_HIT));
    EXPECT_EQ(7, stats->getTickerCount(BLOCK_CACHE_DATA_MISS));
  }
@@ -1576,6 +1584,7 @@ class DBIteratorTestForPinnedData : public DBIteratorTest {
        ASSERT_EQ("1", prop_value);
        all_keys.push_back(iter->key());
      }
+      ASSERT_OK(iter->status());
      ASSERT_EQ(all_keys.size(), true_data.size());

      // Verify that all keys slices are valid (backward)
@@ -1679,7 +1688,7 @@ TEST_P(DBIteratorTest, PinnedDataIteratorMultipleFiles) {
    ASSERT_EQ(kv.first, data_iter->first);
    ASSERT_EQ(kv.second, data_iter->second);
  }
-
+  ASSERT_OK(iter->status());
  delete iter;
 }

@@ -1725,6 +1734,7 @@ TEST_P(DBIteratorTest, PinnedDataIteratorMergeOperator) {
    ASSERT_EQ("1", prop_value);
    results.emplace_back(iter->key(), iter->value().ToString());
  }
+  ASSERT_OK(iter->status());

  ASSERT_EQ(results.size(), 1000);
  for (size_t i = 0; i < results.size(); i++) {
@@ -1782,6 +1792,7 @@ TEST_P(DBIteratorTest, PinnedDataIteratorReadAfterUpdate) {
    ASSERT_EQ("1", prop_value);
    results.emplace_back(iter->key(), iter->value().ToString());
  }
+  ASSERT_OK(iter->status());

  auto data_iter = true_data.begin();
  for (size_t i = 0; i < results.size(); i++, data_iter++) {
@@ -2076,6 +2087,7 @@ TEST_P(DBIteratorTest, IterPrevKeyCrossingBlocksRandomized) {
      ASSERT_EQ(iter->value().ToString(), data_iter->second);
      data_iter++;
    }
+    ASSERT_OK(iter->status());
    ASSERT_EQ(data_iter, true_data.rend());

    delete iter;
@@ -2133,6 +2145,7 @@ TEST_P(DBIteratorTest, IterPrevKeyCrossingBlocksRandomized) {
      entries_right++;
      data_iter++;
    }
+    ASSERT_OK(iter->status());
    ASSERT_EQ(data_iter, true_data.rend());

    delete iter;
@@ -2172,6 +2185,7 @@ TEST_P(DBIteratorTest, IteratorWithLocalStatistics) {
      total_next++;

      if (!iter->Valid()) {
+        EXPECT_OK(iter->status());
        break;
      }
      total_next_found++;
@@ -2199,6 +2213,7 @@ TEST_P(DBIteratorTest, IteratorWithLocalStatistics) {
      total_prev++;

      if (!iter->Valid()) {
+        EXPECT_OK(iter->status());
        break;
      }
      total_prev_found++;
@@ -2413,37 +2428,98 @@ TEST_P(DBIteratorTest, Refresh) {
  ASSERT_EQ(iter->key().compare(Slice("x")), 0);
  iter->Next();
  ASSERT_FALSE(iter->Valid());
+  ASSERT_OK(iter->status());

  iter.reset();
 }

 TEST_P(DBIteratorTest, RefreshWithSnapshot) {
-  ASSERT_OK(Put("x", "y"));
+  // L1 file, uses LevelIterator internally
+  ASSERT_OK(Put(Key(0), "val0"));
+  ASSERT_OK(Put(Key(5), "val5"));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(1);
+
+  // L0 file, uses table iterator internally
+  ASSERT_OK(Put(Key(1), "val1"));
+  ASSERT_OK(Put(Key(4), "val4"));
+  ASSERT_OK(Flush());
+
+  // Memtable
+  ASSERT_OK(Put(Key(2), "val2"));
+  ASSERT_OK(Put(Key(3), "val3"));
  const Snapshot* snapshot = db_->GetSnapshot();
+  ASSERT_OK(Put(Key(2), "new val"));
+  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(4),
+                             Key(7)));
+  const Snapshot* snapshot2 = db_->GetSnapshot();
+
+  ASSERT_EQ(1, NumTableFilesAtLevel(1));
+  ASSERT_EQ(1, NumTableFilesAtLevel(0));
+
  ReadOptions options;
  options.snapshot = snapshot;
  Iterator* iter = NewIterator(options);
+  ASSERT_OK(Put(Key(6), "val6"));
  ASSERT_OK(iter->status());

-  iter->Seek(Slice("a"));
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("x")), 0);
-  iter->Next();
-  ASSERT_FALSE(iter->Valid());
+  auto verify_iter = [&](int start, int end, bool new_key2 = false) {
+    for (int i = start; i < end; ++i) {
+      ASSERT_OK(iter->status());
+      ASSERT_TRUE(iter->Valid());
+      ASSERT_EQ(iter->key(), Key(i));
+      if (i == 2 && new_key2) {
+        ASSERT_EQ(iter->value(), "new val");
+      } else {
+        ASSERT_EQ(iter->value(), "val" + std::to_string(i));
+      }
+      iter->Next();
+    }
+  };

-  ASSERT_OK(Put("c", "d"));
+  for (int j = 0; j < 2; j++) {
+    iter->Seek(Key(1));
+    verify_iter(1, 3);
+    // Refresh to same snapshot
+    ASSERT_OK(iter->Refresh(snapshot));
+    ASSERT_TRUE(!iter->Valid() && iter->status().ok());
+    iter->Seek(Key(3));
+    verify_iter(3, 6);
+    ASSERT_TRUE(!iter->Valid() && iter->status().ok());

-  iter->Seek(Slice("a"));
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("x")), 0);
-  iter->Next();
-  ASSERT_FALSE(iter->Valid());
+    // Refresh to a newer snapshot
+    ASSERT_OK(iter->Refresh(snapshot2));
+    ASSERT_TRUE(!iter->Valid() && iter->status().ok());
+    iter->SeekToFirst();
+    verify_iter(0, 4, /*new_key2=*/true);
+    ASSERT_TRUE(!iter->Valid() && iter->status().ok());
+
+    // Refresh to an older snapshot
+    ASSERT_OK(iter->Refresh(snapshot));
+    ASSERT_TRUE(!iter->Valid() && iter->status().ok());
+    iter->Seek(Key(3));
+    verify_iter(3, 6);
+    ASSERT_TRUE(!iter->Valid() && iter->status().ok());
+
+    // Refresh to no snapshot
+    ASSERT_OK(iter->Refresh());
+    ASSERT_TRUE(!iter->Valid() && iter->status().ok());
+    iter->Seek(Key(2));
+    verify_iter(2, 4, /*new_key2=*/true);
+    verify_iter(6, 7);
+    ASSERT_TRUE(!iter->Valid() && iter->status().ok());
+
+    // Change LSM shape, new SuperVersion is created.
+    ASSERT_OK(Flush());
+
+    // Refresh back to original snapshot
+    ASSERT_OK(iter->Refresh(snapshot));
+  }

-  ASSERT_OK(iter->status());
-  Status s = iter->Refresh();
-  ASSERT_TRUE(s.IsNotSupported());
-  db_->ReleaseSnapshot(snapshot);
  delete iter;
+  db_->ReleaseSnapshot(snapshot);
+  db_->ReleaseSnapshot(snapshot2);
+  ASSERT_OK(db_->Close());
 }

 TEST_P(DBIteratorTest, CreationFailure) {
@@ -2536,6 +2612,7 @@ TEST_P(DBIteratorTest, TableFilter) {
    ASSERT_EQ(IterStatus(iter), "f->6");
    iter->Next();
    ASSERT_FALSE(iter->Valid());
+    ASSERT_OK(iter->status());
    ASSERT_TRUE(unseen.empty());
    delete iter;
  }
@@ -2558,6 +2635,7 @@ TEST_P(DBIteratorTest, TableFilter) {
    ASSERT_EQ(IterStatus(iter), "f->6");
    iter->Next();
    ASSERT_FALSE(iter->Valid());
+    ASSERT_OK(iter->status());
    delete iter;
  }
 }
@@ -2642,6 +2720,7 @@ TEST_P(DBIteratorTest, SkipStatistics) {
    ASSERT_OK(iter->status());
    count++;
  }
+  ASSERT_OK(iter->status());
  ASSERT_EQ(count, 3);
  delete iter;
  skip_count += 8;  // Same as above, but in reverse order
@@ -2677,6 +2756,7 @@ TEST_P(DBIteratorTest, SkipStatistics) {
    ASSERT_OK(iter->status());
    count++;
  }
+  ASSERT_OK(iter->status());
  ASSERT_EQ(count, 2);
  delete iter;
  // 3 deletes + 3 original keys + lower sequence of "a"
@@ -3130,8 +3210,10 @@ TEST_F(DBIteratorWithReadCallbackTest, ReadCallback) {
      static_cast_with_check<ColumnFamilyHandleImpl>(db_->DefaultColumnFamily())
          ->cfd();
  // The iterator are suppose to see data before seq1.
-  Iterator* iter =
-      dbfull()->NewIteratorImpl(ReadOptions(), cfd, seq2, &callback1);
+  DBImpl* db_impl = dbfull();
+  SuperVersion* super_version = cfd->GetReferencedSuperVersion(db_impl);
+  Iterator* iter = db_impl->NewIteratorImpl(ReadOptions(), cfd, super_version,
+                                            seq2, &callback1);

  // Seek
  // The latest value of "foo" before seq1 is "v3"
@@ -3209,7 +3291,9 @@ TEST_F(DBIteratorWithReadCallbackTest, ReadCallback) {
  SequenceNumber seq4 = db_->GetLatestSequenceNumber();

  // The iterator is suppose to see data before seq3.
-  iter = dbfull()->NewIteratorImpl(ReadOptions(), cfd, seq4, &callback2);
+  super_version = cfd->GetReferencedSuperVersion(db_impl);
+  iter = db_impl->NewIteratorImpl(ReadOptions(), cfd, super_version, seq4,
+                                  &callback2);
  // Seek to "z", which is visible.
  iter->Seek("z");
  ASSERT_TRUE(iter->Valid());
@@ -3255,6 +3339,7 @@ TEST_F(DBIteratorTest, BackwardIterationOnInplaceUpdateMemtable) {
    for (iter->SeekToLast(); iter->Valid(); iter->Prev()) {
      ++count;
    }
+    ASSERT_OK(iter->status());
    ASSERT_EQ(kNumKeys, count);
  }

@@ -3295,6 +3380,176 @@ TEST_F(DBIteratorTest, IteratorRefreshReturnSV) {
  Close();
 }

+TEST_F(DBIteratorTest, ErrorWhenReadFile) {
+  // This is to test a bug that is fixed in
+  // https://github.com/facebook/rocksdb/pull/11782.
+  //
+  // Ingest error when reading from a file, and
+  // see if Iterator handles it correctly.
+  Options opts = CurrentOptions();
+  opts.num_levels = 7;
+  opts.compression = kNoCompression;
+  BlockBasedTableOptions bbto;
+  // Always do I/O
+  bbto.no_block_cache = true;
+  opts.table_factory.reset(NewBlockBasedTableFactory(bbto));
+  DestroyAndReopen(opts);
+
+  // Set up LSM
+  // L5: F1 [key0, key99], F2 [key100, key199]
+  // L6:        F3 [key50, key149]
+  Random rnd(301);
+  const int kValLen = 100;
+  for (int i = 50; i < 150; ++i) {
+    ASSERT_OK(Put(Key(i), rnd.RandomString(kValLen)));
+  }
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(6);
+
+  std::vector<std::string> values;
+  for (int i = 0; i < 100; ++i) {
+    values.emplace_back(rnd.RandomString(kValLen));
+    ASSERT_OK(Put(Key(i), values.back()));
+  }
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(5);
+
+  for (int i = 100; i < 200; ++i) {
+    values.emplace_back(rnd.RandomString(kValLen));
+    ASSERT_OK(Put(Key(i), values.back()));
+  }
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(5);
+
+  ASSERT_EQ(2, NumTableFilesAtLevel(5));
+  ASSERT_EQ(1, NumTableFilesAtLevel(6));
+
+  std::vector<LiveFileMetaData> files;
+  db_->GetLiveFilesMetaData(&files);
+  // Get file names for F1, F2 and F3.
+  // These are file names, not full paths.
+  std::string f1, f2, f3;
+  for (auto& file_meta : files) {
+    if (file_meta.level == 6) {
+      f3 = file_meta.name;
+    } else {
+      if (file_meta.smallestkey == Key(0)) {
+        f1 = file_meta.name;
+      } else {
+        f2 = file_meta.name;
+      }
+    }
+  }
+  ASSERT_TRUE(!f1.empty());
+  ASSERT_TRUE(!f2.empty());
+  ASSERT_TRUE(!f3.empty());
+
+  std::string error_file;
+  SyncPoint::GetInstance()->SetCallBack(
+      "RandomAccessFileReader::Read::BeforeReturn",
+      [&error_file](void* io_s_ptr) {
+        auto p =
+            reinterpret_cast<std::pair<std::string*, IOStatus*>*>(io_s_ptr);
+        if (p->first->find(error_file) != std::string::npos) {
+          *p->second = IOStatus::IOError();
+          p->second->SetRetryable(true);
+        }
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+  // Error reading F1
+  error_file = f1;
+  std::unique_ptr<Iterator> iter{db_->NewIterator(ReadOptions())};
+  iter->SeekToFirst();
+  ASSERT_NOK(iter->status());
+  ASSERT_TRUE(iter->status().IsIOError());
+  // This does not require reading the first block.
+  iter->Seek(Key(90));
+  ASSERT_OK(iter->status());
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_EQ(iter->value(), values[90]);
+  // iter has ok status before this Seek.
+  iter->Seek(Key(1));
+  ASSERT_NOK(iter->status());
+  ASSERT_TRUE(iter->status().IsIOError());
+
+  // Error reading F2
+  error_file = f2;
+  iter.reset(db_->NewIterator(ReadOptions()));
+  iter->Seek(Key(99));
+  ASSERT_OK(iter->status());
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_EQ(iter->value(), values[99]);
+  // Need to read from F2.
+  iter->Next();
+  ASSERT_NOK(iter->status());
+  ASSERT_TRUE(iter->status().IsIOError());
+  iter->Seek(Key(190));
+  ASSERT_OK(iter->status());
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_EQ(iter->value(), values[190]);
+  // Seek for first key of F2.
+  iter->Seek(Key(100));
+  ASSERT_NOK(iter->status());
+  ASSERT_TRUE(iter->status().IsIOError());
+  iter->SeekToLast();
+  ASSERT_OK(iter->status());
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_EQ(iter->value(), values[199]);
+  // SeekForPrev for first key of F2.
+  iter->SeekForPrev(Key(100));
+  ASSERT_NOK(iter->status());
+  ASSERT_TRUE(iter->status().IsIOError());
+  // Does not read first block (offset 0).
+  iter->SeekForPrev(Key(98));
+  ASSERT_OK(iter->status());
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_EQ(iter->value(), values[98]);
+
+  // Error reading F3
+  error_file = f3;
+  iter.reset(db_->NewIterator(ReadOptions()));
+  iter->SeekToFirst();
+  ASSERT_NOK(iter->status());
+  ASSERT_TRUE(iter->status().IsIOError());
+  iter->Seek(Key(50));
+  ASSERT_NOK(iter->status());
+  ASSERT_TRUE(iter->status().IsIOError());
+  iter->SeekForPrev(Key(50));
+  ASSERT_NOK(iter->status());
+  ASSERT_TRUE(iter->status().IsIOError());
+  // Does not read file 3
+  iter->Seek(Key(150));
+  ASSERT_OK(iter->status());
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_EQ(iter->value(), values[150]);
+
+  // Test when file read error occurs during Prev().
+  // This requires returning an error when reading near the end of a file
+  // instead of offset 0.
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+  SyncPoint::GetInstance()->SetCallBack(
+      "RandomAccessFileReader::Read::AnyOffset", [&f1](void* pair_ptr) {
+        auto p =
+            reinterpret_cast<std::pair<std::string*, IOStatus*>*>(pair_ptr);
+        if (p->first->find(f1) != std::string::npos) {
+          *p->second = IOStatus::IOError();
+          p->second->SetRetryable(true);
+        }
+      });
+  iter->SeekForPrev(Key(101));
+  ASSERT_OK(iter->status());
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_EQ(iter->value(), values[101]);
+  // DBIter will not stop at Key(100) since it needs
+  // to make sure the key it returns has the max sequence number for Key(100).
+  // So it will call MergingIterator::Prev() which will read F1.
+  iter->Prev();
+  ASSERT_NOK(iter->status());
+  ASSERT_TRUE(iter->status().IsIOError());
+  SyncPoint::GetInstance()->DisableProcessing();
+  iter->Reset();
+}
+
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -145,6 +145,41 @@ TEST_F(DBTestXactLogIterator, TransactionLogIteratorRace) {
    } while (ChangeCompactOptions());
  }
 }
+
+TEST_F(DBTestXactLogIterator, TransactionLogIteratorCheckWhenArchive) {
+  do {
+    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearTrace();
+    Options options = OptionsForLogIterTest();
+    DestroyAndReopen(options);
+    ColumnFamilyHandle* cf;
+    auto s = dbfull()->CreateColumnFamily(ColumnFamilyOptions(), "CF", &cf);
+    ASSERT_TRUE(s.ok());
+
+    ASSERT_OK(dbfull()->Put(WriteOptions(), cf, "key1", DummyString(1024)));
+
+    ASSERT_OK(dbfull()->Put(WriteOptions(), "key2", DummyString(1024)));
+
+    ASSERT_OK(dbfull()->Flush(FlushOptions()));
+
+    ASSERT_OK(dbfull()->Put(WriteOptions(), "key3", DummyString(1024)));
+
+    ASSERT_OK(dbfull()->Flush(FlushOptions()));
+
+    ASSERT_OK(dbfull()->Put(WriteOptions(), "key4", DummyString(1024)));
+    ASSERT_OK(dbfull()->Flush(FlushOptions()));
+
+    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+        "WalManager::PurgeObsoleteFiles:1", [&](void*) {
+          auto iter = OpenTransactionLogIter(0);
+          ExpectRecords(4, iter);
+        });
+
+    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+    ASSERT_OK(dbfull()->Flush(FlushOptions(), cf));
+
+    delete cf;
+  } while (ChangeCompactOptions());
+}
 #endif

 TEST_F(DBTestXactLogIterator, TransactionLogIteratorStallAtLastRecord) {
@@ -201,7 +236,7 @@ TEST_F(DBTestXactLogIterator, TransactionLogIteratorCorruptedLog) {
    ASSERT_OK(test::TruncateFile(env_, logfile_path,
                                 wal_files.front()->SizeFileBytes() / 2));

-    ASSERT_OK(db_->EnableFileDeletions());
+    ASSERT_OK(db_->EnableFileDeletions(/*force=*/false));

    // Insert a new entry to a new log file
    ASSERT_OK(Put("key1025", DummyString(10)));
@@ -6,9 +6,12 @@
 #include <vector>

 #include "db/db_test_util.h"
+#include "db/dbformat.h"
 #include "db/forward_iterator.h"
 #include "port/stack_trace.h"
 #include "rocksdb/merge_operator.h"
+#include "rocksdb/snapshot.h"
+#include "rocksdb/utilities/debug.h"
 #include "util/random.h"
 #include "utilities/merge_operators.h"
 #include "utilities/merge_operators/string_append/stringappend2.h"
@@ -202,7 +205,6 @@ TEST_F(DBMergeOperatorTest, MergeErrorOnIteration) {
  VerifyDBInternal({{"k1", "v1"}, {"k2", "corrupted"}, {"k2", "v2"}});
 }

-
 TEST_F(DBMergeOperatorTest, MergeOperatorFailsWithMustMerge) {
  // This is like a mini-stress test dedicated to `OpFailureScope::kMustMerge`.
  // Some or most of it might be deleted upon adding that option to the actual
@@ -358,6 +360,98 @@ TEST_F(DBMergeOperatorTest, MergeOperatorFailsWithMustMerge) {
  }
 }

+TEST_F(DBMergeOperatorTest, MergeOperandThresholdExceeded) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.merge_operator = MergeOperators::CreatePutOperator();
+  options.env = env_;
+  Reopen(options);
+
+  std::vector<Slice> keys{"foo", "bar", "baz"};
+
+  // Write base values.
+  for (const auto& key : keys) {
+    ASSERT_OK(Put(key, key.ToString() + "0"));
+  }
+
+  // Write merge operands. Note that the first key has 1 merge operand, the
+  // second one has 2 merge operands, and the third one has 3 merge operands.
+  // Also, we'll take some snapshots to make sure the merge operands are
+  // preserved during flush.
+  std::vector<ManagedSnapshot> snapshots;
+  snapshots.reserve(3);
+
+  for (size_t i = 0; i < keys.size(); ++i) {
+    snapshots.emplace_back(db_);
+
+    const std::string suffix = std::to_string(i + 1);
+
+    for (size_t j = i; j < keys.size(); ++j) {
+      ASSERT_OK(Merge(keys[j], keys[j].ToString() + suffix));
+    }
+  }
+
+  // Verify the results and status codes of various types of point lookups.
+  auto verify = [&](const std::optional<size_t>& threshold) {
+    ReadOptions read_options;
+    read_options.merge_operand_count_threshold = threshold;
+
+    // Check Get()
+    {
+      for (size_t i = 0; i < keys.size(); ++i) {
+        PinnableSlice value;
+        const Status status =
+            db_->Get(read_options, db_->DefaultColumnFamily(), keys[i], &value);
+        ASSERT_OK(status);
+        ASSERT_EQ(status.IsOkMergeOperandThresholdExceeded(),
+                  threshold.has_value() && i + 1 > threshold.value());
+        ASSERT_EQ(value, keys[i].ToString() + std::to_string(i + 1));
+      }
+    }
+
+    // Check old-style MultiGet()
+    {
+      std::vector<std::string> values;
+      std::vector<Status> statuses = db_->MultiGet(read_options, keys, &values);
+
+      for (size_t i = 0; i < keys.size(); ++i) {
+        ASSERT_OK(statuses[i]);
+        ASSERT_EQ(statuses[i].IsOkMergeOperandThresholdExceeded(),
+                  threshold.has_value() && i + 1 > threshold.value());
+        ASSERT_EQ(values[i], keys[i].ToString() + std::to_string(i + 1));
+      }
+    }
+
+    // Check batched MultiGet()
+    {
+      std::vector<PinnableSlice> values(keys.size());
+      std::vector<Status> statuses(keys.size());
+      db_->MultiGet(read_options, db_->DefaultColumnFamily(), keys.size(),
+                    keys.data(), values.data(), statuses.data());
+
+      for (size_t i = 0; i < keys.size(); ++i) {
+        ASSERT_OK(statuses[i]);
+        ASSERT_EQ(statuses[i].IsOkMergeOperandThresholdExceeded(),
+                  threshold.has_value() && i + 1 > threshold.value());
+        ASSERT_EQ(values[i], keys[i].ToString() + std::to_string(i + 1));
+      }
+    }
+  };
+
+  // Test the case when the feature is disabled as well as various thresholds.
+  verify(std::nullopt);
+  for (size_t i = 0; i < 5; ++i) {
+    verify(i);
+  }
+
+  // Flush and try again to test the case when results are served from SSTs.
+  ASSERT_OK(Flush());
+  verify(std::nullopt);
+  for (size_t i = 0; i < 5; ++i) {
+    verify(i);
+  }
+}
+
 TEST_F(DBMergeOperatorTest, DataBlockBinaryAndHash) {
  // Basic test to check that merge operator works with data block index type
  // DataBlockBinaryAndHash.
@@ -857,6 +951,98 @@ TEST_P(PerConfigMergeOperatorPinningTest, Randomized) {
  VerifyDBFromMap(true_data);
 }

+TEST_F(DBMergeOperatorTest, MaxSuccessiveMergesBaseValues) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.merge_operator = MergeOperators::CreatePutOperator();
+  options.max_successive_merges = 1;
+  options.env = env_;
+  Reopen(options);
+
+  constexpr char foo[] = "foo";
+  constexpr char bar[] = "bar";
+  constexpr char baz[] = "baz";
+  constexpr char qux[] = "qux";
+  constexpr char corge[] = "corge";
+
+  // No base value
+  {
+    constexpr char key[] = "key1";
+
+    ASSERT_OK(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), key, foo));
+    ASSERT_OK(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), key, bar));
+
+    PinnableSlice result;
+    ASSERT_OK(
+        db_->Get(ReadOptions(), db_->DefaultColumnFamily(), key, &result));
+    ASSERT_EQ(result, bar);
+
+    // We expect the second Merge to be converted to a Put because of
+    // max_successive_merges.
+    constexpr size_t max_key_versions = 8;
+    std::vector<KeyVersion> key_versions;
+    ASSERT_OK(GetAllKeyVersions(db_, db_->DefaultColumnFamily(), key, key,
+                                max_key_versions, &key_versions));
+    ASSERT_EQ(key_versions.size(), 2);
+    ASSERT_EQ(key_versions[0].type, kTypeValue);
+    ASSERT_EQ(key_versions[1].type, kTypeMerge);
+  }
+
+  // Plain base value
+  {
+    constexpr char key[] = "key2";
+
+    ASSERT_OK(db_->Put(WriteOptions(), db_->DefaultColumnFamily(), key, foo));
+    ASSERT_OK(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), key, bar));
+    ASSERT_OK(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), key, baz));
+
+    PinnableSlice result;
+    ASSERT_OK(
+        db_->Get(ReadOptions(), db_->DefaultColumnFamily(), key, &result));
+    ASSERT_EQ(result, baz);
+
+    // We expect the second Merge to be converted to a Put because of
+    // max_successive_merges.
+    constexpr size_t max_key_versions = 8;
+    std::vector<KeyVersion> key_versions;
+    ASSERT_OK(GetAllKeyVersions(db_, db_->DefaultColumnFamily(), key, key,
+                                max_key_versions, &key_versions));
+    ASSERT_EQ(key_versions.size(), 3);
+    ASSERT_EQ(key_versions[0].type, kTypeValue);
+    ASSERT_EQ(key_versions[1].type, kTypeMerge);
+    ASSERT_EQ(key_versions[2].type, kTypeValue);
+  }
+
+  // Wide-column base value
+  {
+    constexpr char key[] = "key3";
+    const WideColumns columns{{kDefaultWideColumnName, foo}, {bar, baz}};
+
+    ASSERT_OK(db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(), key,
+                             columns));
+    ASSERT_OK(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), key, qux));
+    ASSERT_OK(
+        db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), key, corge));
+
+    PinnableWideColumns result;
+    ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), key,
+                             &result));
+    const WideColumns expected{{kDefaultWideColumnName, corge}, {bar, baz}};
+    ASSERT_EQ(result.columns(), expected);
+
+    // We expect the second Merge to be converted to a PutEntity because of
+    // max_successive_merges.
+    constexpr size_t max_key_versions = 8;
+    std::vector<KeyVersion> key_versions;
+    ASSERT_OK(GetAllKeyVersions(db_, db_->DefaultColumnFamily(), key, key,
+                                max_key_versions, &key_versions));
+    ASSERT_EQ(key_versions.size(), 3);
+    ASSERT_EQ(key_versions[0].type, kTypeWideColumnEntity);
+    ASSERT_EQ(key_versions[1].type, kTypeMerge);
+    ASSERT_EQ(key_versions[2].type, kTypeWideColumnEntity);
+  }
+}
+
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -19,6 +19,8 @@
 #include "rocksdb/convenience.h"
 #include "rocksdb/rate_limiter.h"
 #include "rocksdb/stats_history.h"
+#include "rocksdb/utilities/options_util.h"
+#include "test_util/mock_time_env.h"
 #include "test_util/sync_point.h"
 #include "test_util/testutil.h"
 #include "util/random.h"
@@ -740,6 +742,55 @@ TEST_F(DBOptionsTest, SetStatsDumpPeriodSec) {
  Close();
 }

+TEST_F(DBOptionsTest, SetStatsDumpPeriodSecRace) {
+  // This is a mini-stress test looking for inconsistency between the reported
+  // state of the option and the behavior in effect for the DB, after the last
+  // modification to that option (indefinite inconsistency).
+  std::vector<std::thread> threads;
+  for (int i = 0; i < 12; i++) {
+    threads.emplace_back([this, i]() {
+      ASSERT_OK(dbfull()->SetDBOptions(
+          {{"stats_dump_period_sec", i % 2 ? "100" : "0"}}));
+    });
+  }
+
+  for (auto& t : threads) {
+    t.join();
+  }
+
+  bool stats_dump_set = dbfull()->GetDBOptions().stats_dump_period_sec > 0;
+  bool task_enabled = dbfull()->TEST_GetPeriodicTaskScheduler().TEST_HasTask(
+      PeriodicTaskType::kDumpStats);
+
+  ASSERT_EQ(stats_dump_set, task_enabled);
+}
+
+TEST_F(DBOptionsTest, SetOptionsAndFileRace) {
+  // This is a mini-stress test looking for inconsistency between the reported
+  // state of the option and what is persisted in the options file, after the
+  // last modification to that option (indefinite inconsistency).
+  std::vector<std::thread> threads;
+  for (int i = 0; i < 12; i++) {
+    threads.emplace_back([this, i]() {
+      ASSERT_OK(dbfull()->SetOptions({{"ttl", std::to_string(i * 100)}}));
+    });
+  }
+
+  for (auto& t : threads) {
+    t.join();
+  }
+
+  auto setting_in_mem = dbfull()->GetOptions().ttl;
+
+  std::vector<ColumnFamilyDescriptor> cf_descs;
+  DBOptions db_options;
+  ConfigOptions cfg;
+  cfg.env = env_;
+  ASSERT_OK(LoadLatestOptions(cfg, dbname_, &db_options, &cf_descs, nullptr));
+  ASSERT_EQ(cf_descs.size(), 1);
+  ASSERT_EQ(setting_in_mem, cf_descs[0].options.ttl);
+}
+
 TEST_F(DBOptionsTest, SetOptionsStatsPersistPeriodSec) {
  Options options;
  options.create_if_missing = true;
@@ -908,6 +959,7 @@ TEST_F(DBOptionsTest, SetFIFOCompactionOptions) {
  Options options;
  options.env = CurrentOptions().env;
  options.compaction_style = kCompactionStyleFIFO;
+  options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
  options.write_buffer_size = 10 << 10;  // 10KB
  options.arena_block_size = 4096;
  options.compression = kNoCompression;
@@ -941,6 +993,9 @@ TEST_F(DBOptionsTest, SetFIFOCompactionOptions) {
  ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
  ASSERT_EQ(NumTableFilesAtLevel(0), 10);

+  ASSERT_EQ(options.statistics->getTickerCount(FIFO_TTL_COMPACTIONS), 0);
+  ASSERT_EQ(options.statistics->getTickerCount(FIFO_MAX_SIZE_COMPACTIONS), 0);
+
  // Set ttl to 1 minute. So all files should get deleted.
  ASSERT_OK(dbfull()->SetOptions({{"ttl", "60"}}));
  ASSERT_EQ(dbfull()->GetOptions().ttl, 60);
@@ -948,6 +1003,10 @@ TEST_F(DBOptionsTest, SetFIFOCompactionOptions) {
  ASSERT_OK(dbfull()->TEST_WaitForCompact());
  ASSERT_EQ(NumTableFilesAtLevel(0), 0);

+  ASSERT_GT(options.statistics->getTickerCount(FIFO_TTL_COMPACTIONS), 0);
+  ASSERT_EQ(options.statistics->getTickerCount(FIFO_MAX_SIZE_COMPACTIONS), 0);
+  ASSERT_OK(options.statistics->Reset());
+
  // NOTE: Presumed unnecessary and removed: resetting mock time in env

  // Test dynamically changing compaction_options_fifo.max_table_files_size
@@ -971,6 +1030,9 @@ TEST_F(DBOptionsTest, SetFIFOCompactionOptions) {
  ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
  ASSERT_EQ(NumTableFilesAtLevel(0), 10);

+  ASSERT_EQ(options.statistics->getTickerCount(FIFO_MAX_SIZE_COMPACTIONS), 0);
+  ASSERT_EQ(options.statistics->getTickerCount(FIFO_TTL_COMPACTIONS), 0);
+
  // Set max_table_files_size to 12 KB. So only 1 file should remain now.
  ASSERT_OK(dbfull()->SetOptions(
      {{"compaction_options_fifo", "{max_table_files_size=12288;}"}}));
@@ -980,6 +1042,10 @@ TEST_F(DBOptionsTest, SetFIFOCompactionOptions) {
  ASSERT_OK(dbfull()->TEST_WaitForCompact());
  ASSERT_EQ(NumTableFilesAtLevel(0), 1);

+  ASSERT_GT(options.statistics->getTickerCount(FIFO_MAX_SIZE_COMPACTIONS), 0);
+  ASSERT_EQ(options.statistics->getTickerCount(FIFO_TTL_COMPACTIONS), 0);
+  ASSERT_OK(options.statistics->Reset());
+
  // Test dynamically changing compaction_options_fifo.allow_compaction
  options.compaction_options_fifo.max_table_files_size = 500 << 10;  // 500KB
  options.ttl = 0;
@@ -1033,33 +1099,218 @@ TEST_F(DBOptionsTest, SetFIFOCompactionOptions) {
  ASSERT_EQ(fifo_temp_opt[1].age, 30000);
 }

-TEST_F(DBOptionsTest, CompactionReadaheadSizeChange) {
-  SpecialEnv env(env_);
+TEST_F(DBOptionsTest, OffpeakTimes) {
  Options options;
-  options.env = &env;
+  options.create_if_missing = true;
+  Random rnd(test::RandomSeed());

-  options.compaction_readahead_size = 0;
-  options.level0_file_num_compaction_trigger = 2;
-  const std::string kValue(1024, 'v');
-  Reopen(options);
+  auto verify_invalid = [&]() {
+    Status s = DBImpl::TEST_ValidateOptions(options);
+    ASSERT_NOK(s);
+    ASSERT_TRUE(s.IsInvalidArgument());
+  };

-  ASSERT_EQ(1024 * 1024 * 2,
-            dbfull()->GetDBOptions().compaction_readahead_size);
-  ASSERT_OK(dbfull()->SetDBOptions({{"compaction_readahead_size", "256"}}));
-  ASSERT_EQ(256, dbfull()->GetDBOptions().compaction_readahead_size);
-  for (int i = 0; i < 1024; i++) {
-    ASSERT_OK(Put(Key(i), kValue));
+  auto verify_valid = [&]() {
+    Status s = DBImpl::TEST_ValidateOptions(options);
+    ASSERT_OK(s);
+    ASSERT_FALSE(s.IsInvalidArgument());
+  };
+  std::vector<std::string> invalid_cases = {
+      "06:30-",
+      "-23:30",  // Both need to be set
+      "00:00-00:00",
+      "06:30-06:30"  //  Start time cannot be the same as end time
+      "12:30 PM-23:30",
+      "12:01AM-11:00PM",  // Invalid format
+      "01:99-22:00",      // Invalid value for minutes
+      "00:00-24:00",      // 24:00 is an invalid value
+      "6-7",
+      "6:-7",
+      "06:31.42-7:00",
+      "6.31:42-7:00",
+      "6:0-7:",
+      "15:0.2-3:.7",
+      ":00-00:02",
+      "02:00-:00",
+      "random-value",
+      "No:No-Hi:Hi",
+  };
+
+  std::vector<std::string> valid_cases = {
+      "",  // Not enabled. Valid case
+      "06:30-11:30",
+      "06:30-23:30",
+      "13:30-14:30",
+      "00:00-23:59",  // Entire Day
+      "23:30-01:15",  // From 11:30PM to 1:15AM next day. Valid case.
+      "1:0000000000000-2:000000000042",  // Weird, but we can parse the int.
+  };
+
+  for (std::string invalid_case : invalid_cases) {
+    options.daily_offpeak_time_utc = invalid_case;
+    verify_invalid();
  }
-  ASSERT_OK(Flush());
-  for (int i = 0; i < 1024 * 2; i++) {
-    ASSERT_OK(Put(Key(i), kValue));
+  for (std::string valid_case : valid_cases) {
+    options.daily_offpeak_time_utc = valid_case;
+    verify_valid();
  }
-  ASSERT_OK(Flush());
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
-  ASSERT_EQ(256, env_->compaction_readahead_size_);
+
+  auto verify_offpeak_info = [&](bool expected_is_now_off_peak,
+                                 int expected_seconds_till_next_offpeak_start,
+                                 int now_utc_hour, int now_utc_minute,
+                                 int now_utc_second = 0) {
+    auto mock_clock = std::make_shared<MockSystemClock>(env_->GetSystemClock());
+    // Add some extra random days to current time
+    int days = rnd.Uniform(100);
+    mock_clock->SetCurrentTime(
+        days * OffpeakTimeOption::kSecondsPerDay +
+        now_utc_hour * OffpeakTimeOption::kSecondsPerHour +
+        now_utc_minute * OffpeakTimeOption::kSecondsPerMinute + now_utc_second);
+    Status s = DBImpl::TEST_ValidateOptions(options);
+    ASSERT_OK(s);
+    auto offpeak_option = OffpeakTimeOption(options.daily_offpeak_time_utc);
+    int64_t now;
+    ASSERT_OK(mock_clock.get()->GetCurrentTime(&now));
+    auto offpeak_info = offpeak_option.GetOffpeakTimeInfo(now);
+    ASSERT_EQ(expected_is_now_off_peak, offpeak_info.is_now_offpeak);
+    ASSERT_EQ(expected_seconds_till_next_offpeak_start,
+              offpeak_info.seconds_till_next_offpeak_start);
+  };
+
+  options.daily_offpeak_time_utc = "";
+  verify_offpeak_info(false, 0, 12, 30);
+
+  options.daily_offpeak_time_utc = "06:30-11:30";
+  verify_offpeak_info(false, 1 * OffpeakTimeOption::kSecondsPerHour, 5, 30);
+  verify_offpeak_info(true, 24 * OffpeakTimeOption::kSecondsPerHour, 6, 30);
+  verify_offpeak_info(true, 20 * OffpeakTimeOption::kSecondsPerHour, 10, 30);
+  verify_offpeak_info(true, 19 * OffpeakTimeOption::kSecondsPerHour, 11, 30);
+  verify_offpeak_info(false, 17 * OffpeakTimeOption::kSecondsPerHour, 13, 30);
+
+  options.daily_offpeak_time_utc = "23:30-04:30";
+  verify_offpeak_info(false, 17 * OffpeakTimeOption::kSecondsPerHour, 6, 30);
+  verify_offpeak_info(true, 24 * OffpeakTimeOption::kSecondsPerHour, 23, 30);
+  verify_offpeak_info(true,
+                      23 * OffpeakTimeOption::kSecondsPerHour +
+                          30 * OffpeakTimeOption::kSecondsPerMinute,
+                      0, 0);
+  verify_offpeak_info(true,
+                      22 * OffpeakTimeOption::kSecondsPerHour +
+                          30 * OffpeakTimeOption::kSecondsPerMinute,
+                      1, 0);
+  verify_offpeak_info(true, 19 * OffpeakTimeOption::kSecondsPerHour, 4, 30);
+  verify_offpeak_info(false,
+                      18 * OffpeakTimeOption::kSecondsPerHour +
+                          59 * OffpeakTimeOption::kSecondsPerMinute,
+                      4, 31);
+
+  // Entire day offpeak
+  options.daily_offpeak_time_utc = "00:00-23:59";
+  verify_offpeak_info(true, 24 * OffpeakTimeOption::kSecondsPerHour, 0, 0);
+  verify_offpeak_info(true, 12 * OffpeakTimeOption::kSecondsPerHour, 12, 00);
+  verify_offpeak_info(true, 1 * OffpeakTimeOption::kSecondsPerMinute, 23, 59);
+  verify_offpeak_info(true, 59, 23, 59, 1);
+  verify_offpeak_info(true, 1, 23, 59, 59);
+
+  // Start with a valid option
+  options.daily_offpeak_time_utc = "01:30-04:15";
+  DestroyAndReopen(options);
+  ASSERT_EQ("01:30-04:15", dbfull()->GetDBOptions().daily_offpeak_time_utc);
+
+  int may_schedule_compaction_called = 0;
+  SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::MaybeScheduleFlushOrCompaction:Start",
+      [&](void*) { may_schedule_compaction_called++; });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  // Make sure calling SetDBOptions with invalid option does not change the
+  // value nor call MaybeScheduleFlushOrCompaction()
+  for (std::string invalid_case : invalid_cases) {
+    ASSERT_NOK(
+        dbfull()->SetDBOptions({{"daily_offpeak_time_utc", invalid_case}}));
+    ASSERT_EQ("01:30-04:15", dbfull()
+                                 ->GetVersionSet()
+                                 ->offpeak_time_option()
+                                 .daily_offpeak_time_utc);
+    ASSERT_EQ(1 * kSecondInHour + 30 * kSecondInMinute,
+              dbfull()
+                  ->GetVersionSet()
+                  ->offpeak_time_option()
+                  .daily_offpeak_start_time_utc);
+    ASSERT_EQ(4 * kSecondInHour + 15 * kSecondInMinute,
+              dbfull()
+                  ->GetVersionSet()
+                  ->offpeak_time_option()
+                  .daily_offpeak_end_time_utc);
+  }
+  ASSERT_EQ(0, may_schedule_compaction_called);
+
+  // Changing to new valid values should call MaybeScheduleFlushOrCompaction()
+  // and sets the offpeak_time_option in VersionSet
+  int expected_count = 0;
+  for (std::string valid_case : valid_cases) {
+    if (dbfull()
+            ->GetVersionSet()
+            ->offpeak_time_option()
+            .daily_offpeak_time_utc != valid_case) {
+      expected_count++;
+    }
+    ASSERT_OK(dbfull()->SetDBOptions({{"daily_offpeak_time_utc", valid_case}}));
+    ASSERT_EQ(valid_case, dbfull()->GetDBOptions().daily_offpeak_time_utc);
+    ASSERT_EQ(valid_case, dbfull()
+                              ->GetVersionSet()
+                              ->offpeak_time_option()
+                              .daily_offpeak_time_utc);
+  }
+  ASSERT_EQ(expected_count, may_schedule_compaction_called);
+
+  // Changing to the same value should not call MaybeScheduleFlushOrCompaction()
+  ASSERT_OK(
+      dbfull()->SetDBOptions({{"daily_offpeak_time_utc", "06:30-11:30"}}));
+  may_schedule_compaction_called = 0;
+  ASSERT_OK(
+      dbfull()->SetDBOptions({{"daily_offpeak_time_utc", "06:30-11:30"}}));
+  ASSERT_EQ(0, may_schedule_compaction_called);
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
  Close();
 }

+TEST_F(DBOptionsTest, CompactionReadaheadSizeChange) {
+  for (bool use_direct_reads : {true, false}) {
+    SpecialEnv env(env_);
+    Options options;
+    options.env = &env;
+
+    options.use_direct_reads = use_direct_reads;
+    options.level0_file_num_compaction_trigger = 2;
+    const std::string kValue(1024, 'v');
+    Status s = TryReopen(options);
+    if (use_direct_reads && (s.IsNotSupported() || s.IsInvalidArgument())) {
+      continue;
+    } else {
+      ASSERT_OK(s);
+    }
+
+    ASSERT_EQ(1024 * 1024 * 2,
+              dbfull()->GetDBOptions().compaction_readahead_size);
+    ASSERT_OK(dbfull()->SetDBOptions({{"compaction_readahead_size", "256"}}));
+    ASSERT_EQ(256, dbfull()->GetDBOptions().compaction_readahead_size);
+    for (int i = 0; i < 1024; i++) {
+      ASSERT_OK(Put(Key(i), kValue));
+    }
+    ASSERT_OK(Flush());
+    for (int i = 0; i < 1024 * 2; i++) {
+      ASSERT_OK(Put(Key(i), kValue));
+    }
+    ASSERT_OK(Flush());
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
+    ASSERT_EQ(256, env_->compaction_readahead_size_);
+    Close();
+  }
+}
+
 TEST_F(DBOptionsTest, FIFOTtlBackwardCompatible) {
  Options options;
  options.compaction_style = kCompactionStyleFIFO;
@@ -107,12 +107,12 @@ TEST_F(DBPropertiesTest, Empty) {
        dbfull()->GetProperty("rocksdb.is-file-deletions-enabled", &num));
    ASSERT_EQ("0", num);

-    ASSERT_OK(db_->EnableFileDeletions(false));
+    ASSERT_OK(db_->EnableFileDeletions(/*force=*/false));
    ASSERT_TRUE(
        dbfull()->GetProperty("rocksdb.is-file-deletions-enabled", &num));
    ASSERT_EQ("0", num);

-    ASSERT_OK(db_->EnableFileDeletions());
+    ASSERT_OK(db_->EnableFileDeletions(/*force=*/true));
    ASSERT_TRUE(
        dbfull()->GetProperty("rocksdb.is-file-deletions-enabled", &num));
    ASSERT_EQ("1", num);
@@ -1084,12 +1084,14 @@ class CountingUserTblPropCollector : public TablePropertiesCollector {
  const char* Name() const override { return "CountingUserTblPropCollector"; }

  Status Finish(UserCollectedProperties* properties) override {
+    assert(!finish_called_);
    std::string encoded;
    PutVarint32(&encoded, count_);
    *properties = UserCollectedProperties{
        {"CountingUserTblPropCollector", message_},
        {"Count", encoded},
    };
+    finish_called_ = true;
    return Status::OK();
  }

@@ -1101,12 +1103,14 @@ class CountingUserTblPropCollector : public TablePropertiesCollector {
  }

  UserCollectedProperties GetReadableProperties() const override {
+    assert(finish_called_);
    return UserCollectedProperties{};
  }

 private:
  std::string message_ = "Rocksdb";
  uint32_t count_ = 0;
+  bool finish_called_ = false;
 };

 class CountingUserTblPropCollectorFactory
@@ -1740,7 +1744,7 @@ TEST_F(DBPropertiesTest, SstFilesSize) {
  ASSERT_EQ(obsolete_sst_size, sst_size);

  // Let the obsolete files be deleted.
-  ASSERT_OK(db_->EnableFileDeletions());
+  ASSERT_OK(db_->EnableFileDeletions(/*force=*/false));
  ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kObsoleteSstFilesSize,
                                  &obsolete_sst_size));
  ASSERT_EQ(obsolete_sst_size, 0);
@@ -682,6 +682,7 @@ TEST_F(DBRangeDelTest, TableEvictedDuringScan) {
    // soon as its refcount drops to zero.
    bbto.block_cache->EraseUnRefEntries();
  }
+  ASSERT_OK(iter->status());
  ASSERT_EQ(kNum, expected);
  delete iter;
  db_->ReleaseSnapshot(snapshot);
@@ -840,6 +841,7 @@ TEST_F(DBRangeDelTest, IteratorRemovesCoveredKeys) {
      ++expected;
    }
  }
+  ASSERT_OK(iter->status());
  ASSERT_EQ(kNum, expected);
  delete iter;
 }
@@ -908,6 +910,7 @@ TEST_F(DBRangeDelTest, IteratorIgnoresRangeDeletions) {
    std::string key;
    ASSERT_EQ(expected[i], iter->key());
  }
+  ASSERT_OK(iter->status());
  ASSERT_EQ(3, i);
  delete iter;
  db_->ReleaseSnapshot(snapshot);
@@ -1382,6 +1385,7 @@ TEST_F(DBRangeDelTest, UntruncatedTombstoneDoesNotDeleteNewerKey) {
    for (; iter->Valid(); iter->Next()) {
      ++keys_found;
    }
+    EXPECT_OK(iter->status());
    delete iter;
    return keys_found;
  };
@@ -1485,6 +1489,7 @@ TEST_F(DBRangeDelTest, DeletedMergeOperandReappearsIterPrev) {
  for (; iter->Valid(); iter->Prev()) {
    ++keys_found;
  }
+  ASSERT_OK(iter->status());
  delete iter;
  ASSERT_EQ(kNumKeys, keys_found);

@@ -1519,6 +1524,7 @@ TEST_F(DBRangeDelTest, SnapshotPreventsDroppedKeys) {

  iter->Next();
  ASSERT_FALSE(iter->Valid());
+  ASSERT_OK(iter->status());

  delete iter;
  db_->ReleaseSnapshot(snapshot);
@@ -1564,6 +1570,7 @@ TEST_F(DBRangeDelTest, SnapshotPreventsDroppedKeysInImmMemTables) {

  iter->Next();
  ASSERT_FALSE(iter->Valid());
+  ASSERT_OK(iter->status());
 }

 TEST_F(DBRangeDelTest, RangeTombstoneWrittenToMinimalSsts) {
@@ -1978,6 +1985,7 @@ TEST_F(DBRangeDelTest, IteratorRefresh) {
    ASSERT_EQ("key1", iter->key());
    iter->Next();
    ASSERT_FALSE(iter->Valid());
+    ASSERT_OK(iter->status());

    delete iter;
  }
@@ -3511,6 +3519,256 @@ TEST_F(DBRangeDelTest, MemtableMaxRangeDeletions) {
  ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
  ASSERT_EQ(3, NumTableFilesAtLevel(0));
 }
+
+TEST_F(DBRangeDelTest, RangeDelReseekAfterFileReadError) {
+  // This is to test a bug that is fixed in
+  // https://github.com/facebook/rocksdb/pull/11786.
+  Options opts = CurrentOptions();
+  opts.num_levels = 7;
+
+  // Set up LSM
+  //
+  // L4: F1: [key1] F2: [key2]
+  // L5:   F3:[DeleteRange(key3, key6)]
+  // L6:               F4:[key3, key6]
+  // Will inject error when reading from F2.
+  // SeekToFirst() should land on key1.
+  // Next() should encounter error when reading from F2,
+  // and range del reseek should not reset this status.
+  Random rnd(301);
+  // L6
+  ASSERT_OK(Put(Key(3), rnd.RandomString(100)));
+  ASSERT_OK(Put(Key(6), rnd.RandomString(100)));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(6);
+  // L5
+  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(3),
+                             Key(6)));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(5);
+  // L4
+  ASSERT_OK(Put(Key(2), rnd.RandomString(100)));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(4);
+  std::string fname;
+  std::vector<LiveFileMetaData> live_files;
+  db_->GetLiveFilesMetaData(&live_files);
+  for (auto& meta : live_files) {
+    if (meta.level == 4) {
+      fname = meta.name;
+      break;
+    }
+  }
+  ASSERT_TRUE(!fname.empty());
+  ASSERT_OK(Put(Key(1), rnd.RandomString(100)));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(4);
+
+  SyncPoint::GetInstance()->SetCallBack(
+      "RandomAccessFileReader::Read::BeforeReturn", [&fname](void* pair_ptr) {
+        auto p =
+            reinterpret_cast<std::pair<std::string*, IOStatus*>*>(pair_ptr);
+        if (p->first->find(fname) != std::string::npos) {
+          *p->second = IOStatus::IOError();
+          p->second->SetRetryable(true);
+        }
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+  std::unique_ptr<Iterator> iter{db_->NewIterator(ReadOptions())};
+  iter->SeekToFirst();
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_OK(iter->status());
+  ASSERT_EQ(iter->key(), Key(1));
+  iter->Next();
+  ASSERT_FALSE(iter->Valid());
+  ASSERT_NOK(iter->status());
+  ASSERT_TRUE(iter->status().IsIOError());
+  iter.reset();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+  SyncPoint::GetInstance()->DisableProcessing();
+
+  // Reverse scan
+  // LSM setup
+  // L4: F1: [key2]  F2: [key7, key8]
+  // L5:        F3:[[key3, key6)]
+  // L6:        F4:[key1, key5]
+  // Ingest error when read from F1.
+  // SeekToLast() should land on key8.
+  // During Prev(), MergingIterator will encounter error when reading from F1
+  // and do a range del reseek (it sees key5 covered by a range tombstone).
+  DestroyAndReopen(opts);
+  // L6
+  ASSERT_OK(Put(Key(1), rnd.RandomString(100)));
+  ASSERT_OK(Put(Key(5), rnd.RandomString(100)));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(6);
+  // L5
+  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(3),
+                             Key(6)));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(5);
+  // L4
+  ASSERT_OK(Put(Key(2), rnd.RandomString(100)));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(4);
+  live_files.clear();
+  db_->GetLiveFilesMetaData(&live_files);
+  for (auto& meta : live_files) {
+    if (meta.level == 4) {
+      fname = meta.name;
+      break;
+    }
+  }
+  ASSERT_TRUE(!fname.empty());
+  ASSERT_OK(Put(Key(7), rnd.RandomString(100)));
+  ASSERT_OK(Put(Key(8), rnd.RandomString(100)));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(4);
+
+  SyncPoint::GetInstance()->SetCallBack(
+      "RandomAccessFileReader::Read::AnyOffset", [&fname](void* pair_ptr) {
+        auto p =
+            reinterpret_cast<std::pair<std::string*, IOStatus*>*>(pair_ptr);
+        if (p->first->find(fname) != std::string::npos) {
+          *p->second = IOStatus::IOError();
+          p->second->SetRetryable(true);
+        }
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+  iter.reset(db_->NewIterator(ReadOptions()));
+  iter->SeekToLast();
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_OK(iter->status());
+  ASSERT_EQ(iter->key(), Key(8));
+  // Note that for reverse scan, DBIter will need to ensure
+  // the key it returns is the one with the highest sequence number.
+  // To return key7, it internally calls MergingIterator::Prev()
+  // until it reaches a previous user key.
+  iter->Prev();
+  ASSERT_FALSE(iter->Valid());
+  ASSERT_NOK(iter->status());
+  ASSERT_TRUE(iter->status().IsIOError());
+
+  iter.reset();
+}
+
+TEST_F(DBRangeDelTest, ReleaseSnapshotAfterIteratorCreation) {
+  // Test that range tombstone code path in LevelIterator
+  // does access ReadOptions::snapshot after Iterator creation.
+  //
+  // Put some data in L2 so that range tombstone in L1 will not be dropped.
+  ASSERT_OK(Put(Key(0), "v"));
+  ASSERT_OK(Put(Key(100), "v"));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(2);
+
+  // two L1 file with range del
+  ASSERT_OK(Put(Key(1), "v"));
+  ASSERT_OK(Put(Key(2), "v"));
+  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(3),
+                             Key(4)));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(1);
+
+  ASSERT_OK(Put(Key(5), "v"));
+  ASSERT_OK(Put(Key(6), "v"));
+  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(5),
+                             Key(6)));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(1);
+
+  ASSERT_EQ(2, NumTableFilesAtLevel(1));
+  ASSERT_EQ(1, NumTableFilesAtLevel(2));
+
+  const Snapshot* snapshot = db_->GetSnapshot();
+  ReadOptions ro;
+  ro.snapshot = snapshot;
+
+  Iterator* iter = db_->NewIterator(ro);
+  db_->ReleaseSnapshot(snapshot);
+
+  iter->Seek(Key(1));
+  std::vector<int> expected_keys{1, 2, 6, 100};
+  for (int i : expected_keys) {
+    ASSERT_OK(iter->status());
+    ASSERT_TRUE(iter->Valid());
+    ASSERT_EQ(iter->key(), Key(i));
+    iter->Next();
+  }
+  ASSERT_TRUE(!iter->Valid() && iter->status().ok());
+
+  delete iter;
+}
+
+TEST_F(DBRangeDelTest, RefreshWithSnapshot) {
+  ASSERT_OK(Put(Key(4), "4"));
+  ASSERT_OK(Put(Key(6), "6"));
+  const Snapshot* snapshot = db_->GetSnapshot();
+  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(3),
+                             Key(5)));
+
+  std::unique_ptr<Iterator> iter{db_->NewIterator(ReadOptions())};
+  // Live Memtable
+  iter->SeekToFirst();
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_EQ(iter->key(), Key(6));
+  ASSERT_OK(iter->Refresh(snapshot));
+  iter->SeekToFirst();
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_EQ(iter->key(), Key(4));
+  // Immutable Memtable
+  ASSERT_OK(dbfull()->TEST_SwitchMemtable());
+  ASSERT_OK(iter->Refresh(nullptr));
+  iter->SeekToFirst();
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_EQ(iter->key(), Key(6));
+  ASSERT_OK(iter->Refresh(snapshot));
+  iter->SeekToFirst();
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_EQ(iter->key(), Key(4));
+  // L0
+  ASSERT_OK(Flush());
+  ASSERT_EQ(1, NumTableFilesAtLevel(0));
+  ASSERT_OK(iter->Refresh(nullptr));
+  iter->SeekToFirst();
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_EQ(iter->key(), Key(6));
+  ASSERT_OK(iter->Refresh(snapshot));
+  iter->SeekToFirst();
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_EQ(iter->key(), Key(4));
+  // L1
+  MoveFilesToLevel(1);
+  ASSERT_EQ(1, NumTableFilesAtLevel(1));
+  ASSERT_OK(iter->Refresh(nullptr));
+  iter->SeekToFirst();
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_EQ(iter->key(), Key(6));
+  ASSERT_OK(iter->Refresh(snapshot));
+  iter->SeekToFirst();
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_EQ(iter->key(), Key(4));
+  // L1 with two file.
+  // Test that when LevelIterator enters a new file,
+  // it remembers which snapshot sequence number to use.
+  ASSERT_OK(Put(Key(2), "2"));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(1);
+  ASSERT_EQ(2, NumTableFilesAtLevel(1));
+  ASSERT_OK(iter->Refresh(nullptr));
+  iter->SeekToFirst();
+  ASSERT_TRUE(iter->Valid());
+  // LevelIterator is at the first file
+  ASSERT_EQ(iter->key(), Key(2));
+  ASSERT_OK(iter->Refresh(snapshot));
+  // Will enter the second file, and create a new range tombstone iterator.
+  // It should use the snapshot sequence number.
+  iter->SeekToFirst();
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_EQ(iter->key(), Key(4));
+  iter.reset();
+  db_->ReleaseSnapshot(snapshot);
+}
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -220,6 +220,7 @@ TEST_P(DBRateLimiterOnReadTest, Iterator) {
      ++expected;
    }
  }
+  ASSERT_OK(iter->status());
  // Reverse scan does not read evenly (one block per iteration) due to
  // descending seqno ordering, so wait until after the loop to check total.
  ASSERT_EQ(expected, options_.rate_limiter->GetTotalRequests(Env::IO_USER));
@@ -240,6 +240,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, IteratorAndGet) {
      get_value_and_check(db_, read_opts, it->key(), it->value(),
                          write_timestamps[i]);
    }
+    ASSERT_OK(it->status());
    size_t expected_count = kMaxKey - start_keys[i] + 1;
    ASSERT_EQ(expected_count, count);

@@ -252,6 +253,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, IteratorAndGet) {
      get_value_and_check(db_, read_opts, it->key(), it->value(),
                          write_timestamps[i]);
    }
+    ASSERT_OK(it->status());
    ASSERT_EQ(static_cast<size_t>(kMaxKey) - start_keys[i] + 1, count);

    // SeekToFirst()/SeekToLast() with lower/upper bounds.
@@ -273,6 +275,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, IteratorAndGet) {
        get_value_and_check(db_, read_opts, it->key(), it->value(),
                            write_timestamps[i]);
      }
+      ASSERT_OK(it->status());
      ASSERT_EQ(r - std::max(l, start_keys[i]), count);

      for (it->SeekToLast(), key = std::min(r, kMaxKey + 1), count = 0;
@@ -282,6 +285,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, IteratorAndGet) {
        get_value_and_check(db_, read_opts, it->key(), it->value(),
                            write_timestamps[i]);
      }
+      ASSERT_OK(it->status());
      l += (kMaxKey / 100);
      r -= (kMaxKey / 100);
    }
@@ -328,6 +332,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, Iterators) {
    CheckIterUserEntry(iters[0], Key1(key), kTypeValue,
                       "value" + std::to_string(key), write_timestamp);
  }
+  ASSERT_OK(iters[0]->status());

  size_t expected_count = kMaxKey - 0 + 1;
  ASSERT_EQ(expected_count, count);
@@ -336,6 +341,53 @@ TEST_F(DBReadOnlyTestWithTimestamp, Iterators) {
  Close();
 }

+TEST_F(DBReadOnlyTestWithTimestamp, FullHistoryTsLowSanityCheckFail) {
+  Options options = CurrentOptions();
+  options.env = env_;
+  options.comparator = test::BytewiseComparatorWithU64TsWrapper();
+  // Use UDT in memtable only feature for this test, so we can control that
+  // newly set `full_history_ts_low` collapse history when Flush happens.
+  options.persist_user_defined_timestamps = false;
+  options.allow_concurrent_memtable_write = false;
+  DestroyAndReopen(options);
+
+  std::string write_ts;
+  PutFixed64(&write_ts, 1);
+  ASSERT_OK(db_->Put(WriteOptions(), "foo", write_ts, "val1"));
+
+  std::string full_history_ts_low;
+  PutFixed64(&full_history_ts_low, 3);
+  ASSERT_OK(db_->IncreaseFullHistoryTsLow(db_->DefaultColumnFamily(),
+                                          full_history_ts_low));
+  ASSERT_OK(Flush(0));
+
+  // Reopen the database in read only mode to test its timestamp support.
+  Close();
+  ASSERT_OK(ReadOnlyReopen(options));
+
+  // Reading below full_history_ts_low fails a sanity check.
+  std::string read_ts;
+  PutFixed64(&read_ts, 2);
+  Slice read_ts_slice = read_ts;
+  ReadOptions read_opts;
+  read_opts.timestamp = &read_ts_slice;
+
+  // Get()
+  std::string value;
+  ASSERT_TRUE(db_->Get(read_opts, "foo", &value).IsInvalidArgument());
+  // NewIterator()
+  std::unique_ptr<Iterator> iter(
+      db_->NewIterator(read_opts, db_->DefaultColumnFamily()));
+  ASSERT_TRUE(iter->status().IsInvalidArgument());
+
+  // NewIterators()
+  std::vector<ColumnFamilyHandle*> cfhs = {db_->DefaultColumnFamily()};
+  std::vector<Iterator*> iterators;
+  ASSERT_TRUE(
+      db_->NewIterators(read_opts, cfhs, &iterators).IsInvalidArgument());
+  Close();
+}
+
 TEST_F(DBReadOnlyTestWithTimestamp, IteratorsReadTimestampSizeMismatch) {
  const int kNumKeysPerFile = 128;
  const uint64_t kMaxKey = 1024;
@@ -164,12 +164,22 @@ TEST_F(DBSecondaryTest, ReopenAsSecondary) {
  Reopen(options);
  ASSERT_OK(Put("foo", "foo_value"));
  ASSERT_OK(Put("bar", "bar_value"));
+  WideColumns columns{{kDefaultWideColumnName, "attr_default_val"},
+                      {"attr_name1", "attr_value_1"},
+                      {"attr_name2", "attr_value_2"}};
+  ASSERT_OK(db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(), "baz",
+                           columns));
  ASSERT_OK(dbfull()->Flush(FlushOptions()));
  Close();

  ASSERT_OK(ReopenAsSecondary(options));
  ASSERT_EQ("foo_value", Get("foo"));
  ASSERT_EQ("bar_value", Get("bar"));
+  PinnableWideColumns result;
+  ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), "baz",
+                           &result));
+  ASSERT_EQ(result.columns(), columns);
+
  ReadOptions ropts;
  ropts.verify_checksums = true;
  auto db1 = static_cast<DBImplSecondary*>(db_);
@@ -182,13 +192,17 @@ TEST_F(DBSecondaryTest, ReopenAsSecondary) {
      ASSERT_EQ("bar", iter->key().ToString());
      ASSERT_EQ("bar_value", iter->value().ToString());
    } else if (1 == count) {
+      ASSERT_EQ("baz", iter->key().ToString());
+      ASSERT_EQ(columns, iter->columns());
+    } else if (2 == count) {
      ASSERT_EQ("foo", iter->key().ToString());
      ASSERT_EQ("foo_value", iter->value().ToString());
    }
    ++count;
  }
+  ASSERT_OK(iter->status());
  delete iter;
-  ASSERT_EQ(2, count);
+  ASSERT_EQ(3, count);
 }

 TEST_F(DBSecondaryTest, SimpleInternalCompaction) {
@@ -521,6 +535,8 @@ TEST_F(DBSecondaryTest, SecondaryCloseFiles) {
    }
    ASSERT_FALSE(iter1->Valid());
    ASSERT_FALSE(iter2->Valid());
+    ASSERT_OK(iter1->status());
+    ASSERT_OK(iter2->status());
  };

  ASSERT_OK(Put("a", "value"));
@@ -793,6 +809,7 @@ TEST_F(DBSecondaryTest, MissingTableFileDuringOpen) {
  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
    ++count;
  }
+  ASSERT_OK(iter->status());
  ASSERT_EQ(2, count);
  delete iter;
 }
@@ -850,6 +867,7 @@ TEST_F(DBSecondaryTest, MissingTableFile) {
  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
    ++count;
  }
+  ASSERT_OK(iter->status());
  ASSERT_EQ(2, count);
  delete iter;
 }
@@ -922,6 +940,7 @@ TEST_F(DBSecondaryTest, SwitchManifest) {
      ASSERT_EQ("value_" + std::to_string(kNumFiles - 1),
                iter->value().ToString());
    }
+    EXPECT_OK(iter->status());
  };

  range_scan_db();
@@ -1472,6 +1491,7 @@ TEST_F(DBSecondaryTestWithTimestamp, IteratorAndGet) {
      get_value_and_check(db_, read_opts, it->key(), it->value(),
                          write_timestamps[i]);
    }
+    ASSERT_OK(it->status());
    size_t expected_count = kMaxKey - start_keys[i] + 1;
    ASSERT_EQ(expected_count, count);

@@ -1484,6 +1504,7 @@ TEST_F(DBSecondaryTestWithTimestamp, IteratorAndGet) {
      get_value_and_check(db_, read_opts, it->key(), it->value(),
                          write_timestamps[i]);
    }
+    ASSERT_OK(it->status());
    ASSERT_EQ(static_cast<size_t>(kMaxKey) - start_keys[i] + 1, count);

    // SeekToFirst()/SeekToLast() with lower/upper bounds.
@@ -1505,6 +1526,7 @@ TEST_F(DBSecondaryTestWithTimestamp, IteratorAndGet) {
        get_value_and_check(db_, read_opts, it->key(), it->value(),
                            write_timestamps[i]);
      }
+      ASSERT_OK(it->status());
      ASSERT_EQ(r - std::max(l, start_keys[i]), count);

      for (it->SeekToLast(), key = std::min(r, kMaxKey + 1), count = 0;
@@ -1514,6 +1536,7 @@ TEST_F(DBSecondaryTestWithTimestamp, IteratorAndGet) {
        get_value_and_check(db_, read_opts, it->key(), it->value(),
                            write_timestamps[i]);
      }
+      ASSERT_OK(it->status());
      l += (kMaxKey / 100);
      r -= (kMaxKey / 100);
    }
@@ -1561,6 +1584,55 @@ TEST_F(DBSecondaryTestWithTimestamp, IteratorsReadTimestampSizeMismatch) {
  Close();
 }

+TEST_F(DBSecondaryTestWithTimestamp, FullHistoryTsLowSanityCheckFail) {
+  Options options = CurrentOptions();
+  options.env = env_;
+  options.comparator = test::BytewiseComparatorWithU64TsWrapper();
+  // Use UDT in memtable only feature for this test, so we can control that
+  // newly set `full_history_ts_low` collapse history when Flush happens.
+  options.persist_user_defined_timestamps = false;
+  options.allow_concurrent_memtable_write = false;
+  DestroyAndReopen(options);
+
+  std::string write_ts;
+  PutFixed64(&write_ts, 1);
+  ASSERT_OK(db_->Put(WriteOptions(), "foo", write_ts, "val1"));
+
+  std::string full_history_ts_low;
+  PutFixed64(&full_history_ts_low, 3);
+  ASSERT_OK(db_->IncreaseFullHistoryTsLow(db_->DefaultColumnFamily(),
+                                          full_history_ts_low));
+  ASSERT_OK(Flush(0));
+
+  // Reopen the database as secondary instance to test its timestamp support.
+  Close();
+  options.max_open_files = -1;
+  ASSERT_OK(ReopenAsSecondary(options));
+
+  // Reading below full_history_ts_low fails a sanity check.
+  std::string read_ts;
+  PutFixed64(&read_ts, 2);
+  Slice read_ts_slice = read_ts;
+  ReadOptions read_opts;
+  read_opts.timestamp = &read_ts_slice;
+
+  // Get()
+  std::string value;
+  ASSERT_TRUE(db_->Get(read_opts, "foo", &value).IsInvalidArgument());
+
+  // NewIterator()
+  std::unique_ptr<Iterator> iter(
+      db_->NewIterator(read_opts, db_->DefaultColumnFamily()));
+  ASSERT_TRUE(iter->status().IsInvalidArgument());
+
+  // NewIterators()
+  std::vector<ColumnFamilyHandle*> cfhs = {db_->DefaultColumnFamily()};
+  std::vector<Iterator*> iterators;
+  ASSERT_TRUE(
+      db_->NewIterators(read_opts, cfhs, &iterators).IsInvalidArgument());
+  Close();
+}
+
 TEST_F(DBSecondaryTestWithTimestamp,
       IteratorsReadTimestampSpecifiedWithoutWriteTimestamp) {
  const int kNumKeysPerFile = 128;
@@ -1674,6 +1746,7 @@ TEST_F(DBSecondaryTestWithTimestamp, Iterators) {
    CheckIterUserEntry(iters[0], Key1(key), kTypeValue,
                       "value" + std::to_string(key), write_timestamp);
  }
+  ASSERT_OK(iters[0]->status());

  size_t expected_count = kMaxKey - 0 + 1;
  ASSERT_EQ(expected_count, count);
@@ -937,12 +937,21 @@ INSTANTIATE_TEST_CASE_P(DBWALTestWithParam, DBWALTestWithParam,
                        ::testing::Values(std::make_tuple("", true),
                                          std::make_tuple("_wal_dir", false)));

-TEST_F(DBSSTTest, OpenDBWithExistingTrash) {
+TEST_F(DBSSTTest, OpenDBWithExistingTrashAndObsoleteSstFile) {
  Options options = CurrentOptions();
-
  options.sst_file_manager.reset(
      NewSstFileManager(env_, nullptr, "", 1024 * 1024 /* 1 MB/sec */));
  auto sfm = static_cast<SstFileManagerImpl*>(options.sst_file_manager.get());
+  // Set an extra high trash ratio to prevent immediate/non-rate limited
+  // deletions
+  sfm->SetDeleteRateBytesPerSecond(1024 * 1024);
+  sfm->delete_scheduler()->SetMaxTrashDBRatio(1000.0);
+
+  int bg_delete_file = 0;
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "DeleteScheduler::DeleteTrashFile:DeleteFile",
+      [&](void* /*arg*/) { bg_delete_file++; });
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();

  Destroy(last_options_);

@@ -951,13 +960,22 @@ TEST_F(DBSSTTest, OpenDBWithExistingTrash) {
  ASSERT_OK(WriteStringToFile(env_, "abc", dbname_ + "/" + "001.sst.trash"));
  ASSERT_OK(WriteStringToFile(env_, "abc", dbname_ + "/" + "002.sst.trash"));
  ASSERT_OK(WriteStringToFile(env_, "abc", dbname_ + "/" + "003.sst.trash"));
+  // Manually add an obsolete sst file. Obsolete SST files are discovered and
+  // deleted upon recovery.
+  constexpr uint64_t kSstFileNumber = 100;
+  const std::string kObsoleteSstFile =
+      MakeTableFileName(dbname_, kSstFileNumber);
+  ASSERT_OK(WriteStringToFile(env_, "abc", kObsoleteSstFile));

-  // Reopen the DB and verify that it deletes existing trash files
+  // Reopen the DB and verify that it deletes existing trash files and obsolete
+  // SST files with rate limiting.
  Reopen(options);
  sfm->WaitForEmptyTrash();
  ASSERT_NOK(env_->FileExists(dbname_ + "/" + "001.sst.trash"));
  ASSERT_NOK(env_->FileExists(dbname_ + "/" + "002.sst.trash"));
  ASSERT_NOK(env_->FileExists(dbname_ + "/" + "003.sst.trash"));
+  ASSERT_NOK(env_->FileExists(kObsoleteSstFile));
+  ASSERT_EQ(bg_delete_file, 4);
 }

 // Create a DB with 2 db_paths, and generate multiple files in the 2
@@ -1520,6 +1538,11 @@ TEST_F(DBSSTTest, OpenDBWithInfiniteMaxOpenFilesSubjectToMemoryLimit) {
 }

 TEST_F(DBSSTTest, GetTotalSstFilesSize) {
+  // FIXME: L0 file and L1+ file also differ in size of `oldest_key_time`.
+  //  L0 file has non-zero `oldest_key_time` while L1+ files have 0.
+  //  The test passes since L1+ file uses current time instead of 0
+  //  as oldest_ancestor_time.
+  //
  // We don't propagate oldest-key-time table property on compaction and
  // just write 0 as default value. This affect the exact table size, since
  // we encode table properties as varint64. Force time to be 0 to work around
@@ -6,9 +6,11 @@
 #include <string>

 #include "db/db_test_util.h"
+#include "db/write_batch_internal.h"
 #include "monitoring/thread_status_util.h"
 #include "port/stack_trace.h"
 #include "rocksdb/statistics.h"
+#include "rocksdb/utilities/transaction_db.h"
 #include "util/random.h"

 namespace ROCKSDB_NAMESPACE {
@@ -283,6 +285,78 @@ TEST_F(DBStatisticsTest, BlockChecksumStats) {
            options.statistics->getTickerCount(BLOCK_CHECKSUM_MISMATCH_COUNT));
 }

+TEST_F(DBStatisticsTest, BytesWrittenStats) {
+  Options options = CurrentOptions();
+  options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
+  options.statistics->set_stats_level(StatsLevel::kExceptHistogramOrTimers);
+  Reopen(options);
+
+  EXPECT_EQ(0, options.statistics->getAndResetTickerCount(WAL_FILE_BYTES));
+  EXPECT_EQ(0, options.statistics->getAndResetTickerCount(BYTES_WRITTEN));
+
+  const int kNumKeysWritten = 100;
+
+  // Scenario 0: Not using transactions.
+  // This will write to WAL and memtable directly.
+  ASSERT_OK(options.statistics->Reset());
+
+  for (int i = 0; i < kNumKeysWritten; ++i) {
+    ASSERT_OK(Put(Key(i), "val"));
+  }
+
+  EXPECT_EQ(options.statistics->getAndResetTickerCount(WAL_FILE_BYTES),
+            options.statistics->getAndResetTickerCount(BYTES_WRITTEN));
+
+  // Scenario 1: Using transactions.
+  // This should not double count BYTES_WRITTEN (issue #12061).
+  for (bool enable_pipelined_write : {false, true}) {
+    ASSERT_OK(options.statistics->Reset());
+
+    // Destroy the DB to recreate as a TransactionDB.
+    Destroy(options, true);
+
+    // Create a TransactionDB.
+    TransactionDB* txn_db = nullptr;
+    TransactionDBOptions txn_db_opts;
+    txn_db_opts.write_policy = TxnDBWritePolicy::WRITE_COMMITTED;
+    options.enable_pipelined_write = enable_pipelined_write;
+    ASSERT_OK(TransactionDB::Open(options, txn_db_opts, dbname_, &txn_db));
+    ASSERT_NE(txn_db, nullptr);
+    db_ = txn_db->GetBaseDB();
+
+    WriteOptions wopts;
+    TransactionOptions txn_opts;
+    Transaction* txn = txn_db->BeginTransaction(wopts, txn_opts, nullptr);
+    ASSERT_NE(txn, nullptr);
+    ASSERT_OK(txn->SetName("txn1"));
+
+    for (int i = 0; i < kNumKeysWritten; ++i) {
+      ASSERT_OK(txn->Put(Key(i), "val"));
+    }
+
+    // Prepare() writes to WAL, but not to memtable. (WriteCommitted)
+    ASSERT_OK(txn->Prepare());
+    EXPECT_NE(0, options.statistics->getTickerCount(WAL_FILE_BYTES));
+    // BYTES_WRITTEN would have been non-zero previously (issue #12061).
+    EXPECT_EQ(0, options.statistics->getTickerCount(BYTES_WRITTEN));
+
+    // Commit() writes to memtable and also a commit marker to WAL.
+    ASSERT_OK(txn->Commit());
+    delete txn;
+
+    // The WAL has an extra header of size `kHeader` written to it,
+    // as we are writing twice to it (first during Prepare, second during
+    // Commit).
+    EXPECT_EQ(options.statistics->getAndResetTickerCount(WAL_FILE_BYTES),
+              options.statistics->getAndResetTickerCount(BYTES_WRITTEN) +
+                  WriteBatchInternal::kHeader);
+
+    // Cleanup
+    db_ = nullptr;
+    delete txn_db;
+  }
+}
+
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -22,6 +22,7 @@
 #include "table/table_properties_internal.h"
 #include "test_util/testharness.h"
 #include "test_util/testutil.h"
+#include "util/atomic.h"
 #include "util/random.h"


@@ -417,6 +418,71 @@ TEST_F(DBTablePropertiesTest, GetDbIdentifiersProperty) {
  }
 }

+TEST_F(DBTablePropertiesTest, FactoryReturnsNull) {
+  struct JunkTablePropertiesCollector : public TablePropertiesCollector {
+    const char* Name() const override { return "JunkTablePropertiesCollector"; }
+    Status Finish(UserCollectedProperties* properties) override {
+      properties->insert({"Junk", "Junk"});
+      return Status::OK();
+    }
+    UserCollectedProperties GetReadableProperties() const override {
+      return {};
+    }
+  };
+
+  // Alternates between putting a "Junk" property and using `nullptr` to
+  // opt out.
+  static RelaxedAtomic<int> count{0};
+  struct SometimesTablePropertiesCollectorFactory
+      : public TablePropertiesCollectorFactory {
+    const char* Name() const override {
+      return "SometimesTablePropertiesCollectorFactory";
+    }
+    TablePropertiesCollector* CreateTablePropertiesCollector(
+        TablePropertiesCollectorFactory::Context /*context*/) override {
+      if (count.FetchAddRelaxed(1) & 1) {
+        return nullptr;
+      } else {
+        return new JunkTablePropertiesCollector();
+      }
+    }
+  };
+
+  Options options = CurrentOptions();
+  options.table_properties_collector_factories.emplace_back(
+      std::make_shared<SometimesTablePropertiesCollectorFactory>());
+  // For plain table
+  options.prefix_extractor.reset(NewFixedPrefixTransform(4));
+  for (std::shared_ptr<TableFactory> tf :
+       {options.table_factory,
+        std::shared_ptr<TableFactory>(NewPlainTableFactory({}))}) {
+    SCOPED_TRACE("Table factory = " + std::string(tf->Name()));
+    options.table_factory = tf;
+
+    DestroyAndReopen(options);
+
+    ASSERT_OK(Put("key0", "value1"));
+    ASSERT_OK(Flush());
+    ASSERT_OK(Put("key0", "value2"));
+    ASSERT_OK(Flush());
+
+    TablePropertiesCollection props;
+    ASSERT_OK(db_->GetPropertiesOfAllTables(&props));
+    int no_junk_count = 0;
+    int junk_count = 0;
+    for (const auto& item : props) {
+      if (item.second->user_collected_properties.find("Junk") !=
+          item.second->user_collected_properties.end()) {
+        junk_count++;
+      } else {
+        no_junk_count++;
+      }
+    }
+    EXPECT_EQ(1, no_junk_count);
+    EXPECT_EQ(1, junk_count);
+  }
+}
+
 class DBTableHostnamePropertyTest
    : public DBTestBase,
      public ::testing::WithParamInterface<std::tuple<int, std::string>> {