Fix the ZStd checksum (#12005 )

Summary: Somehow we had the wrong checksum when validating the ZStd 1.5.5 download for RocksJava in the previous Pull Request - https://github.com/facebook/rocksdb/pull/9304. This PR fixes that. Pull Request resolved: https://github.com/facebook/rocksdb/pull/12005 Reviewed By: jaykorean Differential Revision: D50840338 Pulled By: cbi42 fbshipit-source-id: 8a92779d3bef013d812eecb89aaaf33fc73991ec
Update HISTORY.md and version for 8.7.3
2026-11-06 11:02:36 -05:00 · 2023-10-31 13:22:47 -07:00 · 2023-10-30 14:07:10 -07:00 · 2023-10-30 11:43:56 -07:00 · 2023-10-25 21:20:16 -07:00 · 2023-10-25 20:48:44 -07:00
469 changed files with 27704 additions and 7008 deletions
@@ -16,7 +16,8 @@ commands:
      - run:
          name: Install JDK 8 on macos
          command: |
-            brew install --cask adoptopenjdk/openjdk/adoptopenjdk8
+            HOMEBREW_NO_AUTO_UPDATE=1 brew tap bell-sw/liberica
+            HOMEBREW_NO_AUTO_UPDATE=1 brew install --cask liberica-jdk8

  increase-max-open-files-on-macos:
    steps:
@@ -205,25 +206,28 @@ executors:
      #  $ docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -it zjay437/rocksdb:0.5 bash
      # option `--cap-add=SYS_PTRACE --security-opt seccomp=unconfined` is used to enable gdb to attach an existing process
      - image: zjay437/rocksdb:0.6
+  linux-java-docker:
+    docker:
+      - image: evolvedbinary/rocksjava:centos6_x64-be

 jobs:
  build-macos:
    macos:
-      xcode: 12.5.1
-    resource_class: large
+      xcode: 14.3.1
+    resource_class: macos.m1.medium.gen1
    environment:
      ROCKSDB_DISABLE_JEMALLOC: 1 # jemalloc cause env_test hang, disable it for now
    steps:
      - increase-max-open-files-on-macos
      - install-gflags-on-macos
      - pre-steps-macos
-      - run: ulimit -S -n `ulimit -H -n` && OPT=-DCIRCLECI make V=1 J=32 -j32 all
+      - run: ulimit -S -n `ulimit -H -n` && OPT=-DCIRCLECI make V=1 J=16 -j16 all
      - post-steps

  build-macos-cmake:
    macos:
-      xcode: 12.5.1
-    resource_class: large
+      xcode: 14.3.1
+    resource_class: macos.m1.medium.gen1
    parameters:
      run_even_tests:
        description: run even or odd tests, used to split tests to 2 groups
@@ -239,20 +243,20 @@ jobs:
          command: ulimit -S -n `ulimit -H -n` && mkdir build && cd build && cmake -DWITH_GFLAGS=1 ..
      - run:
          name: "Build tests"
-          command: cd build && make V=1 -j32
+          command: cd build && make V=1 -j16
      - when:
          condition: << parameters.run_even_tests >>
          steps:
            - run:
                name: "Run even tests"
-                command: ulimit -S -n `ulimit -H -n` && cd build && ctest -j32 -I 0,,2
+                command: ulimit -S -n `ulimit -H -n` && cd build && ctest -j16 -I 0,,2
      - when:
          condition:
            not: << parameters.run_even_tests >>
          steps:
            - run:
                name: "Run odd tests"
-                command: ulimit -S -n `ulimit -H -n` && cd build && ctest -j32 -I 1,,2
+                command: ulimit -S -n `ulimit -H -n` && cd build && ctest -j16 -I 1,,2
      - post-steps

  build-linux:
@@ -589,7 +593,7 @@ jobs:
      - post-steps

  build-linux-java-static:
-    executor: linux-docker
+    executor: linux-java-docker
    resource_class: large
    steps:
      - pre-steps
@@ -602,15 +606,15 @@ jobs:
            which javac && javac -version
      - run:
          name: "Build RocksDBJava Static Library"
-          command: make V=1 J=8 -j8 rocksdbjavastatic
+          command: scl enable devtoolset-7 'make V=1 J=8 -j8 rocksdbjavastatic'
      - post-steps

  build-macos-java:
    macos:
-      xcode: 12.5.1
-    resource_class: large
+      xcode: 14.3.1
+    resource_class: macos.m1.medium.gen1
    environment:
-      JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
+      JAVA_HOME: /Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home
      ROCKSDB_DISABLE_JEMALLOC: 1 # jemalloc causes java 8 crash
    steps:
      - increase-max-open-files-on-macos
@@ -632,10 +636,10 @@ jobs:

  build-macos-java-static:
    macos:
-      xcode: 12.5.1
-    resource_class: large
+      xcode: 14.3.1
+    resource_class: macos.m1.medium.gen1
    environment:
-      JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
+      JAVA_HOME: /Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home
    steps:
      - increase-max-open-files-on-macos
      - install-gflags-on-macos
@@ -657,10 +661,10 @@ jobs:

  build-macos-java-static-universal:
    macos:
-      xcode: 12.5.1
-    resource_class: large
+      xcode: 14.3.1
+    resource_class: macos.m1.medium.gen1
    environment:
-      JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
+      JAVA_HOME: /Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home
    steps:
      - increase-max-open-files-on-macos
      - install-gflags-on-macos
@@ -33,9 +33,7 @@ jobs:
      run: pip install argparse

    - name: Download clang-format-diff.py
-      uses: wei/wget@v1
-      with:
-        args: https://raw.githubusercontent.com/llvm/llvm-project/release/12.x/clang/tools/clang-format/clang-format-diff.py
+      run: wget https://raw.githubusercontent.com/llvm/llvm-project/release/12.x/clang/tools/clang-format/clang-format-diff.py

    - name: Check format
      run: VERBOSE_CHECK=1 make check-format
@@ -85,6 +85,7 @@ fbcode/
 fbcode
 buckifier/*.pyc
 buckifier/__pycache__
+.arcconfig

 compile_commands.json
 clang-format-diff.py
@@ -95,4 +96,5 @@ fuzz/crash-*

 cmake-build-*
 third-party/folly/
-.cache
+.cache
+*.sublime-*
@@ -254,7 +254,7 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "loongarch64")
 endif(CMAKE_SYSTEM_PROCESSOR MATCHES "loongarch64")

 set(PORTABLE 0 CACHE STRING "Minimum CPU arch to support, or 0 = current CPU, 1 = baseline CPU")
-if(PORTABLE STREQUAL 1)
+if(PORTABLE MATCHES "1|ON|YES|TRUE|Y")
  # Usually nothing to do; compiler default is typically the most general
  if(NOT MSVC)
    if(CMAKE_SYSTEM_PROCESSOR MATCHES "^s390x")
@@ -264,14 +264,7 @@ if(PORTABLE STREQUAL 1)
      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=loongarch64")
    endif()
  endif()
-elseif(PORTABLE MATCHES [^0]+)
-  # Name of a CPU arch spec or feature set to require
-  if(MSVC)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:${PORTABLE}")
-  else()
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=${PORTABLE}")
-  endif()
-else()
+elseif(PORTABLE MATCHES "0|OFF|NO|FALSE|N")
  if(MSVC)
    # NOTE: No auto-detection of current CPU, but instead assume some useful
    # level of optimization is supported
@@ -285,6 +278,13 @@ else()
      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
    endif()
  endif()
+else()
+  # Name of a CPU arch spec or feature set to require
+  if(MSVC)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:${PORTABLE}")
+  else()
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=${PORTABLE}")
+  endif()
 endif()

 include(CheckCXXSourceCompiles)
@@ -547,7 +547,7 @@ if(HAVE_SCHED_GETCPU)
  add_definitions(-DROCKSDB_SCHED_GETCPU_PRESENT)
 endif()

-check_cxx_symbol_exists(getauxval auvx.h HAVE_AUXV_GETAUXVAL)
+check_cxx_symbol_exists(getauxval "sys/auxv.h" HAVE_AUXV_GETAUXVAL)
 if(HAVE_AUXV_GETAUXVAL)
  add_definitions(-DROCKSDB_AUXV_GETAUXVAL_PRESENT)
 endif()
@@ -632,6 +632,7 @@ set(SOURCES
        cache/secondary_cache.cc
        cache/secondary_cache_adapter.cc
        cache/sharded_cache.cc
+        cache/tiered_secondary_cache.cc
        db/arena_wrapped_db_iter.cc
        db/blob/blob_contents.cc
        db/blob/blob_fetcher.cc
@@ -714,6 +715,7 @@ set(SOURCES
        db/wal_manager.cc
        db/wide/wide_column_serialization.cc
        db/wide/wide_columns.cc
+        db/wide/wide_columns_helper.cc
        db/write_batch.cc
        db/write_batch_base.cc
        db/write_controller.cc
@@ -865,6 +867,8 @@ set(SOURCES
        util/string_util.cc
        util/thread_local.cc
        util/threadpool_imp.cc
+        util/udt_util.cc
+        util/write_batch_util.cc
        util/xxhash.cc
        utilities/agg_merge/agg_merge.cc
        utilities/backup/backup_engine.cc
@@ -1260,6 +1264,7 @@ if(WITH_TESTS)
        cache/cache_test.cc
        cache/compressed_secondary_cache_test.cc
        cache/lru_cache_test.cc
+        cache/tiered_secondary_cache_test.cc
        db/blob/blob_counting_iterator_test.cc
        db/blob/blob_file_addition_test.cc
        db/blob/blob_file_builder_test.cc
@@ -1355,6 +1360,7 @@ if(WITH_TESTS)
        db/wal_edit_test.cc
        db/wide/db_wide_basic_test.cc
        db/wide/wide_column_serialization_test.cc
+        db/wide/wide_columns_helper_test.cc
        db/write_batch_test.cc
        db/write_callback_test.cc
        db/write_controller_test.cc
@@ -1421,6 +1427,7 @@ if(WITH_TESTS)
        util/timer_test.cc
        util/thread_list_test.cc
        util/thread_local_test.cc
+        util/udt_util_test.cc
        util/work_queue_test.cc
        utilities/agg_merge/agg_merge_test.cc
        utilities/backup/backup_engine_test.cc
@@ -1589,3 +1596,6 @@ option(WITH_BENCHMARK "build benchmark tests" OFF)
 if(WITH_BENCHMARK)
  add_subdirectory(${PROJECT_SOURCE_DIR}/microbench/)
 endif()
+
+target_include_directories(${PROJECT_NAME} PUBLIC
+  $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>)
@@ -1,5 +1,143 @@
 # Rocksdb Change Log
-## Unreleased
+> NOTE: Entries for next release do not go here. Follow instructions in `unreleased_history/README.txt`
+
+## 8.7.3 (10/30/2023)
+### Behavior Changes
+* Deleting stale files upon recovery are delegated to SstFileManger if available so they can be rate limited.
+
+## 8.7.2 (10/25/2023)
+### Public API Changes
+* Add new Cache APIs GetSecondaryCacheCapacity() and GetSecondaryCachePinnedUsage() to return the configured capacity, and cache reservation charged to the secondary cache.
+
+### Bug Fixes
+* Fixed a possible underflow when computing the compressed secondary cache share of memory reservations while updating the compressed secondary to total block cache ratio.
+* Fix an assertion failure when UpdeteTieredCache() is called in an idempotent manner.
+
+## 8.7.1 (10/20/2023)
+### Bug Fixes
+* Fix a bug in auto_readahead_size where first_internal_key of index blocks wasn't copied properly resulting in corruption error when first_internal_key was used for comparison.
+* Add bounds check in WBWIIteratorImpl and make BaseDeltaIterator, WriteUnpreparedTxn and WritePreparedTxn respect the upper bound and lower bound in ReadOption. See 11680.
+
+## 8.7.0 (09/22/2023)
+### New Features
+* Added an experimental new "automatic" variant of HyperClockCache that does not require a prior estimate of the average size of cache entries. This variant is activated when HyperClockCacheOptions::estimated\_entry\_charge = 0 and has essentially the same concurrency benefits as the existing HyperClockCache.
+* Add a new statistic `COMPACTION_CPU_TOTAL_TIME` that records cumulative compaction cpu time. This ticker is updated regularly while a compaction is running.
+* Add `GetEntity()` API for ReadOnly DB and Secondary DB.
+* Add a new iterator API `Iterator::Refresh(const Snapshot *)` that allows iterator to be refreshed while using the input snapshot to read.
+* Added a new read option `merge_operand_count_threshold`. When the number of merge operands applied during a successful point lookup exceeds this threshold, the query will return a special OK status with a new subcode `kMergeOperandThresholdExceeded`. Applications might use this signal to take action to reduce the number of merge operands for the affected key(s), for example by running a compaction.
+* For `NewRibbonFilterPolicy()`, made the `bloom_before_level` option mutable through the Configurable interface and the SetOptions API, allowing dynamic switching between all-Bloom and all-Ribbon configurations, and configurations in between. See comments on `NewRibbonFilterPolicy()`
+* RocksDB now allows the block cache to be stacked on top of a compressed secondary cache and a non-volatile secondary cache, thus creating a three-tier cache. To set it up, use the `NewTieredCache()` API in rocksdb/cache.h..
+* Added a new wide-column aware full merge API called `FullMergeV3` to `MergeOperator`. `FullMergeV3` supports wide columns both as base value and merge result, which enables the application to perform more general transformations during merges. For backward compatibility, the default implementation implements the earlier logic of applying the merge operation to the default column of any wide-column entities. Specifically, if there is no base value or the base value is a plain key-value, the default implementation falls back to `FullMergeV2`. If the base value is a wide-column entity, the default implementation invokes `FullMergeV2` to perform the merge on the default column, and leaves any other columns unchanged.
+* Add wide column support to ldb commands (scan, dump, idump, dump_wal) and sst_dump tool's scan command
+
+### Public API Changes
+* Expose more information about input files used in table creation (if any) in `CompactionFilter::Context`. See `CompactionFilter::Context::input_start_level`,`CompactionFilter::Context::input_table_properties` for more.
+* `Options::compaction_readahead_size` 's default value is changed from 0 to 2MB.
+* When using LZ4 compression, the `acceleration` parameter is configurable by setting the negated value in `CompressionOptions::level`. For example, `CompressionOptions::level=-10` will set `acceleration=10`
+* The `NewTieredCache` API has been changed to take the total cache capacity (inclusive of both the primary and the compressed secondary cache) and the ratio of total capacity to allocate to the compressed cache. These are specified in `TieredCacheOptions`. Any capacity specified in `LRUCacheOptions`, `HyperClockCacheOptions` and `CompressedSecondaryCacheOptions` is ignored. A new API, `UpdateTieredCache` is provided to dynamically update the total capacity, ratio of compressed cache, and admission policy.
+* The `NewTieredVolatileCache()` API in rocksdb/cache.h has been renamed to `NewTieredCache()`.
+
+### Behavior Changes
+* Compaction read performance will regress when `Options::compaction_readahead_size` is explicitly set to 0
+* Universal size amp compaction will conditionally exclude some of the newest L0 files when selecting input with a small negative impact to size amp. This is to prevent a large number of L0 files from being locked by a size amp compaction, potentially leading to write stop with a few more flushes.
+* Change ldb scan command delimiter from ':' to '==>'.
+* For non direct IO, eliminate the file system prefetching attempt for compaction read when `Options::compaction_readahead_size` is 0
+
+### Bug Fixes
+* Fix a bug where if there is an error reading from offset 0 of a file from L1+ and that the file is not the first file in the sorted run, data can be lost in compaction and read/scan can return incorrect results.
+* Fix a bug where iterator may return incorrect result for DeleteRange() users if there was an error reading from a file.
+* Fix a bug with atomic_flush=true that can cause DB to stuck after a flush fails (#11872).
+* Fix a bug where RocksDB (with atomic_flush=false) can delete output SST files of pending flushes when a previous concurrent flush fails (#11865). This can result in DB entering read-only state with error message like `IO error: No such file or directory: While open a file for random read: /tmp/rocksdbtest-501/db_flush_test_87732_4230653031040984171/000013.sst`.
+* Fix an assertion fault during seek with async_io when readahead trimming is enabled.
+* When the compressed secondary cache capacity is reduced to 0, it should be completely disabled. Before this fix, inserts and lookups would still go to the backing `LRUCache` before returning, thus incurring locking overhead. With this fix, inserts and lookups are no-ops and do not add any overhead.
+* Updating the tiered cache (cache allocated using NewTieredCache()) by calling SetCapacity() on it was not working properly. The initial creation would set the primary cache capacity to the combined primary and compressed secondary cache capacity. But SetCapacity() would just set the primary cache capacity. With this fix, the user always specifies the total budget and compressed secondary cache ratio on creation. Subsequently, SetCapacity() will distribute the new capacity across the two caches by the same ratio.
+* Fixed a bug in `MultiGet` for cleaning up SuperVersion acquired with locking db mutex.
+* Fix a bug where row cache can falsely return kNotFound even though row cache entry is hit.
+* Fixed a race condition in `GenericRateLimiter` that could cause it to stop granting requests
+* Fix a bug (Issue #10257) where DB can hang after write stall since no compaction is scheduled (#11764).
+* Add a fix for async_io where during seek, when reading a block for seeking a target key in a file without any readahead, the iterator aligned the read on a page boundary and reading more than necessary. This increased the storage read bandwidth usage.
+* Fix an issue in sst dump tool to handle bounds specified for data with user-defined timestamps.
+* When auto_readahead_size is enabled, update readahead upper bound during readahead trimming when reseek changes iterate_upper_bound dynamically.
+* Fixed a bug where `rocksdb.file.read.verify.file.checksums.micros` is not populated
+* Fixed a bug where compaction read under non direct IO still falls back to RocksDB internal prefetching after file system's prefetching returns non-OK status other than `Status::NotSupported()`
+
+### Performance Improvements
+* Added additional improvements in tuning readahead_size during Scans when auto_readahead_size is enabled. However it's not recommended for backward scans and might impact the performance. More details in options.h.
+* During async_io, the Seek happens in 2 phases. Phase 1 starts an asynchronous read on a block cache miss, and phase 2 waits for it to complete and finishes the seek. In both phases, it tries to lookup the block cache for the data block first before looking in the prefetch buffer. It's optimized by doing the block cache lookup only in the first phase that would save some CPU.
+
+## 8.6.0 (08/18/2023)
+### New Features
+* Added enhanced data integrity checking on SST files with new format_version=6. Performance impact is very small or negligible. Previously if SST data was misplaced or re-arranged by the storage layer, it could pass block checksum with higher than 1 in 4 billion probability. With format_version=6, block checksums depend on what file they are in and location within the file. This way, misplaced SST data is no more likely to pass checksum verification than randomly corrupted data. Also in format_version=6, SST footers are checksum-protected.
+* Add a new feature to trim readahead_size during scans upto upper_bound when iterate_upper_bound is specified. It's enabled through ReadOptions.auto_readahead_size. Users must also specify ReadOptions.iterate_upper_bound.
+* RocksDB will compare the number of input keys to the number of keys processed after each compaction. Compaction will fail and report Corruption status if the verification fails. Option `compaction_verify_record_count` is introduced for this purpose and is enabled by default.
+* Add a CF option `bottommost_file_compaction_delay` to allow specifying the delay of bottommost level single-file compactions.
+* Add support to allow enabling / disabling user-defined timestamps feature for an existing column family in combination with the in-Memtable only feature.
+* Implement a new admission policy for the compressed secondary cache that admits blocks evicted from the primary cache with the hit bit set. This policy can be specified in TieredVolatileCacheOptions by setting the newly added adm_policy option.
+* Add a column family option `memtable_max_range_deletions` that limits the number of range deletions in a memtable. RocksDB will try to do an automatic flush after the limit is reached. (#11358)
+* Add PutEntity API in sst_file_writer
+* Add `timeout` in microsecond option to `WaitForCompactOptions` to allow timely termination of prolonged waiting in scenarios like recurring recoverable errors, such as out-of-space situations and continuous write streams that sustain ongoing flush and compactions
+* New statistics `rocksdb.file.read.{get|multiget|db.iterator|verify.checksum|verify.file.checksums}.micros` measure read time of block-based SST tables or blob files during db open, `Get()`, `MultiGet()`, using db iterator, `VerifyFileChecksums()` and `VerifyChecksum()`. They require stats level greater than `StatsLevel::kExceptDetailedTimers`.
+* Add close_db option to `WaitForCompactOptions` to call Close() after waiting is done.
+* Add a new compression option `CompressionOptions::checksum` for enabling ZSTD's checksum feature to detect corruption during decompression.
+
+### Public API Changes
+* Mark `Options::access_hint_on_compaction_start` related APIs as deprecated. See #11631 for alternative behavior.
+
+### Behavior Changes
+* Statistics `rocksdb.sst.read.micros` now includes time spent on multi read and async read into the file
+* For Universal Compaction users, periodic compaction (option `periodic_compaction_seconds`) will be set to 30 days by default if block based table is used.
+
+### Bug Fixes
+* Fix a bug in FileTTLBooster that can cause users with a large number of levels (more than 65) to see errors like "runtime error: shift exponent .. is too large.." (#11673).
+
+## 8.5.0 (07/21/2023)
+### Public API Changes
+* Removed recently added APIs `GeneralCache` and `MakeSharedGeneralCache()` as our plan changed to stop exposing a general-purpose cache interface. The old forms of these APIs, `Cache` and `NewLRUCache()`, are still available, although general-purpose caching support will be dropped eventually.
+
+### Behavior Changes
+* Option `periodic_compaction_seconds` no longer supports FIFO compaction: setting it has no effect on FIFO compactions. FIFO compaction users should only set option `ttl` instead.
+* Move prefetching responsibility to page cache for compaction read for non directIO use case
+
+### Performance Improvements
+* In case of direct_io, if buffer passed by callee is already aligned, RandomAccessFileRead::Read will avoid realloacting a new buffer, reducing memcpy and use already passed aligned buffer.
+* Small efficiency improvement to HyperClockCache by reducing chance of compiler-generated heap allocations
+
+### Bug Fixes
+* Fix use_after_free bug in async_io MultiReads when underlying FS enabled kFSBuffer. kFSBuffer is when underlying FS pass their own buffer instead of using RocksDB scratch in FSReadRequest. Right now it's an experimental feature.
+
+## 8.4.0 (06/26/2023)
+### New Features
+* Add FSReadRequest::fs_scratch which is a data buffer allocated and provided by underlying FileSystem to RocksDB during reads, when FS wants to provide its own buffer with data instead of using RocksDB provided FSReadRequest::scratch. This can help in cpu optimization by avoiding copy from file system's buffer to RocksDB buffer. More details on how to use/enable it in file_system.h. Right now its supported only for MultiReads(async + sync) with non direct io.
+* Start logging non-zero user-defined timestamp sizes in WAL to signal user key format in subsequent records and use it during recovery. This change will break recovery from WAL files written by early versions that contain user-defined timestamps. The workaround is to ensure there are no WAL files to recover (i.e. by flushing before close) before upgrade.
+* Added new property "rocksdb.obsolete-sst-files-size-property" that reports the size of SST files that have become obsolete but have not yet been deleted or scheduled for deletion
+* Start to record the value of the flag `AdvancedColumnFamilyOptions.persist_user_defined_timestamps` in the Manifest and table properties for a SST file when it is created. And use the recorded flag when creating a table reader for the SST file. This flag is only explicitly record if it's false.
+* Add a new option OptimisticTransactionDBOptions::shared_lock_buckets that enables sharing mutexes for validating transactions between DB instances, for better balancing memory efficiency and validation contention across DB instances. Different column families and DBs also now use different hash seeds in this validation, so that the same set of key names will not contend across DBs or column families.
+* Add a new ticker `rocksdb.files.marked.trash.deleted` to track the number of trash files deleted by background thread from the trash queue.
+* Add an API NewTieredVolatileCache() in include/rocksdb/cache.h to allocate an instance of a block cache with a primary block cache tier and a compressed secondary cache tier. A cache of this type distributes memory reservations against the block cache, such as WriteBufferManager, table reader memory etc., proportionally across both the primary and compressed secondary cache.
+* Add `WaitForCompact()` to wait for all flush and compactions jobs to finish. Jobs to wait include the unscheduled (queued, but not scheduled yet).
+* Add `WriteBatch::Release()` that releases the batch's serialized data to the caller.
+
+### Public API Changes
+* Add C API `rocksdb_options_add_compact_on_deletion_collector_factory_del_ratio`.
+* change the FileSystem::use_async_io() API to SupportedOps API in order to extend it to various operations supported by underlying FileSystem. Right now it contains FSSupportedOps::kAsyncIO and FSSupportedOps::kFSBuffer. More details about FSSupportedOps in filesystem.h
+* Add new tickers: `rocksdb.error.handler.bg.error.count`, `rocksdb.error.handler.bg.io.error.count`, `rocksdb.error.handler.bg.retryable.io.error.count` to replace the misspelled ones: `rocksdb.error.handler.bg.errro.count`, `rocksdb.error.handler.bg.io.errro.count`, `rocksdb.error.handler.bg.retryable.io.errro.count` ('error' instead of 'errro'). Users should switch to use the new tickers before 9.0 release as the misspelled old tickers will be completely removed then.
+* Overload the API CreateColumnFamilyWithImport() to support creating ColumnFamily by importing multiple ColumnFamilies It requires that CFs should not overlap in user key range.
+
+### Behavior Changes
+* Change the default value for option `level_compaction_dynamic_level_bytes` to true. This affects users who use leveled compaction and do not set this option explicitly. These users may see additional background compactions following DB open. These compactions help to shape the LSM according to `level_compaction_dynamic_level_bytes` such that the size of each level Ln is approximately size of Ln-1 * `max_bytes_for_level_multiplier`. Turning on this option has other benefits too: see more detail in wiki: https://github.com/facebook/rocksdb/wiki/Leveled-Compaction#option-level_compaction_dynamic_level_bytes-and-levels-target-size and in option comment in advanced_options.h (#11525).
+* For Leveled Compaction users, `CompactRange()` will now always try to compact to the last non-empty level. (#11468)
+For Leveled Compaction users, `CompactRange()` with `bottommost_level_compaction = BottommostLevelCompaction::kIfHaveCompactionFilter` will behave similar to `kForceOptimized` in that it will skip files created during this manual compaction when compacting files in the bottommost level. (#11468)
+* RocksDB will try to drop range tombstones during non-bottommost compaction when it is safe to do so. (#11459)
+* When a DB is openend with `allow_ingest_behind=true` (currently only Universal compaction is supported), files in the last level, i.e. the ingested files,  will not be included in any compaction. (#11489)
+* Statistics `rocksdb.sst.read.micros` scope is expanded to all SST reads except for file ingestion and column family import (some compaction reads were previously excluded).
+
+### Bug Fixes
+* Reduced cases of illegally using Env::Default() during static destruction by never destroying the internal PosixEnv itself (except for builds checking for memory leaks). (#11538)
+* Fix extra prefetching during seek in async_io when BlockBasedTableOptions.num_file_reads_for_auto_readahead is 1 leading to extra reads than required.
+* Fix a bug where compactions that are qualified to be run as 2 subcompactions were only run as one subcompaction.
+* Fix a use-after-move bug in block.cc.
+
+## 8.3.0 (05/19/2023)
 ### New Features
 * Introduced a new option `block_protection_bytes_per_key`, which can be used to enable per key-value integrity protection for in-memory blocks in block cache (#11287).
 * Added `JemallocAllocatorOptions::num_arenas`. Setting `num_arenas > 1` may mitigate mutex contention in the allocator, particularly in scenarios where block allocations commonly bypass jemalloc tcache.
@@ -88,6 +88,25 @@ endif

 $(info $$DEBUG_LEVEL is $(DEBUG_LEVEL), $$LIB_MODE is $(LIB_MODE))

+# Detect what platform we're building on.
+# Export some common variables that might have been passed as Make variables
+# instead of environment variables.
+dummy := $(shell (export ROCKSDB_ROOT="$(CURDIR)"; \
+                  export CXXFLAGS="$(EXTRA_CXXFLAGS)"; \
+                  export LDFLAGS="$(EXTRA_LDFLAGS)"; \
+                  export COMPILE_WITH_ASAN="$(COMPILE_WITH_ASAN)"; \
+                  export COMPILE_WITH_TSAN="$(COMPILE_WITH_TSAN)"; \
+                  export COMPILE_WITH_UBSAN="$(COMPILE_WITH_UBSAN)"; \
+                  export PORTABLE="$(PORTABLE)"; \
+                  export ROCKSDB_NO_FBCODE="$(ROCKSDB_NO_FBCODE)"; \
+                  export USE_CLANG="$(USE_CLANG)"; \
+                  export LIB_MODE="$(LIB_MODE)"; \
+		  export ROCKSDB_CXX_STANDARD="$(ROCKSDB_CXX_STANDARD)"; \
+		  export USE_FOLLY="$(USE_FOLLY)"; \
+                  "$(CURDIR)/build_tools/build_detect_platform" "$(CURDIR)/make_config.mk"))
+# this file is generated by the previous line to set build flags and sources
+include make_config.mk
+
 # Figure out optimize level.
 ifneq ($(DEBUG_LEVEL), 2)
 	OPTIMIZE_LEVEL ?= -O2
@@ -223,25 +242,6 @@ am__v_AR_1 =
 AM_LINK = $(AM_V_CCLD)$(CXX) -L. $(patsubst lib%.a, -l%, $(patsubst lib%.$(PLATFORM_SHARED_EXT), -l%, $^)) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 AM_SHARE = $(AM_V_CCLD) $(CXX) $(PLATFORM_SHARED_LDFLAGS)$@ -L. $(patsubst lib%.$(PLATFORM_SHARED_EXT), -l%, $^) $(EXEC_LDFLAGS) $(LDFLAGS) -o $@

-# Detect what platform we're building on.
-# Export some common variables that might have been passed as Make variables
-# instead of environment variables.
-dummy := $(shell (export ROCKSDB_ROOT="$(CURDIR)"; \
-                  export CXXFLAGS="$(EXTRA_CXXFLAGS)"; \
-                  export LDFLAGS="$(EXTRA_LDFLAGS)"; \
-                  export COMPILE_WITH_ASAN="$(COMPILE_WITH_ASAN)"; \
-                  export COMPILE_WITH_TSAN="$(COMPILE_WITH_TSAN)"; \
-                  export COMPILE_WITH_UBSAN="$(COMPILE_WITH_UBSAN)"; \
-                  export PORTABLE="$(PORTABLE)"; \
-                  export ROCKSDB_NO_FBCODE="$(ROCKSDB_NO_FBCODE)"; \
-                  export USE_CLANG="$(USE_CLANG)"; \
-                  export LIB_MODE="$(LIB_MODE)"; \
-		  export ROCKSDB_CXX_STANDARD="$(ROCKSDB_CXX_STANDARD)"; \
-		  export USE_FOLLY="$(USE_FOLLY)"; \
-                  "$(CURDIR)/build_tools/build_detect_platform" "$(CURDIR)/make_config.mk"))
-# this file is generated by the previous line to set build flags and sources
-include make_config.mk
-
 ROCKSDB_PLUGIN_MKS = $(foreach plugin, $(ROCKSDB_PLUGINS), plugin/$(plugin)/*.mk)
 include $(ROCKSDB_PLUGIN_MKS)
 ROCKSDB_PLUGIN_PROTO =ROCKSDB_NAMESPACE::ObjectLibrary\&, const std::string\&
@@ -1417,6 +1417,9 @@ thread_local_test: $(OBJ_DIR)/util/thread_local_test.o $(TEST_LIBRARY) $(LIBRARY
 work_queue_test: $(OBJ_DIR)/util/work_queue_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)

+udt_util_test: $(OBJ_DIR)/util/udt_util_test.o $(TEST_LIBRARY) $(LIBRARY)
+	$(AM_LINK)
+
 corruption_test: $(OBJ_DIR)/db/corruption_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)

@@ -1882,6 +1885,9 @@ compressed_secondary_cache_test: $(OBJ_DIR)/cache/compressed_secondary_cache_tes
 lru_cache_test: $(OBJ_DIR)/cache/lru_cache_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)

+tiered_secondary_cache_test: $(OBJ_DIR)/cache/tiered_secondary_cache_test.o $(TEST_LIBRARY) $(LIBRARY)
+	$(AM_LINK)
+
 range_del_aggregator_test: $(OBJ_DIR)/db/range_del_aggregator_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)

@@ -1981,6 +1987,9 @@ cache_reservation_manager_test: $(OBJ_DIR)/cache/cache_reservation_manager_test.
 wide_column_serialization_test: $(OBJ_DIR)/db/wide/wide_column_serialization_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)

+wide_columns_helper_test: $(OBJ_DIR)/db/wide/wide_columns_helper_test.o $(TEST_LIBRARY) $(LIBRARY)
+	$(AM_LINK)
+
 #-------------------------------------------------
 # make install related stuff
 PREFIX ?= /usr/local
@@ -2085,8 +2094,8 @@ ROCKSDB_JAVADOCS_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-javadoc.jar
 ROCKSDB_SOURCES_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-sources.jar
 SHA256_CMD = sha256sum

-ZLIB_VER ?= 1.2.13
-ZLIB_SHA256 ?= b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30
+ZLIB_VER ?= 1.3
+ZLIB_SHA256 ?= ff0ba4c292013dbc27530b3a81e1f9a813cd39de01ca5e0f8bf355702efa593e
 ZLIB_DOWNLOAD_BASE ?= http://zlib.net
 BZIP2_VER ?= 1.0.8
 BZIP2_SHA256 ?= ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269
@@ -2094,11 +2103,11 @@ BZIP2_DOWNLOAD_BASE ?= http://sourceware.org/pub/bzip2
 SNAPPY_VER ?= 1.1.8
 SNAPPY_SHA256 ?= 16b677f07832a612b0836178db7f374e414f94657c138e6993cbfc5dcc58651f
 SNAPPY_DOWNLOAD_BASE ?= https://github.com/google/snappy/archive
-LZ4_VER ?= 1.9.3
-LZ4_SHA256 ?= 030644df4611007ff7dc962d981f390361e6c97a34e5cbc393ddfbe019ffe2c1
+LZ4_VER ?= 1.9.4
+LZ4_SHA256 ?= 0b0e3aa07c8c063ddf40b082bdf7e37a1562bda40a0ff5272957f3e987e0e54b
 LZ4_DOWNLOAD_BASE ?= https://github.com/lz4/lz4/archive
-ZSTD_VER ?= 1.4.9
-ZSTD_SHA256 ?= acf714d98e3db7b876e5b540cbf6dee298f60eb3c0723104f6d3f065cd60d6a8
+ZSTD_VER ?= 1.5.5
+ZSTD_SHA256 ?= 98e9c3d949d1b924e28e01eccb7deed865eefebf25c2f21c702e5cd5b63b85e1
 ZSTD_DOWNLOAD_BASE ?= https://github.com/facebook/zstd/archive
 CURL_SSL_OPTS ?= --tlsv1

@@ -21,6 +21,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
        "cache/secondary_cache.cc",
        "cache/secondary_cache_adapter.cc",
        "cache/sharded_cache.cc",
+        "cache/tiered_secondary_cache.cc",
        "db/arena_wrapped_db_iter.cc",
        "db/blob/blob_contents.cc",
        "db/blob/blob_fetcher.cc",
@@ -103,6 +104,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
        "db/wal_manager.cc",
        "db/wide/wide_column_serialization.cc",
        "db/wide/wide_columns.cc",
+        "db/wide/wide_columns_helper.cc",
        "db/write_batch.cc",
        "db/write_batch_base.cc",
        "db/write_controller.cc",
@@ -264,6 +266,8 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
        "util/string_util.cc",
        "util/thread_local.cc",
        "util/threadpool_imp.cc",
+        "util/udt_util.cc",
+        "util/write_batch_util.cc",
        "util/xxhash.cc",
        "utilities/agg_merge/agg_merge.cc",
        "utilities/backup/backup_engine.cc",
@@ -5472,6 +5476,12 @@ cpp_unittest_wrapper(name="tiered_compaction_test",
            extra_compiler_flags=[])


+cpp_unittest_wrapper(name="tiered_secondary_cache_test",
+            srcs=["cache/tiered_secondary_cache_test.cc"],
+            deps=[":rocksdb_test_lib"],
+            extra_compiler_flags=[])
+
+
 cpp_unittest_wrapper(name="timer_queue_test",
            srcs=["util/timer_queue_test.cc"],
            deps=[":rocksdb_test_lib"],
@@ -5508,6 +5518,12 @@ cpp_unittest_wrapper(name="ttl_test",
            extra_compiler_flags=[])


+cpp_unittest_wrapper(name="udt_util_test",
+            srcs=["util/udt_util_test.cc"],
+            deps=[":rocksdb_test_lib"],
+            extra_compiler_flags=[])
+
+
 cpp_unittest_wrapper(name="util_merge_operators_test",
            srcs=["utilities/util_merge_operators_test.cc"],
            deps=[":rocksdb_test_lib"],
@@ -5544,6 +5560,12 @@ cpp_unittest_wrapper(name="wide_column_serialization_test",
            extra_compiler_flags=[])


+cpp_unittest_wrapper(name="wide_columns_helper_test",
+            srcs=["db/wide/wide_columns_helper_test.cc"],
+            deps=[":rocksdb_test_lib"],
+            extra_compiler_flags=[])
+
+
 cpp_unittest_wrapper(name="work_queue_test",
            srcs=["util/work_queue_test.cc"],
            deps=[":rocksdb_test_lib"],
@@ -161,5 +161,9 @@ LzLabs is using RocksDB as a storage engine in their multi-database distributed
 ## Solana Labs
 [Solana](https://github.com/solana-labs/solana) is a fast, secure, scalable, and decentralized blockchain.  It uses RocksDB as the underlying storage for its ledger store.

+## Apache Kvrocks
+
+[Apache Kvrocks](https://github.com/apache/kvrocks) is an open-source distributed key-value NoSQL database built on top of RocksDB. It serves as a cost-saving and capacity-increasing alternative drop-in replacement for Redis.
+
 ## Others
 More databases using RocksDB can be found at [dbdb.io](https://dbdb.io/browse?embeds=rocksdb).
@@ -23,7 +23,7 @@ def pretty_list(lst, indent=8):
    return res


-class TARGETSBuilder(object):
+class TARGETSBuilder:
    def __init__(self, path, extra_argv):
        self.path = path
        header = targets_cfg.rocksdb_target_header_template.format(
@@ -14,7 +14,7 @@ import sys
 import time


-class ColorString(object):
+class ColorString:
    """Generate colorful strings on terminal"""

    HEADER = "\033[95m"
@@ -419,7 +419,7 @@ EOF

    if ! test $ROCKSDB_DISABLE_JEMALLOC; then
        # Test whether jemalloc is available
-        if echo 'int main() {}' | $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o -ljemalloc \
+        if echo 'int main() {}' | $CXX $PLATFORM_CXXFLAGS $LDFLAGS -x c++ - -o test.o -ljemalloc \
          2>/dev/null; then
            # This will enable some preprocessor identifiers in the Makefile
            JEMALLOC=1
@@ -428,12 +428,19 @@ EOF
            WITH_JEMALLOC_FLAG=1
            # check for JEMALLOC installed with HomeBrew
            if [ "$PLATFORM" == "OS_MACOSX" ]; then
+                if [ "$TARGET_ARCHITECTURE" = "arm64" ]; then
+                    # on M1 Macs, homebrew installs here instead of /usr/local
+                    JEMALLOC_PREFIX="/opt/homebrew"
+                else
+                    JEMALLOC_PREFIX="/usr/local"
+                fi
                if hash brew 2>/dev/null && brew ls --versions jemalloc > /dev/null; then
                    JEMALLOC_VER=$(brew ls --versions jemalloc | tail -n 1 | cut -f 2 -d ' ')
-                    JEMALLOC_INCLUDE="-I/usr/local/Cellar/jemalloc/${JEMALLOC_VER}/include"
-                    JEMALLOC_LIB="/usr/local/Cellar/jemalloc/${JEMALLOC_VER}/lib/libjemalloc_pic.a"
-                    PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS $JEMALLOC_LIB"
-                    JAVA_STATIC_LDFLAGS="$JAVA_STATIC_LDFLAGS $JEMALLOC_LIB"
+                    JEMALLOC_INCLUDE="-I${JEMALLOC_PREFIX}/Cellar/jemalloc/${JEMALLOC_VER}/include"
+                    JEMALLOC_LIB="${JEMALLOC_PREFIX}/Cellar/jemalloc/${JEMALLOC_VER}/lib/libjemalloc_pic.a"
+                    PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -L${JEMALLOC_PREFIX}/lib $JEMALLOC_LIB"
+                    JAVA_LDFLAGS="$JAVA_LDFLAGS -L${JEMALLOC_PREFIX}/lib $JEMALLOC_LIB"
+                    JAVA_STATIC_LDFLAGS="$JAVA_STATIC_LDFLAGS -L${JEMALLOC_PREFIX}/lib $JEMALLOC_LIB"
                fi
            fi
        fi
@@ -667,13 +674,13 @@ else
  fi

  if [[ "${PLATFORM}" == "OS_MACOSX" ]]; then
-    # For portability compile for macOS 10.13 (2017) or newer
-    COMMON_FLAGS="$COMMON_FLAGS -mmacosx-version-min=10.13"
-    PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -mmacosx-version-min=10.13"
+    # For portability compile for macOS 10.14 (2018) or newer
+    COMMON_FLAGS="$COMMON_FLAGS -mmacosx-version-min=10.14"
+    PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -mmacosx-version-min=10.14"
    # -mmacosx-version-min must come first here.
-    PLATFORM_SHARED_LDFLAGS="-mmacosx-version-min=10.13 $PLATFORM_SHARED_LDFLAGS"
-    PLATFORM_CMAKE_FLAGS="-DCMAKE_OSX_DEPLOYMENT_TARGET=10.13"
-    JAVA_STATIC_DEPS_COMMON_FLAGS="-mmacosx-version-min=10.13"
+    PLATFORM_SHARED_LDFLAGS="-mmacosx-version-min=10.14 $PLATFORM_SHARED_LDFLAGS"
+    PLATFORM_CMAKE_FLAGS="-DCMAKE_OSX_DEPLOYMENT_TARGET=10.14"
+    JAVA_STATIC_DEPS_COMMON_FLAGS="-mmacosx-version-min=10.14"
    JAVA_STATIC_DEPS_LDFLAGS="$JAVA_STATIC_DEPS_COMMON_FLAGS"
    JAVA_STATIC_DEPS_CCFLAGS="$JAVA_STATIC_DEPS_COMMON_FLAGS"
    JAVA_STATIC_DEPS_CXXFLAGS="$JAVA_STATIC_DEPS_COMMON_FLAGS"
@@ -15,7 +15,7 @@ import re
 import sys


-class ErrorParserBase(object):
+class ErrorParserBase:
    def parse_error(self, line):
        """Parses a line of test output. If it contains an error, returns a
        formatted message describing the error; otherwise, returns None.
@@ -66,6 +66,41 @@ static std::unordered_map<std::string, OptionTypeInfo>
          OptionTypeFlags::kMutable}},
 };

+namespace {
+static void NoopDelete(Cache::ObjectPtr /*obj*/,
+                       MemoryAllocator* /*allocator*/) {
+  assert(false);
+}
+
+static size_t SliceSize(Cache::ObjectPtr obj) {
+  return static_cast<Slice*>(obj)->size();
+}
+
+static Status SliceSaveTo(Cache::ObjectPtr from_obj, size_t from_offset,
+                          size_t length, char* out) {
+  const Slice& slice = *static_cast<Slice*>(from_obj);
+  std::memcpy(out, slice.data() + from_offset, length);
+  return Status::OK();
+}
+
+static Status NoopCreate(const Slice& /*data*/, CompressionType /*type*/,
+                         CacheTier /*source*/, Cache::CreateContext* /*ctx*/,
+                         MemoryAllocator* /*allocator*/,
+                         Cache::ObjectPtr* /*out_obj*/,
+                         size_t* /*out_charge*/) {
+  assert(false);
+  return Status::NotSupported();
+}
+
+static Cache::CacheItemHelper kBasicCacheItemHelper(CacheEntryRole::kMisc,
+                                                    &NoopDelete);
+}  // namespace
+
+const Cache::CacheItemHelper kSliceCacheItemHelper{
+    CacheEntryRole::kMisc, &NoopDelete, &SliceSize,
+    &SliceSaveTo,          &NoopCreate, &kBasicCacheItemHelper,
+};
+
 Status SecondaryCache::CreateFromString(
    const ConfigOptions& config_options, const std::string& value,
    std::shared_ptr<SecondaryCache>* result) {
@@ -3,7 +3,6 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).

-#include "cache_key.h"
 #ifdef GFLAGS
 #include <cinttypes>
 #include <cstddef>
@@ -13,9 +12,12 @@
 #include <set>
 #include <sstream>

+#include "cache/cache_key.h"
+#include "cache/sharded_cache.h"
 #include "db/db_impl/db_impl.h"
 #include "monitoring/histogram.h"
 #include "port/port.h"
+#include "port/stack_trace.h"
 #include "rocksdb/advanced_cache.h"
 #include "rocksdb/convenience.h"
 #include "rocksdb/db.h"
@@ -31,6 +33,7 @@
 #include "util/hash.h"
 #include "util/mutexlock.h"
 #include "util/random.h"
+#include "util/stderr_logger.h"
 #include "util/stop_watch.h"
 #include "util/string_util.h"

@@ -43,14 +46,21 @@ static constexpr uint64_t GiB = MiB << 10;
 DEFINE_uint32(threads, 16, "Number of concurrent threads to run.");
 DEFINE_uint64(cache_size, 1 * GiB,
              "Number of bytes to use as a cache of uncompressed data.");
-DEFINE_uint32(num_shard_bits, 6, "shard_bits.");
+DEFINE_int32(num_shard_bits, -1,
+             "ShardedCacheOptions::shard_bits. Default = auto");

 DEFINE_double(resident_ratio, 0.25,
              "Ratio of keys fitting in cache to keyspace.");
 DEFINE_uint64(ops_per_thread, 2000000U, "Number of operations per thread.");
 DEFINE_uint32(value_bytes, 8 * KiB, "Size of each value added.");
+DEFINE_uint32(value_bytes_estimate, 0,
+              "If > 0, overrides estimated_entry_charge or "
+              "min_avg_entry_charge depending on cache_type.");

-DEFINE_uint32(skew, 5, "Degree of skew in key selection");
+DEFINE_int32(
+    degenerate_hash_bits, 0,
+    "With HCC, fix this many hash bits to increase table hash collisions");
+DEFINE_uint32(skew, 5, "Degree of skew in key selection. 0 = no skew");
 DEFINE_bool(populate_cache, true, "Populate cache before operations");

 DEFINE_uint32(lookup_insert_percent, 87,
@@ -71,18 +81,41 @@ DEFINE_uint32(

 DEFINE_uint32(gather_stats_entries_per_lock, 256,
              "For Cache::ApplyToAllEntries");
-DEFINE_bool(skewed, false, "If true, skew the key access distribution");
+
+DEFINE_uint32(usleep, 0, "Sleep up to this many microseconds after each op.");

 DEFINE_bool(lean, false,
            "If true, no additional computation is performed besides cache "
            "operations.");

+DEFINE_bool(early_exit, false,
+            "Exit before deallocating most memory. Good for malloc stats, e.g."
+            "MALLOC_CONF=\"stats_print:true\"");
+
+DEFINE_bool(histograms, true,
+            "Whether to track and print histogram statistics.");
+
+DEFINE_bool(report_problems, true, "Whether to ReportProblems() at the end.");
+
+DEFINE_uint32(seed, 0, "Hashing/random seed to use. 0 = choose at random");
+
 DEFINE_string(secondary_cache_uri, "",
              "Full URI for creating a custom secondary cache object");
 static class std::shared_ptr<ROCKSDB_NAMESPACE::SecondaryCache> secondary_cache;

 DEFINE_string(cache_type, "lru_cache", "Type of block cache.");

+DEFINE_bool(use_jemalloc_no_dump_allocator, false,
+            "Whether to use JemallocNoDumpAllocator");
+
+DEFINE_uint32(jemalloc_no_dump_allocator_num_arenas,
+              ROCKSDB_NAMESPACE::JemallocAllocatorOptions().num_arenas,
+              "JemallocNodumpAllocator::num_arenas");
+
+DEFINE_bool(jemalloc_no_dump_allocator_limit_tcache_size,
+            ROCKSDB_NAMESPACE::JemallocAllocatorOptions().limit_tcache_size,
+            "JemallocNodumpAllocator::limit_tcache_size");
+
 // ## BEGIN stress_cache_key sub-tool options ##
 // See class StressCacheKey below.
 DEFINE_bool(stress_cache_key, false,
@@ -145,9 +178,6 @@ class SharedState {
 public:
  explicit SharedState(CacheBench* cache_bench)
      : cv_(&mu_),
-        num_initialized_(0),
-        start_(false),
-        num_done_(0),
        cache_bench_(cache_bench) {}

  ~SharedState() {}
@@ -170,15 +200,27 @@ class SharedState {

  bool Started() const { return start_; }

+  void AddLookupStats(uint64_t hits, uint64_t misses) {
+    MutexLock l(&mu_);
+    lookup_count_ += hits + misses;
+    lookup_hits_ += hits;
+  }
+
+  double GetLookupHitRatio() const {
+    return 1.0 * lookup_hits_ / lookup_count_;
+  }
+
 private:
  port::Mutex mu_;
  port::CondVar cv_;

-  uint64_t num_initialized_;
-  bool start_;
-  uint64_t num_done_;
-
  CacheBench* cache_bench_;
+
+  uint64_t num_initialized_ = 0;
+  bool start_ = false;
+  uint64_t num_done_ = 0;
+  uint64_t lookup_count_ = 0;
+  uint64_t lookup_hits_ = 0;
 };

 // Per-thread state for concurrent executions of the same benchmark.
@@ -190,26 +232,32 @@ struct ThreadState {
  uint64_t duration_us = 0;

  ThreadState(uint32_t index, SharedState* _shared)
-      : tid(index), rnd(1000 + index), shared(_shared) {}
+      : tid(index), rnd(FLAGS_seed + 1 + index), shared(_shared) {}
 };

 struct KeyGen {
  char key_data[27];

-  Slice GetRand(Random64& rnd, uint64_t max_key, int max_log) {
-    uint64_t key = 0;
-    if (!FLAGS_skewed) {
-      uint64_t raw = rnd.Next();
-      // Skew according to setting
-      for (uint32_t i = 0; i < FLAGS_skew; ++i) {
-        raw = std::min(raw, rnd.Next());
-      }
-      key = FastRange64(raw, max_key);
-    } else {
-      key = rnd.Skewed(max_log);
-      if (key > max_key) {
-        key -= max_key;
-      }
+  Slice GetRand(Random64& rnd, uint64_t max_key, uint32_t skew) {
+    uint64_t raw = rnd.Next();
+    // Skew according to setting
+    for (uint32_t i = 0; i < skew; ++i) {
+      raw = std::min(raw, rnd.Next());
+    }
+    uint64_t key = FastRange64(raw, max_key);
+    if (FLAGS_degenerate_hash_bits) {
+      uint64_t key_hash =
+          Hash64(reinterpret_cast<const char*>(&key), sizeof(key));
+      // HCC uses the high 64 bits and a lower bit mask for starting probe
+      // location, so we fix hash bits starting at the bottom of that word.
+      auto hi_hash = uint64_t{0x9e3779b97f4a7c13U} ^
+                     (key_hash << 1 << (FLAGS_degenerate_hash_bits - 1));
+      uint64_t un_hi, un_lo;
+      BijectiveUnhash2x64(hi_hash, key_hash, &un_hi, &un_lo);
+      un_lo ^= BitwiseAnd(FLAGS_seed, INT32_MAX);
+      EncodeFixed64(key_data, un_lo);
+      EncodeFixed64(key_data + 8, un_hi);
+      return Slice(key_data, kCacheKeySize);
    }
    // Variable size and alignment
    size_t off = key % 8;
@@ -224,8 +272,8 @@ struct KeyGen {
  }
 };

-Cache::ObjectPtr createValue(Random64& rnd) {
-  char* rv = new char[FLAGS_value_bytes];
+Cache::ObjectPtr createValue(Random64& rnd, MemoryAllocator* alloc) {
+  char* rv = AllocateBlock(FLAGS_value_bytes, alloc).release();
  // Fill with some filler data, and take some CPU time
  for (uint32_t i = 0; i < FLAGS_value_bytes; i += 8) {
    EncodeFixed64(rv + i, rnd.Next());
@@ -242,7 +290,8 @@ Status SaveToFn(Cache::ObjectPtr from_obj, size_t /*from_offset*/,
  return Status::OK();
 }

-Status CreateFn(const Slice& data, Cache::CreateContext* /*context*/,
+Status CreateFn(const Slice& data, CompressionType /*type*/,
+                CacheTier /*source*/, Cache::CreateContext* /*context*/,
                MemoryAllocator* /*allocator*/, Cache::ObjectPtr* out_obj,
                size_t* out_charge) {
  *out_obj = new char[data.size()];
@@ -251,8 +300,8 @@ Status CreateFn(const Slice& data, Cache::CreateContext* /*context*/,
  return Status::OK();
 };

-void DeleteFn(Cache::ObjectPtr value, MemoryAllocator* /*alloc*/) {
-  delete[] static_cast<char*>(value);
+void DeleteFn(Cache::ObjectPtr value, MemoryAllocator* alloc) {
+  CustomDeleter{alloc}(static_cast<char*>(value));
 }

 Cache::CacheItemHelper helper1_wos(CacheEntryRole::kDataBlock, DeleteFn);
@@ -281,31 +330,49 @@ class CacheBench {
        lookup_threshold_(insert_threshold_ +
                          kHundredthUint64 * FLAGS_lookup_percent),
        erase_threshold_(lookup_threshold_ +
-                         kHundredthUint64 * FLAGS_erase_percent),
-        skewed_(FLAGS_skewed) {
+                         kHundredthUint64 * FLAGS_erase_percent) {
    if (erase_threshold_ != 100U * kHundredthUint64) {
      fprintf(stderr, "Percentages must add to 100.\n");
      exit(1);
    }

-    max_log_ = 0;
-    if (skewed_) {
-      uint64_t max_key = max_key_;
-      while (max_key >>= 1) max_log_++;
-      if (max_key > (static_cast<uint64_t>(1) << max_log_)) max_log_++;
+    std::shared_ptr<MemoryAllocator> allocator;
+    if (FLAGS_use_jemalloc_no_dump_allocator) {
+      JemallocAllocatorOptions opts;
+      opts.num_arenas = FLAGS_jemalloc_no_dump_allocator_num_arenas;
+      opts.limit_tcache_size =
+          FLAGS_jemalloc_no_dump_allocator_limit_tcache_size;
+      Status s = NewJemallocNodumpAllocator(opts, &allocator);
+      assert(s.ok());
    }
-
    if (FLAGS_cache_type == "clock_cache") {
      fprintf(stderr, "Old clock cache implementation has been removed.\n");
      exit(1);
-    } else if (FLAGS_cache_type == "hyper_clock_cache") {
-      cache_ = HyperClockCacheOptions(FLAGS_cache_size, FLAGS_value_bytes,
-                                      FLAGS_num_shard_bits)
-                   .MakeSharedCache();
+    } else if (EndsWith(FLAGS_cache_type, "hyper_clock_cache")) {
+      HyperClockCacheOptions opts(
+          FLAGS_cache_size, /*estimated_entry_charge=*/0, FLAGS_num_shard_bits);
+      opts.hash_seed = BitwiseAnd(FLAGS_seed, INT32_MAX);
+      opts.memory_allocator = allocator;
+      if (FLAGS_cache_type == "fixed_hyper_clock_cache" ||
+          FLAGS_cache_type == "hyper_clock_cache") {
+        opts.estimated_entry_charge = FLAGS_value_bytes_estimate > 0
+                                          ? FLAGS_value_bytes_estimate
+                                          : FLAGS_value_bytes;
+      } else if (FLAGS_cache_type == "auto_hyper_clock_cache") {
+        if (FLAGS_value_bytes_estimate > 0) {
+          opts.min_avg_entry_charge = FLAGS_value_bytes_estimate;
+        }
+      } else {
+        fprintf(stderr, "Cache type not supported.\n");
+        exit(1);
+      }
+      cache_ = opts.MakeSharedCache();
    } else if (FLAGS_cache_type == "lru_cache") {
      LRUCacheOptions opts(FLAGS_cache_size, FLAGS_num_shard_bits,
                           false /* strict_capacity_limit */,
                           0.5 /* high_pri_pool_ratio */);
+      opts.hash_seed = BitwiseAnd(FLAGS_seed, INT32_MAX);
+      opts.memory_allocator = allocator;
      if (!FLAGS_secondary_cache_uri.empty()) {
        Status s = SecondaryCache::CreateFromString(
            ConfigOptions(), FLAGS_secondary_cache_uri, &secondary_cache);
@@ -321,7 +388,7 @@ class CacheBench {

      cache_ = NewLRUCache(opts);
    } else {
-      fprintf(stderr, "Cache type not supported.");
+      fprintf(stderr, "Cache type not supported.\n");
      exit(1);
    }
  }
@@ -329,13 +396,51 @@ class CacheBench {
  ~CacheBench() {}

  void PopulateCache() {
-    Random64 rnd(1);
+    Random64 rnd(FLAGS_seed);
    KeyGen keygen;
-    for (uint64_t i = 0; i < 2 * FLAGS_cache_size; i += FLAGS_value_bytes) {
-      Status s = cache_->Insert(keygen.GetRand(rnd, max_key_, max_log_),
-                                createValue(rnd), &helper1, FLAGS_value_bytes);
+    size_t max_occ = 0;
+    size_t inserts_since_max_occ_increase = 0;
+    size_t keys_since_last_not_found = 0;
+
+    // Avoid redundant insertions by checking Lookup before Insert.
+    // Loop until insertions consistently fail to increase max occupancy or
+    // it becomes difficult to find keys not already inserted.
+    while (inserts_since_max_occ_increase < 100 &&
+           keys_since_last_not_found < 100) {
+      Slice key = keygen.GetRand(rnd, max_key_, FLAGS_skew);
+
+      Cache::Handle* handle = cache_->Lookup(key);
+      if (handle != nullptr) {
+        cache_->Release(handle);
+        ++keys_since_last_not_found;
+        continue;
+      }
+      keys_since_last_not_found = 0;
+
+      Status s =
+          cache_->Insert(key, createValue(rnd, cache_->memory_allocator()),
+                         &helper1, FLAGS_value_bytes);
      assert(s.ok());
+
+      handle = cache_->Lookup(key);
+      if (!handle) {
+        fprintf(stderr, "Failed to lookup key just inserted.\n");
+        assert(false);
+        exit(42);
+      } else {
+        cache_->Release(handle);
+      }
+
+      size_t occ = cache_->GetOccupancyCount();
+      if (occ > max_occ) {
+        max_occ = occ;
+        inserts_since_max_occ_increase = 0;
+      } else {
+        ++inserts_since_max_occ_increase;
+      }
    }
+    printf("Population complete (%zu entries, %g average charge)\n", max_occ,
+           1.0 * FLAGS_cache_size / max_occ);
  }

  bool Run() {
@@ -394,19 +499,33 @@ class CacheBench {
                                        FLAGS_ops_per_thread / elapsed_secs);
    printf("Thread ops/sec = %u\n", ops_per_sec);

-    printf("\nOperation latency (ns):\n");
-    HistogramImpl combined;
-    for (uint32_t i = 0; i < FLAGS_threads; i++) {
-      combined.Merge(threads[i]->latency_ns_hist);
-    }
-    printf("%s", combined.ToString().c_str());
+    printf("Lookup hit ratio: %g\n", shared.GetLookupHitRatio());

-    if (FLAGS_gather_stats) {
-      printf("\nGather stats latency (us):\n");
-      printf("%s", stats_hist.ToString().c_str());
+    size_t occ = cache_->GetOccupancyCount();
+    size_t slot = cache_->GetTableAddressCount();
+    printf("Final load factor: %g (%zu / %zu)\n", 1.0 * occ / slot, occ, slot);
+
+    if (FLAGS_histograms) {
+      printf("\nOperation latency (ns):\n");
+      HistogramImpl combined;
+      for (uint32_t i = 0; i < FLAGS_threads; i++) {
+        combined.Merge(threads[i]->latency_ns_hist);
+      }
+      printf("%s", combined.ToString().c_str());
+
+      if (FLAGS_gather_stats) {
+        printf("\nGather stats latency (us):\n");
+        printf("%s", stats_hist.ToString().c_str());
+      }
    }

-    printf("\n%s", stats_report.c_str());
+    if (FLAGS_report_problems) {
+      printf("\n");
+      std::shared_ptr<Logger> logger =
+          std::make_shared<StderrLogger>(InfoLogLevel::DEBUG_LEVEL);
+      cache_->ReportProblems(logger);
+    }
+    printf("%s", stats_report.c_str());

    return true;
  }
@@ -419,8 +538,6 @@ class CacheBench {
  const uint64_t insert_threshold_;
  const uint64_t lookup_threshold_;
  const uint64_t erase_threshold_;
-  const bool skewed_;
-  int max_log_;

  // A benchmark version of gathering stats on an active block cache by
  // iterating over it. The primary purpose is to measure the impact of
@@ -453,7 +570,7 @@ class CacheBench {
        for (;;) {
          if (shared->AllDone()) {
            std::ostringstream ostr;
-            ostr << "Most recent cache entry stats:\n"
+            ostr << "\nMost recent cache entry stats:\n"
                 << "Number of entries: " << total_entry_count << "\n"
                 << "Table occupancy: " << table_occupancy << " / "
                 << table_size << " = "
@@ -490,13 +607,17 @@ class CacheBench {
        // Something slightly more expensive as in stats by category
        helpers.insert(helper);
      };
-      timer.Start();
+      if (FLAGS_histograms) {
+        timer.Start();
+      }
      Cache::ApplyToAllEntriesOptions opts;
      opts.average_entries_per_lock = FLAGS_gather_stats_entries_per_lock;
      shared->GetCacheBench()->cache_->ApplyToAllEntries(fn, opts);
      table_occupancy = shared->GetCacheBench()->cache_->GetOccupancyCount();
      table_size = shared->GetCacheBench()->cache_->GetTableAddressCount();
-      stats_hist->Add(timer.ElapsedNanos() / 1000);
+      if (FLAGS_histograms) {
+        stats_hist->Add(timer.ElapsedNanos() / 1000);
+      }
    }
  }

@@ -527,18 +648,23 @@ class CacheBench {
  void OperateCache(ThreadState* thread) {
    // To use looked-up values
    uint64_t result = 0;
+    uint64_t lookup_misses = 0;
+    uint64_t lookup_hits = 0;
    // To hold handles for a non-trivial amount of time
    Cache::Handle* handle = nullptr;
    KeyGen gen;
    const auto clock = SystemClock::Default().get();
    uint64_t start_time = clock->NowMicros();
    StopWatchNano timer(clock);
+    auto system_clock = SystemClock::Default();

    for (uint64_t i = 0; i < FLAGS_ops_per_thread; i++) {
-      Slice key = gen.GetRand(thread->rnd, max_key_, max_log_);
+      Slice key = gen.GetRand(thread->rnd, max_key_, FLAGS_skew);
      uint64_t random_op = thread->rnd.Next();

-      timer.Start();
+      if (FLAGS_histograms) {
+        timer.Start();
+      }

      if (random_op < lookup_insert_threshold_) {
        if (handle) {
@@ -549,15 +675,18 @@ class CacheBench {
        handle = cache_->Lookup(key, &helper2, /*context*/ nullptr,
                                Cache::Priority::LOW);
        if (handle) {
+          ++lookup_hits;
          if (!FLAGS_lean) {
            // do something with the data
            result += NPHash64(static_cast<char*>(cache_->Value(handle)),
                               FLAGS_value_bytes);
          }
        } else {
+          ++lookup_misses;
          // do insert
-          Status s = cache_->Insert(key, createValue(thread->rnd), &helper2,
-                                    FLAGS_value_bytes, &handle);
+          Status s = cache_->Insert(
+              key, createValue(thread->rnd, cache_->memory_allocator()),
+              &helper2, FLAGS_value_bytes, &handle);
          assert(s.ok());
        }
      } else if (random_op < insert_threshold_) {
@@ -566,8 +695,9 @@ class CacheBench {
          handle = nullptr;
        }
        // do insert
-        Status s = cache_->Insert(key, createValue(thread->rnd), &helper3,
-                                  FLAGS_value_bytes, &handle);
+        Status s = cache_->Insert(
+            key, createValue(thread->rnd, cache_->memory_allocator()), &helper3,
+            FLAGS_value_bytes, &handle);
        assert(s.ok());
      } else if (random_op < lookup_threshold_) {
        if (handle) {
@@ -578,11 +708,14 @@ class CacheBench {
        handle = cache_->Lookup(key, &helper2, /*context*/ nullptr,
                                Cache::Priority::LOW);
        if (handle) {
+          ++lookup_hits;
          if (!FLAGS_lean) {
            // do something with the data
            result += NPHash64(static_cast<char*>(cache_->Value(handle)),
                               FLAGS_value_bytes);
          }
+        } else {
+          ++lookup_misses;
        }
      } else if (random_op < erase_threshold_) {
        // do erase
@@ -591,7 +724,21 @@ class CacheBench {
        // Should be extremely unlikely (noop)
        assert(random_op >= kHundredthUint64 * 100U);
      }
-      thread->latency_ns_hist.Add(timer.ElapsedNanos());
+      if (FLAGS_histograms) {
+        thread->latency_ns_hist.Add(timer.ElapsedNanos());
+      }
+      thread->shared->AddLookupStats(lookup_hits, lookup_misses);
+      if (FLAGS_usleep > 0) {
+        unsigned us =
+            static_cast<unsigned>(thread->rnd.Uniform(FLAGS_usleep + 1));
+        if (us > 0) {
+          system_clock->SleepForMicroseconds(us);
+        }
+      }
+    }
+    if (FLAGS_early_exit) {
+      MutexLock l(thread->shared->GetMutex());
+      exit(0);
    }
    if (handle) {
      cache_->Release(handle);
@@ -613,13 +760,17 @@ class CacheBench {
 #ifndef NDEBUG
    printf("WARNING: Assertions are enabled; benchmarks unnecessarily slow\n");
 #endif
+    printf("----------------------------\n");
    printf("RocksDB version     : %d.%d\n", kMajorVersion, kMinorVersion);
+    printf("Cache impl name     : %s\n", cache_->Name());
    printf("DMutex impl name    : %s\n", DMutex::kName());
    printf("Number of threads   : %u\n", FLAGS_threads);
    printf("Ops per thread      : %" PRIu64 "\n", FLAGS_ops_per_thread);
    printf("Cache size          : %s\n",
           BytesToHumanString(FLAGS_cache_size).c_str());
-    printf("Num shard bits      : %u\n", FLAGS_num_shard_bits);
+    printf("Num shard bits      : %d\n",
+           static_cast_with_check<ShardedCacheBase>(cache_.get())
+               ->GetNumShardBits());
    printf("Max key             : %" PRIu64 "\n", max_key_);
    printf("Resident ratio      : %g\n", FLAGS_resident_ratio);
    printf("Skew degree         : %u\n", FLAGS_skew);
@@ -939,6 +1090,7 @@ class StressCacheKey {
 };

 int cache_bench_tool(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ParseCommandLineFlags(&argc, &argv, true);

  if (FLAGS_stress_cache_key) {
@@ -952,11 +1104,14 @@ int cache_bench_tool(int argc, char** argv) {
    exit(1);
  }

+  if (FLAGS_seed == 0) {
+    FLAGS_seed = static_cast<uint32_t>(port::GetProcessID());
+    printf("Using seed = %" PRIu32 "\n", FLAGS_seed);
+  }
+
  ROCKSDB_NAMESPACE::CacheBench bench;
  if (FLAGS_populate_cache) {
    bench.PopulateCache();
-    printf("Population complete\n");
-    printf("----------------------------\n");
  }
  if (bench.Run()) {
    return 0;
@@ -25,7 +25,8 @@ Status WarmInCache(Cache* cache, const Slice& key, const Slice& saved,
  assert(helper->create_cb);
  Cache::ObjectPtr value;
  size_t charge;
-  Status st = helper->create_cb(saved, create_context,
+  Status st = helper->create_cb(saved, CompressionType::kNoCompression,
+                                CacheTier::kVolatileTier, create_context,
                                cache->memory_allocator(), &value, &charge);
  if (st.ok()) {
    st =
@@ -273,9 +273,10 @@ class ConcurrentCacheReservationManager
    std::size_t total_mem_used = cache_res_mgr_->GetTotalMemoryUsed();
    Status s;
    if (!increase) {
-      assert(total_mem_used >= memory_used_delta);
-      s = cache_res_mgr_->UpdateCacheReservation(total_mem_used -
-                                                 memory_used_delta);
+      s = cache_res_mgr_->UpdateCacheReservation(
+          (total_mem_used > memory_used_delta)
+              ? (total_mem_used - memory_used_delta)
+              : 0);
    } else {
      s = cache_res_mgr_->UpdateCacheReservation(total_mem_used +
                                                 memory_used_delta);
@@ -70,18 +70,11 @@ const Cache::CacheItemHelper kDumbHelper{
    CacheEntryRole::kMisc,
    [](Cache::ObjectPtr /*value*/, MemoryAllocator* /*alloc*/) {}};

-const Cache::CacheItemHelper kEraseOnDeleteHelper1{
+const Cache::CacheItemHelper kInvokeOnDeleteHelper{
    CacheEntryRole::kMisc,
    [](Cache::ObjectPtr value, MemoryAllocator* /*alloc*/) {
-      Cache* cache = static_cast<Cache*>(value);
-      cache->Erase("foo");
-    }};
-
-const Cache::CacheItemHelper kEraseOnDeleteHelper2{
-    CacheEntryRole::kMisc,
-    [](Cache::ObjectPtr value, MemoryAllocator* /*alloc*/) {
-      Cache* cache = static_cast<Cache*>(value);
-      cache->Erase(EncodeKey16Bytes(1234));
+      auto& fn = *static_cast<std::function<void()>*>(value);
+      fn();
    }};
 }  // anonymous namespace

@@ -120,8 +113,7 @@ class CacheTest : public testing::Test,
  // Currently, HyperClockCache requires keys to be 16B long, whereas
  // LRUCache doesn't, so the encoding depends on the cache type.
  std::string EncodeKey(int k) {
-    auto type = GetParam();
-    if (type == kHyperClock) {
+    if (IsHyperClock()) {
      return EncodeKey16Bytes(k);
    } else {
      return EncodeKey32Bits(k);
@@ -129,8 +121,7 @@ class CacheTest : public testing::Test,
  }

  int DecodeKey(const Slice& k) {
-    auto type = GetParam();
-    if (type == kHyperClock) {
+    if (IsHyperClock()) {
      return DecodeKey16Bytes(k);
    } else {
      return DecodeKey32Bits(k);
@@ -182,15 +173,13 @@ std::string CacheTest::type_;
 class LRUCacheTest : public CacheTest {};

 TEST_P(CacheTest, UsageTest) {
-  auto type = GetParam();
-
  // cache is std::shared_ptr and will be automatically cleaned up.
  const size_t kCapacity = 100000;
-  auto cache = NewCache(kCapacity, 8, false, kDontChargeCacheMetadata);
+  auto cache = NewCache(kCapacity, 6, false, kDontChargeCacheMetadata);
  auto precise_cache = NewCache(kCapacity, 0, false, kFullChargeCacheMetadata);
  ASSERT_EQ(0, cache->GetUsage());
  size_t baseline_meta_usage = precise_cache->GetUsage();
-  if (type != kHyperClock) {
+  if (!IsHyperClock()) {
    ASSERT_EQ(0, baseline_meta_usage);
  }

@@ -198,20 +187,19 @@ TEST_P(CacheTest, UsageTest) {
  char value[10] = "abcdef";
  // make sure everything will be cached
  for (int i = 1; i < 100; ++i) {
-    std::string key;
-    if (type == kLRU) {
-      key = std::string(i, 'a');
-    } else {
-      key = EncodeKey(i);
-    }
+    std::string key = EncodeKey(i);
    auto kv_size = key.size() + 5;
    ASSERT_OK(cache->Insert(key, value, &kDumbHelper, kv_size));
    ASSERT_OK(precise_cache->Insert(key, value, &kDumbHelper, kv_size));
    usage += kv_size;
    ASSERT_EQ(usage, cache->GetUsage());
-    if (type == kHyperClock) {
+    if (GetParam() == kFixedHyperClock) {
      ASSERT_EQ(baseline_meta_usage + usage, precise_cache->GetUsage());
    } else {
+      // AutoHyperClockCache meta usage grows in proportion to lifetime
+      // max number of entries. LRUCache in proportion to resident number of
+      // entries, though there is an untracked component proportional to
+      // lifetime max number of entries.
      ASSERT_LT(usage, precise_cache->GetUsage());
    }
  }
@@ -219,16 +207,15 @@ TEST_P(CacheTest, UsageTest) {
  cache->EraseUnRefEntries();
  precise_cache->EraseUnRefEntries();
  ASSERT_EQ(0, cache->GetUsage());
-  ASSERT_EQ(baseline_meta_usage, precise_cache->GetUsage());
+  if (GetParam() != kAutoHyperClock) {
+    // NOTE: AutoHyperClockCache meta usage grows in proportion to lifetime
+    // max number of entries.
+    ASSERT_EQ(baseline_meta_usage, precise_cache->GetUsage());
+  }

  // make sure the cache will be overloaded
  for (size_t i = 1; i < kCapacity; ++i) {
-    std::string key;
-    if (type == kLRU) {
-      key = std::to_string(i);
-    } else {
-      key = EncodeKey(static_cast<int>(1000 + i));
-    }
+    std::string key = EncodeKey(static_cast<int>(1000 + i));
    ASSERT_OK(cache->Insert(key, value, &kDumbHelper, key.size() + 5));
    ASSERT_OK(precise_cache->Insert(key, value, &kDumbHelper, key.size() + 5));
  }
@@ -237,7 +224,7 @@ TEST_P(CacheTest, UsageTest) {
  ASSERT_GT(kCapacity, cache->GetUsage());
  ASSERT_GT(kCapacity, precise_cache->GetUsage());
  ASSERT_LT(kCapacity * 0.95, cache->GetUsage());
-  if (type != kHyperClock) {
+  if (!IsHyperClock()) {
    ASSERT_LT(kCapacity * 0.95, precise_cache->GetUsage());
  } else {
    // estimated value size of 1 is weird for clock cache, because
@@ -248,22 +235,20 @@ TEST_P(CacheTest, UsageTest) {
  }
 }

-// TODO: This test takes longer than expected on ClockCache. This is
-// because the values size estimate at construction is too sloppy.
+// TODO: This test takes longer than expected on FixedHyperClockCache.
+// This is because the values size estimate at construction is too sloppy.
 // Fix this.
 // Why is it so slow? The cache is constructed with an estimate of 1, but
 // then the charge is claimed to be 21. This will cause the hash table
 // to be extremely sparse, which in turn means clock needs to scan too
 // many slots to find victims.
 TEST_P(CacheTest, PinnedUsageTest) {
-  auto type = GetParam();
-
  // cache is std::shared_ptr and will be automatically cleaned up.
  const size_t kCapacity = 200000;
  auto cache = NewCache(kCapacity, 8, false, kDontChargeCacheMetadata);
  auto precise_cache = NewCache(kCapacity, 8, false, kFullChargeCacheMetadata);
  size_t baseline_meta_usage = precise_cache->GetUsage();
-  if (type != kHyperClock) {
+  if (!IsHyperClock()) {
    ASSERT_EQ(0, baseline_meta_usage);
  }

@@ -276,12 +261,7 @@ TEST_P(CacheTest, PinnedUsageTest) {
  // Add entries. Unpin some of them after insertion. Then, pin some of them
  // again. Check GetPinnedUsage().
  for (int i = 1; i < 100; ++i) {
-    std::string key;
-    if (type == kLRU) {
-      key = std::string(i, 'a');
-    } else {
-      key = EncodeKey(i);
-    }
+    std::string key = EncodeKey(i);
    auto kv_size = key.size() + 5;
    Cache::Handle* handle;
    Cache::Handle* handle_in_precise_cache;
@@ -322,12 +302,7 @@ TEST_P(CacheTest, PinnedUsageTest) {

  // check that overloading the cache does not change the pinned usage
  for (size_t i = 1; i < 2 * kCapacity; ++i) {
-    std::string key;
-    if (type == kLRU) {
-      key = std::to_string(i);
-    } else {
-      key = EncodeKey(static_cast<int>(1000 + i));
-    }
+    std::string key = EncodeKey(static_cast<int>(1000 + i));
    ASSERT_OK(cache->Insert(key, value, &kDumbHelper, key.size() + 5));
    ASSERT_OK(precise_cache->Insert(key, value, &kDumbHelper, key.size() + 5));
  }
@@ -351,7 +326,11 @@ TEST_P(CacheTest, PinnedUsageTest) {
  cache->EraseUnRefEntries();
  precise_cache->EraseUnRefEntries();
  ASSERT_EQ(0, cache->GetUsage());
-  ASSERT_EQ(baseline_meta_usage, precise_cache->GetUsage());
+  if (GetParam() != kAutoHyperClock) {
+    // NOTE: AutoHyperClockCache meta usage grows in proportion to lifetime
+    // max number of entries.
+    ASSERT_EQ(baseline_meta_usage, precise_cache->GetUsage());
+  }
 }

 TEST_P(CacheTest, HitAndMiss) {
@@ -368,7 +347,7 @@ TEST_P(CacheTest, HitAndMiss) {
  ASSERT_EQ(-1, Lookup(300));

  Insert(100, 102);
-  if (GetParam() == kHyperClock) {
+  if (IsHyperClock()) {
    // ClockCache usually doesn't overwrite on Insert
    ASSERT_EQ(101, Lookup(100));
  } else {
@@ -378,7 +357,7 @@ TEST_P(CacheTest, HitAndMiss) {
  ASSERT_EQ(-1, Lookup(300));

  ASSERT_EQ(1U, deleted_values_.size());
-  if (GetParam() == kHyperClock) {
+  if (IsHyperClock()) {
    ASSERT_EQ(102, deleted_values_[0]);
  } else {
    ASSERT_EQ(101, deleted_values_[0]);
@@ -386,7 +365,7 @@ TEST_P(CacheTest, HitAndMiss) {
 }

 TEST_P(CacheTest, InsertSameKey) {
-  if (GetParam() == kHyperClock) {
+  if (IsHyperClock()) {
    ROCKSDB_GTEST_BYPASS(
        "ClockCache doesn't guarantee Insert overwrite same key.");
    return;
@@ -415,7 +394,7 @@ TEST_P(CacheTest, Erase) {
 }

 TEST_P(CacheTest, EntriesArePinned) {
-  if (GetParam() == kHyperClock) {
+  if (IsHyperClock()) {
    ROCKSDB_GTEST_BYPASS(
        "ClockCache doesn't guarantee Insert overwrite same key.");
    return;
@@ -479,7 +458,7 @@ TEST_P(CacheTest, ExternalRefPinsEntries) {
      Insert(1000 + j, 2000 + j);
    }
    // Clock cache is even more stateful and needs more churn to evict
-    if (GetParam() == kHyperClock) {
+    if (IsHyperClock()) {
      for (int j = 0; j < kCacheSize; j++) {
        Insert(11000 + j, 11000 + j);
      }
@@ -517,20 +496,20 @@ TEST_P(CacheTest, EvictionPolicyRef) {
  // Check whether the entries inserted in the beginning
  // are evicted. Ones without extra ref are evicted and
  // those with are not.
-  ASSERT_EQ(-1, Lookup(100));
-  ASSERT_EQ(-1, Lookup(101));
-  ASSERT_EQ(-1, Lookup(102));
-  ASSERT_EQ(-1, Lookup(103));
+  EXPECT_EQ(-1, Lookup(100));
+  EXPECT_EQ(-1, Lookup(101));
+  EXPECT_EQ(-1, Lookup(102));
+  EXPECT_EQ(-1, Lookup(103));

-  ASSERT_EQ(-1, Lookup(300));
-  ASSERT_EQ(-1, Lookup(301));
-  ASSERT_EQ(-1, Lookup(302));
-  ASSERT_EQ(-1, Lookup(303));
+  EXPECT_EQ(-1, Lookup(300));
+  EXPECT_EQ(-1, Lookup(301));
+  EXPECT_EQ(-1, Lookup(302));
+  EXPECT_EQ(-1, Lookup(303));

-  ASSERT_EQ(101, Lookup(200));
-  ASSERT_EQ(102, Lookup(201));
-  ASSERT_EQ(103, Lookup(202));
-  ASSERT_EQ(104, Lookup(203));
+  EXPECT_EQ(101, Lookup(200));
+  EXPECT_EQ(102, Lookup(201));
+  EXPECT_EQ(103, Lookup(202));
+  EXPECT_EQ(104, Lookup(203));

  // Cleaning up all the handles
  cache_->Release(h201);
@@ -540,37 +519,22 @@ TEST_P(CacheTest, EvictionPolicyRef) {
 }

 TEST_P(CacheTest, EvictEmptyCache) {
-  auto type = GetParam();
-
  // Insert item large than capacity to trigger eviction on empty cache.
  auto cache = NewCache(1, 0, false);
-  if (type == kLRU) {
-    ASSERT_OK(cache->Insert("foo", nullptr, &kDumbHelper, 10));
-  } else {
-    ASSERT_OK(cache->Insert(EncodeKey(1000), nullptr, &kDumbHelper, 10));
-  }
+  ASSERT_OK(cache->Insert(EncodeKey(1000), nullptr, &kDumbHelper, 10));
 }

 TEST_P(CacheTest, EraseFromDeleter) {
-  auto type = GetParam();
-
  // Have deleter which will erase item from cache, which will re-enter
  // the cache at that point.
  std::shared_ptr<Cache> cache = NewCache(10, 0, false);
-  std::string foo, bar;
-  const Cache::CacheItemHelper* erase_helper;
-  if (type == kLRU) {
-    foo = "foo";
-    bar = "bar";
-    erase_helper = &kEraseOnDeleteHelper1;
-  } else {
-    foo = EncodeKey(1234);
-    bar = EncodeKey(5678);
-    erase_helper = &kEraseOnDeleteHelper2;
-  }
+  std::string foo = EncodeKey(1234);
+  std::string bar = EncodeKey(5678);
+
+  std::function<void()> erase_fn = [&]() { cache->Erase(foo); };

  ASSERT_OK(cache->Insert(foo, nullptr, &kDumbHelper, 1));
-  ASSERT_OK(cache->Insert(bar, cache.get(), erase_helper, 1));
+  ASSERT_OK(cache->Insert(bar, &erase_fn, &kInvokeOnDeleteHelper, 1));

  cache->Erase(bar);
  ASSERT_EQ(nullptr, cache->Lookup(foo));
@@ -678,10 +642,10 @@ using TypedHandle = SharedCache::TypedHandle;
 }  // namespace

 TEST_P(CacheTest, SetCapacity) {
-  auto type = GetParam();
-  if (type == kHyperClock) {
+  if (IsHyperClock()) {
+    // TODO: update test & code for limited supoort
    ROCKSDB_GTEST_BYPASS(
-        "FastLRUCache and HyperClockCache don't support arbitrary capacity "
+        "HyperClockCache doesn't support arbitrary capacity "
        "adjustments.");
    return;
  }
@@ -811,7 +775,7 @@ TEST_P(CacheTest, OverCapacity) {
    cache.Release(handles[i]);
  }

-  if (GetParam() == kHyperClock) {
+  if (IsHyperClock()) {
    // Make sure eviction is triggered.
    ASSERT_OK(cache.Insert(EncodeKey(-1), nullptr, 1, &handles[0]));

@@ -923,8 +887,7 @@ TEST_P(CacheTest, DefaultShardBits) {
  // Prevent excessive allocation (to save time & space)
  estimated_value_size_ = 100000;
  // Implementations use different minimum shard sizes
-  size_t min_shard_size =
-      (GetParam() == kHyperClock ? 32U * 1024U : 512U) * 1024U;
+  size_t min_shard_size = (IsHyperClock() ? 32U * 1024U : 512U) * 1024U;

  std::shared_ptr<Cache> cache = NewCache(32U * min_shard_size);
  ShardedCacheBase* sc = dynamic_cast<ShardedCacheBase*>(cache.get());
@@ -19,8 +19,10 @@ ChargedCache::ChargedCache(std::shared_ptr<Cache> cache,

 Status ChargedCache::Insert(const Slice& key, ObjectPtr obj,
                            const CacheItemHelper* helper, size_t charge,
-                            Handle** handle, Priority priority) {
-  Status s = target_->Insert(key, obj, helper, charge, handle, priority);
+                            Handle** handle, Priority priority,
+                            const Slice& compressed_val, CompressionType type) {
+  Status s = target_->Insert(key, obj, helper, charge, handle, priority,
+                             compressed_val, type);
  if (s.ok()) {
    // Insert may cause the cache entry eviction if the cache is full. So we
    // directly call the reservation manager to update the total memory used
@@ -22,9 +22,11 @@ class ChargedCache : public CacheWrapper {
  ChargedCache(std::shared_ptr<Cache> cache,
               std::shared_ptr<Cache> block_cache);

-  Status Insert(const Slice& key, ObjectPtr obj, const CacheItemHelper* helper,
-                size_t charge, Handle** handle = nullptr,
-                Priority priority = Priority::LOW) override;
+  Status Insert(
+      const Slice& key, ObjectPtr obj, const CacheItemHelper* helper,
+      size_t charge, Handle** handle = nullptr,
+      Priority priority = Priority::LOW, const Slice& compressed_val = Slice(),
+      CompressionType type = CompressionType::kNoCompression) override;

  Cache::Handle* Lookup(const Slice& key, const CacheItemHelper* helper,
                        CreateContext* create_context,
@@ -20,16 +20,19 @@
 #include "cache/sharded_cache.h"
 #include "port/lang.h"
 #include "port/malloc.h"
+#include "port/mmap.h"
 #include "port/port.h"
 #include "rocksdb/cache.h"
 #include "rocksdb/secondary_cache.h"
 #include "util/autovector.h"
+#include "util/math.h"

 namespace ROCKSDB_NAMESPACE {

 namespace clock_cache {

 // Forward declaration of friend class.
+template <class ClockCache>
 class ClockCacheTest;

 // HyperClockCache is an alternative to LRUCache specifically tailored for
@@ -37,24 +40,31 @@ class ClockCacheTest;
 //
 // Benefits
 // --------
-// * Fully lock free (no waits or spins) for efficiency under high concurrency
+// * Lock/wait free (no waits or spins) for efficiency under high concurrency
+//   * Fixed version (estimated_entry_charge > 0) is fully lock/wait free
+//   * Automatic version (estimated_entry_charge = 0) has rare waits among
+//     certain insertion or erase operations that involve the same very small
+//     set of entries.
 // * Optimized for hot path reads. For concurrency control, most Lookup() and
 // essentially all Release() are a single atomic add operation.
-// * Eviction on insertion is fully parallel and lock-free.
+// * Eviction on insertion is fully parallel.
 // * Uses a generalized + aging variant of CLOCK eviction that might outperform
 // LRU in some cases. (For background, see
 // https://en.wikipedia.org/wiki/Page_replacement_algorithm)
 //
 // Costs
 // -----
-// * Hash table is not resizable (for lock-free efficiency) so capacity is not
-// dynamically changeable. Rely on an estimated average value (block) size for
+// * FixedHyperClockCache (estimated_entry_charge > 0) - Hash table is not
+// resizable (for lock-free efficiency) so capacity is not dynamically
+// changeable. Rely on an estimated average value (block) size for
 // space+time efficiency. (See estimated_entry_charge option details.)
+// EXPERIMENTAL - This limitation is fixed in AutoHyperClockCache, activated
+// with estimated_entry_charge == 0.
 // * Insert usually does not (but might) overwrite a previous entry associated
-// with a cache key. This is OK for RocksDB uses of Cache.
+// with a cache key. This is OK for RocksDB uses of Cache, though it does mess
+// up our REDUNDANT block cache insertion statistics.
 // * Only supports keys of exactly 16 bytes, which is what RocksDB uses for
-// block cache (not row cache or table cache).
-// * SecondaryCache is not supported.
+// block cache (but not row cache or table cache).
 // * Cache priorities are less aggressively enforced. Unlike LRUCache, enough
 // transient LOW or BOTTOM priority items can evict HIGH priority entries that
 // are not referenced recently (or often) enough.
@@ -137,7 +147,8 @@ class ClockCacheTest;
 // * Empty - slot is not in use and unowned. All other metadata and data is
 // in an undefined state.
 // * Construction - slot is exclusively owned by one thread, the thread
-// successfully entering this state, for populating or freeing data.
+// successfully entering this state, for populating or freeing data
+// (de-construction, same state marker).
 // * Shareable (group) - slot holds an entry with counted references for
 // pinning and reading, including
 //   * Visible - slot holds an entry that can be returned by Lookup
@@ -185,15 +196,19 @@ class ClockCacheTest;
 // know from our "redundant" stats that overwrites are very rare for the block
 // cache, so we should not spend much to make them effective.
 //
-// So instead we Insert as soon as we find an empty slot in the probing
-// sequence without seeing an existing (visible) entry for the same key. This
-// way we only insert if we can improve the probing performance, and we don't
-// need to probe beyond our insert position, assuming we are willing to let
-// the previous entry for the same key die of old age (eventual eviction from
-// not being used). We can reach a similar state with concurrent insertions,
-// where one will pass over the other while it is "under construction."
-// This temporary duplication is acceptable for RocksDB block cache because
-// we know redundant insertion is rare.
+// FixedHyperClockCache: Instead we Insert as soon as we find an empty slot in
+// the probing sequence without seeing an existing (visible) entry for the same
+// key. This way we only insert if we can improve the probing performance, and
+// we don't need to probe beyond our insert position, assuming we are willing
+// to let the previous entry for the same key die of old age (eventual eviction
+// from not being used). We can reach a similar state with concurrent
+// insertions, where one will pass over the other while it is "under
+// construction." This temporary duplication is acceptable for RocksDB block
+// cache because we know redundant insertion is rare.
+// AutoHyperClockCache: Similar, except we only notice and return an existing
+// match if it is found in the search for a suitable empty slot (starting with
+// the same slot as the head pointer), not by following the existing chain of
+// entries. Insertions are always made to the head of the chain.
 //
 // Another problem to solve is what to return to the caller when we find an
 // existing entry whose probing position we cannot improve on, or when the
@@ -281,29 +296,6 @@ class ClockCacheTest;

 // ----------------------------------------------------------------------- //

-// The load factor p is a real number in (0, 1) such that at all
-// times at most a fraction p of all slots, without counting tombstones,
-// are occupied by elements. This means that the probability that a random
-// probe hits an occupied slot is at most p, and thus at most 1/p probes
-// are required on average. For example, p = 70% implies that between 1 and 2
-// probes are needed on average (bear in mind that this reasoning doesn't
-// consider the effects of clustering over time, which should be negligible
-// with double hashing).
-// Because the size of the hash table is always rounded up to the next
-// power of 2, p is really an upper bound on the actual load factor---the
-// actual load factor is anywhere between p/2 and p. This is a bit wasteful,
-// but bear in mind that slots only hold metadata, not actual values.
-// Since space cost is dominated by the values (the LSM blocks),
-// overprovisioning the table with metadata only increases the total cache space
-// usage by a tiny fraction.
-constexpr double kLoadFactor = 0.7;
-
-// The user can exceed kLoadFactor if the sizes of the inserted values don't
-// match estimated_value_size, or in some rare cases with
-// strict_capacity_limit == false. To avoid degenerate performance, we set a
-// strict upper bound on the load factor.
-constexpr double kStrictLoadFactor = 0.84;
-
 struct ClockHandleBasicData {
  Cache::ObjectPtr value = nullptr;
  const Cache::CacheItemHelper* helper = nullptr;
@@ -326,7 +318,7 @@ struct ClockHandle : public ClockHandleBasicData {
  // state of the handle. The meta word looks like this:
  // low bits                                                     high bits
  // -----------------------------------------------------------------------
-  // | acquire counter          | release counter           | state marker |
+  // | acquire counter      | release counter     | hit bit | state marker |
  // -----------------------------------------------------------------------

  // For reading or updating counters in meta word.
@@ -340,8 +332,12 @@ struct ClockHandle : public ClockHandleBasicData {
  static constexpr uint64_t kReleaseIncrement = uint64_t{1}
                                                << kReleaseCounterShift;

+  // For setting the hit bit
+  static constexpr uint8_t kHitBitShift = 2U * kCounterNumBits;
+  static constexpr uint64_t kHitBitMask = uint64_t{1} << kHitBitShift;
+
  // For reading or updating the state marker in meta word
-  static constexpr uint8_t kStateShift = 2U * kCounterNumBits;
+  static constexpr uint8_t kStateShift = kHitBitShift + 1;

  // Bits contribution to state marker.
  // Occupied means any state other than empty
@@ -371,14 +367,133 @@ struct ClockHandle : public ClockHandleBasicData {
  static constexpr uint8_t kMaxCountdown = kHighCountdown;
  // TODO: make these coundown values tuning parameters for eviction?

-  // See above
-  std::atomic<uint64_t> meta{};
-
-  // Anticipating use for SecondaryCache support
-  void* reserved_for_future_use = nullptr;
+  // See above. Mutable for read reference counting.
+  mutable std::atomic<uint64_t> meta{};
 };  // struct ClockHandle

-class HyperClockTable {
+class BaseClockTable {
+ public:
+  BaseClockTable(CacheMetadataChargePolicy metadata_charge_policy,
+                 MemoryAllocator* allocator,
+                 const Cache::EvictionCallback* eviction_callback,
+                 const uint32_t* hash_seed)
+      : metadata_charge_policy_(metadata_charge_policy),
+        allocator_(allocator),
+        eviction_callback_(*eviction_callback),
+        hash_seed_(*hash_seed) {}
+
+  template <class Table>
+  typename Table::HandleImpl* CreateStandalone(ClockHandleBasicData& proto,
+                                               size_t capacity,
+                                               bool strict_capacity_limit,
+                                               bool allow_uncharged);
+
+  template <class Table>
+  Status Insert(const ClockHandleBasicData& proto,
+                typename Table::HandleImpl** handle, Cache::Priority priority,
+                size_t capacity, bool strict_capacity_limit);
+
+  void Ref(ClockHandle& handle);
+
+  size_t GetOccupancy() const {
+    return occupancy_.load(std::memory_order_relaxed);
+  }
+
+  size_t GetUsage() const { return usage_.load(std::memory_order_relaxed); }
+
+  size_t GetStandaloneUsage() const {
+    return standalone_usage_.load(std::memory_order_relaxed);
+  }
+
+  uint32_t GetHashSeed() const { return hash_seed_; }
+
+  uint64_t GetYieldCount() const { return yield_count_.load(); }
+
+  struct EvictionData {
+    size_t freed_charge = 0;
+    size_t freed_count = 0;
+  };
+
+  void TrackAndReleaseEvictedEntry(ClockHandle* h, EvictionData* data);
+
+#ifndef NDEBUG
+  // Acquire N references
+  void TEST_RefN(ClockHandle& handle, size_t n);
+  // Helper for TEST_ReleaseN
+  void TEST_ReleaseNMinus1(ClockHandle* handle, size_t n);
+#endif
+
+ private:  // fns
+  // Creates a "standalone" handle for returning from an Insert operation that
+  // cannot be completed by actually inserting into the table.
+  // Updates `standalone_usage_` but not `usage_` nor `occupancy_`.
+  template <class HandleImpl>
+  HandleImpl* StandaloneInsert(const ClockHandleBasicData& proto);
+
+  // Helper for updating `usage_` for new entry with given `total_charge`
+  // and evicting if needed under strict_capacity_limit=true rules. This
+  // means the operation might fail with Status::MemoryLimit. If
+  // `need_evict_for_occupancy`, then eviction of at least one entry is
+  // required, and the operation should fail if not possible.
+  // NOTE: Otherwise, occupancy_ is not managed in this function
+  template <class Table>
+  Status ChargeUsageMaybeEvictStrict(size_t total_charge, size_t capacity,
+                                     bool need_evict_for_occupancy,
+                                     typename Table::InsertState& state);
+
+  // Helper for updating `usage_` for new entry with given `total_charge`
+  // and evicting if needed under strict_capacity_limit=false rules. This
+  // means that updating `usage_` always succeeds even if forced to exceed
+  // capacity. If `need_evict_for_occupancy`, then eviction of at least one
+  // entry is required, and the operation should return false if such eviction
+  // is not possible. `usage_` is not updated in that case. Otherwise, returns
+  // true, indicating success.
+  // NOTE: occupancy_ is not managed in this function
+  template <class Table>
+  bool ChargeUsageMaybeEvictNonStrict(size_t total_charge, size_t capacity,
+                                      bool need_evict_for_occupancy,
+                                      typename Table::InsertState& state);
+
+ protected:  // data
+  // We partition the following members into different cache lines
+  // to avoid false sharing among Lookup, Release, Erase and Insert
+  // operations in ClockCacheShard.
+
+  // Clock algorithm sweep pointer.
+  std::atomic<uint64_t> clock_pointer_{};
+
+  // Counter for number of times we yield to wait on another thread.
+  std::atomic<uint64_t> yield_count_{};
+
+  // TODO: is this separation needed if we don't do background evictions?
+  ALIGN_AS(CACHE_LINE_SIZE)
+  // Number of elements in the table.
+  std::atomic<size_t> occupancy_{};
+
+  // Memory usage by entries tracked by the cache (including standalone)
+  std::atomic<size_t> usage_{};
+
+  // Part of usage by standalone entries (not in table)
+  std::atomic<size_t> standalone_usage_{};
+
+  ALIGN_AS(CACHE_LINE_SIZE)
+  const CacheMetadataChargePolicy metadata_charge_policy_;
+
+  // From Cache, for deleter
+  MemoryAllocator* const allocator_;
+
+  // A reference to Cache::eviction_callback_
+  const Cache::EvictionCallback& eviction_callback_;
+
+  // A reference to ShardedCacheBase::hash_seed_
+  const uint32_t& hash_seed_;
+};
+
+// Hash table for cache entries with size determined at creation time.
+// Uses open addressing and double hashing. Since entries cannot be moved,
+// the "displacements" count ensures probing sequences find entries even when
+// entries earlier in the probing sequence have been removed.
+class FixedHyperClockTable : public BaseClockTable {
 public:
  // Target size to be exactly a common cache line size (see static_assert in
  // clock_cache.cc)
@@ -400,88 +515,104 @@ class HyperClockTable {
  };  // struct HandleImpl

  struct Opts {
+    explicit Opts(size_t _estimated_value_size)
+        : estimated_value_size(_estimated_value_size) {}
+    explicit Opts(const HyperClockCacheOptions& opts) {
+      assert(opts.estimated_entry_charge > 0);
+      estimated_value_size = opts.estimated_entry_charge;
+    }
    size_t estimated_value_size;
  };

-  HyperClockTable(size_t capacity, bool strict_capacity_limit,
-                  CacheMetadataChargePolicy metadata_charge_policy,
-                  MemoryAllocator* allocator,
-                  const Cache::EvictionCallback* eviction_callback,
-                  const uint32_t* hash_seed, const Opts& opts);
-  ~HyperClockTable();
+  FixedHyperClockTable(size_t capacity, bool strict_capacity_limit,
+                       CacheMetadataChargePolicy metadata_charge_policy,
+                       MemoryAllocator* allocator,
+                       const Cache::EvictionCallback* eviction_callback,
+                       const uint32_t* hash_seed, const Opts& opts);
+  ~FixedHyperClockTable();

-  Status Insert(const ClockHandleBasicData& proto, HandleImpl** handle,
-                Cache::Priority priority, size_t capacity,
-                bool strict_capacity_limit);
+  // For BaseClockTable::Insert
+  struct InsertState {};

-  HandleImpl* CreateStandalone(ClockHandleBasicData& proto, size_t capacity,
-                               bool strict_capacity_limit,
-                               bool allow_uncharged);
+  void StartInsert(InsertState& state);
+
+  // Returns true iff there is room for the proposed number of entries.
+  bool GrowIfNeeded(size_t new_occupancy, InsertState& state);
+
+  HandleImpl* DoInsert(const ClockHandleBasicData& proto,
+                       uint64_t initial_countdown, bool take_ref,
+                       InsertState& state);
+
+  // Runs the clock eviction algorithm trying to reclaim at least
+  // requested_charge. Returns how much is evicted, which could be less
+  // if it appears impossible to evict the requested amount without blocking.
+  void Evict(size_t requested_charge, InsertState& state, EvictionData* data);

  HandleImpl* Lookup(const UniqueId64x2& hashed_key);

  bool Release(HandleImpl* handle, bool useful, bool erase_if_last_ref);

-  void Ref(HandleImpl& handle);
-
  void Erase(const UniqueId64x2& hashed_key);

-  void ConstApplyToEntriesRange(std::function<void(const HandleImpl&)> func,
-                                size_t index_begin, size_t index_end,
-                                bool apply_if_will_be_deleted) const;
-
  void EraseUnRefEntries();

  size_t GetTableSize() const { return size_t{1} << length_bits_; }

-  int GetLengthBits() const { return length_bits_; }
-
-  size_t GetOccupancy() const {
-    return occupancy_.load(std::memory_order_relaxed);
-  }
-
  size_t GetOccupancyLimit() const { return occupancy_limit_; }

-  size_t GetUsage() const { return usage_.load(std::memory_order_relaxed); }
+  const HandleImpl* HandlePtr(size_t idx) const { return &array_[idx]; }

-  size_t GetStandaloneUsage() const {
-    return standalone_usage_.load(std::memory_order_relaxed);
+#ifndef NDEBUG
+  size_t& TEST_MutableOccupancyLimit() {
+    return const_cast<size_t&>(occupancy_limit_);
  }

-  uint32_t GetHashSeed() const { return hash_seed_; }
-
-  // Acquire/release N references
-  void TEST_RefN(HandleImpl& handle, size_t n);
+  // Release N references
  void TEST_ReleaseN(HandleImpl* handle, size_t n);
+#endif
+
+  // The load factor p is a real number in (0, 1) such that at all
+  // times at most a fraction p of all slots, without counting tombstones,
+  // are occupied by elements. This means that the probability that a random
+  // probe hits an occupied slot is at most p, and thus at most 1/p probes
+  // are required on average. For example, p = 70% implies that between 1 and 2
+  // probes are needed on average (bear in mind that this reasoning doesn't
+  // consider the effects of clustering over time, which should be negligible
+  // with double hashing).
+  // Because the size of the hash table is always rounded up to the next
+  // power of 2, p is really an upper bound on the actual load factor---the
+  // actual load factor is anywhere between p/2 and p. This is a bit wasteful,
+  // but bear in mind that slots only hold metadata, not actual values.
+  // Since space cost is dominated by the values (the LSM blocks),
+  // overprovisioning the table with metadata only increases the total cache
+  // space usage by a tiny fraction.
+  static constexpr double kLoadFactor = 0.7;
+
+  // The user can exceed kLoadFactor if the sizes of the inserted values don't
+  // match estimated_value_size, or in some rare cases with
+  // strict_capacity_limit == false. To avoid degenerate performance, we set a
+  // strict upper bound on the load factor.
+  static constexpr double kStrictLoadFactor = 0.84;

 private:  // functions
  // Returns x mod 2^{length_bits_}.
  inline size_t ModTableSize(uint64_t x) {
-    return static_cast<size_t>(x) & length_bits_mask_;
+    return BitwiseAnd(x, length_bits_mask_);
  }

-  // Runs the clock eviction algorithm trying to reclaim at least
-  // requested_charge. Returns how much is evicted, which could be less
-  // if it appears impossible to evict the requested amount without blocking.
-  inline void Evict(size_t requested_charge, size_t* freed_charge,
-                    size_t* freed_count);
-
-  // Returns the first slot in the probe sequence, starting from the given
-  // probe number, with a handle e such that match(e) is true. At every
-  // step, the function first tests whether match(e) holds. If this is false,
-  // it evaluates abort(e) to decide whether the search should be aborted,
-  // and in the affirmative returns -1. For every handle e probed except
-  // the last one, the function runs update(e).
-  // The probe parameter is modified as follows. We say a probe to a handle
-  // e is aborting if match(e) is false and abort(e) is true. Then the final
-  // value of probe is one more than the last non-aborting probe during the
-  // call. This is so that that the variable can be used to keep track of
-  // progress across consecutive calls to FindSlot.
+  // Returns the first slot in the probe sequence with a handle e such that
+  // match_fn(e) is true. At every step, the function first tests whether
+  // match_fn(e) holds. If this is false, it evaluates abort_fn(e) to decide
+  // whether the search should be aborted, and if so, FindSlot immediately
+  // returns nullptr. For every handle e that is not a match and not aborted,
+  // FindSlot runs update_fn(e, is_last) where is_last is set to true iff that
+  // slot will be the last probed because the next would cycle back to the first
+  // slot probed. This function uses templates instead of std::function to
+  // minimize the risk of heap-allocated closures being created.
+  template <typename MatchFn, typename AbortFn, typename UpdateFn>
  inline HandleImpl* FindSlot(const UniqueId64x2& hashed_key,
-                              std::function<bool(HandleImpl*)> match,
-                              std::function<bool(HandleImpl*)> stop,
-                              std::function<void(HandleImpl*)> update,
-                              size_t& probe);
+                              const MatchFn& match_fn, const AbortFn& abort_fn,
+                              const UpdateFn& update_fn);

  // Re-decrement all displacements in probe path starting from beginning
  // until (not including) the given handle
@@ -494,33 +625,6 @@ class HyperClockTable {
  // before releasing it so that it can be provided to this function.
  inline void ReclaimEntryUsage(size_t total_charge);

-  // Helper for updating `usage_` for new entry with given `total_charge`
-  // and evicting if needed under strict_capacity_limit=true rules. This
-  // means the operation might fail with Status::MemoryLimit. If
-  // `need_evict_for_occupancy`, then eviction of at least one entry is
-  // required, and the operation should fail if not possible.
-  // NOTE: Otherwise, occupancy_ is not managed in this function
-  inline Status ChargeUsageMaybeEvictStrict(size_t total_charge,
-                                            size_t capacity,
-                                            bool need_evict_for_occupancy);
-
-  // Helper for updating `usage_` for new entry with given `total_charge`
-  // and evicting if needed under strict_capacity_limit=false rules. This
-  // means that updating `usage_` always succeeds even if forced to exceed
-  // capacity. If `need_evict_for_occupancy`, then eviction of at least one
-  // entry is required, and the operation should return false if such eviction
-  // is not possible. `usage_` is not updated in that case. Otherwise, returns
-  // true, indicating success.
-  // NOTE: occupancy_ is not managed in this function
-  inline bool ChargeUsageMaybeEvictNonStrict(size_t total_charge,
-                                             size_t capacity,
-                                             bool need_evict_for_occupancy);
-
-  // Creates a "standalone" handle for returning from an Insert operation that
-  // cannot be completed by actually inserting into the table.
-  // Updates `standalone_usage_` but not `usage_` nor `occupancy_`.
-  inline HandleImpl* StandaloneInsert(const ClockHandleBasicData& proto);
-
  MemoryAllocator* GetAllocator() const { return allocator_; }

  // Returns the number of bits used to hash an element in the hash
@@ -541,39 +645,323 @@ class HyperClockTable {

  // Array of slots comprising the hash table.
  const std::unique_ptr<HandleImpl[]> array_;
+};  // class FixedHyperClockTable

-  // From Cache, for deleter
-  MemoryAllocator* const allocator_;
+// Hash table for cache entries that resizes automatically based on occupancy.
+// However, it depends on a contiguous memory region to grow into
+// incrementally, using linear hashing, so uses an anonymous mmap so that
+// only the used portion of the memory region is mapped to physical memory
+// (part of RSS).
+//
+// This table implementation uses the same "low-level protocol" for managing
+// the contens of an entry slot as FixedHyperClockTable does, captured in the
+// ClockHandle struct. The provides most of the essential data safety, but
+// AutoHyperClockTable is another "high-level protocol" for organizing entries
+// into a hash table, with automatic resizing.
+//
+// This implementation is not fully wait-free but we can call it "essentially
+// wait-free," and here's why. First, like FixedHyperClockCache, there is no
+// locking nor other forms of waiting at the cache or shard level. Also like
+// FixedHCC there is essentially an entry-level read-write lock implemented
+// with atomics, but our relaxed atomicity/consistency guarantees (e.g.
+// duplicate inserts are possible) mean we do not need to wait for entry
+// locking. Lookups, non-erasing Releases, and non-evicting non-growing Inserts
+// are all fully wait-free. Of course, these waits are not dependent on any
+// external factors such as I/O.
+//
+// For operations that remove entries from a chain or grow the table by
+// splitting a chain, there is a chain-level locking mechanism that we call a
+// "rewrite" lock, and the only waits are for these locks. On average, each
+// chain lock is relevant to < 2 entries each. (The average would be less than
+// one entry each, but we do not lock when there's no entry to remove or
+// migrate.) And a given thread can only hold two such chain locks at a time,
+// more typically just one. So in that sense alone, the waiting that does exist
+// is very localized.
+//
+// If we look closer at the operations utilizing that locking mechanism, we
+// can see why it's "essentially wait-free."
+// * Grow operations to increase the size of the table: each operation splits
+// an existing chain into two, and chains for splitting are chosen in table
+// order. Grow operations are fully parallel except for the chain locking, but
+// for one Grow operation to wait on another, it has to be feeding into the
+// other, which means the table has doubled in size already from other Grow
+// operations without the original one finishing. So Grow operations are very
+// low latency (unlike LRUCache doubling the table size in one operation) and
+// very parallelizeable. (We use some tricks to break up dependencies in
+// updating metadata on the usable size of the table.) And obviously Grow
+// operations are very rare after the initial population of the table.
+// * Evict operations (part of many Inserts): clock updates and evictions
+// sweep through the structure in table order, so like Grow operations,
+// parallel Evict can only wait on each other if an Evict has lingered (slept)
+// long enough that the clock pointer has wrapped around the entire structure.
+// * Random erasures (Erase, Release with erase_if_last_ref, etc.): these
+// operations are rare and not really considered performance critical.
+// Currently they're mostly used for removing placeholder cache entries, e.g.
+// for memory tracking, though that could use standalone entries instead to
+// avoid potential contention in table operations. It's possible that future
+// enhancements could pro-actively remove cache entries from obsolete files,
+// but that's not yet implemented.
+class AutoHyperClockTable : public BaseClockTable {
+ public:
+  // Target size to be exactly a common cache line size (see static_assert in
+  // clock_cache.cc)
+  struct ALIGN_AS(64U) HandleImpl : public ClockHandle {
+    // To orgainize AutoHyperClockTable entries into a hash table while
+    // allowing the table size to grow without existing entries being moved,
+    // a version of chaining is used. Rather than being heap allocated (and
+    // incurring overheads to ensure memory safety) entries must go into
+    // Handles ("slots") in the pre-allocated array. To improve CPU cache
+    // locality, the chain head pointers are interleved with the entries;
+    // specifically, a Handle contains
+    // * A head pointer for a chain of entries with this "home" location.
+    // * A ClockHandle, for an entry that may or may not be in the chain
+    // starting from that head (but for performance ideally is on that
+    // chain).
+    // * A next pointer for the continuation of the chain containing this
+    // entry.
+    //
+    // The pointers are not raw pointers, but are indices into the array,
+    // and are decorated in two ways to help detect and recover from
+    // relevant concurrent modifications during Lookup, so that Lookup is
+    // fully wait-free:
+    // * Each "with_shift" pointer contains a shift count that indicates
+    // how many hash bits were used in chosing the home address for the
+    // chain--specifically the next entry in the chain.
+    // * The end of a chain is given a special "end" marker and refers back
+    // to the head of the chain.
+    //
+    // Why do we need shift on each pointer? To make Lookup wait-free, we need
+    // to be able to query a chain without missing anything, and preferably
+    // avoid synchronously double-checking the length_info. Without the shifts,
+    // there is a risk that we start down a chain and while paused on an entry
+    // that goes to a new home, we then follow the rest of the
+    // partially-migrated chain to see the shared ending with the old home, but
+    // for a time were following the chain for the new home, missing some
+    // entries for the old home.
+    //
+    // Why do we need the end of the chain to loop back? If Lookup pauses
+    // at an "under construction" entry, and sees that "next" is null after
+    // waking up, we need something to tell whether the "under construction"
+    // entry was freed and reused for another chain. Otherwise, we could
+    // miss entries still on the original chain due in the presence of a
+    // concurrent modification. Until an entry is fully erased from a chain,
+    // it is normal to see "under construction" entries on the chain, and it
+    // is not safe to read their hashed key without either a read reference
+    // on the entry or a rewrite lock on the chain.

-  // A reference to Cache::eviction_callback_
-  const Cache::EvictionCallback& eviction_callback_;
+    // Marker in a "with_shift" head pointer for some thread owning writes
+    // to the chain structure (except for inserts), but only if not an
+    // "end" pointer. Also called the "rewrite lock."
+    static constexpr uint64_t kHeadLocked = uint64_t{1} << 7;

-  // A reference to ShardedCacheBase::hash_seed_
-  const uint32_t& hash_seed_;
+    // Marker in a "with_shift" pointer for the end of a chain. Must also
+    // point back to the head of the chain (with end marker removed).
+    // Also includes the "locked" bit so that attempting to lock an empty
+    // chain has no effect (not needed, as the lock is only needed for
+    // removals).
+    static constexpr uint64_t kNextEndFlags = (uint64_t{1} << 6) | kHeadLocked;

-  // We partition the following members into different cache lines
-  // to avoid false sharing among Lookup, Release, Erase and Insert
-  // operations in ClockCacheShard.
+    static inline bool IsEnd(uint64_t next_with_shift) {
+      // Assuming certain values never used, suffices to check this one bit
+      constexpr auto kCheckBit = kNextEndFlags ^ kHeadLocked;
+      return next_with_shift & kCheckBit;
+    }

-  ALIGN_AS(CACHE_LINE_SIZE)
-  // Clock algorithm sweep pointer.
-  std::atomic<uint64_t> clock_pointer_{};
+    // Bottom bits to right shift away to get an array index from a
+    // "with_shift" pointer.
+    static constexpr int kNextShift = 8;

-  ALIGN_AS(CACHE_LINE_SIZE)
-  // Number of elements in the table.
-  std::atomic<size_t> occupancy_{};
+    // A bit mask for the "shift" associated with each "with_shift" pointer.
+    // Always bottommost bits.
+    static constexpr int kShiftMask = 63;

-  // Memory usage by entries tracked by the cache (including standalone)
-  std::atomic<size_t> usage_{};
+    // A marker for head_next_with_shift that indicates this HandleImpl is
+    // heap allocated (standalone) rather than in the table.
+    static constexpr uint64_t kStandaloneMarker = UINT64_MAX;

-  // Part of usage by standalone entries (not in table)
-  std::atomic<size_t> standalone_usage_{};
-};  // class HyperClockTable
+    // A marker for head_next_with_shift indicating the head is not yet part
+    // of the usable table, or for chain_next_with_shift indicating that the
+    // entry is not present or is not yet part of a chain (must not be
+    // "shareable" state).
+    static constexpr uint64_t kUnusedMarker = 0;
+
+    // See above. The head pointer is logically independent of the rest of
+    // the entry, including the chain next pointer.
+    std::atomic<uint64_t> head_next_with_shift{kUnusedMarker};
+    std::atomic<uint64_t> chain_next_with_shift{kUnusedMarker};
+
+    // For supporting CreateStandalone and some fallback cases.
+    inline bool IsStandalone() const {
+      return head_next_with_shift.load(std::memory_order_acquire) ==
+             kStandaloneMarker;
+    }
+
+    inline void SetStandalone() {
+      head_next_with_shift.store(kStandaloneMarker, std::memory_order_release);
+    }
+  };  // struct HandleImpl
+
+  struct Opts {
+    explicit Opts(size_t _min_avg_value_size)
+        : min_avg_value_size(_min_avg_value_size) {}
+
+    explicit Opts(const HyperClockCacheOptions& opts) {
+      assert(opts.estimated_entry_charge == 0);
+      min_avg_value_size = opts.min_avg_entry_charge;
+    }
+    size_t min_avg_value_size;
+  };
+
+  AutoHyperClockTable(size_t capacity, bool strict_capacity_limit,
+                      CacheMetadataChargePolicy metadata_charge_policy,
+                      MemoryAllocator* allocator,
+                      const Cache::EvictionCallback* eviction_callback,
+                      const uint32_t* hash_seed, const Opts& opts);
+  ~AutoHyperClockTable();
+
+  // For BaseClockTable::Insert
+  struct InsertState {
+    uint64_t saved_length_info = 0;
+    size_t likely_empty_slot = 0;
+  };
+
+  void StartInsert(InsertState& state);
+
+  // Does initial check for whether there's hash table room for another
+  // inserted entry, possibly growing if needed. Returns true iff (after
+  // the call) there is room for the proposed number of entries.
+  bool GrowIfNeeded(size_t new_occupancy, InsertState& state);
+
+  HandleImpl* DoInsert(const ClockHandleBasicData& proto,
+                       uint64_t initial_countdown, bool take_ref,
+                       InsertState& state);
+
+  // Runs the clock eviction algorithm trying to reclaim at least
+  // requested_charge. Returns how much is evicted, which could be less
+  // if it appears impossible to evict the requested amount without blocking.
+  void Evict(size_t requested_charge, InsertState& state, EvictionData* data);
+
+  HandleImpl* Lookup(const UniqueId64x2& hashed_key);
+
+  bool Release(HandleImpl* handle, bool useful, bool erase_if_last_ref);
+
+  void Erase(const UniqueId64x2& hashed_key);
+
+  void EraseUnRefEntries();
+
+  size_t GetTableSize() const;
+
+  size_t GetOccupancyLimit() const;
+
+  const HandleImpl* HandlePtr(size_t idx) const { return &array_[idx]; }
+
+#ifndef NDEBUG
+  size_t& TEST_MutableOccupancyLimit() {
+    return *reinterpret_cast<size_t*>(&occupancy_limit_);
+  }
+
+  // Release N references
+  void TEST_ReleaseN(HandleImpl* handle, size_t n);
+#endif
+
+  // Maximum ratio of number of occupied slots to number of usable slots. The
+  // actual load factor should float pretty close to this number, which should
+  // be a nice space/time trade-off, though large swings in WriteBufferManager
+  // memory could lead to low (but very much safe) load factors (only after
+  // seeing high load factors). Linear hashing along with (modified) linear
+  // probing to find an available slot increases potential risks of high
+  // load factors, so are disallowed.
+  static constexpr double kMaxLoadFactor = 0.60;
+
+ private:  // functions
+  // Returns true iff increased usable length. Due to load factor
+  // considerations, GrowIfNeeded might call this more than once to make room
+  // for one more entry.
+  bool Grow(InsertState& state);
+
+  // Operational details of splitting a chain into two for Grow().
+  void SplitForGrow(size_t grow_home, size_t old_home, int old_shift);
+
+  // Takes an "under construction" entry and ensures it is no longer connected
+  // to its home chain (in preparaion for completing erasure and freeing the
+  // slot). Note that previous operations might have already noticed it being
+  // "under (de)construction" and removed it from its chain.
+  void Remove(HandleImpl* h);
+
+  // Try to take ownership of an entry and erase+remove it from the table.
+  // Returns true if successful. Could fail if
+  // * There are other references to the entry
+  // * Some other thread has exclusive ownership or has freed it.
+  bool TryEraseHandle(HandleImpl* h, bool holding_ref, bool mark_invisible);
+
+  // Calculates the appropriate maximum table size, for creating the memory
+  // mapping.
+  static size_t CalcMaxUsableLength(
+      size_t capacity, size_t min_avg_value_size,
+      CacheMetadataChargePolicy metadata_charge_policy);
+
+  // Shared helper function that implements removing entries from a chain
+  // with proper handling to ensure all existing data is seen even in the
+  // presence of concurrent insertions, etc. (See implementation.)
+  template <class OpData>
+  void PurgeImpl(OpData* op_data, size_t home = SIZE_MAX);
+
+  // An RAII wrapper for locking a chain of entries for removals. See
+  // implementation.
+  class ChainRewriteLock;
+
+  // Helper function for PurgeImpl while holding a ChainRewriteLock. See
+  // implementation.
+  template <class OpData>
+  void PurgeImplLocked(OpData* op_data, ChainRewriteLock& rewrite_lock,
+                       size_t home);
+
+  // Update length_info_ as much as possible without waiting, given a known
+  // usable (ready for inserts and lookups) grow_home. (Previous grow_homes
+  // might not be usable yet, but we can check if they are by looking at
+  // the corresponding old home.)
+  void CatchUpLengthInfoNoWait(size_t known_usable_grow_home);
+
+ private:  // data
+  // mmaped area holding handles
+  const TypedMemMapping<HandleImpl> array_;
+
+  // Metadata for table size under linear hashing.
+  //
+  // Lowest 8 bits are the minimum number of lowest hash bits to use
+  // ("min shift"). The upper 56 bits are a threshold. If that minumum number
+  // of bits taken from a hash value is < this threshold, then one more bit of
+  // hash value is taken and used.
+  //
+  // Other mechanisms (shift amounts on pointers) ensure complete availability
+  // of data already in the table even if a reader only sees a completely
+  // out-of-date version of this value. In the worst case, it could take
+  // log time to find the correct chain, but normally this value enables
+  // readers to find the correct chain on the first try.
+  //
+  // NOTES: length_info_ is only updated at the end of a Grow operation,
+  // so that waiting in Grow operations isn't done while entries are pinned
+  // for internal operation purposes. Thus, Lookup and Insert have to
+  // detect and support cases where length_info hasn't caught up to updated
+  // chains. Winning grow thread is the one that transitions
+  // head_next_with_shift from zeros. Grow threads can spin/yield wait for
+  // preconditions and postconditions to be met.
+  std::atomic<uint64_t> length_info_;
+
+  // An already-computed version of the usable length times the max load
+  // factor. Could be slightly out of date but GrowIfNeeded()/Grow() handle
+  // that internally.
+  std::atomic<size_t> occupancy_limit_;
+
+  // See explanation in AutoHyperClockTable::Evict
+  std::atomic<size_t> clock_pointer_mask_;
+};  // class AutoHyperClockTable

 // A single shard of sharded cache.
-template <class Table>
+template <class TableT>
 class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShardBase {
 public:
+  using Table = TableT;
  ClockCacheShard(size_t capacity, bool strict_capacity_limit,
                  CacheMetadataChargePolicy metadata_charge_policy,
                  MemoryAllocator* allocator,
@@ -666,9 +1054,17 @@ class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShardBase {
    return Lookup(key, hashed_key);
  }

+  Table& GetTable() { return table_; }
+  const Table& GetTable() const { return table_; }
+
+#ifndef NDEBUG
+  size_t& TEST_MutableOccupancyLimit() {
+    return table_.TEST_MutableOccupancyLimit();
+  }
  // Acquire/release N references
  void TEST_RefN(HandleImpl* handle, size_t n);
  void TEST_ReleaseN(HandleImpl* handle, size_t n);
+#endif

 private:  // data
  Table table_;
@@ -680,17 +1076,14 @@ class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShardBase {
  std::atomic<bool> strict_capacity_limit_;
 };  // class ClockCacheShard

-class HyperClockCache
-#ifdef NDEBUG
-    final
-#endif
-    : public ShardedCache<ClockCacheShard<HyperClockTable>> {
+template <class Table>
+class BaseHyperClockCache : public ShardedCache<ClockCacheShard<Table>> {
 public:
-  using Shard = ClockCacheShard<HyperClockTable>;
+  using Shard = ClockCacheShard<Table>;
+  using Handle = Cache::Handle;
+  using CacheItemHelper = Cache::CacheItemHelper;

-  explicit HyperClockCache(const HyperClockCacheOptions& opts);
-
-  const char* Name() const override { return "HyperClockCache"; }
+  explicit BaseHyperClockCache(const HyperClockCacheOptions& opts);

  Cache::ObjectPtr Value(Handle* handle) override;

@@ -700,7 +1093,35 @@ class HyperClockCache

  void ReportProblems(
      const std::shared_ptr<Logger>& /*info_log*/) const override;
-};  // class HyperClockCache
+};
+
+class FixedHyperClockCache
+#ifdef NDEBUG
+    final
+#endif
+    : public BaseHyperClockCache<FixedHyperClockTable> {
+ public:
+  using BaseHyperClockCache::BaseHyperClockCache;
+
+  const char* Name() const override { return "FixedHyperClockCache"; }
+
+  void ReportProblems(
+      const std::shared_ptr<Logger>& /*info_log*/) const override;
+};  // class FixedHyperClockCache
+
+class AutoHyperClockCache
+#ifdef NDEBUG
+    final
+#endif
+    : public BaseHyperClockCache<AutoHyperClockTable> {
+ public:
+  using BaseHyperClockCache::BaseHyperClockCache;
+
+  const char* Name() const override { return "AutoHyperClockCache"; }
+
+  void ReportProblems(
+      const std::shared_ptr<Logger>& /*info_log*/) const override;
+};  // class AutoHyperClockCache

 }  // namespace clock_cache

@@ -11,6 +11,7 @@

 #include "memory/memory_allocator_impl.h"
 #include "monitoring/perf_context_imp.h"
+#include "util/coding.h"
 #include "util/compression.h"
 #include "util/string_util.h"

@@ -18,15 +19,24 @@ namespace ROCKSDB_NAMESPACE {

 CompressedSecondaryCache::CompressedSecondaryCache(
    const CompressedSecondaryCacheOptions& opts)
-    : cache_(opts.LRUCacheOptions::MakeSharedCache()), cache_options_(opts) {}
+    : cache_(opts.LRUCacheOptions::MakeSharedCache()),
+      cache_options_(opts),
+      cache_res_mgr_(std::make_shared<ConcurrentCacheReservationManager>(
+          std::make_shared<CacheReservationManagerImpl<CacheEntryRole::kMisc>>(
+              cache_))),
+      disable_cache_(opts.capacity == 0) {}

-CompressedSecondaryCache::~CompressedSecondaryCache() { cache_.reset(); }
+CompressedSecondaryCache::~CompressedSecondaryCache() {}

 std::unique_ptr<SecondaryCacheResultHandle> CompressedSecondaryCache::Lookup(
    const Slice& key, const Cache::CacheItemHelper* helper,
    Cache::CreateContext* create_context, bool /*wait*/, bool advise_erase,
    bool& kept_in_sec_cache) {
  assert(helper);
+  if (disable_cache_) {
+    return nullptr;
+  }
+
  std::unique_ptr<SecondaryCacheResultHandle> handle;
  kept_in_sec_cache = false;
  Cache::Handle* lru_handle = cache_->Lookup(key);
@@ -43,40 +53,65 @@ std::unique_ptr<SecondaryCacheResultHandle> CompressedSecondaryCache::Lookup(
  CacheAllocationPtr* ptr{nullptr};
  CacheAllocationPtr merged_value;
  size_t handle_value_charge{0};
+  const char* data_ptr = nullptr;
+  CacheTier source = CacheTier::kVolatileCompressedTier;
+  CompressionType type = cache_options_.compression_type;
  if (cache_options_.enable_custom_split_merge) {
    CacheValueChunk* value_chunk_ptr =
        reinterpret_cast<CacheValueChunk*>(handle_value);
    merged_value = MergeChunksIntoValue(value_chunk_ptr, handle_value_charge);
    ptr = &merged_value;
+    data_ptr = ptr->get();
  } else {
+    uint32_t type_32 = static_cast<uint32_t>(type);
+    uint32_t source_32 = static_cast<uint32_t>(source);
    ptr = reinterpret_cast<CacheAllocationPtr*>(handle_value);
    handle_value_charge = cache_->GetCharge(lru_handle);
+    data_ptr = ptr->get();
+    data_ptr = GetVarint32Ptr(data_ptr, data_ptr + 1,
+                              static_cast<uint32_t*>(&type_32));
+    type = static_cast<CompressionType>(type_32);
+    data_ptr = GetVarint32Ptr(data_ptr, data_ptr + 1,
+                              static_cast<uint32_t*>(&source_32));
+    source = static_cast<CacheTier>(source_32);
+    handle_value_charge -= (data_ptr - ptr->get());
  }
  MemoryAllocator* allocator = cache_options_.memory_allocator.get();

  Status s;
  Cache::ObjectPtr value{nullptr};
  size_t charge{0};
-  if (cache_options_.compression_type == kNoCompression ||
-      cache_options_.do_not_compress_roles.Contains(helper->role)) {
-    s = helper->create_cb(Slice(ptr->get(), handle_value_charge),
-                          create_context, allocator, &value, &charge);
-  } else {
-    UncompressionContext uncompression_context(cache_options_.compression_type);
-    UncompressionInfo uncompression_info(uncompression_context,
-                                         UncompressionDict::GetEmptyDict(),
-                                         cache_options_.compression_type);
+  if (source == CacheTier::kVolatileCompressedTier) {
+    if (cache_options_.compression_type == kNoCompression ||
+        cache_options_.do_not_compress_roles.Contains(helper->role)) {
+      s = helper->create_cb(Slice(data_ptr, handle_value_charge),
+                            kNoCompression, CacheTier::kVolatileTier,
+                            create_context, allocator, &value, &charge);
+    } else {
+      UncompressionContext uncompression_context(
+          cache_options_.compression_type);
+      UncompressionInfo uncompression_info(uncompression_context,
+                                           UncompressionDict::GetEmptyDict(),
+                                           cache_options_.compression_type);

-    size_t uncompressed_size{0};
-    CacheAllocationPtr uncompressed = UncompressData(
-        uncompression_info, (char*)ptr->get(), handle_value_charge,
-        &uncompressed_size, cache_options_.compress_format_version, allocator);
+      size_t uncompressed_size{0};
+      CacheAllocationPtr uncompressed =
+          UncompressData(uncompression_info, (char*)data_ptr,
+                         handle_value_charge, &uncompressed_size,
+                         cache_options_.compress_format_version, allocator);

-    if (!uncompressed) {
-      cache_->Release(lru_handle, /*erase_if_last_ref=*/true);
-      return nullptr;
+      if (!uncompressed) {
+        cache_->Release(lru_handle, /*erase_if_last_ref=*/true);
+        return nullptr;
+      }
+      s = helper->create_cb(Slice(uncompressed.get(), uncompressed_size),
+                            kNoCompression, CacheTier::kVolatileTier,
+                            create_context, allocator, &value, &charge);
    }
-    s = helper->create_cb(Slice(uncompressed.get(), uncompressed_size),
+  } else {
+    // The item was not compressed by us. Let the helper create_cb
+    // uncompress it
+    s = helper->create_cb(Slice(data_ptr, handle_value_charge), type, source,
                          create_context, allocator, &value, &charge);
  }

@@ -101,40 +136,59 @@ std::unique_ptr<SecondaryCacheResultHandle> CompressedSecondaryCache::Lookup(
  return handle;
 }

-Status CompressedSecondaryCache::Insert(const Slice& key,
-                                        Cache::ObjectPtr value,
-                                        const Cache::CacheItemHelper* helper) {
-  if (value == nullptr) {
-    return Status::InvalidArgument();
-  }
-
-  Cache::Handle* lru_handle = cache_->Lookup(key);
+bool CompressedSecondaryCache::MaybeInsertDummy(const Slice& key) {
  auto internal_helper = GetHelper(cache_options_.enable_custom_split_merge);
+  Cache::Handle* lru_handle = cache_->Lookup(key);
  if (lru_handle == nullptr) {
    PERF_COUNTER_ADD(compressed_sec_cache_insert_dummy_count, 1);
    // Insert a dummy handle if the handle is evicted for the first time.
-    return cache_->Insert(key, /*obj=*/nullptr, internal_helper,
-                          /*charge=*/0);
+    cache_->Insert(key, /*obj=*/nullptr, internal_helper, /*charge=*/0)
+        .PermitUncheckedError();
+    return true;
  } else {
    cache_->Release(lru_handle, /*erase_if_last_ref=*/false);
  }

-  size_t size = (*helper->size_cb)(value);
-  CacheAllocationPtr ptr =
-      AllocateBlock(size, cache_options_.memory_allocator.get());
+  return false;
+}

-  Status s = (*helper->saveto_cb)(value, 0, size, ptr.get());
+Status CompressedSecondaryCache::InsertInternal(
+    const Slice& key, Cache::ObjectPtr value,
+    const Cache::CacheItemHelper* helper, CompressionType type,
+    CacheTier source) {
+  if (source != CacheTier::kVolatileCompressedTier &&
+      cache_options_.enable_custom_split_merge) {
+    // We don't support custom split/merge for the tiered case
+    return Status::OK();
+  }
+
+  auto internal_helper = GetHelper(cache_options_.enable_custom_split_merge);
+  char header[10];
+  char* payload = header;
+  payload = EncodeVarint32(payload, static_cast<uint32_t>(type));
+  payload = EncodeVarint32(payload, static_cast<uint32_t>(source));
+
+  size_t header_size = payload - header;
+  size_t data_size = (*helper->size_cb)(value);
+  size_t total_size = data_size + header_size;
+  CacheAllocationPtr ptr =
+      AllocateBlock(total_size, cache_options_.memory_allocator.get());
+  char* data_ptr = ptr.get() + header_size;
+
+  Status s = (*helper->saveto_cb)(value, 0, data_size, data_ptr);
  if (!s.ok()) {
    return s;
  }
-  Slice val(ptr.get(), size);
+  Slice val(data_ptr, data_size);

  std::string compressed_val;
  if (cache_options_.compression_type != kNoCompression &&
+      type == kNoCompression &&
      !cache_options_.do_not_compress_roles.Contains(helper->role)) {
-    PERF_COUNTER_ADD(compressed_sec_cache_uncompressed_bytes, size);
+    PERF_COUNTER_ADD(compressed_sec_cache_uncompressed_bytes, data_size);
    CompressionOptions compression_opts;
-    CompressionContext compression_context(cache_options_.compression_type);
+    CompressionContext compression_context(cache_options_.compression_type,
+                                           compression_opts);
    uint64_t sample_for_compression{0};
    CompressionInfo compression_info(
        compression_opts, compression_context, CompressionDict::GetEmptyDict(),
@@ -149,12 +203,14 @@ Status CompressedSecondaryCache::Insert(const Slice& key,
    }

    val = Slice(compressed_val);
-    size = compressed_val.size();
-    PERF_COUNTER_ADD(compressed_sec_cache_compressed_bytes, size);
+    data_size = compressed_val.size();
+    total_size = header_size + data_size;
+    PERF_COUNTER_ADD(compressed_sec_cache_compressed_bytes, data_size);

    if (!cache_options_.enable_custom_split_merge) {
-      ptr = AllocateBlock(size, cache_options_.memory_allocator.get());
-      memcpy(ptr.get(), compressed_val.data(), size);
+      ptr = AllocateBlock(total_size, cache_options_.memory_allocator.get());
+      data_ptr = ptr.get() + header_size;
+      memcpy(data_ptr, compressed_val.data(), data_size);
    }
  }

@@ -165,17 +221,52 @@ Status CompressedSecondaryCache::Insert(const Slice& key,
        SplitValueIntoChunks(val, cache_options_.compression_type, charge);
    return cache_->Insert(key, value_chunks_head, internal_helper, charge);
  } else {
+    std::memcpy(ptr.get(), header, header_size);
    CacheAllocationPtr* buf = new CacheAllocationPtr(std::move(ptr));
-    return cache_->Insert(key, buf, internal_helper, size);
+    return cache_->Insert(key, buf, internal_helper, total_size);
  }
 }

+Status CompressedSecondaryCache::Insert(const Slice& key,
+                                        Cache::ObjectPtr value,
+                                        const Cache::CacheItemHelper* helper,
+                                        bool force_insert) {
+  if (value == nullptr) {
+    return Status::InvalidArgument();
+  }
+
+  if (!force_insert && MaybeInsertDummy(key)) {
+    return Status::OK();
+  }
+
+  return InsertInternal(key, value, helper, kNoCompression,
+                        CacheTier::kVolatileCompressedTier);
+}
+
+Status CompressedSecondaryCache::InsertSaved(
+    const Slice& key, const Slice& saved, CompressionType type = kNoCompression,
+    CacheTier source = CacheTier::kVolatileTier) {
+  if (type == kNoCompression) {
+    return Status::OK();
+  }
+
+  auto slice_helper = &kSliceCacheItemHelper;
+  if (MaybeInsertDummy(key)) {
+    return Status::OK();
+  }
+
+  return InsertInternal(
+      key, static_cast<Cache::ObjectPtr>(const_cast<Slice*>(&saved)),
+      slice_helper, type, source);
+}
+
 void CompressedSecondaryCache::Erase(const Slice& key) { cache_->Erase(key); }

 Status CompressedSecondaryCache::SetCapacity(size_t capacity) {
  MutexLock l(&capacity_mutex_);
  cache_options_.capacity = capacity;
  cache_->SetCapacity(capacity);
+  disable_cache_ = capacity == 0;
  return Status::OK();
 }

@@ -301,4 +392,12 @@ CompressedSecondaryCacheOptions::MakeSharedSecondaryCache() const {
  return std::make_shared<CompressedSecondaryCache>(*this);
 }

+Status CompressedSecondaryCache::Deflate(size_t decrease) {
+  return cache_res_mgr_->UpdateCacheReservation(decrease, /*increase=*/true);
+}
+
+Status CompressedSecondaryCache::Inflate(size_t increase) {
+  return cache_res_mgr_->UpdateCacheReservation(increase, /*increase=*/false);
+}
+
 }  // namespace ROCKSDB_NAMESPACE
@@ -9,6 +9,7 @@
 #include <cstddef>
 #include <memory>

+#include "cache/cache_reservation_manager.h"
 #include "cache/lru_cache.h"
 #include "memory/memory_allocator_impl.h"
 #include "rocksdb/secondary_cache.h"
@@ -76,7 +77,11 @@ class CompressedSecondaryCache : public SecondaryCache {
  const char* Name() const override { return "CompressedSecondaryCache"; }

  Status Insert(const Slice& key, Cache::ObjectPtr value,
-                const Cache::CacheItemHelper* helper) override;
+                const Cache::CacheItemHelper* helper,
+                bool force_insert) override;
+
+  Status InsertSaved(const Slice& key, const Slice& saved, CompressionType type,
+                     CacheTier source) override;

  std::unique_ptr<SecondaryCacheResultHandle> Lookup(
      const Slice& key, const Cache::CacheItemHelper* helper,
@@ -93,8 +98,14 @@ class CompressedSecondaryCache : public SecondaryCache {

  Status GetCapacity(size_t& capacity) override;

+  Status Deflate(size_t decrease) override;
+
+  Status Inflate(size_t increase) override;
+
  std::string GetPrintableOptions() const override;

+  size_t TEST_GetUsage() { return cache_->GetUsage(); }
+
 private:
  friend class CompressedSecondaryCacheTestBase;
  static constexpr std::array<uint16_t, 8> malloc_bin_sizes_{
@@ -122,11 +133,19 @@ class CompressedSecondaryCache : public SecondaryCache {
  CacheAllocationPtr MergeChunksIntoValue(const void* chunks_head,
                                          size_t& charge);

+  bool MaybeInsertDummy(const Slice& key);
+
+  Status InsertInternal(const Slice& key, Cache::ObjectPtr value,
+                        const Cache::CacheItemHelper* helper,
+                        CompressionType type, CacheTier source);
+
  // TODO: clean up to use cleaner interfaces in typed_cache.h
  const Cache::CacheItemHelper* GetHelper(bool enable_custom_split_merge) const;
  std::shared_ptr<Cache> cache_;
  CompressedSecondaryCacheOptions cache_options_;
  mutable port::Mutex capacity_mutex_;
+  std::shared_ptr<ConcurrentCacheReservationManager> cache_res_mgr_;
+  bool disable_cache_;
 };

 }  // namespace ROCKSDB_NAMESPACE
@@ -10,11 +10,14 @@
 #include <memory>
 #include <tuple>

+#include "cache/secondary_cache_adapter.h"
 #include "memory/jemalloc_nodump_allocator.h"
+#include "rocksdb/cache.h"
 #include "rocksdb/convenience.h"
 #include "test_util/secondary_cache_test_util.h"
 #include "test_util/testharness.h"
 #include "test_util/testutil.h"
+#include "util/cast_util.h"

 namespace ROCKSDB_NAMESPACE {

@@ -49,7 +52,7 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
    std::string str1(rnd.RandomString(1000));
    TestItem item1(str1.data(), str1.length());
    // A dummy handle is inserted if the item is inserted for the first time.
-    ASSERT_OK(sec_cache->Insert(key1, &item1, GetHelper()));
+    ASSERT_OK(sec_cache->Insert(key1, &item1, GetHelper(), false));
    ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 1);
    ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, 0);
    ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, 0);
@@ -60,7 +63,7 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
    ASSERT_EQ(handle1_1, nullptr);

    // Insert and Lookup the item k1 for the second time and advise erasing it.
-    ASSERT_OK(sec_cache->Insert(key1, &item1, GetHelper()));
+    ASSERT_OK(sec_cache->Insert(key1, &item1, GetHelper(), false));
    ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 1);

    std::unique_ptr<SecondaryCacheResultHandle> handle1_2 =
@@ -92,14 +95,14 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
    // Insert and Lookup the item k2.
    std::string str2(rnd.RandomString(1000));
    TestItem item2(str2.data(), str2.length());
-    ASSERT_OK(sec_cache->Insert(key2, &item2, GetHelper()));
+    ASSERT_OK(sec_cache->Insert(key2, &item2, GetHelper(), false));
    ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 2);
    std::unique_ptr<SecondaryCacheResultHandle> handle2_1 =
        sec_cache->Lookup(key2, GetHelper(), this, true, /*advise_erase=*/false,
                          kept_in_sec_cache);
    ASSERT_EQ(handle2_1, nullptr);

-    ASSERT_OK(sec_cache->Insert(key2, &item2, GetHelper()));
+    ASSERT_OK(sec_cache->Insert(key2, &item2, GetHelper(), false));
    ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 2);
    if (sec_cache_is_compressed) {
      ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes,
@@ -181,15 +184,15 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
    std::string str1(rnd.RandomString(1000));
    TestItem item1(str1.data(), str1.length());
    // Insert a dummy handle.
-    ASSERT_OK(sec_cache->Insert(key1, &item1, GetHelper()));
+    ASSERT_OK(sec_cache->Insert(key1, &item1, GetHelper(), false));
    // Insert k1.
-    ASSERT_OK(sec_cache->Insert(key1, &item1, GetHelper()));
+    ASSERT_OK(sec_cache->Insert(key1, &item1, GetHelper(), false));

    // Insert and Lookup the second item.
    std::string str2(rnd.RandomString(200));
    TestItem item2(str2.data(), str2.length());
    // Insert a dummy handle, k1 is not evicted.
-    ASSERT_OK(sec_cache->Insert(key2, &item2, GetHelper()));
+    ASSERT_OK(sec_cache->Insert(key2, &item2, GetHelper(), false));
    bool kept_in_sec_cache{false};
    std::unique_ptr<SecondaryCacheResultHandle> handle1 =
        sec_cache->Lookup(key1, GetHelper(), this, true, /*advise_erase=*/false,
@@ -197,7 +200,7 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
    ASSERT_EQ(handle1, nullptr);

    // Insert k2 and k1 is evicted.
-    ASSERT_OK(sec_cache->Insert(key2, &item2, GetHelper()));
+    ASSERT_OK(sec_cache->Insert(key2, &item2, GetHelper(), false));
    std::unique_ptr<SecondaryCacheResultHandle> handle2 =
        sec_cache->Lookup(key2, GetHelper(), this, true, /*advise_erase=*/false,
                          kept_in_sec_cache);
@@ -208,7 +211,7 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
    ASSERT_EQ(memcmp(val2->Buf(), item2.Buf(), item2.Size()), 0);

    // Insert k1 again and a dummy handle is inserted.
-    ASSERT_OK(sec_cache->Insert(key1, &item1, GetHelper()));
+    ASSERT_OK(sec_cache->Insert(key1, &item1, GetHelper(), false));

    std::unique_ptr<SecondaryCacheResultHandle> handle1_1 =
        sec_cache->Lookup(key1, GetHelper(), this, true, /*advise_erase=*/false,
@@ -226,8 +229,8 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
    std::string str3 = rnd.RandomString(10);
    TestItem item3(str3.data(), str3.length());
    // The Status is OK because a dummy handle is inserted.
-    ASSERT_OK(sec_cache->Insert(key3, &item3, GetHelperFail()));
-    ASSERT_NOK(sec_cache->Insert(key3, &item3, GetHelperFail()));
+    ASSERT_OK(sec_cache->Insert(key3, &item3, GetHelperFail(), false));
+    ASSERT_NOK(sec_cache->Insert(key3, &item3, GetHelperFail(), false));

    sec_cache.reset();
  }
@@ -735,7 +738,7 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
 class CompressedSecondaryCacheTest
    : public CompressedSecondaryCacheTestBase,
      public testing::WithParamInterface<std::string> {
-  const std::string& Type() override { return GetParam(); }
+  const std::string& Type() const override { return GetParam(); }
 };

 INSTANTIATE_TEST_CASE_P(CompressedSecondaryCacheTest,
@@ -750,7 +753,7 @@ class CompressedSecCacheTestWithCompressAndAllocatorParam
    sec_cache_is_compressed_ = std::get<0>(GetParam());
    use_jemalloc_ = std::get<1>(GetParam());
  }
-  const std::string& Type() override { return std::get<2>(GetParam()); }
+  const std::string& Type() const override { return std::get<2>(GetParam()); }
  bool sec_cache_is_compressed_;
  bool use_jemalloc_;
 };
@@ -771,7 +774,7 @@ class CompressedSecondaryCacheTestWithCompressionParam
  CompressedSecondaryCacheTestWithCompressionParam() {
    sec_cache_is_compressed_ = std::get<0>(GetParam());
  }
-  const std::string& Type() override { return std::get<1>(GetParam()); }
+  const std::string& Type() const override { return std::get<1>(GetParam()); }
  bool sec_cache_is_compressed_;
 };

@@ -902,10 +905,10 @@ TEST_P(CompressedSecondaryCacheTestWithCompressionParam, EntryRoles) {
    Slice ith_key = Slice(junk.data(), 16);

    get_perf_context()->Reset();
-    ASSERT_OK(sec_cache->Insert(ith_key, &item, GetHelper(role)));
+    ASSERT_OK(sec_cache->Insert(ith_key, &item, GetHelper(role), false));
    ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 1U);

-    ASSERT_OK(sec_cache->Insert(ith_key, &item, GetHelper(role)));
+    ASSERT_OK(sec_cache->Insert(ith_key, &item, GetHelper(role), false));
    ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 1U);

    bool kept_in_sec_cache{true};
@@ -948,7 +951,7 @@ class CompressedSecCacheTestWithCompressAndSplitParam
    sec_cache_is_compressed_ = std::get<0>(GetParam());
    enable_custom_split_merge_ = std::get<1>(GetParam());
  }
-  const std::string& Type() override { return std::get<2>(GetParam()); }
+  const std::string& Type() const override { return std::get<2>(GetParam()); }
  bool sec_cache_is_compressed_;
  bool enable_custom_split_merge_;
 };
@@ -974,6 +977,377 @@ TEST_P(CompressedSecondaryCacheTest, SplictValueAndMergeChunksTest) {
  SplictValueAndMergeChunksTest();
 }

+using secondary_cache_test_util::WithCacheType;
+
+class CompressedSecCacheTestWithTiered
+    : public testing::Test,
+      public WithCacheType,
+      public testing::WithParamInterface<
+          std::tuple<PrimaryCacheType, TieredAdmissionPolicy>> {
+ public:
+  using secondary_cache_test_util::WithCacheType::TestItem;
+  CompressedSecCacheTestWithTiered() {
+    LRUCacheOptions lru_opts;
+    HyperClockCacheOptions hcc_opts(
+        /*_capacity=*/0,
+        /*_estimated_entry_charge=*/256 << 10,
+        /*_num_shard_bits=*/0);
+    TieredCacheOptions opts;
+    lru_opts.capacity = 0;
+    lru_opts.num_shard_bits = 0;
+    lru_opts.high_pri_pool_ratio = 0;
+    opts.cache_type = std::get<0>(GetParam());
+    if (opts.cache_type == PrimaryCacheType::kCacheTypeLRU) {
+      opts.cache_opts = &lru_opts;
+    } else {
+      opts.cache_opts = &hcc_opts;
+    }
+    opts.adm_policy = std::get<1>(GetParam());
+    ;
+    opts.comp_cache_opts.capacity = 0;
+    opts.comp_cache_opts.num_shard_bits = 0;
+    opts.total_capacity = 100 << 20;
+    opts.compressed_secondary_ratio = 0.3;
+    cache_ = NewTieredCache(opts);
+    cache_res_mgr_ =
+        std::make_shared<CacheReservationManagerImpl<CacheEntryRole::kMisc>>(
+            cache_);
+  }
+
+  const std::string& Type() const override {
+    if (std::get<0>(GetParam()) == PrimaryCacheType::kCacheTypeLRU) {
+      return lru_str;
+    } else {
+      return hcc_str;
+    }
+  }
+
+ protected:
+  CacheReservationManager* cache_res_mgr() { return cache_res_mgr_.get(); }
+
+  std::shared_ptr<Cache> GetTieredCache() { return cache_; }
+
+  Cache* GetCache() {
+    return static_cast_with_check<CacheWithSecondaryAdapter, Cache>(
+               cache_.get())
+        ->TEST_GetCache();
+  }
+
+  SecondaryCache* GetSecondaryCache() {
+    return static_cast_with_check<CacheWithSecondaryAdapter, Cache>(
+               cache_.get())
+        ->TEST_GetSecondaryCache();
+  }
+
+  size_t GetPercent(size_t val, unsigned int percent) {
+    return static_cast<size_t>(val * percent / 100);
+  }
+
+ private:
+  std::shared_ptr<Cache> cache_;
+  std::shared_ptr<CacheReservationManager> cache_res_mgr_;
+  static std::string lru_str;
+  static std::string hcc_str;
+};
+
+std::string CompressedSecCacheTestWithTiered::lru_str(WithCacheType::kLRU);
+std::string CompressedSecCacheTestWithTiered::hcc_str(
+    WithCacheType::kFixedHyperClock);
+
+bool CacheUsageWithinBounds(size_t val1, size_t val2, size_t error) {
+  return ((val1 < (val2 + error)) && (val1 > (val2 - error)));
+}
+
+TEST_P(CompressedSecCacheTestWithTiered, CacheReservationManager) {
+  CompressedSecondaryCache* sec_cache =
+      reinterpret_cast<CompressedSecondaryCache*>(GetSecondaryCache());
+
+  // Use EXPECT_PRED3 instead of EXPECT_NEAR to void too many size_t to
+  // double explicit casts
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (30 << 20),
+               GetPercent(30 << 20, 1));
+  EXPECT_EQ(sec_cache->TEST_GetUsage(), 0);
+
+  ASSERT_OK(cache_res_mgr()->UpdateCacheReservation(10 << 20));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (37 << 20),
+               GetPercent(37 << 20, 1));
+  EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (3 << 20),
+               GetPercent(3 << 20, 1));
+
+  ASSERT_OK(cache_res_mgr()->UpdateCacheReservation(0));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (30 << 20),
+               GetPercent(30 << 20, 1));
+  EXPECT_EQ(sec_cache->TEST_GetUsage(), 0);
+}
+
+TEST_P(CompressedSecCacheTestWithTiered,
+       CacheReservationManagerMultipleUpdate) {
+  CompressedSecondaryCache* sec_cache =
+      reinterpret_cast<CompressedSecondaryCache*>(GetSecondaryCache());
+
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (30 << 20),
+               GetPercent(30 << 20, 1));
+  EXPECT_EQ(sec_cache->TEST_GetUsage(), 0);
+
+  int i;
+  for (i = 0; i < 10; ++i) {
+    ASSERT_OK(cache_res_mgr()->UpdateCacheReservation((1 + i) << 20));
+  }
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (37 << 20),
+               GetPercent(37 << 20, 1));
+  EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (3 << 20),
+               GetPercent(3 << 20, 1));
+
+  for (i = 10; i > 0; --i) {
+    ASSERT_OK(cache_res_mgr()->UpdateCacheReservation(((i - 1) << 20)));
+  }
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (30 << 20),
+               GetPercent(30 << 20, 1));
+  EXPECT_EQ(sec_cache->TEST_GetUsage(), 0);
+}
+
+TEST_P(CompressedSecCacheTestWithTiered, AdmissionPolicy) {
+  if (!LZ4_Supported()) {
+    ROCKSDB_GTEST_BYPASS("This test requires LZ4 support\n");
+    return;
+  }
+
+  Cache* tiered_cache = GetTieredCache().get();
+  Cache* cache = GetCache();
+  std::vector<CacheKey> keys;
+  std::vector<std::string> vals;
+  // Make the item size slightly less than 10MB to ensure we can fit the
+  // expected number of items in the cache
+  int item_size = (10 << 20) - (1 << 18);
+  int i;
+  Random rnd(301);
+  for (i = 0; i < 14; ++i) {
+    keys.emplace_back(CacheKey::CreateUniqueForCacheLifetime(cache));
+    vals.emplace_back(rnd.RandomString(item_size));
+  }
+
+  for (i = 0; i < 7; ++i) {
+    TestItem* item = new TestItem(vals[i].data(), vals[i].length());
+    ASSERT_OK(tiered_cache->Insert(keys[i].AsSlice(), item, GetHelper(),
+                                   vals[i].length()));
+  }
+
+  Cache::Handle* handle1;
+  handle1 = tiered_cache->Lookup(keys[0].AsSlice(), GetHelper(),
+                                 /*context*/ this, Cache::Priority::LOW);
+  ASSERT_NE(handle1, nullptr);
+  Cache::Handle* handle2;
+  handle2 = tiered_cache->Lookup(keys[1].AsSlice(), GetHelper(),
+                                 /*context*/ this, Cache::Priority::LOW);
+  ASSERT_NE(handle2, nullptr);
+  tiered_cache->Release(handle1);
+  tiered_cache->Release(handle2);
+
+  // Flush all previous entries out of the primary cache
+  for (i = 7; i < 14; ++i) {
+    TestItem* item = new TestItem(vals[i].data(), vals[i].length());
+    ASSERT_OK(tiered_cache->Insert(keys[i].AsSlice(), item, GetHelper(),
+                                   vals[i].length()));
+  }
+  // keys 0 and 1 should be found as they had the hit bit set
+  handle1 = tiered_cache->Lookup(keys[0].AsSlice(), GetHelper(),
+                                 /*context*/ this, Cache::Priority::LOW);
+  ASSERT_NE(handle1, nullptr);
+  handle2 = tiered_cache->Lookup(keys[1].AsSlice(), GetHelper(),
+                                 /*context*/ this, Cache::Priority::LOW);
+  ASSERT_NE(handle2, nullptr);
+  tiered_cache->Release(handle1);
+  tiered_cache->Release(handle2);
+
+  handle1 = tiered_cache->Lookup(keys[2].AsSlice(), GetHelper(),
+                                 /*context*/ this, Cache::Priority::LOW);
+  ASSERT_EQ(handle1, nullptr);
+  handle1 = tiered_cache->Lookup(keys[3].AsSlice(), GetHelper(),
+                                 /*context*/ this, Cache::Priority::LOW);
+  ASSERT_EQ(handle1, nullptr);
+}
+
+TEST_P(CompressedSecCacheTestWithTiered, DynamicUpdate) {
+  CompressedSecondaryCache* sec_cache =
+      reinterpret_cast<CompressedSecondaryCache*>(GetSecondaryCache());
+  std::shared_ptr<Cache> tiered_cache = GetTieredCache();
+
+  // Use EXPECT_PRED3 instead of EXPECT_NEAR to void too many size_t to
+  // double explicit casts
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (30 << 20),
+               GetPercent(30 << 20, 1));
+  size_t sec_capacity;
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (30 << 20));
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, 130 << 20));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (39 << 20),
+               GetPercent(39 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (39 << 20));
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, 70 << 20));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (21 << 20),
+               GetPercent(21 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (21 << 20));
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, 100 << 20));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (30 << 20),
+               GetPercent(30 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (30 << 20));
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 0.4));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (40 << 20),
+               GetPercent(40 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (40 << 20));
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 0.2));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (20 << 20),
+               GetPercent(20 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (20 << 20));
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 1.0));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (100 << 20),
+               GetPercent(100 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, 100 << 20);
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 0.0));
+  // Only check usage for LRU cache. HCC shows a 64KB usage for some reason
+  if (std::get<0>(GetParam()) == PrimaryCacheType::kCacheTypeLRU) {
+    ASSERT_EQ(GetCache()->GetUsage(), 0);
+  }
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, 0);
+
+  ASSERT_NOK(UpdateTieredCache(tiered_cache, -1, 0.3));
+  // Only check usage for LRU cache. HCC shows a 64KB usage for some reason
+  if (std::get<0>(GetParam()) == PrimaryCacheType::kCacheTypeLRU) {
+    ASSERT_EQ(GetCache()->GetUsage(), 0);
+  }
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, 0);
+}
+
+TEST_P(CompressedSecCacheTestWithTiered, DynamicUpdateWithReservation) {
+  CompressedSecondaryCache* sec_cache =
+      reinterpret_cast<CompressedSecondaryCache*>(GetSecondaryCache());
+  std::shared_ptr<Cache> tiered_cache = GetTieredCache();
+
+  ASSERT_OK(cache_res_mgr()->UpdateCacheReservation(10 << 20));
+  // Use EXPECT_PRED3 instead of EXPECT_NEAR to void too many size_t to
+  // double explicit casts
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (37 << 20),
+               GetPercent(37 << 20, 1));
+  EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (3 << 20),
+               GetPercent(3 << 20, 1));
+  size_t sec_capacity;
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (30 << 20));
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, 70 << 20));
+  // Only check usage for LRU cache. HCC is slightly off for some reason
+  if (std::get<0>(GetParam()) == PrimaryCacheType::kCacheTypeLRU) {
+    EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (28 << 20),
+                 GetPercent(28 << 20, 1));
+  }
+  EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (3 << 20),
+               GetPercent(3 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (21 << 20));
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, 130 << 20));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (46 << 20),
+               GetPercent(46 << 20, 1));
+  EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (3 << 20),
+               GetPercent(3 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (39 << 20));
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, 100 << 20));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (37 << 20),
+               GetPercent(37 << 20, 1));
+  EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (3 << 20),
+               GetPercent(3 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (30 << 20));
+
+  ASSERT_OK(tiered_cache->GetSecondaryCacheCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, 30 << 20);
+  size_t sec_usage;
+  ASSERT_OK(tiered_cache->GetSecondaryCachePinnedUsage(sec_usage));
+  EXPECT_PRED3(CacheUsageWithinBounds, sec_usage, 3 << 20,
+               GetPercent(3 << 20, 1));
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 0.39));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (45 << 20),
+               GetPercent(45 << 20, 1));
+  EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (4 << 20),
+               GetPercent(4 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (39 << 20));
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 0.2));
+  // Only check usage for LRU cache. HCC is slightly off for some reason
+  if (std::get<0>(GetParam()) == PrimaryCacheType::kCacheTypeLRU) {
+    EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (28 << 20),
+                 GetPercent(28 << 20, 1));
+  }
+  EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (2 << 20),
+               GetPercent(2 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (20 << 20));
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 1.0));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (100 << 20),
+               GetPercent(100 << 20, 1));
+  EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (10 << 20),
+               GetPercent(10 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, 100 << 20);
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 0.0));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (10 << 20),
+               GetPercent(10 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, 0);
+
+  ASSERT_OK(cache_res_mgr()->UpdateCacheReservation(0));
+}
+
+TEST_P(CompressedSecCacheTestWithTiered,
+       DynamicUpdateWithReservationUnderflow) {
+  std::shared_ptr<Cache> tiered_cache = GetTieredCache();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
+      {{"CacheWithSecondaryAdapter::Release:ChargeSecCache1",
+        "CacheWithSecondaryAdapter::UpdateCacheReservationRatio:Begin"},
+       {"CacheWithSecondaryAdapter::UpdateCacheReservationRatio:End",
+        "CacheWithSecondaryAdapter::Release:ChargeSecCache2"}});
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  port::Thread reserve_release_thread([&]() {
+    EXPECT_EQ(cache_res_mgr()->UpdateCacheReservation(50), Status::OK());
+    EXPECT_EQ(cache_res_mgr()->UpdateCacheReservation(0), Status::OK());
+  });
+  ASSERT_OK(UpdateTieredCache(tiered_cache, 100 << 20, 0.01));
+  reserve_release_thread.join();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, 100 << 20, 0.3));
+}
+
+INSTANTIATE_TEST_CASE_P(
+    CompressedSecCacheTests, CompressedSecCacheTestWithTiered,
+    ::testing::Values(
+        std::make_tuple(PrimaryCacheType::kCacheTypeLRU,
+                        TieredAdmissionPolicy::kAdmPolicyAllowCacheHits),
+        std::make_tuple(PrimaryCacheType::kCacheTypeHCC,
+                        TieredAdmissionPolicy::kAdmPolicyAllowCacheHits)));
+
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -340,7 +340,8 @@ void LRUCacheShard::NotifyEvicted(
  for (LRUHandle* entry : evicted_handles) {
    if (eviction_callback_ &&
        eviction_callback_(entry->key(),
-                           reinterpret_cast<Cache::Handle*>(entry))) {
+                           reinterpret_cast<Cache::Handle*>(entry),
+                           entry->HasHit())) {
      // Callback took ownership of obj; just free handle
      free(entry);
    } else {
@@ -505,7 +506,8 @@ bool LRUCacheShard::Release(LRUHandle* e, bool /*useful*/,
    // Only call eviction callback if we're sure no one requested erasure
    // FIXME: disabled because of test churn
    if (false && was_in_cache && !erase_if_last_ref && eviction_callback_ &&
-        eviction_callback_(e->key(), reinterpret_cast<Cache::Handle*>(e))) {
+        eviction_callback_(e->key(), reinterpret_cast<Cache::Handle*>(e),
+                           e->HasHit())) {
      // Callback took ownership of obj; just free handle
      free(e);
    } else {
@@ -712,12 +714,12 @@ std::shared_ptr<Cache> LRUCacheOptions::MakeSharedCache() const {
  return cache;
 }

-std::shared_ptr<GeneralCache> LRUCacheOptions::MakeSharedGeneralCache() const {
+std::shared_ptr<RowCache> LRUCacheOptions::MakeSharedRowCache() const {
  if (secondary_cache) {
-    // Not allowed for a GeneralCache
+    // Not allowed for a RowCache
    return nullptr;
  }
-  // Works while GeneralCache is an alias for Cache
+  // Works while RowCache is an alias for Cache
  return MakeSharedCache();
 }
 }  // namespace ROCKSDB_NAMESPACE
@@ -371,11 +371,12 @@ TEST_F(LRUCacheTest, EntriesWithPriority) {

 namespace clock_cache {

+template <class ClockCache>
 class ClockCacheTest : public testing::Test {
 public:
-  using Shard = HyperClockCache::Shard;
-  using Table = HyperClockTable;
-  using HandleImpl = Shard::HandleImpl;
+  using Shard = typename ClockCache::Shard;
+  using Table = typename Shard::Table;
+  using TableOpts = typename Table::Opts;

  ClockCacheTest() {}
  ~ClockCacheTest() override { DeleteShard(); }
@@ -393,8 +394,7 @@ class ClockCacheTest : public testing::Test {
    shard_ =
        reinterpret_cast<Shard*>(port::cacheline_aligned_alloc(sizeof(Shard)));

-    Table::Opts opts;
-    opts.estimated_value_size = 1;
+    TableOpts opts{1 /*value_size*/};
    new (shard_)
        Shard(capacity, strict_capacity_limit, kDontChargeCacheMetadata,
              /*allocator*/ nullptr, &eviction_callback_, &hash_seed_, opts);
@@ -458,43 +458,53 @@ class ClockCacheTest : public testing::Test {
  uint32_t hash_seed_ = 0;
 };

-TEST_F(ClockCacheTest, Misc) {
-  NewShard(3);
+using ClockCacheTypes =
+    ::testing::Types<AutoHyperClockCache, FixedHyperClockCache>;
+TYPED_TEST_CASE(ClockCacheTest, ClockCacheTypes);
+
+TYPED_TEST(ClockCacheTest, Misc) {
+  this->NewShard(3);
+  // NOTE: templated base class prevents simple naming of inherited members,
+  // so lots of `this->`
+  auto& shard = *this->shard_;

  // Key size stuff
-  EXPECT_OK(InsertWithLen('a', 16));
-  EXPECT_NOK(InsertWithLen('b', 15));
-  EXPECT_OK(InsertWithLen('b', 16));
-  EXPECT_NOK(InsertWithLen('c', 17));
-  EXPECT_NOK(InsertWithLen('d', 1000));
-  EXPECT_NOK(InsertWithLen('e', 11));
-  EXPECT_NOK(InsertWithLen('f', 0));
+  EXPECT_OK(this->InsertWithLen('a', 16));
+  EXPECT_NOK(this->InsertWithLen('b', 15));
+  EXPECT_OK(this->InsertWithLen('b', 16));
+  EXPECT_NOK(this->InsertWithLen('c', 17));
+  EXPECT_NOK(this->InsertWithLen('d', 1000));
+  EXPECT_NOK(this->InsertWithLen('e', 11));
+  EXPECT_NOK(this->InsertWithLen('f', 0));

  // Some of this is motivated by code coverage
  std::string wrong_size_key(15, 'x');
-  EXPECT_FALSE(Lookup(wrong_size_key, TestHashedKey('x')));
-  EXPECT_FALSE(shard_->Ref(nullptr));
-  EXPECT_FALSE(shard_->Release(nullptr));
-  shard_->Erase(wrong_size_key, TestHashedKey('x'));  // no-op
+  EXPECT_FALSE(this->Lookup(wrong_size_key, this->TestHashedKey('x')));
+  EXPECT_FALSE(shard.Ref(nullptr));
+  EXPECT_FALSE(shard.Release(nullptr));
+  shard.Erase(wrong_size_key, this->TestHashedKey('x'));  // no-op
 }

-TEST_F(ClockCacheTest, Limits) {
-  constexpr size_t kCapacity = 3;
-  NewShard(kCapacity, false /*strict_capacity_limit*/);
+TYPED_TEST(ClockCacheTest, Limits) {
+  constexpr size_t kCapacity = 64;
+  this->NewShard(kCapacity, false /*strict_capacity_limit*/);
+  auto& shard = *this->shard_;
+  using HandleImpl = typename ClockCacheTest<TypeParam>::Shard::HandleImpl;
+
  for (bool strict_capacity_limit : {false, true, false}) {
    SCOPED_TRACE("strict_capacity_limit = " +
                 std::to_string(strict_capacity_limit));

    // Also tests switching between strict limit and not
-    shard_->SetStrictCapacityLimit(strict_capacity_limit);
+    shard.SetStrictCapacityLimit(strict_capacity_limit);

-    UniqueId64x2 hkey = TestHashedKey('x');
+    UniqueId64x2 hkey = this->TestHashedKey('x');

    // Single entry charge beyond capacity
    {
-      Status s = shard_->Insert(TestKey(hkey), hkey, nullptr /*value*/,
-                                &kNoopCacheItemHelper, 5 /*charge*/,
-                                nullptr /*handle*/, Cache::Priority::LOW);
+      Status s = shard.Insert(this->TestKey(hkey), hkey, nullptr /*value*/,
+                              &kNoopCacheItemHelper, kCapacity + 2 /*charge*/,
+                              nullptr /*handle*/, Cache::Priority::LOW);
      if (strict_capacity_limit) {
        EXPECT_TRUE(s.IsMemoryLimit());
      } else {
@@ -505,11 +515,11 @@ TEST_F(ClockCacheTest, Limits) {
    // Single entry fills capacity
    {
      HandleImpl* h;
-      ASSERT_OK(shard_->Insert(TestKey(hkey), hkey, nullptr /*value*/,
-                               &kNoopCacheItemHelper, 3 /*charge*/, &h,
-                               Cache::Priority::LOW));
+      ASSERT_OK(shard.Insert(this->TestKey(hkey), hkey, nullptr /*value*/,
+                             &kNoopCacheItemHelper, kCapacity /*charge*/, &h,
+                             Cache::Priority::LOW));
      // Try to insert more
-      Status s = Insert('a');
+      Status s = this->Insert('a');
      if (strict_capacity_limit) {
        EXPECT_TRUE(s.IsMemoryLimit());
      } else {
@@ -517,22 +527,22 @@ TEST_F(ClockCacheTest, Limits) {
      }
      // Release entry filling capacity.
      // Cover useful = false case.
-      shard_->Release(h, false /*useful*/, false /*erase_if_last_ref*/);
+      shard.Release(h, false /*useful*/, false /*erase_if_last_ref*/);
    }

    // Insert more than table size can handle to exceed occupancy limit.
    // (Cleverly using mostly zero-charge entries, but some non-zero to
    // verify usage tracking on detached entries.)
    {
-      size_t n = shard_->GetTableAddressCount() + 1;
+      size_t n = kCapacity * 5 + 1;
      std::unique_ptr<HandleImpl* []> ha { new HandleImpl* [n] {} };
      Status s;
      for (size_t i = 0; i < n && s.ok(); ++i) {
        hkey[1] = i;
-        s = shard_->Insert(TestKey(hkey), hkey, nullptr /*value*/,
-                           &kNoopCacheItemHelper,
-                           (i + kCapacity < n) ? 0 : 1 /*charge*/, &ha[i],
-                           Cache::Priority::LOW);
+        s = shard.Insert(this->TestKey(hkey), hkey, nullptr /*value*/,
+                         &kNoopCacheItemHelper,
+                         (i + kCapacity < n) ? 0 : 1 /*charge*/, &ha[i],
+                         Cache::Priority::LOW);
        if (i == 0) {
          EXPECT_OK(s);
        }
@@ -543,130 +553,133 @@ TEST_F(ClockCacheTest, Limits) {
        EXPECT_OK(s);
      }
      // Same result if not keeping a reference
-      s = Insert('a');
+      s = this->Insert('a');
      if (strict_capacity_limit) {
        EXPECT_TRUE(s.IsMemoryLimit());
      } else {
        EXPECT_OK(s);
      }

+      EXPECT_EQ(shard.GetOccupancyCount(), shard.GetOccupancyLimit());
+
      // Regardless, we didn't allow table to actually get full
-      EXPECT_LT(shard_->GetOccupancyCount(), shard_->GetTableAddressCount());
+      EXPECT_LT(shard.GetOccupancyCount(), shard.GetTableAddressCount());

      // Release handles
      for (size_t i = 0; i < n; ++i) {
        if (ha[i]) {
-          shard_->Release(ha[i]);
+          shard.Release(ha[i]);
        }
      }
    }
  }
 }

-TEST_F(ClockCacheTest, ClockEvictionTest) {
+TYPED_TEST(ClockCacheTest, ClockEvictionTest) {
  for (bool strict_capacity_limit : {false, true}) {
    SCOPED_TRACE("strict_capacity_limit = " +
                 std::to_string(strict_capacity_limit));

-    NewShard(6, strict_capacity_limit);
-    EXPECT_OK(Insert('a', Cache::Priority::BOTTOM));
-    EXPECT_OK(Insert('b', Cache::Priority::LOW));
-    EXPECT_OK(Insert('c', Cache::Priority::HIGH));
-    EXPECT_OK(Insert('d', Cache::Priority::BOTTOM));
-    EXPECT_OK(Insert('e', Cache::Priority::LOW));
-    EXPECT_OK(Insert('f', Cache::Priority::HIGH));
+    this->NewShard(6, strict_capacity_limit);
+    auto& shard = *this->shard_;
+    EXPECT_OK(this->Insert('a', Cache::Priority::BOTTOM));
+    EXPECT_OK(this->Insert('b', Cache::Priority::LOW));
+    EXPECT_OK(this->Insert('c', Cache::Priority::HIGH));
+    EXPECT_OK(this->Insert('d', Cache::Priority::BOTTOM));
+    EXPECT_OK(this->Insert('e', Cache::Priority::LOW));
+    EXPECT_OK(this->Insert('f', Cache::Priority::HIGH));

-    EXPECT_TRUE(Lookup('a', /*use*/ false));
-    EXPECT_TRUE(Lookup('b', /*use*/ false));
-    EXPECT_TRUE(Lookup('c', /*use*/ false));
-    EXPECT_TRUE(Lookup('d', /*use*/ false));
-    EXPECT_TRUE(Lookup('e', /*use*/ false));
-    EXPECT_TRUE(Lookup('f', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('a', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('b', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('c', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('d', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('e', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('f', /*use*/ false));

    // Ensure bottom are evicted first, even if new entries are low
-    EXPECT_OK(Insert('g', Cache::Priority::LOW));
-    EXPECT_OK(Insert('h', Cache::Priority::LOW));
+    EXPECT_OK(this->Insert('g', Cache::Priority::LOW));
+    EXPECT_OK(this->Insert('h', Cache::Priority::LOW));

-    EXPECT_FALSE(Lookup('a', /*use*/ false));
-    EXPECT_TRUE(Lookup('b', /*use*/ false));
-    EXPECT_TRUE(Lookup('c', /*use*/ false));
-    EXPECT_FALSE(Lookup('d', /*use*/ false));
-    EXPECT_TRUE(Lookup('e', /*use*/ false));
-    EXPECT_TRUE(Lookup('f', /*use*/ false));
+    EXPECT_FALSE(this->Lookup('a', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('b', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('c', /*use*/ false));
+    EXPECT_FALSE(this->Lookup('d', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('e', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('f', /*use*/ false));
    // Mark g & h useful
-    EXPECT_TRUE(Lookup('g', /*use*/ true));
-    EXPECT_TRUE(Lookup('h', /*use*/ true));
+    EXPECT_TRUE(this->Lookup('g', /*use*/ true));
+    EXPECT_TRUE(this->Lookup('h', /*use*/ true));

    // Then old LOW entries
-    EXPECT_OK(Insert('i', Cache::Priority::LOW));
-    EXPECT_OK(Insert('j', Cache::Priority::LOW));
+    EXPECT_OK(this->Insert('i', Cache::Priority::LOW));
+    EXPECT_OK(this->Insert('j', Cache::Priority::LOW));

-    EXPECT_FALSE(Lookup('b', /*use*/ false));
-    EXPECT_TRUE(Lookup('c', /*use*/ false));
-    EXPECT_FALSE(Lookup('e', /*use*/ false));
-    EXPECT_TRUE(Lookup('f', /*use*/ false));
+    EXPECT_FALSE(this->Lookup('b', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('c', /*use*/ false));
+    EXPECT_FALSE(this->Lookup('e', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('f', /*use*/ false));
    // Mark g & h useful once again
-    EXPECT_TRUE(Lookup('g', /*use*/ true));
-    EXPECT_TRUE(Lookup('h', /*use*/ true));
-    EXPECT_TRUE(Lookup('i', /*use*/ false));
-    EXPECT_TRUE(Lookup('j', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('g', /*use*/ true));
+    EXPECT_TRUE(this->Lookup('h', /*use*/ true));
+    EXPECT_TRUE(this->Lookup('i', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('j', /*use*/ false));

    // Then old HIGH entries
-    EXPECT_OK(Insert('k', Cache::Priority::LOW));
-    EXPECT_OK(Insert('l', Cache::Priority::LOW));
+    EXPECT_OK(this->Insert('k', Cache::Priority::LOW));
+    EXPECT_OK(this->Insert('l', Cache::Priority::LOW));

-    EXPECT_FALSE(Lookup('c', /*use*/ false));
-    EXPECT_FALSE(Lookup('f', /*use*/ false));
-    EXPECT_TRUE(Lookup('g', /*use*/ false));
-    EXPECT_TRUE(Lookup('h', /*use*/ false));
-    EXPECT_TRUE(Lookup('i', /*use*/ false));
-    EXPECT_TRUE(Lookup('j', /*use*/ false));
-    EXPECT_TRUE(Lookup('k', /*use*/ false));
-    EXPECT_TRUE(Lookup('l', /*use*/ false));
+    EXPECT_FALSE(this->Lookup('c', /*use*/ false));
+    EXPECT_FALSE(this->Lookup('f', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('g', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('h', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('i', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('j', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('k', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('l', /*use*/ false));

    // Then the (roughly) least recently useful
-    EXPECT_OK(Insert('m', Cache::Priority::HIGH));
-    EXPECT_OK(Insert('n', Cache::Priority::HIGH));
+    EXPECT_OK(this->Insert('m', Cache::Priority::HIGH));
+    EXPECT_OK(this->Insert('n', Cache::Priority::HIGH));

-    EXPECT_TRUE(Lookup('g', /*use*/ false));
-    EXPECT_TRUE(Lookup('h', /*use*/ false));
-    EXPECT_FALSE(Lookup('i', /*use*/ false));
-    EXPECT_FALSE(Lookup('j', /*use*/ false));
-    EXPECT_TRUE(Lookup('k', /*use*/ false));
-    EXPECT_TRUE(Lookup('l', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('g', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('h', /*use*/ false));
+    EXPECT_FALSE(this->Lookup('i', /*use*/ false));
+    EXPECT_FALSE(this->Lookup('j', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('k', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('l', /*use*/ false));

    // Now try changing capacity down
-    shard_->SetCapacity(4);
+    shard.SetCapacity(4);
    // Insert to ensure evictions happen
-    EXPECT_OK(Insert('o', Cache::Priority::LOW));
-    EXPECT_OK(Insert('p', Cache::Priority::LOW));
+    EXPECT_OK(this->Insert('o', Cache::Priority::LOW));
+    EXPECT_OK(this->Insert('p', Cache::Priority::LOW));

-    EXPECT_FALSE(Lookup('g', /*use*/ false));
-    EXPECT_FALSE(Lookup('h', /*use*/ false));
-    EXPECT_FALSE(Lookup('k', /*use*/ false));
-    EXPECT_FALSE(Lookup('l', /*use*/ false));
-    EXPECT_TRUE(Lookup('m', /*use*/ false));
-    EXPECT_TRUE(Lookup('n', /*use*/ false));
-    EXPECT_TRUE(Lookup('o', /*use*/ false));
-    EXPECT_TRUE(Lookup('p', /*use*/ false));
+    EXPECT_FALSE(this->Lookup('g', /*use*/ false));
+    EXPECT_FALSE(this->Lookup('h', /*use*/ false));
+    EXPECT_FALSE(this->Lookup('k', /*use*/ false));
+    EXPECT_FALSE(this->Lookup('l', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('m', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('n', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('o', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('p', /*use*/ false));

    // Now try changing capacity up
-    EXPECT_TRUE(Lookup('m', /*use*/ true));
-    EXPECT_TRUE(Lookup('n', /*use*/ true));
-    shard_->SetCapacity(6);
-    EXPECT_OK(Insert('q', Cache::Priority::HIGH));
-    EXPECT_OK(Insert('r', Cache::Priority::HIGH));
-    EXPECT_OK(Insert('s', Cache::Priority::HIGH));
-    EXPECT_OK(Insert('t', Cache::Priority::HIGH));
+    EXPECT_TRUE(this->Lookup('m', /*use*/ true));
+    EXPECT_TRUE(this->Lookup('n', /*use*/ true));
+    shard.SetCapacity(6);
+    EXPECT_OK(this->Insert('q', Cache::Priority::HIGH));
+    EXPECT_OK(this->Insert('r', Cache::Priority::HIGH));
+    EXPECT_OK(this->Insert('s', Cache::Priority::HIGH));
+    EXPECT_OK(this->Insert('t', Cache::Priority::HIGH));

-    EXPECT_FALSE(Lookup('o', /*use*/ false));
-    EXPECT_FALSE(Lookup('p', /*use*/ false));
-    EXPECT_TRUE(Lookup('m', /*use*/ false));
-    EXPECT_TRUE(Lookup('n', /*use*/ false));
-    EXPECT_TRUE(Lookup('q', /*use*/ false));
-    EXPECT_TRUE(Lookup('r', /*use*/ false));
-    EXPECT_TRUE(Lookup('s', /*use*/ false));
-    EXPECT_TRUE(Lookup('t', /*use*/ false));
+    EXPECT_FALSE(this->Lookup('o', /*use*/ false));
+    EXPECT_FALSE(this->Lookup('p', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('m', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('n', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('q', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('r', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('s', /*use*/ false));
+    EXPECT_TRUE(this->Lookup('t', /*use*/ false));
  }
 }

@@ -682,113 +695,178 @@ const Cache::CacheItemHelper kDeleteCounterHelper{
 }  // namespace

 // Testing calls to CorrectNearOverflow in Release
-TEST_F(ClockCacheTest, ClockCounterOverflowTest) {
-  NewShard(6, /*strict_capacity_limit*/ false);
+TYPED_TEST(ClockCacheTest, ClockCounterOverflowTest) {
+  this->NewShard(6, /*strict_capacity_limit*/ false);
+  auto& shard = *this->shard_;
+  using HandleImpl = typename ClockCacheTest<TypeParam>::Shard::HandleImpl;
+
  HandleImpl* h;
  DeleteCounter val;
-  UniqueId64x2 hkey = TestHashedKey('x');
-  ASSERT_OK(shard_->Insert(TestKey(hkey), hkey, &val, &kDeleteCounterHelper, 1,
-                           &h, Cache::Priority::HIGH));
+  UniqueId64x2 hkey = this->TestHashedKey('x');
+  ASSERT_OK(shard.Insert(this->TestKey(hkey), hkey, &val, &kDeleteCounterHelper,
+                         1, &h, Cache::Priority::HIGH));

  // Some large number outstanding
-  shard_->TEST_RefN(h, 123456789);
+  shard.TEST_RefN(h, 123456789);
  // Simulate many lookup/ref + release, plenty to overflow counters
  for (int i = 0; i < 10000; ++i) {
-    shard_->TEST_RefN(h, 1234567);
-    shard_->TEST_ReleaseN(h, 1234567);
+    shard.TEST_RefN(h, 1234567);
+    shard.TEST_ReleaseN(h, 1234567);
  }
  // Mark it invisible (to reach a different CorrectNearOverflow() in Release)
-  shard_->Erase(TestKey(hkey), hkey);
+  shard.Erase(this->TestKey(hkey), hkey);
  // Simulate many more lookup/ref + release (one-by-one would be too
  // expensive for unit test)
  for (int i = 0; i < 10000; ++i) {
-    shard_->TEST_RefN(h, 1234567);
-    shard_->TEST_ReleaseN(h, 1234567);
+    shard.TEST_RefN(h, 1234567);
+    shard.TEST_ReleaseN(h, 1234567);
  }
  // Free all but last 1
-  shard_->TEST_ReleaseN(h, 123456789);
+  shard.TEST_ReleaseN(h, 123456789);
  // Still alive
  ASSERT_EQ(val.deleted, 0);
  // Free last ref, which will finalize erasure
-  shard_->Release(h);
+  shard.Release(h);
  // Deleted
  ASSERT_EQ(val.deleted, 1);
 }

+TYPED_TEST(ClockCacheTest, ClockTableFull) {
+  // Force clock cache table to fill up (not usually allowed) in order
+  // to test full probe sequence that is theoretically possible due to
+  // parallel operations
+  this->NewShard(6, /*strict_capacity_limit*/ false);
+  auto& shard = *this->shard_;
+  using HandleImpl = typename ClockCacheTest<TypeParam>::Shard::HandleImpl;
+
+  size_t size = shard.GetTableAddressCount();
+  ASSERT_LE(size + 3, 256);  // for using char keys
+  // Modify occupancy and capacity limits to attempt insert on full
+  shard.TEST_MutableOccupancyLimit() = size + 100;
+  shard.SetCapacity(size + 100);
+
+  DeleteCounter val;
+  std::vector<HandleImpl*> handles;
+  // NOTE: the three extra insertions should create standalone entries
+  for (size_t i = 0; i < size + 3; ++i) {
+    UniqueId64x2 hkey = this->TestHashedKey(static_cast<char>(i));
+    ASSERT_OK(shard.Insert(this->TestKey(hkey), hkey, &val,
+                           &kDeleteCounterHelper, 1, &handles.emplace_back(),
+                           Cache::Priority::HIGH));
+  }
+
+  for (size_t i = 0; i < size + 3; ++i) {
+    UniqueId64x2 hkey = this->TestHashedKey(static_cast<char>(i));
+    HandleImpl* h = shard.Lookup(this->TestKey(hkey), hkey);
+    if (i < size) {
+      ASSERT_NE(h, nullptr);
+      shard.Release(h);
+    } else {
+      // Standalone entries not visible by lookup
+      ASSERT_EQ(h, nullptr);
+    }
+  }
+
+  for (size_t i = 0; i < size + 3; ++i) {
+    ASSERT_NE(handles[i], nullptr);
+    shard.Release(handles[i]);
+    if (i < size) {
+      // Everything still in cache
+      ASSERT_EQ(val.deleted, 0);
+    } else {
+      // Standalone entries freed on release
+      ASSERT_EQ(val.deleted, i + 1 - size);
+    }
+  }
+
+  for (size_t i = size + 3; i > 0; --i) {
+    UniqueId64x2 hkey = this->TestHashedKey(static_cast<char>(i - 1));
+    shard.Erase(this->TestKey(hkey), hkey);
+    if (i - 1 > size) {
+      ASSERT_EQ(val.deleted, 3);
+    } else {
+      ASSERT_EQ(val.deleted, 3 + size - (i - 1));
+    }
+  }
+}
+
 // This test is mostly to exercise some corner case logic, by forcing two
 // keys to have the same hash, and more
-TEST_F(ClockCacheTest, CollidingInsertEraseTest) {
-  NewShard(6, /*strict_capacity_limit*/ false);
+TYPED_TEST(ClockCacheTest, CollidingInsertEraseTest) {
+  this->NewShard(6, /*strict_capacity_limit*/ false);
+  auto& shard = *this->shard_;
+  using HandleImpl = typename ClockCacheTest<TypeParam>::Shard::HandleImpl;
+
  DeleteCounter val;
-  UniqueId64x2 hkey1 = TestHashedKey('x');
-  Slice key1 = TestKey(hkey1);
-  UniqueId64x2 hkey2 = TestHashedKey('y');
-  Slice key2 = TestKey(hkey2);
-  UniqueId64x2 hkey3 = TestHashedKey('z');
-  Slice key3 = TestKey(hkey3);
+  UniqueId64x2 hkey1 = this->TestHashedKey('x');
+  Slice key1 = this->TestKey(hkey1);
+  UniqueId64x2 hkey2 = this->TestHashedKey('y');
+  Slice key2 = this->TestKey(hkey2);
+  UniqueId64x2 hkey3 = this->TestHashedKey('z');
+  Slice key3 = this->TestKey(hkey3);
  HandleImpl* h1;
-  ASSERT_OK(shard_->Insert(key1, hkey1, &val, &kDeleteCounterHelper, 1, &h1,
-                           Cache::Priority::HIGH));
+  ASSERT_OK(shard.Insert(key1, hkey1, &val, &kDeleteCounterHelper, 1, &h1,
+                         Cache::Priority::HIGH));
  HandleImpl* h2;
-  ASSERT_OK(shard_->Insert(key2, hkey2, &val, &kDeleteCounterHelper, 1, &h2,
-                           Cache::Priority::HIGH));
+  ASSERT_OK(shard.Insert(key2, hkey2, &val, &kDeleteCounterHelper, 1, &h2,
+                         Cache::Priority::HIGH));
  HandleImpl* h3;
-  ASSERT_OK(shard_->Insert(key3, hkey3, &val, &kDeleteCounterHelper, 1, &h3,
-                           Cache::Priority::HIGH));
+  ASSERT_OK(shard.Insert(key3, hkey3, &val, &kDeleteCounterHelper, 1, &h3,
+                         Cache::Priority::HIGH));

  // Can repeatedly lookup+release despite the hash collision
  HandleImpl* tmp_h;
  for (bool erase_if_last_ref : {true, false}) {  // but not last ref
-    tmp_h = shard_->Lookup(key1, hkey1);
+    tmp_h = shard.Lookup(key1, hkey1);
    ASSERT_EQ(h1, tmp_h);
-    ASSERT_FALSE(shard_->Release(tmp_h, erase_if_last_ref));
+    ASSERT_FALSE(shard.Release(tmp_h, erase_if_last_ref));

-    tmp_h = shard_->Lookup(key2, hkey2);
+    tmp_h = shard.Lookup(key2, hkey2);
    ASSERT_EQ(h2, tmp_h);
-    ASSERT_FALSE(shard_->Release(tmp_h, erase_if_last_ref));
+    ASSERT_FALSE(shard.Release(tmp_h, erase_if_last_ref));

-    tmp_h = shard_->Lookup(key3, hkey3);
+    tmp_h = shard.Lookup(key3, hkey3);
    ASSERT_EQ(h3, tmp_h);
-    ASSERT_FALSE(shard_->Release(tmp_h, erase_if_last_ref));
+    ASSERT_FALSE(shard.Release(tmp_h, erase_if_last_ref));
  }

  // Make h1 invisible
-  shard_->Erase(key1, hkey1);
+  shard.Erase(key1, hkey1);
  // Redundant erase
-  shard_->Erase(key1, hkey1);
+  shard.Erase(key1, hkey1);

  // All still alive
  ASSERT_EQ(val.deleted, 0);

  // Invisible to Lookup
-  tmp_h = shard_->Lookup(key1, hkey1);
+  tmp_h = shard.Lookup(key1, hkey1);
  ASSERT_EQ(nullptr, tmp_h);

  // Can still find h2, h3
  for (bool erase_if_last_ref : {true, false}) {  // but not last ref
-    tmp_h = shard_->Lookup(key2, hkey2);
+    tmp_h = shard.Lookup(key2, hkey2);
    ASSERT_EQ(h2, tmp_h);
-    ASSERT_FALSE(shard_->Release(tmp_h, erase_if_last_ref));
+    ASSERT_FALSE(shard.Release(tmp_h, erase_if_last_ref));

-    tmp_h = shard_->Lookup(key3, hkey3);
+    tmp_h = shard.Lookup(key3, hkey3);
    ASSERT_EQ(h3, tmp_h);
-    ASSERT_FALSE(shard_->Release(tmp_h, erase_if_last_ref));
+    ASSERT_FALSE(shard.Release(tmp_h, erase_if_last_ref));
  }

  // Also Insert with invisible entry there
-  ASSERT_OK(shard_->Insert(key1, hkey1, &val, &kDeleteCounterHelper, 1, nullptr,
-                           Cache::Priority::HIGH));
-  tmp_h = shard_->Lookup(key1, hkey1);
+  ASSERT_OK(shard.Insert(key1, hkey1, &val, &kDeleteCounterHelper, 1, nullptr,
+                         Cache::Priority::HIGH));
+  tmp_h = shard.Lookup(key1, hkey1);
  // Found but distinct handle
  ASSERT_NE(nullptr, tmp_h);
  ASSERT_NE(h1, tmp_h);
-  ASSERT_TRUE(shard_->Release(tmp_h, /*erase_if_last_ref*/ true));
+  ASSERT_TRUE(shard.Release(tmp_h, /*erase_if_last_ref*/ true));

  // tmp_h deleted
  ASSERT_EQ(val.deleted--, 1);

  // Release last ref on h1 (already invisible)
-  ASSERT_TRUE(shard_->Release(h1, /*erase_if_last_ref*/ false));
+  ASSERT_TRUE(shard.Release(h1, /*erase_if_last_ref*/ false));

  // h1 deleted
  ASSERT_EQ(val.deleted--, 1);
@@ -796,57 +874,57 @@ TEST_F(ClockCacheTest, CollidingInsertEraseTest) {

  // Can still find h2, h3
  for (bool erase_if_last_ref : {true, false}) {  // but not last ref
-    tmp_h = shard_->Lookup(key2, hkey2);
+    tmp_h = shard.Lookup(key2, hkey2);
    ASSERT_EQ(h2, tmp_h);
-    ASSERT_FALSE(shard_->Release(tmp_h, erase_if_last_ref));
+    ASSERT_FALSE(shard.Release(tmp_h, erase_if_last_ref));

-    tmp_h = shard_->Lookup(key3, hkey3);
+    tmp_h = shard.Lookup(key3, hkey3);
    ASSERT_EQ(h3, tmp_h);
-    ASSERT_FALSE(shard_->Release(tmp_h, erase_if_last_ref));
+    ASSERT_FALSE(shard.Release(tmp_h, erase_if_last_ref));
  }

  // Release last ref on h2
-  ASSERT_FALSE(shard_->Release(h2, /*erase_if_last_ref*/ false));
+  ASSERT_FALSE(shard.Release(h2, /*erase_if_last_ref*/ false));

  // h2 still not deleted (unreferenced in cache)
  ASSERT_EQ(val.deleted, 0);

  // Can still find it
-  tmp_h = shard_->Lookup(key2, hkey2);
+  tmp_h = shard.Lookup(key2, hkey2);
  ASSERT_EQ(h2, tmp_h);

  // Release last ref on h2, with erase
-  ASSERT_TRUE(shard_->Release(h2, /*erase_if_last_ref*/ true));
+  ASSERT_TRUE(shard.Release(h2, /*erase_if_last_ref*/ true));

  // h2 deleted
  ASSERT_EQ(val.deleted--, 1);
-  tmp_h = shard_->Lookup(key2, hkey2);
+  tmp_h = shard.Lookup(key2, hkey2);
  ASSERT_EQ(nullptr, tmp_h);

  // Can still find h3
  for (bool erase_if_last_ref : {true, false}) {  // but not last ref
-    tmp_h = shard_->Lookup(key3, hkey3);
+    tmp_h = shard.Lookup(key3, hkey3);
    ASSERT_EQ(h3, tmp_h);
-    ASSERT_FALSE(shard_->Release(tmp_h, erase_if_last_ref));
+    ASSERT_FALSE(shard.Release(tmp_h, erase_if_last_ref));
  }

  // Release last ref on h3, without erase
-  ASSERT_FALSE(shard_->Release(h3, /*erase_if_last_ref*/ false));
+  ASSERT_FALSE(shard.Release(h3, /*erase_if_last_ref*/ false));

  // h3 still not deleted (unreferenced in cache)
  ASSERT_EQ(val.deleted, 0);

  // Explicit erase
-  shard_->Erase(key3, hkey3);
+  shard.Erase(key3, hkey3);

  // h3 deleted
  ASSERT_EQ(val.deleted--, 1);
-  tmp_h = shard_->Lookup(key3, hkey3);
+  tmp_h = shard.Lookup(key3, hkey3);
  ASSERT_EQ(nullptr, tmp_h);
 }

 // This uses the public API to effectively test CalcHashBits etc.
-TEST_F(ClockCacheTest, TableSizesTest) {
+TYPED_TEST(ClockCacheTest, TableSizesTest) {
  for (size_t est_val_size : {1U, 5U, 123U, 2345U, 345678U}) {
    SCOPED_TRACE("est_val_size = " + std::to_string(est_val_size));
    for (double est_count : {1.1, 2.2, 511.9, 512.1, 2345.0}) {
@@ -859,8 +937,10 @@ TEST_F(ClockCacheTest, TableSizesTest) {
                       /*memory_allocator*/ nullptr, kDontChargeCacheMetadata)
                       .MakeSharedCache();
      // Table sizes are currently only powers of two
-      EXPECT_GE(cache->GetTableAddressCount(), est_count / kLoadFactor);
-      EXPECT_LE(cache->GetTableAddressCount(), est_count / kLoadFactor * 2.0);
+      EXPECT_GE(cache->GetTableAddressCount(),
+                est_count / FixedHyperClockTable::kLoadFactor);
+      EXPECT_LE(cache->GetTableAddressCount(),
+                est_count / FixedHyperClockTable::kLoadFactor * 2.0);
      EXPECT_EQ(cache->GetUsage(), 0);

      // kFullChargeMetaData
@@ -877,9 +957,10 @@ TEST_F(ClockCacheTest, TableSizesTest) {
        double est_count_after_meta =
            (capacity - cache->GetUsage()) * 1.0 / est_val_size;
        EXPECT_GE(cache->GetTableAddressCount(),
-                  est_count_after_meta / kLoadFactor);
-        EXPECT_LE(cache->GetTableAddressCount(),
-                  est_count_after_meta / kLoadFactor * 2.0);
+                  est_count_after_meta / FixedHyperClockTable::kLoadFactor);
+        EXPECT_LE(
+            cache->GetTableAddressCount(),
+            est_count_after_meta / FixedHyperClockTable::kLoadFactor * 2.0);
      }
    }
  }
@@ -902,13 +983,14 @@ class TestSecondaryCache : public SecondaryCache {

  using ResultMap = std::unordered_map<std::string, ResultType>;

-  explicit TestSecondaryCache(size_t capacity)
+  explicit TestSecondaryCache(size_t capacity, bool insert_saved = false)
      : cache_(NewLRUCache(capacity, 0, false, 0.5 /* high_pri_pool_ratio */,
                           nullptr, kDefaultToAdaptiveMutex,
                           kDontChargeCacheMetadata)),
        num_inserts_(0),
        num_lookups_(0),
-        inject_failure_(false) {}
+        inject_failure_(false),
+        insert_saved_(insert_saved) {}

  const char* Name() const override { return "TestSecondaryCache"; }

@@ -917,7 +999,8 @@ class TestSecondaryCache : public SecondaryCache {
  void ResetInjectFailure() { inject_failure_ = false; }

  Status Insert(const Slice& key, Cache::ObjectPtr value,
-                const Cache::CacheItemHelper* helper) override {
+                const Cache::CacheItemHelper* helper,
+                bool /*force_insert*/) override {
    if (inject_failure_) {
      return Status::Corruption("Insertion Data Corrupted");
    }
@@ -938,6 +1021,17 @@ class TestSecondaryCache : public SecondaryCache {
    return cache_.Insert(key, buf, size);
  }

+  Status InsertSaved(const Slice& key, const Slice& saved,
+                     CompressionType /*type*/ = kNoCompression,
+                     CacheTier /*source*/ = CacheTier::kVolatileTier) override {
+    if (insert_saved_) {
+      return Insert(key, const_cast<Slice*>(&saved), &kSliceCacheItemHelper,
+                    /*force_insert=*/true);
+    } else {
+      return Status::OK();
+    }
+  }
+
  std::unique_ptr<SecondaryCacheResultHandle> Lookup(
      const Slice& key, const Cache::CacheItemHelper* helper,
      Cache::CreateContext* create_context, bool /*wait*/,
@@ -966,7 +1060,8 @@ class TestSecondaryCache : public SecondaryCache {
        char* ptr = cache_.Value(handle);
        size_t size = DecodeFixed64(ptr);
        ptr += sizeof(uint64_t);
-        s = helper->create_cb(Slice(ptr, size), create_context,
+        s = helper->create_cb(Slice(ptr, size), kNoCompression,
+                              CacheTier::kVolatileTier, create_context,
                              /*alloc*/ nullptr, &value, &charge);
      }
      if (s.ok()) {
@@ -1055,6 +1150,7 @@ class TestSecondaryCache : public SecondaryCache {
  uint32_t num_inserts_;
  uint32_t num_lookups_;
  bool inject_failure_;
+  bool insert_saved_;
  std::string ckey_prefix_;
  ResultMap result_map_;
 };
@@ -1085,7 +1181,7 @@ INSTANTIATE_TEST_CASE_P(DBSecondaryCacheTest, DBSecondaryCacheTest,

 TEST_P(BasicSecondaryCacheTest, BasicTest) {
  std::shared_ptr<TestSecondaryCache> secondary_cache =
-      std::make_shared<TestSecondaryCache>(4096);
+      std::make_shared<TestSecondaryCache>(4096, true);
  std::shared_ptr<Cache> cache =
      NewCache(1024 /* capacity */, 0 /* num_shard_bits */,
               false /* strict_capacity_limit */, secondary_cache);
@@ -1142,7 +1238,7 @@ TEST_P(BasicSecondaryCacheTest, BasicTest) {

 TEST_P(BasicSecondaryCacheTest, StatsTest) {
  std::shared_ptr<TestSecondaryCache> secondary_cache =
-      std::make_shared<TestSecondaryCache>(4096);
+      std::make_shared<TestSecondaryCache>(4096, true);
  std::shared_ptr<Cache> cache =
      NewCache(1024 /* capacity */, 0 /* num_shard_bits */,
               false /* strict_capacity_limit */, secondary_cache);
@@ -1196,7 +1292,7 @@ TEST_P(BasicSecondaryCacheTest, StatsTest) {

 TEST_P(BasicSecondaryCacheTest, BasicFailTest) {
  std::shared_ptr<TestSecondaryCache> secondary_cache =
-      std::make_shared<TestSecondaryCache>(2048);
+      std::make_shared<TestSecondaryCache>(2048, true);
  std::shared_ptr<Cache> cache =
      NewCache(1024 /* capacity */, 0 /* num_shard_bits */,
               false /* strict_capacity_limit */, secondary_cache);
@@ -1238,7 +1334,7 @@ TEST_P(BasicSecondaryCacheTest, BasicFailTest) {

 TEST_P(BasicSecondaryCacheTest, SaveFailTest) {
  std::shared_ptr<TestSecondaryCache> secondary_cache =
-      std::make_shared<TestSecondaryCache>(2048);
+      std::make_shared<TestSecondaryCache>(2048, true);
  std::shared_ptr<Cache> cache =
      NewCache(1024 /* capacity */, 0 /* num_shard_bits */,
               false /* strict_capacity_limit */, secondary_cache);
@@ -1279,7 +1375,7 @@ TEST_P(BasicSecondaryCacheTest, SaveFailTest) {

 TEST_P(BasicSecondaryCacheTest, CreateFailTest) {
  std::shared_ptr<TestSecondaryCache> secondary_cache =
-      std::make_shared<TestSecondaryCache>(2048);
+      std::make_shared<TestSecondaryCache>(2048, true);
  std::shared_ptr<Cache> cache =
      NewCache(1024 /* capacity */, 0 /* num_shard_bits */,
               false /* strict_capacity_limit */, secondary_cache);
@@ -1320,7 +1416,7 @@ TEST_P(BasicSecondaryCacheTest, CreateFailTest) {
 TEST_P(BasicSecondaryCacheTest, FullCapacityTest) {
  for (bool strict_capacity_limit : {false, true}) {
    std::shared_ptr<TestSecondaryCache> secondary_cache =
-        std::make_shared<TestSecondaryCache>(2048);
+        std::make_shared<TestSecondaryCache>(2048, true);
    std::shared_ptr<Cache> cache =
        NewCache(1024 /* capacity */, 0 /* num_shard_bits */,
                 strict_capacity_limit, secondary_cache);
@@ -1367,7 +1463,7 @@ TEST_P(BasicSecondaryCacheTest, FullCapacityTest) {
        k2.AsSlice(),
        GetHelper(CacheEntryRole::kDataBlock, /*secondary_compatible=*/false),
        /*context*/ this, Cache::Priority::LOW);
-    if (strict_capacity_limit || GetParam() == kHyperClock) {
+    if (strict_capacity_limit || IsHyperClock()) {
      ASSERT_NE(handle2, nullptr);
      cache->Release(handle2);
      ASSERT_EQ(secondary_cache->num_inserts(), 1u);
@@ -1392,12 +1488,12 @@ TEST_P(BasicSecondaryCacheTest, FullCapacityTest) {
 // CORRECTION: this is not quite right. block_1 can be inserted into the block
 // cache because strict_capacity_limit=false, but it is removed from the cache
 // in Release() because of being over-capacity, without demoting to secondary
-// cache. HyperClockCache doesn't check capacity on release (for efficiency)
-// so can demote the over-capacity item to secondary cache. Also, we intend to
-// add support for demotion in Release, but that currently causes too much
-// unit test churn.
+// cache. FixedHyperClockCache doesn't check capacity on release (for
+// efficiency) so can demote the over-capacity item to secondary cache. Also, we
+// intend to add support for demotion in Release, but that currently causes too
+// much unit test churn.
 TEST_P(DBSecondaryCacheTest, TestSecondaryCacheCorrectness1) {
-  if (GetParam() == kHyperClock) {
+  if (IsHyperClock()) {
    // See CORRECTION above
    ROCKSDB_GTEST_BYPASS("Test depends on LRUCache-specific behaviors");
    return;
@@ -1495,7 +1591,7 @@ TEST_P(DBSecondaryCacheTest, TestSecondaryCacheCorrectness1) {
 // insert and cache block_1 in the block cache (this is the different place
 // from TestSecondaryCacheCorrectness1)
 TEST_P(DBSecondaryCacheTest, TestSecondaryCacheCorrectness2) {
-  if (GetParam() == kHyperClock) {
+  if (IsHyperClock()) {
    ROCKSDB_GTEST_BYPASS("Test depends on LRUCache-specific behaviors");
    return;
  }
@@ -1683,7 +1779,7 @@ TEST_P(DBSecondaryCacheTest, SecondaryCacheIntensiveTesting) {
 // if we try to insert block_1 to the block cache, it will always fails. Only
 // block_2 will be successfully inserted into the block cache.
 TEST_P(DBSecondaryCacheTest, SecondaryCacheFailureTest) {
-  if (GetParam() == kHyperClock) {
+  if (IsHyperClock()) {
    ROCKSDB_GTEST_BYPASS("Test depends on LRUCache-specific behaviors");
    return;
  }
@@ -1793,7 +1889,7 @@ TEST_P(BasicSecondaryCacheTest, BasicWaitAllTest) {
                            str.length()));
  }
  // Force all entries to be evicted to the secondary cache
-  if (GetParam() == kHyperClock) {
+  if (IsHyperClock()) {
    // HCC doesn't respond immediately to SetCapacity
    for (int i = 9000; i < 9030; ++i) {
      ASSERT_OK(cache->Insert(ock.WithOffset(i).AsSlice(), nullptr,
@@ -1848,7 +1944,7 @@ TEST_P(BasicSecondaryCacheTest, BasicWaitAllTest) {
 // a sync point callback in TestSecondaryCache::Lookup. We then control the
 // lookup result by setting the ResultMap.
 TEST_P(DBSecondaryCacheTest, TestSecondaryCacheMultiGet) {
-  if (GetParam() == kHyperClock) {
+  if (IsHyperClock()) {
    ROCKSDB_GTEST_BYPASS("Test depends on LRUCache-specific behaviors");
    return;
  }
@@ -1939,8 +2035,9 @@ class CacheWithStats : public CacheWrapper {

  Status Insert(const Slice& key, Cache::ObjectPtr value,
                const CacheItemHelper* helper, size_t charge,
-                Handle** handle = nullptr,
-                Priority priority = Priority::LOW) override {
+                Handle** handle = nullptr, Priority priority = Priority::LOW,
+                const Slice& /*compressed*/ = Slice(),
+                CompressionType /*type*/ = kNoCompression) override {
    insert_count_++;
    return target_->Insert(key, value, helper, charge, handle, priority);
  }
@@ -2033,7 +2130,7 @@ TEST_P(DBSecondaryCacheTest, LRUCacheDumpLoadBasic) {
  // we have a new cache it is empty, then, before we do the Get, we do the
  // dumpload
  std::shared_ptr<TestSecondaryCache> secondary_cache =
-      std::make_shared<TestSecondaryCache>(2048 * 1024);
+      std::make_shared<TestSecondaryCache>(2048 * 1024, true);
  // This time with secondary cache
  base_cache = NewCache(1024 * 1024 /* capacity */, 0 /* num_shard_bits */,
                        false /* strict_capacity_limit */, secondary_cache);
@@ -2189,7 +2286,7 @@ TEST_P(DBSecondaryCacheTest, LRUCacheDumpLoadWithFilter) {
  // we have a new cache it is empty, then, before we do the Get, we do the
  // dumpload
  std::shared_ptr<TestSecondaryCache> secondary_cache =
-      std::make_shared<TestSecondaryCache>(2048 * 1024);
+      std::make_shared<TestSecondaryCache>(2048 * 1024, true);
  // This time with secondary_cache
  base_cache = NewCache(1024 * 1024 /* capacity */, 0 /* num_shard_bits */,
                        false /* strict_capacity_limit */, secondary_cache);
@@ -2349,7 +2446,7 @@ TEST_P(DBSecondaryCacheTest, TestSecondaryCacheOptionBasic) {
 // with new options, which set the lowest_used_cache_tier to
 // kNonVolatileBlockTier. So secondary cache will be used.
 TEST_P(DBSecondaryCacheTest, TestSecondaryCacheOptionChange) {
-  if (GetParam() == kHyperClock) {
+  if (IsHyperClock()) {
    ROCKSDB_GTEST_BYPASS("Test depends on LRUCache-specific behaviors");
    return;
  }
@@ -2444,7 +2541,7 @@ TEST_P(DBSecondaryCacheTest, TestSecondaryCacheOptionChange) {
 // Two DB test. We create 2 DBs sharing the same block cache and secondary
 // cache. We diable the secondary cache option for DB2.
 TEST_P(DBSecondaryCacheTest, TestSecondaryCacheOptionTwoDB) {
-  if (GetParam() == kHyperClock) {
+  if (IsHyperClock()) {
    ROCKSDB_GTEST_BYPASS("Test depends on LRUCache-specific behaviors");
    return;
  }
@@ -9,36 +9,4 @@

 namespace ROCKSDB_NAMESPACE {

-namespace {
-
-void NoopDelete(Cache::ObjectPtr, MemoryAllocator*) {}
-
-size_t SliceSize(Cache::ObjectPtr obj) {
-  return static_cast<Slice*>(obj)->size();
-}
-
-Status SliceSaveTo(Cache::ObjectPtr from_obj, size_t from_offset, size_t length,
-                   char* out) {
-  const Slice& slice = *static_cast<Slice*>(from_obj);
-  std::memcpy(out, slice.data() + from_offset, length);
-  return Status::OK();
-}
-
-Status FailCreate(const Slice&, Cache::CreateContext*, MemoryAllocator*,
-                  Cache::ObjectPtr*, size_t*) {
-  return Status::NotSupported("Only for dumping data into SecondaryCache");
-}
-
-}  // namespace
-
-Status SecondaryCache::InsertSaved(const Slice& key, const Slice& saved) {
-  static Cache::CacheItemHelper helper_no_secondary{CacheEntryRole::kMisc,
-                                                    &NoopDelete};
-  static Cache::CacheItemHelper helper{
-      CacheEntryRole::kMisc, &NoopDelete, &SliceSize,
-      &SliceSaveTo,          &FailCreate, &helper_no_secondary};
-  // NOTE: depends on Insert() being synchronous, not keeping pointer `&saved`
-  return Insert(key, const_cast<Slice*>(&saved), &helper);
-}
-
 }  // namespace ROCKSDB_NAMESPACE
@@ -5,7 +5,12 @@

 #include "cache/secondary_cache_adapter.h"

+#include <atomic>
+
+#include "cache/tiered_secondary_cache.h"
 #include "monitoring/perf_context_imp.h"
+#include "test_util/sync_point.h"
+#include "util/cast_util.h"

 namespace ROCKSDB_NAMESPACE {

@@ -16,33 +21,121 @@ struct Dummy {
 };
 const Dummy kDummy{};
 Cache::ObjectPtr const kDummyObj = const_cast<Dummy*>(&kDummy);
+const char* kTieredCacheName = "TieredCache";
 }  // namespace

+// When CacheWithSecondaryAdapter is constructed with the distribute_cache_res
+// parameter set to true, it manages the entire memory budget across the
+// primary and secondary cache. The secondary cache is assumed to be in
+// memory, such as the CompressedSecondaryCache. When a placeholder entry
+// is inserted by a CacheReservationManager instance to reserve memory,
+// the CacheWithSecondaryAdapter ensures that the reservation is distributed
+// proportionally across the primary/secondary caches.
+//
+// The primary block cache is initially sized to the sum of the primary cache
+// budget + teh secondary cache budget, as follows -
+//   |---------    Primary Cache Configured Capacity  -----------|
+//   |---Secondary Cache Budget----|----Primary Cache Budget-----|
+//
+// A ConcurrentCacheReservationManager member in the CacheWithSecondaryAdapter,
+// pri_cache_res_,
+// is used to help with tracking the distribution of memory reservations.
+// Initially, it accounts for the entire secondary cache budget as a
+// reservation against the primary cache. This shrinks the usable capacity of
+// the primary cache to the budget that the user originally desired.
+//
+//   |--Reservation for Sec Cache--|-Pri Cache Usable Capacity---|
+//
+// When a reservation placeholder is inserted into the adapter, it is inserted
+// directly into the primary cache. This means the entire charge of the
+// placeholder is counted against the primary cache. To compensate and count
+// a portion of it against the secondary cache, the secondary cache Deflate()
+// method is called to shrink it. Since the Deflate() causes the secondary
+// actual usage to shrink, it is refelcted here by releasing an equal amount
+// from the pri_cache_res_ reservation. The Deflate() in the secondary cache
+// can be, but is not required to be, implemented using its own cache
+// reservation manager.
+//
+// For example, if the pri/sec ratio is 70/30, and the combined capacity is
+// 100MB, the intermediate and final  state after inserting a reservation
+// placeholder for 10MB would be as follows -
+//
+//   |-Reservation for Sec Cache-|-Pri Cache Usable Capacity-|---R---|
+// 1. After inserting the placeholder in primary
+//   |-------  30MB -------------|------- 60MB -------------|-10MB--|
+// 2. After deflating the secondary and adjusting the reservation for
+//    secondary against the primary
+//   |-------  27MB -------------|------- 63MB -------------|-10MB--|
+//
+// Likewise, when the user inserted placeholder is released, the secondary
+// cache Inflate() method is called to grow it, and the pri_cache_res_
+// reservation is increased by an equal amount.
+//
+// Another way of implementing this would have been to simply split the user
+// reservation into primary and seconary components. However, this would
+// require allocating a structure to track the associated secondary cache
+// reservation, which adds some complexity and overhead.
+//
 CacheWithSecondaryAdapter::CacheWithSecondaryAdapter(
    std::shared_ptr<Cache> target,
-    std::shared_ptr<SecondaryCache> secondary_cache)
+    std::shared_ptr<SecondaryCache> secondary_cache,
+    TieredAdmissionPolicy adm_policy, bool distribute_cache_res)
    : CacheWrapper(std::move(target)),
-      secondary_cache_(std::move(secondary_cache)) {
-  target_->SetEvictionCallback([this](const Slice& key, Handle* handle) {
-    return EvictionHandler(key, handle);
-  });
+      secondary_cache_(std::move(secondary_cache)),
+      adm_policy_(adm_policy),
+      distribute_cache_res_(distribute_cache_res) {
+  target_->SetEvictionCallback(
+      [this](const Slice& key, Handle* handle, bool was_hit) {
+        return EvictionHandler(key, handle, was_hit);
+      });
+  if (distribute_cache_res_) {
+    size_t sec_capacity = 0;
+    pri_cache_res_ = std::make_shared<ConcurrentCacheReservationManager>(
+        std::make_shared<CacheReservationManagerImpl<CacheEntryRole::kMisc>>(
+            target_));
+    Status s = secondary_cache_->GetCapacity(sec_capacity);
+    assert(s.ok());
+    // Initially, the primary cache is sized to uncompressed cache budget plsu
+    // compressed secondary cache budget. The secondary cache budget is then
+    // taken away from the primary cache through cache reservations. Later,
+    // when a placeholder entry is inserted by the caller, its inserted
+    // into the primary cache and the portion that should be assigned to the
+    // secondary cache is freed from the reservation.
+    s = pri_cache_res_->UpdateCacheReservation(sec_capacity);
+    assert(s.ok());
+    sec_cache_res_ratio_.store((double)sec_capacity / target_->GetCapacity(),
+                               std::memory_order_relaxed);
+  }
 }

 CacheWithSecondaryAdapter::~CacheWithSecondaryAdapter() {
  // `*this` will be destroyed before `*target_`, so we have to prevent
  // use after free
  target_->SetEvictionCallback({});
+#ifndef NDEBUG
+  if (distribute_cache_res_ && !ratio_changed_) {
+    size_t sec_capacity = 0;
+    Status s = secondary_cache_->GetCapacity(sec_capacity);
+    assert(s.ok());
+    assert(pri_cache_res_->GetTotalMemoryUsed() == sec_capacity);
+  }
+#endif  // NDEBUG
 }

 bool CacheWithSecondaryAdapter::EvictionHandler(const Slice& key,
-                                                Handle* handle) {
+                                                Handle* handle, bool was_hit) {
  auto helper = GetCacheItemHelper(handle);
-  if (helper->IsSecondaryCacheCompatible()) {
+  if (helper->IsSecondaryCacheCompatible() &&
+      adm_policy_ != TieredAdmissionPolicy::kAdmPolicyThreeQueue) {
    auto obj = target_->Value(handle);
    // Ignore dummy entry
    if (obj != kDummyObj) {
+      bool hit = false;
+      if (adm_policy_ == TieredAdmissionPolicy::kAdmPolicyAllowCacheHits) {
+        hit = was_hit;
+      }
      // Spill into secondary cache.
-      secondary_cache_->Insert(key, obj, helper).PermitUncheckedError();
+      secondary_cache_->Insert(key, obj, helper, hit).PermitUncheckedError();
    }
  }
  // Never takes ownership of obj
@@ -136,6 +229,32 @@ Cache::Handle* CacheWithSecondaryAdapter::Promote(
  return result;
 }

+Status CacheWithSecondaryAdapter::Insert(const Slice& key, ObjectPtr value,
+                                         const CacheItemHelper* helper,
+                                         size_t charge, Handle** handle,
+                                         Priority priority,
+                                         const Slice& compressed_value,
+                                         CompressionType type) {
+  Status s = target_->Insert(key, value, helper, charge, handle, priority);
+  if (s.ok() && value == nullptr && distribute_cache_res_) {
+    size_t sec_charge = static_cast<size_t>(
+        charge * (sec_cache_res_ratio_.load(std::memory_order_relaxed)));
+    s = secondary_cache_->Deflate(sec_charge);
+    assert(s.ok());
+    s = pri_cache_res_->UpdateCacheReservation(sec_charge, /*increase=*/false);
+    assert(s.ok());
+  }
+  // Warm up the secondary cache with the compressed block. The secondary
+  // cache may choose to ignore it based on the admission policy.
+  if (value != nullptr && !compressed_value.empty() &&
+      adm_policy_ == TieredAdmissionPolicy::kAdmPolicyThreeQueue) {
+    Status status = secondary_cache_->InsertSaved(key, compressed_value, type);
+    assert(status.ok() || status.IsNotSupported());
+  }
+
+  return s;
+}
+
 Cache::Handle* CacheWithSecondaryAdapter::Lookup(const Slice& key,
                                                 const CacheItemHelper* helper,
                                                 CreateContext* create_context,
@@ -162,6 +281,25 @@ Cache::Handle* CacheWithSecondaryAdapter::Lookup(const Slice& key,
  return result;
 }

+bool CacheWithSecondaryAdapter::Release(Handle* handle,
+                                        bool erase_if_last_ref) {
+  if (erase_if_last_ref) {
+    ObjectPtr v = target_->Value(handle);
+    if (v == nullptr && distribute_cache_res_) {
+      size_t charge = target_->GetCharge(handle);
+      size_t sec_charge = static_cast<size_t>(
+          charge * (sec_cache_res_ratio_.load(std::memory_order_relaxed)));
+      TEST_SYNC_POINT("CacheWithSecondaryAdapter::Release:ChargeSecCache1");
+      TEST_SYNC_POINT("CacheWithSecondaryAdapter::Release:ChargeSecCache2");
+      Status s = secondary_cache_->Inflate(sec_charge);
+      assert(s.ok());
+      s = pri_cache_res_->UpdateCacheReservation(sec_charge, /*increase=*/true);
+      assert(s.ok());
+    }
+  }
+  return target_->Release(handle, erase_if_last_ref);
+}
+
 Cache::ObjectPtr CacheWithSecondaryAdapter::Value(Handle* handle) {
  ObjectPtr v = target_->Value(handle);
  // TODO with stacked secondaries: might fail in EvictionHandler
@@ -265,7 +403,7 @@ void CacheWithSecondaryAdapter::WaitAll(AsyncLookupHandle* async_handles,
    for (AsyncLookupHandle* cur : my_pending) {
      my_secondary_handles.push_back(cur->pending_handle);
    }
-    secondary_cache_->WaitAll(my_secondary_handles);
+    secondary_cache_->WaitAll(std::move(my_secondary_handles));
  }

  // Process results
@@ -288,8 +426,297 @@ std::string CacheWithSecondaryAdapter::GetPrintableOptions() const {
 }

 const char* CacheWithSecondaryAdapter::Name() const {
-  // To the user, at least for now, configure the underlying cache with
-  // a secondary cache. So we pretend to be that cache
-  return target_->Name();
+  if (distribute_cache_res_) {
+    return kTieredCacheName;
+  } else {
+    // To the user, at least for now, configure the underlying cache with
+    // a secondary cache. So we pretend to be that cache
+    return target_->Name();
+  }
+}
+
+// Update the total cache capacity. If we're distributing cache reservations
+// to both primary and secondary, then update the pri_cache_res_reservation
+// as well. At the moment, we don't have a good way of handling the case
+// where the new capacity < total cache reservations.
+void CacheWithSecondaryAdapter::SetCapacity(size_t capacity) {
+  size_t sec_capacity = static_cast<size_t>(
+      capacity * (distribute_cache_res_
+                      ? sec_cache_res_ratio_.load(std::memory_order_relaxed)
+                      : 0.0));
+  size_t old_sec_capacity = 0;
+
+  if (distribute_cache_res_) {
+    MutexLock m(&mutex_);
+
+    Status s = secondary_cache_->GetCapacity(old_sec_capacity);
+    if (!s.ok()) {
+      return;
+    }
+    if (old_sec_capacity > sec_capacity) {
+      // We're shrinking the cache. We do things in the following order to
+      // avoid a temporary spike in usage over the configured capacity -
+      // 1. Lower the secondary cache capacity
+      // 2. Credit an equal amount (by decreasing pri_cache_res_) to the
+      //    primary cache
+      // 3. Decrease the primary cache capacity to the total budget
+      s = secondary_cache_->SetCapacity(sec_capacity);
+      if (s.ok()) {
+        s = pri_cache_res_->UpdateCacheReservation(
+            old_sec_capacity - sec_capacity,
+            /*increase=*/false);
+        assert(s.ok());
+        target_->SetCapacity(capacity);
+      }
+    } else {
+      // We're expanding the cache. Do it in the following order to avoid
+      // unnecessary evictions -
+      // 1. Increase the primary cache capacity to total budget
+      // 2. Reserve additional memory in primary on behalf of secondary (by
+      //    increasing pri_cache_res_ reservation)
+      // 3. Increase secondary cache capacity
+      target_->SetCapacity(capacity);
+      s = pri_cache_res_->UpdateCacheReservation(
+          sec_capacity - old_sec_capacity,
+          /*increase=*/true);
+      assert(s.ok());
+      s = secondary_cache_->SetCapacity(sec_capacity);
+      assert(s.ok());
+    }
+  } else {
+    // No cache reservation distribution. Just set the primary cache capacity.
+    target_->SetCapacity(capacity);
+  }
+}
+
+Status CacheWithSecondaryAdapter::GetSecondaryCacheCapacity(
+    size_t& size) const {
+  return secondary_cache_->GetCapacity(size);
+}
+
+Status CacheWithSecondaryAdapter::GetSecondaryCachePinnedUsage(
+    size_t& size) const {
+  Status s;
+  if (distribute_cache_res_) {
+    MutexLock m(&mutex_);
+    size_t capacity = 0;
+    s = secondary_cache_->GetCapacity(capacity);
+    if (s.ok()) {
+      size = capacity - pri_cache_res_->GetTotalMemoryUsed();
+    } else {
+      size = 0;
+    }
+  } else {
+    size = 0;
+  }
+  return s;
+}
+
+// Update the secondary/primary allocation ratio (remember, the primary
+// capacity is the total memory budget when distribute_cache_res_ is true).
+// When the ratio changes, we may accumulate some error in the calculations
+// for secondary cache inflate/deflate and pri_cache_res_ reservations.
+// This is due to the rounding of the reservation amount.
+//
+// We rely on the current pri_cache_res_ total memory used to estimate the
+// new secondary cache reservation after the ratio change. For this reason,
+// once the ratio is lowered to 0.0 (effectively disabling the secondary
+// cache and pri_cache_res_ total mem used going down to 0), we cannot
+// increase the ratio and re-enable it, We might remove this limitation
+// in the future.
+Status CacheWithSecondaryAdapter::UpdateCacheReservationRatio(
+    double compressed_secondary_ratio) {
+  if (!distribute_cache_res_ ||
+      sec_cache_res_ratio_.load(std::memory_order_relaxed) == 0.0) {
+    return Status::NotSupported();
+  }
+
+  MutexLock m(&mutex_);
+  size_t pri_capacity = target_->GetCapacity();
+  size_t sec_capacity =
+      static_cast<size_t>(pri_capacity * compressed_secondary_ratio);
+  size_t old_sec_capacity;
+  Status s = secondary_cache_->GetCapacity(old_sec_capacity);
+  if (!s.ok()) {
+    return s;
+  }
+
+  TEST_SYNC_POINT(
+      "CacheWithSecondaryAdapter::UpdateCacheReservationRatio:Begin");
+
+  // There's a possible race condition here. Since the read of pri_cache_res_
+  // memory used (secondary cache usage charged to primary cache), and the
+  // change to sec_cache_res_ratio_ are not guarded by a mutex, its possible
+  // that an Insert/Release in another thread might decrease/increase the
+  // pri_cache_res_ reservation by the wrong amount. This should not be a
+  // problem because updating the sec/pri ratio is a rare operation, and
+  // the worst that can happen is we may over/under charge the secondary
+  // cache usage by a little bit. But we do need to protect against
+  // underflow of old_sec_reserved.
+  // TODO: Make the accounting more accurate by tracking the total memory
+  // reservation on the primary cache. This will also allow us to remove
+  // the restriction of not being able to change the sec/pri ratio from
+  // 0.0 to higher.
+  size_t sec_charge_to_pri = pri_cache_res_->GetTotalMemoryUsed();
+  size_t old_sec_reserved = (old_sec_capacity > sec_charge_to_pri)
+                                ? (old_sec_capacity - sec_charge_to_pri)
+                                : 0;
+  // Calculate the new secondary cache reservation
+  size_t sec_reserved = static_cast<size_t>(
+      old_sec_reserved *
+      (double)(compressed_secondary_ratio /
+               sec_cache_res_ratio_.load(std::memory_order_relaxed)));
+  sec_cache_res_ratio_.store(compressed_secondary_ratio,
+                             std::memory_order_relaxed);
+  if (sec_capacity > old_sec_capacity) {
+    // We're increasing the ratio, thus ending up with a larger secondary
+    // cache and a smaller usable primary cache capacity. Similar to
+    // SetCapacity(), we try to avoid a temporary increase in total usage
+    // beyond teh configured capacity -
+    // 1. A higher secondary cache ratio means it gets a higher share of
+    //    cache reservations. So first account for that by deflating the
+    //    secondary cache
+    // 2. Increase pri_cache_res_ reservation to reflect the new secondary
+    //    cache utilization (increase in capacity - increase in share of cache
+    //    reservation)
+    // 3. Increase secondary cache capacity
+    s = secondary_cache_->Deflate(sec_reserved - old_sec_reserved);
+    assert(s.ok());
+    s = pri_cache_res_->UpdateCacheReservation(
+        (sec_capacity - old_sec_capacity) - (sec_reserved - old_sec_reserved),
+        /*increase=*/true);
+    assert(s.ok());
+    s = secondary_cache_->SetCapacity(sec_capacity);
+    assert(s.ok());
+  } else {
+    // We're shrinking the ratio. Try to avoid unnecessary evictions -
+    // 1. Lower the secondary cache capacity
+    // 2. Decrease pri_cache_res_ reservation to relect lower secondary
+    //    cache utilization (decrease in capacity - decrease in share of cache
+    //    reservations)
+    // 3. Inflate the secondary cache to give it back the reduction in its
+    //    share of cache reservations
+    s = secondary_cache_->SetCapacity(sec_capacity);
+    if (s.ok()) {
+      s = pri_cache_res_->UpdateCacheReservation(
+          (old_sec_capacity - sec_capacity) - (old_sec_reserved - sec_reserved),
+          /*increase=*/false);
+      assert(s.ok());
+      s = secondary_cache_->Inflate(old_sec_reserved - sec_reserved);
+      assert(s.ok());
+    }
+  }
+
+  TEST_SYNC_POINT("CacheWithSecondaryAdapter::UpdateCacheReservationRatio:End");
+#ifndef NDEBUG
+  // As mentioned in the function comments, we may accumulate some erros when
+  // the ratio is changed. We set a flag here which disables some assertions
+  // in the destructor
+  ratio_changed_ = true;
+#endif
+  return s;
+}
+
+Status CacheWithSecondaryAdapter::UpdateAdmissionPolicy(
+    TieredAdmissionPolicy adm_policy) {
+  adm_policy_ = adm_policy;
+  return Status::OK();
+}
+
+std::shared_ptr<Cache> NewTieredCache(const TieredCacheOptions& _opts) {
+  if (!_opts.cache_opts) {
+    return nullptr;
+  }
+
+  TieredCacheOptions opts = _opts;
+  {
+    bool valid_adm_policy = true;
+
+    switch (_opts.adm_policy) {
+      case TieredAdmissionPolicy::kAdmPolicyAuto:
+        // Select an appropriate default policy
+        if (opts.adm_policy == TieredAdmissionPolicy::kAdmPolicyAuto) {
+          if (opts.nvm_sec_cache) {
+            opts.adm_policy = TieredAdmissionPolicy::kAdmPolicyThreeQueue;
+          } else {
+            opts.adm_policy = TieredAdmissionPolicy::kAdmPolicyPlaceholder;
+          }
+        }
+        break;
+      case TieredAdmissionPolicy::kAdmPolicyPlaceholder:
+      case TieredAdmissionPolicy::kAdmPolicyAllowCacheHits:
+        if (opts.nvm_sec_cache) {
+          valid_adm_policy = false;
+        }
+        break;
+      case TieredAdmissionPolicy::kAdmPolicyThreeQueue:
+        if (!opts.nvm_sec_cache) {
+          valid_adm_policy = false;
+        }
+        break;
+      default:
+        valid_adm_policy = false;
+    }
+    if (!valid_adm_policy) {
+      return nullptr;
+    }
+  }
+
+  std::shared_ptr<Cache> cache;
+  if (opts.cache_type == PrimaryCacheType::kCacheTypeLRU) {
+    LRUCacheOptions cache_opts =
+        *(static_cast_with_check<LRUCacheOptions, ShardedCacheOptions>(
+            opts.cache_opts));
+    cache_opts.capacity = opts.total_capacity;
+    cache = cache_opts.MakeSharedCache();
+  } else if (opts.cache_type == PrimaryCacheType::kCacheTypeHCC) {
+    HyperClockCacheOptions cache_opts =
+        *(static_cast_with_check<HyperClockCacheOptions, ShardedCacheOptions>(
+            opts.cache_opts));
+    cache_opts.capacity = opts.total_capacity;
+    cache = cache_opts.MakeSharedCache();
+  } else {
+    return nullptr;
+  }
+  std::shared_ptr<SecondaryCache> sec_cache;
+  opts.comp_cache_opts.capacity = static_cast<size_t>(
+      opts.total_capacity * opts.compressed_secondary_ratio);
+  sec_cache = NewCompressedSecondaryCache(opts.comp_cache_opts);
+
+  if (opts.nvm_sec_cache) {
+    if (opts.adm_policy == TieredAdmissionPolicy::kAdmPolicyThreeQueue) {
+      sec_cache = std::make_shared<TieredSecondaryCache>(
+          sec_cache, opts.nvm_sec_cache,
+          TieredAdmissionPolicy::kAdmPolicyThreeQueue);
+    } else {
+      return nullptr;
+    }
+  }
+
+  return std::make_shared<CacheWithSecondaryAdapter>(
+      cache, sec_cache, opts.adm_policy, /*distribute_cache_res=*/true);
+}
+
+Status UpdateTieredCache(const std::shared_ptr<Cache>& cache,
+                         int64_t total_capacity,
+                         double compressed_secondary_ratio,
+                         TieredAdmissionPolicy adm_policy) {
+  if (!cache || strcmp(cache->Name(), kTieredCacheName)) {
+    return Status::InvalidArgument();
+  }
+  CacheWithSecondaryAdapter* tiered_cache =
+      static_cast<CacheWithSecondaryAdapter*>(cache.get());
+
+  Status s;
+  if (total_capacity > 0) {
+    tiered_cache->SetCapacity(total_capacity);
+  }
+  if (compressed_secondary_ratio >= 0.0 && compressed_secondary_ratio <= 1.0) {
+    s = tiered_cache->UpdateCacheReservationRatio(compressed_secondary_ratio);
+  }
+  if (adm_policy < TieredAdmissionPolicy::kAdmPolicyMax) {
+    s = tiered_cache->UpdateAdmissionPolicy(adm_policy);
+  }
+  return s;
 }
 }  // namespace ROCKSDB_NAMESPACE
@@ -5,6 +5,7 @@

 #pragma once

+#include "cache/cache_reservation_manager.h"
 #include "rocksdb/secondary_cache.h"

 namespace ROCKSDB_NAMESPACE {
@@ -13,15 +14,27 @@ class CacheWithSecondaryAdapter : public CacheWrapper {
 public:
  explicit CacheWithSecondaryAdapter(
      std::shared_ptr<Cache> target,
-      std::shared_ptr<SecondaryCache> secondary_cache);
+      std::shared_ptr<SecondaryCache> secondary_cache,
+      TieredAdmissionPolicy adm_policy = TieredAdmissionPolicy::kAdmPolicyAuto,
+      bool distribute_cache_res = false);

  ~CacheWithSecondaryAdapter() override;

+  Status Insert(
+      const Slice& key, ObjectPtr value, const CacheItemHelper* helper,
+      size_t charge, Handle** handle = nullptr,
+      Priority priority = Priority::LOW,
+      const Slice& compressed_value = Slice(),
+      CompressionType type = CompressionType::kNoCompression) override;
+
  Handle* Lookup(const Slice& key, const CacheItemHelper* helper,
                 CreateContext* create_context,
                 Priority priority = Priority::LOW,
                 Statistics* stats = nullptr) override;

+  using Cache::Release;
+  bool Release(Handle* handle, bool erase_if_last_ref = false) override;
+
  ObjectPtr Value(Handle* handle) override;

  void StartAsyncLookup(AsyncLookupHandle& async_handle) override;
@@ -32,8 +45,22 @@ class CacheWithSecondaryAdapter : public CacheWrapper {

  const char* Name() const override;

+  void SetCapacity(size_t capacity) override;
+
+  Status GetSecondaryCacheCapacity(size_t& size) const override;
+
+  Status GetSecondaryCachePinnedUsage(size_t& size) const override;
+
+  Status UpdateCacheReservationRatio(double ratio);
+
+  Status UpdateAdmissionPolicy(TieredAdmissionPolicy adm_policy);
+
+  Cache* TEST_GetCache() { return target_.get(); }
+
+  SecondaryCache* TEST_GetSecondaryCache() { return secondary_cache_.get(); }
+
 private:
-  bool EvictionHandler(const Slice& key, Handle* handle);
+  bool EvictionHandler(const Slice& key, Handle* handle, bool was_hit);

  void StartAsyncLookupOnMySecondary(AsyncLookupHandle& async_handle);

@@ -47,6 +74,21 @@ class CacheWithSecondaryAdapter : public CacheWrapper {
  void CleanupCacheObject(ObjectPtr obj, const CacheItemHelper* helper);

  std::shared_ptr<SecondaryCache> secondary_cache_;
+  TieredAdmissionPolicy adm_policy_;
+  // Whether to proportionally distribute cache memory reservations, i.e
+  // placeholder entries with null value and a non-zero charge, across
+  // the primary and secondary caches.
+  bool distribute_cache_res_;
+  // A cache reservation manager to keep track of secondary cache memory
+  // usage by reserving equivalent capacity against the primary cache
+  std::shared_ptr<ConcurrentCacheReservationManager> pri_cache_res_;
+  // Fraction of a cache memory reservation to be assigned to the secondary
+  // cache
+  std::atomic<double> sec_cache_res_ratio_;
+  mutable port::Mutex mutex_;
+#ifndef NDEBUG
+  bool ratio_changed_ = false;
+#endif
 };

 }  // namespace ROCKSDB_NAMESPACE
@@ -38,7 +38,7 @@ uint32_t DetermineSeed(int32_t hash_seed_option) {
      return GetSliceHash(hostname) & kSeedMask;
    } else {
      // Fall back on something stable within the process.
-      return static_cast<uint32_t>(gen.GetBaseUpper()) & kSeedMask;
+      return BitwiseAnd(gen.GetBaseUpper(), kSeedMask);
    }
  } else {
    // for kQuasiRandomHashSeed and fallback
@@ -83,6 +83,16 @@ size_t ShardedCacheBase::GetCapacity() const {
  return capacity_;
 }

+Status ShardedCacheBase::GetSecondaryCacheCapacity(size_t& size) const {
+  size = 0;
+  return Status::OK();
+}
+
+Status ShardedCacheBase::GetSecondaryCachePinnedUsage(size_t& size) const {
+  size = 0;
+  return Status::OK();
+}
+
 bool ShardedCacheBase::HasStrictCapacityLimit() const {
  MutexLock l(&config_mutex_);
  return strict_capacity_limit_;
@@ -99,6 +99,8 @@ class ShardedCacheBase : public Cache {

  bool HasStrictCapacityLimit() const override;
  size_t GetCapacity() const override;
+  Status GetSecondaryCacheCapacity(size_t& size) const override;
+  Status GetSecondaryCachePinnedUsage(size_t& size) const override;

  using Cache::GetUsage;
  size_t GetUsage(Handle* handle) const override;
@@ -170,9 +172,12 @@ class ShardedCache : public ShardedCacheBase {
        [s_c_l](CacheShard* cs) { cs->SetStrictCapacityLimit(s_c_l); });
  }

-  Status Insert(const Slice& key, ObjectPtr obj, const CacheItemHelper* helper,
-                size_t charge, Handle** handle = nullptr,
-                Priority priority = Priority::LOW) override {
+  Status Insert(
+      const Slice& key, ObjectPtr obj, const CacheItemHelper* helper,
+      size_t charge, Handle** handle = nullptr,
+      Priority priority = Priority::LOW,
+      const Slice& /*compressed_value*/ = Slice(),
+      CompressionType /*type*/ = CompressionType::kNoCompression) override {
    assert(helper);
    HashVal hash = CacheShard::ComputeHash(key, hash_seed_);
    auto h_out = reinterpret_cast<HandleImpl**>(handle);
@@ -273,6 +278,14 @@ class ShardedCache : public ShardedCacheBase {
    }
  }

+  inline void ForEachShard(
+      const std::function<void(const CacheShard*)>& fn) const {
+    uint32_t num_shards = GetNumShards();
+    for (uint32_t i = 0; i < num_shards; i++) {
+      fn(shards_ + i);
+    }
+  }
+
  inline size_t SumOverShards(
      const std::function<size_t(CacheShard&)>& fn) const {
    uint32_t num_shards = GetNumShards();
@@ -0,0 +1,119 @@
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#include "cache/tiered_secondary_cache.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+// Creation callback for use in the lookup path. It calls the upper layer
+// create_cb to create the object, and optionally calls the compressed
+// secondary cache InsertSaved to save the compressed block. If
+// advise_erase is set, it means the primary cache wants the block to be
+// erased in the secondary cache, so we skip calling InsertSaved.
+//
+// For the time being, we assume that all blocks in the nvm tier belong to
+// the primary block cache (i.e CacheTier::kVolatileTier). That can be changed
+// if we implement demotion from the compressed secondary cache to the nvm
+// cache in the future.
+Status TieredSecondaryCache::MaybeInsertAndCreate(
+    const Slice& data, CompressionType type, CacheTier source,
+    Cache::CreateContext* ctx, MemoryAllocator* allocator,
+    Cache::ObjectPtr* out_obj, size_t* out_charge) {
+  TieredSecondaryCache::CreateContext* context =
+      static_cast<TieredSecondaryCache::CreateContext*>(ctx);
+  assert(source == CacheTier::kVolatileTier);
+  if (!context->advise_erase && type != kNoCompression) {
+    // Attempt to insert into compressed secondary cache
+    // TODO: Don't hardcode the source
+    context->comp_sec_cache->InsertSaved(*context->key, data, type, source)
+        .PermitUncheckedError();
+  }
+  // Primary cache will accept the object, so call its helper to create
+  // the object
+  return context->helper->create_cb(data, type, source, context->inner_ctx,
+                                    allocator, out_obj, out_charge);
+}
+
+// The lookup first looks up in the compressed secondary cache. If its a miss,
+// then the nvm cache lookup is called. The cache item helper and create
+// context are wrapped in order to intercept the creation callback to make
+// the decision on promoting to the compressed secondary cache.
+std::unique_ptr<SecondaryCacheResultHandle> TieredSecondaryCache::Lookup(
+    const Slice& key, const Cache::CacheItemHelper* helper,
+    Cache::CreateContext* create_context, bool wait, bool advise_erase,
+    bool& kept_in_sec_cache) {
+  bool dummy = false;
+  std::unique_ptr<SecondaryCacheResultHandle> result =
+      target()->Lookup(key, helper, create_context, wait, advise_erase,
+                       /*kept_in_sec_cache=*/dummy);
+  // We never want the item to spill back into the secondary cache
+  kept_in_sec_cache = true;
+  if (result) {
+    assert(result->IsReady());
+    return result;
+  }
+
+  // If wait is true, then we can be a bit more efficient and avoid a memory
+  // allocation for the CReateContext.
+  const Cache::CacheItemHelper* outer_helper =
+      TieredSecondaryCache::GetHelper();
+  if (wait) {
+    TieredSecondaryCache::CreateContext ctx;
+    ctx.key = &key;
+    ctx.advise_erase = advise_erase;
+    ctx.helper = helper;
+    ctx.inner_ctx = create_context;
+    ctx.comp_sec_cache = target();
+
+    return nvm_sec_cache_->Lookup(key, outer_helper, &ctx, wait, advise_erase,
+                                  kept_in_sec_cache);
+  }
+
+  // If wait is false, i.e its an async lookup, we have to allocate a result
+  // handle for tracking purposes. Embed the CreateContext inside the handle
+  // so we need only allocate memory once instead of twice.
+  std::unique_ptr<ResultHandle> handle(new ResultHandle());
+  handle->ctx()->key = &key;
+  handle->ctx()->advise_erase = advise_erase;
+  handle->ctx()->helper = helper;
+  handle->ctx()->inner_ctx = create_context;
+  handle->ctx()->comp_sec_cache = target();
+  handle->SetInnerHandle(nvm_sec_cache_->Lookup(
+      key, outer_helper, handle->ctx(), wait, advise_erase, kept_in_sec_cache));
+  if (!handle->inner_handle()) {
+    handle.reset();
+  } else {
+    result.reset(handle.release());
+  }
+
+  return result;
+}
+
+// Call the nvm cache WaitAll to complete the lookups
+void TieredSecondaryCache::WaitAll(
+    std::vector<SecondaryCacheResultHandle*> handles) {
+  std::vector<SecondaryCacheResultHandle*> nvm_handles;
+  std::vector<ResultHandle*> my_handles;
+  nvm_handles.reserve(handles.size());
+  for (auto handle : handles) {
+    // The handle could belong to the compressed secondary cache. Skip if
+    // that's the case.
+    if (handle->IsReady()) {
+      continue;
+    }
+    ResultHandle* hdl = static_cast<ResultHandle*>(handle);
+    nvm_handles.push_back(hdl->inner_handle());
+    my_handles.push_back(hdl);
+  }
+  nvm_sec_cache_->WaitAll(nvm_handles);
+  for (auto handle : my_handles) {
+    assert(handle->IsReady());
+    auto nvm_handle = handle->inner_handle();
+    handle->SetSize(nvm_handle->Size());
+    handle->SetValue(nvm_handle->Value());
+  }
+}
+
+}  // namespace ROCKSDB_NAMESPACE
@@ -0,0 +1,155 @@
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include "rocksdb/cache.h"
+#include "rocksdb/secondary_cache.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+// A SecondaryCache that implements stacking of a compressed secondary cache
+// and a non-volatile (local flash) cache. It implements an admission
+// policy of warming the bottommost tier (local flash) with compressed
+// blocks from the SST on misses, and on hits in the bottommost tier,
+// promoting to the compressed and/or primary block cache. The admission
+// policies of the primary block cache and compressed secondary cache remain
+// unchanged - promote on second access. There is no demotion ofablocks
+// evicted from a tier. They are just discarded.
+//
+// In order to properly handle compressed blocks directly read from SSTs, and
+// to allow writeback of blocks compressed by the compressed secondary
+// cache in the future, we make use of the compression type and source
+// cache tier arguments in InsertSaved.
+class TieredSecondaryCache : public SecondaryCacheWrapper {
+ public:
+  TieredSecondaryCache(std::shared_ptr<SecondaryCache> comp_sec_cache,
+                       std::shared_ptr<SecondaryCache> nvm_sec_cache,
+                       TieredAdmissionPolicy adm_policy)
+      : SecondaryCacheWrapper(comp_sec_cache), nvm_sec_cache_(nvm_sec_cache) {
+#ifndef NDEBUG
+    assert(adm_policy == TieredAdmissionPolicy::kAdmPolicyThreeQueue);
+#else
+    (void)adm_policy;
+#endif
+  }
+
+  ~TieredSecondaryCache() override {}
+
+  const char* Name() const override { return "TieredSecondaryCache"; }
+
+  // This is a no-op as we currently don't allow demotion (i.e
+  // insertion by the upper layer) of evicted blocks.
+  virtual Status Insert(const Slice& /*key*/, Cache::ObjectPtr /*obj*/,
+                        const Cache::CacheItemHelper* /*helper*/,
+                        bool /*force_insert*/) override {
+    return Status::OK();
+  }
+
+  // Warm up the nvm tier directly
+  virtual Status InsertSaved(
+      const Slice& key, const Slice& saved,
+      CompressionType type = CompressionType::kNoCompression,
+      CacheTier source = CacheTier::kVolatileTier) override {
+    return nvm_sec_cache_->InsertSaved(key, saved, type, source);
+  }
+
+  virtual std::unique_ptr<SecondaryCacheResultHandle> Lookup(
+      const Slice& key, const Cache::CacheItemHelper* helper,
+      Cache::CreateContext* create_context, bool wait, bool advise_erase,
+      bool& kept_in_sec_cache) override;
+
+  virtual void WaitAll(
+      std::vector<SecondaryCacheResultHandle*> handles) override;
+
+ private:
+  struct CreateContext : public Cache::CreateContext {
+    const Slice* key;
+    bool advise_erase;
+    const Cache::CacheItemHelper* helper;
+    Cache::CreateContext* inner_ctx;
+    std::shared_ptr<SecondaryCacheResultHandle> inner_handle;
+    SecondaryCache* comp_sec_cache;
+  };
+
+  class ResultHandle : public SecondaryCacheResultHandle {
+   public:
+    ~ResultHandle() override {}
+
+    bool IsReady() override {
+      return !inner_handle_ || inner_handle_->IsReady();
+    }
+
+    void Wait() override {
+      inner_handle_->Wait();
+      Complete();
+    }
+
+    size_t Size() override { return size_; }
+
+    Cache::ObjectPtr Value() override { return value_; }
+
+    void Complete() {
+      assert(IsReady());
+      size_ = inner_handle_->Size();
+      value_ = inner_handle_->Value();
+      inner_handle_.reset();
+    }
+
+    void SetInnerHandle(std::unique_ptr<SecondaryCacheResultHandle>&& handle) {
+      inner_handle_ = std::move(handle);
+    }
+
+    void SetSize(size_t size) { size_ = size; }
+
+    void SetValue(Cache::ObjectPtr val) { value_ = val; }
+
+    CreateContext* ctx() { return &ctx_; }
+
+    SecondaryCacheResultHandle* inner_handle() { return inner_handle_.get(); }
+
+   private:
+    std::unique_ptr<SecondaryCacheResultHandle> inner_handle_;
+    CreateContext ctx_;
+    size_t size_;
+    Cache::ObjectPtr value_;
+  };
+
+  static void NoopDelete(Cache::ObjectPtr /*obj*/,
+                         MemoryAllocator* /*allocator*/) {
+    assert(false);
+  }
+  static size_t ZeroSize(Cache::ObjectPtr /*obj*/) {
+    assert(false);
+    return 0;
+  }
+  static Status NoopSaveTo(Cache::ObjectPtr /*from_obj*/,
+                           size_t /*from_offset*/, size_t /*length*/,
+                           char* /*out_buf*/) {
+    assert(false);
+    return Status::OK();
+  }
+  static Status MaybeInsertAndCreate(const Slice& data, CompressionType type,
+                                     CacheTier source,
+                                     Cache::CreateContext* ctx,
+                                     MemoryAllocator* allocator,
+                                     Cache::ObjectPtr* out_obj,
+                                     size_t* out_charge);
+
+  static const Cache::CacheItemHelper* GetHelper() {
+    const static Cache::CacheItemHelper basic_helper(CacheEntryRole::kMisc,
+                                                     &NoopDelete);
+    const static Cache::CacheItemHelper maybe_insert_and_create_helper{
+        CacheEntryRole::kMisc, &NoopDelete,           &ZeroSize,
+        &NoopSaveTo,           &MaybeInsertAndCreate, &basic_helper,
+    };
+    return &maybe_insert_and_create_helper;
+  }
+
+  std::shared_ptr<SecondaryCache> comp_sec_cache_;
+  std::shared_ptr<SecondaryCache> nvm_sec_cache_;
+};
+
+}  // namespace ROCKSDB_NAMESPACE
@@ -0,0 +1,711 @@
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+//
+#include "cache/compressed_secondary_cache.h"
+#include "cache/secondary_cache_adapter.h"
+#include "db/db_test_util.h"
+#include "rocksdb/cache.h"
+#include "rocksdb/secondary_cache.h"
+#include "typed_cache.h"
+#include "util/random.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+class TestSecondaryCache : public SecondaryCache {
+ public:
+  explicit TestSecondaryCache(size_t capacity)
+      : cache_(NewLRUCache(capacity, 0, false, 0.5 /* high_pri_pool_ratio */,
+                           nullptr, kDefaultToAdaptiveMutex,
+                           kDontChargeCacheMetadata)),
+        num_insert_saved_(0),
+        num_hits_(0),
+        num_misses_(0) {}
+
+  const char* Name() const override { return "TestSecondaryCache"; }
+
+  Status Insert(const Slice& /*key*/, Cache::ObjectPtr /*value*/,
+                const Cache::CacheItemHelper* /*helper*/,
+                bool /*force_insert*/) override {
+    assert(false);
+    return Status::NotSupported();
+  }
+
+  Status InsertSaved(const Slice& key, const Slice& saved,
+                     CompressionType type = kNoCompression,
+                     CacheTier source = CacheTier::kVolatileTier) override {
+    CheckCacheKeyCommonPrefix(key);
+    size_t size;
+    char* buf;
+    Status s;
+
+    num_insert_saved_++;
+    size = saved.size();
+    buf = new char[size + sizeof(uint64_t) + 2 * sizeof(uint16_t)];
+    EncodeFixed64(buf, size);
+    buf += sizeof(uint64_t);
+    EncodeFixed16(buf, type);
+    buf += sizeof(uint16_t);
+    EncodeFixed16(buf, (uint16_t)source);
+    buf += sizeof(uint16_t);
+    memcpy(buf, saved.data(), size);
+    buf -= sizeof(uint64_t) + 2 * sizeof(uint16_t);
+    if (!s.ok()) {
+      delete[] buf;
+      return s;
+    }
+    return cache_.Insert(key, buf, size);
+  }
+
+  std::unique_ptr<SecondaryCacheResultHandle> Lookup(
+      const Slice& key, const Cache::CacheItemHelper* helper,
+      Cache::CreateContext* create_context, bool wait, bool /*advise_erase*/,
+      bool& kept_in_sec_cache) override {
+    std::string key_str = key.ToString();
+    TEST_SYNC_POINT_CALLBACK("TestSecondaryCache::Lookup", &key_str);
+
+    std::unique_ptr<SecondaryCacheResultHandle> secondary_handle;
+    kept_in_sec_cache = false;
+
+    TypedHandle* handle = cache_.Lookup(key);
+    if (handle) {
+      num_hits_++;
+      Cache::ObjectPtr value = nullptr;
+      size_t charge = 0;
+      Status s;
+      char* ptr = cache_.Value(handle);
+      CompressionType type;
+      CacheTier source;
+      size_t size = DecodeFixed64(ptr);
+      ptr += sizeof(uint64_t);
+      type = static_cast<CompressionType>(DecodeFixed16(ptr));
+      ptr += sizeof(uint16_t);
+      source = static_cast<CacheTier>(DecodeFixed16(ptr));
+      assert(source == CacheTier::kVolatileTier);
+      ptr += sizeof(uint16_t);
+      s = helper->create_cb(Slice(ptr, size), type, source, create_context,
+                            /*alloc*/ nullptr, &value, &charge);
+      if (s.ok()) {
+        secondary_handle.reset(new TestSecondaryCacheResultHandle(
+            cache_.get(), handle, value, charge, /*ready=*/wait));
+        kept_in_sec_cache = true;
+      } else {
+        cache_.Release(handle);
+      }
+    } else {
+      num_misses_++;
+    }
+    return secondary_handle;
+  }
+
+  bool SupportForceErase() const override { return false; }
+
+  void Erase(const Slice& /*key*/) override {}
+
+  void WaitAll(std::vector<SecondaryCacheResultHandle*> handles) override {
+    for (SecondaryCacheResultHandle* handle : handles) {
+      TestSecondaryCacheResultHandle* sec_handle =
+          static_cast<TestSecondaryCacheResultHandle*>(handle);
+      EXPECT_FALSE(sec_handle->IsReady());
+      sec_handle->SetReady();
+    }
+  }
+
+  std::string GetPrintableOptions() const override { return ""; }
+
+  uint32_t num_insert_saved() { return num_insert_saved_; }
+
+  uint32_t num_hits() { return num_hits_; }
+
+  uint32_t num_misses() { return num_misses_; }
+
+  void CheckCacheKeyCommonPrefix(const Slice& key) {
+    Slice current_prefix(key.data(), OffsetableCacheKey::kCommonPrefixSize);
+    if (ckey_prefix_.empty()) {
+      ckey_prefix_ = current_prefix.ToString();
+    } else {
+      EXPECT_EQ(ckey_prefix_, current_prefix.ToString());
+    }
+  }
+
+ private:
+  class TestSecondaryCacheResultHandle : public SecondaryCacheResultHandle {
+   public:
+    TestSecondaryCacheResultHandle(Cache* cache, Cache::Handle* handle,
+                                   Cache::ObjectPtr value, size_t size,
+                                   bool ready)
+        : cache_(cache),
+          handle_(handle),
+          value_(value),
+          size_(size),
+          is_ready_(ready) {}
+
+    ~TestSecondaryCacheResultHandle() override { cache_->Release(handle_); }
+
+    bool IsReady() override { return is_ready_; }
+
+    void Wait() override {}
+
+    Cache::ObjectPtr Value() override {
+      assert(is_ready_);
+      return value_;
+    }
+
+    size_t Size() override { return Value() ? size_ : 0; }
+
+    void SetReady() { is_ready_ = true; }
+
+   private:
+    Cache* cache_;
+    Cache::Handle* handle_;
+    Cache::ObjectPtr value_;
+    size_t size_;
+    bool is_ready_;
+  };
+
+  using SharedCache =
+      BasicTypedSharedCacheInterface<char[], CacheEntryRole::kMisc>;
+  using TypedHandle = SharedCache::TypedHandle;
+  SharedCache cache_;
+  uint32_t num_insert_saved_;
+  uint32_t num_hits_;
+  uint32_t num_misses_;
+  std::string ckey_prefix_;
+};
+
+class DBTieredSecondaryCacheTest : public DBTestBase {
+ public:
+  DBTieredSecondaryCacheTest()
+      : DBTestBase("db_tiered_secondary_cache_test", /*env_do_fsync=*/true) {}
+
+  std::shared_ptr<Cache> NewCache(size_t pri_capacity,
+                                  size_t compressed_capacity,
+                                  size_t nvm_capacity,
+                                  TieredAdmissionPolicy adm_policy =
+                                      TieredAdmissionPolicy::kAdmPolicyAuto) {
+    LRUCacheOptions lru_opts;
+    TieredCacheOptions opts;
+    lru_opts.capacity = 0;
+    lru_opts.num_shard_bits = 0;
+    lru_opts.high_pri_pool_ratio = 0;
+    opts.cache_opts = &lru_opts;
+    opts.cache_type = PrimaryCacheType::kCacheTypeLRU;
+    opts.comp_cache_opts.capacity = 0;
+    opts.comp_cache_opts.num_shard_bits = 0;
+    opts.total_capacity = pri_capacity + compressed_capacity;
+    opts.compressed_secondary_ratio =
+        (double)compressed_capacity / opts.total_capacity;
+    if (nvm_capacity > 0) {
+      nvm_sec_cache_.reset(new TestSecondaryCache(nvm_capacity));
+      opts.nvm_sec_cache = nvm_sec_cache_;
+    }
+    opts.adm_policy = adm_policy;
+    cache_ = NewTieredCache(opts);
+    assert(cache_ != nullptr);
+
+    return cache_;
+  }
+
+  TestSecondaryCache* nvm_sec_cache() { return nvm_sec_cache_.get(); }
+
+  CompressedSecondaryCache* compressed_secondary_cache() {
+    return static_cast<CompressedSecondaryCache*>(
+        static_cast<CacheWithSecondaryAdapter*>(cache_.get())
+            ->TEST_GetSecondaryCache());
+  }
+
+ private:
+  std::shared_ptr<Cache> cache_;
+  std::shared_ptr<TestSecondaryCache> nvm_sec_cache_;
+};
+
+// In this test, the block size is set to 4096. Each value is 1007 bytes, so
+// each data block contains exactly 4 KV pairs. Metadata blocks are not
+// cached, so we can accurately estimate the cache usage.
+TEST_F(DBTieredSecondaryCacheTest, BasicTest) {
+  if (!LZ4_Supported()) {
+    ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
+    return;
+  }
+
+  BlockBasedTableOptions table_options;
+  // We want a block cache of size 5KB, and a compressed secondary cache of
+  // size 5KB. However, we specify a block cache size of 256KB here in order
+  // to take into account the cache reservation in the block cache on
+  // behalf of the compressed cache. The unit of cache reservation is 256KB.
+  // The effective block cache capacity will be calculated as 256 + 5 = 261KB,
+  // and 256KB will be reserved for the compressed cache, leaving 5KB for
+  // the primary block cache. We only have to worry about this here because
+  // the cache size is so small.
+  table_options.block_cache = NewCache(256 * 1024, 5 * 1024, 256 * 1024);
+  table_options.block_size = 4 * 1024;
+  table_options.cache_index_and_filter_blocks = false;
+  Options options = GetDefaultOptions();
+  options.create_if_missing = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+
+  // Disable paranoid_file_checks so that flush will not read back the newly
+  // written file
+  options.paranoid_file_checks = false;
+  DestroyAndReopen(options);
+  Random rnd(301);
+  const int N = 256;
+  for (int i = 0; i < N; i++) {
+    std::string p_v;
+    test::CompressibleString(&rnd, 0.5, 1007, &p_v);
+    ASSERT_OK(Put(Key(i), p_v));
+  }
+
+  ASSERT_OK(Flush());
+
+  // The first 2 Gets, for keys 0 and 5, will load the corresponding data
+  // blocks as they will be cache misses. The nvm secondary cache will be
+  // warmed up with the compressed blocks
+  std::string v = Get(Key(0));
+  ASSERT_EQ(1007, v.size());
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 1u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 1u);
+
+  v = Get(Key(5));
+  ASSERT_EQ(1007, v.size());
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 2u);
+
+  // At this point, the nvm cache is warmed up with the data blocks for 0
+  // and 5. The next Get will lookup the block in nvm and will be a hit.
+  // It will be created as a standalone entry in memory, and a placeholder
+  // will be inserted in the primary and compressed caches.
+  v = Get(Key(0));
+  ASSERT_EQ(1007, v.size());
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 1u);
+
+  // For this Get, the primary and compressed only have placeholders for
+  // the required data block. So we will lookup the nvm cache and find the
+  // block there. This time, the block will be promoted to the primary
+  // block cache. No promotion to the compressed secondary cache happens,
+  // and it will retain the placeholder.
+  v = Get(Key(0));
+  ASSERT_EQ(1007, v.size());
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 2u);
+
+  // This Get will find the data block in the primary cache.
+  v = Get(Key(0));
+  ASSERT_EQ(1007, v.size());
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 2u);
+
+  // We repeat the sequence for key 5. This will end up evicting the block
+  // for 0 from the in-memory cache.
+  v = Get(Key(5));
+  ASSERT_EQ(1007, v.size());
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 3u);
+
+  v = Get(Key(5));
+  ASSERT_EQ(1007, v.size());
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 4u);
+
+  v = Get(Key(5));
+  ASSERT_EQ(1007, v.size());
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 4u);
+
+  // This Get for key 0 will find the data block in nvm. Since the compressed
+  // cache still has the placeholder, the block (compressed) will be
+  // admitted. It is theh inserted into the primary as a standalone entry.
+  v = Get(Key(0));
+  ASSERT_EQ(1007, v.size());
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 5u);
+
+  // This Get for key 0 will find the data block in the compressed secondary
+  // cache.
+  v = Get(Key(0));
+  ASSERT_EQ(1007, v.size());
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 2u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 5u);
+
+  Destroy(options);
+}
+
+// This test is very similar to BasicTest, except it calls MultiGet rather
+// than Get, in order to exercise the async lookup and WaitAll path.
+TEST_F(DBTieredSecondaryCacheTest, BasicMultiGetTest) {
+  if (!LZ4_Supported()) {
+    ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
+    return;
+  }
+
+  BlockBasedTableOptions table_options;
+  table_options.block_cache = NewCache(260 * 1024, 10 * 1024, 256 * 1024);
+  table_options.block_size = 4 * 1024;
+  table_options.cache_index_and_filter_blocks = false;
+  Options options = GetDefaultOptions();
+  options.create_if_missing = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+
+  options.paranoid_file_checks = false;
+  DestroyAndReopen(options);
+  Random rnd(301);
+  const int N = 256;
+  for (int i = 0; i < N; i++) {
+    std::string p_v;
+    test::CompressibleString(&rnd, 0.5, 1007, &p_v);
+    ASSERT_OK(Put(Key(i), p_v));
+  }
+
+  ASSERT_OK(Flush());
+
+  std::vector<std::string> keys;
+  std::vector<std::string> values;
+
+  keys.push_back(Key(0));
+  keys.push_back(Key(4));
+  keys.push_back(Key(8));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 3u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 3u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 0u);
+
+  keys.clear();
+  values.clear();
+  keys.push_back(Key(12));
+  keys.push_back(Key(16));
+  keys.push_back(Key(20));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 0u);
+
+  keys.clear();
+  values.clear();
+  keys.push_back(Key(0));
+  keys.push_back(Key(4));
+  keys.push_back(Key(8));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 3u);
+
+  keys.clear();
+  values.clear();
+  keys.push_back(Key(0));
+  keys.push_back(Key(4));
+  keys.push_back(Key(8));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 6u);
+
+  keys.clear();
+  values.clear();
+  keys.push_back(Key(0));
+  keys.push_back(Key(4));
+  keys.push_back(Key(8));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 6u);
+
+  keys.clear();
+  values.clear();
+  keys.push_back(Key(12));
+  keys.push_back(Key(16));
+  keys.push_back(Key(20));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 9u);
+
+  keys.clear();
+  values.clear();
+  keys.push_back(Key(12));
+  keys.push_back(Key(16));
+  keys.push_back(Key(20));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 12u);
+
+  keys.clear();
+  values.clear();
+  keys.push_back(Key(12));
+  keys.push_back(Key(16));
+  keys.push_back(Key(20));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 12u);
+
+  Destroy(options);
+}
+
+TEST_F(DBTieredSecondaryCacheTest, WaitAllTest) {
+  if (!LZ4_Supported()) {
+    ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
+    return;
+  }
+
+  BlockBasedTableOptions table_options;
+  table_options.block_cache = NewCache(250 * 1024, 20 * 1024, 256 * 1024);
+  table_options.block_size = 4 * 1024;
+  table_options.cache_index_and_filter_blocks = false;
+  Options options = GetDefaultOptions();
+  options.create_if_missing = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+
+  options.paranoid_file_checks = false;
+  DestroyAndReopen(options);
+  Random rnd(301);
+  const int N = 256;
+  for (int i = 0; i < N; i++) {
+    std::string p_v;
+    test::CompressibleString(&rnd, 0.5, 1007, &p_v);
+    ASSERT_OK(Put(Key(i), p_v));
+  }
+
+  ASSERT_OK(Flush());
+
+  std::vector<std::string> keys;
+  std::vector<std::string> values;
+
+  keys.push_back(Key(0));
+  keys.push_back(Key(4));
+  keys.push_back(Key(8));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 3u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 3u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 0u);
+
+  keys.clear();
+  values.clear();
+  keys.push_back(Key(12));
+  keys.push_back(Key(16));
+  keys.push_back(Key(20));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 6u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 0u);
+
+  // Insert placeholders for 4 in primary and compressed
+  std::string val = Get(Key(4));
+
+  // Force placeholder 4 out of primary
+  keys.clear();
+  values.clear();
+  keys.push_back(Key(24));
+  keys.push_back(Key(28));
+  keys.push_back(Key(32));
+  keys.push_back(Key(36));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 10u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 10u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 1u);
+
+  // Now read 4 again. This will create a placeholder in primary, and insert
+  // in compressed secondary since it already has a placeholder
+  val = Get(Key(4));
+
+  // Now read 0, 4 and 8. While 4 is already in the compressed secondary
+  // cache, 0 and 8 will be read asynchronously from the nvm tier. The
+  // WaitAll will be called for all 3 blocks.
+  keys.clear();
+  values.clear();
+  keys.push_back(Key(0));
+  keys.push_back(Key(4));
+  keys.push_back(Key(8));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (auto value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 10u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 10u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 4u);
+
+  Destroy(options);
+}
+
+// This test is for iteration. It iterates through a set of keys in two
+// passes. First pass loads the compressed blocks into the nvm tier, and
+// the second pass should hit all of those blocks.
+TEST_F(DBTieredSecondaryCacheTest, IterateTest) {
+  if (!LZ4_Supported()) {
+    ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
+    return;
+  }
+
+  BlockBasedTableOptions table_options;
+  table_options.block_cache = NewCache(250 * 1024, 10 * 1024, 256 * 1024);
+  table_options.block_size = 4 * 1024;
+  table_options.cache_index_and_filter_blocks = false;
+  Options options = GetDefaultOptions();
+  options.create_if_missing = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+
+  options.paranoid_file_checks = false;
+  DestroyAndReopen(options);
+  Random rnd(301);
+  const int N = 256;
+  for (int i = 0; i < N; i++) {
+    std::string p_v;
+    test::CompressibleString(&rnd, 0.5, 1007, &p_v);
+    ASSERT_OK(Put(Key(i), p_v));
+  }
+
+  ASSERT_OK(Flush());
+
+  ReadOptions ro;
+  ro.readahead_size = 256 * 1024;
+  auto iter = dbfull()->NewIterator(ro);
+  iter->SeekToFirst();
+  for (int i = 0; i < 31; ++i) {
+    ASSERT_EQ(Key(i), iter->key().ToString());
+    ASSERT_EQ(1007, iter->value().size());
+    iter->Next();
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 8u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 8u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 0u);
+  delete iter;
+
+  iter = dbfull()->NewIterator(ro);
+  iter->SeekToFirst();
+  for (int i = 0; i < 31; ++i) {
+    ASSERT_EQ(Key(i), iter->key().ToString());
+    ASSERT_EQ(1007, iter->value().size());
+    iter->Next();
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 8u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 8u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 8u);
+  delete iter;
+
+  Destroy(options);
+}
+
+class DBTieredAdmPolicyTest
+    : public DBTieredSecondaryCacheTest,
+      public testing::WithParamInterface<TieredAdmissionPolicy> {};
+
+TEST_P(DBTieredAdmPolicyTest, CompressedOnlyTest) {
+  if (!LZ4_Supported()) {
+    ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
+    return;
+  }
+
+  BlockBasedTableOptions table_options;
+  // We want a block cache of size 10KB, and a compressed secondary cache of
+  // size 10KB. However, we specify a block cache size of 256KB here in order
+  // to take into account the cache reservation in the block cache on
+  // behalf of the compressed cache. The unit of cache reservation is 256KB.
+  // The effective block cache capacity will be calculated as 256 + 10 = 266KB,
+  // and 256KB will be reserved for the compressed cache, leaving 10KB for
+  // the primary block cache. We only have to worry about this here because
+  // the cache size is so small.
+  table_options.block_cache = NewCache(256 * 1024, 10 * 1024, 0, GetParam());
+  table_options.block_size = 4 * 1024;
+  table_options.cache_index_and_filter_blocks = false;
+  Options options = GetDefaultOptions();
+  options.create_if_missing = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+
+  size_t comp_cache_usage = compressed_secondary_cache()->TEST_GetUsage();
+  // Disable paranoid_file_checks so that flush will not read back the newly
+  // written file
+  options.paranoid_file_checks = false;
+  DestroyAndReopen(options);
+  Random rnd(301);
+  const int N = 256;
+  for (int i = 0; i < N; i++) {
+    std::string p_v;
+    test::CompressibleString(&rnd, 0.5, 1007, &p_v);
+    ASSERT_OK(Put(Key(i), p_v));
+  }
+
+  ASSERT_OK(Flush());
+
+  // The first 2 Gets, for keys 0 and 5, will load the corresponding data
+  // blocks as they will be cache misses. Since this is a 2-tier cache (
+  // primary and compressed), no warm-up should happen with the compressed
+  // blocks.
+  std::string v = Get(Key(0));
+  ASSERT_EQ(1007, v.size());
+
+  v = Get(Key(5));
+  ASSERT_EQ(1007, v.size());
+
+  ASSERT_EQ(compressed_secondary_cache()->TEST_GetUsage(), comp_cache_usage);
+
+  Destroy(options);
+}
+
+INSTANTIATE_TEST_CASE_P(
+    DBTieredAdmPolicyTest, DBTieredAdmPolicyTest,
+    ::testing::Values(TieredAdmissionPolicy::kAdmPolicyAuto,
+                      TieredAdmissionPolicy::kAdmPolicyPlaceholder,
+                      TieredAdmissionPolicy::kAdmPolicyAllowCacheHits));
+
+}  // namespace ROCKSDB_NAMESPACE
+
+int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
@@ -234,15 +234,19 @@ class FullTypedCacheHelperFns : public BasicTypedCacheHelperFns<TValue> {
    return Status::OK();
  }

-  static Status Create(const Slice& data, CreateContext* context,
+  static Status Create(const Slice& data, CompressionType type,
+                       CacheTier source, CreateContext* context,
                       MemoryAllocator* allocator, ObjectPtr* out_obj,
                       size_t* out_charge) {
    std::unique_ptr<TValue> value = nullptr;
+    if (source != CacheTier::kVolatileTier) {
+      return Status::InvalidArgument();
+    }
    if constexpr (sizeof(TCreateContext) > 0) {
      TCreateContext* tcontext = static_cast<TCreateContext*>(context);
-      tcontext->Create(&value, out_charge, data, allocator);
+      tcontext->Create(&value, out_charge, data, type, allocator);
    } else {
-      TCreateContext::Create(&value, out_charge, data, allocator);
+      TCreateContext::Create(&value, out_charge, data, type, allocator);
    }
    *out_obj = UpCastValue(value.release());
    return Status::OK();
@@ -301,13 +305,15 @@ class FullTypedCacheInterface
  inline Status InsertFull(
      const Slice& key, TValuePtr value, size_t charge,
      TypedHandle** handle = nullptr, Priority priority = Priority::LOW,
-      CacheTier lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier) {
+      CacheTier lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier,
+      const Slice& compressed = Slice(),
+      CompressionType type = CompressionType::kNoCompression) {
    auto untyped_handle = reinterpret_cast<Handle**>(handle);
-    auto helper = lowest_used_cache_tier == CacheTier::kNonVolatileBlockTier
+    auto helper = lowest_used_cache_tier > CacheTier::kVolatileTier
                      ? GetFullHelper()
                      : GetBasicHelper();
    return this->cache_->Insert(key, UpCastValue(value), helper, charge,
-                                untyped_handle, priority);
+                                untyped_handle, priority, compressed, type);
  }

  // Like SecondaryCache::InsertSaved, with SecondaryCache compatibility
@@ -319,9 +325,9 @@ class FullTypedCacheInterface
      size_t* out_charge = nullptr) {
    ObjectPtr value;
    size_t charge;
-    Status st = GetFullHelper()->create_cb(data, create_context,
-                                           this->cache_->memory_allocator(),
-                                           &value, &charge);
+    Status st = GetFullHelper()->create_cb(
+        data, kNoCompression, CacheTier::kVolatileTier, create_context,
+        this->cache_->memory_allocator(), &value, &charge);
    if (out_charge) {
      *out_charge = charge;
    }
@@ -340,7 +346,7 @@ class FullTypedCacheInterface
      const Slice& key, TCreateContext* create_context = nullptr,
      Priority priority = Priority::LOW, Statistics* stats = nullptr,
      CacheTier lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier) {
-    if (lowest_used_cache_tier == CacheTier::kNonVolatileBlockTier) {
+    if (lowest_used_cache_tier > CacheTier::kVolatileTier) {
      return reinterpret_cast<TypedHandle*>(this->cache_->Lookup(
          key, GetFullHelper(), create_context, priority, stats));
    } else {
@@ -352,7 +358,7 @@ class FullTypedCacheInterface
  inline void StartAsyncLookupFull(
      TypedAsyncLookupHandle& async_handle,
      CacheTier lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier) {
-    if (lowest_used_cache_tier == CacheTier::kNonVolatileBlockTier) {
+    if (lowest_used_cache_tier > CacheTier::kVolatileTier) {
      async_handle.helper = GetFullHelper();
      this->cache_->StartAsyncLookup(async_handle);
    } else {
@@ -21,6 +21,8 @@ CRASHTEST_PY=$(PYTHON) -u tools/db_crashtest.py --stress_cmd=$(DB_STRESS_CMD) --
 	blackbox_crash_test_with_multiops_wp_txn \
 	crash_test_with_tiered_storage blackbox_crash_test_with_tiered_storage \
 	whitebox_crash_test_with_tiered_storage \
+	whitebox_crash_test_with_optimistic_txn \
+	blackbox_crash_test_with_optimistic_txn \

 crash_test: $(DB_STRESS_CMD)
 # Do not parallelize
@@ -37,6 +39,11 @@ crash_test_with_txn: $(DB_STRESS_CMD)
 	$(CRASHTEST_MAKE) whitebox_crash_test_with_txn
 	$(CRASHTEST_MAKE) blackbox_crash_test_with_txn

+crash_test_with_optimistic_txn: $(DB_STRESS_CMD)
+# Do not parallelize
+	$(CRASHTEST_MAKE) whitebox_crash_test_with_optimistic_txn
+	$(CRASHTEST_MAKE) blackbox_crash_test_with_optimistic_txn
+
 crash_test_with_best_efforts_recovery: blackbox_crash_test_with_best_efforts_recovery

 crash_test_with_ts: $(DB_STRESS_CMD)
@@ -80,6 +87,9 @@ blackbox_crash_test_with_multiops_wp_txn: $(DB_STRESS_CMD)
 blackbox_crash_test_with_tiered_storage: $(DB_STRESS_CMD)
 	$(CRASHTEST_PY) --test_tiered_storage blackbox $(CRASH_TEST_EXT_ARGS)

+blackbox_crash_test_with_optimistic_txn: $(DB_STRESS_CMD)
+	$(CRASHTEST_PY) --optimistic_txn blackbox $(CRASH_TEST_EXT_ARGS)
+
 ifeq ($(CRASH_TEST_KILL_ODD),)
  CRASH_TEST_KILL_ODD=888887
 endif
@@ -105,3 +115,7 @@ whitebox_crash_test_with_ts: $(DB_STRESS_CMD)
 whitebox_crash_test_with_tiered_storage: $(DB_STRESS_CMD)
 	$(CRASHTEST_PY) --test_tiered_storage whitebox --random_kill_odd \
      $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
+
+whitebox_crash_test_with_optimistic_txn: $(DB_STRESS_CMD)
+	$(CRASHTEST_PY) --optimistic_txn whitebox --random_kill_odd \
+      $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
@@ -19,6 +19,14 @@

 namespace ROCKSDB_NAMESPACE {

+inline static SequenceNumber GetSeqNum(const DBImpl* db, const Snapshot* s) {
+  if (s) {
+    return s->GetSequenceNumber();
+  } else {
+    return db->GetLatestSequenceNumber();
+  }
+}
+
 Status ArenaWrappedDBIter::GetProperty(std::string prop_name,
                                       std::string* prop) {
  if (prop_name == "rocksdb.iterator.super-version-number") {
@@ -47,12 +55,16 @@ void ArenaWrappedDBIter::Init(
  read_options_ = read_options;
  allow_refresh_ = allow_refresh;
  memtable_range_tombstone_iter_ = nullptr;
-  if (!env->GetFileSystem()->use_async_io()) {
+
+  if (!CheckFSFeatureSupport(env->GetFileSystem().get(),
+                             FSSupportedOps::kAsyncIO)) {
    read_options_.async_io = false;
  }
 }

-Status ArenaWrappedDBIter::Refresh() {
+Status ArenaWrappedDBIter::Refresh() { return Refresh(nullptr); }
+
+Status ArenaWrappedDBIter::Refresh(const Snapshot* snapshot) {
  if (cfd_ == nullptr || db_impl_ == nullptr || !allow_refresh_) {
    return Status::NotSupported("Creating renew iterator is not allowed.");
  }
@@ -61,6 +73,10 @@ Status ArenaWrappedDBIter::Refresh() {
  // correct behavior. Will be corrected automatically when we take a snapshot
  // here for the case of WritePreparedTxnDB.
  uint64_t cur_sv_number = cfd_->GetSuperVersionNumber();
+  // If we recreate a new internal iterator below (NewInternalIterator()),
+  // we will pass in read_options_. We need to make sure it
+  // has the right snapshot.
+  read_options_.snapshot = snapshot;
  TEST_SYNC_POINT("ArenaWrappedDBIter::Refresh:1");
  TEST_SYNC_POINT("ArenaWrappedDBIter::Refresh:2");
  auto reinit_internal_iter = [&]() {
@@ -70,18 +86,19 @@ Status ArenaWrappedDBIter::Refresh() {
    new (&arena_) Arena();

    SuperVersion* sv = cfd_->GetReferencedSuperVersion(db_impl_);
-    SequenceNumber latest_seq = db_impl_->GetLatestSequenceNumber();
+    assert(sv->version_number >= cur_sv_number);
+    SequenceNumber read_seq = GetSeqNum(db_impl_, snapshot);
    if (read_callback_) {
-      read_callback_->Refresh(latest_seq);
+      read_callback_->Refresh(read_seq);
    }
    Init(env, read_options_, *(cfd_->ioptions()), sv->mutable_cf_options,
-         sv->current, latest_seq,
+         sv->current, read_seq,
         sv->mutable_cf_options.max_sequential_skip_in_iterations,
-         cur_sv_number, read_callback_, db_impl_, cfd_, expose_blob_index_,
+         sv->version_number, read_callback_, db_impl_, cfd_, expose_blob_index_,
         allow_refresh_);

    InternalIterator* internal_iter = db_impl_->NewInternalIterator(
-        read_options_, cfd_, sv, &arena_, latest_seq,
+        read_options_, cfd_, sv, &arena_, read_seq,
        /* allow_unprepared_value */ true, /* db_iter */ this);
    SetIterUnderDBIter(internal_iter);
  };
@@ -90,13 +107,13 @@ Status ArenaWrappedDBIter::Refresh() {
      reinit_internal_iter();
      break;
    } else {
-      SequenceNumber latest_seq = db_impl_->GetLatestSequenceNumber();
+      SequenceNumber read_seq = GetSeqNum(db_impl_, snapshot);
      // Refresh range-tombstones in MemTable
      if (!read_options_.ignore_range_deletions) {
        SuperVersion* sv = cfd_->GetThreadLocalSuperVersion(db_impl_);
        TEST_SYNC_POINT_CALLBACK("ArenaWrappedDBIter::Refresh:SV", nullptr);
        auto t = sv->mem->NewRangeTombstoneIterator(
-            read_options_, latest_seq, false /* immutable_memtable */);
+            read_options_, read_seq, false /* immutable_memtable */);
        if (!t || t->empty()) {
          // If memtable_range_tombstone_iter_ points to a non-empty tombstone
          // iterator, then it means sv->mem is not the memtable that
@@ -126,9 +143,6 @@ Status ArenaWrappedDBIter::Refresh() {
        }
        db_impl_->ReturnAndCleanupSuperVersion(cfd_, sv);
      }
-      // Refresh latest sequence number
-      db_iter_->set_sequence(latest_seq);
-      db_iter_->set_valid(false);
      // Check again if the latest super version number is changed
      uint64_t latest_sv_number = cfd_->GetSuperVersionNumber();
      if (latest_sv_number != cur_sv_number) {
@@ -137,6 +151,8 @@ Status ArenaWrappedDBIter::Refresh() {
        cur_sv_number = latest_sv_number;
        continue;
      }
+      db_iter_->set_sequence(read_seq);
+      db_iter_->set_valid(false);
      break;
    }
  }
@@ -80,6 +80,7 @@ class ArenaWrappedDBIter : public Iterator {
  Status GetProperty(std::string prop_name, std::string* prop) override;

  Status Refresh() override;
+  Status Refresh(const Snapshot*) override;

  void Init(Env* env, const ReadOptions& read_options,
            const ImmutableOptions& ioptions,
@@ -46,7 +46,8 @@ class BlobContents {
 class BlobContentsCreator : public Cache::CreateContext {
 public:
  static void Create(std::unique_ptr<BlobContents>* out, size_t* out_charge,
-                     const Slice& contents, MemoryAllocator* alloc) {
+                     const Slice& contents, CompressionType /*type*/,
+                     MemoryAllocator* alloc) {
    auto raw = new BlobContents(AllocateAndCopyBlock(contents, alloc),
                                contents.size());
    out->reset(raw);
@@ -261,7 +261,7 @@ Status BlobFileBuilder::CompressBlobIfNeeded(

  // TODO: allow user CompressionOptions, including max_compressed_bytes_per_kb
  CompressionOptions opts;
-  CompressionContext context(blob_compression_type_);
+  CompressionContext context(blob_compression_type_, opts);
  constexpr uint64_t sample_for_compression = 0;

  CompressionInfo info(opts, context, CompressionDict::GetEmptyDict(),
@@ -406,7 +406,7 @@ TEST_F(BlobFileBuilderTest, Compression) {
  ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), 1);

  CompressionOptions opts;
-  CompressionContext context(kSnappyCompression);
+  CompressionContext context(kSnappyCompression, opts);
  constexpr uint64_t sample_for_compression = 0;

  CompressionInfo info(opts, context, CompressionDict::GetEmptyDict(),
@@ -25,7 +25,7 @@ BlobFileCache::BlobFileCache(Cache* cache,
                             HistogramImpl* blob_file_read_hist,
                             const std::shared_ptr<IOTracer>& io_tracer)
    : cache_(cache),
-      mutex_(kNumberOfMutexStripes, kGetSliceNPHash64UnseededFnPtr),
+      mutex_(kNumberOfMutexStripes),
      immutable_options_(immutable_options),
      file_options_(file_options),
      column_family_id_(column_family_id),
@@ -55,7 +55,7 @@ Status BlobFileCache::GetBlobFileReader(
  TEST_SYNC_POINT("BlobFileCache::GetBlobFileReader:DoubleCheck");

  // Check again while holding mutex
-  MutexLock lock(mutex_.get(key));
+  MutexLock lock(&mutex_.Get(key));

  handle = cache_.Lookup(key);
  if (handle) {
@@ -43,7 +43,7 @@ class BlobFileCache {
  CacheInterface cache_;
  // Note: mutex_ below is used to guard against multiple threads racing to open
  // the same file.
-  Striped<port::Mutex, Slice> mutex_;
+  Striped<CacheAlignedWrapper<port::Mutex>> mutex_;
  const ImmutableOptions* immutable_options_;
  const FileOptions* file_options_;
  uint32_t column_family_id_;
@@ -154,11 +154,9 @@ Status BlobFileReader::ReadHeader(const RandomAccessFileReader* file_reader,
    constexpr uint64_t read_offset = 0;
    constexpr size_t read_size = BlobLogHeader::kSize;

-    // TODO: rate limit reading headers from blob files.
    const Status s =
        ReadFromFile(file_reader, read_options, read_offset, read_size,
-                     statistics, &header_slice, &buf, &aligned_buf,
-                     Env::IO_TOTAL /* rate_limiter_priority */);
+                     statistics, &header_slice, &buf, &aligned_buf);
    if (!s.ok()) {
      return s;
    }
@@ -207,11 +205,9 @@ Status BlobFileReader::ReadFooter(const RandomAccessFileReader* file_reader,
    const uint64_t read_offset = file_size - BlobLogFooter::kSize;
    constexpr size_t read_size = BlobLogFooter::kSize;

-    // TODO: rate limit reading footers from blob files.
    const Status s =
        ReadFromFile(file_reader, read_options, read_offset, read_size,
-                     statistics, &footer_slice, &buf, &aligned_buf,
-                     Env::IO_TOTAL /* rate_limiter_priority */);
+                     statistics, &footer_slice, &buf, &aligned_buf);
    if (!s.ok()) {
      return s;
    }
@@ -242,8 +238,7 @@ Status BlobFileReader::ReadFromFile(const RandomAccessFileReader* file_reader,
                                    const ReadOptions& read_options,
                                    uint64_t read_offset, size_t read_size,
                                    Statistics* statistics, Slice* slice,
-                                    Buffer* buf, AlignedBuf* aligned_buf,
-                                    Env::IOPriority rate_limiter_priority) {
+                                    Buffer* buf, AlignedBuf* aligned_buf) {
  assert(slice);
  assert(buf);
  assert(aligned_buf);
@@ -264,13 +259,13 @@ Status BlobFileReader::ReadFromFile(const RandomAccessFileReader* file_reader,
    constexpr char* scratch = nullptr;

    s = file_reader->Read(io_options, read_offset, read_size, slice, scratch,
-                          aligned_buf, rate_limiter_priority);
+                          aligned_buf);
  } else {
    buf->reset(new char[read_size]);
    constexpr AlignedBuf* aligned_scratch = nullptr;

    s = file_reader->Read(io_options, read_offset, read_size, slice, buf->get(),
-                          aligned_scratch, rate_limiter_priority);
+                          aligned_scratch);
  }

  if (!s.ok()) {
@@ -345,8 +340,7 @@ Status BlobFileReader::GetBlob(
    }
    prefetched = prefetch_buffer->TryReadFromCache(
        io_options, file_reader_.get(), record_offset,
-        static_cast<size_t>(record_size), &record_slice, &s,
-        read_options.rate_limiter_priority, for_compaction);
+        static_cast<size_t>(record_size), &record_slice, &s, for_compaction);
    if (!s.ok()) {
      return s;
    }
@@ -357,10 +351,10 @@ Status BlobFileReader::GetBlob(
    PERF_COUNTER_ADD(blob_read_count, 1);
    PERF_COUNTER_ADD(blob_read_byte, record_size);
    PERF_TIMER_GUARD(blob_read_time);
-    const Status s = ReadFromFile(
-        file_reader_.get(), read_options, record_offset,
-        static_cast<size_t>(record_size), statistics_, &record_slice, &buf,
-        &aligned_buf, read_options.rate_limiter_priority);
+    const Status s =
+        ReadFromFile(file_reader_.get(), read_options, record_offset,
+                     static_cast<size_t>(record_size), statistics_,
+                     &record_slice, &buf, &aligned_buf);
    if (!s.ok()) {
      return s;
    }
@@ -439,11 +433,11 @@ void BlobFileReader::MultiGetBlob(
    assert(req->offset >= adjustment);
    adjustments.push_back(adjustment);

-    FSReadRequest read_req = {};
+    FSReadRequest read_req;
    read_req.offset = req->offset - adjustment;
    read_req.len = req->len + adjustment;
-    read_reqs.emplace_back(read_req);
    total_len += read_req.len;
+    read_reqs.emplace_back(std::move(read_req));
  }

  RecordTick(statistics_, BLOB_DB_BLOB_FILE_BYTES_READ, total_len);
@@ -468,9 +462,12 @@ void BlobFileReader::MultiGetBlob(
  TEST_SYNC_POINT("BlobFileReader::MultiGetBlob:ReadFromFile");
  PERF_COUNTER_ADD(blob_read_count, num_blobs);
  PERF_COUNTER_ADD(blob_read_byte, total_len);
-  s = file_reader_->MultiRead(IOOptions(), read_reqs.data(), read_reqs.size(),
-                              direct_io ? &aligned_buf : nullptr,
-                              read_options.rate_limiter_priority);
+  IOOptions opts;
+  s = file_reader_->PrepareIOOptions(read_options, opts);
+  if (s.ok()) {
+    s = file_reader_->MultiRead(opts, read_reqs.data(), read_reqs.size(),
+                                direct_io ? &aligned_buf : nullptr);
+  }
  if (!s.ok()) {
    for (auto& req : read_reqs) {
      req.status.PermitUncheckedError();
@@ -588,7 +585,8 @@ Status BlobFileReader::UncompressBlobIfNeeded(
  assert(result);

  if (compression_type == kNoCompression) {
-    BlobContentsCreator::Create(result, nullptr, value_slice, allocator);
+    BlobContentsCreator::Create(result, nullptr, value_slice, kNoCompression,
+                                allocator);
    return Status::OK();
  }

@@ -89,8 +89,7 @@ class BlobFileReader {
                             const ReadOptions& read_options,
                             uint64_t read_offset, size_t read_size,
                             Statistics* statistics, Slice* slice, Buffer* buf,
-                             AlignedBuf* aligned_buf,
-                             Env::IOPriority rate_limiter_priority);
+                             AlignedBuf* aligned_buf);

  static Status VerifyBlob(const Slice& record_slice, const Slice& user_key,
                           uint64_t value_size);
@@ -74,7 +74,7 @@ void WriteBlobFile(const ImmutableOptions& immutable_options,
    }
  } else {
    CompressionOptions opts;
-    CompressionContext context(compression);
+    CompressionContext context(compression, opts);
    constexpr uint64_t sample_for_compression = 0;
    CompressionInfo info(opts, context, CompressionDict::GetEmptyDict(),
                         compression, sample_for_compression);
@@ -29,9 +29,8 @@ Status BlobLogSequentialReader::ReadSlice(uint64_t size, Slice* slice,

  StopWatch read_sw(clock_, statistics_, BLOB_DB_BLOB_FILE_READ_MICROS);
  // TODO: rate limit `BlobLogSequentialReader` reads (it appears unused?)
-  Status s =
-      file_->Read(IOOptions(), next_byte_, static_cast<size_t>(size), slice,
-                  buf, nullptr, Env::IO_TOTAL /* rate_limiter_priority */);
+  Status s = file_->Read(IOOptions(), next_byte_, static_cast<size_t>(size),
+                         slice, buf, nullptr);
  next_byte_ += size;
  if (!s.ok()) {
    return s;
@@ -76,7 +76,7 @@ void WriteBlobFile(const ImmutableOptions& immutable_options,
    }
  } else {
    CompressionOptions opts;
-    CompressionContext context(compression);
+    CompressionContext context(compression, opts);
    constexpr uint64_t sample_for_compression = 0;
    CompressionInfo info(opts, context, CompressionDict::GetEmptyDict(),
                         compression, sample_for_compression);
@@ -96,9 +96,13 @@ class DBBlobIndexTest : public DBTestBase {
  }

  ArenaWrappedDBIter* GetBlobIterator() {
-    return dbfull()->NewIteratorImpl(
-        ReadOptions(), cfd(), dbfull()->GetLatestSequenceNumber(),
-        nullptr /*read_callback*/, true /*expose_blob_index*/);
+    ColumnFamilyData* column_family = cfd();
+    DBImpl* db_impl = dbfull();
+    return db_impl->NewIteratorImpl(
+        ReadOptions(), column_family,
+        column_family->GetReferencedSuperVersion(db_impl),
+        db_impl->GetLatestSequenceNumber(), nullptr /*read_callback*/,
+        true /*expose_blob_index*/);
  }

  Options GetTestOptions() {
@@ -15,6 +15,7 @@

 #include "db/blob/blob_file_builder.h"
 #include "db/compaction/compaction_iterator.h"
+#include "db/dbformat.h"
 #include "db/event_helpers.h"
 #include "db/internal_stats.h"
 #include "db/merge_helper.h"
@@ -202,24 +203,42 @@ Status BuildTable(
        blob_file_builder.get(), ioptions.allow_data_in_errors,
        ioptions.enforce_single_del_contracts,
        /*manual_compaction_canceled=*/kManualCompactionCanceledFalse,
+        true /* must_count_input_entries */,
        /*compaction=*/nullptr, compaction_filter.get(),
        /*shutting_down=*/nullptr, db_options.info_log, full_history_ts_low);

+    const size_t ts_sz = ucmp->timestamp_size();
+    const bool strip_timestamp =
+        ts_sz > 0 && !ioptions.persist_user_defined_timestamps;
+
+    std::string key_after_flush_buf;
    c_iter.SeekToFirst();
    for (; c_iter.Valid(); c_iter.Next()) {
      const Slice& key = c_iter.key();
      const Slice& value = c_iter.value();
      const ParsedInternalKey& ikey = c_iter.ikey();
-      // Generate a rolling 64-bit hash of the key and values
-      // Note :
-      // Here "key" integrates 'sequence_number'+'kType'+'user key'.
-      s = output_validator.Add(key, value);
+      Slice key_after_flush = key;
+      // If user defined timestamps will be stripped from user key after flush,
+      // the in memory version of the key act logically the same as one with a
+      // minimum timestamp. We update the timestamp here so file boundary and
+      // output validator, block builder all see the effect of the stripping.
+      if (strip_timestamp) {
+        key_after_flush_buf.clear();
+        ReplaceInternalKeyWithMinTimestamp(&key_after_flush_buf, key, ts_sz);
+        key_after_flush = key_after_flush_buf;
+      }
+
+      //  Generate a rolling 64-bit hash of the key and values
+      //  Note :
+      //  Here "key" integrates 'sequence_number'+'kType'+'user key'.
+      s = output_validator.Add(key_after_flush, value);
      if (!s.ok()) {
        break;
      }
-      builder->Add(key, value);
+      builder->Add(key_after_flush, value);

-      s = meta->UpdateBoundaries(key, value, ikey.sequence, ikey.type);
+      s = meta->UpdateBoundaries(key_after_flush, value, ikey.sequence,
+                                 ikey.type);
      if (!s.ok()) {
        break;
      }
@@ -244,6 +263,7 @@ Status BuildTable(
           range_del_it->Next()) {
        auto tombstone = range_del_it->Tombstone();
        auto kv = tombstone.Serialize();
+        // TODO(yuzhangyu): handle range deletion for UDT in memtables only.
        builder->Add(kv.first.Encode(), kv.second);
        InternalKey tombstone_end = tombstone.SerializeEndKey();
        meta->UpdateBoundariesForRange(kv.first, tombstone_end, tombstone.seq_,
@@ -267,8 +287,9 @@ Status BuildTable(
    TEST_SYNC_POINT("BuildTable:BeforeFinishBuildTable");
    const bool empty = builder->IsEmpty();
    if (num_input_entries != nullptr) {
+      assert(c_iter.HasNumInputEntryScanned());
      *num_input_entries =
-          c_iter.num_input_entry_scanned() + num_unfragmented_tombstones;
+          c_iter.NumInputEntryScanned() + num_unfragmented_tombstones;
    }
    if (!s.ok() || empty) {
      builder->Abandon();
@@ -293,6 +314,8 @@ Status BuildTable(
      meta->fd.file_size = file_size;
      meta->tail_size = builder->GetTailSize();
      meta->marked_for_compaction = builder->NeedCompact();
+      meta->user_defined_timestamps_persisted =
+          ioptions.persist_user_defined_timestamps;
      assert(meta->fd.GetFileSize() > 0);
      tp = builder
               ->GetTableProperties();  // refresh now that builder is finished
@@ -352,6 +375,8 @@ Status BuildTable(
      s = *io_status;
    }

+    // TODO(yuzhangyu): handle the key copy in the blob when ts should be
+    // stripped.
    if (blob_file_builder) {
      if (s.ok()) {
        s = blob_file_builder->Finish();
@@ -77,6 +77,7 @@ using ROCKSDB_NAMESPACE::EnvOptions;
 using ROCKSDB_NAMESPACE::FileLock;
 using ROCKSDB_NAMESPACE::FilterPolicy;
 using ROCKSDB_NAMESPACE::FlushOptions;
+using ROCKSDB_NAMESPACE::HistogramData;
 using ROCKSDB_NAMESPACE::HyperClockCacheOptions;
 using ROCKSDB_NAMESPACE::InfoLogLevel;
 using ROCKSDB_NAMESPACE::IngestExternalFileOptions;
@@ -119,6 +120,7 @@ using ROCKSDB_NAMESPACE::TransactionDB;
 using ROCKSDB_NAMESPACE::TransactionDBOptions;
 using ROCKSDB_NAMESPACE::TransactionLogIterator;
 using ROCKSDB_NAMESPACE::TransactionOptions;
+using ROCKSDB_NAMESPACE::WaitForCompactOptions;
 using ROCKSDB_NAMESPACE::WALRecoveryMode;
 using ROCKSDB_NAMESPACE::WritableFile;
 using ROCKSDB_NAMESPACE::WriteBatch;
@@ -274,11 +276,19 @@ struct rocksdb_optimistictransactiondb_t {
 struct rocksdb_optimistictransaction_options_t {
  OptimisticTransactionOptions rep;
 };
+struct rocksdb_wait_for_compact_options_t {
+  WaitForCompactOptions rep;
+};

 struct rocksdb_compactionfiltercontext_t {
  CompactionFilter::Context rep;
 };

+struct rocksdb_statistics_histogram_data_t {
+  rocksdb_statistics_histogram_data_t() : rep() {}
+  HistogramData rep;
+};
+
 struct rocksdb_compactionfilter_t : public CompactionFilter {
  void* state_;
  void (*destructor_)(void*);
@@ -1057,6 +1067,36 @@ rocksdb_column_family_handle_t* rocksdb_create_column_family(
  return handle;
 }

+rocksdb_column_family_handle_t** rocksdb_create_column_families(
+    rocksdb_t* db, const rocksdb_options_t* column_family_options,
+    int num_column_families, const char* const* column_family_names,
+    size_t* lencfs, char** errptr) {
+  std::vector<ColumnFamilyHandle*> handles;
+  std::vector<std::string> names;
+  for (int i = 0; i != num_column_families; ++i) {
+    names.push_back(std::string(column_family_names[i]));
+  }
+  SaveError(errptr, db->rep->CreateColumnFamilies(
+                        ColumnFamilyOptions(column_family_options->rep), names,
+                        &handles));
+
+  *lencfs = handles.size();
+  rocksdb_column_family_handle_t** c_handles =
+      static_cast<rocksdb_column_family_handle_t**>(
+          malloc(sizeof(rocksdb_column_family_handle_t*) * handles.size()));
+  for (size_t i = 0; i != handles.size(); ++i) {
+    c_handles[i] = new rocksdb_column_family_handle_t;
+    c_handles[i]->rep = handles[i];
+  }
+
+  return c_handles;
+}
+
+void rocksdb_create_column_families_destroy(
+    rocksdb_column_family_handle_t** list) {
+  free(list);
+}
+
 rocksdb_column_family_handle_t* rocksdb_create_column_family_with_ttl(
    rocksdb_t* db, const rocksdb_options_t* column_family_options,
    const char* column_family_name, int ttl, char** errptr) {
@@ -2993,6 +3033,29 @@ void rocksdb_options_enable_statistics(rocksdb_options_t* opt) {
  opt->rep.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
 }

+void rocksdb_options_set_statistics_level(rocksdb_options_t* opt, int level) {
+  if (!opt->rep.statistics) {
+    return;
+  }
+
+  if (level < rocksdb_statistics_level_disable_all) {
+    level = rocksdb_statistics_level_disable_all;
+  }
+  if (level > rocksdb_statistics_level_all) {
+    level = rocksdb_statistics_level_all;
+  }
+  opt->rep.statistics->set_stats_level(
+      static_cast<ROCKSDB_NAMESPACE::StatsLevel>(level));
+}
+
+int rocksdb_options_get_statistics_level(rocksdb_options_t* opt) {
+  if (!opt->rep.statistics) {
+    return ROCKSDB_NAMESPACE::StatsLevel::kDisableAll;
+  }
+
+  return static_cast<int>(opt->rep.statistics->get_stats_level());
+}
+
 void rocksdb_options_set_skip_stats_update_on_db_open(rocksdb_options_t* opt,
                                                      unsigned char val) {
  opt->rep.skip_stats_update_on_db_open = val;
@@ -3740,16 +3803,21 @@ void rocksdb_options_set_hash_link_list_rep(rocksdb_options_t* opt,
      ROCKSDB_NAMESPACE::NewHashLinkListRepFactory(bucket_count));
 }

-void rocksdb_options_set_plain_table_factory(rocksdb_options_t* opt,
-                                             uint32_t user_key_len,
-                                             int bloom_bits_per_key,
-                                             double hash_table_ratio,
-                                             size_t index_sparseness) {
+void rocksdb_options_set_plain_table_factory(
+    rocksdb_options_t* opt, uint32_t user_key_len, int bloom_bits_per_key,
+    double hash_table_ratio, size_t index_sparseness, size_t huge_page_tlb_size,
+    char encoding_type, unsigned char full_scan_mode,
+    unsigned char store_index_in_file) {
  ROCKSDB_NAMESPACE::PlainTableOptions options;
  options.user_key_len = user_key_len;
  options.bloom_bits_per_key = bloom_bits_per_key;
  options.hash_table_ratio = hash_table_ratio;
  options.index_sparseness = index_sparseness;
+  options.huge_page_tlb_size = huge_page_tlb_size;
+  options.encoding_type =
+      static_cast<ROCKSDB_NAMESPACE::EncodingType>(encoding_type);
+  options.full_scan_mode = full_scan_mode;
+  options.store_index_in_file = store_index_in_file;

  ROCKSDB_NAMESPACE::TableFactory* factory =
      ROCKSDB_NAMESPACE::NewPlainTableFactory(options);
@@ -3827,6 +3895,26 @@ char* rocksdb_options_statistics_get_string(rocksdb_options_t* opt) {
  return nullptr;
 }

+uint64_t rocksdb_options_statistics_get_ticker_count(rocksdb_options_t* opt,
+                                                     uint32_t ticker_type) {
+  ROCKSDB_NAMESPACE::Statistics* statistics = opt->rep.statistics.get();
+  if (statistics) {
+    return statistics->getTickerCount(ticker_type);
+  }
+  return 0;
+}
+
+void rocksdb_options_statistics_get_histogram_data(
+    rocksdb_options_t* opt, uint32_t type,
+    rocksdb_statistics_histogram_data_t* const data) {
+  ROCKSDB_NAMESPACE::Statistics* statistics = opt->rep.statistics.get();
+  if (statistics) {
+    statistics->histogramData(type, &data->rep);
+  } else {
+    *data = rocksdb_statistics_histogram_data_t{};
+  }
+}
+
 void rocksdb_options_set_ratelimiter(rocksdb_options_t* opt,
                                     rocksdb_ratelimiter_t* limiter) {
  if (limiter) {
@@ -3888,6 +3976,15 @@ void rocksdb_options_add_compact_on_deletion_collector_factory(
  opt->rep.table_properties_collector_factories.emplace_back(compact_on_del);
 }

+void rocksdb_options_add_compact_on_deletion_collector_factory_del_ratio(
+    rocksdb_options_t* opt, size_t window_size, size_t num_dels_trigger,
+    double deletion_ratio) {
+  std::shared_ptr<ROCKSDB_NAMESPACE::TablePropertiesCollectorFactory>
+      compact_on_del = NewCompactOnDeletionCollectorFactory(
+          window_size, num_dels_trigger, deletion_ratio);
+  opt->rep.table_properties_collector_factories.emplace_back(compact_on_del);
+}
+
 void rocksdb_set_perf_level(int v) {
  PerfLevel level = static_cast<PerfLevel>(v);
  SetPerfLevel(level);
@@ -4492,6 +4589,11 @@ void rocksdb_readoptions_set_iter_start_ts(rocksdb_readoptions_t* opt,
  }
 }

+void rocksdb_readoptions_set_auto_readahead_size(rocksdb_readoptions_t* opt,
+                                                 unsigned char v) {
+  opt->rep.auto_readahead_size = v;
+}
+
 rocksdb_writeoptions_t* rocksdb_writeoptions_create() {
  return new rocksdb_writeoptions_t;
 }
@@ -5150,7 +5252,8 @@ rocksdb_fifo_compaction_options_t* rocksdb_fifo_compaction_options_create() {
 }

 void rocksdb_fifo_compaction_options_set_allow_compaction(
-    rocksdb_fifo_compaction_options_t* fifo_opts, unsigned char allow_compaction) {
+    rocksdb_fifo_compaction_options_t* fifo_opts,
+    unsigned char allow_compaction) {
  fifo_opts->rep.allow_compaction = allow_compaction;
 }

@@ -5579,6 +5682,20 @@ int rocksdb_transactiondb_property_int(rocksdb_transactiondb_t* db,
  }
 }

+rocksdb_t* rocksdb_transactiondb_get_base_db(rocksdb_transactiondb_t* txn_db) {
+  DB* base_db = txn_db->rep->GetBaseDB();
+
+  if (base_db != nullptr) {
+    rocksdb_t* result = new rocksdb_t;
+    result->rep = base_db;
+    return result;
+  }
+
+  return nullptr;
+}
+
+void rocksdb_transactiondb_close_base_db(rocksdb_t* base_db) { delete base_db; }
+
 rocksdb_transaction_t* rocksdb_transaction_begin(
    rocksdb_transactiondb_t* txn_db,
    const rocksdb_writeoptions_t* write_options,
@@ -6556,4 +6673,114 @@ void rocksdb_enable_manual_compaction(rocksdb_t* db) {
  db->rep->EnableManualCompaction();
 }

+rocksdb_statistics_histogram_data_t*
+rocksdb_statistics_histogram_data_create() {
+  return new rocksdb_statistics_histogram_data_t{};
+}
+
+void rocksdb_statistics_histogram_data_destroy(
+    rocksdb_statistics_histogram_data_t* data) {
+  delete data;
+}
+
+double rocksdb_statistics_histogram_data_get_median(
+    rocksdb_statistics_histogram_data_t* data) {
+  return data->rep.median;
+}
+
+double rocksdb_statistics_histogram_data_get_p95(
+    rocksdb_statistics_histogram_data_t* data) {
+  return data->rep.percentile95;
+}
+
+double rocksdb_statistics_histogram_data_get_p99(
+    rocksdb_statistics_histogram_data_t* data) {
+  return data->rep.percentile99;
+}
+
+double rocksdb_statistics_histogram_data_get_average(
+    rocksdb_statistics_histogram_data_t* data) {
+  return data->rep.average;
+}
+
+double rocksdb_statistics_histogram_data_get_std_dev(
+    rocksdb_statistics_histogram_data_t* data) {
+  return data->rep.standard_deviation;
+}
+
+double rocksdb_statistics_histogram_data_get_max(
+    rocksdb_statistics_histogram_data_t* data) {
+  return data->rep.max;
+}
+
+uint64_t rocksdb_statistics_histogram_data_get_count(
+    rocksdb_statistics_histogram_data_t* data) {
+  return data->rep.count;
+}
+
+uint64_t rocksdb_statistics_histogram_data_get_sum(
+    rocksdb_statistics_histogram_data_t* data) {
+  return data->rep.sum;
+}
+
+double rocksdb_statistics_histogram_data_get_min(
+    rocksdb_statistics_histogram_data_t* data) {
+  return data->rep.min;
+}
+
+void rocksdb_wait_for_compact(rocksdb_t* db,
+                              rocksdb_wait_for_compact_options_t* options,
+                              char** errptr) {
+  SaveError(errptr, db->rep->WaitForCompact(options->rep));
+}
+
+rocksdb_wait_for_compact_options_t* rocksdb_wait_for_compact_options_create() {
+  return new rocksdb_wait_for_compact_options_t;
+}
+
+void rocksdb_wait_for_compact_options_destroy(
+    rocksdb_wait_for_compact_options_t* opt) {
+  delete opt;
+}
+
+void rocksdb_wait_for_compact_options_set_abort_on_pause(
+    rocksdb_wait_for_compact_options_t* opt, unsigned char v) {
+  opt->rep.abort_on_pause = v;
+}
+
+unsigned char rocksdb_wait_for_compact_options_get_abort_on_pause(
+    rocksdb_wait_for_compact_options_t* opt) {
+  return opt->rep.abort_on_pause;
+}
+
+void rocksdb_wait_for_compact_options_set_flush(
+    rocksdb_wait_for_compact_options_t* opt, unsigned char v) {
+  opt->rep.flush = v;
+}
+
+unsigned char rocksdb_wait_for_compact_options_get_flush(
+    rocksdb_wait_for_compact_options_t* opt) {
+  return opt->rep.flush;
+}
+
+void rocksdb_wait_for_compact_options_set_close_db(
+    rocksdb_wait_for_compact_options_t* opt, unsigned char v) {
+  opt->rep.close_db = v;
+}
+
+unsigned char rocksdb_wait_for_compact_options_get_close_db(
+    rocksdb_wait_for_compact_options_t* opt) {
+  return opt->rep.close_db;
+}
+
+void rocksdb_wait_for_compact_options_set_timeout(
+    rocksdb_wait_for_compact_options_t* opt, uint64_t microseconds) {
+  opt->rep.timeout = std::chrono::microseconds(microseconds);
+}
+
+uint64_t rocksdb_wait_for_compact_options_get_timeout(
+    rocksdb_wait_for_compact_options_t* opt) {
+  return opt->rep.timeout.count();
+}
+
 }  // end extern "C"
@@ -3,15 +3,14 @@
   found in the LICENSE file. See the AUTHORS file for names of contributors. */
 // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.

-#include <stdio.h>
+#include "rocksdb/c.h"

 #include <assert.h>
 #include <stddef.h>
+#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <sys/types.h>
-
-#include "rocksdb/c.h"
 #ifndef OS_WIN
 #include <unistd.h>
 #endif
@@ -376,6 +375,11 @@ static rocksdb_t* CheckCompaction(rocksdb_t* db, rocksdb_options_t* options,

  // Force compaction
  rocksdb_compact_range(db, NULL, 0, NULL, 0);
+  rocksdb_wait_for_compact_options_t* wco;
+  wco = rocksdb_wait_for_compact_options_create();
+  rocksdb_wait_for_compact(db, wco, &err);
+  CheckNoError(err);
+  rocksdb_wait_for_compact_options_destroy(wco);
  // should have filtered bar, but not foo
  CheckGet(db, roptions, "foo", "foovalue");
  CheckGet(db, roptions, "bar", NULL);
@@ -721,6 +725,8 @@ int main(int argc, char** argv) {

  rocksdb_options_add_compact_on_deletion_collector_factory(options, 10000,
                                                            10001);
+  rocksdb_options_add_compact_on_deletion_collector_factory_del_ratio(
+      options, 10000, 10001, 0.0);

  StartPhase("destroy");
  rocksdb_destroy_db(options, dbname, &err);
@@ -1672,7 +1678,8 @@ int main(int argc, char** argv) {
    rocksdb_options_set_prefix_extractor(
        options, rocksdb_slicetransform_create_fixed_prefix(3));
    rocksdb_options_set_hash_skip_list_rep(options, 5000, 4, 4);
-    rocksdb_options_set_plain_table_factory(options, 4, 10, 0.75, 16);
+    rocksdb_options_set_plain_table_factory(options, 4, 10, 0.75, 16, 0, 0, 0,
+                                            0);
    rocksdb_options_set_allow_concurrent_memtable_write(options, 0);

    db = rocksdb_open(options, dbname, &err);
@@ -2058,6 +2065,15 @@ int main(int argc, char** argv) {
    CheckCondition(29.0 ==
                   rocksdb_options_get_experimental_mempurge_threshold(o));

+    CheckCondition(rocksdb_statistics_level_disable_all ==
+                   rocksdb_options_get_statistics_level(o));
+    rocksdb_options_enable_statistics(o);
+    CheckCondition(rocksdb_statistics_level_disable_all !=
+                   rocksdb_options_get_statistics_level(o));
+    rocksdb_options_set_statistics_level(o, rocksdb_statistics_level_all);
+    CheckCondition(rocksdb_statistics_level_all ==
+                   rocksdb_options_get_statistics_level(o));
+
    /* Blob Options */
    rocksdb_options_set_enable_blob_files(o, 1);
    CheckCondition(1 == rocksdb_options_get_enable_blob_files(o));
@@ -3119,6 +3135,17 @@ int main(int argc, char** argv) {
    CheckTxnDBGetCF(txn_db, roptions, cfh, "cf_foo", NULL);
    CheckTxnDBPinGetCF(txn_db, roptions, cfh, "cf_foo", NULL);

+    // memory usage
+    rocksdb_t* base_db = rocksdb_transactiondb_get_base_db(txn_db);
+    rocksdb_memory_consumers_t* consumers = rocksdb_memory_consumers_create();
+    rocksdb_memory_consumers_add_db(consumers, base_db);
+    rocksdb_memory_usage_t* usage =
+        rocksdb_approximate_memory_usage_create(consumers, &err);
+    CheckNoError(err);
+    rocksdb_approximate_memory_usage_destroy(usage);
+    rocksdb_memory_consumers_destroy(consumers);
+    rocksdb_transactiondb_close_base_db(base_db);
+
    // flush
    rocksdb_flushoptions_t* flush_options = rocksdb_flushoptions_create();
    rocksdb_flushoptions_set_wait(flush_options, 1);
@@ -3374,8 +3401,19 @@ int main(int argc, char** argv) {
    rocksdb_put(db, woptions, "key", 3, "value", 5, &err);
    CheckNoError(err);
    rocksdb_column_family_handle_t *cfh1, *cfh2;
-    cfh1 = rocksdb_create_column_family(db, db_options, "txn_db_cf1", &err);
-    cfh2 = rocksdb_create_column_family(db, db_options, "txn_db_cf2", &err);
+    char** list_const_cf_names = (char**)malloc(2 * sizeof(char*));
+    list_const_cf_names[0] = "txn_db_cf1";
+    list_const_cf_names[1] = "txn_db_cf2";
+    size_t cflen;
+    rocksdb_column_family_handle_t** list_cfh = rocksdb_create_column_families(
+        db, db_options, 2, (const char* const*)list_const_cf_names, &cflen,
+        &err);
+    free(list_const_cf_names);
+    CheckNoError(err);
+    assert(cflen == 2);
+    cfh1 = list_cfh[0];
+    cfh2 = list_cfh[1];
+    rocksdb_create_column_families_destroy(list_cfh);
    txn = rocksdb_optimistictransaction_begin(otxn_db, woptions, otxn_options,
                                              NULL);
    rocksdb_transaction_put_cf(txn, cfh1, "key_cf1", 7, "val_cf1", 7, &err);
@@ -3589,6 +3627,102 @@ int main(int argc, char** argv) {
    rocksdb_readoptions_destroy(ropts);
  }

+  StartPhase("statistics");
+  {
+    const uint32_t BYTES_WRITTEN_TICKER = 40;
+    const uint32_t DB_WRITE_HIST = 1;
+
+    rocksdb_statistics_histogram_data_t* hist =
+        rocksdb_statistics_histogram_data_create();
+    {
+      // zero by default
+      CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_median(hist));
+      CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_p95(hist));
+      CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_p99(hist));
+      CheckCondition(0.0 ==
+                     rocksdb_statistics_histogram_data_get_average(hist));
+      CheckCondition(0.0 ==
+                     rocksdb_statistics_histogram_data_get_std_dev(hist));
+      CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_max(hist));
+      CheckCondition(0 == rocksdb_statistics_histogram_data_get_count(hist));
+      CheckCondition(0 == rocksdb_statistics_histogram_data_get_sum(hist));
+      CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_min(hist));
+    }
+
+    rocksdb_close(db);
+    rocksdb_destroy_db(options, dbname, &err);
+    CheckNoError(err);
+
+    rocksdb_options_enable_statistics(options);
+    rocksdb_options_set_statistics_level(options, rocksdb_statistics_level_all);
+
+    db = rocksdb_open(options, dbname, &err);
+    CheckNoError(err);
+
+    CheckCondition(0 == rocksdb_options_statistics_get_ticker_count(
+                            options, BYTES_WRITTEN_TICKER));
+    rocksdb_options_statistics_get_histogram_data(options, DB_WRITE_HIST, hist);
+    CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_median(hist));
+    CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_p95(hist));
+    CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_p99(hist));
+    CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_average(hist));
+    CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_std_dev(hist));
+    CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_max(hist));
+    CheckCondition(0 == rocksdb_statistics_histogram_data_get_count(hist));
+    CheckCondition(0 == rocksdb_statistics_histogram_data_get_sum(hist));
+
+    int i;
+    for (i = 0; i < 10; ++i) {
+      char key = '0' + (char)i;
+      rocksdb_put(db, woptions, &key, 1, "", 1, &err);
+      CheckNoError(err);
+    }
+    CheckCondition(0 != rocksdb_options_statistics_get_ticker_count(
+                            options, BYTES_WRITTEN_TICKER));
+    rocksdb_options_statistics_get_histogram_data(options, DB_WRITE_HIST, hist);
+    CheckCondition(0.0 != rocksdb_statistics_histogram_data_get_median(hist));
+    CheckCondition(0.0 != rocksdb_statistics_histogram_data_get_p95(hist));
+    CheckCondition(0.0 != rocksdb_statistics_histogram_data_get_p99(hist));
+    CheckCondition(0.0 != rocksdb_statistics_histogram_data_get_average(hist));
+    CheckCondition(0.0 != rocksdb_statistics_histogram_data_get_std_dev(hist));
+    CheckCondition(0.0 != rocksdb_statistics_histogram_data_get_max(hist));
+    CheckCondition(0 != rocksdb_statistics_histogram_data_get_count(hist));
+    CheckCondition(0 != rocksdb_statistics_histogram_data_get_sum(hist));
+
+    rocksdb_statistics_histogram_data_destroy(hist);
+  }
+
+  StartPhase("wait_for_compact_options");
+  {
+    rocksdb_wait_for_compact_options_t* wco;
+    wco = rocksdb_wait_for_compact_options_create();
+
+    rocksdb_wait_for_compact_options_set_abort_on_pause(wco, 1);
+    CheckCondition(1 ==
+                   rocksdb_wait_for_compact_options_get_abort_on_pause(wco));
+
+    rocksdb_wait_for_compact_options_set_flush(wco, 1);
+    CheckCondition(1 == rocksdb_wait_for_compact_options_get_flush(wco));
+
+    rocksdb_wait_for_compact_options_set_close_db(wco, 1);
+    CheckCondition(1 == rocksdb_wait_for_compact_options_get_close_db(wco));
+
+    rocksdb_wait_for_compact_options_set_timeout(wco, 342);
+    CheckCondition(342 == rocksdb_wait_for_compact_options_get_timeout(wco));
+
+    rocksdb_wait_for_compact_options_destroy(wco);
+  }
+  StartPhase("wait_for_compact");
+  {
+    rocksdb_wait_for_compact_options_t* wco;
+    wco = rocksdb_wait_for_compact_options_create();
+    rocksdb_wait_for_compact_options_set_flush(wco, 1);
+
+    rocksdb_wait_for_compact(db, wco, &err);
+    CheckNoError(err);
+    rocksdb_wait_for_compact_options_destroy(wco);
+  }
+
  StartPhase("cancel_all_background_work");
  rocksdb_cancel_all_background_work(db, 1);

@@ -382,8 +382,9 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,

  const uint64_t kAdjustedTtl = 30 * 24 * 60 * 60;
  if (result.ttl == kDefaultTtl) {
-    if (is_block_based_table &&
-        result.compaction_style != kCompactionStyleFIFO) {
+    if (is_block_based_table) {
+      // FIFO also requires max_open_files=-1, which is checked in
+      // ValidateOptions().
      result.ttl = kAdjustedTtl;
    } else {
      result.ttl = 0;
@@ -391,40 +392,35 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
  }

  const uint64_t kAdjustedPeriodicCompSecs = 30 * 24 * 60 * 60;
-
-  // Turn on periodic compactions and set them to occur once every 30 days if
-  // compaction filters are used and periodic_compaction_seconds is set to the
-  // default value.
-  if (result.compaction_style != kCompactionStyleFIFO) {
+  if (result.compaction_style == kCompactionStyleLevel) {
    if ((result.compaction_filter != nullptr ||
         result.compaction_filter_factory != nullptr) &&
        result.periodic_compaction_seconds == kDefaultPeriodicCompSecs &&
        is_block_based_table) {
      result.periodic_compaction_seconds = kAdjustedPeriodicCompSecs;
    }
-  } else {
-    // result.compaction_style == kCompactionStyleFIFO
-    if (result.ttl == 0) {
-      if (is_block_based_table) {
-        if (result.periodic_compaction_seconds == kDefaultPeriodicCompSecs) {
-          result.periodic_compaction_seconds = kAdjustedPeriodicCompSecs;
-        }
-        result.ttl = result.periodic_compaction_seconds;
-      }
-    } else if (result.periodic_compaction_seconds != 0) {
-      result.ttl = std::min(result.ttl, result.periodic_compaction_seconds);
+  } else if (result.compaction_style == kCompactionStyleUniversal) {
+    if (result.periodic_compaction_seconds == kDefaultPeriodicCompSecs &&
+        is_block_based_table) {
+      result.periodic_compaction_seconds = kAdjustedPeriodicCompSecs;
+    }
+  } else if (result.compaction_style == kCompactionStyleFIFO) {
+    if (result.periodic_compaction_seconds != kDefaultPeriodicCompSecs) {
+      ROCKS_LOG_WARN(
+          db_options.info_log.get(),
+          "periodic_compaction_seconds does not support FIFO compaction. You"
+          "may want to set option TTL instead.");
    }
  }

-  // TTL compactions would work similar to Periodic Compactions in Universal in
-  // most of the cases. So, if ttl is set, execute the periodic compaction
-  // codepath.
-  if (result.compaction_style == kCompactionStyleUniversal && result.ttl != 0) {
-    if (result.periodic_compaction_seconds != 0) {
+  // For universal compaction, `ttl` and `periodic_compaction_seconds` mean the
+  // same thing, take the stricter value.
+  if (result.compaction_style == kCompactionStyleUniversal) {
+    if (result.periodic_compaction_seconds == 0) {
+      result.periodic_compaction_seconds = result.ttl;
+    } else if (result.ttl != 0) {
      result.periodic_compaction_seconds =
          std::min(result.ttl, result.periodic_compaction_seconds);
-    } else {
-      result.periodic_compaction_seconds = result.ttl;
    }
  }

@@ -480,6 +476,7 @@ void SuperVersion::Init(ColumnFamilyData* new_cfd, MemTable* new_mem,
  mem = new_mem;
  imm = new_imm;
  current = new_current;
+  full_history_ts_low = cfd->GetFullHistoryTsLow();
  cfd->Ref();
  mem->Ref();
  imm->Ref();
@@ -1122,7 +1119,7 @@ Compaction* ColumnFamilyData::PickCompaction(
      GetName(), mutable_options, mutable_db_options, current_->storage_info(),
      log_buffer);
  if (result != nullptr) {
-    result->SetInputVersion(current_);
+    result->FinalizeInputInfo(current_);
  }
  return result;
 }
@@ -1206,7 +1203,7 @@ Compaction* ColumnFamilyData::CompactRange(
      compact_range_options, begin, end, compaction_end, conflict,
      max_file_num_to_ignore, trim_ts);
  if (result != nullptr) {
-    result->SetInputVersion(current_);
+    result->FinalizeInputInfo(current_);
  }
  TEST_SYNC_POINT("ColumnFamilyData::CompactRange:Return");
  return result;
@@ -1245,30 +1242,11 @@ SuperVersion* ColumnFamilyData::GetThreadLocalSuperVersion(DBImpl* db) {
  // (if no Scrape happens).
  assert(ptr != SuperVersion::kSVInUse);
  SuperVersion* sv = static_cast<SuperVersion*>(ptr);
-  if (sv == SuperVersion::kSVObsolete ||
-      sv->version_number != super_version_number_.load()) {
+  if (sv == SuperVersion::kSVObsolete) {
    RecordTick(ioptions_.stats, NUMBER_SUPERVERSION_ACQUIRES);
-    SuperVersion* sv_to_delete = nullptr;
-
-    if (sv && sv->Unref()) {
-      RecordTick(ioptions_.stats, NUMBER_SUPERVERSION_CLEANUPS);
-      db->mutex()->Lock();
-      // NOTE: underlying resources held by superversion (sst files) might
-      // not be released until the next background job.
-      sv->Cleanup();
-      if (db->immutable_db_options().avoid_unnecessary_blocking_io) {
-        db->AddSuperVersionsToFreeQueue(sv);
-        db->SchedulePurge();
-      } else {
-        sv_to_delete = sv;
-      }
-    } else {
-      db->mutex()->Lock();
-    }
+    db->mutex()->Lock();
    sv = super_version_->Ref();
    db->mutex()->Unlock();
-
-    delete sv_to_delete;
  }
  assert(sv != nullptr);
  return sv;
@@ -1306,8 +1284,6 @@ void ColumnFamilyData::InstallSuperVersion(
  new_superversion->Init(this, mem_, imm_.current(), current_);
  SuperVersion* old_superversion = super_version_;
  super_version_ = new_superversion;
-  ++super_version_number_;
-  super_version_->version_number = super_version_number_;
  if (old_superversion == nullptr || old_superversion->current != current() ||
      old_superversion->mem != mem_ ||
      old_superversion->imm != imm_.current()) {
@@ -1342,6 +1318,8 @@ void ColumnFamilyData::InstallSuperVersion(
      sv_context->superversions_to_free.push_back(old_superversion);
    }
  }
+  ++super_version_number_;
+  super_version_->version_number = super_version_number_;
 }

 void ColumnFamilyData::ResetThreadLocalSuperVersions() {
@@ -1399,6 +1377,33 @@ Status ColumnFamilyData::ValidateOptions(
    }
  }

+  const auto* ucmp = cf_options.comparator;
+  assert(ucmp);
+  if (ucmp->timestamp_size() > 0 &&
+      !cf_options.persist_user_defined_timestamps) {
+    if (db_options.atomic_flush) {
+      return Status::NotSupported(
+          "Not persisting user-defined timestamps feature is not supported"
+          "in combination with atomic flush.");
+    }
+    if (db_options.allow_concurrent_memtable_write) {
+      return Status::NotSupported(
+          "Not persisting user-defined timestamps feature is not supported"
+          " in combination with concurrent memtable write.");
+    }
+    const char* comparator_name = cf_options.comparator->Name();
+    size_t name_size = strlen(comparator_name);
+    const char* suffix = ".u64ts";
+    size_t suffix_size = strlen(suffix);
+    if (name_size <= suffix_size ||
+        strcmp(comparator_name + name_size - suffix_size, suffix) != 0) {
+      return Status::NotSupported(
+          "Not persisting user-defined timestamps"
+          "feature only support user-defined timestamps formatted as "
+          "uint64_t.");
+    }
+  }
+
  if (cf_options.enable_blob_garbage_collection) {
    if (cf_options.blob_garbage_collection_age_cutoff < 0.0 ||
        cf_options.blob_garbage_collection_age_cutoff > 1.0) {
@@ -1538,6 +1543,34 @@ FSDirectory* ColumnFamilyData::GetDataDir(size_t path_id) const {
  return data_dirs_[path_id].get();
 }

+bool ColumnFamilyData::ShouldPostponeFlushToRetainUDT(
+    uint64_t max_memtable_id) {
+  const Comparator* ucmp = user_comparator();
+  const size_t ts_sz = ucmp->timestamp_size();
+  if (ts_sz == 0 || ioptions_.persist_user_defined_timestamps) {
+    return false;
+  }
+  // If users set the `persist_user_defined_timestamps` flag to false, they
+  // should also set the `full_history_ts_low` flag to indicate the range of
+  // user-defined timestamps to retain in memory. Otherwise, we do not
+  // explicitly postpone flush to retain UDTs.
+  const std::string& full_history_ts_low = GetFullHistoryTsLow();
+  if (full_history_ts_low.empty()) {
+    return false;
+  }
+  for (const Slice& table_newest_udt :
+       imm()->GetTablesNewestUDT(max_memtable_id)) {
+    assert(table_newest_udt.size() == full_history_ts_low.size());
+    // Checking the newest UDT contained in MemTable with ascending ID up to
+    // `max_memtable_id`. Return immediately on finding the first MemTable that
+    // needs postponing.
+    if (ucmp->CompareTimestamp(table_newest_udt, full_history_ts_low) >= 0) {
+      return true;
+    }
+  }
+  return false;
+}
+
 void ColumnFamilyData::RecoverEpochNumbers() {
  assert(current_);
  auto* vstorage = current_->storage_info();
@@ -1640,6 +1673,13 @@ ColumnFamilyData* ColumnFamilySet::CreateColumnFamily(
      db_id_, db_session_id_);
  column_families_.insert({name, id});
  column_family_data_.insert({id, new_cfd});
+  auto ucmp = new_cfd->user_comparator();
+  assert(ucmp);
+  size_t ts_sz = ucmp->timestamp_size();
+  running_ts_sz_.insert({id, ts_sz});
+  if (ts_sz > 0) {
+    ts_sz_for_record_.insert({id, ts_sz});
+  }
  max_column_family_ = std::max(max_column_family_, id);
  // add to linked list
  new_cfd->next_ = dummy_cfd_;
@@ -1655,10 +1695,13 @@ ColumnFamilyData* ColumnFamilySet::CreateColumnFamily(

 // under a DB mutex AND from a write thread
 void ColumnFamilySet::RemoveColumnFamily(ColumnFamilyData* cfd) {
-  auto cfd_iter = column_family_data_.find(cfd->GetID());
+  uint32_t cf_id = cfd->GetID();
+  auto cfd_iter = column_family_data_.find(cf_id);
  assert(cfd_iter != column_family_data_.end());
  column_family_data_.erase(cfd_iter);
  column_families_.erase(cfd->GetName());
+  running_ts_sz_.erase(cf_id);
+  ts_sz_for_record_.erase(cf_id);
 }

 // under a DB mutex OR from a write thread
@@ -211,6 +211,12 @@ struct SuperVersion {
  // Version number of the current SuperVersion
  uint64_t version_number;
  WriteStallCondition write_stall_condition;
+  // Each time `full_history_ts_low` collapses history, a new SuperVersion is
+  // installed. This field tracks the effective `full_history_ts_low` for that
+  // SuperVersion, to be used by read APIs for sanity checks. This field is
+  // immutable once SuperVersion is installed. For column family that doesn't
+  // enable UDT feature, this is an empty string.
+  std::string full_history_ts_low;

  // should be called outside the mutex
  SuperVersion() = default;
@@ -506,6 +512,12 @@ class ColumnFamilyData {
    return full_history_ts_low_;
  }

+  // REQUIRES: DB mutex held.
+  // Return true if flushing up to MemTables with ID `max_memtable_id`
+  // should be postponed to retain user-defined timestamps according to the
+  // user's setting. Called by background flush job.
+  bool ShouldPostponeFlushToRetainUDT(uint64_t max_memtable_id);
+
  ThreadLocalPtr* TEST_GetLocalSV() { return local_sv_.get(); }
  WriteBufferManager* write_buffer_mgr() { return write_buffer_manager_; }
  std::shared_ptr<CacheReservationManager>
@@ -513,8 +525,6 @@ class ColumnFamilyData {
    return file_metadata_cache_res_mgr_;
  }

-  SequenceNumber GetFirstMemtableSequenceNumber() const;
-
  static const uint32_t kDummyColumnFamilyDataId;

  // Keep track of whether the mempurge feature was ever used.
@@ -705,6 +715,16 @@ class ColumnFamilySet {
                                       Version* dummy_version,
                                       const ColumnFamilyOptions& options);

+  const UnorderedMap<uint32_t, size_t>& GetRunningColumnFamiliesTimestampSize()
+      const {
+    return running_ts_sz_;
+  }
+
+  const UnorderedMap<uint32_t, size_t>&
+  GetColumnFamiliesTimestampSizeForRecord() const {
+    return ts_sz_for_record_;
+  }
+
  iterator begin() { return iterator(dummy_cfd_->next_); }
  iterator end() { return iterator(dummy_cfd_); }

@@ -730,6 +750,15 @@ class ColumnFamilySet {
  UnorderedMap<std::string, uint32_t> column_families_;
  UnorderedMap<uint32_t, ColumnFamilyData*> column_family_data_;

+  // Mutating / reading `running_ts_sz_` and `ts_sz_for_record_` follow
+  // the same requirements as `column_families_` and `column_family_data_`.
+  // Mapping from column family id to user-defined timestamp size for all
+  // running column families.
+  UnorderedMap<uint32_t, size_t> running_ts_sz_;
+  // Mapping from column family id to user-defined timestamp size for
+  // column families with non-zero user-defined timestamp size.
+  UnorderedMap<uint32_t, size_t> ts_sz_for_record_;
+
  uint32_t max_column_family_;
  const FileOptions file_options_;

@@ -17,6 +17,7 @@
 #include "options/options_parser.h"
 #include "port/port.h"
 #include "port/stack_trace.h"
+#include "rocksdb/comparator.h"
 #include "rocksdb/convenience.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
@@ -63,6 +64,9 @@ class ColumnFamilyTestBase : public testing::Test {
    db_options_.create_if_missing = true;
    db_options_.fail_if_options_file_error = true;
    db_options_.env = env_;
+  }
+
+  void SetUp() override {
    EXPECT_OK(DestroyDB(dbname_, Options(db_options_, column_family_options_)));
  }

@@ -1255,6 +1259,7 @@ TEST_P(ColumnFamilyTest, DifferentCompactionStyles) {
  ColumnFamilyOptions default_cf, one, two;
  db_options_.max_open_files = 20;  // only 10 files in file cache

+  default_cf.level_compaction_dynamic_level_bytes = false;
  default_cf.compaction_style = kCompactionStyleLevel;
  default_cf.num_levels = 3;
  default_cf.write_buffer_size = 64 << 10;  // 64KB
@@ -1272,6 +1277,7 @@ TEST_P(ColumnFamilyTest, DifferentCompactionStyles) {
  one.level0_file_num_compaction_trigger = 4;
  one.write_buffer_size = 120000;

+  two.level_compaction_dynamic_level_bytes = false;
  two.compaction_style = kCompactionStyleLevel;
  two.num_levels = 4;
  two.level0_file_num_compaction_trigger = 3;
@@ -1326,6 +1332,7 @@ TEST_P(ColumnFamilyTest, MultipleManualCompactions) {
  db_options_.max_open_files = 20;  // only 10 files in file cache
  db_options_.max_background_compactions = 3;

+  default_cf.level_compaction_dynamic_level_bytes = false;
  default_cf.compaction_style = kCompactionStyleLevel;
  default_cf.num_levels = 3;
  default_cf.write_buffer_size = 64 << 10;  // 64KB
@@ -1342,6 +1349,7 @@ TEST_P(ColumnFamilyTest, MultipleManualCompactions) {
  one.level0_file_num_compaction_trigger = 4;
  one.write_buffer_size = 120000;

+  two.level_compaction_dynamic_level_bytes = false;
  two.compaction_style = kCompactionStyleLevel;
  two.num_levels = 4;
  two.level0_file_num_compaction_trigger = 3;
@@ -1424,13 +1432,14 @@ TEST_P(ColumnFamilyTest, AutomaticAndManualCompactions) {
  db_options_.max_open_files = 20;  // only 10 files in file cache
  db_options_.max_background_compactions = 3;

+  default_cf.level_compaction_dynamic_level_bytes = false;
  default_cf.compaction_style = kCompactionStyleLevel;
  default_cf.num_levels = 3;
  default_cf.write_buffer_size = 64 << 10;  // 64KB
  default_cf.target_file_size_base = 30 << 10;
  default_cf.max_compaction_bytes = default_cf.target_file_size_base * 1100;
  BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
-  ;
+
  table_options.no_block_cache = true;
  default_cf.table_factory.reset(NewBlockBasedTableFactory(table_options));

@@ -1441,6 +1450,7 @@ TEST_P(ColumnFamilyTest, AutomaticAndManualCompactions) {
  one.level0_file_num_compaction_trigger = 4;
  one.write_buffer_size = 120000;

+  two.level_compaction_dynamic_level_bytes = false;
  two.compaction_style = kCompactionStyleLevel;
  two.num_levels = 4;
  two.level0_file_num_compaction_trigger = 3;
@@ -1519,13 +1529,14 @@ TEST_P(ColumnFamilyTest, ManualAndAutomaticCompactions) {
  db_options_.max_open_files = 20;  // only 10 files in file cache
  db_options_.max_background_compactions = 3;

+  default_cf.level_compaction_dynamic_level_bytes = false;
  default_cf.compaction_style = kCompactionStyleLevel;
  default_cf.num_levels = 3;
  default_cf.write_buffer_size = 64 << 10;  // 64KB
  default_cf.target_file_size_base = 30 << 10;
  default_cf.max_compaction_bytes = default_cf.target_file_size_base * 1100;
  BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
-  ;
+
  table_options.no_block_cache = true;
  default_cf.table_factory.reset(NewBlockBasedTableFactory(table_options));

@@ -1536,6 +1547,7 @@ TEST_P(ColumnFamilyTest, ManualAndAutomaticCompactions) {
  one.level0_file_num_compaction_trigger = 4;
  one.write_buffer_size = 120000;

+  two.level_compaction_dynamic_level_bytes = false;
  two.compaction_style = kCompactionStyleLevel;
  two.num_levels = 4;
  two.level0_file_num_compaction_trigger = 3;
@@ -3372,6 +3384,205 @@ TEST(ColumnFamilyTest, ValidateMemtableKVChecksumOption) {
  ASSERT_OK(ColumnFamilyData::ValidateOptions(db_options, cf_options));
 }

+// Tests the flushing behavior of a column family to retain user-defined
+// timestamp when `persist_user_defined_timestamp` is false.
+class ColumnFamilyRetainUDTTest : public ColumnFamilyTestBase {
+ public:
+  ColumnFamilyRetainUDTTest() : ColumnFamilyTestBase(kLatestFormatVersion) {}
+
+  void SetUp() override {
+    db_options_.allow_concurrent_memtable_write = false;
+    column_family_options_.comparator =
+        test::BytewiseComparatorWithU64TsWrapper();
+    column_family_options_.persist_user_defined_timestamps = false;
+    ColumnFamilyTestBase::SetUp();
+  }
+
+  Status Put(int cf, const std::string& key, const std::string& ts,
+             const std::string& value) {
+    return db_->Put(WriteOptions(), handles_[cf], Slice(key), Slice(ts),
+                    Slice(value));
+  }
+};
+
+class TestTsComparator : public Comparator {
+ public:
+  TestTsComparator() : Comparator(8 /*ts_sz*/) {}
+
+  int Compare(const ROCKSDB_NAMESPACE::Slice& /*a*/,
+              const ROCKSDB_NAMESPACE::Slice& /*b*/) const override {
+    return 0;
+  }
+  const char* Name() const override { return "TestTs"; }
+  void FindShortestSeparator(
+      std::string* /*start*/,
+      const ROCKSDB_NAMESPACE::Slice& /*limit*/) const override {}
+  void FindShortSuccessor(std::string* /*key*/) const override {}
+};
+
+TEST_F(ColumnFamilyRetainUDTTest, SanityCheck) {
+  Open();
+  ColumnFamilyOptions cf_options;
+  cf_options.persist_user_defined_timestamps = false;
+  TestTsComparator test_comparator;
+  cf_options.comparator = &test_comparator;
+  ColumnFamilyHandle* handle;
+  // Not persisting user-defined timestamps feature only supports user-defined
+  // timestamps formatted as uint64_t.
+  ASSERT_TRUE(
+      db_->CreateColumnFamily(cf_options, "pikachu", &handle).IsNotSupported());
+
+  Destroy();
+  // Not persisting user-defined timestamps feature doesn't work in combination
+  // with atomic flush.
+  db_options_.atomic_flush = true;
+  ASSERT_TRUE(TryOpen({"default"}).IsNotSupported());
+
+  // Not persisting user-defined timestamps feature doesn't work in combination
+  // with concurrent memtable write.
+  db_options_.atomic_flush = false;
+  db_options_.allow_concurrent_memtable_write = true;
+  ASSERT_TRUE(TryOpen({"default"}).IsNotSupported());
+  Close();
+}
+
+TEST_F(ColumnFamilyRetainUDTTest, FullHistoryTsLowNotSet) {
+  SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::BackgroundFlush:CheckFlushRequest:cb", [&](void* arg) {
+        ASSERT_NE(nullptr, arg);
+        auto reschedule_count = *static_cast<int*>(arg);
+        ASSERT_EQ(1, reschedule_count);
+      });
+
+  SyncPoint::GetInstance()->EnableProcessing();
+  Open();
+  std::string write_ts;
+  PutFixed64(&write_ts, 1);
+  ASSERT_OK(Put(0, "foo", write_ts, "v1"));
+  // No `full_history_ts_low` explicitly set by user, flush is continued
+  // without checking if its UDTs expired.
+  ASSERT_OK(Flush(0));
+
+  // After flush, `full_history_ts_low` should be automatically advanced to
+  // the effective cutoff timestamp: write_ts + 1
+  std::string cutoff_ts;
+  PutFixed64(&cutoff_ts, 2);
+  std::string effective_full_history_ts_low;
+  ASSERT_OK(
+      db_->GetFullHistoryTsLow(handles_[0], &effective_full_history_ts_low));
+  ASSERT_EQ(cutoff_ts, effective_full_history_ts_low);
+  Close();
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+}
+
+TEST_F(ColumnFamilyRetainUDTTest, AllKeysExpired) {
+  SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::BackgroundFlush:CheckFlushRequest:cb", [&](void* arg) {
+        ASSERT_NE(nullptr, arg);
+        auto reschedule_count = *static_cast<int*>(arg);
+        ASSERT_EQ(1, reschedule_count);
+      });
+
+  SyncPoint::GetInstance()->EnableProcessing();
+  Open();
+  std::string write_ts;
+  PutFixed64(&write_ts, 1);
+  ASSERT_OK(Put(0, "foo", write_ts, "v1"));
+  std::string cutoff_ts;
+  PutFixed64(&cutoff_ts, 3);
+  ASSERT_OK(db_->IncreaseFullHistoryTsLow(handles_[0], cutoff_ts));
+  // All keys expired w.r.t the configured `full_history_ts_low`, flush continue
+  // without the need for a re-schedule.
+  ASSERT_OK(Flush(0));
+
+  // `full_history_ts_low` stays unchanged after flush.
+  std::string effective_full_history_ts_low;
+  ASSERT_OK(
+      db_->GetFullHistoryTsLow(handles_[0], &effective_full_history_ts_low));
+  ASSERT_EQ(cutoff_ts, effective_full_history_ts_low);
+  Close();
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+}
+TEST_F(ColumnFamilyRetainUDTTest, NotAllKeysExpiredFlushToAvoidWriteStall) {
+  SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::BackgroundFlush:CheckFlushRequest:cb", [&](void* arg) {
+        ASSERT_NE(nullptr, arg);
+        auto reschedule_count = *static_cast<int*>(arg);
+        ASSERT_EQ(1, reschedule_count);
+      });
+
+  SyncPoint::GetInstance()->EnableProcessing();
+  Open();
+  std::string cutoff_ts;
+  std::string write_ts;
+  PutFixed64(&write_ts, 1);
+  ASSERT_OK(Put(0, "foo", write_ts, "v1"));
+  PutFixed64(&cutoff_ts, 1);
+  ASSERT_OK(db_->IncreaseFullHistoryTsLow(handles_[0], cutoff_ts));
+  ASSERT_OK(db_->SetOptions(handles_[0], {{"max_write_buffer_number", "1"}}));
+  // Not all keys expired, but flush is continued without a re-schedule because
+  // of risk of write stall.
+  ASSERT_OK(Flush(0));
+
+  // After flush, `full_history_ts_low` should be automatically advanced to
+  // the effective cutoff timestamp: write_ts + 1
+  std::string effective_full_history_ts_low;
+  ASSERT_OK(
+      db_->GetFullHistoryTsLow(handles_[0], &effective_full_history_ts_low));
+
+  cutoff_ts.clear();
+  PutFixed64(&cutoff_ts, 2);
+  ASSERT_EQ(cutoff_ts, effective_full_history_ts_low);
+  Close();
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+}
+
+TEST_F(ColumnFamilyRetainUDTTest, NotAllKeysExpiredFlushRescheduled) {
+  std::string cutoff_ts;
+  SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::AfterRetainUDTReschedule:cb", [&](void* /*arg*/) {
+        // Increasing full_history_ts_low so all keys expired after the initial
+        // FlushRequest is rescheduled
+        cutoff_ts.clear();
+        PutFixed64(&cutoff_ts, 3);
+        ASSERT_OK(db_->IncreaseFullHistoryTsLow(handles_[0], cutoff_ts));
+      });
+  SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::BackgroundFlush:CheckFlushRequest:cb", [&](void* arg) {
+        ASSERT_NE(nullptr, arg);
+        auto reschedule_count = *static_cast<int*>(arg);
+        ASSERT_EQ(2, reschedule_count);
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  Open();
+  std::string write_ts;
+  PutFixed64(&write_ts, 1);
+  ASSERT_OK(Put(0, "foo", write_ts, "v1"));
+  PutFixed64(&cutoff_ts, 1);
+  ASSERT_OK(db_->IncreaseFullHistoryTsLow(handles_[0], cutoff_ts));
+  // Not all keys expired, and there is no risk of write stall. Flush is
+  // rescheduled. The actual flush happens after `full_history_ts_low` is
+  // increased to mark all keys expired.
+  ASSERT_OK(Flush(0));
+
+  std::string effective_full_history_ts_low;
+  ASSERT_OK(
+      db_->GetFullHistoryTsLow(handles_[0], &effective_full_history_ts_low));
+  // `full_history_ts_low` stays unchanged.
+  ASSERT_EQ(cutoff_ts, effective_full_history_ts_low);
+  Close();
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+}
+
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -66,6 +66,7 @@ TEST_F(CompactFilesTest, L0ConflictsFiles) {
  const int kWriteBufferSize = 10000;
  const int kLevel0Trigger = 2;
  options.create_if_missing = true;
+  options.level_compaction_dynamic_level_bytes = false;
  options.compaction_style = kCompactionStyleLevel;
  // Small slowdown and stop trigger for experimental purpose.
  options.level0_slowdown_writes_trigger = 20;
@@ -344,7 +345,7 @@ TEST_F(CompactFilesTest, CompactionFilterWithGetSv) {
        return true;
      }
      std::string res;
-      db_->Get(ReadOptions(), "", &res);
+      EXPECT_TRUE(db_->Get(ReadOptions(), "", &res).IsNotFound());
      return true;
    }

@@ -359,6 +360,7 @@ TEST_F(CompactFilesTest, CompactionFilterWithGetSv) {
  std::shared_ptr<FilterWithGet> cf(new FilterWithGet());

  Options options;
+  options.level_compaction_dynamic_level_bytes = false;
  options.create_if_missing = true;
  options.compaction_filter = cf.get();

@@ -401,6 +403,7 @@ TEST_F(CompactFilesTest, SentinelCompressionType) {
                                CompactionStyle::kCompactionStyleNone}) {
    ASSERT_OK(DestroyDB(db_name_, Options()));
    Options options;
+    options.level_compaction_dynamic_level_bytes = false;
    options.compaction_style = compaction_style;
    // L0: Snappy, L1: ZSTD, L2: Snappy
    options.compression_per_level = {CompressionType::kSnappyCompression,
@@ -13,6 +13,7 @@
 #include <vector>

 #include "db/column_family.h"
+#include "logging/logging.h"
 #include "rocksdb/compaction_filter.h"
 #include "rocksdb/sst_partitioner.h"
 #include "test_util/sync_point.h"
@@ -23,14 +24,13 @@ namespace ROCKSDB_NAMESPACE {
 const uint64_t kRangeTombstoneSentinel =
    PackSequenceAndType(kMaxSequenceNumber, kTypeRangeDeletion);

-int sstableKeyCompare(const Comparator* user_cmp, const InternalKey& a,
-                      const InternalKey& b) {
-  auto c = user_cmp->CompareWithoutTimestamp(a.user_key(), b.user_key());
+int sstableKeyCompare(const Comparator* uc, const Slice& a, const Slice& b) {
+  auto c = uc->CompareWithoutTimestamp(ExtractUserKey(a), ExtractUserKey(b));
  if (c != 0) {
    return c;
  }
-  auto a_footer = ExtractInternalKeyFooter(a.Encode());
-  auto b_footer = ExtractInternalKeyFooter(b.Encode());
+  auto a_footer = ExtractInternalKeyFooter(a);
+  auto b_footer = ExtractInternalKeyFooter(b);
  if (a_footer == kRangeTombstoneSentinel) {
    if (b_footer != kRangeTombstoneSentinel) {
      return -1;
@@ -204,6 +204,38 @@ bool Compaction::IsFullCompaction(
  return num_files_in_compaction == total_num_files;
 }

+Status Compaction::InitInputTableProperties() {
+  if (!input_table_properties_.empty()) {
+    return Status::OK();
+  }
+
+  Status s;
+  const ReadOptions read_options(Env::IOActivity::kCompaction);
+  assert(input_version_);
+  for (size_t i = 0; i < num_input_levels(); ++i) {
+    for (const FileMetaData* fmd : *(this->inputs(i))) {
+      std::shared_ptr<const TableProperties> tp;
+      std::string file_name =
+          TableFileName(immutable_options_.cf_paths, fmd->fd.GetNumber(),
+                        fmd->fd.GetPathId());
+      s = input_version_->GetTableProperties(read_options, &tp, fmd,
+                                             &file_name);
+      if (s.ok()) {
+        input_table_properties_[file_name] = tp;
+      } else {
+        ROCKS_LOG_ERROR(immutable_options_.info_log,
+                        "Unable to load table properties for file %" PRIu64
+                        " --- %s\n",
+                        fmd->fd.GetNumber(), s.ToString().c_str());
+        input_table_properties_.clear();
+        return s;
+      }
+    }
+  }
+
+  return s;
+}
+
 Compaction::Compaction(
    VersionStorageInfo* vstorage, const ImmutableOptions& _immutable_options,
    const MutableCFOptions& _mutable_cf_options,
@@ -486,26 +518,25 @@ bool Compaction::IsTrivialMove() const {

  // assert inputs_.size() == 1

-  std::unique_ptr<SstPartitioner> partitioner = CreateSstPartitioner();
-
-  for (const auto& file : inputs_.front().files) {
-    std::vector<FileMetaData*> file_grand_parents;
-    if (output_level_ + 1 >= number_levels_) {
-      continue;
-    }
-    input_vstorage_->GetOverlappingInputs(output_level_ + 1, &file->smallest,
-                                          &file->largest, &file_grand_parents);
-    const auto compaction_size =
-        file->fd.GetFileSize() + TotalFileSize(file_grand_parents);
-    if (compaction_size > max_compaction_bytes_) {
-      return false;
-    }
-
-    if (partitioner.get() != nullptr) {
-      if (!partitioner->CanDoTrivialMove(file->smallest.user_key(),
-                                         file->largest.user_key())) {
+  if (output_level_ + 1 < number_levels_) {
+    std::unique_ptr<SstPartitioner> partitioner = CreateSstPartitioner();
+    for (const auto& file : inputs_.front().files) {
+      std::vector<FileMetaData*> file_grand_parents;
+      input_vstorage_->GetOverlappingInputs(output_level_ + 1, &file->smallest,
+                                            &file->largest,
+                                            &file_grand_parents);
+      const auto compaction_size =
+          file->fd.GetFileSize() + TotalFileSize(file_grand_parents);
+      if (compaction_size > max_compaction_bytes_) {
        return false;
      }
+
+      if (partitioner.get() != nullptr) {
+        if (!partitioner->CanDoTrivialMove(file->smallest.user_key(),
+                                           file->largest.user_key())) {
+          return false;
+        }
+      }
    }
  }

@@ -563,6 +594,49 @@ bool Compaction::KeyNotExistsBeyondOutputLevel(
  return false;
 }

+bool Compaction::KeyRangeNotExistsBeyondOutputLevel(
+    const Slice& begin_key, const Slice& end_key,
+    std::vector<size_t>* level_ptrs) const {
+  assert(input_version_ != nullptr);
+  assert(level_ptrs != nullptr);
+  assert(level_ptrs->size() == static_cast<size_t>(number_levels_));
+  assert(cfd_->user_comparator()->CompareWithoutTimestamp(begin_key, end_key) <
+         0);
+  if (bottommost_level_) {
+    return true /* does not overlap */;
+  } else if (output_level_ != 0 &&
+             cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
+    const Comparator* user_cmp = cfd_->user_comparator();
+    for (int lvl = output_level_ + 1; lvl < number_levels_; lvl++) {
+      const std::vector<FileMetaData*>& files =
+          input_vstorage_->LevelFiles(lvl);
+      for (; level_ptrs->at(lvl) < files.size(); level_ptrs->at(lvl)++) {
+        auto* f = files[level_ptrs->at(lvl)];
+        // Advance until the first file with begin_key <= f->largest.user_key()
+        if (user_cmp->CompareWithoutTimestamp(begin_key,
+                                              f->largest.user_key()) > 0) {
+          continue;
+        }
+        // We know that the previous file prev_f, if exists, has
+        // prev_f->largest.user_key() < begin_key.
+        if (user_cmp->CompareWithoutTimestamp(end_key,
+                                              f->smallest.user_key()) <= 0) {
+          // not overlapping with this level
+          break;
+        } else {
+          // We have:
+          // - begin_key < end_key,
+          // - begin_key <= f->largest.user_key(), and
+          // - end_key > f->smallest.user_key()
+          return false /* overlap */;
+        }
+      }
+    }
+    return true /* does not overlap */;
+  }
+  return false /* overlaps */;
+};
+
 // Mark (or clear) each file that is being compacted
 void Compaction::MarkFilesBeingCompacted(bool mark_as_compacted) {
  for (size_t i = 0; i < num_input_levels(); i++) {
@@ -704,8 +778,17 @@ std::unique_ptr<CompactionFilter> Compaction::CreateCompactionFilter() const {
  CompactionFilter::Context context;
  context.is_full_compaction = is_full_compaction_;
  context.is_manual_compaction = is_manual_compaction_;
+  context.input_start_level = start_level_;
  context.column_family_id = cfd_->GetID();
  context.reason = TableFileCreationReason::kCompaction;
+  context.input_table_properties = GetInputTableProperties();
+  if (context.input_table_properties.empty()) {
+    ROCKS_LOG_WARN(
+        immutable_options_.info_log,
+        "Unable to set `input_table_properties` of `CompactionFilter::Context` "
+        "for compaction.");
+  }
+
  return cfd_->ioptions()->compaction_filter_factory->CreateCompactionFilter(
      context);
 }
@@ -31,8 +31,19 @@ namespace ROCKSDB_NAMESPACE {
 // that key never appears in the database. We don't want adjacent sstables to
 // be considered overlapping if they are separated by the range tombstone
 // sentinel.
-int sstableKeyCompare(const Comparator* user_cmp, const InternalKey& a,
-                      const InternalKey& b);
+int sstableKeyCompare(const Comparator* user_cmp, const Slice&, const Slice&);
+inline int sstableKeyCompare(const Comparator* user_cmp, const Slice& a,
+                             const InternalKey& b) {
+  return sstableKeyCompare(user_cmp, a, b.Encode());
+}
+inline int sstableKeyCompare(const Comparator* user_cmp, const InternalKey& a,
+                             const Slice& b) {
+  return sstableKeyCompare(user_cmp, a.Encode(), b);
+}
+inline int sstableKeyCompare(const Comparator* user_cmp, const InternalKey& a,
+                             const InternalKey& b) {
+  return sstableKeyCompare(user_cmp, a.Encode(), b.Encode());
+}
 int sstableKeyCompare(const Comparator* user_cmp, const InternalKey* a,
                      const InternalKey& b);
 int sstableKeyCompare(const Comparator* user_cmp, const InternalKey& a,
@@ -203,10 +214,18 @@ class Compaction {
  void AddInputDeletions(VersionEdit* edit);

  // Returns true if the available information we have guarantees that
-  // the input "user_key" does not exist in any level beyond "output_level()".
+  // the input "user_key" does not exist in any level beyond `output_level()`.
  bool KeyNotExistsBeyondOutputLevel(const Slice& user_key,
                                     std::vector<size_t>* level_ptrs) const;

+  // Returns true if the user key range [begin_key, end_key) does not exist
+  // in any level beyond `output_level()`.
+  // Used for checking range tombstones, so we assume begin_key < end_key.
+  // begin_key and end_key should include timestamp if enabled.
+  bool KeyRangeNotExistsBeyondOutputLevel(
+      const Slice& begin_key, const Slice& end_key,
+      std::vector<size_t>* level_ptrs) const;
+
  // Clear all files to indicate that they are not being compacted
  // Delete this compaction from the list of running compactions.
  //
@@ -270,7 +289,14 @@ class Compaction {
  // is the sum of all input file sizes.
  uint64_t OutputFilePreallocationSize() const;

-  void SetInputVersion(Version* input_version);
+  // TODO(hx235): eventually we should consider `InitInputTableProperties()`'s
+  // status and fail the compaction if needed
+  // TODO(hx235): consider making this function part of the construction so we
+  // don't forget to call it
+  void FinalizeInputInfo(Version* input_version) {
+    SetInputVersion(input_version);
+    InitInputTableProperties().PermitUncheckedError();
+  }

  struct InputLevelSummaryBuffer {
    char buffer[128];
@@ -307,12 +333,20 @@ class Compaction {
      int output_level, VersionStorageInfo* vstorage,
      const std::vector<CompactionInputFiles>& inputs);

-  TablePropertiesCollection GetOutputTableProperties() const {
-    return output_table_properties_;
+  const TablePropertiesCollection& GetInputTableProperties() const {
+    return input_table_properties_;
  }

-  void SetOutputTableProperties(TablePropertiesCollection tp) {
-    output_table_properties_ = std::move(tp);
+  // TODO(hx235): consider making this function symmetric to
+  // InitInputTableProperties()
+  void SetOutputTableProperties(
+      const std::string& file_name,
+      const std::shared_ptr<const TableProperties>& tp) {
+    output_table_properties_[file_name] = tp;
+  }
+
+  const TablePropertiesCollection& GetOutputTableProperties() const {
+    return output_table_properties_;
  }

  Slice GetSmallestUserKey() const { return smallest_user_key_; }
@@ -409,6 +443,10 @@ class Compaction {
                                      const int output_level);

 private:
+  void SetInputVersion(Version* input_version);
+
+  Status InitInputTableProperties();
+
  // mark (or clear) all files that are being compacted
  void MarkFilesBeingCompacted(bool mark_as_compacted);

@@ -499,7 +537,7 @@ class Compaction {
  // Does input compression match the output compression?
  bool InputCompressionMatchesOutput() const;

-  // table properties of output files
+  TablePropertiesCollection input_table_properties_;
  TablePropertiesCollection output_table_properties_;

  // smallest user keys in compaction
@@ -14,6 +14,7 @@
 #include "db/blob/prefetch_buffer_collection.h"
 #include "db/snapshot_checker.h"
 #include "db/wide/wide_column_serialization.h"
+#include "db/wide/wide_columns_helper.h"
 #include "logging/logging.h"
 #include "port/likely.h"
 #include "rocksdb/listener.h"
@@ -31,7 +32,8 @@ CompactionIterator::CompactionIterator(
    BlobFileBuilder* blob_file_builder, bool allow_data_in_errors,
    bool enforce_single_del_contracts,
    const std::atomic<bool>& manual_compaction_canceled,
-    const Compaction* compaction, const CompactionFilter* compaction_filter,
+    bool must_count_input_entries, const Compaction* compaction,
+    const CompactionFilter* compaction_filter,
    const std::atomic<bool>* shutting_down,
    const std::shared_ptr<Logger> info_log,
    const std::string* full_history_ts_low,
@@ -45,8 +47,9 @@ CompactionIterator::CompactionIterator(
          manual_compaction_canceled,
          std::unique_ptr<CompactionProxy>(
              compaction ? new RealCompaction(compaction) : nullptr),
-          compaction_filter, shutting_down, info_log, full_history_ts_low,
-          preserve_time_min_seqno, preclude_last_level_min_seqno) {}
+          must_count_input_entries, compaction_filter, shutting_down, info_log,
+          full_history_ts_low, preserve_time_min_seqno,
+          preclude_last_level_min_seqno) {}

 CompactionIterator::CompactionIterator(
    InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
@@ -58,15 +61,14 @@ CompactionIterator::CompactionIterator(
    BlobFileBuilder* blob_file_builder, bool allow_data_in_errors,
    bool enforce_single_del_contracts,
    const std::atomic<bool>& manual_compaction_canceled,
-    std::unique_ptr<CompactionProxy> compaction,
+    std::unique_ptr<CompactionProxy> compaction, bool must_count_input_entries,
    const CompactionFilter* compaction_filter,
    const std::atomic<bool>* shutting_down,
    const std::shared_ptr<Logger> info_log,
    const std::string* full_history_ts_low,
    const SequenceNumber preserve_time_min_seqno,
    const SequenceNumber preclude_last_level_min_seqno)
-    : input_(input, cmp,
-             !compaction || compaction->DoesInputReferenceBlobFiles()),
+    : input_(input, cmp, must_count_input_entries),
      cmp_(cmp),
      merge_helper_(merge_helper),
      snapshots_(snapshots),
@@ -422,16 +424,13 @@ bool CompactionIterator::InvokeFilterIfNeeded(bool* need_skip,
    return false;
  } else if (decision == CompactionFilter::Decision::kChangeWideColumnEntity) {
    WideColumns sorted_columns;
-
    sorted_columns.reserve(new_columns.size());
+
    for (const auto& column : new_columns) {
      sorted_columns.emplace_back(column.first, column.second);
    }

-    std::sort(sorted_columns.begin(), sorted_columns.end(),
-              [](const WideColumn& lhs, const WideColumn& rhs) {
-                return lhs.name().compare(rhs.name()) < 0;
-              });
+    WideColumnsHelper::SortColumns(sorted_columns);

    {
      const Status s = WideColumnSerialization::Serialize(
@@ -38,15 +38,18 @@ class SequenceIterWrapper : public InternalIterator {
  bool Valid() const override { return inner_iter_->Valid(); }
  Status status() const override { return inner_iter_->status(); }
  void Next() override {
-    num_itered_++;
+    if (!inner_iter_->IsDeleteRangeSentinelKey()) {
+      num_itered_++;
+    }
    inner_iter_->Next();
  }
  void Seek(const Slice& target) override {
    if (!need_count_entries_) {
+      has_num_itered_ = false;
      inner_iter_->Seek(target);
    } else {
-      // For flush cases, we need to count total number of entries, so we
-      // do Next() rather than Seek().
+      // Need to count total number of entries,
+      // so we do Next() rather than Seek().
      while (inner_iter_->Valid() &&
             icmp_.Compare(inner_iter_->key(), target) < 0) {
        Next();
@@ -62,7 +65,8 @@ class SequenceIterWrapper : public InternalIterator {
  void SeekForPrev(const Slice& /* target */) override { assert(false); }
  void SeekToLast() override { assert(false); }

-  uint64_t num_itered() const { return num_itered_; }
+  uint64_t NumItered() const { return num_itered_; }
+  bool HasNumItered() const { return has_num_itered_; }
  bool IsDeleteRangeSentinelKey() const override {
    assert(Valid());
    return inner_iter_->IsDeleteRangeSentinelKey();
@@ -73,6 +77,7 @@ class SequenceIterWrapper : public InternalIterator {
  InternalIterator* inner_iter_;  // not owned
  uint64_t num_itered_ = 0;
  bool need_count_entries_;
+  bool has_num_itered_ = true;
 };

 class CompactionIterator {
@@ -189,6 +194,10 @@ class CompactionIterator {
    const Compaction* compaction_;
  };

+  // @param must_count_input_entries  if true, `NumInputEntryScanned()` will
+  // return the number of input keys scanned. If false, `NumInputEntryScanned()`
+  // will return this number if no Seek was called on `input`. User should call
+  // `HasNumInputEntryScanned()` first in this case.
  CompactionIterator(
      InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
      SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
@@ -199,7 +208,7 @@ class CompactionIterator {
      BlobFileBuilder* blob_file_builder, bool allow_data_in_errors,
      bool enforce_single_del_contracts,
      const std::atomic<bool>& manual_compaction_canceled,
-      const Compaction* compaction = nullptr,
+      bool must_count_input_entries, const Compaction* compaction = nullptr,
      const CompactionFilter* compaction_filter = nullptr,
      const std::atomic<bool>* shutting_down = nullptr,
      const std::shared_ptr<Logger> info_log = nullptr,
@@ -219,6 +228,7 @@ class CompactionIterator {
      bool enforce_single_del_contracts,
      const std::atomic<bool>& manual_compaction_canceled,
      std::unique_ptr<CompactionProxy> compaction,
+      bool must_count_input_entries,
      const CompactionFilter* compaction_filter = nullptr,
      const std::atomic<bool>* shutting_down = nullptr,
      const std::shared_ptr<Logger> info_log = nullptr,
@@ -253,7 +263,8 @@ class CompactionIterator {
    return current_user_key_;
  }
  const CompactionIterationStats& iter_stats() const { return iter_stats_; }
-  uint64_t num_input_entry_scanned() const { return input_.num_itered(); }
+  bool HasNumInputEntryScanned() const { return input_.HasNumItered(); }
+  uint64_t NumInputEntryScanned() const { return input_.NumItered(); }
  // If the current key should be placed on penultimate level, only valid if
  // per_key_placement is supported
  bool output_to_penultimate_level() const {
@@ -293,8 +293,8 @@ class CompactionIteratorTest : public testing::TestWithParam<bool> {
        nullptr /* blob_file_builder */, true /*allow_data_in_errors*/,
        true /*enforce_single_del_contracts*/,
        /*manual_compaction_canceled=*/kManualCompactionCanceledFalse_,
-        std::move(compaction), filter, &shutting_down_, /*info_log=*/nullptr,
-        full_history_ts_low));
+        std::move(compaction), /*must_count_input_entries=*/false, filter,
+        &shutting_down_, /*info_log=*/nullptr, full_history_ts_low));
  }

  void AddSnapshot(SequenceNumber snapshot,
@@ -260,7 +260,7 @@ void CompactionJob::Prepare() {
    StopWatch sw(db_options_.clock, stats_, SUBCOMPACTION_SETUP_TIME);
    GenSubcompactionBoundaries();
  }
-  if (boundaries_.size() > 1) {
+  if (boundaries_.size() >= 1) {
    for (size_t i = 0; i <= boundaries_.size(); i++) {
      compact_->sub_compact_states.emplace_back(
          c, (i != 0) ? std::optional<Slice>(boundaries_[i - 1]) : std::nullopt,
@@ -796,24 +796,56 @@ Status CompactionJob::Run() {
      auto fn =
          TableFileName(state.compaction->immutable_options()->cf_paths,
                        output.meta.fd.GetNumber(), output.meta.fd.GetPathId());
-      tp[fn] = output.table_properties;
+      compact_->compaction->SetOutputTableProperties(fn,
+                                                     output.table_properties);
    }
  }
-  compact_->compaction->SetOutputTableProperties(std::move(tp));

-  // Finish up all book-keeping to unify the subcompaction results
+  // Finish up all bookkeeping to unify the subcompaction results.
  compact_->AggregateCompactionStats(compaction_stats_, *compaction_job_stats_);
-  UpdateCompactionStats();
-
+  uint64_t num_input_range_del = 0;
+  bool ok = UpdateCompactionStats(&num_input_range_del);
+  // (Sub)compactions returned ok, do sanity check on the number of input keys.
+  if (status.ok() && ok && compaction_job_stats_->has_num_input_records) {
+    size_t ts_sz = compact_->compaction->column_family_data()
+                       ->user_comparator()
+                       ->timestamp_size();
+    // When trim_ts_ is non-empty, CompactionIterator takes
+    // HistoryTrimmingIterator as input iterator and sees a trimmed view of
+    // input keys. So the number of keys it processed is not suitable for
+    // verification here.
+    // TODO: support verification when trim_ts_ is non-empty.
+    if (!(ts_sz > 0 && !trim_ts_.empty()) &&
+        db_options_.compaction_verify_record_count) {
+      assert(compaction_stats_.stats.num_input_records > 0);
+      // TODO: verify the number of range deletion entries.
+      uint64_t expected =
+          compaction_stats_.stats.num_input_records - num_input_range_del;
+      uint64_t actual = compaction_job_stats_->num_input_records;
+      if (expected != actual) {
+        std::string msg =
+            "Total number of input records: " + std::to_string(expected) +
+            ", but processed " + std::to_string(actual) + " records.";
+        ROCKS_LOG_WARN(
+            db_options_.info_log, "[%s] [JOB %d] Compaction %s",
+            compact_->compaction->column_family_data()->GetName().c_str(),
+            job_context_->job_id, msg.c_str());
+        status = Status::Corruption(
+            "Compaction number of input keys does not match number of keys "
+            "processed.");
+      }
+    }
+  }
  RecordCompactionIOStats();
  LogFlush(db_options_.info_log);
  TEST_SYNC_POINT("CompactionJob::Run():End");
-
  compact_->status = status;
+  TEST_SYNC_POINT_CALLBACK("CompactionJob::Run():EndStatusSet", &status);
  return status;
 }

-Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) {
+Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options,
+                              bool* compaction_released) {
  assert(compact_);

  AutoThreadOperationStageUpdater stage_updater(
@@ -829,7 +861,7 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) {
                                            compaction_stats_);

  if (status.ok()) {
-    status = InstallCompactionResults(mutable_cf_options);
+    status = InstallCompactionResults(mutable_cf_options, compaction_released);
  }
  if (!versions_->io_status().ok()) {
    io_status_ = versions_->io_status();
@@ -1096,17 +1128,17 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
  // GenSubcompactionBoundaries doesn't strip away the timestamp.
  size_t ts_sz = cfd->user_comparator()->timestamp_size();
  if (start.has_value()) {
-    read_options.iterate_lower_bound = &start.value();
+    read_options.iterate_lower_bound = &(*start);
    if (ts_sz > 0) {
-      start_without_ts = StripTimestampFromUserKey(start.value(), ts_sz);
-      read_options.iterate_lower_bound = &start_without_ts.value();
+      start_without_ts = StripTimestampFromUserKey(*start, ts_sz);
+      read_options.iterate_lower_bound = &(*start_without_ts);
    }
  }
  if (end.has_value()) {
-    read_options.iterate_upper_bound = &end.value();
+    read_options.iterate_upper_bound = &(*end);
    if (ts_sz > 0) {
-      end_without_ts = StripTimestampFromUserKey(end.value(), ts_sz);
-      read_options.iterate_upper_bound = &end_without_ts.value();
+      end_without_ts = StripTimestampFromUserKey(*end, ts_sz);
+      read_options.iterate_upper_bound = &(*end_without_ts);
    }
  }

@@ -1138,8 +1170,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
  }

  if (start.has_value()) {
-    start_ikey.SetInternalKey(start.value(), kMaxSequenceNumber,
-                              kValueTypeForSeek);
+    start_ikey.SetInternalKey(*start, kMaxSequenceNumber, kValueTypeForSeek);
    if (ts_sz > 0) {
      start_ikey.UpdateInternalKey(kMaxSequenceNumber, kValueTypeForSeek,
                                   &ts_slice);
@@ -1148,7 +1179,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
    start_user_key = start_ikey.GetUserKey();
  }
  if (end.has_value()) {
-    end_ikey.SetInternalKey(end.value(), kMaxSequenceNumber, kValueTypeForSeek);
+    end_ikey.SetInternalKey(*end, kMaxSequenceNumber, kValueTypeForSeek);
    if (ts_sz > 0) {
      end_ikey.UpdateInternalKey(kMaxSequenceNumber, kValueTypeForSeek,
                                 &ts_slice);
@@ -1253,6 +1284,8 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
      /*expect_valid_internal_key=*/true, range_del_agg.get(),
      blob_file_builder.get(), db_options_.allow_data_in_errors,
      db_options_.enforce_single_del_contracts, manual_compaction_canceled_,
+      sub_compact->compaction
+          ->DoesInputReferenceBlobFiles() /* must_count_input_entries */,
      sub_compact->compaction, compaction_filter, shutting_down_,
      db_options_.info_log, full_history_ts_low, preserve_time_min_seqno_,
      preclude_last_level_min_seqno_);
@@ -1286,17 +1319,24 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
      "CompactionJob::ProcessKeyValueCompaction()::Processing",
      reinterpret_cast<void*>(
          const_cast<Compaction*>(sub_compact->compaction)));
+  uint64_t last_cpu_micros = prev_cpu_micros;
  while (status.ok() && !cfd->IsDropped() && c_iter->Valid()) {
    // Invariant: c_iter.status() is guaranteed to be OK if c_iter->Valid()
    // returns true.
-    assert(!end.has_value() || cfd->user_comparator()->Compare(
-                                   c_iter->user_key(), end.value()) < 0);
+    assert(!end.has_value() ||
+           cfd->user_comparator()->Compare(c_iter->user_key(), *end) < 0);

    if (c_iter_stats.num_input_records % kRecordStatsEvery ==
        kRecordStatsEvery - 1) {
      RecordDroppedKeys(c_iter_stats, &sub_compact->compaction_job_stats);
      c_iter->ResetRecordCounts();
      RecordCompactionIOStats();
+
+      uint64_t cur_cpu_micros = db_options_.clock->CPUMicros();
+      assert(cur_cpu_micros >= last_cpu_micros);
+      RecordTick(stats_, COMPACTION_CPU_TOTAL_TIME,
+                 cur_cpu_micros - last_cpu_micros);
+      last_cpu_micros = cur_cpu_micros;
    }

    // Add current compaction_iterator key to target compaction output, if the
@@ -1317,8 +1357,25 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
    if (c_iter->status().IsManualCompactionPaused()) {
      break;
    }
+
+#ifndef NDEBUG
+    bool stop = false;
+    TEST_SYNC_POINT_CALLBACK("CompactionJob::ProcessKeyValueCompaction()::stop",
+                             static_cast<void*>(&stop));
+    if (stop) {
+      break;
+    }
+#endif  // NDEBUG
  }

+  // This number may not be accurate when CompactionIterator was created
+  // with `must_count_input_entries=false`.
+  assert(!sub_compact->compaction->DoesInputReferenceBlobFiles() ||
+         c_iter->HasNumInputEntryScanned());
+  sub_compact->compaction_job_stats.has_num_input_records =
+      c_iter->HasNumInputEntryScanned();
+  sub_compact->compaction_job_stats.num_input_records =
+      c_iter->NumInputEntryScanned();
  sub_compact->compaction_job_stats.num_blobs_read =
      c_iter_stats.num_blobs_read;
  sub_compact->compaction_job_stats.total_blob_bytes_read =
@@ -1387,8 +1444,11 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
    sub_compact->Current().UpdateBlobStats();
  }

+  uint64_t cur_cpu_micros = db_options_.clock->CPUMicros();
  sub_compact->compaction_job_stats.cpu_micros =
-      db_options_.clock->CPUMicros() - prev_cpu_micros;
+      cur_cpu_micros - prev_cpu_micros;
+  RecordTick(stats_, COMPACTION_CPU_TOTAL_TIME,
+             cur_cpu_micros - last_cpu_micros);

  if (measure_io_stats_) {
    sub_compact->compaction_job_stats.file_write_nanos +=
@@ -1638,7 +1698,7 @@ Status CompactionJob::FinishCompactionOutputFile(
 }

 Status CompactionJob::InstallCompactionResults(
-    const MutableCFOptions& mutable_cf_options) {
+    const MutableCFOptions& mutable_cf_options, bool* compaction_released) {
  assert(compact_);

  db_mutex_->AssertHeld();
@@ -1720,9 +1780,15 @@ Status CompactionJob::InstallCompactionResults(
    }
  }

-  return versions_->LogAndApply(compaction->column_family_data(),
-                                mutable_cf_options, read_options, edit,
-                                db_mutex_, db_directory_);
+  auto manifest_wcb = [&compaction, &compaction_released](const Status& s) {
+    compaction->ReleaseCompactionFiles(s);
+    *compaction_released = true;
+  };
+
+  return versions_->LogAndApply(
+      compaction->column_family_data(), mutable_cf_options, read_options, edit,
+      db_mutex_, db_directory_, /*new_descriptor_log=*/false,
+      /*column_family_options=*/nullptr, manifest_wcb);
 }

 void CompactionJob::RecordCompactionIOStats() {
@@ -1818,10 +1884,10 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
  uint64_t current_time = static_cast<uint64_t>(temp_current_time);
  InternalKey tmp_start, tmp_end;
  if (sub_compact->start.has_value()) {
-    tmp_start.SetMinPossibleForUserKey(sub_compact->start.value());
+    tmp_start.SetMinPossibleForUserKey(*(sub_compact->start));
  }
  if (sub_compact->end.has_value()) {
-    tmp_end.SetMinPossibleForUserKey(sub_compact->end.value());
+    tmp_end.SetMinPossibleForUserKey(*(sub_compact->end));
  }
  uint64_t oldest_ancester_time =
      sub_compact->compaction->MinInputFileOldestAncesterTime(
@@ -1872,6 +1938,7 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
      db_options_.stats, listeners, db_options_.file_checksum_gen_factory.get(),
      tmp_set.Contains(FileType::kTableFile), false));

+  // TODO(hx235): pass in the correct `oldest_key_time` instead of `0`
  TableBuilderOptions tboptions(
      *cfd->ioptions(), *(sub_compact->compaction->mutable_cf_options()),
      cfd->internal_comparator(), cfd->int_tbl_prop_collector_factories(),
@@ -1904,24 +1971,53 @@ void CopyPrefix(const Slice& src, size_t prefix_length, std::string* dst) {
 }
 }  // namespace

-
-void CompactionJob::UpdateCompactionStats() {
+bool CompactionJob::UpdateCompactionStats(uint64_t* num_input_range_del) {
  assert(compact_);

  Compaction* compaction = compact_->compaction;
  compaction_stats_.stats.num_input_files_in_non_output_levels = 0;
  compaction_stats_.stats.num_input_files_in_output_level = 0;
+
+  bool has_error = false;
+  const ReadOptions read_options(Env::IOActivity::kCompaction);
+  const auto& input_table_properties = compaction->GetInputTableProperties();
  for (int input_level = 0;
       input_level < static_cast<int>(compaction->num_input_levels());
       ++input_level) {
+    size_t num_input_files = compaction->num_input_files(input_level);
+    uint64_t* bytes_read;
    if (compaction->level(input_level) != compaction->output_level()) {
-      UpdateCompactionInputStatsHelper(
-          &compaction_stats_.stats.num_input_files_in_non_output_levels,
-          &compaction_stats_.stats.bytes_read_non_output_levels, input_level);
+      compaction_stats_.stats.num_input_files_in_non_output_levels +=
+          static_cast<int>(num_input_files);
+      bytes_read = &compaction_stats_.stats.bytes_read_non_output_levels;
    } else {
-      UpdateCompactionInputStatsHelper(
-          &compaction_stats_.stats.num_input_files_in_output_level,
-          &compaction_stats_.stats.bytes_read_output_level, input_level);
+      compaction_stats_.stats.num_input_files_in_output_level +=
+          static_cast<int>(num_input_files);
+      bytes_read = &compaction_stats_.stats.bytes_read_output_level;
+    }
+    for (size_t i = 0; i < num_input_files; ++i) {
+      const FileMetaData* file_meta = compaction->input(input_level, i);
+      *bytes_read += file_meta->fd.GetFileSize();
+      uint64_t file_input_entries = file_meta->num_entries;
+      uint64_t file_num_range_del = file_meta->num_range_deletions;
+      if (file_input_entries == 0) {
+        uint64_t file_number = file_meta->fd.GetNumber();
+        // Try getting info from table property
+        std::string fn =
+            TableFileName(compaction->immutable_options()->cf_paths,
+                          file_number, file_meta->fd.GetPathId());
+        const auto& tp = input_table_properties.find(fn);
+        if (tp != input_table_properties.end()) {
+          file_input_entries = tp->second->num_entries;
+          file_num_range_del = tp->second->num_range_deletions;
+        } else {
+          has_error = true;
+        }
+      }
+      compaction_stats_.stats.num_input_records += file_input_entries;
+      if (num_input_range_del) {
+        *num_input_range_del += file_num_range_del;
+      }
    }
  }

@@ -1931,21 +2027,7 @@ void CompactionJob::UpdateCompactionStats() {

  compaction_stats_.stats.num_dropped_records =
      compaction_stats_.DroppedRecords();
-}
-
-void CompactionJob::UpdateCompactionInputStatsHelper(int* num_files,
-                                                     uint64_t* bytes_read,
-                                                     int input_level) {
-  const Compaction* compaction = compact_->compaction;
-  auto num_input_files = compaction->num_input_files(input_level);
-  *num_files += static_cast<int>(num_input_files);
-
-  for (size_t i = 0; i < num_input_files; ++i) {
-    const auto* file_meta = compaction->input(input_level, i);
-    *bytes_read += file_meta->fd.GetFileSize();
-    compaction_stats_.stats.num_input_records +=
-        static_cast<uint64_t>(file_meta->num_entries);
-  }
+  return !has_error;
 }

 void CompactionJob::UpdateCompactionJobStats(
@@ -186,13 +186,30 @@ class CompactionJob {

  // REQUIRED: mutex held
  // Add compaction input/output to the current version
-  Status Install(const MutableCFOptions& mutable_cf_options);
+  // Releases compaction file through Compaction::ReleaseCompactionFiles().
+  // Sets *compaction_released to true if compaction is released.
+  Status Install(const MutableCFOptions& mutable_cf_options,
+                 bool* compaction_released);

  // Return the IO status
  IOStatus io_status() const { return io_status_; }

 protected:
-  void UpdateCompactionStats();
+  // Update the following stats in compaction_stats_.stats
+  // - num_input_files_in_non_output_levels
+  // - num_input_files_in_output_level
+  // - bytes_read_non_output_levels
+  // - bytes_read_output_level
+  // - num_input_records
+  // - bytes_read_blob
+  // - num_dropped_records
+  //
+  // @param num_input_range_del if non-null, will be set to the number of range
+  // deletion entries in this compaction input.
+  //
+  // Returns true iff compaction_stats_.stats.num_input_records and
+  // num_input_range_del are calculated successfully.
+  bool UpdateCompactionStats(uint64_t* num_input_range_del = nullptr);
  void LogCompaction();
  virtual void RecordCompactionIOStats();
  void CleanupCompaction();
@@ -259,7 +276,8 @@ class CompactionJob {
                                    const Slice& next_table_min_key,
                                    const Slice* comp_start_user_key,
                                    const Slice* comp_end_user_key);
-  Status InstallCompactionResults(const MutableCFOptions& mutable_cf_options);
+  Status InstallCompactionResults(const MutableCFOptions& mutable_cf_options,
+                                  bool* compaction_released);
  Status OpenCompactionOutputFile(SubcompactionState* sub_compact,
                                  CompactionOutputs& outputs);
  void UpdateCompactionJobStats(
@@ -267,9 +285,6 @@ class CompactionJob {
  void RecordDroppedKeys(const CompactionIterationStats& c_iter_stats,
                         CompactionJobStats* compaction_job_stats = nullptr);

-  void UpdateCompactionInputStatsHelper(int* num_files, uint64_t* bytes_read,
-                                        int input_level);
-
  void NotifyOnSubcompactionBegin(SubcompactionState* sub_compact);

  void NotifyOnSubcompactionCompleted(SubcompactionState* sub_compact);
@@ -616,6 +616,7 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
  // via AddExpectedStats().
  auto* stats_checker = new CompactionJobStatsChecker();
  Options options;
+  options.level_compaction_dynamic_level_bytes = false;
  options.listeners.emplace_back(stats_checker);
  options.create_if_missing = true;
  // just enough setting to hold off auto-compaction.
@@ -815,6 +816,7 @@ TEST_P(CompactionJobStatsTest, DeletionStatsTest) {
  // what we expect.
  auto* stats_checker = new CompactionJobDeletionStatsChecker();
  Options options;
+  options.level_compaction_dynamic_level_bytes = false;
  options.listeners.emplace_back(stats_checker);
  options.create_if_missing = true;
  options.level0_file_num_compaction_trigger = kTestScale + 1;
@@ -386,7 +386,8 @@ class CompactionJobTestBase : public testing::Test {
        kUnknownFileCreationTime,
        versions_->GetColumnFamilySet()->GetDefault()->NewEpochNumber(),
        kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2,
-        0, 0);
+        /*compensated_range_deletion_size=*/0, /*tail_size=*/0,
+        /*user_defined_timestamps_persisted=*/true);

    mutex_.Lock();
    EXPECT_OK(versions_->LogAndApply(
@@ -643,7 +644,7 @@ class CompactionJobTestBase : public testing::Test {
        mutable_cf_options_.max_compaction_bytes, 0, kNoCompression,
        cfd->GetLatestMutableCFOptions()->compression_opts,
        Temperature::kUnknown, max_subcompactions, grandparents, true);
-    compaction.SetInputVersion(cfd->current());
+    compaction.FinalizeInputInfo(cfd->current());

    assert(db_options_.info_log);
    LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, db_options_.info_log.get());
@@ -654,11 +655,12 @@ class CompactionJobTestBase : public testing::Test {
    ASSERT_TRUE(full_history_ts_low_.empty() ||
                ucmp_->timestamp_size() == full_history_ts_low_.size());
    const std::atomic<bool> kManualCompactionCanceledFalse{false};
+    JobContext job_context(1, false /* create_superversion */);
    CompactionJob compaction_job(
        0, &compaction, db_options_, mutable_db_options_, env_options_,
        versions_.get(), &shutting_down_, &log_buffer, nullptr, nullptr,
        nullptr, nullptr, &mutex_, &error_handler_, snapshots,
-        earliest_write_conflict_snapshot, snapshot_checker, nullptr,
+        earliest_write_conflict_snapshot, snapshot_checker, &job_context,
        table_cache_, &event_logger, false, false, dbname_,
        &compaction_job_stats_, Env::Priority::USER, nullptr /* IOTracer */,
        /*manual_compaction_canceled=*/kManualCompactionCanceledFalse,
@@ -672,7 +674,9 @@ class CompactionJobTestBase : public testing::Test {
    ASSERT_OK(s);
    ASSERT_OK(compaction_job.io_status());
    mutex_.Lock();
-    ASSERT_OK(compaction_job.Install(*cfd->GetLatestMutableCFOptions()));
+    bool compaction_released = false;
+    ASSERT_OK(compaction_job.Install(*cfd->GetLatestMutableCFOptions(),
+                                     &compaction_released));
    ASSERT_OK(compaction_job.io_status());
    mutex_.Unlock();
    log_buffer.FlushBufferToLog();
@@ -45,6 +45,8 @@ Status CompactionOutputs::Finish(const Status& intput_status,
    meta->fd.file_size = current_bytes;
    meta->tail_size = builder_->GetTailSize();
    meta->marked_for_compaction = builder_->NeedCompact();
+    meta->user_defined_timestamps_persisted = static_cast<bool>(
+        builder_->GetTableProperties().user_defined_timestamps_persisted);
  }
  current_output().finished = true;
  stats_.bytes_written += current_bytes;
@@ -125,11 +127,6 @@ size_t CompactionOutputs::UpdateGrandparentBoundaryInfo(
  if (grandparents.empty()) {
    return curr_key_boundary_switched_num;
  }
-  assert(!internal_key.empty());
-  InternalKey ikey;
-  ikey.DecodeFrom(internal_key);
-  assert(ikey.Valid());
-
  const Comparator* ucmp = compaction_->column_family_data()->user_comparator();

  // Move the grandparent_index_ to the file containing the current user_key.
@@ -137,7 +134,7 @@ size_t CompactionOutputs::UpdateGrandparentBoundaryInfo(
  // index points to the last file containing the key.
  while (grandparent_index_ < grandparents.size()) {
    if (being_grandparent_gap_) {
-      if (sstableKeyCompare(ucmp, ikey,
+      if (sstableKeyCompare(ucmp, internal_key,
                            grandparents[grandparent_index_]->smallest) < 0) {
        break;
      }
@@ -150,13 +147,13 @@ size_t CompactionOutputs::UpdateGrandparentBoundaryInfo(
      being_grandparent_gap_ = false;
    } else {
      int cmp_result = sstableKeyCompare(
-          ucmp, ikey, grandparents[grandparent_index_]->largest);
+          ucmp, internal_key, grandparents[grandparent_index_]->largest);
      // If it's same key, make sure grandparent_index_ is pointing to the last
      // one.
      if (cmp_result < 0 ||
          (cmp_result == 0 &&
           (grandparent_index_ == grandparents.size() - 1 ||
-            sstableKeyCompare(ucmp, ikey,
+            sstableKeyCompare(ucmp, internal_key,
                              grandparents[grandparent_index_ + 1]->smallest) <
                0))) {
        break;
@@ -598,10 +595,12 @@ Status CompactionOutputs::AddRangeDels(
    // in any snapshot. trim_ts_ is passed to the constructor for
    // range_del_agg_, and range_del_agg_ internally drops tombstones above
    // trim_ts_.
-    if (bottommost_level && tombstone.seq_ <= earliest_snapshot &&
+    bool consider_drop =
+        tombstone.seq_ <= earliest_snapshot &&
        (ts_sz == 0 ||
         (!full_history_ts_low.empty() &&
-          ucmp->CompareTimestamp(tombstone.ts_, full_history_ts_low) < 0))) {
+          ucmp->CompareTimestamp(tombstone.ts_, full_history_ts_low) < 0));
+    if (consider_drop && bottommost_level) {
      // TODO(andrewkr): tombstones that span multiple output files are
      // counted for each compaction output file, so lots of double
      // counting.
@@ -635,6 +634,20 @@ Status CompactionOutputs::AddRangeDels(
        icmp.Compare(*upper_bound, tombstone_start.Encode()) < 0) {
      break;
    }
+    if (lower_bound &&
+        icmp.Compare(tombstone_start.Encode(), *lower_bound) < 0) {
+      tombstone_start.DecodeFrom(*lower_bound);
+    }
+    if (upper_bound && icmp.Compare(*upper_bound, tombstone_end.Encode()) < 0) {
+      tombstone_end.DecodeFrom(*upper_bound);
+    }
+    if (consider_drop && compaction_->KeyRangeNotExistsBeyondOutputLevel(
+                             tombstone_start.user_key(),
+                             tombstone_end.user_key(), &level_ptrs_)) {
+      range_del_out_stats.num_range_del_drop_obsolete++;
+      range_del_out_stats.num_record_drop_obsolete++;
+      continue;
+    }
    // Here we show that *only* range tombstones that overlap with
    // [lower_bound, upper_bound] are added to the current file, and
    // sanity checking invariants that should hold:
@@ -688,13 +701,6 @@ Status CompactionOutputs::AddRangeDels(

    // Range tombstone is not supported by output validator yet.
    builder_->Add(kv.first.Encode(), kv.second);
-    if (lower_bound &&
-        icmp.Compare(tombstone_start.Encode(), *lower_bound) < 0) {
-      tombstone_start.DecodeFrom(*lower_bound);
-    }
-    if (upper_bound && icmp.Compare(*upper_bound, tombstone_end.Encode()) < 0) {
-      tombstone_end.DecodeFrom(*upper_bound);
-    }
    assert(icmp.Compare(tombstone_start, tombstone_end) <= 0);
    meta.UpdateBoundariesForRange(tombstone_start, tombstone_end,
                                  tombstone.seq_, icmp);
@@ -779,6 +785,8 @@ CompactionOutputs::CompactionOutputs(const Compaction* compaction,
  if (compaction->output_level() != 0) {
    FillFilesToCutForTtl();
  }
+
+  level_ptrs_ = std::vector<size_t>(compaction_->number_levels(), 0);
 }

 }  // namespace ROCKSDB_NAMESPACE
@@ -206,10 +206,10 @@ class CompactionOutputs {
      // We may only split the output when the cursor is in the range. Split
      if ((!end.has_value() ||
           icmp->user_comparator()->Compare(
-               ExtractUserKey(output_split_key->Encode()), end.value()) < 0) &&
-          (!start.has_value() || icmp->user_comparator()->Compare(
-                                     ExtractUserKey(output_split_key->Encode()),
-                                     start.value()) > 0)) {
+               ExtractUserKey(output_split_key->Encode()), *end) < 0) &&
+          (!start.has_value() ||
+           icmp->user_comparator()->Compare(
+               ExtractUserKey(output_split_key->Encode()), *start) > 0)) {
        local_output_split_key_ = output_split_key;
      }
    }
@@ -356,6 +356,15 @@ class CompactionOutputs {
  // The smallest key of the current output file, this is set when current
  // output file's smallest key is a range tombstone start key.
  InternalKey range_tombstone_lower_bound_;
+
+  // Used for calls to compaction->KeyRangeNotExistsBeyondOutputLevel() in
+  // CompactionOutputs::AddRangeDels().
+  // level_ptrs_[i] holds index of the file that was checked during the last
+  // call to compaction->KeyRangeNotExistsBeyondOutputLevel(). This allows
+  // future calls to the function to pick up where it left off, since each
+  // range tombstone added to output file within each subcompaction is in
+  // increasing key range.
+  std::vector<size_t> level_ptrs_;
 };

 // helper struct to concatenate the last level and penultimate level outputs
@@ -611,23 +611,21 @@ Compaction* CompactionPicker::CompactRange(
    // Universal compaction with more than one level always compacts all the
    // files together to the last level.
    assert(vstorage->num_levels() > 1);
+    int max_output_level =
+        vstorage->MaxOutputLevel(ioptions_.allow_ingest_behind);
    // DBImpl::CompactRange() set output level to be the last level
-    if (ioptions_.allow_ingest_behind) {
-      assert(output_level == vstorage->num_levels() - 2);
-    } else {
-      assert(output_level == vstorage->num_levels() - 1);
-    }
+    assert(output_level == max_output_level);
    // DBImpl::RunManualCompaction will make full range for universal compaction
    assert(begin == nullptr);
    assert(end == nullptr);
    *compaction_end = nullptr;

    int start_level = 0;
-    for (; start_level < vstorage->num_levels() &&
+    for (; start_level <= max_output_level &&
           vstorage->NumLevelFiles(start_level) == 0;
         start_level++) {
    }
-    if (start_level == vstorage->num_levels()) {
+    if (start_level > max_output_level) {
      return nullptr;
    }

@@ -637,9 +635,9 @@ Compaction* CompactionPicker::CompactRange(
      return nullptr;
    }

-    std::vector<CompactionInputFiles> inputs(vstorage->num_levels() -
+    std::vector<CompactionInputFiles> inputs(max_output_level + 1 -
                                             start_level);
-    for (int level = start_level; level < vstorage->num_levels(); level++) {
+    for (int level = start_level; level <= max_output_level; level++) {
      inputs[level - start_level].level = level;
      auto& files = inputs[level - start_level].files;
      for (FileMetaData* f : vstorage->LevelFiles(level)) {
@@ -753,8 +751,10 @@ Compaction* CompactionPicker::CompactRange(

  // for BOTTOM LEVEL compaction only, use max_file_num_to_ignore to filter out
  // files that are created during the current compaction.
-  if (compact_range_options.bottommost_level_compaction ==
-          BottommostLevelCompaction::kForceOptimized &&
+  if ((compact_range_options.bottommost_level_compaction ==
+           BottommostLevelCompaction::kForceOptimized ||
+       compact_range_options.bottommost_level_compaction ==
+           BottommostLevelCompaction::kIfHaveCompactionFilter) &&
      max_file_num_to_ignore != std::numeric_limits<uint64_t>::max()) {
    assert(input_level == output_level);
    // inputs_shrunk holds a continuous subset of input files which were all
@@ -70,6 +70,11 @@ class CompactionPickerTestBase : public testing::Test {
    mutable_cf_options_.RefreshDerivedOptions(ioptions_);
    ioptions_.cf_paths.emplace_back("dummy",
                                    std::numeric_limits<uint64_t>::max());
+    // When the default value of this option is true, universal compaction
+    // tests can encounter assertion failure since SanitizeOption() is
+    // not run to set this option to false. So we do the sanitization
+    // here. Tests that test this option set this option to true explicitly.
+    ioptions_.level_compaction_dynamic_level_bytes = false;
  }

  ~CompactionPickerTestBase() override {}
@@ -79,7 +84,8 @@ class CompactionPickerTestBase : public testing::Test {
    options_.num_levels = num_levels;
    vstorage_.reset(new VersionStorageInfo(
        &icmp_, ucmp_, options_.num_levels, style, nullptr, false,
-        EpochNumberRequirement::kMustPresent));
+        EpochNumberRequirement::kMustPresent, ioptions_.clock,
+        options_.bottommost_file_compaction_delay));
    vstorage_->PrepareForVersionAppend(ioptions_, mutable_cf_options_);
  }

@@ -88,7 +94,8 @@ class CompactionPickerTestBase : public testing::Test {
  void AddVersionStorage() {
    temp_vstorage_.reset(new VersionStorageInfo(
        &icmp_, ucmp_, options_.num_levels, ioptions_.compaction_style,
-        vstorage_.get(), false, EpochNumberRequirement::kMustPresent));
+        vstorage_.get(), false, EpochNumberRequirement::kMustPresent,
+        ioptions_.clock, options_.bottommost_file_compaction_delay));
  }

  void DeleteVersionStorage() {
@@ -148,7 +155,8 @@ class CompactionPickerTestBase : public testing::Test {
        smallest_seq, largest_seq, marked_for_compact, temperature,
        kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
        kUnknownFileCreationTime, epoch_number, kUnknownFileChecksum,
-        kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0);
+        kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0,
+        true /* user_defined_timestamps_persisted */);
    f->compensated_file_size =
        (compensated_file_size != 0) ? compensated_file_size : file_size;
    f->oldest_ancester_time = oldest_ancestor_time;
@@ -505,7 +513,7 @@ TEST_F(CompactionPickerTest, NeedsCompactionUniversal) {

 TEST_F(CompactionPickerTest, CompactionUniversalIngestBehindReservedLevel) {
  const uint64_t kFileSize = 100000;
-  NewVersionStorage(1, kCompactionStyleUniversal);
+  NewVersionStorage(3 /* num_levels */, kCompactionStyleUniversal);
  ioptions_.allow_ingest_behind = true;
  ioptions_.num_levels = 3;
  UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
@@ -532,6 +540,14 @@ TEST_F(CompactionPickerTest, CompactionUniversalIngestBehindReservedLevel) {

  // output level should be the one above the bottom-most
  ASSERT_EQ(1, compaction->output_level());
+
+  // input should not include the reserved level
+  const std::vector<CompactionInputFiles>* inputs = compaction->inputs();
+  for (const auto& compaction_input : *inputs) {
+    if (!compaction_input.empty()) {
+      ASSERT_LT(compaction_input.level, 2);
+    }
+  }
 }
 // Tests if the files can be trivially moved in multi level
 // universal compaction when allow_trivial_move option is set
@@ -978,6 +994,61 @@ TEST_F(CompactionPickerTest, UniversalIncrementalSpace5) {
  ASSERT_EQ(13, compaction->num_input_files(1));
 }

+TEST_F(CompactionPickerTest,
+       PartiallyExcludeL0ToReduceWriteStopForSizeAmpCompaction) {
+  const uint64_t kFileSize = 100000;
+  const uint64_t kL0FileCount = 30;
+  const uint64_t kLastLevelFileCount = 1;
+  const uint64_t kNumLevels = 5;
+
+  for (const uint64_t test_no_exclusion : {false, true}) {
+    const uint64_t kExpectedNumExcludedL0 =
+        test_no_exclusion ? 0 : kL0FileCount * 1 / 10;
+
+    mutable_cf_options_.level0_stop_writes_trigger = 36;
+    mutable_cf_options_.compaction_options_universal
+        .max_size_amplification_percent = 1;
+    mutable_cf_options_.compaction_options_universal.max_merge_width =
+        test_no_exclusion
+            // In universal compaction, sorted runs from non L0 levels are
+            // counted toward `level0_stop_writes_trigger`. Therefore we need to
+            // subtract the total number of sorted runs picked originally for
+            // this compaction (i.e, kL0FileCount + kLastLevelFileCount) from
+            // `level0_stop_writes_trigger` to calculate `max_merge_width` that
+            // results in no L0 exclusion for testing purpose.
+            ? mutable_cf_options_.level0_stop_writes_trigger -
+                  (kL0FileCount + kLastLevelFileCount)
+            : UINT_MAX;
+
+    UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
+    NewVersionStorage(kNumLevels, kCompactionStyleUniversal);
+
+    for (uint64_t i = 1; i <= kL0FileCount + kLastLevelFileCount; ++i) {
+      Add(i <= kL0FileCount ? 0 : kNumLevels - 1, static_cast<uint32_t>(i),
+          std::to_string((i + 100) * 1000).c_str(),
+          std::to_string((i + 100) * 1000 + 999).c_str(), kFileSize, 0, i * 100,
+          i * 100 + 99);
+    }
+
+    UpdateVersionStorageInfo();
+
+    ASSERT_TRUE(universal_compaction_picker.NeedsCompaction(vstorage_.get()));
+    std::unique_ptr<Compaction> compaction(
+        universal_compaction_picker.PickCompaction(
+            cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(),
+            &log_buffer_));
+    ASSERT_TRUE(compaction.get() != nullptr);
+    ASSERT_EQ(compaction->compaction_reason(),
+              CompactionReason::kUniversalSizeAmplification);
+    ASSERT_EQ(compaction->num_input_files(0),
+              kL0FileCount - kExpectedNumExcludedL0);
+    ASSERT_EQ(compaction->num_input_files(kNumLevels - 1), kLastLevelFileCount);
+    for (uint64_t level = 1; level <= kNumLevels - 2; level++) {
+      ASSERT_EQ(compaction->num_input_files(level), 0);
+    }
+  }
+}
+
 TEST_F(CompactionPickerTest, NeedsCompactionFIFO) {
  NewVersionStorage(1, kCompactionStyleFIFO);
  const int kFileCount =
@@ -1954,6 +2025,15 @@ TEST_F(CompactionPickerTest, OverlappingUserKeys11) {
  ASSERT_EQ(7U, compaction->input(1, 0)->fd.GetNumber());
 }

+TEST_F(CompactionPickerTest, FileTtlBoosterLargeNumLevels) {
+  const uint64_t kCurrentTime = 1000000;
+  FileTtlBooster booster(kCurrentTime, /*ttl=*/2048,
+                         /*num_non_empty_levels=*/100, /*level=*/1);
+  FileMetaData meta;
+  meta.oldest_ancester_time = kCurrentTime - 1023;
+  ASSERT_EQ(1, booster.GetBoostScore(&meta));
+}
+
 TEST_F(CompactionPickerTest, FileTtlBooster) {
  // Set TTL to 2048
  // TTL boosting for all levels starts at 1024,
@@ -3355,6 +3435,9 @@ TEST_F(CompactionPickerTest, UniversalSizeAmpTierCompactionNonLastLevel) {
  ioptions_.preclude_last_level_data_seconds = 1000;
  mutable_cf_options_.compaction_options_universal
      .max_size_amplification_percent = 200;
+  // To avoid any L0 file exclusion in size amp compaction intended for reducing
+  // write stop
+  mutable_cf_options_.compaction_options_universal.max_merge_width = 2;
  UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);

  NewVersionStorage(kNumLevels, kCompactionStyleUniversal);
@@ -3428,6 +3511,9 @@ TEST_F(CompactionPickerTest, UniversalSizeAmpTierCompactionNotSuport) {
  ioptions_.preclude_last_level_data_seconds = 1000;
  mutable_cf_options_.compaction_options_universal
      .max_size_amplification_percent = 200;
+  // To avoid any L0 file exclusion in size amp compaction intended for reducing
+  // write stop
+  mutable_cf_options_.compaction_options_universal.max_merge_width = 2;
  UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);

  NewVersionStorage(kNumLevels, kCompactionStyleUniversal);
@@ -9,7 +9,7 @@

 #include "db/compaction/compaction_picker_universal.h"

-#include <cinttypes>
+#include <cstdint>
 #include <limits>
 #include <queue>
 #include <string>
@@ -114,6 +114,13 @@ class UniversalCompactionBuilder {
  // because some files are being compacted.
  Compaction* PickPeriodicCompaction();

+  bool ShouldSkipLastSortedRunForSizeAmpCompaction() const {
+    assert(!sorted_runs_.empty());
+    return ioptions_.preclude_last_level_data_seconds > 0 &&
+           ioptions_.num_levels > 2 &&
+           sorted_runs_.back().level == ioptions_.num_levels - 1 &&
+           sorted_runs_.size() > 1;
+  }
  // Used in universal compaction when the allow_trivial_move
  // option is set. Checks whether there are any overlapping files
  // in the input. Returns true if the input files are non
@@ -122,6 +129,100 @@ class UniversalCompactionBuilder {

  uint64_t GetMaxOverlappingBytes() const;

+  // To conditionally exclude some of the newest L0 files
+  // from a size amp compaction. This is to prevent a large number of L0
+  // files from being locked by a size amp compaction, potentially leading to
+  // write stop with a few more flushes.
+  //
+  // Such exclusion is based on `num_l0_input_pre_exclusion`,
+  // `level0_stop_writes_trigger`, `max/min_merge_width` and the pre-exclusion
+  // compaction score. Noted that it will not make the size amp compaction of
+  // interest invalid from running as a size amp compaction as long as its
+  // pre-exclusion compaction score satisfies the condition to run.
+  //
+  // @param `num_l0_input_pre_exclusion` Number of L0 input files prior to
+  // exclusion
+  // @param `end_index` Index of the last sorted run selected as compaction
+  // input. Will not be affected by this exclusion.
+  // @param `start_index` Index of the first input sorted run prior to
+  // exclusion. Will be modified as output based on the exclusion.
+  // @param `candidate_size` Total size of all except for the last input sorted
+  // runs prior to exclusion. Will be modified as output based on the exclusion.
+  //
+  // @return Number of L0 files to exclude. `start_index` and
+  // `candidate_size` will be modified accordingly
+  std::size_t MightExcludeNewL0sToReduceWriteStop(
+      std::size_t num_l0_input_pre_exclusion, std::size_t end_index,
+      std::size_t& start_index, uint64_t& candidate_size) const {
+    if (num_l0_input_pre_exclusion == 0) {
+      return 0;
+    }
+
+    assert(start_index <= end_index && sorted_runs_.size() > end_index);
+    assert(mutable_cf_options_.level0_stop_writes_trigger > 0);
+    const std::size_t level0_stop_writes_trigger = static_cast<std::size_t>(
+        mutable_cf_options_.level0_stop_writes_trigger);
+    const std::size_t max_merge_width = static_cast<std::size_t>(
+        mutable_cf_options_.compaction_options_universal.max_merge_width);
+    const std::size_t min_merge_width = static_cast<std::size_t>(
+        mutable_cf_options_.compaction_options_universal.min_merge_width);
+    const uint64_t max_size_amplification_percent =
+        mutable_cf_options_.compaction_options_universal
+            .max_size_amplification_percent;
+    const uint64_t base_sr_size = sorted_runs_[end_index].size;
+
+    // Leave at least 1 L0 file and 2 input sorted runs after exclusion
+    const std::size_t max_num_l0_to_exclude =
+        std::min(num_l0_input_pre_exclusion - 1, end_index - start_index - 1);
+    // In universal compaction, sorted runs from non L0 levels are counted
+    // toward `level0_stop_writes_trigger`. Therefore we need to subtract the
+    // total number of sorted runs picked originally for this compaction from
+    // `level0_stop_writes_trigger` to calculate
+    // `num_extra_l0_before_write_stop`
+    const std::size_t num_extra_l0_before_write_stop =
+        level0_stop_writes_trigger -
+        std::min(level0_stop_writes_trigger, end_index - start_index + 1);
+    const std::size_t num_l0_to_exclude_for_max_merge_width =
+        std::min(max_merge_width -
+                     std::min(max_merge_width, num_extra_l0_before_write_stop),
+                 max_num_l0_to_exclude);
+    const std::size_t num_l0_to_exclude_for_min_merge_width =
+        std::min(min_merge_width -
+                     std::min(min_merge_width, num_extra_l0_before_write_stop),
+                 max_num_l0_to_exclude);
+
+    std::size_t num_l0_to_exclude = 0;
+    uint64_t candidate_size_post_exclusion = candidate_size;
+
+    for (std::size_t possible_num_l0_to_exclude =
+             num_l0_to_exclude_for_min_merge_width;
+         possible_num_l0_to_exclude <= num_l0_to_exclude_for_max_merge_width;
+         ++possible_num_l0_to_exclude) {
+      uint64_t current_candidate_size = candidate_size_post_exclusion;
+      for (std::size_t j = num_l0_to_exclude; j < possible_num_l0_to_exclude;
+           ++j) {
+        current_candidate_size -=
+            sorted_runs_.at(start_index + j).compensated_file_size;
+      }
+
+      // To ensure the compaction score before and after exclusion is similar
+      // so this exclusion will not make the size amp compaction of
+      // interest invalid from running as a size amp compaction as long as its
+      // pre-exclusion compaction score satisfies the condition to run.
+      if (current_candidate_size * 100 <
+              max_size_amplification_percent * base_sr_size ||
+          current_candidate_size < candidate_size * 9 / 10) {
+        break;
+      }
+      num_l0_to_exclude = possible_num_l0_to_exclude;
+      candidate_size_post_exclusion = current_candidate_size;
+    }
+
+    start_index += num_l0_to_exclude;
+    candidate_size = candidate_size_post_exclusion;
+    return num_l0_to_exclude;
+  }
+
  const ImmutableOptions& ioptions_;
  const InternalKeyComparator* icmp_;
  double score_;
@@ -133,8 +234,8 @@ class UniversalCompactionBuilder {
  UniversalCompactionPicker* picker_;
  LogBuffer* log_buffer_;

-  static std::vector<SortedRun> CalculateSortedRuns(
-      const VersionStorageInfo& vstorage);
+  static std::vector<UniversalCompactionBuilder::SortedRun> CalculateSortedRuns(
+      const VersionStorageInfo& vstorage, int last_level);

  // Pick a path ID to place a newly generated file, with its estimated file
  // size.
@@ -339,13 +440,13 @@ void UniversalCompactionBuilder::SortedRun::DumpSizeInfo(

 std::vector<UniversalCompactionBuilder::SortedRun>
 UniversalCompactionBuilder::CalculateSortedRuns(
-    const VersionStorageInfo& vstorage) {
+    const VersionStorageInfo& vstorage, int last_level) {
  std::vector<UniversalCompactionBuilder::SortedRun> ret;
  for (FileMetaData* f : vstorage.LevelFiles(0)) {
    ret.emplace_back(0, f, f->fd.GetFileSize(), f->compensated_file_size,
                     f->being_compacted);
  }
-  for (int level = 1; level < vstorage.num_levels(); level++) {
+  for (int level = 1; level <= last_level; level++) {
    uint64_t total_compensated_size = 0U;
    uint64_t total_size = 0U;
    bool being_compacted = false;
@@ -374,7 +475,9 @@ UniversalCompactionBuilder::CalculateSortedRuns(
 Compaction* UniversalCompactionBuilder::PickCompaction() {
  const int kLevel0 = 0;
  score_ = vstorage_->CompactionScore(kLevel0);
-  sorted_runs_ = CalculateSortedRuns(*vstorage_);
+  int max_output_level =
+      vstorage_->MaxOutputLevel(ioptions_.allow_ingest_behind);
+  sorted_runs_ = CalculateSortedRuns(*vstorage_, max_output_level);

  if (sorted_runs_.size() == 0 ||
      (vstorage_->FilesMarkedForPeriodicCompaction().empty() &&
@@ -471,6 +574,8 @@ Compaction* UniversalCompactionBuilder::PickCompaction() {
        "UniversalCompactionBuilder::PickCompaction:Return", nullptr);
    return nullptr;
  }
+  assert(c->output_level() <=
+         vstorage_->MaxOutputLevel(ioptions_.allow_ingest_behind));

  if (mutable_cf_options_.compaction_options_universal.allow_trivial_move ==
          true &&
@@ -698,22 +803,18 @@ Compaction* UniversalCompactionBuilder::PickCompactionToReduceSortedRuns(
      GetPathId(ioptions_, mutable_cf_options_, estimated_total_size);
  int start_level = sorted_runs_[start_index].level;
  int output_level;
+  // last level is reserved for the files ingested behind
+  int max_output_level =
+      vstorage_->MaxOutputLevel(ioptions_.allow_ingest_behind);
  if (first_index_after == sorted_runs_.size()) {
-    output_level = vstorage_->num_levels() - 1;
+    output_level = max_output_level;
  } else if (sorted_runs_[first_index_after].level == 0) {
    output_level = 0;
  } else {
    output_level = sorted_runs_[first_index_after].level - 1;
  }

-  // last level is reserved for the files ingested behind
-  if (ioptions_.allow_ingest_behind &&
-      (output_level == vstorage_->num_levels() - 1)) {
-    assert(output_level > 1);
-    output_level--;
-  }
-
-  std::vector<CompactionInputFiles> inputs(vstorage_->num_levels());
+  std::vector<CompactionInputFiles> inputs(max_output_level + 1);
  for (size_t i = 0; i < inputs.size(); ++i) {
    inputs[i].level = start_level + static_cast<int>(i);
  }
@@ -778,85 +879,67 @@ Compaction* UniversalCompactionBuilder::PickCompactionToReduceSortedRuns(

 // Look at overall size amplification. If size amplification
 // exceeds the configured value, then do a compaction
-// of the candidate files all the way upto the earliest
-// base file (overrides configured values of file-size ratios,
-// min_merge_width and max_merge_width).
-//
+// on longest span of candidate files without conflict with other compactions
+// ending at the earliest base file (overriding configured values of file-size
+// ratios, min_merge_width and max_merge_width).
 Compaction* UniversalCompactionBuilder::PickCompactionToReduceSizeAmp() {
-  // percentage flexibility while reducing size amplification
-  uint64_t ratio = mutable_cf_options_.compaction_options_universal
-                       .max_size_amplification_percent;
-
-  unsigned int candidate_count = 0;
-  uint64_t candidate_size = 0;
-  size_t start_index = 0;
-  const SortedRun* sr = nullptr;
-
  assert(!sorted_runs_.empty());
-  if (sorted_runs_.back().being_compacted) {
+
+  const size_t end_index = ShouldSkipLastSortedRunForSizeAmpCompaction()
+                               ? sorted_runs_.size() - 2
+                               : sorted_runs_.size() - 1;
+  if (sorted_runs_[end_index].being_compacted) {
+    return nullptr;
+  }
+  const uint64_t base_sr_size = sorted_runs_[end_index].size;
+  size_t start_index = end_index;
+  uint64_t candidate_size = 0;
+  size_t num_l0_files = 0;
+
+  // Get longest span (i.e, [start_index, end_index]) of available sorted runs
+  while (start_index > 0) {
+    const SortedRun* sr = &sorted_runs_[start_index - 1];
+    if (sr->being_compacted) {
+      char file_num_buf[kFormatFileNumberBufSize];
+      sr->Dump(file_num_buf, sizeof(file_num_buf), true);
+      ROCKS_LOG_BUFFER(
+          log_buffer_,
+          "[%s] Universal: stopping at sorted run undergoing compaction: "
+          "%s[%" ROCKSDB_PRIszt "]",
+          cf_name_.c_str(), file_num_buf, start_index - 1);
+      break;
+    }
+    candidate_size += sr->compensated_file_size;
+    num_l0_files += sr->level == 0 ? 1 : 0;
+    --start_index;
+  }
+
+  if (start_index == end_index) {
    return nullptr;
  }

-  // Skip files that are already being compacted
-  for (size_t loop = 0; loop + 1 < sorted_runs_.size(); loop++) {
-    sr = &sorted_runs_[loop];
-    if (!sr->being_compacted) {
-      start_index = loop;  // Consider this as the first candidate.
-      break;
-    }
-    char file_num_buf[kFormatFileNumberBufSize];
-    sr->Dump(file_num_buf, sizeof(file_num_buf), true);
+  {
+    const size_t num_l0_to_exclude = MightExcludeNewL0sToReduceWriteStop(
+        num_l0_files, end_index, start_index, candidate_size);
    ROCKS_LOG_BUFFER(log_buffer_,
-                     "[%s] Universal: skipping %s[%d] compacted %s",
-                     cf_name_.c_str(), file_num_buf, loop,
-                     " cannot be a candidate to reduce size amp.\n");
-    sr = nullptr;
+                     "[%s] Universal: Excluding %" ROCKSDB_PRIszt
+                     " latest L0 files to reduce potential write stop "
+                     "triggered by `level0_stop_writes_trigger`",
+                     cf_name_.c_str(), num_l0_to_exclude);
  }

-  if (sr == nullptr) {
-    return nullptr;  // no candidate files
-  }
  {
    char file_num_buf[kFormatFileNumberBufSize];
-    sr->Dump(file_num_buf, sizeof(file_num_buf), true);
+    sorted_runs_[start_index].Dump(file_num_buf, sizeof(file_num_buf), true);
    ROCKS_LOG_BUFFER(
        log_buffer_,
        "[%s] Universal: First candidate %s[%" ROCKSDB_PRIszt "] %s",
        cf_name_.c_str(), file_num_buf, start_index, " to reduce size amp.\n");
  }

-  // size of the base sorted run for size amp calculation
-  uint64_t base_sr_size = sorted_runs_.back().size;
-  size_t sr_end_idx = sorted_runs_.size() - 1;
-  // If tiered compaction is enabled and the last sorted run is the last level
-  if (ioptions_.preclude_last_level_data_seconds > 0 &&
-      ioptions_.num_levels > 2 &&
-      sorted_runs_.back().level == ioptions_.num_levels - 1 &&
-      sorted_runs_.size() > 1) {
-    sr_end_idx = sorted_runs_.size() - 2;
-    base_sr_size = sorted_runs_[sr_end_idx].size;
-  }
-
-  // keep adding up all the remaining files
-  for (size_t loop = start_index; loop < sr_end_idx; loop++) {
-    sr = &sorted_runs_[loop];
-    if (sr->being_compacted) {
-      // TODO with incremental compaction is supported, we might want to
-      // schedule some incremental compactions in parallel if needed.
-      char file_num_buf[kFormatFileNumberBufSize];
-      sr->Dump(file_num_buf, sizeof(file_num_buf), true);
-      ROCKS_LOG_BUFFER(
-          log_buffer_, "[%s] Universal: Possible candidate %s[%d] %s",
-          cf_name_.c_str(), file_num_buf, start_index,
-          " is already being compacted. No size amp reduction possible.\n");
-      return nullptr;
-    }
-    candidate_size += sr->compensated_file_size;
-    candidate_count++;
-  }
-  if (candidate_count == 0) {
-    return nullptr;
-  }
+  // percentage flexibility while reducing size amplification
+  const uint64_t ratio = mutable_cf_options_.compaction_options_universal
+                             .max_size_amplification_percent;

  // size amplification = percentage of additional size
  if (candidate_size * 100 < ratio * base_sr_size) {
@@ -893,7 +976,7 @@ Compaction* UniversalCompactionBuilder::PickCompactionToReduceSizeAmp() {
    }
  }
  return PickCompactionWithSortedRunRange(
-      start_index, sr_end_idx, CompactionReason::kUniversalSizeAmplification);
+      start_index, end_index, CompactionReason::kUniversalSizeAmplification);
 }

 Compaction* UniversalCompactionBuilder::PickIncrementalForReduceSizeAmp(
@@ -1192,8 +1275,10 @@ Compaction* UniversalCompactionBuilder::PickDeleteTriggeredCompaction() {
      return nullptr;
    }

+    int max_output_level =
+        vstorage_->MaxOutputLevel(ioptions_.allow_ingest_behind);
    // Pick the first non-empty level after the start_level
-    for (output_level = start_level + 1; output_level < vstorage_->num_levels();
+    for (output_level = start_level + 1; output_level <= max_output_level;
         output_level++) {
      if (vstorage_->NumLevelFiles(output_level) != 0) {
        break;
@@ -1201,9 +1286,9 @@ Compaction* UniversalCompactionBuilder::PickDeleteTriggeredCompaction() {
    }

    // If all higher levels are empty, pick the highest level as output level
-    if (output_level == vstorage_->num_levels()) {
+    if (output_level > max_output_level) {
      if (start_level == 0) {
-        output_level = vstorage_->num_levels() - 1;
+        output_level = max_output_level;
      } else {
        // If start level is non-zero and all higher levels are empty, this
        // compaction will translate into a trivial move. Since the idea is
@@ -1212,11 +1297,7 @@ Compaction* UniversalCompactionBuilder::PickDeleteTriggeredCompaction() {
        return nullptr;
      }
    }
-    if (ioptions_.allow_ingest_behind &&
-        output_level == vstorage_->num_levels() - 1) {
-      assert(output_level > 1);
-      output_level--;
-    }
+    assert(output_level <= max_output_level);

    if (output_level != 0) {
      if (start_level == 0) {
@@ -1293,8 +1374,9 @@ Compaction* UniversalCompactionBuilder::PickCompactionWithSortedRunRange(
  uint32_t path_id =
      GetPathId(ioptions_, mutable_cf_options_, estimated_total_size);
  int start_level = sorted_runs_[start_index].level;
-
-  std::vector<CompactionInputFiles> inputs(vstorage_->num_levels());
+  int max_output_level =
+      vstorage_->MaxOutputLevel(ioptions_.allow_ingest_behind);
+  std::vector<CompactionInputFiles> inputs(max_output_level + 1);
  for (size_t i = 0; i < inputs.size(); ++i) {
    inputs[i].level = start_level + static_cast<int>(i);
  }
@@ -1331,13 +1413,7 @@ Compaction* UniversalCompactionBuilder::PickCompactionWithSortedRunRange(

  int output_level;
  if (end_index == sorted_runs_.size() - 1) {
-    // output files at the last level, unless it's reserved
-    output_level = vstorage_->num_levels() - 1;
-    // last level is reserved for the files ingested behind
-    if (ioptions_.allow_ingest_behind) {
-      assert(output_level > 1);
-      output_level--;
-    }
+    output_level = max_output_level;
  } else {
    // if it's not including all sorted_runs, it can only output to the level
    // above the `end_index + 1` sorted_run.
@@ -1449,4 +1525,3 @@ uint64_t UniversalCompactionBuilder::GetMaxOverlappingBytes() const {
  }
 }
 }  // namespace ROCKSDB_NAMESPACE
-
@@ -53,8 +53,10 @@ class FileTtlBooster {
      enabled_ = true;
      uint64_t all_boost_start_age = ttl / 2;
      uint64_t all_boost_age_range = (ttl / 32) * 31 - all_boost_start_age;
+      // TODO(cbi): more reasonable algorithm that gives different values
+      //   when num_non_empty_levels - level - 1 > 63.
      uint64_t boost_age_range =
-          all_boost_age_range >> (num_non_empty_levels - level - 1);
+          all_boost_age_range >> std::min(63, num_non_empty_levels - level - 1);
      boost_age_start_ = all_boost_start_age + boost_age_range;
      const uint64_t kBoostRatio = 16;
      // prevent 0 value to avoid divide 0 error.
@@ -209,7 +209,7 @@ TEST_P(TieredCompactionTest, SequenceBasedTieredStorageUniversal) {
    seq_history.emplace_back(dbfull()->GetLatestSequenceNumber());
    expect_stats[0].Add(kBasicFlushStats);
  }
-  ASSERT_OK(dbfull()->WaitForCompact());
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());

  // the penultimate level file temperature is not cold, all data are output to
  // the penultimate level.
@@ -374,7 +374,7 @@ TEST_P(TieredCompactionTest, RangeBasedTieredStorageUniversal) {
    ASSERT_OK(Flush());
    expect_stats[0].Add(kBasicFlushStats);
  }
-  ASSERT_OK(dbfull()->WaitForCompact());
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
  ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel());
  ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0);
  ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
@@ -446,7 +446,7 @@ TEST_P(TieredCompactionTest, RangeBasedTieredStorageUniversal) {
    ASSERT_OK(Flush());
  }
  // make sure the compaction is able to finish
-  ASSERT_OK(dbfull()->WaitForCompact());
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
  ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel());
  ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0);
  ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
@@ -911,7 +911,7 @@ TEST_P(TieredCompactionTest, SequenceBasedTieredStorageLevel) {
    ASSERT_OK(Flush());
    expect_stats[0].Add(kBasicFlushStats);
  }
-  ASSERT_OK(dbfull()->WaitForCompact());
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());

  // non last level is hot
  ASSERT_EQ("0,1", FilesPerLevel());
@@ -954,7 +954,7 @@ TEST_P(TieredCompactionTest, SequenceBasedTieredStorageLevel) {
    ASSERT_OK(Flush());
    seq_history.emplace_back(dbfull()->GetLatestSequenceNumber());
  }
-  ASSERT_OK(dbfull()->WaitForCompact());
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
  ASSERT_EQ("0,1,0,0,0,0,1", FilesPerLevel());
  ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0);
  ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
@@ -1005,7 +1005,7 @@ TEST_P(TieredCompactionTest, SequenceBasedTieredStorageLevel) {
    ASSERT_OK(Flush());
    seq_history.emplace_back(dbfull()->GetLatestSequenceNumber());
  }
-  ASSERT_OK(dbfull()->WaitForCompact());
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());

  latest_cold_seq = seq_history[0];
  ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
@@ -1135,7 +1135,7 @@ TEST_P(TieredCompactionTest, RangeBasedTieredStorageLevel) {
    }
    ASSERT_OK(Flush());
  }
-  ASSERT_OK(dbfull()->WaitForCompact());
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
  ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel());
  ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0);
  ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
@@ -1265,7 +1265,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeManualCompaction) {
    }
    ASSERT_OK(Flush());
  }
-  ASSERT_OK(dbfull()->WaitForCompact());
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());

  // all data is pushed to the last level
  ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel());
@@ -1327,7 +1327,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeAutoCompaction) {
    }
    ASSERT_OK(Flush());
  }
-  ASSERT_OK(dbfull()->WaitForCompact());
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());

  // all data is pushed to the last level
  ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel());
@@ -1360,7 +1360,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeAutoCompaction) {
      });
    }
    ASSERT_OK(Flush());
-    ASSERT_OK(dbfull()->WaitForCompact());
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
  }

  // all data is moved up to the penultimate level
@@ -1403,7 +1403,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimePartial) {
    }
    ASSERT_OK(Flush());
  }
-  ASSERT_OK(dbfull()->WaitForCompact());
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());

  // all data is pushed to the last level
  ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel());
@@ -1530,7 +1530,7 @@ TEST_F(PrecludeLastLevelTest, LastLevelOnlyCompactionPartial) {
    }
    ASSERT_OK(Flush());
  }
-  ASSERT_OK(dbfull()->WaitForCompact());
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());

  // all data is pushed to the last level
  ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel());
@@ -1609,7 +1609,7 @@ TEST_P(PrecludeLastLevelTestWithParms, LastLevelOnlyCompactionNoPreclude) {
    }
    ASSERT_OK(Flush());
  }
-  ASSERT_OK(dbfull()->WaitForCompact());
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());

  // all data is pushed to the last level
  ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel());
@@ -1705,7 +1705,7 @@ TEST_P(PrecludeLastLevelTestWithParms, LastLevelOnlyCompactionNoPreclude) {

  manual_compaction_thread.join();

-  ASSERT_OK(dbfull()->WaitForCompact());
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());

  if (enable_preclude_last_level) {
    ASSERT_NE("0,0,0,0,0,1,1", FilesPerLevel());
@@ -1841,7 +1841,7 @@ TEST_P(PrecludeLastLevelTestWithParms, PeriodicCompactionToPenultimateLevel) {
  }
  ASSERT_OK(Flush());

-  ASSERT_OK(dbfull()->WaitForCompact());
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());

  stop_token.reset();

@@ -1940,7 +1940,7 @@ TEST_F(PrecludeLastLevelTest, PartialPenultimateLevelCompaction) {
    ASSERT_OK(Flush());
  }

-  ASSERT_OK(dbfull()->WaitForCompact());
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());

  // L5: [0,19] [20,39] [40,299]
  // L6: [0,                299]
@@ -2106,7 +2106,7 @@ TEST_F(PrecludeLastLevelTest, RangeDelsCauseFileEndpointsToOverlap) {
  Slice begin_key(begin_key_buf), end_key(end_key_buf);
  ASSERT_OK(db_->SuggestCompactRange(db_->DefaultColumnFamily(), &begin_key,
                                     &end_key));
-  ASSERT_OK(dbfull()->WaitForCompact());
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
  ASSERT_EQ("0,0,0,0,0,3,3", FilesPerLevel());
  ASSERT_EQ(1, per_key_comp_num);
  verify_db();
@@ -2116,7 +2116,7 @@ TEST_F(PrecludeLastLevelTest, RangeDelsCauseFileEndpointsToOverlap) {
  db_->ReleaseSnapshot(snap2);
  ASSERT_OK(db_->SuggestCompactRange(db_->DefaultColumnFamily(), &begin_key,
                                     &end_key));
-  ASSERT_OK(dbfull()->WaitForCompact());
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
  ASSERT_EQ("0,0,0,0,0,3,3", FilesPerLevel());
  ASSERT_EQ(2, per_key_comp_num);
  verify_db();
@@ -2126,7 +2126,7 @@ TEST_F(PrecludeLastLevelTest, RangeDelsCauseFileEndpointsToOverlap) {
  db_->ReleaseSnapshot(snap1);
  ASSERT_OK(db_->SuggestCompactRange(db_->DefaultColumnFamily(), &begin_key,
                                     &end_key));
-  ASSERT_OK(dbfull()->WaitForCompact());
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
  ASSERT_EQ("0,0,0,0,0,2,3", FilesPerLevel());
  ASSERT_EQ(3, per_key_comp_num);
  verify_db();
@@ -77,7 +77,7 @@ void DoRandomIteraratorTest(DB* db, std::vector<std::string> source_strings,

  for (int i = 0; i < num_writes; i++) {
    if (num_trigger_flush > 0 && i != 0 && i % num_trigger_flush == 0) {
-      db->Flush(FlushOptions());
+      ASSERT_OK(db->Flush(FlushOptions()));
    }

    int type = rnd->Uniform(2);
@@ -156,6 +156,7 @@ void DoRandomIteraratorTest(DB* db, std::vector<std::string> source_strings,
        if (map.find(key) == map.end()) {
          ASSERT_TRUE(status.IsNotFound());
        } else {
+          ASSERT_OK(status);
          ASSERT_EQ(map[key], result);
        }
        break;
@@ -7,6 +7,7 @@

 #include "rocksdb/convenience.h"

+#include "db/convenience_impl.h"
 #include "db/db_impl/db_impl.h"
 #include "util/cast_util.h"

@@ -39,9 +40,25 @@ Status VerifySstFileChecksum(const Options& options,
 }
 Status VerifySstFileChecksum(const Options& options,
                             const EnvOptions& env_options,
-                             const ReadOptions& read_options,
+                             const ReadOptions& _read_options,
                             const std::string& file_path,
                             const SequenceNumber& largest_seqno) {
+  if (_read_options.io_activity != Env::IOActivity::kUnknown) {
+    return Status::InvalidArgument(
+        "Can only call VerifySstFileChecksum with `ReadOptions::io_activity` "
+        "is "
+        "`Env::IOActivity::kUnknown`");
+  }
+  ReadOptions read_options(_read_options);
+  return VerifySstFileChecksumInternal(options, env_options, read_options,
+                                       file_path, largest_seqno);
+}
+
+Status VerifySstFileChecksumInternal(const Options& options,
+                                     const EnvOptions& env_options,
+                                     const ReadOptions& read_options,
+                                     const std::string& file_path,
+                                     const SequenceNumber& largest_seqno) {
  std::unique_ptr<FSRandomAccessFile> file;
  uint64_t file_size;
  InternalKeyComparator internal_comparator(options.comparator);
@@ -68,8 +85,8 @@ Status VerifySstFileChecksum(const Options& options,
      !kImmortal, false /* force_direct_prefetch */, -1 /* level */);
  reader_options.largest_seqno = largest_seqno;
  s = ioptions.table_factory->NewTableReader(
-      reader_options, std::move(file_reader), file_size, &table_reader,
-      false /* prefetch_index_and_filter_in_cache */);
+      read_options, reader_options, std::move(file_reader), file_size,
+      &table_reader, false /* prefetch_index_and_filter_in_cache */);
  if (!s.ok()) {
    return s;
  }
@@ -0,0 +1,15 @@
+// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+#include "rocksdb/db.h"
+
+namespace ROCKSDB_NAMESPACE {
+Status VerifySstFileChecksumInternal(const Options& options,
+                                     const EnvOptions& env_options,
+                                     const ReadOptions& read_options,
+                                     const std::string& file_path,
+                                     const SequenceNumber& largest_seqno = 0);
+}  // namespace ROCKSDB_NAMESPACE
@@ -450,6 +450,7 @@ TEST_F(CorruptionTest, TableFile) {

 TEST_F(CorruptionTest, VerifyChecksumReadahead) {
  Options options;
+  options.level_compaction_dynamic_level_bytes = false;
  SpecialEnv senv(base_env_);
  options.env = &senv;
  // Disable block cache as we are going to check checksum for
@@ -503,6 +504,7 @@ TEST_F(CorruptionTest, VerifyChecksumReadahead) {

 TEST_F(CorruptionTest, TableFileIndexData) {
  Options options;
+  options.level_compaction_dynamic_level_bytes = false;
  // very big, we'll trigger flushes manually
  options.write_buffer_size = 100 * 1024 * 1024;
  Reopen(&options);
@@ -659,6 +661,7 @@ TEST_F(CorruptionTest, CorruptedDescriptor) {

 TEST_F(CorruptionTest, CompactionInputError) {
  Options options;
+  options.level_compaction_dynamic_level_bytes = false;
  options.env = env_.get();
  Reopen(&options);
  Build(10);
@@ -680,6 +683,7 @@ TEST_F(CorruptionTest, CompactionInputError) {

 TEST_F(CorruptionTest, CompactionInputErrorParanoid) {
  Options options;
+  options.level_compaction_dynamic_level_bytes = false;
  options.env = env_.get();
  options.paranoid_checks = true;
  options.write_buffer_size = 131072;
@@ -777,6 +781,7 @@ TEST_F(CorruptionTest, RangeDeletionCorrupted) {
 TEST_F(CorruptionTest, FileSystemStateCorrupted) {
  for (int iter = 0; iter < 2; ++iter) {
    Options options;
+    options.level_compaction_dynamic_level_bytes = false;
    options.env = env_.get();
    options.paranoid_checks = true;
    options.create_if_missing = true;
@@ -816,6 +821,7 @@ static const auto& corruption_modes = {

 TEST_F(CorruptionTest, ParanoidFileChecksOnFlush) {
  Options options;
+  options.level_compaction_dynamic_level_bytes = false;
  options.env = env_.get();
  options.check_flush_compaction_key_order = false;
  options.paranoid_file_checks = true;
@@ -844,6 +850,7 @@ TEST_F(CorruptionTest, ParanoidFileChecksOnFlush) {

 TEST_F(CorruptionTest, ParanoidFileChecksOnCompact) {
  Options options;
+  options.level_compaction_dynamic_level_bytes = false;
  options.env = env_.get();
  options.paranoid_file_checks = true;
  options.create_if_missing = true;
@@ -877,6 +884,7 @@ TEST_F(CorruptionTest, ParanoidFileChecksOnCompact) {

 TEST_F(CorruptionTest, ParanoidFileChecksWithDeleteRangeFirst) {
  Options options;
+  options.level_compaction_dynamic_level_bytes = false;
  options.env = env_.get();
  options.check_flush_compaction_key_order = false;
  options.paranoid_file_checks = true;
@@ -913,6 +921,7 @@ TEST_F(CorruptionTest, ParanoidFileChecksWithDeleteRangeFirst) {

 TEST_F(CorruptionTest, ParanoidFileChecksWithDeleteRange) {
  Options options;
+  options.level_compaction_dynamic_level_bytes = false;
  options.env = env_.get();
  options.check_flush_compaction_key_order = false;
  options.paranoid_file_checks = true;
@@ -952,6 +961,7 @@ TEST_F(CorruptionTest, ParanoidFileChecksWithDeleteRange) {

 TEST_F(CorruptionTest, ParanoidFileChecksWithDeleteRangeLast) {
  Options options;
+  options.level_compaction_dynamic_level_bytes = false;
  options.env = env_.get();
  options.check_flush_compaction_key_order = false;
  options.paranoid_file_checks = true;
@@ -988,6 +998,7 @@ TEST_F(CorruptionTest, ParanoidFileChecksWithDeleteRangeLast) {

 TEST_F(CorruptionTest, LogCorruptionErrorsInCompactionIterator) {
  Options options;
+  options.level_compaction_dynamic_level_bytes = false;
  options.env = env_.get();
  options.create_if_missing = true;
  options.allow_data_in_errors = true;
@@ -1017,6 +1028,7 @@ TEST_F(CorruptionTest, LogCorruptionErrorsInCompactionIterator) {

 TEST_F(CorruptionTest, CompactionKeyOrderCheck) {
  Options options;
+  options.level_compaction_dynamic_level_bytes = false;
  options.env = env_.get();
  options.paranoid_file_checks = false;
  options.create_if_missing = true;
@@ -1044,6 +1056,7 @@ TEST_F(CorruptionTest, CompactionKeyOrderCheck) {

 TEST_F(CorruptionTest, FlushKeyOrderCheck) {
  Options options;
+  options.level_compaction_dynamic_level_bytes = false;
  options.env = env_.get();
  options.paranoid_file_checks = false;
  options.create_if_missing = true;
@@ -1097,6 +1110,7 @@ TEST_F(CorruptionTest, DisableKeyOrderCheck) {
 TEST_F(CorruptionTest, VerifyWholeTableChecksum) {
  CloseDb();
  Options options;
+  options.level_compaction_dynamic_level_bytes = false;
  options.env = env_.get();
  ASSERT_OK(DestroyDB(dbname_, options));
  options.create_if_missing = true;
@@ -1182,6 +1196,7 @@ INSTANTIATE_TEST_CASE_P(CorruptionTest, CrashDuringRecoveryWithCorruptionTest,
 TEST_P(CrashDuringRecoveryWithCorruptionTest, CrashDuringRecovery) {
  CloseDb();
  Options options;
+  options.level_compaction_dynamic_level_bytes = false;
  options.track_and_verify_wals_in_manifest =
      track_and_verify_wals_in_manifest_;
  options.wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery;
@@ -1354,6 +1369,7 @@ TEST_P(CrashDuringRecoveryWithCorruptionTest, CrashDuringRecovery) {
 TEST_P(CrashDuringRecoveryWithCorruptionTest, TxnDbCrashDuringRecovery) {
  CloseDb();
  Options options;
+  options.level_compaction_dynamic_level_bytes = false;
  options.wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery;
  options.track_and_verify_wals_in_manifest =
      track_and_verify_wals_in_manifest_;
@@ -1551,6 +1567,7 @@ TEST_P(CrashDuringRecoveryWithCorruptionTest, TxnDbCrashDuringRecovery) {
 TEST_P(CrashDuringRecoveryWithCorruptionTest, CrashDuringRecoveryWithFlush) {
  CloseDb();
  Options options;
+  options.level_compaction_dynamic_level_bytes = false;
  options.wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery;
  options.avoid_flush_during_recovery = false;
  options.env = env_.get();
@@ -39,6 +39,7 @@ class CuckooTableDBTest : public testing::Test {

  Options CurrentOptions() {
    Options options;
+    options.level_compaction_dynamic_level_bytes = false;
    options.table_factory.reset(NewCuckooTableFactory());
    options.memtable_factory.reset(NewHashLinkListRepFactory(4, 0, 3, true));
    options.allow_mmap_reads = true;
@@ -1204,9 +1204,23 @@ TEST_F(DBBasicTest, DBClose) {
  delete db;
  ASSERT_EQ(env->GetCloseCount(), 2);

+  // close by WaitForCompact() with close_db option
+  options.create_if_missing = false;
+  s = DB::Open(options, dbname, &db);
+  ASSERT_OK(s);
+  ASSERT_TRUE(db != nullptr);
+  WaitForCompactOptions wait_for_compact_options = WaitForCompactOptions();
+  wait_for_compact_options.close_db = true;
+  s = db->WaitForCompact(wait_for_compact_options);
+  ASSERT_EQ(env->GetCloseCount(), 3);
+  // see TestLogger::CloseHelper()
+  ASSERT_EQ(s, Status::IOError());
+
+  delete db;
+  ASSERT_EQ(env->GetCloseCount(), 3);
+
  // Provide our own logger and ensure DB::Close() does not close it
  options.info_log.reset(new TestEnv::TestLogger(env));
-  options.create_if_missing = false;
  s = DB::Open(options, dbname, &db);
  ASSERT_OK(s);
  ASSERT_TRUE(db != nullptr);
@@ -1214,9 +1228,9 @@ TEST_F(DBBasicTest, DBClose) {
  s = db->Close();
  ASSERT_EQ(s, Status::OK());
  delete db;
-  ASSERT_EQ(env->GetCloseCount(), 2);
-  options.info_log.reset();
  ASSERT_EQ(env->GetCloseCount(), 3);
+  options.info_log.reset();
+  ASSERT_EQ(env->GetCloseCount(), 4);
 }

 TEST_F(DBBasicTest, DBCloseAllDirectoryFDs) {
@@ -1413,10 +1427,7 @@ TEST_P(DBMultiGetTestWithParam, MultiGetMultiCFMutex) {
  int retries = 0;
  bool last_try = false;
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::MultiGet::LastTry", [&](void* /*arg*/) {
-        last_try = true;
-        ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
-      });
+      "DBImpl::MultiGet::LastTry", [&](void* /*arg*/) { last_try = true; });
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "DBImpl::MultiGet::AfterRefSV", [&](void* /*arg*/) {
        if (last_try) {
@@ -1433,8 +1444,28 @@ TEST_P(DBMultiGetTestWithParam, MultiGetMultiCFMutex) {
          }
        }
      });
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({
+      {"DBImpl::MultiGet::AfterLastTryRefSV",
+       "DBMultiGetTestWithParam::MultiGetMultiCFMutex:BeforeCreateSV"},
+      {"DBMultiGetTestWithParam::MultiGetMultiCFMutex:AfterCreateSV",
+       "DBImpl::MultiGet::BeforeLastTryUnRefSV"},
+  });
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();

+  port::Thread create_sv_thread([this]() {
+    TEST_SYNC_POINT(
+        "DBMultiGetTestWithParam::MultiGetMultiCFMutex:BeforeCreateSV");
+    // Create a new SuperVersion for each column family after last_try
+    // of MultiGet ref SuperVersion and before unref it.
+    for (int i = 0; i < 8; ++i) {
+      ASSERT_OK(Put(i, "cf" + std::to_string(i) + "_key",
+                    "cf" + std::to_string(i) + "_val_after_last_try"));
+      ASSERT_OK(Flush(i));
+    }
+    TEST_SYNC_POINT(
+        "DBMultiGetTestWithParam::MultiGetMultiCFMutex:AfterCreateSV");
+  });
+
  std::vector<int> cfs;
  std::vector<std::string> keys;
  std::vector<std::string> values;
@@ -1446,6 +1477,7 @@ TEST_P(DBMultiGetTestWithParam, MultiGetMultiCFMutex) {

  values = MultiGet(cfs, keys, nullptr, std::get<0>(GetParam()),
                    std::get<1>(GetParam()));
+  create_sv_thread.join();
  ASSERT_TRUE(last_try);
  ASSERT_EQ(values.size(), 8);
  for (unsigned int j = 0; j < values.size(); ++j) {
@@ -1459,6 +1491,7 @@ TEST_P(DBMultiGetTestWithParam, MultiGetMultiCFMutex) {
            ->cfd();
    ASSERT_NE(cfd->TEST_GetLocalSV()->Get(), SuperVersion::kSVInUse);
  }
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }

 TEST_P(DBMultiGetTestWithParam, MultiGetMultiCFSnapshot) {
@@ -3621,10 +3654,12 @@ class DBBasicTestMultiGet : public DBTestBase {

    Status Insert(const Slice& key, Cache::ObjectPtr value,
                  const CacheItemHelper* helper, size_t charge,
-                  Handle** handle = nullptr,
-                  Priority priority = Priority::LOW) override {
+                  Handle** handle = nullptr, Priority priority = Priority::LOW,
+                  const Slice& compressed = Slice(),
+                  CompressionType type = kNoCompression) override {
      num_inserts_++;
-      return target_->Insert(key, value, helper, charge, handle, priority);
+      return target_->Insert(key, value, helper, charge, handle, priority,
+                             compressed, type);
    }

    Handle* Lookup(const Slice& key, const CacheItemHelper* helper,
@@ -294,7 +294,9 @@ class ReadOnlyCacheWrapper : public CacheWrapper {

  Status Insert(const Slice& /*key*/, Cache::ObjectPtr /*value*/,
                const CacheItemHelper* /*helper*/, size_t /*charge*/,
-                Handle** /*handle*/, Priority /*priority*/) override {
+                Handle** /*handle*/, Priority /*priority*/,
+                const Slice& /*compressed*/,
+                CompressionType /*type*/) override {
    return Status::NotSupported();
  }
 };
@@ -628,13 +630,15 @@ class MockCache : public LRUCache {

  Status Insert(const Slice& key, Cache::ObjectPtr value,
                const Cache::CacheItemHelper* helper, size_t charge,
-                Handle** handle, Priority priority) override {
+                Handle** handle, Priority priority, const Slice& compressed,
+                CompressionType type) override {
    if (priority == Priority::LOW) {
      low_pri_insert_count++;
    } else {
      high_pri_insert_count++;
    }
-    return LRUCache::Insert(key, value, helper, charge, handle, priority);
+    return LRUCache::Insert(key, value, helper, charge, handle, priority,
+                            compressed, type);
  }
 };

@@ -741,10 +745,15 @@ TEST_F(DBBlockCacheTest, AddRedundantStats) {
  int iterations_tested = 0;
  for (std::shared_ptr<Cache> base_cache :
       {NewLRUCache(capacity, num_shard_bits),
+        // FixedHyperClockCache
        HyperClockCacheOptions(
            capacity,
            BlockBasedTableOptions().block_size /*estimated_value_size*/,
            num_shard_bits)
+            .MakeSharedCache(),
+        // AutoHyperClockCache
+        HyperClockCacheOptions(capacity, 0 /*estimated_value_size*/,
+                               num_shard_bits)
            .MakeSharedCache()}) {
    if (!base_cache) {
      // Skip clock cache when not supported
@@ -654,7 +654,7 @@ TEST_P(DBBloomFilterTestWithParam, SkipFilterOnEssentiallyZeroBpk) {
    for (i = 0; i < maxKey; i++) {
      ASSERT_OK(Put(Key(i), Key(i)));
    }
-    Flush();
+    ASSERT_OK(Flush());
  };
  auto GetFn = [&]() {
    int i;
@@ -792,7 +792,7 @@ TEST_F(DBBloomFilterTest, BloomFilterRate) {
    }
    // Add a large key to make the file contain wide range
    ASSERT_OK(Put(1, Key(maxKey + 55555), Key(maxKey + 55555)));
-    Flush(1);
+    ASSERT_OK(Flush(1));

    // Check if they can be found
    for (int i = 0; i < maxKey; i++) {
@@ -1696,7 +1696,7 @@ TEST_F(DBBloomFilterTest, ContextCustomFilterPolicy) {
    table_options.format_version = 5;
    options.table_factory.reset(NewBlockBasedTableFactory(table_options));

-    TryReopen(options);
+    ASSERT_OK(TryReopen(options));
    CreateAndReopenWithCF({fifo ? "abe" : "bob"}, options);

    const int maxKey = 10000;
@@ -1705,7 +1705,7 @@ TEST_F(DBBloomFilterTest, ContextCustomFilterPolicy) {
    }
    // Add a large key to make the file contain wide range
    ASSERT_OK(Put(1, Key(maxKey + 55555), Key(maxKey + 55555)));
-    Flush(1);
+    ASSERT_OK(Flush(1));
    EXPECT_EQ(policy->DumpTestReport(),
              fifo ? "cf=abe,s=kCompactionStyleFIFO,n=7,l=0,b=0,r=kFlush\n"
                   : "cf=bob,s=kCompactionStyleLevel,n=7,l=0,b=0,r=kFlush\n");
@@ -1713,7 +1713,7 @@ TEST_F(DBBloomFilterTest, ContextCustomFilterPolicy) {
    for (int i = maxKey / 2; i < maxKey; i++) {
      ASSERT_OK(Put(1, Key(i), Key(i)));
    }
-    Flush(1);
+    ASSERT_OK(Flush(1));
    EXPECT_EQ(policy->DumpTestReport(),
              fifo ? "cf=abe,s=kCompactionStyleFIFO,n=7,l=0,b=0,r=kFlush\n"
                   : "cf=bob,s=kCompactionStyleLevel,n=7,l=0,b=0,r=kFlush\n");
@@ -1775,6 +1775,64 @@ TEST_F(DBBloomFilterTest, ContextCustomFilterPolicy) {
  }
 }

+TEST_F(DBBloomFilterTest, MutatingRibbonFilterPolicy) {
+  // Test that RibbonFilterPolicy has a mutable bloom_before_level fields that
+  // can be updated through SetOptions
+
+  Options options = CurrentOptions();
+  options.statistics = CreateDBStatistics();
+  auto& stats = *options.statistics;
+  BlockBasedTableOptions table_options;
+  // First config forces Bloom filter, to establish a baseline before
+  // SetOptions().
+  table_options.filter_policy.reset(NewRibbonFilterPolicy(10, INT_MAX));
+  double expected_bpk = 10.0;
+  // Other configs to try, with approx expected bits per key
+  std::vector<std::pair<std::string, double>> configs = {{"-1", 7.0},
+                                                         {"0", 10.0}};
+
+  table_options.cache_index_and_filter_blocks = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+
+  ASSERT_OK(TryReopen(options));
+
+  char v[] = "a";
+
+  for (;; ++(v[0])) {
+    const int maxKey = 8000;
+    for (int i = 0; i < maxKey; i++) {
+      ASSERT_OK(Put(Key(i), v));
+    }
+    ASSERT_OK(Flush());
+
+    for (int i = 0; i < maxKey; i++) {
+      ASSERT_EQ(Get(Key(i)), v);
+    }
+
+    uint64_t filter_bytes =
+        stats.getAndResetTickerCount(BLOCK_CACHE_FILTER_BYTES_INSERT);
+
+    EXPECT_NEAR(filter_bytes * 8.0 / maxKey, expected_bpk, 0.3);
+
+    if (configs.empty()) {
+      break;
+    }
+
+    ASSERT_OK(
+        db_->SetOptions({{"table_factory.filter_policy.bloom_before_level",
+                          configs.back().first}}));
+
+    // Ensure original object is mutated
+    std::string val;
+    ASSERT_OK(
+        table_options.filter_policy->GetOption({}, "bloom_before_level", &val));
+    ASSERT_EQ(configs.back().first, val);
+
+    expected_bpk = configs.back().second;
+    configs.pop_back();
+  }
+}
+
 class SliceTransformLimitedDomain : public SliceTransform {
  const char* Name() const override { return "SliceTransformLimitedDomain"; }

@@ -2261,7 +2319,7 @@ TEST_P(BloomStatsTestWithParam, BloomStatsTest) {
  ASSERT_EQ(0, get_perf_context()->bloom_sst_hit_count);
  ASSERT_EQ(0, get_perf_context()->bloom_sst_miss_count);

-  Flush();
+  ASSERT_OK(Flush());

  // sanity checks
  ASSERT_EQ(0, get_perf_context()->bloom_sst_hit_count);
@@ -2311,7 +2369,7 @@ TEST_P(BloomStatsTestWithParam, BloomStatsTestWithIter) {
  ASSERT_EQ(1, get_perf_context()->bloom_memtable_miss_count);
  ASSERT_EQ(2, get_perf_context()->bloom_memtable_hit_count);

-  Flush();
+  ASSERT_OK(Flush());

  iter.reset(dbfull()->NewIterator(ReadOptions()));

@@ -2379,7 +2437,7 @@ void PrefixScanInit(DBBloomFilterTest* dbtest) {
    snprintf(buf, sizeof(buf), "%02d______:end", i + 1);
    keystr = std::string(buf);
    ASSERT_OK(dbtest->Put(keystr, keystr));
-    dbtest->Flush();
+    ASSERT_OK(dbtest->Flush());
  }

  // GROUP 2
@@ -2390,7 +2448,7 @@ void PrefixScanInit(DBBloomFilterTest* dbtest) {
    snprintf(buf, sizeof(buf), "%02d______:end", small_range_sstfiles + i + 1);
    keystr = std::string(buf);
    ASSERT_OK(dbtest->Put(keystr, keystr));
-    dbtest->Flush();
+    ASSERT_OK(dbtest->Flush());
  }
 }
 }  // anonymous namespace
@@ -2614,7 +2672,7 @@ TEST_F(DBBloomFilterTest, OptimizeFiltersForHits) {
  compact_options.target_level = 7;
  ASSERT_OK(db_->CompactRange(compact_options, handles_[1], nullptr, nullptr));

-  ASSERT_EQ(trivial_move, 1);
+  ASSERT_GE(trivial_move, 1);
  ASSERT_EQ(non_trivial_move, 0);

  prev_cache_filter_hits = TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT);
@@ -2853,7 +2911,7 @@ TEST_F(DBBloomFilterTest, DynamicBloomFilterMultipleSST) {
    ASSERT_OK(Put("foo", "bar"));
    ASSERT_OK(Put("foq1", "bar1"));
    ASSERT_OK(Put("fpa", "0"));
-    dbfull()->Flush(FlushOptions());
+    ASSERT_OK(dbfull()->Flush(FlushOptions()));
    std::unique_ptr<Iterator> iter_old(db_->NewIterator(read_options));
    ASSERT_EQ(CountIter(iter_old, "foo"), 4);
    EXPECT_EQ(PopTicker(options, NON_LAST_LEVEL_SEEK_FILTERED), 0);
@@ -2981,7 +3039,7 @@ TEST_F(DBBloomFilterTest, DynamicBloomFilterNewColumnFamily) {
    ASSERT_OK(Put(2, "foo5", "bar5"));
    ASSERT_OK(Put(2, "foq6", "bar6"));
    ASSERT_OK(Put(2, "fpq7", "bar7"));
-    dbfull()->Flush(FlushOptions());
+    ASSERT_OK(dbfull()->Flush(FlushOptions()));
    {
      std::unique_ptr<Iterator> iter(
          db_->NewIterator(read_options, handles_[2]));
@@ -3031,17 +3089,17 @@ TEST_F(DBBloomFilterTest, DynamicBloomFilterOptions) {
    ASSERT_OK(Put("foo", "bar"));
    ASSERT_OK(Put("foo1", "bar1"));
    ASSERT_OK(Put("fpa", "0"));
-    dbfull()->Flush(FlushOptions());
+    ASSERT_OK(dbfull()->Flush(FlushOptions()));
    ASSERT_OK(Put("foo3", "bar3"));
    ASSERT_OK(Put("foo4", "bar4"));
    ASSERT_OK(Put("foo5", "bar5"));
    ASSERT_OK(Put("fpb", "1"));
-    dbfull()->Flush(FlushOptions());
+    ASSERT_OK(dbfull()->Flush(FlushOptions()));
    ASSERT_OK(Put("foo6", "bar6"));
    ASSERT_OK(Put("foo7", "bar7"));
    ASSERT_OK(Put("foo8", "bar8"));
    ASSERT_OK(Put("fpc", "2"));
-    dbfull()->Flush(FlushOptions());
+    ASSERT_OK(dbfull()->Flush(FlushOptions()));

    ReadOptions read_options;
    read_options.prefix_same_as_start = true;
@@ -166,9 +166,12 @@ class ChangeFilter : public CompactionFilter {
 class KeepFilterFactory : public CompactionFilterFactory {
 public:
  explicit KeepFilterFactory(bool check_context = false,
-                             bool check_context_cf_id = false)
+                             bool check_context_cf_id = false,
+                             bool check_context_input_table_properties = false)
      : check_context_(check_context),
        check_context_cf_id_(check_context_cf_id),
+        check_context_input_table_properties_(
+            check_context_input_table_properties),
        compaction_filter_created_(false) {}

  std::unique_ptr<CompactionFilter> CreateCompactionFilter(
@@ -176,6 +179,11 @@ class KeepFilterFactory : public CompactionFilterFactory {
    if (check_context_) {
      EXPECT_EQ(expect_full_compaction_.load(), context.is_full_compaction);
      EXPECT_EQ(expect_manual_compaction_.load(), context.is_manual_compaction);
+      EXPECT_EQ(expect_input_start_level_.load(), context.input_start_level);
+    }
+    if (check_context_input_table_properties_) {
+      EXPECT_TRUE(expect_input_table_properties_ ==
+                  context.input_table_properties);
    }
    if (check_context_cf_id_) {
      EXPECT_EQ(expect_cf_id_.load(), context.column_family_id);
@@ -189,9 +197,15 @@ class KeepFilterFactory : public CompactionFilterFactory {
  const char* Name() const override { return "KeepFilterFactory"; }
  bool check_context_;
  bool check_context_cf_id_;
+  // `check_context_input_table_properties_` can be true only when access to
+  // `expect_input_table_properties_` is syncronized since we can't have
+  // std::atomic<TablePropertiesCollection> unfortunately
+  bool check_context_input_table_properties_;
  std::atomic_bool expect_full_compaction_;
  std::atomic_bool expect_manual_compaction_;
  std::atomic<uint32_t> expect_cf_id_;
+  std::atomic<int> expect_input_start_level_;
+  TablePropertiesCollection expect_input_table_properties_;
  bool compaction_filter_created_;
 };

@@ -654,7 +668,9 @@ TEST_F(DBTestCompactionFilter, CompactionFilterWithMergeOperator) {
 }

 TEST_F(DBTestCompactionFilter, CompactionFilterContextManual) {
-  KeepFilterFactory* filter = new KeepFilterFactory(true, true);
+  KeepFilterFactory* filter = new KeepFilterFactory(
+      true /* check_context */, true /* check_context_cf_id */,
+      true /* check_context_input_table_properties */);

  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleUniversal;
@@ -662,8 +678,9 @@ TEST_F(DBTestCompactionFilter, CompactionFilterContextManual) {
  options.compression = kNoCompression;
  options.level0_file_num_compaction_trigger = 8;
  Reopen(options);
+  const int kNumFiles = 3;
  int num_keys_per_file = 400;
-  for (int j = 0; j < 3; j++) {
+  for (int j = 0; j < kNumFiles; j++) {
    // Write several keys.
    const std::string value(10, 'x');
    for (int i = 0; i < num_keys_per_file; i++) {
@@ -683,6 +700,11 @@ TEST_F(DBTestCompactionFilter, CompactionFilterContextManual) {
  filter->expect_manual_compaction_.store(true);
  filter->expect_full_compaction_.store(true);
  filter->expect_cf_id_.store(0);
+  filter->expect_input_start_level_.store(0);
+  ASSERT_OK(dbfull()->GetPropertiesOfAllTables(
+      &filter->expect_input_table_properties_));
+  ASSERT_TRUE(filter->expect_input_table_properties_.size() == kNumFiles);
+
  ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
  ASSERT_EQ(cfilter_count, 700);
  ASSERT_EQ(NumSortedRuns(0), 1);
@@ -29,35 +29,8 @@
 namespace ROCKSDB_NAMESPACE {

 Status DBImpl::FlushForGetLiveFiles() {
-  mutex_.AssertHeld();
-
-  // flush all dirty data to disk.
-  Status status;
-  if (immutable_db_options_.atomic_flush) {
-    mutex_.Unlock();
-    status = AtomicFlushMemTables(FlushOptions(), FlushReason::kGetLiveFiles);
-    if (status.IsColumnFamilyDropped()) {
-      status = Status::OK();
-    }
-    mutex_.Lock();
-  } else {
-    for (auto cfd : versions_->GetRefedColumnFamilySet()) {
-      if (cfd->IsDropped()) {
-        continue;
-      }
-      mutex_.Unlock();
-      status = FlushMemTable(cfd, FlushOptions(), FlushReason::kGetLiveFiles);
-      TEST_SYNC_POINT("DBImpl::GetLiveFiles:1");
-      TEST_SYNC_POINT("DBImpl::GetLiveFiles:2");
-      mutex_.Lock();
-      if (!status.ok() && !status.IsColumnFamilyDropped()) {
-        break;
-      } else if (status.IsColumnFamilyDropped()) {
-        status = Status::OK();
-      }
-    }
-  }
-  return status;
+  return DBImpl::FlushAllColumnFamilies(FlushOptions(),
+                                        FlushReason::kGetLiveFiles);
 }

 Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
@@ -222,7 +222,7 @@ TEST_F(DBFlushTest, CloseDBWhenFlushInLowPri) {
  sleeping_task_low.WaitUntilDone();
  ASSERT_EQ(0, num_flushes);

-  TryReopenWithColumnFamilies({"default", "cf1", "cf2"}, options);
+  ASSERT_OK(TryReopenWithColumnFamilies({"default", "cf1", "cf2"}, options));
  ASSERT_OK(Put(0, "key3", DummyString(8192)));
  ASSERT_OK(Flush(0));
  ASSERT_EQ(1, num_flushes);
@@ -3193,6 +3193,279 @@ INSTANTIATE_TEST_CASE_P(DBFlushDirectIOTest, DBFlushDirectIOTest,

 INSTANTIATE_TEST_CASE_P(DBAtomicFlushTest, DBAtomicFlushTest, testing::Bool());

+TEST_F(DBFlushTest, NonAtomicFlushRollbackPendingFlushes) {
+  // Fix a bug in when atomic_flush=false.
+  // The bug can happen as follows:
+  // Start Flush0 for memtable M0 to SST0
+  // Start Flush1 for memtable M1 to SST1
+  // Flush1 returns OK, but don't install to MANIFEST and let whoever flushes
+  // M0 to take care of it
+  // Flush0 finishes with a retryable IOError
+  //   - It rollbacks M0, (incorrectly) not M1
+  //   - Deletes SST1 and SST2
+  //
+  // Auto-recovery will start Flush2 for M0, it does not pick up M1 since it
+  // thinks that M1 is flushed
+  // Flush2 writes SST3 and finishes OK, tries to install SST3 and SST2
+  // Error opening SST2 since it's already deleted
+  //
+  // The fix is to let Flush0 also rollback M1.
+  Options opts = CurrentOptions();
+  opts.atomic_flush = false;
+  opts.memtable_factory.reset(test::NewSpecialSkipListFactory(1));
+  opts.max_write_buffer_number = 64;
+  opts.max_background_flushes = 4;
+  env_->SetBackgroundThreads(4, Env::HIGH);
+  DestroyAndReopen(opts);
+  std::atomic_int flush_count = 0;
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->SetCallBack(
+      "FlushJob::WriteLevel0Table:s", [&](void* s_ptr) {
+        int c = flush_count.fetch_add(1);
+        if (c == 0) {
+          Status* s = (Status*)(s_ptr);
+          IOStatus io_error = IOStatus::IOError("injected foobar");
+          io_error.SetRetryable(true);
+          *s = io_error;
+          TEST_SYNC_POINT("Let mem1 flush start");
+          TEST_SYNC_POINT("Wait for mem1 flush to finish");
+        }
+      });
+  SyncPoint::GetInstance()->LoadDependency(
+      {{"Let mem1 flush start", "Mem1 flush starts"},
+       {"DBImpl::BGWorkFlush:done", "Wait for mem1 flush to finish"},
+       {"RecoverFromRetryableBGIOError:RecoverSuccess",
+        "Wait for error recover"}});
+  // Need first flush to wait for the second flush to finish
+  SyncPoint::GetInstance()->EnableProcessing();
+  ASSERT_OK(Put(Key(1), "val1"));
+  // trigger bg flush mem0
+  ASSERT_OK(Put(Key(2), "val2"));
+  TEST_SYNC_POINT("Mem1 flush starts");
+  // trigger bg flush mem1
+  ASSERT_OK(Put(Key(3), "val3"));
+
+  TEST_SYNC_POINT("Wait for error recover");
+  ASSERT_EQ(1, NumTableFilesAtLevel(0));
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+  SyncPoint::GetInstance()->DisableProcessing();
+}
+
+TEST_F(DBFlushTest, AbortNonAtomicFlushWhenBGError) {
+  // Fix a bug in when atomic_flush=false.
+  // The bug can happen as follows:
+  // Start Flush0 for memtable M0 to SST0
+  // Start Flush1 for memtable M1 to SST1
+  // Flush1 returns OK, but doesn't install output MANIFEST and let whoever
+  // flushes M0 to take care of it
+  // Start Flush2 for memtable M2 to SST2
+  // Flush0 finishes with a retryable IOError
+  //   - It rollbacks M0 AND M1
+  //   - Deletes SST1 and SST2
+  // Flush2 finishes, does not rollback M2,
+  //  - releases the pending file number that keeps SST2 alive
+  //  - deletes SST2
+  //
+  // Then auto-recovery starts, error opening SST2 when try to install
+  // flush result
+  //
+  // The fix is to let Flush2 rollback M2 if it finds that
+  // there is a background error.
+  Options opts = CurrentOptions();
+  opts.atomic_flush = false;
+  opts.memtable_factory.reset(test::NewSpecialSkipListFactory(1));
+  opts.max_write_buffer_number = 64;
+  opts.max_background_flushes = 4;
+  env_->SetBackgroundThreads(4, Env::HIGH);
+  DestroyAndReopen(opts);
+  std::atomic_int flush_count = 0;
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->SetCallBack(
+      "FlushJob::WriteLevel0Table:s", [&](void* s_ptr) {
+        int c = flush_count.fetch_add(1);
+        if (c == 0) {
+          Status* s = (Status*)(s_ptr);
+          IOStatus io_error = IOStatus::IOError("injected foobar");
+          io_error.SetRetryable(true);
+          *s = io_error;
+          TEST_SYNC_POINT("Let mem1 flush start");
+          TEST_SYNC_POINT("Wait for mem1 flush to finish");
+
+          TEST_SYNC_POINT("Let mem2 flush start");
+          TEST_SYNC_POINT("Wait for mem2 to start writing table");
+        }
+      });
+
+  SyncPoint::GetInstance()->SetCallBack(
+      "FlushJob::WriteLevel0Table", [&](void* mems) {
+        autovector<MemTable*>* mems_ptr = (autovector<MemTable*>*)mems;
+        if ((*mems_ptr)[0]->GetID() == 3) {
+          TEST_SYNC_POINT("Mem2 flush starts writing table");
+          TEST_SYNC_POINT("Mem2 flush waits until rollback");
+        }
+      });
+  SyncPoint::GetInstance()->LoadDependency(
+      {{"Let mem1 flush start", "Mem1 flush starts"},
+       {"DBImpl::BGWorkFlush:done", "Wait for mem1 flush to finish"},
+       {"Let mem2 flush start", "Mem2 flush starts"},
+       {"Mem2 flush starts writing table",
+        "Wait for mem2 to start writing table"},
+       {"RollbackMemtableFlush", "Mem2 flush waits until rollback"},
+       {"RecoverFromRetryableBGIOError:RecoverSuccess",
+        "Wait for error recover"}});
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  ASSERT_OK(Put(Key(1), "val1"));
+  // trigger bg flush mem0
+  ASSERT_OK(Put(Key(2), "val2"));
+  TEST_SYNC_POINT("Mem1 flush starts");
+  // trigger bg flush mem1
+  ASSERT_OK(Put(Key(3), "val3"));
+
+  TEST_SYNC_POINT("Mem2 flush starts");
+  ASSERT_OK(Put(Key(4), "val4"));
+
+  TEST_SYNC_POINT("Wait for error recover");
+  // Recovery flush writes 3 memtables together into 1 file.
+  ASSERT_EQ(1, NumTableFilesAtLevel(0));
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+  SyncPoint::GetInstance()->DisableProcessing();
+}
+
+TEST_F(DBFlushTest, NonAtomicNormalFlushAbortWhenBGError) {
+  Options opts = CurrentOptions();
+  opts.atomic_flush = false;
+  opts.memtable_factory.reset(test::NewSpecialSkipListFactory(1));
+  opts.max_write_buffer_number = 64;
+  opts.max_background_flushes = 1;
+  env_->SetBackgroundThreads(2, Env::HIGH);
+  DestroyAndReopen(opts);
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+  SyncPoint::GetInstance()->DisableProcessing();
+  std::atomic_int flush_write_table_count = 0;
+  SyncPoint::GetInstance()->SetCallBack(
+      "FlushJob::WriteLevel0Table:s", [&](void* s_ptr) {
+        int c = flush_write_table_count.fetch_add(1);
+        if (c == 0) {
+          Status* s = (Status*)(s_ptr);
+          IOStatus io_error = IOStatus::IOError("injected foobar");
+          io_error.SetRetryable(true);
+          *s = io_error;
+        }
+      });
+
+  SyncPoint::GetInstance()->EnableProcessing();
+  SyncPoint::GetInstance()->LoadDependency(
+      {{"Let error recovery start",
+        "RecoverFromRetryableBGIOError:BeforeStart"},
+       {"RecoverFromRetryableBGIOError:RecoverSuccess",
+        "Wait for error recover"}});
+
+  ASSERT_OK(Put(Key(1), "val1"));
+  // trigger bg flush0 for mem0
+  ASSERT_OK(Put(Key(2), "val2"));
+  // Not checking status since this wait can finish before flush starts.
+  dbfull()->TEST_WaitForFlushMemTable().PermitUncheckedError();
+
+  // trigger bg flush1 for mem1, should see bg error and abort
+  // before picking a memtable to flush
+  ASSERT_OK(Put(Key(3), "val3"));
+  ASSERT_NOK(dbfull()->TEST_WaitForFlushMemTable());
+  ASSERT_EQ(0, NumTableFilesAtLevel(0));
+
+  TEST_SYNC_POINT("Let error recovery start");
+  TEST_SYNC_POINT("Wait for error recover");
+  // Recovery flush writes 2 memtables together into 1 file.
+  ASSERT_EQ(1, NumTableFilesAtLevel(0));
+  // 1 for flush 0 and 1 for recovery flush
+  ASSERT_EQ(2, flush_write_table_count);
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+  SyncPoint::GetInstance()->DisableProcessing();
+}
+
+TEST_F(DBFlushTest, DBStuckAfterAtomicFlushError) {
+  // Test for a bug with atomic flush where DB can become stuck
+  // after a flush error. A repro timeline:
+  //
+  // Start Flush0 for mem0
+  // Start Flush1 for mem1
+  // Now Flush1 will wait for Flush0 to install mem0
+  // Flush0 finishes with retryable IOError, rollbacks mem0
+  // Resume starts and waits for background job to finish, i.e., Flush1
+  // Fill memtable again, trigger Flush2 for mem0
+  // Flush2 will get error status, and not rollback mem0, see code in
+  // https://github.com/facebook/rocksdb/blob/b927ba5936216861c2c35ab68f50ba4a78e65747/db/db_impl/db_impl_compaction_flush.cc#L725
+  //
+  // DB is stuck since mem0 can never be picked now
+  //
+  // The fix is to rollback mem0 in Flush2, and let Flush1 also abort upon
+  // background error besides waiting for older memtables to be installed.
+  // The recovery flush in this case should pick up all memtables
+  // and write them to a single L0 file.
+  Options opts = CurrentOptions();
+  opts.atomic_flush = true;
+  opts.memtable_factory.reset(test::NewSpecialSkipListFactory(1));
+  opts.max_write_buffer_number = 64;
+  opts.max_background_flushes = 4;
+  env_->SetBackgroundThreads(4, Env::HIGH);
+  DestroyAndReopen(opts);
+
+  std::atomic_int flush_count = 0;
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->SetCallBack(
+      "FlushJob::WriteLevel0Table:s", [&](void* s_ptr) {
+        int c = flush_count.fetch_add(1);
+        if (c == 0) {
+          Status* s = (Status*)(s_ptr);
+          IOStatus io_error = IOStatus::IOError("injected foobar");
+          io_error.SetRetryable(true);
+          *s = io_error;
+          TEST_SYNC_POINT("Let flush for mem1 start");
+          // Wait for Flush1 to start waiting to install flush result
+          TEST_SYNC_POINT("Wait for flush for mem1");
+        }
+      });
+  SyncPoint::GetInstance()->LoadDependency(
+      {{"Let flush for mem1 start", "Flush for mem1"},
+       {"DBImpl::AtomicFlushMemTablesToOutputFiles:WaitCV",
+        "Wait for flush for mem1"},
+       {"RecoverFromRetryableBGIOError:BeforeStart",
+        "Wait for resume to start"},
+       {"Recovery should continue here",
+        "RecoverFromRetryableBGIOError:BeforeStart2"},
+       {"RecoverFromRetryableBGIOError:RecoverSuccess",
+        "Wait for error recover"}});
+  SyncPoint::GetInstance()->EnableProcessing();
+  ASSERT_OK(Put(Key(1), "val1"));
+  // trigger Flush0 for mem0
+  ASSERT_OK(Put(Key(2), "val2"));
+
+  // trigger Flush1 for mem1
+  TEST_SYNC_POINT("Flush for mem1");
+  ASSERT_OK(Put(Key(3), "val3"));
+
+  // Wait until resume started to schedule another flush
+  TEST_SYNC_POINT("Wait for resume to start");
+  // This flush should not be scheduled due to bg error
+  ASSERT_OK(Put(Key(4), "val4"));
+
+  // TEST_WaitForBackgroundWork() returns background error
+  // after all background work is done.
+  ASSERT_NOK(dbfull()->TEST_WaitForBackgroundWork());
+  // Flush should abort and not writing any table
+  ASSERT_EQ(0, NumTableFilesAtLevel(0));
+
+  // Wait until this flush is done.
+  TEST_SYNC_POINT("Recovery should continue here");
+  TEST_SYNC_POINT("Wait for error recover");
+  // error recovery can schedule new flushes, but should not
+  // encounter error
+  ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork());
+  ASSERT_EQ(1, NumTableFilesAtLevel(0));
+}
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -43,21 +43,34 @@ Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*,
             /*timestamp*/ nullptr);
 }

-Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*,
-                            const Slice& key, PinnableSlice* value,
-                            std::string* timestamp) {
-  if (options.io_activity != Env::IOActivity::kUnknown) {
+Status CompactedDBImpl::Get(const ReadOptions& _read_options,
+                            ColumnFamilyHandle*, const Slice& key,
+                            PinnableSlice* value, std::string* timestamp) {
+  if (_read_options.io_activity != Env::IOActivity::kUnknown &&
+      _read_options.io_activity != Env::IOActivity::kGet) {
    return Status::InvalidArgument(
-        "Cannot call Get with `ReadOptions::io_activity` != "
-        "`Env::IOActivity::kUnknown`");
+        "Can only call Get with `ReadOptions::io_activity` is "
+        "`Env::IOActivity::kUnknown` or `Env::IOActivity::kGet`");
  }
+  ReadOptions read_options(_read_options);
+  if (read_options.io_activity == Env::IOActivity::kUnknown) {
+    read_options.io_activity = Env::IOActivity::kGet;
+  }
+
  assert(user_comparator_);
-  if (options.timestamp) {
-    const Status s = FailIfTsMismatchCf(
-        DefaultColumnFamily(), *(options.timestamp), /*ts_for_read=*/true);
+  if (read_options.timestamp) {
+    Status s =
+        FailIfTsMismatchCf(DefaultColumnFamily(), *(read_options.timestamp));
    if (!s.ok()) {
      return s;
    }
+    if (read_options.timestamp->size() > 0) {
+      s = FailIfReadCollapsedHistory(cfd_, cfd_->GetSuperVersion(),
+                                     *(read_options.timestamp));
+      if (!s.ok()) {
+        return s;
+      }
+    }
  } else {
    const Status s = FailIfCfHasTs(DefaultColumnFamily());
    if (!s.ok()) {
@@ -74,7 +87,7 @@ Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*,
  GetWithTimestampReadCallback read_cb(kMaxSequenceNumber);
  std::string* ts =
      user_comparator_->timestamp_size() > 0 ? timestamp : nullptr;
-  LookupKey lkey(key, kMaxSequenceNumber, options.timestamp);
+  LookupKey lkey(key, kMaxSequenceNumber, read_options.timestamp);
  GetContext get_context(user_comparator_, nullptr, nullptr, nullptr,
                         GetContext::kNotFound, lkey.user_key(), value,
                         /*columns=*/nullptr, ts, nullptr, nullptr, true,
@@ -88,8 +101,8 @@ Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*,
          /*b_has_ts=*/false) < 0) {
    return Status::NotFound();
  }
-  Status s = f.fd.table_reader->Get(options, lkey.internal_key(), &get_context,
-                                    nullptr);
+  Status s = f.fd.table_reader->Get(read_options, lkey.internal_key(),
+                                    &get_context, nullptr);
  if (!s.ok() && !s.IsNotFound()) {
    return s;
  }
@@ -106,18 +119,37 @@ std::vector<Status> CompactedDBImpl::MultiGet(
 }

 std::vector<Status> CompactedDBImpl::MultiGet(
-    const ReadOptions& options, const std::vector<ColumnFamilyHandle*>&,
+    const ReadOptions& _read_options, const std::vector<ColumnFamilyHandle*>&,
    const std::vector<Slice>& keys, std::vector<std::string>* values,
    std::vector<std::string>* timestamps) {
  assert(user_comparator_);
  size_t num_keys = keys.size();
+  if (_read_options.io_activity != Env::IOActivity::kUnknown &&
+      _read_options.io_activity != Env::IOActivity::kMultiGet) {
+    Status s = Status::InvalidArgument(
+        "Can only call MultiGet with `ReadOptions::io_activity` is "
+        "`Env::IOActivity::kUnknown` or `Env::IOActivity::kMultiGet`");
+    return std::vector<Status>(num_keys, s);
+  }

-  if (options.timestamp) {
-    Status s = FailIfTsMismatchCf(DefaultColumnFamily(), *(options.timestamp),
-                                  /*ts_for_read=*/true);
+  ReadOptions read_options(_read_options);
+  if (read_options.io_activity == Env::IOActivity::kUnknown) {
+    read_options.io_activity = Env::IOActivity::kMultiGet;
+  }
+
+  if (read_options.timestamp) {
+    Status s =
+        FailIfTsMismatchCf(DefaultColumnFamily(), *(read_options.timestamp));
    if (!s.ok()) {
      return std::vector<Status>(num_keys, s);
    }
+    if (read_options.timestamp->size() > 0) {
+      s = FailIfReadCollapsedHistory(cfd_, cfd_->GetSuperVersion(),
+                                     *(read_options.timestamp));
+      if (!s.ok()) {
+        return std::vector<Status>(num_keys, s);
+      }
+    }
  } else {
    Status s = FailIfCfHasTs(DefaultColumnFamily());
    if (!s.ok()) {
@@ -136,7 +168,7 @@ std::vector<Status> CompactedDBImpl::MultiGet(
  GetWithTimestampReadCallback read_cb(kMaxSequenceNumber);
  autovector<TableReader*, 16> reader_list;
  for (const auto& key : keys) {
-    LookupKey lkey(key, kMaxSequenceNumber, options.timestamp);
+    LookupKey lkey(key, kMaxSequenceNumber, read_options.timestamp);
    const FdWithKeyRange& f = files_.files[FindFile(lkey.user_key())];
    if (user_comparator_->CompareWithoutTimestamp(
            key, /*a_has_ts=*/false,
@@ -159,14 +191,15 @@ std::vector<Status> CompactedDBImpl::MultiGet(
    if (r != nullptr) {
      PinnableSlice pinnable_val;
      std::string& value = (*values)[idx];
-      LookupKey lkey(keys[idx], kMaxSequenceNumber, options.timestamp);
+      LookupKey lkey(keys[idx], kMaxSequenceNumber, read_options.timestamp);
      std::string* timestamp = timestamps ? &(*timestamps)[idx] : nullptr;
      GetContext get_context(
          user_comparator_, nullptr, nullptr, nullptr, GetContext::kNotFound,
          lkey.user_key(), &pinnable_val, /*columns=*/nullptr,
          user_comparator_->timestamp_size() > 0 ? timestamp : nullptr, nullptr,
          nullptr, true, nullptr, nullptr, nullptr, nullptr, &read_cb);
-      Status s = r->Get(options, lkey.internal_key(), &get_context, nullptr);
+      Status s =
+          r->Get(read_options, lkey.internal_key(), &get_context, nullptr);
      assert(static_cast<size_t>(idx) < statuses.size());
      if (!s.ok() && !s.IsNotFound()) {
        statuses[idx] = s;
@@ -30,9 +30,9 @@ class CompactedDBImpl : public DBImpl {
                     ColumnFamilyHandle* column_family, const Slice& key,
                     PinnableSlice* value) override;

-  Status Get(const ReadOptions& options, ColumnFamilyHandle* column_family,
-             const Slice& key, PinnableSlice* value,
-             std::string* timestamp) override;
+  Status Get(const ReadOptions& _read_options,
+             ColumnFamilyHandle* column_family, const Slice& key,
+             PinnableSlice* value, std::string* timestamp) override;

  using DB::MultiGet;
  // Note that CompactedDBImpl::MultiGet is not the optimized version of
@@ -43,7 +43,7 @@ class CompactedDBImpl : public DBImpl {
      const std::vector<Slice>& keys,
      std::vector<std::string>* values) override;

-  std::vector<Status> MultiGet(const ReadOptions& options,
+  std::vector<Status> MultiGet(const ReadOptions& _read_options,
                               const std::vector<ColumnFamilyHandle*>&,
                               const std::vector<Slice>& keys,
                               std::vector<std::string>* values,
@@ -118,12 +118,13 @@ class CompactedDBImpl : public DBImpl {
      const IngestExternalFileOptions& /*ingestion_options*/) override {
    return Status::NotSupported("Not supported in compacted db mode.");
  }
+
  using DB::CreateColumnFamilyWithImport;
  virtual Status CreateColumnFamilyWithImport(
      const ColumnFamilyOptions& /*options*/,
      const std::string& /*column_family_name*/,
      const ImportColumnFamilyOptions& /*import_options*/,
-      const ExportImportFilesMetaData& /*metadata*/,
+      const std::vector<const ExportImportFilesMetaData*>& /*metadatas*/,
      ColumnFamilyHandle** /*handle*/) override {
    return Status::NotSupported("Not supported in compacted db mode.");
  }
@@ -234,7 +234,7 @@ class DBImpl : public DB {
  virtual Status Get(const ReadOptions& options,
                     ColumnFamilyHandle* column_family, const Slice& key,
                     PinnableSlice* value) override;
-  virtual Status Get(const ReadOptions& options,
+  virtual Status Get(const ReadOptions& _read_options,
                     ColumnFamilyHandle* column_family, const Slice& key,
                     PinnableSlice* value, std::string* timestamp) override;

@@ -265,7 +265,7 @@ class DBImpl : public DB {
      const std::vector<Slice>& keys,
      std::vector<std::string>* values) override;
  virtual std::vector<Status> MultiGet(
-      const ReadOptions& options,
+      const ReadOptions& _read_options,
      const std::vector<ColumnFamilyHandle*>& column_family,
      const std::vector<Slice>& keys, std::vector<std::string>* values,
      std::vector<std::string>* timestamps) override;
@@ -280,8 +280,9 @@ class DBImpl : public DB {
  void MultiGet(const ReadOptions& options, ColumnFamilyHandle* column_family,
                const size_t num_keys, const Slice* keys, PinnableSlice* values,
                Status* statuses, const bool sorted_input = false) override;
-  void MultiGet(const ReadOptions& options, ColumnFamilyHandle* column_family,
-                const size_t num_keys, const Slice* keys, PinnableSlice* values,
+  void MultiGet(const ReadOptions& _read_options,
+                ColumnFamilyHandle* column_family, const size_t num_keys,
+                const Slice* keys, PinnableSlice* values,
                std::string* timestamps, Status* statuses,
                const bool sorted_input = false) override;

@@ -289,13 +290,13 @@ class DBImpl : public DB {
                ColumnFamilyHandle** column_families, const Slice* keys,
                PinnableSlice* values, Status* statuses,
                const bool sorted_input = false) override;
-  void MultiGet(const ReadOptions& options, const size_t num_keys,
+  void MultiGet(const ReadOptions& _read_options, const size_t num_keys,
                ColumnFamilyHandle** column_families, const Slice* keys,
                PinnableSlice* values, std::string* timestamps,
                Status* statuses, const bool sorted_input = false) override;

  void MultiGetWithCallback(
-      const ReadOptions& options, ColumnFamilyHandle* column_family,
+      const ReadOptions& _read_options, ColumnFamilyHandle* column_family,
      ReadCallback* callback,
      autovector<KeyContext*, MultiGetContext::MAX_BATCH_SIZE>* sorted_keys);

@@ -336,10 +337,10 @@ class DBImpl : public DB {
                           bool* value_found = nullptr) override;

  using DB::NewIterator;
-  virtual Iterator* NewIterator(const ReadOptions& options,
+  virtual Iterator* NewIterator(const ReadOptions& _read_options,
                                ColumnFamilyHandle* column_family) override;
  virtual Status NewIterators(
-      const ReadOptions& options,
+      const ReadOptions& _read_options,
      const std::vector<ColumnFamilyHandle*>& column_families,
      std::vector<Iterator*>* iterators) override;

@@ -531,7 +532,7 @@ class DBImpl : public DB {
  virtual Status CreateColumnFamilyWithImport(
      const ColumnFamilyOptions& options, const std::string& column_family_name,
      const ImportColumnFamilyOptions& import_options,
-      const ExportImportFilesMetaData& metadata,
+      const std::vector<const ExportImportFilesMetaData*>& metadatas,
      ColumnFamilyHandle** handle) override;

  using DB::ClipColumnFamily;
@@ -627,6 +628,14 @@ class DBImpl : public DB {
    int* number_of_operands = nullptr;
  };

+  Status GetImpl(const ReadOptions& read_options,
+                 ColumnFamilyHandle* column_family, const Slice& key,
+                 PinnableSlice* value);
+
+  Status GetImpl(const ReadOptions& read_options,
+                 ColumnFamilyHandle* column_family, const Slice& key,
+                 PinnableSlice* value, std::string* timestamp);
+
  // Function that Get and KeyMayExist call with no_io true or false
  // Note: 'value_found' from KeyMayExist propagates here
  // This function is also called by GetMergeOperands
@@ -634,12 +643,12 @@ class DBImpl : public DB {
  // get_impl_options.key via get_impl_options.value
  // If get_impl_options.get_value = false get merge operands associated with
  // get_impl_options.key via get_impl_options.merge_operands
-  Status GetImpl(const ReadOptions& options, const Slice& key,
-                 GetImplOptions& get_impl_options);
+  virtual Status GetImpl(const ReadOptions& options, const Slice& key,
+                         GetImplOptions& get_impl_options);

  // If `snapshot` == kMaxSequenceNumber, set a recent one inside the file.
  ArenaWrappedDBIter* NewIteratorImpl(const ReadOptions& options,
-                                      ColumnFamilyData* cfd,
+                                      ColumnFamilyData* cfd, SuperVersion* sv,
                                      SequenceNumber snapshot,
                                      ReadCallback* read_callback,
                                      bool expose_blob_index = false,
@@ -811,6 +820,8 @@ class DBImpl : public DB {
  // being deleted.
  uint64_t MinObsoleteSstNumberToKeep();

+  uint64_t GetObsoleteSstFilesSize();
+
  // Returns the list of live files in 'live' and the list
  // of all files in the filesystem in 'candidate_files'.
  // If force == false and the last call was less than
@@ -1058,16 +1069,8 @@ class DBImpl : public DB {

  VersionSet* GetVersionSet() const { return versions_.get(); }

-  // Wait for all flush and compactions jobs to finish. Jobs to wait include the
-  // unscheduled (queued, but not scheduled yet). If the db is shutting down,
-  // Status::ShutdownInProgress will be returned. If PauseBackgroundWork() was
-  // called prior to this, this may potentially wait for unscheduled jobs
-  // indefinitely. abort_on_pause can be set to true to abort, and
-  // Status::Aborted will be returned immediately. This may also never return if
-  // there's sufficient ongoing writes that keeps flush and compaction going
-  // without stopping. The user would have to cease all the writes to DB to make
-  // this eventually return in a stable state.
-  Status WaitForCompact(bool abort_on_pause = false);
+  Status WaitForCompact(
+      const WaitForCompactOptions& wait_for_compact_options) override;

 #ifndef NDEBUG
  // Compact any files in the named level that overlap [*begin, *end]
@@ -1106,8 +1109,9 @@ class DBImpl : public DB {
  // Wait for memtable compaction
  Status TEST_WaitForFlushMemTable(ColumnFamilyHandle* column_family = nullptr);

-  // Wait for any compaction
-  Status TEST_WaitForCompact(bool abort_on_pause = false);
+  Status TEST_WaitForCompact();
+  Status TEST_WaitForCompact(
+      const WaitForCompactOptions& wait_for_compact_options);

  // Wait for any background purge
  Status TEST_WaitForPurge();
@@ -1345,6 +1349,11 @@ class DBImpl : public DB {

  std::atomic<bool> shutting_down_;

+  // No new background jobs can be queued if true. This is used to prevent new
+  // background jobs from being queued after WaitForCompact() completes waiting
+  // all background jobs then attempts to close when close_db_ option is true.
+  bool reject_new_background_jobs_;
+
  // RecoveryContext struct stores the context about version edits along
  // with corresponding column_family_data and column_family_options.
  class RecoveryContext {
@@ -1374,8 +1383,10 @@ class DBImpl : public DB {
    autovector<ColumnFamilyData*> cfds_;
    autovector<const MutableCFOptions*> mutable_cf_opts_;
    autovector<autovector<VersionEdit*>> edit_lists_;
-    // files_to_delete_ contains sst files
-    std::unordered_set<std::string> files_to_delete_;
+    // Stale SST files to delete found upon recovery. This stores a mapping from
+    // such a file's absolute path to its parent directory.
+    std::unordered_map<std::string, std::string> files_to_delete_;
+    bool is_new_db_ = false;
  };

  // Except in DB::Open(), WriteOptionsFile can only be called when:
@@ -1417,6 +1428,9 @@ class DBImpl : public DB {
  void NotifyOnExternalFileIngested(
      ColumnFamilyData* cfd, const ExternalSstFileIngestionJob& ingestion_job);

+  Status FlushAllColumnFamilies(const FlushOptions& flush_options,
+                                FlushReason flush_reason);
+
  virtual Status FlushForGetLiveFiles();

  void NewThreadStatusCfInfo(ColumnFamilyData* cfd) const;
@@ -1531,8 +1545,18 @@ class DBImpl : public DB {
  void SetDbSessionId();

  Status FailIfCfHasTs(const ColumnFamilyHandle* column_family) const;
-  Status FailIfTsMismatchCf(ColumnFamilyHandle* column_family, const Slice& ts,
-                            bool ts_for_read) const;
+  Status FailIfTsMismatchCf(ColumnFamilyHandle* column_family,
+                            const Slice& ts) const;
+
+  // Check that the read timestamp `ts` is at or above the `full_history_ts_low`
+  // timestamp in a `SuperVersion`. It's necessary to do this check after
+  // grabbing the SuperVersion. If the check passed, the referenced SuperVersion
+  // this read holds on to can ensure the read won't be affected if
+  // `full_history_ts_low` is increased concurrently, and this achieves that
+  // without explicitly locking by piggybacking the SuperVersion.
+  Status FailIfReadCollapsedHistory(const ColumnFamilyData* cfd,
+                                    const SuperVersion* sv,
+                                    const Slice& ts) const;

  // recovery_ctx stores the context about version edits and
  // LogAndApplyForRecovery persist all those edits to new Manifest after
@@ -1568,12 +1592,14 @@ class DBImpl : public DB {
  friend class ForwardIterator;
  friend struct SuperVersion;
  friend class CompactedDBImpl;
+#ifndef NDEBUG
  friend class DBTest_ConcurrentFlushWAL_Test;
  friend class DBTest_MixedSlowdownOptionsStop_Test;
  friend class DBCompactionTest_CompactBottomLevelFilesWithDeletions_Test;
  friend class DBCompactionTest_CompactionDuringShutdown_Test;
+  friend class DBCompactionTest_DelayCompactBottomLevelFilesWithDeletions_Test;
+  friend class DBCompactionTest_DisableCompactBottomLevelFiles_Test;
  friend class StatsHistoryTest_PersistentStatsCreateColumnFamilies_Test;
-#ifndef NDEBUG
  friend class DBTest2_ReadCallbackTest_Test;
  friend class WriteCallbackPTest_WriteWithCallbackTest_Test;
  friend class XFTransactionWriteHandler;
@@ -1828,7 +1854,8 @@ class DBImpl : public DB {
  void ReleaseFileNumberFromPendingOutputs(
      std::unique_ptr<std::list<uint64_t>::iterator>& v);

-  IOStatus SyncClosedLogs(JobContext* job_context, VersionEdit* synced_wals);
+  IOStatus SyncClosedLogs(JobContext* job_context, VersionEdit* synced_wals,
+                          bool error_recovery_in_prog);

  // Flush the in-memory write buffer to storage.  Switches to a new
  // log-file/memtable and writes a new descriptor iff successful. Then
@@ -2065,6 +2092,10 @@ class DBImpl : public DB {
    // flush is considered complete.
    std::unordered_map<ColumnFamilyData*, uint64_t>
        cfd_to_max_mem_id_to_persist;
+
+#ifndef NDEBUG
+    int reschedule_count = 1;
+#endif /* !NDEBUG */
  };

  void GenerateFlushRequest(const autovector<ColumnFamilyData*>& cfds,
@@ -2093,6 +2124,7 @@ class DBImpl : public DB {
                              Env::Priority thread_pri);
  Status BackgroundFlush(bool* madeProgress, JobContext* job_context,
                         LogBuffer* log_buffer, FlushReason* reason,
+                         bool* flush_rescheduled_to_retain_udt,
                         Env::Priority thread_pri);

  bool EnoughRoomForCompaction(ColumnFamilyData* cfd,
@@ -2105,9 +2137,18 @@ class DBImpl : public DB {
                              std::unique_ptr<TaskLimiterToken>* token,
                              LogBuffer* log_buffer);

+  // Return true if the `FlushRequest` can be rescheduled to retain the UDT.
+  // Only true if there are user-defined timestamps in the involved MemTables
+  // with newer than cutoff timestamp `full_history_ts_low` and not flushing
+  // immediately will not cause entering write stall mode.
+  bool ShouldRescheduleFlushRequestToRetainUDT(const FlushRequest& flush_req);
+
  // Schedule background tasks
  Status StartPeriodicTaskScheduler();

+  // Cancel scheduled periodic tasks
+  Status CancelPeriodicTaskScheduler();
+
  Status RegisterRecordSeqnoTimeWorker();

  void PrintStatistics();
@@ -2191,7 +2232,7 @@ class DBImpl : public DB {
  void BuildCompactionJobInfo(const ColumnFamilyData* cfd, Compaction* c,
                              const Status& st,
                              const CompactionJobStats& compaction_job_stats,
-                              const int job_id, const Version* current,
+                              const int job_id,
                              CompactionJobInfo* compaction_job_info) const;
  // Reserve the next 'num' file numbers for to-be-ingested external SST files,
  // and return the current file_number in 'next_file_number'.
@@ -2284,15 +2325,18 @@ class DBImpl : public DB {
  // If callback is non-null, the callback is refreshed with the snapshot
  // sequence number
  //
-  // A return value of true indicates that the SuperVersions were obtained
-  // from the ColumnFamilyData, whereas false indicates they are thread
-  // local
+  // `sv_from_thread_local` being set to false indicates that the SuperVersion
+  // obtained from the ColumnFamilyData, whereas true indicates they are thread
+  // local.
+  // A non-OK status will be returned if for a column family that enables
+  // user-defined timestamp feature, the specified `ReadOptions.timestamp`
+  // attemps to read collapsed history.
  template <class T>
-  bool MultiCFSnapshot(
+  Status MultiCFSnapshot(
      const ReadOptions& read_options, ReadCallback* callback,
      std::function<MultiGetColumnFamilyData*(typename T::iterator&)>&
          iter_deref_func,
-      T* cf_list, SequenceNumber* snapshot);
+      T* cf_list, SequenceNumber* snapshot, bool* sv_from_thread_local);

  // The actual implementation of the batching MultiGet. The caller is expected
  // to have acquired the SuperVersion and pass in a snapshot sequence number
@@ -2303,8 +2347,17 @@ class DBImpl : public DB {
      autovector<KeyContext*, MultiGetContext::MAX_BATCH_SIZE>* sorted_keys,
      SuperVersion* sv, SequenceNumber snap_seqnum, ReadCallback* callback);

+  void MultiGetWithCallbackImpl(
+      const ReadOptions& read_options, ColumnFamilyHandle* column_family,
+      ReadCallback* callback,
+      autovector<KeyContext*, MultiGetContext::MAX_BATCH_SIZE>* sorted_keys);
+
  Status DisableFileDeletionsWithLock();

+  // Safely decrease `disable_delete_obsolete_files_` by one while holding lock
+  // and return its remaning value.
+  int EnableFileDeletionsWithLock();
+
  Status IncreaseFullHistoryTsLowImpl(ColumnFamilyData* cfd,
                                      std::string ts_low);

@@ -2592,9 +2645,6 @@ class DBImpl : public DB {
  // initialized with startup time.
  uint64_t delete_obsolete_files_last_run_;

-  // last time stats were dumped to LOG
-  std::atomic<uint64_t> last_stats_dump_time_microsec_;
-
  // The thread that wants to switch memtable, can wait on this cv until the
  // pending writes to memtable finishes.
  std::condition_variable switch_cv_;
@@ -2801,8 +2851,7 @@ inline Status DBImpl::FailIfCfHasTs(
 }

 inline Status DBImpl::FailIfTsMismatchCf(ColumnFamilyHandle* column_family,
-                                         const Slice& ts,
-                                         bool ts_for_read) const {
+                                         const Slice& ts) const {
  if (!column_family) {
    return Status::InvalidArgument("column family handle cannot be null");
  }
@@ -2822,20 +2871,28 @@ inline Status DBImpl::FailIfTsMismatchCf(ColumnFamilyHandle* column_family,
        << ts_sz << " given";
    return Status::InvalidArgument(oss.str());
  }
-  if (ts_for_read) {
-    auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(column_family);
-    auto cfd = cfh->cfd();
-    std::string current_ts_low = cfd->GetFullHistoryTsLow();
-    if (!current_ts_low.empty() &&
-        ucmp->CompareTimestamp(ts, current_ts_low) < 0) {
-      std::stringstream oss;
-      oss << "Read timestamp: " << ts.ToString(true)
-          << " is smaller than full_history_ts_low: "
-          << Slice(current_ts_low).ToString(true) << std::endl;
-      return Status::InvalidArgument(oss.str());
-    }
-  }
  return Status::OK();
 }

+inline Status DBImpl::FailIfReadCollapsedHistory(const ColumnFamilyData* cfd,
+                                                 const SuperVersion* sv,
+                                                 const Slice& ts) const {
+  // Reaching to this point means the timestamp size matching sanity check in
+  // `DBImpl::FailIfTsMismatchCf` already passed. So we skip that and assume
+  // column family has the same user-defined timestamp format as `ts`.
+  const Comparator* const ucmp = cfd->user_comparator();
+  assert(ucmp);
+  const std::string& full_history_ts_low = sv->full_history_ts_low;
+  assert(full_history_ts_low.empty() ||
+         full_history_ts_low.size() == ts.size());
+  if (!full_history_ts_low.empty() &&
+      ucmp->CompareTimestamp(ts, full_history_ts_low) < 0) {
+    std::stringstream oss;
+    oss << "Read timestamp: " << ts.ToString(true)
+        << " is smaller than full_history_ts_low: "
+        << Slice(full_history_ts_low).ToString(true) << std::endl;
+    return Status::InvalidArgument(oss.str());
+  }
+  return Status::OK();
+}
 }  // namespace ROCKSDB_NAMESPACE
@@ -21,7 +21,9 @@
 #include "monitoring/thread_status_util.h"
 #include "test_util/sync_point.h"
 #include "util/cast_util.h"
+#include "util/coding.h"
 #include "util/concurrent_task_limiter_impl.h"
+#include "util/udt_util.h"

 namespace ROCKSDB_NAMESPACE {

@@ -76,8 +78,43 @@ bool DBImpl::RequestCompactionToken(ColumnFamilyData* cfd, bool force,
  return false;
 }

+bool DBImpl::ShouldRescheduleFlushRequestToRetainUDT(
+    const FlushRequest& flush_req) {
+  mutex_.AssertHeld();
+  assert(flush_req.cfd_to_max_mem_id_to_persist.size() == 1);
+  ColumnFamilyData* cfd = flush_req.cfd_to_max_mem_id_to_persist.begin()->first;
+  uint64_t max_memtable_id =
+      flush_req.cfd_to_max_mem_id_to_persist.begin()->second;
+  if (cfd->IsDropped() ||
+      !cfd->ShouldPostponeFlushToRetainUDT(max_memtable_id)) {
+    return false;
+  }
+  // Check if holding on the flush will cause entering write stall mode.
+  // Write stall entered because of the accumulation of write buffers can be
+  // alleviated if we continue with the flush instead of postponing it.
+  const auto& mutable_cf_options = *cfd->GetLatestMutableCFOptions();
+
+  // Taking the status of the active Memtable into consideration so that we are
+  // not just checking if DB is currently already in write stall mode.
+  int mem_to_flush = cfd->mem()->ApproximateMemoryUsageFast() >=
+                             cfd->mem()->write_buffer_size() / 2
+                         ? 1
+                         : 0;
+  WriteStallCondition write_stall =
+      ColumnFamilyData::GetWriteStallConditionAndCause(
+          cfd->imm()->NumNotFlushed() + mem_to_flush, /*num_l0_files=*/0,
+          /*num_compaction_needed_bytes=*/0, mutable_cf_options,
+          *cfd->ioptions())
+          .first;
+  if (write_stall != WriteStallCondition::kNormal) {
+    return false;
+  }
+  return true;
+}
+
 IOStatus DBImpl::SyncClosedLogs(JobContext* job_context,
-                                VersionEdit* synced_wals) {
+                                VersionEdit* synced_wals,
+                                bool error_recovery_in_prog) {
  TEST_SYNC_POINT("DBImpl::SyncClosedLogs:Start");
  InstrumentedMutexLock l(&log_write_mutex_);
  autovector<log::Writer*, 1> logs_to_sync;
@@ -103,7 +140,7 @@ IOStatus DBImpl::SyncClosedLogs(JobContext* job_context,
      ROCKS_LOG_INFO(immutable_db_options_.info_log,
                     "[JOB %d] Syncing log #%" PRIu64, job_context->job_id,
                     log->get_log_number());
-      if (error_handler_.IsRecoveryInProgress()) {
+      if (error_recovery_in_prog) {
        log->file()->reset_seen_error();
      }
      io_s = log->file()->Sync(immutable_db_options_.use_fsync);
@@ -112,7 +149,7 @@ IOStatus DBImpl::SyncClosedLogs(JobContext* job_context,
      }

      if (immutable_db_options_.recycle_log_file_num > 0) {
-        if (error_handler_.IsRecoveryInProgress()) {
+        if (error_recovery_in_prog) {
          log->file()->reset_seen_error();
        }
        io_s = log->Close();
@@ -225,8 +262,10 @@ Status DBImpl::FlushMemTableToOutputFile(
    // SyncClosedLogs() may unlock and re-lock the log_write_mutex multiple
    // times.
    VersionEdit synced_wals;
+    bool error_recovery_in_prog = error_handler_.IsRecoveryInProgress();
    mutex_.Unlock();
-    log_io_s = SyncClosedLogs(job_context, &synced_wals);
+    log_io_s =
+        SyncClosedLogs(job_context, &synced_wals, error_recovery_in_prog);
    mutex_.Lock();
    if (log_io_s.ok() && synced_wals.IsWalAddition()) {
      const ReadOptions read_options(Env::IOActivity::kFlush);
@@ -248,6 +287,24 @@ Status DBImpl::FlushMemTableToOutputFile(
  // If the log sync failed, we do not need to pick memtable. Otherwise,
  // num_flush_not_started_ needs to be rollback.
  TEST_SYNC_POINT("DBImpl::FlushMemTableToOutputFile:BeforePickMemtables");
+  // Exit a flush due to bg error should not set bg error again.
+  bool skip_set_bg_error = false;
+  if (s.ok() && !error_handler_.GetBGError().ok() &&
+      error_handler_.IsBGWorkStopped() &&
+      flush_reason != FlushReason::kErrorRecovery &&
+      flush_reason != FlushReason::kErrorRecoveryRetryFlush) {
+    // Error recovery in progress, should not pick memtable which excludes
+    // them from being picked up by recovery flush.
+    // This ensures that when bg error is set, no new flush can pick
+    // memtables.
+    skip_set_bg_error = true;
+    s = error_handler_.GetBGError();
+    assert(!s.ok());
+    ROCKS_LOG_BUFFER(log_buffer,
+                     "[JOB %d] Skip flush due to background error %s",
+                     job_context->job_id, s.ToString().c_str());
+  }
+
  if (s.ok()) {
    flush_job.PickMemTable();
    need_cancel = true;
@@ -268,7 +325,8 @@ Status DBImpl::FlushMemTableToOutputFile(
  // is unlocked by the current thread.
  if (s.ok()) {
    s = flush_job.Run(&logs_with_prep_tracker_, &file_meta,
-                      &switched_to_mempurge);
+                      &switched_to_mempurge, &skip_set_bg_error,
+                      &error_handler_);
    need_cancel = false;
  }

@@ -309,7 +367,8 @@ Status DBImpl::FlushMemTableToOutputFile(
    }
  }

-  if (!s.ok() && !s.IsShutdownInProgress() && !s.IsColumnFamilyDropped()) {
+  if (!s.ok() && !s.IsShutdownInProgress() && !s.IsColumnFamilyDropped() &&
+      !skip_set_bg_error) {
    if (log_io_s.ok()) {
      // Error while writing to MANIFEST.
      // In fact, versions_->io_status() can also be the result of renaming
@@ -490,8 +549,10 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
    // TODO (yanqin) investigate whether we should sync the closed logs for
    // single column family case.
    VersionEdit synced_wals;
+    bool error_recovery_in_prog = error_handler_.IsRecoveryInProgress();
    mutex_.Unlock();
-    log_io_s = SyncClosedLogs(job_context, &synced_wals);
+    log_io_s =
+        SyncClosedLogs(job_context, &synced_wals, error_recovery_in_prog);
    mutex_.Lock();
    if (log_io_s.ok() && synced_wals.IsWalAddition()) {
      const ReadOptions read_options(Env::IOActivity::kFlush);
@@ -521,6 +582,21 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
    pick_status.push_back(false);
  }

+  bool flush_for_recovery =
+      bg_flush_args[0].flush_reason_ == FlushReason::kErrorRecovery ||
+      bg_flush_args[0].flush_reason_ == FlushReason::kErrorRecoveryRetryFlush;
+  bool skip_set_bg_error = false;
+
+  if (s.ok() && !error_handler_.GetBGError().ok() &&
+      error_handler_.IsBGWorkStopped() && !flush_for_recovery) {
+    s = error_handler_.GetBGError();
+    skip_set_bg_error = true;
+    assert(!s.ok());
+    ROCKS_LOG_BUFFER(log_buffer,
+                     "[JOB %d] Skip flush due to background error %s",
+                     job_context->job_id, s.ToString().c_str());
+  }
+
  if (s.ok()) {
    for (int i = 0; i != num_cfs; ++i) {
      jobs[i]->PickMemTable();
@@ -585,7 +661,10 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
        }
      }
    }
-  } else {
+  } else if (!skip_set_bg_error) {
+    // When `skip_set_bg_error` is true, no memtable is picked so
+    // there is no need to call Cancel() or RollbackMemtableFlush().
+    //
    // Need to undo atomic flush if something went wrong, i.e. s is not OK and
    // it is not because of CF drop.
    // Have to cancel the flush jobs that have NOT executed because we need to
@@ -598,8 +677,8 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
    for (int i = 0; i != num_cfs; ++i) {
      if (exec_status[i].second.ok() && exec_status[i].first) {
        auto& mems = jobs[i]->GetMemTables();
-        cfds[i]->imm()->RollbackMemtableFlush(mems,
-                                              file_meta[i].fd.GetNumber());
+        cfds[i]->imm()->RollbackMemtableFlush(
+            mems, /*rollback_succeeding_memtables=*/false);
      }
    }
  }
@@ -641,10 +720,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
    };

    bool resuming_from_bg_err =
-        error_handler_.IsDBStopped() ||
-        (bg_flush_args[0].flush_reason_ == FlushReason::kErrorRecovery ||
-         bg_flush_args[0].flush_reason_ ==
-             FlushReason::kErrorRecoveryRetryFlush);
+        error_handler_.IsDBStopped() || flush_for_recovery;
    while ((!resuming_from_bg_err || error_handler_.GetRecoveryError().ok())) {
      std::pair<Status, bool> res = wait_to_install_func();

@@ -655,15 +731,27 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
        s = res.first;
        break;
      } else if (!res.second) {
+        // we are the oldest immutable memtable
+        break;
+      }
+      // We are not the oldest immutable memtable
+      TEST_SYNC_POINT_CALLBACK(
+          "DBImpl::AtomicFlushMemTablesToOutputFiles:WaitCV", &res);
+      //
+      // If bg work is stopped, recovery thread first calls
+      // WaitForBackgroundWork() before proceeding to flush for recovery. This
+      // flush can block WaitForBackgroundWork() while waiting for recovery
+      // flush to install result. To avoid this deadlock, we should abort here
+      // if there is background error.
+      if (!flush_for_recovery && error_handler_.IsBGWorkStopped() &&
+          !error_handler_.GetBGError().ok()) {
+        s = error_handler_.GetBGError();
+        assert(!s.ok());
        break;
      }
      atomic_flush_install_cv_.Wait();

-      resuming_from_bg_err =
-          error_handler_.IsDBStopped() ||
-          (bg_flush_args[0].flush_reason_ == FlushReason::kErrorRecovery ||
-           bg_flush_args[0].flush_reason_ ==
-               FlushReason::kErrorRecoveryRetryFlush);
+      resuming_from_bg_err = error_handler_.IsDBStopped() || flush_for_recovery;
    }

    if (!resuming_from_bg_err) {
@@ -679,6 +767,17 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
      // installation.
      s = error_handler_.GetRecoveryError();
    }
+    // Since we are not installing these memtables, need to rollback
+    // to allow future flush job to pick up these memtables.
+    if (!s.ok()) {
+      for (int i = 0; i != num_cfs; ++i) {
+        assert(exec_status[i].first);
+        assert(exec_status[i].second.ok());
+        auto& mems = jobs[i]->GetMemTables();
+        cfds[i]->imm()->RollbackMemtableFlush(
+            mems, /*rollback_succeeding_memtables=*/false);
+      }
+    }
  }

  if (s.ok()) {
@@ -782,7 +881,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(

  // Need to undo atomic flush if something went wrong, i.e. s is not OK and
  // it is not because of CF drop.
-  if (!s.ok() && !s.IsColumnFamilyDropped()) {
+  if (!s.ok() && !s.IsColumnFamilyDropped() && !skip_set_bg_error) {
    if (log_io_s.ok()) {
      // Error while writing to MANIFEST.
      // In fact, versions_->io_status() can also be the result of renaming
@@ -852,8 +951,8 @@ void DBImpl::NotifyOnFlushBegin(ColumnFamilyData* cfd, FileMetaData* file_meta,
    }
  }
  mutex_.Lock();
-// no need to signal bg_cv_ as it will be signaled at the end of the
-// flush process.
+  // no need to signal bg_cv_ as it will be signaled at the end of the
+  // flush process.
 }

 void DBImpl::NotifyOnFlushCompleted(
@@ -912,26 +1011,14 @@ Status DBImpl::CompactRange(const CompactRangeOptions& options,
                                end_without_ts, "" /*trim_ts*/);
  }

-  std::string begin_str;
-  std::string end_str;
+  std::string begin_str, end_str;
+  auto [begin, end] =
+      MaybeAddTimestampsToRange(begin_without_ts, end_without_ts, ts_sz,
+                                &begin_str, &end_str, false /*exclusive_end*/);

-  // CompactRange compact all keys: [begin, end] inclusively. Add maximum
-  // timestamp to include all `begin` keys, and add minimal timestamp to include
-  // all `end` keys.
-  if (begin_without_ts != nullptr) {
-    AppendKeyWithMaxTimestamp(&begin_str, *begin_without_ts, ts_sz);
-  }
-  if (end_without_ts != nullptr) {
-    AppendKeyWithMinTimestamp(&end_str, *end_without_ts, ts_sz);
-  }
-  Slice begin(begin_str);
-  Slice end(end_str);
-
-  Slice* begin_with_ts = begin_without_ts ? &begin : nullptr;
-  Slice* end_with_ts = end_without_ts ? &end : nullptr;
-
-  return CompactRangeInternal(options, column_family, begin_with_ts,
-                              end_with_ts, "" /*trim_ts*/);
+  return CompactRangeInternal(
+      options, column_family, begin.has_value() ? &begin.value() : nullptr,
+      end.has_value() ? &end.value() : nullptr, "" /*trim_ts*/);
 }

 Status DBImpl::IncreaseFullHistoryTsLow(ColumnFamilyHandle* column_family,
@@ -1066,7 +1153,6 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
                            std::numeric_limits<uint64_t>::max(), trim_ts);
  } else {
    int first_overlapped_level = kInvalidLevel;
-    int max_overlapped_level = kInvalidLevel;
    {
      SuperVersion* super_version = cfd->GetReferencedSuperVersion(this);
      Version* current_version = super_version->current;
@@ -1142,10 +1228,8 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
                                                                    begin, end);
        }
        if (overlap) {
-          if (first_overlapped_level == kInvalidLevel) {
-            first_overlapped_level = level;
-          }
-          max_overlapped_level = level;
+          first_overlapped_level = level;
+          break;
        }
      }
      CleanupSuperVersion(super_version);
@@ -1159,7 +1243,7 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
            end, exclusive, true /* disallow_trivial_move */,
            std::numeric_limits<uint64_t>::max() /* max_file_num_to_ignore */,
            trim_ts);
-        final_output_level = max_overlapped_level;
+        final_output_level = first_overlapped_level;
      } else {
        assert(cfd->ioptions()->compaction_style == kCompactionStyleLevel);
        uint64_t next_file_number = versions_->current_next_file_number();
@@ -1171,7 +1255,29 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
        int level = first_overlapped_level;
        final_output_level = level;
        int output_level = 0, base_level = 0;
-        while (level < max_overlapped_level || level == 0) {
+        for (;;) {
+          // Always allow L0 -> L1 compaction
+          if (level > 0) {
+            if (cfd->ioptions()->level_compaction_dynamic_level_bytes) {
+              assert(final_output_level < cfd->ioptions()->num_levels);
+              if (final_output_level + 1 == cfd->ioptions()->num_levels) {
+                break;
+              }
+            } else {
+              // TODO(cbi): there is still a race condition here where
+              //  if a background compaction compacts some file beyond
+              //  current()->storage_info()->num_non_empty_levels() right after
+              //  the check here.This should happen very infrequently and should
+              //  not happen once a user populates the last level of the LSM.
+              InstrumentedMutexLock l(&mutex_);
+              // num_non_empty_levels may be lower after a compaction, so
+              // we check for >= here.
+              if (final_output_level + 1 >=
+                  cfd->current()->storage_info()->num_non_empty_levels()) {
+                break;
+              }
+            }
+          }
          output_level = level + 1;
          if (cfd->ioptions()->level_compaction_dynamic_level_bytes &&
              level == 0) {
@@ -1203,17 +1309,8 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
        if (s.ok()) {
          assert(final_output_level > 0);
          // bottommost level intra-level compaction
-          // TODO(cbi): this preserves earlier behavior where if
-          //  max_overlapped_level = 0 and bottommost_level_compaction is
-          //  kIfHaveCompactionFilter, we only do a L0 -> LBase compaction
-          //  and do not do intra-LBase compaction even when user configures
-          //  compaction filter. We may want to still do a LBase -> LBase
-          //  compaction in case there is some file in LBase that did not go
-          //  through L0 -> LBase compaction, and hence did not go through
-          //  compaction filter.
          if ((options.bottommost_level_compaction ==
                   BottommostLevelCompaction::kIfHaveCompactionFilter &&
-               max_overlapped_level != 0 &&
               (cfd->ioptions()->compaction_filter != nullptr ||
                cfd->ioptions()->compaction_filter_factory != nullptr)) ||
              options.bottommost_level_compaction ==
@@ -1221,10 +1318,11 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
              options.bottommost_level_compaction ==
                  BottommostLevelCompaction::kForce) {
            // Use `next_file_number` as `max_file_num_to_ignore` to avoid
-            // rewriting newly compacted files when it is kForceOptimized.
+            // rewriting newly compacted files when it is kForceOptimized
+            // or kIfHaveCompactionFilter with compaction filter set.
            s = RunManualCompaction(
                cfd, final_output_level, final_output_level, options, begin,
-                end, exclusive, !trim_ts.empty() /* disallow_trivial_move */,
+                end, exclusive, true /* disallow_trivial_move */,
                next_file_number /* max_file_num_to_ignore */, trim_ts);
          }
        }
@@ -1375,6 +1473,14 @@ Status DBImpl::CompactFilesImpl(
    }
  }

+  if (cfd->ioptions()->allow_ingest_behind &&
+      output_level >= cfd->ioptions()->num_levels - 1) {
+    return Status::InvalidArgument(
+        "Exceed the maximum output level defined by "
+        "the current compaction algorithm with ingest_behind --- " +
+        std::to_string(cfd->ioptions()->num_levels - 1));
+  }
+
  Status s = cfd->compaction_picker()->SanitizeCompactionInputFiles(
      &input_set, cf_meta, output_level);
  TEST_SYNC_POINT("DBImpl::CompactFilesImpl::PostSanitizeCompactionInputFiles");
@@ -1419,7 +1525,8 @@ Status DBImpl::CompactFilesImpl(
  // without releasing the lock, so we're guaranteed a compaction can be formed.
  assert(c != nullptr);

-  c->SetInputVersion(version);
+  c->FinalizeInputInfo(version);
+
  // deletion compaction currently not allowed in CompactFiles.
  assert(!c->deletion_compaction());

@@ -1469,7 +1576,12 @@ Status DBImpl::CompactFilesImpl(
  TEST_SYNC_POINT("CompactFilesImpl:3");
  mutex_.Lock();

-  Status status = compaction_job.Install(*c->mutable_cf_options());
+  bool compaction_released = false;
+  Status status =
+      compaction_job.Install(*c->mutable_cf_options(), &compaction_released);
+  if (!compaction_released) {
+    c->ReleaseCompactionFiles(s);
+  }
  if (status.ok()) {
    assert(compaction_job.io_status().ok());
    InstallSuperVersionAndScheduleWork(c->column_family_data(),
@@ -1480,7 +1592,6 @@ Status DBImpl::CompactFilesImpl(
  // not check compaction_job.io_status() explicitly if we're not calling
  // SetBGError
  compaction_job.io_status().PermitUncheckedError();
-  c->ReleaseCompactionFiles(s);
  // Need to make sure SstFileManager does its bookkeeping
  auto sfm = static_cast<SstFileManagerImpl*>(
      immutable_db_options_.sst_file_manager.get());
@@ -1492,7 +1603,7 @@ Status DBImpl::CompactFilesImpl(

  if (compaction_job_info != nullptr) {
    BuildCompactionJobInfo(cfd, c.get(), s, compaction_job_stats,
-                           job_context->job_id, version, compaction_job_info);
+                           job_context->job_id, compaction_job_info);
  }

  if (status.ok()) {
@@ -1589,21 +1700,18 @@ void DBImpl::NotifyOnCompactionBegin(ColumnFamilyData* cfd, Compaction* c,
  }

  c->SetNotifyOnCompactionCompleted();
-  Version* current = cfd->current();
-  current->Ref();
  // release lock while notifying events
  mutex_.Unlock();
  TEST_SYNC_POINT("DBImpl::NotifyOnCompactionBegin::UnlockMutex");
  {
    CompactionJobInfo info{};
-    BuildCompactionJobInfo(cfd, c, st, job_stats, job_id, current, &info);
+    BuildCompactionJobInfo(cfd, c, st, job_stats, job_id, &info);
    for (auto listener : immutable_db_options_.listeners) {
      listener->OnCompactionBegin(this, info);
    }
    info.status.PermitUncheckedError();
  }
  mutex_.Lock();
-  current->Unref();
 }

 void DBImpl::NotifyOnCompactionCompleted(
@@ -1621,21 +1729,17 @@ void DBImpl::NotifyOnCompactionCompleted(
    return;
  }

-  Version* current = cfd->current();
-  current->Ref();
  // release lock while notifying events
  mutex_.Unlock();
  TEST_SYNC_POINT("DBImpl::NotifyOnCompactionCompleted::UnlockMutex");
  {
    CompactionJobInfo info{};
-    BuildCompactionJobInfo(cfd, c, st, compaction_job_stats, job_id, current,
-                           &info);
+    BuildCompactionJobInfo(cfd, c, st, compaction_job_stats, job_id, &info);
    for (auto listener : immutable_db_options_.listeners) {
      listener->OnCompactionCompleted(this, info);
    }
  }
  mutex_.Lock();
-  current->Unref();
  // no need to signal bg_cv_ as it will be signaled at the end of the
  // flush process.
 }
@@ -1758,7 +1862,8 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
          f->marked_for_compaction, f->temperature, f->oldest_blob_file_number,
          f->oldest_ancester_time, f->file_creation_time, f->epoch_number,
          f->file_checksum, f->file_checksum_func_name, f->unique_id,
-          f->compensated_range_deletion_size, f->tail_size);
+          f->compensated_range_deletion_size, f->tail_size,
+          f->user_defined_timestamps_persisted);
    }
    ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
                    "[%s] Apply version edit:\n%s", cfd->GetName().c_str(),
@@ -1808,6 +1913,37 @@ int DBImpl::Level0StopWriteTrigger(ColumnFamilyHandle* column_family) {
      ->mutable_cf_options.level0_stop_writes_trigger;
 }

+Status DBImpl::FlushAllColumnFamilies(const FlushOptions& flush_options,
+                                      FlushReason flush_reason) {
+  mutex_.AssertHeld();
+  Status status;
+  if (immutable_db_options_.atomic_flush) {
+    mutex_.Unlock();
+    status = AtomicFlushMemTables(flush_options, flush_reason);
+    if (status.IsColumnFamilyDropped()) {
+      status = Status::OK();
+    }
+    mutex_.Lock();
+  } else {
+    for (auto cfd : versions_->GetRefedColumnFamilySet()) {
+      if (cfd->IsDropped()) {
+        continue;
+      }
+      mutex_.Unlock();
+      status = FlushMemTable(cfd, flush_options, flush_reason);
+      TEST_SYNC_POINT("DBImpl::FlushAllColumnFamilies:1");
+      TEST_SYNC_POINT("DBImpl::FlushAllColumnFamilies:2");
+      mutex_.Lock();
+      if (!status.ok() && !status.IsColumnFamilyDropped()) {
+        break;
+      } else if (status.IsColumnFamilyDropped()) {
+        status = Status::OK();
+      }
+    }
+  }
+  return status;
+}
+
 Status DBImpl::Flush(const FlushOptions& flush_options,
                     ColumnFamilyHandle* column_family) {
  auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(column_family);
@@ -2144,9 +2280,13 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd,
    WaitForPendingWrites();

    if (flush_reason != FlushReason::kErrorRecoveryRetryFlush &&
+        flush_reason != FlushReason::kCatchUpAfterErrorRecovery &&
        (!cfd->mem()->IsEmpty() || !cached_recoverable_state_empty_.load())) {
      // Note that, when flush reason is kErrorRecoveryRetryFlush, during the
      // auto retry resume, we want to avoid creating new small memtables.
+      // If flush reason is kCatchUpAfterErrorRecovery, we try to flush any new
+      // memtable that filled up during recovery, and we also want to avoid
+      // switching memtable to create small memtables.
      // Therefore, SwitchMemtable will not be called. Also, since ResumeImpl
      // will iterate through all the CFs and call FlushMemtable during auto
      // retry resume, it is possible that in some CFs,
@@ -2337,7 +2477,8 @@ Status DBImpl::AtomicFlushMemTables(

    for (auto cfd : cfds) {
      if ((cfd->mem()->IsEmpty() && cached_recoverable_state_empty_.load()) ||
-          flush_reason == FlushReason::kErrorRecoveryRetryFlush) {
+          flush_reason == FlushReason::kErrorRecoveryRetryFlush ||
+          flush_reason == FlushReason::kCatchUpAfterErrorRecovery) {
        continue;
      }
      cfd->Ref();
@@ -2455,8 +2596,11 @@ Status DBImpl::WaitUntilFlushWouldNotStallWrites(ColumnFamilyData* cfd,
      // check whether one extra immutable memtable or an extra L0 file would
      // cause write stalling mode to be entered. It could still enter stall
      // mode due to pending compaction bytes, but that's less common
+      // No extra immutable Memtable will be created if the current Memtable is
+      // empty.
+      int mem_to_flush = cfd->mem()->IsEmpty() ? 0 : 1;
      write_stall_condition = ColumnFamilyData::GetWriteStallConditionAndCause(
-                                  cfd->imm()->NumNotFlushed() + 1,
+                                  cfd->imm()->NumNotFlushed() + mem_to_flush,
                                  vstorage->l0_delay_trigger_count() + 1,
                                  vstorage->estimated_compaction_needed_bytes(),
                                  mutable_cf_options, *cfd->ioptions())
@@ -2602,6 +2746,11 @@ void DBImpl::MaybeScheduleFlushOrCompaction() {
    // There has been a hard error and this call is not part of the recovery
    // sequence. Bail out here so we don't get into an endless loop of
    // scheduling BG work which will again call this function
+    //
+    // Note that a non-recovery flush can still be scheduled if
+    // error_handler_.IsRecoveryInProgress() returns true. We rely on
+    // BackgroundCallFlush() to check flush reason and drop non-recovery
+    // flushes.
    return;
  } else if (shutting_down_.load(std::memory_order_acquire)) {
    // DB is being deleted; no more background compactions
@@ -2612,6 +2761,9 @@ void DBImpl::MaybeScheduleFlushOrCompaction() {
      env_->GetBackgroundThreads(Env::Priority::HIGH) == 0;
  while (!is_flush_pool_empty && unscheduled_flushes_ > 0 &&
         bg_flush_scheduled_ < bg_job_limits.max_flushes) {
+    TEST_SYNC_POINT_CALLBACK(
+        "DBImpl::MaybeScheduleFlushOrCompaction:BeforeSchedule",
+        &unscheduled_flushes_);
    bg_flush_scheduled_++;
    FlushThreadArg* fta = new FlushThreadArg;
    fta->db_ = this;
@@ -2721,7 +2873,7 @@ ColumnFamilyData* DBImpl::PopFirstFromCompactionQueue() {

 DBImpl::FlushRequest DBImpl::PopFirstFromFlushQueue() {
  assert(!flush_queue_.empty());
-  FlushRequest flush_req = flush_queue_.front();
+  FlushRequest flush_req = std::move(flush_queue_.front());
  flush_queue_.pop_front();
  if (!immutable_db_options_.atomic_flush) {
    assert(flush_req.cfd_to_max_mem_id_to_persist.size() == 1);
@@ -2765,6 +2917,9 @@ ColumnFamilyData* DBImpl::PickCompactionFromQueue(

 void DBImpl::SchedulePendingFlush(const FlushRequest& flush_req) {
  mutex_.AssertHeld();
+  if (reject_new_background_jobs_) {
+    return;
+  }
  if (flush_req.cfd_to_max_mem_id_to_persist.empty()) {
    return;
  }
@@ -2794,6 +2949,9 @@ void DBImpl::SchedulePendingFlush(const FlushRequest& flush_req) {

 void DBImpl::SchedulePendingCompaction(ColumnFamilyData* cfd) {
  mutex_.AssertHeld();
+  if (reject_new_background_jobs_) {
+    return;
+  }
  if (!cfd->queued_for_compaction() && cfd->NeedsCompaction()) {
    AddToCompactionQueue(cfd);
    ++unscheduled_compactions_;
@@ -2803,6 +2961,9 @@ void DBImpl::SchedulePendingCompaction(ColumnFamilyData* cfd) {
 void DBImpl::SchedulePendingPurge(std::string fname, std::string dir_to_sync,
                                  FileType type, uint64_t number, int job_id) {
  mutex_.AssertHeld();
+  if (reject_new_background_jobs_) {
+    return;
+  }
  PurgeFileInfo file_info(fname, dir_to_sync, type, number, job_id);
  purge_files_.insert({{number, std::move(file_info)}});
 }
@@ -2891,6 +3052,7 @@ void DBImpl::UnscheduleFlushCallback(void* arg) {

 Status DBImpl::BackgroundFlush(bool* made_progress, JobContext* job_context,
                               LogBuffer* log_buffer, FlushReason* reason,
+                               bool* flush_rescheduled_to_retain_udt,
                               Env::Priority thread_pri) {
  mutex_.AssertHeld();

@@ -2916,14 +3078,61 @@ Status DBImpl::BackgroundFlush(bool* made_progress, JobContext* job_context,
  autovector<ColumnFamilyData*> column_families_not_to_flush;
  while (!flush_queue_.empty()) {
    // This cfd is already referenced
-    const FlushRequest& flush_req = PopFirstFromFlushQueue();
+    FlushRequest flush_req = PopFirstFromFlushQueue();
    FlushReason flush_reason = flush_req.flush_reason;
+    if (!error_handler_.GetBGError().ok() && error_handler_.IsBGWorkStopped() &&
+        flush_reason != FlushReason::kErrorRecovery &&
+        flush_reason != FlushReason::kErrorRecoveryRetryFlush) {
+      // Stop non-recovery flush when bg work is stopped
+      // Note that we drop the flush request here.
+      // Recovery thread should schedule further flushes after bg error
+      // is cleared.
+      status = error_handler_.GetBGError();
+      assert(!status.ok());
+      ROCKS_LOG_BUFFER(log_buffer,
+                       "[JOB %d] Abort flush due to background error %s",
+                       job_context->job_id, status.ToString().c_str());
+      *reason = flush_reason;
+      for (auto item : flush_req.cfd_to_max_mem_id_to_persist) {
+        item.first->UnrefAndTryDelete();
+      }
+      return status;
+    }
+    if (!immutable_db_options_.atomic_flush &&
+        ShouldRescheduleFlushRequestToRetainUDT(flush_req)) {
+      assert(flush_req.cfd_to_max_mem_id_to_persist.size() == 1);
+      ColumnFamilyData* cfd =
+          flush_req.cfd_to_max_mem_id_to_persist.begin()->first;
+      if (cfd->UnrefAndTryDelete()) {
+        return Status::OK();
+      }
+      ROCKS_LOG_BUFFER(log_buffer,
+                       "FlushRequest for column family %s is re-scheduled to "
+                       "retain user-defined timestamps.",
+                       cfd->GetName().c_str());
+      // Reschedule the `FlushRequest` as is without checking dropped column
+      // family etc. The follow-up job will do the check anyways, so save the
+      // duplication. Column family is deduplicated by `SchdulePendingFlush` and
+      // `PopFirstFromFlushQueue` contains at flush request enqueueing and
+      // dequeueing time.
+      // This flush request is rescheduled right after it's popped from the
+      // queue while the db mutex is held, so there should be no other
+      // FlushRequest for the same column family with higher `max_memtable_id`
+      // in the queue to block the reschedule from succeeding.
+#ifndef NDEBUG
+      flush_req.reschedule_count += 1;
+#endif /* !NDEBUG */
+      SchedulePendingFlush(flush_req);
+      *reason = flush_reason;
+      *flush_rescheduled_to_retain_udt = true;
+      return Status::TryAgain();
+    }
    superversion_contexts.clear();
    superversion_contexts.reserve(
        flush_req.cfd_to_max_mem_id_to_persist.size());

-    for (const auto& iter : flush_req.cfd_to_max_mem_id_to_persist) {
-      ColumnFamilyData* cfd = iter.first;
+    for (const auto& [cfd, max_memtable_id] :
+         flush_req.cfd_to_max_mem_id_to_persist) {
      if (cfd->GetMempurgeUsed()) {
        // If imm() contains silent memtables (e.g.: because
        // MemPurge was activated), requesting a flush will
@@ -2937,10 +3146,16 @@ Status DBImpl::BackgroundFlush(bool* made_progress, JobContext* job_context,
        continue;
      }
      superversion_contexts.emplace_back(SuperVersionContext(true));
-      bg_flush_args.emplace_back(cfd, iter.second,
+      bg_flush_args.emplace_back(cfd, max_memtable_id,
                                 &(superversion_contexts.back()), flush_reason);
    }
-    if (!bg_flush_args.empty()) {
+    // `MaybeScheduleFlushOrCompaction` schedules as many `BackgroundCallFlush`
+    // jobs as the number of `FlushRequest` in the `flush_queue_`, a.k.a
+    // `unscheduled_flushes_`. So it's sufficient to make each `BackgroundFlush`
+    // handle one `FlushRequest` and each have a Status returned.
+    if (!bg_flush_args.empty() || !column_families_not_to_flush.empty()) {
+      TEST_SYNC_POINT_CALLBACK("DBImpl::BackgroundFlush:CheckFlushRequest:cb",
+                               const_cast<int*>(&flush_req.reschedule_count));
      break;
    }
  }
@@ -3002,11 +3217,20 @@ void DBImpl::BackgroundCallFlush(Env::Priority thread_pri) {
        pending_outputs_inserted_elem(new std::list<uint64_t>::iterator(
            CaptureCurrentFileNumberInPendingOutputs()));
    FlushReason reason;
-
-    Status s = BackgroundFlush(&made_progress, &job_context, &log_buffer,
-                               &reason, thread_pri);
-    if (!s.ok() && !s.IsShutdownInProgress() && !s.IsColumnFamilyDropped() &&
-        reason != FlushReason::kErrorRecovery) {
+    bool flush_rescheduled_to_retain_udt = false;
+    Status s =
+        BackgroundFlush(&made_progress, &job_context, &log_buffer, &reason,
+                        &flush_rescheduled_to_retain_udt, thread_pri);
+    if (s.IsTryAgain() && flush_rescheduled_to_retain_udt) {
+      bg_cv_.SignalAll();  // In case a waiter can proceed despite the error
+      mutex_.Unlock();
+      TEST_SYNC_POINT_CALLBACK("DBImpl::AfterRetainUDTReschedule:cb", nullptr);
+      immutable_db_options_.clock->SleepForMicroseconds(
+          100000);  // prevent hot loop
+      mutex_.Lock();
+    } else if (!s.ok() && !s.IsShutdownInProgress() &&
+               !s.IsColumnFamilyDropped() &&
+               reason != FlushReason::kErrorRecovery) {
      // Wait a little bit before retrying background flush in
      // case this is an environmental problem and we do not want to
      // chew up resources for failed flushes for the duration of
@@ -3016,9 +3240,9 @@ void DBImpl::BackgroundCallFlush(Env::Priority thread_pri) {
      bg_cv_.SignalAll();  // In case a waiter can proceed despite the error
      mutex_.Unlock();
      ROCKS_LOG_ERROR(immutable_db_options_.info_log,
-                      "Waiting after background flush error: %s"
+                      "[JOB %d] Waiting after background flush error: %s"
                      "Accumulated background error counts: %" PRIu64,
-                      s.ToString().c_str(), error_cnt);
+                      job_context.job_id, s.ToString().c_str(), error_cnt);
      log_buffer.FlushBufferToLog();
      LogFlush(immutable_db_options_.info_log);
      immutable_db_options_.clock->SleepForMicroseconds(1000000);
@@ -3027,29 +3251,33 @@ void DBImpl::BackgroundCallFlush(Env::Priority thread_pri) {

    TEST_SYNC_POINT("DBImpl::BackgroundCallFlush:FlushFinish:0");
    ReleaseFileNumberFromPendingOutputs(pending_outputs_inserted_elem);
-
-    // If flush failed, we want to delete all temporary files that we might have
-    // created. Thus, we force full scan in FindObsoleteFiles()
-    FindObsoleteFiles(&job_context, !s.ok() && !s.IsShutdownInProgress() &&
-                                        !s.IsColumnFamilyDropped());
-    // delete unnecessary files if any, this is done outside the mutex
-    if (job_context.HaveSomethingToClean() ||
-        job_context.HaveSomethingToDelete() || !log_buffer.IsEmpty()) {
-      mutex_.Unlock();
-      TEST_SYNC_POINT("DBImpl::BackgroundCallFlush:FilesFound");
-      // Have to flush the info logs before bg_flush_scheduled_--
-      // because if bg_flush_scheduled_ becomes 0 and the lock is
-      // released, the deconstructor of DB can kick in and destroy all the
-      // states of DB so info_log might not be available after that point.
-      // It also applies to access other states that DB owns.
-      log_buffer.FlushBufferToLog();
-      if (job_context.HaveSomethingToDelete()) {
-        PurgeObsoleteFiles(job_context);
+    // There is no need to do these clean up if the flush job is rescheduled
+    // to retain user-defined timestamps because the job doesn't get to the
+    // stage of actually flushing the MemTables.
+    if (!flush_rescheduled_to_retain_udt) {
+      // If flush failed, we want to delete all temporary files that we might
+      // have created. Thus, we force full scan in FindObsoleteFiles()
+      FindObsoleteFiles(&job_context, !s.ok() && !s.IsShutdownInProgress() &&
+                                          !s.IsColumnFamilyDropped());
+      // delete unnecessary files if any, this is done outside the mutex
+      if (job_context.HaveSomethingToClean() ||
+          job_context.HaveSomethingToDelete() || !log_buffer.IsEmpty()) {
+        mutex_.Unlock();
+        TEST_SYNC_POINT("DBImpl::BackgroundCallFlush:FilesFound");
+        // Have to flush the info logs before bg_flush_scheduled_--
+        // because if bg_flush_scheduled_ becomes 0 and the lock is
+        // released, the deconstructor of DB can kick in and destroy all the
+        // states of DB so info_log might not be available after that point.
+        // It also applies to access other states that DB owns.
+        log_buffer.FlushBufferToLog();
+        if (job_context.HaveSomethingToDelete()) {
+          PurgeObsoleteFiles(job_context);
+        }
+        job_context.Clean();
+        mutex_.Lock();
      }
-      job_context.Clean();
-      mutex_.Lock();
+      TEST_SYNC_POINT("DBImpl::BackgroundCallFlush:ContextCleanedUp");
    }
-    TEST_SYNC_POINT("DBImpl::BackgroundCallFlush:ContextCleanedUp");

    assert(num_running_flushes_ > 0);
    num_running_flushes_--;
@@ -3256,8 +3484,6 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,

  std::unique_ptr<TaskLimiterToken> task_token;

-  // InternalKey manual_end_storage;
-  // InternalKey* manual_end = &manual_end_storage;
  bool sfm_reserved_compact_space = false;
  if (is_manual) {
    ManualCompactionState* m = manual_compaction;
@@ -3393,6 +3619,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
  }

  IOStatus io_s;
+  bool compaction_released = false;
  if (!c) {
    // Nothing to do
    ROCKS_LOG_BUFFER(log_buffer, "Compaction nothing to do");
@@ -3415,7 +3642,12 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
    }
    status = versions_->LogAndApply(
        c->column_family_data(), *c->mutable_cf_options(), read_options,
-        c->edit(), &mutex_, directories_.GetDbDir());
+        c->edit(), &mutex_, directories_.GetDbDir(),
+        /*new_descriptor_log=*/false, /*column_family_options=*/nullptr,
+        [&c, &compaction_released](const Status& s) {
+          c->ReleaseCompactionFiles(s);
+          compaction_released = true;
+        });
    io_s = versions_->io_status();
    InstallSuperVersionAndScheduleWork(c->column_family_data(),
                                       &job_context->superversion_contexts[0],
@@ -3457,7 +3689,8 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
            f->oldest_blob_file_number, f->oldest_ancester_time,
            f->file_creation_time, f->epoch_number, f->file_checksum,
            f->file_checksum_func_name, f->unique_id,
-            f->compensated_range_deletion_size, f->tail_size);
+            f->compensated_range_deletion_size, f->tail_size,
+            f->user_defined_timestamps_persisted);

        ROCKS_LOG_BUFFER(
            log_buffer,
@@ -3480,7 +3713,12 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
    }
    status = versions_->LogAndApply(
        c->column_family_data(), *c->mutable_cf_options(), read_options,
-        c->edit(), &mutex_, directories_.GetDbDir());
+        c->edit(), &mutex_, directories_.GetDbDir(),
+        /*new_descriptor_log=*/false, /*column_family_options=*/nullptr,
+        [&c, &compaction_released](const Status& s) {
+          c->ReleaseCompactionFiles(s);
+          compaction_released = true;
+        });
    io_s = versions_->io_status();
    // Use latest MutableCFOptions
    InstallSuperVersionAndScheduleWork(c->column_family_data(),
@@ -3530,6 +3768,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
    // Transfer requested token, so it doesn't need to do it again.
    ca->prepicked_compaction->task_token = std::move(task_token);
    ++bg_bottom_compaction_scheduled_;
+    assert(c == nullptr);
    env_->Schedule(&DBImpl::BGWorkBottomCompaction, ca, Env::Priority::BOTTOM,
                   this, &DBImpl::UnscheduleCompactionCallback);
  } else {
@@ -3573,8 +3812,8 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
    compaction_job.Run().PermitUncheckedError();
    TEST_SYNC_POINT("DBImpl::BackgroundCompaction:NonTrivial:AfterRun");
    mutex_.Lock();
-
-    status = compaction_job.Install(*c->mutable_cf_options());
+    status =
+        compaction_job.Install(*c->mutable_cf_options(), &compaction_released);
    io_s = compaction_job.io_status();
    if (status.ok()) {
      InstallSuperVersionAndScheduleWork(c->column_family_data(),
@@ -3593,7 +3832,23 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
  }

  if (c != nullptr) {
-    c->ReleaseCompactionFiles(status);
+    if (!compaction_released) {
+      c->ReleaseCompactionFiles(status);
+    } else {
+#ifndef NDEBUG
+      // Sanity checking that compaction files are freed.
+      for (size_t i = 0; i < c->num_input_levels(); i++) {
+        for (size_t j = 0; j < c->inputs(i)->size(); j++) {
+          assert(!c->input(i, j)->being_compacted);
+        }
+      }
+      std::unordered_set<Compaction*>* cip = c->column_family_data()
+                                                 ->compaction_picker()
+                                                 ->compactions_in_progress();
+      assert(cip->find(c.get()) == cip->end());
+#endif
+    }
+
    *made_progress = true;

    // Need to make sure SstFileManager does its bookkeeping
@@ -3781,7 +4036,7 @@ bool DBImpl::MCOverlap(ManualCompactionState* m, ManualCompactionState* m1) {
 void DBImpl::BuildCompactionJobInfo(
    const ColumnFamilyData* cfd, Compaction* c, const Status& st,
    const CompactionJobStats& compaction_job_stats, const int job_id,
-    const Version* current, CompactionJobInfo* compaction_job_info) const {
+    CompactionJobInfo* compaction_job_info) const {
  assert(compaction_job_info != nullptr);
  compaction_job_info->cf_id = cfd->GetID();
  compaction_job_info->cf_name = cfd->GetName();
@@ -3791,7 +4046,12 @@ void DBImpl::BuildCompactionJobInfo(
  compaction_job_info->base_input_level = c->start_level();
  compaction_job_info->output_level = c->output_level();
  compaction_job_info->stats = compaction_job_stats;
-  compaction_job_info->table_properties = c->GetOutputTableProperties();
+  const auto& input_table_properties = c->GetInputTableProperties();
+  const auto& output_table_properties = c->GetOutputTableProperties();
+  compaction_job_info->table_properties.insert(input_table_properties.begin(),
+                                               input_table_properties.end());
+  compaction_job_info->table_properties.insert(output_table_properties.begin(),
+                                               output_table_properties.end());
  compaction_job_info->compaction_reason = c->compaction_reason();
  compaction_job_info->compression = c->output_compression();

@@ -3805,15 +4065,9 @@ void DBImpl::BuildCompactionJobInfo(
      compaction_job_info->input_files.push_back(fn);
      compaction_job_info->input_file_infos.push_back(CompactionFileInfo{
          static_cast<int>(i), file_number, fmd->oldest_blob_file_number});
-      if (compaction_job_info->table_properties.count(fn) == 0) {
-        std::shared_ptr<const TableProperties> tp;
-        auto s = current->GetTableProperties(read_options, &tp, fmd, &fn);
-        if (s.ok()) {
-          compaction_job_info->table_properties[fn] = tp;
-        }
-      }
    }
  }
+
  for (const auto& newf : c->edit()->GetNewFiles()) {
    const FileMetaData& meta = newf.second;
    const FileDescriptor& desc = meta.fd;
@@ -3957,20 +4211,54 @@ void DBImpl::GetSnapshotContext(
  *snapshot_seqs = snapshots_.GetAll(earliest_write_conflict_snapshot);
 }

-Status DBImpl::WaitForCompact(bool abort_on_pause) {
+Status DBImpl::WaitForCompact(
+    const WaitForCompactOptions& wait_for_compact_options) {
  InstrumentedMutexLock l(&mutex_);
+  if (wait_for_compact_options.flush) {
+    Status s = DBImpl::FlushAllColumnFamilies(FlushOptions(),
+                                              FlushReason::kManualFlush);
+    if (!s.ok()) {
+      return s;
+    }
+  } else if (wait_for_compact_options.close_db &&
+             has_unpersisted_data_.load(std::memory_order_relaxed) &&
+             !mutable_db_options_.avoid_flush_during_shutdown) {
+    Status s =
+        DBImpl::FlushAllColumnFamilies(FlushOptions(), FlushReason::kShutDown);
+    if (!s.ok()) {
+      return s;
+    }
+  }
+  TEST_SYNC_POINT("DBImpl::WaitForCompact:StartWaiting");
+  const auto deadline = immutable_db_options_.clock->NowMicros() +
+                        wait_for_compact_options.timeout.count();
  for (;;) {
    if (shutting_down_.load(std::memory_order_acquire)) {
      return Status::ShutdownInProgress();
    }
-    if (bg_work_paused_ && abort_on_pause) {
+    if (bg_work_paused_ && wait_for_compact_options.abort_on_pause) {
      return Status::Aborted();
    }
    if ((bg_bottom_compaction_scheduled_ || bg_compaction_scheduled_ ||
         bg_flush_scheduled_ || unscheduled_compactions_ ||
-         unscheduled_flushes_) &&
+         unscheduled_flushes_ || error_handler_.IsRecoveryInProgress()) &&
        (error_handler_.GetBGError().ok())) {
-      bg_cv_.Wait();
+      if (wait_for_compact_options.timeout.count()) {
+        if (bg_cv_.TimedWait(deadline)) {
+          return Status::TimedOut();
+        }
+      } else {
+        bg_cv_.Wait();
+      }
+    } else if (wait_for_compact_options.close_db) {
+      reject_new_background_jobs_ = true;
+      mutex_.Unlock();
+      Status s = Close();
+      mutex_.Lock();
+      if (!s.ok()) {
+        reject_new_background_jobs_ = false;
+      }
+      return s;
    } else {
      return error_handler_.GetBGError();
    }
@@ -178,9 +178,12 @@ Status DBImpl::TEST_WaitForFlushMemTable(ColumnFamilyHandle* column_family) {
  return WaitForFlushMemTable(cfd, nullptr, false);
 }

-Status DBImpl::TEST_WaitForCompact(bool abort_on_pause) {
-  // Wait until the compaction completes
-  return WaitForCompact(abort_on_pause);
+Status DBImpl::TEST_WaitForCompact() {
+  return WaitForCompact(WaitForCompactOptions());
+}
+Status DBImpl::TEST_WaitForCompact(
+    const WaitForCompactOptions& wait_for_compact_options) {
+  return WaitForCompact(wait_for_compact_options);
 }

 Status DBImpl::TEST_WaitForPurge() {
@@ -138,7 +138,8 @@ Status DBImpl::PromoteL0(ColumnFamilyHandle* column_family, int target_level) {
                   f->oldest_blob_file_number, f->oldest_ancester_time,
                   f->file_creation_time, f->epoch_number, f->file_checksum,
                   f->file_checksum_func_name, f->unique_id,
-                   f->compensated_range_deletion_size, f->tail_size);
+                   f->compensated_range_deletion_size, f->tail_size,
+                   f->user_defined_timestamps_persisted);
    }

    status = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),
@@ -35,6 +35,11 @@ uint64_t DBImpl::MinObsoleteSstNumberToKeep() {
  return std::numeric_limits<uint64_t>::max();
 }

+uint64_t DBImpl::GetObsoleteSstFilesSize() {
+  mutex_.AssertHeld();
+  return versions_->GetObsoleteSstFilesSize();
+}
+
 Status DBImpl::DisableFileDeletions() {
  Status s;
  int my_disable_delete_obsolete_files;
@@ -95,6 +100,14 @@ Status DBImpl::EnableFileDeletions(bool force) {
  return Status::OK();
 }

+int DBImpl::EnableFileDeletionsWithLock() {
+  mutex_.AssertHeld();
+  // In case others have called EnableFileDeletions(true /* force */) in between
+  disable_delete_obsolete_files_ =
+      std::max(0, disable_delete_obsolete_files_ - 1);
+  return disable_delete_obsolete_files_;
+}
+
 bool DBImpl::IsFileDeletionsEnabled() const {
  return 0 == disable_delete_obsolete_files_;
 }
@@ -452,12 +465,12 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
  std::sort(candidate_files.begin(), candidate_files.end(),
            [](const JobContext::CandidateFileInfo& lhs,
               const JobContext::CandidateFileInfo& rhs) {
-              if (lhs.file_name > rhs.file_name) {
+              if (lhs.file_name < rhs.file_name) {
                return true;
-              } else if (lhs.file_name < rhs.file_name) {
+              } else if (lhs.file_name > rhs.file_name) {
                return false;
              } else {
-                return (lhs.file_path > rhs.file_path);
+                return (lhs.file_path < rhs.file_path);
              }
            });
  candidate_files.erase(
@@ -990,7 +1003,7 @@ Status DBImpl::DeleteUnreferencedSstFiles(RecoveryContext* recovery_ctx) {
      if (type == kTableFile && number >= next_file_number &&
          recovery_ctx->files_to_delete_.find(normalized_fpath) ==
              recovery_ctx->files_to_delete_.end()) {
-        recovery_ctx->files_to_delete_.emplace(normalized_fpath);
+        recovery_ctx->files_to_delete_.emplace(normalized_fpath, path);
      }
    }
  }
@@ -25,6 +25,7 @@
 #include "rocksdb/wal_filter.h"
 #include "test_util/sync_point.h"
 #include "util/rate_limiter_impl.h"
+#include "util/udt_util.h"

 namespace ROCKSDB_NAMESPACE {
 Options SanitizeOptions(const std::string& dbname, const Options& src,
@@ -142,11 +143,6 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src,
    result.wal_dir = result.wal_dir.substr(0, result.wal_dir.size() - 1);
  }

-  if (result.use_direct_reads && result.compaction_readahead_size == 0) {
-    TEST_SYNC_POINT_CALLBACK("SanitizeOptions:direct_io", nullptr);
-    result.compaction_readahead_size = 1024 * 1024 * 2;
-  }
-
  // Force flush on DB open if 2PC is enabled, since with 2PC we have no
  // guarantee that consecutive log files have consecutive sequence id, which
  // make recovery complicated.
@@ -617,7 +613,7 @@ Status DBImpl::Recover(
                           f->file_creation_time, f->epoch_number,
                           f->file_checksum, f->file_checksum_func_name,
                           f->unique_id, f->compensated_range_deletion_size,
-                           f->tail_size);
+                           f->tail_size, f->user_defined_timestamps_persisted);
              ROCKS_LOG_WARN(immutable_db_options_.info_log,
                             "[%s] Moving #%" PRIu64
                             " from from_level-%d to from_level-%d %" PRIu64
@@ -931,8 +927,11 @@ Status DBImpl::LogAndApplyForRecovery(const RecoveryContext& recovery_ctx) {
      recovery_ctx.edit_lists_, &mutex_, directories_.GetDbDir());
  if (s.ok() && !(recovery_ctx.files_to_delete_.empty())) {
    mutex_.Unlock();
-    for (const auto& fname : recovery_ctx.files_to_delete_) {
-      s = env_->DeleteFile(fname);
+    for (const auto& stale_sst_file : recovery_ctx.files_to_delete_) {
+      s = DeleteDBFile(&immutable_db_options_, stale_sst_file.first,
+                       stale_sst_file.second,
+                       /*force_bg=*/false,
+                       /*force_fg=*/false);
      if (!s.ok()) {
        break;
      }
@@ -1186,6 +1185,9 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& wal_numbers,
    std::string scratch;
    Slice record;

+    const UnorderedMap<uint32_t, size_t>& running_ts_sz =
+        versions_->GetRunningColumnFamiliesTimestampSize();
+
    TEST_SYNC_POINT_CALLBACK("DBImpl::RecoverLogFiles:BeforeReadWal",
                             /*arg=*/nullptr);
    uint64_t record_checksum;
@@ -1199,27 +1201,41 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& wal_numbers,
                            Status::Corruption("log record too small"));
        continue;
      }
-
      // We create a new batch and initialize with a valid prot_info_ to store
      // the data checksums
      WriteBatch batch;
+      std::unique_ptr<WriteBatch> new_batch;

      status = WriteBatchInternal::SetContents(&batch, record);
      if (!status.ok()) {
        return status;
      }
-      TEST_SYNC_POINT_CALLBACK(
-          "DBImpl::RecoverLogFiles:BeforeUpdateProtectionInfo:batch", &batch);
-      TEST_SYNC_POINT_CALLBACK(
-          "DBImpl::RecoverLogFiles:BeforeUpdateProtectionInfo:checksum",
-          &record_checksum);
-      status = WriteBatchInternal::UpdateProtectionInfo(
-          &batch, 8 /* bytes_per_key */, &record_checksum);
+
+      const UnorderedMap<uint32_t, size_t>& record_ts_sz =
+          reader.GetRecordedTimestampSize();
+      status = HandleWriteBatchTimestampSizeDifference(
+          &batch, running_ts_sz, record_ts_sz,
+          TimestampSizeConsistencyMode::kReconcileInconsistency, &new_batch);
      if (!status.ok()) {
        return status;
      }

-      SequenceNumber sequence = WriteBatchInternal::Sequence(&batch);
+      bool batch_updated = new_batch != nullptr;
+      WriteBatch* batch_to_use = batch_updated ? new_batch.get() : &batch;
+      TEST_SYNC_POINT_CALLBACK(
+          "DBImpl::RecoverLogFiles:BeforeUpdateProtectionInfo:batch",
+          batch_to_use);
+      TEST_SYNC_POINT_CALLBACK(
+          "DBImpl::RecoverLogFiles:BeforeUpdateProtectionInfo:checksum",
+          &record_checksum);
+      status = WriteBatchInternal::UpdateProtectionInfo(
+          batch_to_use, 8 /* bytes_per_key */,
+          batch_updated ? nullptr : &record_checksum);
+      if (!status.ok()) {
+        return status;
+      }
+
+      SequenceNumber sequence = WriteBatchInternal::Sequence(batch_to_use);

      if (immutable_db_options_.wal_recovery_mode ==
          WALRecoveryMode::kPointInTimeRecovery) {
@@ -1240,7 +1256,7 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& wal_numbers,
      // and returns true.
      if (!InvokeWalFilterIfNeededOnWalRecord(wal_number, fname, reporter,
                                              status, stop_replay_by_wal_filter,
-                                              batch)) {
+                                              *batch_to_use)) {
        continue;
      }

@@ -1251,7 +1267,7 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& wal_numbers,
      // That's why we set ignore missing column families to true
      bool has_valid_writes = false;
      status = WriteBatchInternal::InsertInto(
-          &batch, column_family_memtables_.get(), &flush_scheduler_,
+          batch_to_use, column_family_memtables_.get(), &flush_scheduler_,
          &trim_history_scheduler_, true, wal_number, this,
          false /* concurrent_memtable_writes */, next_sequence,
          &has_valid_writes, seq_per_batch_, batch_per_txn_);
@@ -1285,7 +1301,7 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& wal_numbers,
          flushed = true;

          cfd->CreateNewMemtable(*cfd->GetLatestMutableCFOptions(),
-                                 *next_sequence);
+                                 *next_sequence - 1);
        }
      }
    }
@@ -1640,6 +1656,7 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
      Version* version = cfd->current();
      version->Ref();
      const ReadOptions read_option(Env::IOActivity::kDBOpen);
+      uint64_t num_input_entries = 0;
      s = BuildTable(
          dbname_, versions_.get(), immutable_db_options_, tboptions,
          file_options_for_compaction_, read_option, cfd->table_cache(),
@@ -1649,7 +1666,8 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
          io_tracer_, BlobFileCreationReason::kRecovery,
          empty_seqno_time_mapping, &event_logger_, job_id, Env::IO_HIGH,
          nullptr /* table_properties */, write_hint,
-          nullptr /*full_history_ts_low*/, &blob_callback_, version);
+          nullptr /*full_history_ts_low*/, &blob_callback_, version,
+          &num_input_entries);
      version->Unref();
      LogFlush(immutable_db_options_.info_log);
      ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
@@ -1663,6 +1681,19 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
      if (!io_s.ok() && s.ok()) {
        s = io_s;
      }
+
+      uint64_t total_num_entries = mem->num_entries();
+      if (s.ok() && total_num_entries != num_input_entries) {
+        std::string msg = "Expected " + std::to_string(total_num_entries) +
+                          " entries in memtable, but read " +
+                          std::to_string(num_input_entries);
+        ROCKS_LOG_WARN(immutable_db_options_.info_log,
+                       "[%s] [JOB %d] Level-0 flush during recover: %s",
+                       cfd->GetName().c_str(), job_id, msg.c_str());
+        if (immutable_db_options_.flush_verify_memtable_count) {
+          s = Status::Corruption(msg);
+        }
+      }
    }
  }
  ReleaseFileNumberFromPendingOutputs(pending_outputs_inserted_elem);
@@ -1674,18 +1705,35 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
  constexpr int level = 0;

  if (s.ok() && has_output) {
-    edit->AddFile(
-        level, meta.fd.GetNumber(), meta.fd.GetPathId(), meta.fd.GetFileSize(),
-        meta.smallest, meta.largest, meta.fd.smallest_seqno,
-        meta.fd.largest_seqno, meta.marked_for_compaction, meta.temperature,
-        meta.oldest_blob_file_number, meta.oldest_ancester_time,
-        meta.file_creation_time, meta.epoch_number, meta.file_checksum,
-        meta.file_checksum_func_name, meta.unique_id,
-        meta.compensated_range_deletion_size, meta.tail_size);
+    edit->AddFile(level, meta.fd.GetNumber(), meta.fd.GetPathId(),
+                  meta.fd.GetFileSize(), meta.smallest, meta.largest,
+                  meta.fd.smallest_seqno, meta.fd.largest_seqno,
+                  meta.marked_for_compaction, meta.temperature,
+                  meta.oldest_blob_file_number, meta.oldest_ancester_time,
+                  meta.file_creation_time, meta.epoch_number,
+                  meta.file_checksum, meta.file_checksum_func_name,
+                  meta.unique_id, meta.compensated_range_deletion_size,
+                  meta.tail_size, meta.user_defined_timestamps_persisted);

    for (const auto& blob : blob_file_additions) {
      edit->AddBlobFile(blob);
    }
+
+    // For UDT in memtable only feature, move up the cutoff timestamp whenever
+    // a flush happens.
+    const Comparator* ucmp = cfd->user_comparator();
+    size_t ts_sz = ucmp->timestamp_size();
+    if (ts_sz > 0 && !cfd->ioptions()->persist_user_defined_timestamps) {
+      Slice mem_newest_udt = mem->GetNewestUDT();
+      std::string full_history_ts_low = cfd->GetFullHistoryTsLow();
+      if (full_history_ts_low.empty() ||
+          ucmp->CompareTimestamp(mem_newest_udt, full_history_ts_low) >= 0) {
+        std::string new_full_history_ts_low;
+        GetFullHistoryTsLowFromU64CutoffTs(&mem_newest_udt,
+                                           &new_full_history_ts_low);
+        edit->SetFullHistoryTsLow(new_full_history_ts_low);
+      }
+    }
  }

  InternalStats::CompactionStats stats(CompactionReason::kFlush, 1);
@@ -29,35 +29,23 @@ DBImplReadOnly::DBImplReadOnly(const DBOptions& db_options,
 DBImplReadOnly::~DBImplReadOnly() {}

 // Implementations of the DB interface
-Status DBImplReadOnly::Get(const ReadOptions& read_options,
-                           ColumnFamilyHandle* column_family, const Slice& key,
-                           PinnableSlice* pinnable_val) {
-  return Get(read_options, column_family, key, pinnable_val,
-             /*timestamp*/ nullptr);
-}
+Status DBImplReadOnly::GetImpl(const ReadOptions& read_options,
+                               const Slice& key,
+                               GetImplOptions& get_impl_options) {
+  assert(get_impl_options.value != nullptr ||
+         get_impl_options.columns != nullptr);
+  assert(get_impl_options.column_family);

-Status DBImplReadOnly::Get(const ReadOptions& read_options,
-                           ColumnFamilyHandle* column_family, const Slice& key,
-                           PinnableSlice* pinnable_val,
-                           std::string* timestamp) {
-  if (read_options.io_activity != Env::IOActivity::kUnknown) {
-    return Status::InvalidArgument(
-        "Cannot call Get with `ReadOptions::io_activity` != "
-        "`Env::IOActivity::kUnknown`");
-  }
-  assert(pinnable_val != nullptr);
-  // TODO: stopwatch DB_GET needed?, perf timer needed?
-  PERF_TIMER_GUARD(get_snapshot_time);
+  Status s;

-  assert(column_family);
  if (read_options.timestamp) {
-    const Status s = FailIfTsMismatchCf(
-        column_family, *(read_options.timestamp), /*ts_for_read=*/true);
+    s = FailIfTsMismatchCf(get_impl_options.column_family,
+                           *(read_options.timestamp));
    if (!s.ok()) {
      return s;
    }
  } else {
-    const Status s = FailIfCfHasTs(column_family);
+    s = FailIfCfHasTs(get_impl_options.column_family);
    if (!s.ok()) {
      return s;
    }
@@ -65,67 +53,99 @@ Status DBImplReadOnly::Get(const ReadOptions& read_options,

  // Clear the timestamps for returning results so that we can distinguish
  // between tombstone or key that has never been written
-  if (timestamp) {
-    timestamp->clear();
+  if (get_impl_options.timestamp) {
+    get_impl_options.timestamp->clear();
  }

-  const Comparator* ucmp = column_family->GetComparator();
-  assert(ucmp);
-  std::string* ts = ucmp->timestamp_size() > 0 ? timestamp : nullptr;
+  PERF_CPU_TIMER_GUARD(get_cpu_nanos, immutable_db_options_.clock);
+  StopWatch sw(immutable_db_options_.clock, stats_, DB_GET);
+  PERF_TIMER_GUARD(get_snapshot_time);

-  Status s;
+  const Comparator* ucmp = get_impl_options.column_family->GetComparator();
+  assert(ucmp);
+  std::string* ts =
+      ucmp->timestamp_size() > 0 ? get_impl_options.timestamp : nullptr;
  SequenceNumber snapshot = versions_->LastSequence();
  GetWithTimestampReadCallback read_cb(snapshot);
-  auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(column_family);
+  auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(
+      get_impl_options.column_family);
  auto cfd = cfh->cfd();
  if (tracer_) {
    InstrumentedMutexLock lock(&trace_mutex_);
    if (tracer_) {
-      tracer_->Get(column_family, key);
+      tracer_->Get(get_impl_options.column_family, key);
    }
  }
+
+  // In read-only mode Get(), no super version operation is needed (i.e.
+  // GetAndRefSuperVersion and ReturnAndCleanupSuperVersion)
  SuperVersion* super_version = cfd->GetSuperVersion();
+  if (read_options.timestamp && read_options.timestamp->size() > 0) {
+    s = FailIfReadCollapsedHistory(cfd, super_version,
+                                   *(read_options.timestamp));
+    if (!s.ok()) {
+      return s;
+    }
+  }
  MergeContext merge_context;
  SequenceNumber max_covering_tombstone_seq = 0;
  LookupKey lkey(key, snapshot, read_options.timestamp);
  PERF_TIMER_STOP(get_snapshot_time);
-  if (super_version->mem->Get(lkey, pinnable_val->GetSelf(),
-                              /*columns=*/nullptr, ts, &s, &merge_context,
-                              &max_covering_tombstone_seq, read_options,
-                              false /* immutable_memtable */, &read_cb)) {
-    pinnable_val->PinSelf();
+
+  // Look up starts here
+  if (super_version->mem->Get(
+          lkey,
+          get_impl_options.value ? get_impl_options.value->GetSelf() : nullptr,
+          get_impl_options.columns, ts, &s, &merge_context,
+          &max_covering_tombstone_seq, read_options,
+          false /* immutable_memtable */, &read_cb)) {
+    if (get_impl_options.value) {
+      get_impl_options.value->PinSelf();
+    }
    RecordTick(stats_, MEMTABLE_HIT);
  } else {
    PERF_TIMER_GUARD(get_from_output_files_time);
    PinnedIteratorsManager pinned_iters_mgr;
    super_version->current->Get(
-        read_options, lkey, pinnable_val, /*columns=*/nullptr, ts, &s,
-        &merge_context, &max_covering_tombstone_seq, &pinned_iters_mgr,
+        read_options, lkey, get_impl_options.value, get_impl_options.columns,
+        ts, &s, &merge_context, &max_covering_tombstone_seq, &pinned_iters_mgr,
        /*value_found*/ nullptr,
        /*key_exists*/ nullptr, /*seq*/ nullptr, &read_cb,
        /*is_blob*/ nullptr,
        /*do_merge*/ true);
    RecordTick(stats_, MEMTABLE_MISS);
  }
-  RecordTick(stats_, NUMBER_KEYS_READ);
-  size_t size = pinnable_val->size();
-  RecordTick(stats_, BYTES_READ, size);
-  RecordInHistogram(stats_, BYTES_PER_READ, size);
-  PERF_COUNTER_ADD(get_read_bytes, size);
+  {
+    RecordTick(stats_, NUMBER_KEYS_READ);
+    size_t size = 0;
+    if (get_impl_options.value) {
+      size = get_impl_options.value->size();
+    } else if (get_impl_options.columns) {
+      size = get_impl_options.columns->serialized_size();
+    }
+    RecordTick(stats_, BYTES_READ, size);
+    RecordInHistogram(stats_, BYTES_PER_READ, size);
+    PERF_COUNTER_ADD(get_read_bytes, size);
+  }
  return s;
 }

-Iterator* DBImplReadOnly::NewIterator(const ReadOptions& read_options,
+Iterator* DBImplReadOnly::NewIterator(const ReadOptions& _read_options,
                                      ColumnFamilyHandle* column_family) {
-  if (read_options.io_activity != Env::IOActivity::kUnknown) {
+  if (_read_options.io_activity != Env::IOActivity::kUnknown &&
+      _read_options.io_activity != Env::IOActivity::kDBIterator) {
    return NewErrorIterator(Status::InvalidArgument(
-        "Cannot call NewIterator with `ReadOptions::io_activity` != "
-        "`Env::IOActivity::kUnknown`"));
+        "Can only call NewIterator with `ReadOptions::io_activity` is "
+        "`Env::IOActivity::kUnknown` or `Env::IOActivity::kDBIterator`"));
+  }
+  ReadOptions read_options(_read_options);
+  if (read_options.io_activity == Env::IOActivity::kUnknown) {
+    read_options.io_activity = Env::IOActivity::kDBIterator;
  }
  assert(column_family);
  if (read_options.timestamp) {
-    const Status s = FailIfTsMismatchCf(
-        column_family, *(read_options.timestamp), /*ts_for_read=*/true);
+    const Status s =
+        FailIfTsMismatchCf(column_family, *(read_options.timestamp));
    if (!s.ok()) {
      return NewErrorIterator(s);
    }
@@ -138,6 +158,14 @@ Iterator* DBImplReadOnly::NewIterator(const ReadOptions& read_options,
  auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(column_family);
  auto cfd = cfh->cfd();
  SuperVersion* super_version = cfd->GetSuperVersion()->Ref();
+  if (read_options.timestamp && read_options.timestamp->size() > 0) {
+    const Status s = FailIfReadCollapsedHistory(cfd, super_version,
+                                                *(read_options.timestamp));
+    if (!s.ok()) {
+      cfd->GetSuperVersion()->Unref();
+      return NewErrorIterator(s);
+    }
+  }
  SequenceNumber latest_snapshot = versions_->LastSequence();
  SequenceNumber read_seq =
      read_options.snapshot != nullptr
@@ -164,8 +192,7 @@ Status DBImplReadOnly::NewIterators(
  if (read_options.timestamp) {
    for (auto* cf : column_families) {
      assert(cf);
-      const Status s = FailIfTsMismatchCf(cf, *(read_options.timestamp),
-                                          /*ts_for_read=*/true);
+      const Status s = FailIfTsMismatchCf(cf, *(read_options.timestamp));
      if (!s.ok()) {
        return s;
      }
@@ -193,9 +220,27 @@ Status DBImplReadOnly::NewIterators(
                ->number_
          : latest_snapshot;

+  autovector<std::tuple<ColumnFamilyData*, SuperVersion*>> cfd_to_sv;
+
+  const bool check_read_ts =
+      read_options.timestamp && read_options.timestamp->size() > 0;
  for (auto cfh : column_families) {
    auto* cfd = static_cast_with_check<ColumnFamilyHandleImpl>(cfh)->cfd();
    auto* sv = cfd->GetSuperVersion()->Ref();
+    cfd_to_sv.emplace_back(cfd, sv);
+    if (check_read_ts) {
+      const Status s =
+          FailIfReadCollapsedHistory(cfd, sv, *(read_options.timestamp));
+      if (!s.ok()) {
+        for (auto prev_entry : cfd_to_sv) {
+          std::get<1>(prev_entry)->Unref();
+        }
+        return s;
+      }
+    }
+  }
+  assert(cfd_to_sv.size() == column_families.size());
+  for (auto [cfd, sv] : cfd_to_sv) {
    auto* db_iter = NewArenaWrappedDbIterator(
        env_, read_options, *cfd->ioptions(), sv->mutable_cf_options,
        sv->current, read_seq,
@@ -24,18 +24,14 @@ class DBImplReadOnly : public DBImpl {
  virtual ~DBImplReadOnly();

  // Implementations of the DB interface
-  using DB::Get;
-  virtual Status Get(const ReadOptions& options,
-                     ColumnFamilyHandle* column_family, const Slice& key,
-                     PinnableSlice* value) override;
-  Status Get(const ReadOptions& options, ColumnFamilyHandle* column_family,
-             const Slice& key, PinnableSlice* value,
-             std::string* timestamp) override;
+  using DBImpl::GetImpl;
+  Status GetImpl(const ReadOptions& options, const Slice& key,
+                 GetImplOptions& get_impl_options) override;

  // TODO: Implement ReadOnly MultiGet?

  using DBImpl::NewIterator;
-  virtual Iterator* NewIterator(const ReadOptions&,
+  virtual Iterator* NewIterator(const ReadOptions& _read_options,
                                ColumnFamilyHandle* column_family) override;

  virtual Status NewIterators(
@@ -142,6 +138,15 @@ class DBImplReadOnly : public DBImpl {
    return Status::NotSupported("Not supported operation in read only mode.");
  }

+  virtual Status CreateColumnFamilyWithImport(
+      const ColumnFamilyOptions& /*options*/,
+      const std::string& /*column_family_name*/,
+      const ImportColumnFamilyOptions& /*import_options*/,
+      const std::vector<const ExportImportFilesMetaData*>& /*metadatas*/,
+      ColumnFamilyHandle** /*handle*/) override {
+    return Status::NotSupported("Not supported operation in read only mode.");
+  }
+
  using DB::ClipColumnFamily;
  virtual Status ClipColumnFamily(ColumnFamilyHandle* /*column_family*/,
                                  const Slice& /*begin*/,
@@ -170,4 +175,3 @@ class DBImplReadOnly : public DBImpl {
  friend class DB;
 };
 }  // namespace ROCKSDB_NAMESPACE
-
@@ -14,6 +14,7 @@
 #include "monitoring/perf_context_imp.h"
 #include "rocksdb/configurable.h"
 #include "util/cast_util.h"
+#include "util/write_batch_util.h"

 namespace ROCKSDB_NAMESPACE {

@@ -197,6 +198,9 @@ Status DBImplSecondary::RecoverLogFiles(
    }
    assert(reader != nullptr);
  }
+
+  const UnorderedMap<uint32_t, size_t>& running_ts_sz =
+      versions_->GetRunningColumnFamiliesTimestampSize();
  for (auto log_number : log_numbers) {
    auto it = log_readers_.find(log_number);
    assert(it != log_readers_.end());
@@ -224,6 +228,14 @@ Status DBImplSecondary::RecoverLogFiles(
      if (!status.ok()) {
        break;
      }
+      const UnorderedMap<uint32_t, size_t>& record_ts_sz =
+          reader->GetRecordedTimestampSize();
+      status = HandleWriteBatchTimestampSizeDifference(
+          &batch, running_ts_sz, record_ts_sz,
+          TimestampSizeConsistencyMode::kVerifyConsistency);
+      if (!status.ok()) {
+        break;
+      }
      SequenceNumber seq_of_batch = WriteBatchInternal::Sequence(&batch);
      std::vector<uint32_t> column_family_ids;
      status = CollectColumnFamilyIdsFromWriteBatch(batch, &column_family_ids);
@@ -327,90 +339,93 @@ Status DBImplSecondary::RecoverLogFiles(
  return status;
 }

-// Implementation of the DB interface
-Status DBImplSecondary::Get(const ReadOptions& read_options,
-                            ColumnFamilyHandle* column_family, const Slice& key,
-                            PinnableSlice* value) {
-  return GetImpl(read_options, column_family, key, value,
-                 /*timestamp*/ nullptr);
-}
-
-Status DBImplSecondary::Get(const ReadOptions& read_options,
-                            ColumnFamilyHandle* column_family, const Slice& key,
-                            PinnableSlice* value, std::string* timestamp) {
-  return GetImpl(read_options, column_family, key, value, timestamp);
-}
-
 Status DBImplSecondary::GetImpl(const ReadOptions& read_options,
-                                ColumnFamilyHandle* column_family,
-                                const Slice& key, PinnableSlice* pinnable_val,
-                                std::string* timestamp) {
-  if (read_options.io_activity != Env::IOActivity::kUnknown) {
-    return Status::InvalidArgument(
-        "Cannot call Get with `ReadOptions::io_activity` != "
-        "`Env::IOActivity::kUnknown`");
-  }
-  assert(pinnable_val != nullptr);
-  PERF_CPU_TIMER_GUARD(get_cpu_nanos, immutable_db_options_.clock);
-  StopWatch sw(immutable_db_options_.clock, stats_, DB_GET);
-  PERF_TIMER_GUARD(get_snapshot_time);
+                                const Slice& key,
+                                GetImplOptions& get_impl_options) {
+  assert(get_impl_options.value != nullptr ||
+         get_impl_options.columns != nullptr);
+  assert(get_impl_options.column_family);
+
+  Status s;

-  assert(column_family);
  if (read_options.timestamp) {
-    const Status s = FailIfTsMismatchCf(
-        column_family, *(read_options.timestamp), /*ts_for_read=*/true);
+    s = FailIfTsMismatchCf(get_impl_options.column_family,
+                           *(read_options.timestamp));
    if (!s.ok()) {
      return s;
    }
  } else {
-    const Status s = FailIfCfHasTs(column_family);
+    s = FailIfCfHasTs(get_impl_options.column_family);
    if (!s.ok()) {
      return s;
    }
  }

-  // Clear the timestamp for returning results so that we can distinguish
-  // between tombstone or key that has never been written later.
-  if (timestamp) {
-    timestamp->clear();
+  // Clear the timestamps for returning results so that we can distinguish
+  // between tombstone or key that has never been written
+  if (get_impl_options.timestamp) {
+    get_impl_options.timestamp->clear();
  }

-  auto cfh = static_cast<ColumnFamilyHandleImpl*>(column_family);
-  ColumnFamilyData* cfd = cfh->cfd();
+  PERF_CPU_TIMER_GUARD(get_cpu_nanos, immutable_db_options_.clock);
+  StopWatch sw(immutable_db_options_.clock, stats_, DB_GET);
+  PERF_TIMER_GUARD(get_snapshot_time);
+
+  const Comparator* ucmp = get_impl_options.column_family->GetComparator();
+  assert(ucmp);
+  std::string* ts =
+      ucmp->timestamp_size() > 0 ? get_impl_options.timestamp : nullptr;
+  SequenceNumber snapshot = versions_->LastSequence();
+  GetWithTimestampReadCallback read_cb(snapshot);
+  auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(
+      get_impl_options.column_family);
+  auto cfd = cfh->cfd();
  if (tracer_) {
    InstrumentedMutexLock lock(&trace_mutex_);
    if (tracer_) {
-      tracer_->Get(column_family, key);
+      tracer_->Get(get_impl_options.column_family, key);
    }
  }
+
  // Acquire SuperVersion
  SuperVersion* super_version = GetAndRefSuperVersion(cfd);
-  SequenceNumber snapshot = versions_->LastSequence();
-  GetWithTimestampReadCallback read_cb(snapshot);
+  if (read_options.timestamp && read_options.timestamp->size() > 0) {
+    s = FailIfReadCollapsedHistory(cfd, super_version,
+                                   *(read_options.timestamp));
+    if (!s.ok()) {
+      ReturnAndCleanupSuperVersion(cfd, super_version);
+      return s;
+    }
+  }
  MergeContext merge_context;
  SequenceNumber max_covering_tombstone_seq = 0;
-  Status s;
  LookupKey lkey(key, snapshot, read_options.timestamp);
  PERF_TIMER_STOP(get_snapshot_time);
-
  bool done = false;
-  const Comparator* ucmp = column_family->GetComparator();
-  assert(ucmp);
-  std::string* ts = ucmp->timestamp_size() > 0 ? timestamp : nullptr;
-  if (super_version->mem->Get(lkey, pinnable_val->GetSelf(),
-                              /*columns=*/nullptr, ts, &s, &merge_context,
-                              &max_covering_tombstone_seq, read_options,
-                              false /* immutable_memtable */, &read_cb)) {
+
+  // Look up starts here
+  if (super_version->mem->Get(
+          lkey,
+          get_impl_options.value ? get_impl_options.value->GetSelf() : nullptr,
+          get_impl_options.columns, ts, &s, &merge_context,
+          &max_covering_tombstone_seq, read_options,
+          false /* immutable_memtable */, &read_cb)) {
    done = true;
-    pinnable_val->PinSelf();
+    if (get_impl_options.value) {
+      get_impl_options.value->PinSelf();
+    }
    RecordTick(stats_, MEMTABLE_HIT);
  } else if ((s.ok() || s.IsMergeInProgress()) &&
             super_version->imm->Get(
-                 lkey, pinnable_val->GetSelf(), /*columns=*/nullptr, ts, &s,
-                 &merge_context, &max_covering_tombstone_seq, read_options,
-                 &read_cb)) {
+                 lkey,
+                 get_impl_options.value ? get_impl_options.value->GetSelf()
+                                        : nullptr,
+                 get_impl_options.columns, ts, &s, &merge_context,
+                 &max_covering_tombstone_seq, read_options, &read_cb)) {
    done = true;
-    pinnable_val->PinSelf();
+    if (get_impl_options.value) {
+      get_impl_options.value->PinSelf();
+    }
    RecordTick(stats_, MEMTABLE_HIT);
  }
  if (!done && !s.ok() && !s.IsMergeInProgress()) {
@@ -421,8 +436,8 @@ Status DBImplSecondary::GetImpl(const ReadOptions& read_options,
    PERF_TIMER_GUARD(get_from_output_files_time);
    PinnedIteratorsManager pinned_iters_mgr;
    super_version->current->Get(
-        read_options, lkey, pinnable_val, /*columns=*/nullptr, ts, &s,
-        &merge_context, &max_covering_tombstone_seq, &pinned_iters_mgr,
+        read_options, lkey, get_impl_options.value, get_impl_options.columns,
+        ts, &s, &merge_context, &max_covering_tombstone_seq, &pinned_iters_mgr,
        /*value_found*/ nullptr,
        /*key_exists*/ nullptr, /*seq*/ nullptr, &read_cb, /*is_blob*/ nullptr,
        /*do_merge*/ true);
@@ -432,7 +447,12 @@ Status DBImplSecondary::GetImpl(const ReadOptions& read_options,
    PERF_TIMER_GUARD(get_post_process_time);
    ReturnAndCleanupSuperVersion(cfd, super_version);
    RecordTick(stats_, NUMBER_KEYS_READ);
-    size_t size = pinnable_val->size();
+    size_t size = 0;
+    if (get_impl_options.value) {
+      size = get_impl_options.value->size();
+    } else if (get_impl_options.columns) {
+      size = get_impl_options.columns->serialized_size();
+    }
    RecordTick(stats_, BYTES_READ, size);
    RecordTimeToHistogram(stats_, BYTES_PER_READ, size);
    PERF_COUNTER_ADD(get_read_bytes, size);
@@ -440,8 +460,18 @@ Status DBImplSecondary::GetImpl(const ReadOptions& read_options,
  return s;
 }

-Iterator* DBImplSecondary::NewIterator(const ReadOptions& read_options,
+Iterator* DBImplSecondary::NewIterator(const ReadOptions& _read_options,
                                       ColumnFamilyHandle* column_family) {
+  if (_read_options.io_activity != Env::IOActivity::kUnknown &&
+      _read_options.io_activity != Env::IOActivity::kDBIterator) {
+    return NewErrorIterator(Status::InvalidArgument(
+        "Can only call NewIterator with `ReadOptions::io_activity` is "
+        "`Env::IOActivity::kUnknown` or `Env::IOActivity::kDBIterator`"));
+  }
+  ReadOptions read_options(_read_options);
+  if (read_options.io_activity == Env::IOActivity::kUnknown) {
+    read_options.io_activity = Env::IOActivity::kDBIterator;
+  }
  if (read_options.managed) {
    return NewErrorIterator(
        Status::NotSupported("Managed iterator is not supported anymore."));
@@ -450,16 +480,11 @@ Iterator* DBImplSecondary::NewIterator(const ReadOptions& read_options,
    return NewErrorIterator(Status::NotSupported(
        "ReadTier::kPersistedData is not yet supported in iterators."));
  }
-  if (read_options.io_activity != Env::IOActivity::kUnknown) {
-    return NewErrorIterator(Status::InvalidArgument(
-        "Cannot call NewIterator with `ReadOptions::io_activity` != "
-        "`Env::IOActivity::kUnknown`"));
-  }

  assert(column_family);
  if (read_options.timestamp) {
-    const Status s = FailIfTsMismatchCf(
-        column_family, *(read_options.timestamp), /*ts_for_read=*/true);
+    const Status s =
+        FailIfTsMismatchCf(column_family, *(read_options.timestamp));
    if (!s.ok()) {
      return NewErrorIterator(s);
    }
@@ -483,17 +508,25 @@ Iterator* DBImplSecondary::NewIterator(const ReadOptions& read_options,
        Status::NotSupported("snapshot not supported in secondary mode"));
  } else {
    SequenceNumber snapshot(kMaxSequenceNumber);
-    result = NewIteratorImpl(read_options, cfd, snapshot, read_callback);
+    SuperVersion* sv = cfd->GetReferencedSuperVersion(this);
+    if (read_options.timestamp && read_options.timestamp->size() > 0) {
+      const Status s =
+          FailIfReadCollapsedHistory(cfd, sv, *(read_options.timestamp));
+      if (!s.ok()) {
+        CleanupSuperVersion(sv);
+        return NewErrorIterator(s);
+      }
+    }
+    result = NewIteratorImpl(read_options, cfd, sv, snapshot, read_callback);
  }
  return result;
 }

 ArenaWrappedDBIter* DBImplSecondary::NewIteratorImpl(
    const ReadOptions& read_options, ColumnFamilyData* cfd,
-    SequenceNumber snapshot, ReadCallback* read_callback,
-    bool expose_blob_index, bool allow_refresh) {
+    SuperVersion* super_version, SequenceNumber snapshot,
+    ReadCallback* read_callback, bool expose_blob_index, bool allow_refresh) {
  assert(nullptr != cfd);
-  SuperVersion* super_version = cfd->GetReferencedSuperVersion(this);
  assert(snapshot == kMaxSequenceNumber);
  snapshot = versions_->LastSequence();
  assert(snapshot != kMaxSequenceNumber);
@@ -502,7 +535,7 @@ ArenaWrappedDBIter* DBImplSecondary::NewIteratorImpl(
      super_version->current, snapshot,
      super_version->mutable_cf_options.max_sequential_skip_in_iterations,
      super_version->version_number, read_callback, this, cfd,
-      expose_blob_index, read_options.snapshot ? false : allow_refresh);
+      expose_blob_index, allow_refresh);
  auto internal_iter = NewInternalIterator(
      db_iter->GetReadOptions(), cfd, super_version, db_iter->GetArena(),
      snapshot, /* allow_unprepared_value */ true, db_iter);
@@ -511,9 +544,19 @@ ArenaWrappedDBIter* DBImplSecondary::NewIteratorImpl(
 }

 Status DBImplSecondary::NewIterators(
-    const ReadOptions& read_options,
+    const ReadOptions& _read_options,
    const std::vector<ColumnFamilyHandle*>& column_families,
    std::vector<Iterator*>* iterators) {
+  if (_read_options.io_activity != Env::IOActivity::kUnknown &&
+      _read_options.io_activity != Env::IOActivity::kDBIterator) {
+    return Status::InvalidArgument(
+        "Can only call NewIterators with `ReadOptions::io_activity` is "
+        "`Env::IOActivity::kUnknown` or `Env::IOActivity::kDBIterator`");
+  }
+  ReadOptions read_options(_read_options);
+  if (read_options.io_activity == Env::IOActivity::kUnknown) {
+    read_options.io_activity = Env::IOActivity::kDBIterator;
+  }
  if (read_options.managed) {
    return Status::NotSupported("Managed iterator is not supported anymore.");
  }
@@ -521,11 +564,6 @@ Status DBImplSecondary::NewIterators(
    return Status::NotSupported(
        "ReadTier::kPersistedData is not yet supported in iterators.");
  }
-  if (read_options.io_activity != Env::IOActivity::kUnknown) {
-    return Status::InvalidArgument(
-        "Cannot call NewIterators with `ReadOptions::io_activity` != "
-        "`Env::IOActivity::kUnknown`");
-  }
  ReadCallback* read_callback = nullptr;  // No read callback provided.
  if (iterators == nullptr) {
    return Status::InvalidArgument("iterators not allowed to be nullptr");
@@ -534,8 +572,7 @@ Status DBImplSecondary::NewIterators(
  if (read_options.timestamp) {
    for (auto* cf : column_families) {
      assert(cf);
-      const Status s = FailIfTsMismatchCf(cf, *(read_options.timestamp),
-                                          /*ts_for_read=*/true);
+      const Status s = FailIfTsMismatchCf(cf, *(read_options.timestamp));
      if (!s.ok()) {
        return s;
      }
@@ -559,10 +596,28 @@ Status DBImplSecondary::NewIterators(
    return Status::NotSupported("snapshot not supported in secondary mode");
  } else {
    SequenceNumber read_seq(kMaxSequenceNumber);
+    autovector<std::tuple<ColumnFamilyData*, SuperVersion*>> cfd_to_sv;
+    const bool check_read_ts =
+        read_options.timestamp && read_options.timestamp->size() > 0;
    for (auto cfh : column_families) {
      ColumnFamilyData* cfd = static_cast<ColumnFamilyHandleImpl*>(cfh)->cfd();
+      SuperVersion* sv = cfd->GetReferencedSuperVersion(this);
+      cfd_to_sv.emplace_back(cfd, sv);
+      if (check_read_ts) {
+        const Status s =
+            FailIfReadCollapsedHistory(cfd, sv, *(read_options.timestamp));
+        if (!s.ok()) {
+          for (auto prev_entry : cfd_to_sv) {
+            CleanupSuperVersion(std::get<1>(prev_entry));
+          }
+          return s;
+        }
+      }
+    }
+    assert(cfd_to_sv.size() == column_families.size());
+    for (auto [cfd, sv] : cfd_to_sv) {
      iterators->push_back(
-          NewIteratorImpl(read_options, cfd, read_seq, read_callback));
+          NewIteratorImpl(read_options, cfd, sv, read_seq, read_callback));
    }
  }
  return Status::OK();
@@ -830,7 +885,7 @@ Status DBImplSecondary::CompactWithoutInstallation(
      *mutable_cf_options, mutable_db_options_, 0));
  assert(c != nullptr);

-  c->SetInputVersion(version);
+  c->FinalizeInputInfo(version);

  // Create output directory if it's not existed yet
  std::unique_ptr<FSDirectory> output_dir;
@@ -948,6 +1003,8 @@ Status DB::OpenAndCompact(
  delete db;
  if (s.ok()) {
    return serialization_status;
+  } else {
+    serialization_status.PermitUncheckedError();
  }
  return s;
 }
@@ -85,8 +85,6 @@ class DBImplSecondary : public DBImpl {
                 bool error_if_data_exists_in_wals, uint64_t* = nullptr,
                 RecoveryContext* recovery_ctx = nullptr) override;

-  // Implementations of the DB interface.
-  using DB::Get;
  // Can return IOError due to files being deleted by the primary. To avoid
  // IOError in this case, application can coordinate between primary and
  // secondaries so that primary will not delete files that are currently being
@@ -96,16 +94,9 @@ class DBImplSecondary : public DBImpl {
  // workaround, the secondaries can be opened with `max_open_files=-1` so that
  // it eagerly keeps all talbe files open and is able to access the contents of
  // deleted files via prior open fd.
-  Status Get(const ReadOptions& options, ColumnFamilyHandle* column_family,
-             const Slice& key, PinnableSlice* value) override;
-
-  Status Get(const ReadOptions& options, ColumnFamilyHandle* column_family,
-             const Slice& key, PinnableSlice* value,
-             std::string* timestamp) override;
-
-  Status GetImpl(const ReadOptions& options, ColumnFamilyHandle* column_family,
-                 const Slice& key, PinnableSlice* value,
-                 std::string* timestamp);
+  using DBImpl::GetImpl;
+  Status GetImpl(const ReadOptions& options, const Slice& key,
+                 GetImplOptions& get_impl_options) override;

  using DBImpl::NewIterator;
  // Operations on the created iterators can return IOError due to files being
@@ -117,17 +108,17 @@ class DBImplSecondary : public DBImpl {
  // deleted. As a partial hacky workaround, the secondaries can be opened with
  // `max_open_files=-1` so that it eagerly keeps all talbe files open and is
  // able to access the contents of deleted files via prior open fd.
-  Iterator* NewIterator(const ReadOptions&,
+  Iterator* NewIterator(const ReadOptions& _read_options,
                        ColumnFamilyHandle* column_family) override;

  ArenaWrappedDBIter* NewIteratorImpl(const ReadOptions& read_options,
-                                      ColumnFamilyData* cfd,
+                                      ColumnFamilyData* cfd, SuperVersion* sv,
                                      SequenceNumber snapshot,
                                      ReadCallback* read_callback,
                                      bool expose_blob_index = false,
                                      bool allow_refresh = true);

-  Status NewIterators(const ReadOptions& options,
+  Status NewIterators(const ReadOptions& _read_options,
                      const std::vector<ColumnFamilyHandle*>& column_families,
                      std::vector<Iterator*>* iterators) override;

@@ -273,85 +264,6 @@ class DBImplSecondary : public DBImpl {
    return Status::OK();
  }

-  // ColumnFamilyCollector is a write batch handler which does nothing
-  // except recording unique column family IDs
-  class ColumnFamilyCollector : public WriteBatch::Handler {
-    std::unordered_set<uint32_t> column_family_ids_;
-
-    Status AddColumnFamilyId(uint32_t column_family_id) {
-      if (column_family_ids_.find(column_family_id) ==
-          column_family_ids_.end()) {
-        column_family_ids_.insert(column_family_id);
-      }
-      return Status::OK();
-    }
-
-   public:
-    explicit ColumnFamilyCollector() {}
-
-    ~ColumnFamilyCollector() override {}
-
-    Status PutCF(uint32_t column_family_id, const Slice&,
-                 const Slice&) override {
-      return AddColumnFamilyId(column_family_id);
-    }
-
-    Status DeleteCF(uint32_t column_family_id, const Slice&) override {
-      return AddColumnFamilyId(column_family_id);
-    }
-
-    Status SingleDeleteCF(uint32_t column_family_id, const Slice&) override {
-      return AddColumnFamilyId(column_family_id);
-    }
-
-    Status DeleteRangeCF(uint32_t column_family_id, const Slice&,
-                         const Slice&) override {
-      return AddColumnFamilyId(column_family_id);
-    }
-
-    Status MergeCF(uint32_t column_family_id, const Slice&,
-                   const Slice&) override {
-      return AddColumnFamilyId(column_family_id);
-    }
-
-    Status PutBlobIndexCF(uint32_t column_family_id, const Slice&,
-                          const Slice&) override {
-      return AddColumnFamilyId(column_family_id);
-    }
-
-    Status MarkBeginPrepare(bool) override { return Status::OK(); }
-
-    Status MarkEndPrepare(const Slice&) override { return Status::OK(); }
-
-    Status MarkRollback(const Slice&) override { return Status::OK(); }
-
-    Status MarkCommit(const Slice&) override { return Status::OK(); }
-
-    Status MarkCommitWithTimestamp(const Slice&, const Slice&) override {
-      return Status::OK();
-    }
-
-    Status MarkNoop(bool) override { return Status::OK(); }
-
-    const std::unordered_set<uint32_t>& column_families() const {
-      return column_family_ids_;
-    }
-  };
-
-  Status CollectColumnFamilyIdsFromWriteBatch(
-      const WriteBatch& batch, std::vector<uint32_t>* column_family_ids) {
-    assert(column_family_ids != nullptr);
-    column_family_ids->clear();
-    ColumnFamilyCollector handler;
-    Status s = batch.Iterate(&handler);
-    if (s.ok()) {
-      for (const auto& cf : handler.column_families()) {
-        column_family_ids->push_back(cf);
-      }
-    }
-    return s;
-  }
-
  bool OwnTablesAndLogs() const override {
    // Currently, the secondary instance does not own the database files. It
    // simply opens the files of the primary instance and tracks their file
@@ -403,4 +315,3 @@ class DBImplSecondary : public DBImpl {
 };

 }  // namespace ROCKSDB_NAMESPACE
-
@@ -30,7 +30,7 @@ Status DBImpl::Put(const WriteOptions& o, ColumnFamilyHandle* column_family,

 Status DBImpl::Put(const WriteOptions& o, ColumnFamilyHandle* column_family,
                   const Slice& key, const Slice& ts, const Slice& val) {
-  const Status s = FailIfTsMismatchCf(column_family, ts, /*ts_for_read=*/false);
+  const Status s = FailIfTsMismatchCf(column_family, ts);
  if (!s.ok()) {
    return s;
  }
@@ -64,7 +64,7 @@ Status DBImpl::Merge(const WriteOptions& o, ColumnFamilyHandle* column_family,

 Status DBImpl::Merge(const WriteOptions& o, ColumnFamilyHandle* column_family,
                     const Slice& key, const Slice& ts, const Slice& val) {
-  const Status s = FailIfTsMismatchCf(column_family, ts, /*ts_for_read=*/false);
+  const Status s = FailIfTsMismatchCf(column_family, ts);
  if (!s.ok()) {
    return s;
  }
@@ -83,7 +83,7 @@ Status DBImpl::Delete(const WriteOptions& write_options,
 Status DBImpl::Delete(const WriteOptions& write_options,
                      ColumnFamilyHandle* column_family, const Slice& key,
                      const Slice& ts) {
-  const Status s = FailIfTsMismatchCf(column_family, ts, /*ts_for_read=*/false);
+  const Status s = FailIfTsMismatchCf(column_family, ts);
  if (!s.ok()) {
    return s;
  }
@@ -103,7 +103,7 @@ Status DBImpl::SingleDelete(const WriteOptions& write_options,
 Status DBImpl::SingleDelete(const WriteOptions& write_options,
                            ColumnFamilyHandle* column_family, const Slice& key,
                            const Slice& ts) {
-  const Status s = FailIfTsMismatchCf(column_family, ts, /*ts_for_read=*/false);
+  const Status s = FailIfTsMismatchCf(column_family, ts);
  if (!s.ok()) {
    return s;
  }
@@ -124,7 +124,7 @@ Status DBImpl::DeleteRange(const WriteOptions& write_options,
                           ColumnFamilyHandle* column_family,
                           const Slice& begin_key, const Slice& end_key,
                           const Slice& ts) {
-  const Status s = FailIfTsMismatchCf(column_family, ts, /*ts_for_read=*/false);
+  const Status s = FailIfTsMismatchCf(column_family, ts);
  if (!s.ok()) {
    return s;
  }
@@ -1328,7 +1328,13 @@ IOStatus DBImpl::WriteToWAL(const WriteBatch& merged_batch,
  if (UNLIKELY(needs_locking)) {
    log_write_mutex_.Lock();
  }
-  IOStatus io_s = log_writer->AddRecord(log_entry, rate_limiter_priority);
+  IOStatus io_s = log_writer->MaybeAddUserDefinedTimestampSizeRecord(
+      versions_->GetColumnFamiliesTimestampSizeForRecord(),
+      rate_limiter_priority);
+  if (!io_s.ok()) {
+    return io_s;
+  }
+  io_s = log_writer->AddRecord(log_entry, rate_limiter_priority);

  if (UNLIKELY(needs_locking)) {
    log_write_mutex_.Unlock();
@@ -34,6 +34,12 @@ void DumpDBFileSummary(const ImmutableDBOptions& options,
  std::string file_info, wal_info;

  Header(options.info_log, "DB SUMMARY\n");
+  {
+    std::string hostname;
+    if (env->GetHostNameString(&hostname).ok()) {
+      Header(options.info_log, "Host name (Env):  %s\n", hostname.c_str());
+    }
+  }
  Header(options.info_log, "DB Session ID:  %s\n", session_id.c_str());

  Status s;
@@ -18,6 +18,7 @@
 #include "db/merge_helper.h"
 #include "db/pinned_iterators_manager.h"
 #include "db/wide/wide_column_serialization.h"
+#include "db/wide/wide_columns_helper.h"
 #include "file/filename.h"
 #include "logging/logging.h"
 #include "memory/arena.h"
@@ -77,11 +78,13 @@ DBIter::DBIter(Env* _env, const ReadOptions& read_options,
      expose_blob_index_(expose_blob_index),
      is_blob_(false),
      arena_mode_(arena_mode),
+      io_activity_(read_options.io_activity),
      db_impl_(db_impl),
      cfd_(cfd),
      timestamp_ub_(read_options.timestamp),
      timestamp_lb_(read_options.iter_start_ts),
-      timestamp_size_(timestamp_ub_ ? timestamp_ub_->size() : 0) {
+      timestamp_size_(timestamp_ub_ ? timestamp_ub_->size() : 0),
+      auto_readahead_size_(read_options.auto_readahead_size) {
  RecordTick(statistics_, NO_ITERATOR_CREATED);
  if (pin_thru_lifetime_) {
    pinned_iters_mgr_.StartPinning();
@@ -196,12 +199,11 @@ bool DBIter::SetBlobValueIfNeeded(const Slice& user_key,

  // TODO: consider moving ReadOptions from ArenaWrappedDBIter to DBIter to
  // avoid having to copy options back and forth.
-  // TODO: plumb Env::IOActivity
  ReadOptions read_options;
  read_options.read_tier = read_tier_;
  read_options.fill_cache = fill_cache_;
  read_options.verify_checksums = verify_checksums_;
-
+  read_options.io_activity = io_activity_;
  constexpr FilePrefetchBuffer* prefetch_buffer = nullptr;
  constexpr uint64_t* bytes_read = nullptr;

@@ -230,14 +232,38 @@ bool DBIter::SetValueAndColumnsFromEntity(Slice slice) {
    return false;
  }

-  if (!wide_columns_.empty() &&
-      wide_columns_[0].name() == kDefaultWideColumnName) {
-    value_ = wide_columns_[0].value();
+  if (WideColumnsHelper::HasDefaultColumn(wide_columns_)) {
+    value_ = WideColumnsHelper::GetDefaultColumn(wide_columns_);
  }

  return true;
 }

+bool DBIter::SetValueAndColumnsFromMergeResult(const Status& merge_status,
+                                               ValueType result_type) {
+  if (!merge_status.ok()) {
+    valid_ = false;
+    status_ = merge_status;
+    return false;
+  }
+
+  if (result_type == kTypeWideColumnEntity) {
+    if (!SetValueAndColumnsFromEntity(saved_value_)) {
+      assert(!valid_);
+      return false;
+    }
+
+    valid_ = true;
+    return true;
+  }
+
+  assert(result_type == kTypeValue);
+  SetValueAndColumnsFromPlain(pinned_value_.data() ? pinned_value_
+                                                   : saved_value_);
+  valid_ = true;
+  return true;
+}
+
 // PRE: saved_key_ has the current user key if skipping_saved_key
 // POST: saved_key_ should have the next user key if valid_,
 //       if the current entry is a result of merge
@@ -554,8 +580,7 @@ bool DBIter::MergeValuesNewToOld() {
    if (kTypeValue == ikey.type) {
      // hit a put, merge the put value with operands and store the
      // final result in saved_value_. We are done!
-      const Slice val = iter_.value();
-      if (!Merge(&val, ikey.user_key)) {
+      if (!MergeWithPlainBaseValue(iter_.value(), ikey.user_key)) {
        return false;
      }
      // iter_ is positioned after put
@@ -584,7 +609,7 @@ bool DBIter::MergeValuesNewToOld() {
        return false;
      }
      valid_ = true;
-      if (!Merge(&blob_value_, ikey.user_key)) {
+      if (!MergeWithPlainBaseValue(blob_value_, ikey.user_key)) {
        return false;
      }

@@ -598,7 +623,7 @@ bool DBIter::MergeValuesNewToOld() {
      }
      return true;
    } else if (kTypeWideColumnEntity == ikey.type) {
-      if (!MergeEntity(iter_.value(), ikey.user_key)) {
+      if (!MergeWithWideColumnBaseValue(iter_.value(), ikey.user_key)) {
        return false;
      }

@@ -628,7 +653,7 @@ bool DBIter::MergeValuesNewToOld() {
  // a deletion marker.
  // feed null as the existing value to the merge operator, such that
  // client can differentiate this scenario and do things accordingly.
-  if (!Merge(nullptr, saved_key_.GetUserKey())) {
+  if (!MergeWithNoBaseValue(saved_key_.GetUserKey())) {
    return false;
  }
  assert(status_.ok());
@@ -719,15 +744,22 @@ bool DBIter::ReverseToBackward() {
  // When current_entry_is_merged_ is true, iter_ may be positioned on the next
  // key, which may not exist or may have prefix different from current.
  // If that's the case, seek to saved_key_.
-  if (current_entry_is_merged_ &&
-      (!expect_total_order_inner_iter() || !iter_.Valid())) {
+  //
+  // In case of auto_readahead_size enabled, index_iter moves forward during
+  // forward scan for block cache lookup and points to different block. If Prev
+  // op is called, it needs to call SeekForPrev to point to right index_iter_ in
+  // BlockBasedTableIterator. This only happens when direction is changed from
+  // forward to backward.
+  if ((current_entry_is_merged_ &&
+       (!expect_total_order_inner_iter() || !iter_.Valid())) ||
+      auto_readahead_size_) {
    IterKey last_key;
    // Using kMaxSequenceNumber and kValueTypeForSeek
    // (not kValueTypeForSeekForPrev) to seek to a key strictly smaller
    // than saved_key_.
    last_key.SetInternalKey(ParsedInternalKey(
        saved_key_.GetUserKey(), kMaxSequenceNumber, kValueTypeForSeek));
-    if (!expect_total_order_inner_iter()) {
+    if (!expect_total_order_inner_iter() || auto_readahead_size_) {
      iter_.SeekForPrev(last_key.GetInternalKey());
    } else {
      // Some iterators may not support SeekForPrev(), so we avoid using it
@@ -979,7 +1011,7 @@ bool DBIter::FindValueForCurrentKey() {
      if (last_not_merge_type == kTypeDeletion ||
          last_not_merge_type == kTypeSingleDeletion ||
          last_not_merge_type == kTypeDeletionWithTimestamp) {
-        if (!Merge(nullptr, saved_key_.GetUserKey())) {
+        if (!MergeWithNoBaseValue(saved_key_.GetUserKey())) {
          return false;
        }
        return true;
@@ -994,7 +1026,7 @@ bool DBIter::FindValueForCurrentKey() {
          return false;
        }
        valid_ = true;
-        if (!Merge(&blob_value_, saved_key_.GetUserKey())) {
+        if (!MergeWithPlainBaseValue(blob_value_, saved_key_.GetUserKey())) {
          return false;
        }

@@ -1002,14 +1034,15 @@ bool DBIter::FindValueForCurrentKey() {

        return true;
      } else if (last_not_merge_type == kTypeWideColumnEntity) {
-        if (!MergeEntity(pinned_value_, saved_key_.GetUserKey())) {
+        if (!MergeWithWideColumnBaseValue(pinned_value_,
+                                          saved_key_.GetUserKey())) {
          return false;
        }

        return true;
      } else {
        assert(last_not_merge_type == kTypeValue);
-        if (!Merge(&pinned_value_, saved_key_.GetUserKey())) {
+        if (!MergeWithPlainBaseValue(pinned_value_, saved_key_.GetUserKey())) {
          return false;
        }
        return true;
@@ -1185,8 +1218,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
    }

    if (ikey.type == kTypeValue) {
-      const Slice val = iter_.value();
-      if (!Merge(&val, saved_key_.GetUserKey())) {
+      if (!MergeWithPlainBaseValue(iter_.value(), saved_key_.GetUserKey())) {
        return false;
      }
      return true;
@@ -1205,7 +1237,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
        return false;
      }
      valid_ = true;
-      if (!Merge(&blob_value_, saved_key_.GetUserKey())) {
+      if (!MergeWithPlainBaseValue(blob_value_, saved_key_.GetUserKey())) {
        return false;
      }

@@ -1213,7 +1245,8 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {

      return true;
    } else if (ikey.type == kTypeWideColumnEntity) {
-      if (!MergeEntity(iter_.value(), saved_key_.GetUserKey())) {
+      if (!MergeWithWideColumnBaseValue(iter_.value(),
+                                        saved_key_.GetUserKey())) {
        return false;
      }

@@ -1227,7 +1260,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
    }
  }

-  if (!Merge(nullptr, saved_key_.GetUserKey())) {
+  if (!MergeWithNoBaseValue(saved_key_.GetUserKey())) {
    return false;
  }

@@ -1250,47 +1283,42 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
  return true;
 }

-bool DBIter::Merge(const Slice* val, const Slice& user_key) {
+bool DBIter::MergeWithNoBaseValue(const Slice& user_key) {
  // `op_failure_scope` (an output parameter) is not provided (set to nullptr)
  // since a failure must be propagated regardless of its value.
-  Status s = MergeHelper::TimedFullMerge(
-      merge_operator_, user_key, val, merge_context_.GetOperands(),
-      &saved_value_, logger_, statistics_, clock_, &pinned_value_,
-      /* update_num_ops_stats */ true,
-      /* op_failure_scope */ nullptr);
-  if (!s.ok()) {
-    valid_ = false;
-    status_ = s;
-    return false;
-  }
-
-  SetValueAndColumnsFromPlain(pinned_value_.data() ? pinned_value_
-                                                   : saved_value_);
-
-  valid_ = true;
-  return true;
+  ValueType result_type;
+  const Status s = MergeHelper::TimedFullMerge(
+      merge_operator_, user_key, MergeHelper::kNoBaseValue,
+      merge_context_.GetOperands(), logger_, statistics_, clock_,
+      /* update_num_ops_stats */ true, &saved_value_, &pinned_value_,
+      &result_type, /* op_failure_scope */ nullptr);
+  return SetValueAndColumnsFromMergeResult(s, result_type);
 }

-bool DBIter::MergeEntity(const Slice& entity, const Slice& user_key) {
+bool DBIter::MergeWithPlainBaseValue(const Slice& value,
+                                     const Slice& user_key) {
  // `op_failure_scope` (an output parameter) is not provided (set to nullptr)
  // since a failure must be propagated regardless of its value.
-  Status s = MergeHelper::TimedFullMergeWithEntity(
-      merge_operator_, user_key, entity, merge_context_.GetOperands(),
-      &saved_value_, logger_, statistics_, clock_,
-      /* update_num_ops_stats */ true,
-      /* op_failure_scope */ nullptr);
-  if (!s.ok()) {
-    valid_ = false;
-    status_ = s;
-    return false;
-  }
+  ValueType result_type;
+  const Status s = MergeHelper::TimedFullMerge(
+      merge_operator_, user_key, MergeHelper::kPlainBaseValue, value,
+      merge_context_.GetOperands(), logger_, statistics_, clock_,
+      /* update_num_ops_stats */ true, &saved_value_, &pinned_value_,
+      &result_type, /* op_failure_scope */ nullptr);
+  return SetValueAndColumnsFromMergeResult(s, result_type);
+}

-  if (!SetValueAndColumnsFromEntity(saved_value_)) {
-    return false;
-  }
-
-  valid_ = true;
-  return true;
+bool DBIter::MergeWithWideColumnBaseValue(const Slice& entity,
+                                          const Slice& user_key) {
+  // `op_failure_scope` (an output parameter) is not provided (set to nullptr)
+  // since a failure must be propagated regardless of its value.
+  ValueType result_type;
+  const Status s = MergeHelper::TimedFullMerge(
+      merge_operator_, user_key, MergeHelper::kWideBaseValue, entity,
+      merge_context_.GetOperands(), logger_, statistics_, clock_,
+      /* update_num_ops_stats */ true, &saved_value_, &pinned_value_,
+      &result_type, /* op_failure_scope */ nullptr);
+  return SetValueAndColumnsFromMergeResult(s, result_type);
 }

 // Move backwards until the key smaller than saved_key_.