Fix deprecated use of 0/NULL in internal_repo_rocksdb/repo/util/xxhash.h + 5

Summary: `nullptr` is typesafe. `0` and `NULL` are not. In the future, only `nullptr` will be allowed. This diff helps us embrace the future _now_ in service of enabling `-Wzero-as-null-pointer-constant`. Reviewed By: dmm-fb Differential Revision: D55559752 fbshipit-source-id: 9f1edc836ded919022c4b53722f6f86208fecf8d
Remove extra semi colon from icsp/lib/logging/IcspLogRpcMessage.cpp
2026-11-07 11:32:34 -05:00 · 2024-04-24 15:20:02 -07:00 · 2024-04-24 15:20:02 -07:00 · 2024-04-24 15:20:02 -07:00 · 2024-04-24 12:45:33 -07:00 · 2024-04-24 12:45:00 -07:00
687 changed files with 22146 additions and 12085 deletions
@@ -547,7 +547,7 @@ jobs:
    resource_class: large
    steps:
      - pre-steps
-      - run: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=960 --max_key=2500000 --use_io_uring=0' blackbox_crash_test_with_atomic_flush
+      - run: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=960 --max_key=2500000' blackbox_crash_test_with_atomic_flush
      - post-steps

  build-linux-crashtest-tiered-storage-bb:
@@ -557,7 +557,7 @@ jobs:
      - pre-steps
      - run:
          name: "run crashtest"
-          command: ulimit -S -n `ulimit -H -n` && make V=1 -j32 CRASH_TEST_EXT_ARGS='--duration=10800 --use_io_uring=0' blackbox_crash_test_with_tiered_storage
+          command: ulimit -S -n `ulimit -H -n` && make V=1 -j32 CRASH_TEST_EXT_ARGS='--duration=10800' blackbox_crash_test_with_tiered_storage
          no_output_timeout: 100m
      - post-steps

@@ -568,7 +568,7 @@ jobs:
      - pre-steps
      - run:
          name: "run crashtest"
-          command: ulimit -S -n `ulimit -H -n` && make V=1 -j32 CRASH_TEST_EXT_ARGS='--duration=10800 --use_io_uring=0' whitebox_crash_test_with_tiered_storage
+          command: ulimit -S -n `ulimit -H -n` && make V=1 -j32 CRASH_TEST_EXT_ARGS='--duration=10800' whitebox_crash_test_with_tiered_storage
          no_output_timeout: 100m
      - post-steps

@@ -352,7 +352,7 @@ jobs:
    steps:
    - uses: actions/checkout@v4.1.0
    - uses: "./.github/actions/pre-steps"
-    - run: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=960 --max_key=2500000 --use_io_uring=0' blackbox_crash_test_with_atomic_flush
+    - run: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=960 --max_key=2500000' blackbox_crash_test_with_atomic_flush
    - uses: "./.github/actions/post-steps"
  # ======================= Linux with Sanitizers ===================== #
  build-linux-clang10-asan:
@@ -403,7 +403,7 @@ jobs:
    - uses: "./.github/actions/pre-steps"
    - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=static OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j24 check
    - uses: "./.github/actions/post-steps"
-  # ========================= MacOS with Tests ======================== #
+  # ========================= MacOS build only ======================== #
  build-macos:
    if: ${{ github.repository_owner == 'facebook' }}
    runs-on: macos-13
@@ -417,9 +417,10 @@ jobs:
    - uses: "./.github/actions/increase-max-open-files-on-macos"
    - uses: "./.github/actions/install-gflags-on-macos"
    - uses: "./.github/actions/pre-steps-macos"
-    - name: Run all tests
+    - name: Build
      run: ulimit -S -n `ulimit -H -n` && make V=1 J=16 -j16 all
    - uses: "./.github/actions/post-steps"
+  # ========================= MacOS with Tests ======================== #
  build-macos-cmake:
    if: ${{ github.repository_owner == 'facebook' }}
    runs-on: macos-13
@@ -605,4 +606,4 @@ jobs:
    - uses: actions/upload-artifact@v4.0.0
      with:
        name: maven-site
-        path: "${{ github.workspace }}/java/target/site"
+        path: "${{ github.workspace }}/java/target/site"
@@ -44,6 +44,29 @@ project(rocksdb
  HOMEPAGE_URL https://rocksdb.org/
  LANGUAGES CXX C ASM)

+if(APPLE)
+  # On macOS Cmake, when cross-compiling, sometimes CMAKE_SYSTEM_PROCESSOR wrongfully stays
+  # the same as CMAKE_HOST_SYSTEM_PROCESSOR regardless the target CPU.
+  # The manual call to  set(CMAKE_SYSTEM_PROCESSOR) has to be set after the project() call.
+  # because project() might reset CMAKE_SYSTEM_PROCESSOR back to the value of CMAKE_HOST_SYSTEM_PROCESSOR.
+  # Check if CMAKE_SYSTEM_PROCESSOR is not equal to CMAKE_OSX_ARCHITECTURES
+  if(NOT CMAKE_OSX_ARCHITECTURES STREQUAL "")
+    if(NOT CMAKE_SYSTEM_PROCESSOR STREQUAL CMAKE_OSX_ARCHITECTURES)
+      # Split CMAKE_OSX_ARCHITECTURES into a list
+      string(REPLACE ";" " " ARCH_LIST ${CMAKE_OSX_ARCHITECTURES})
+      separate_arguments(ARCH_LIST UNIX_COMMAND ${ARCH_LIST})
+      # Count the number of architectures
+      list(LENGTH ARCH_LIST ARCH_COUNT)
+      # Ensure that exactly one architecture is specified
+      if(NOT ARCH_COUNT EQUAL 1)
+          message(FATAL_ERROR "CMAKE_OSX_ARCHITECTURES must have exactly one value. Current value: ${CMAKE_OSX_ARCHITECTURES}")
+      endif()
+      set(CMAKE_SYSTEM_PROCESSOR ${CMAKE_OSX_ARCHITECTURES})
+      message(STATUS "CMAKE_SYSTEM_PROCESSOR is manually set to ${CMAKE_SYSTEM_PROCESSOR}")
+    endif()
+  endif()
+endif()
+
 if(POLICY CMP0042)
  cmake_policy(SET CMP0042 NEW)
 endif()
@@ -166,7 +189,7 @@ else()
  if(WITH_ZSTD)
    find_package(zstd REQUIRED)
    add_definitions(-DZSTD)
-    include_directories(${ZSTD_INCLUDE_DIR})
+    include_directories(${ZSTD_INCLUDE_DIRS})
    list(APPEND THIRDPARTY_LIBS zstd::zstd)
  endif()
 endif()
@@ -182,7 +205,7 @@ endif()

 if(MSVC)
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W4 /wd4127 /wd4800 /wd4996 /wd4351 /wd4100 /wd4204 /wd4324")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W4 /wd4127 /wd4996 /wd4100 /wd4324")
 else()
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall -pthread")
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wshadow -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers -Wno-strict-aliasing -Wno-invalid-offsetof")
@@ -191,6 +214,7 @@ else()
  endif()
  if(MINGW)
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wa,-mbig-obj")
    add_definitions(-D_POSIX_C_SOURCE=1)
  endif()
  if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
@@ -371,7 +395,7 @@ option(WITH_NUMA "build with NUMA policy support" OFF)
 if(WITH_NUMA)
  find_package(NUMA REQUIRED)
  add_definitions(-DNUMA)
-  include_directories(${NUMA_INCLUDE_DIR})
+  include_directories(${NUMA_INCLUDE_DIRS})
  list(APPEND THIRDPARTY_LIBS NUMA::NUMA)
 endif()

@@ -691,6 +715,7 @@ set(SOURCES
        db/memtable_list.cc
        db/merge_helper.cc
        db/merge_operator.cc
+        db/multi_cf_iterator.cc
        db/output_validator.cc
        db/periodic_task_scheduler.cc
        db/range_del_aggregator.cc
@@ -1343,6 +1368,7 @@ if(WITH_TESTS)
        db/memtable_list_test.cc
        db/merge_helper_test.cc
        db/merge_test.cc
+        db/multi_cf_iterator_test.cc
        db/options_file_test.cc
        db/perf_context_test.cc
        db/periodic_task_scheduler_test.cc
@@ -1,25 +1,90 @@
 # Rocksdb Change Log
 > NOTE: Entries for next release do not go here. Follow instructions in `unreleased_history/README.txt`

-## 8.11.5 (11/26/2025)
-### Behavior Changes
-* Backport new value types `kTypeValuePreferredSeqno` and `kTypeColumnFamilyValuePreferredSeqno` introduced in RocksDB 9.1. This fixes a forward compatibility issue to allow reading MANIFEST with entries containing kTypeMaxValid written by RocksDB 9.1+.
-
-## 8.11.4 (04/09/2024)
+## 9.1.2 (04/24/2024)
+### Bug Fixes
+* Fixed hang in `DisableManualCompactions()` where compactions waiting to be scheduled due to conflicts would not be canceled promptly
+
+## 9.1.1 (04/17/2024)
 ### Bug Fixes
-* Fixed CMake Javadoc build
 * Fixed Java `SstFileMetaData` to prevent throwing `java.lang.NoSuchMethodError`
+* Fixed a regression when `ColumnFamilyOptions::max_successive_merges > 0` where the CPU overhead for deciding whether to merge could have increased unless the user had set the option `ColumnFamilyOptions::strict_max_successive_merges`

-## 8.11.3 (02/27/2024)
-* Correct CMake Javadoc and source jar builds
+## 9.1.0 (03/22/2024)
+### New Features
+* Added an option, `GetMergeOperandsOptions::continue_cb`, to give users the ability to end `GetMergeOperands()`'s lookup process before all merge operands were found.
+* *Add sanity checks for ingesting external files that currently checks if the user key comparator used to create the file is compatible with the column family's user key comparator.
+*Support ingesting external files for column family that has user-defined timestamps in memtable only enabled.
+* On file systems that support storage level data checksum and reconstruction, retry SST block reads for point lookups, scans, and flush and compaction if there's a checksum mismatch on the initial read.
+* Some enhancements and fixes to experimental Temperature handling features, including new `default_write_temperature` CF option and opening an `SstFileWriter` with a temperature.
+* `WriteBatchWithIndex` now supports wide-column point lookups via the `GetEntityFromBatch` API. See the API comments for more details.
+* *Implement experimental features: API `Iterator::GetProperty("rocksdb.iterator.write-time")` to allow users to get data's approximate write unix time and write data with a specific write time via `WriteBatch::TimedPut` API.

-## 8.11.2 (02/16/2024)
-* Update zlib to 1.3.1 for Java builds
+### Public API Changes
+* Best-effort recovery (`best_efforts_recovery == true`) may now be used together with atomic flush (`atomic_flush == true`). The all-or-nothing recovery guarantee for atomically flushed data will be upheld.
+* Remove deprecated option `bottommost_temperature`, already replaced by `last_level_temperature`
+* Added new PerfContext counters for block cache bytes read - block_cache_index_read_byte, block_cache_filter_read_byte, block_cache_compression_dict_read_byte, and block_cache_read_byte.
+* Deprecate experimental Remote Compaction APIs - StartV2() and WaitForCompleteV2() and introduce Schedule() and Wait(). The new APIs essentially does the same thing as the old APIs. They allow taking externally generated unique id to wait for remote compaction to complete.
+* *For API `WriteCommittedTransaction::GetForUpdate`, if the column family enables user-defined timestamp, it was mandated that argument `do_validate` cannot be false, and UDT based validation has to be done with a user set read timestamp. It's updated to make the UDT based validation optional if user sets `do_validate` to false and does not set a read timestamp. With this, `GetForUpdate` skips UDT based validation and it's users' responsibility to enforce the UDT invariant. SO DO NOT skip this UDT-based validation if users do not have ways to enforce the UDT invariant. Ways to enforce the invariant on the users side include manage a monotonically increasing timestamp, commit transactions in a single thread etc.
+* Defined a new PerfLevel `kEnableWait` to measure time spent by user threads blocked in RocksDB other than mutex, such as a write thread waiting to be added to a write group, a write thread delayed or stalled etc.
+* `RateLimiter`'s API no longer requires the burst size to be the refill size. Users of `NewGenericRateLimiter()` can now provide burst size in `single_burst_bytes`. Implementors of `RateLimiter::SetSingleBurstBytes()` need to adapt their implementations to match the changed API doc.
+* Add `write_memtable_time` to the newly introduced PerfLevel `kEnableWait`.
+
+### Behavior Changes
+* `RateLimiter`s created by `NewGenericRateLimiter()` no longer modify the refill period when `SetSingleBurstBytes()` is called.
+* Merge writes will only keep merge operand count within `ColumnFamilyOptions::max_successive_merges` when the key's merge operands are all found in memory, unless `strict_max_successive_merges` is explicitly set.

-## 8.11.1 (01/25/2024)
 ### Bug Fixes
+* Fixed `kBlockCacheTier` reads to return `Status::Incomplete` when I/O is needed to fetch a merge chain's base value from a blob file.
+* Fixed `kBlockCacheTier` reads to return `Status::Incomplete` on table cache miss rather than incorrectly returning an empty value.
+* Fixed a data race in WalManager that may affect how frequent PurgeObsoleteWALFiles() runs.
+* Re-enable the recycle_log_file_num option in DBOptions for kPointInTimeRecovery WAL recovery mode, which was previously disabled due to a bug in the recovery logic. This option is incompatible with WriteOptions::disableWAL. A Status::InvalidArgument() will be returned if disableWAL is specified.
+
+### Performance Improvements
+* Java API `multiGet()` variants now take advantage of the underlying batched `multiGet()` performance improvements.
+Before
+```
+Benchmark (columnFamilyTestType) (keyCount) (keySize) (multiGetSize) (valueSize) Mode Cnt Score Error Units
+MultiGetBenchmarks.multiGetList10 no_column_family 10000 16 100 64 thrpt 25 6315.541 ± 8.106 ops/s
+MultiGetBenchmarks.multiGetList10 no_column_family 10000 16 100 1024 thrpt 25 6975.468 ± 68.964 ops/s
+```
+After
+```
+Benchmark (columnFamilyTestType) (keyCount) (keySize) (multiGetSize) (valueSize) Mode Cnt Score Error Units
+MultiGetBenchmarks.multiGetList10 no_column_family 10000 16 100 64 thrpt 25 7046.739 ± 13.299 ops/s
+MultiGetBenchmarks.multiGetList10 no_column_family 10000 16 100 1024 thrpt 25 7654.521 ± 60.121 ops/s
+```
+
+## 9.0.0 (02/16/2024)
+### New Features
+* Provide support for FSBuffer for point lookups. Also added support for scans and compactions that don't go through prefetching.
+* *Make `SstFileWriter` create SST files without persisting user defined timestamps when the `Option.persist_user_defined_timestamps` flag is set to false.
+* Add support for user-defined timestamps in APIs `DeleteFilesInRanges` and `GetPropertiesOfTablesInRange`.
+* Mark wal\_compression feature as production-ready. Currently only compatible with ZSTD compression.
+
+### Public API Changes
+* Allow setting Stderr logger via C API
+* Declare one Get and one MultiGet variant as pure virtual, and make all the other variants non-overridable. The methods required to be implemented by derived classes of DB allow returning timestamps. It is up to the implementation to check and return an error if timestamps are not supported. The non-batched MultiGet APIs are reimplemented in terms of batched MultiGet, so callers might see a performance improvement.
+* Exposed mode option to Rate Limiter via c api.
+* Removed deprecated option `access_hint_on_compaction_start`
+* Removed deprecated option `ColumnFamilyOptions::check_flush_compaction_key_order`
+* *Remove the default `WritableFile::GetFileSize` and `FSWritableFile::GetFileSize` implementation that returns 0 and make it pure virtual, so that subclasses are enforced to explicitly provide an implementation.
+* Removed deprecated option `ColumnFamilyOptions::level_compaction_dynamic_file_size`
+* *Removed tickers with typos "rocksdb.error.handler.bg.errro.count", "rocksdb.error.handler.bg.io.errro.count", "rocksdb.error.handler.bg.retryable.io.errro.count".
+* Remove the force mode for `EnableFileDeletions` API because it is unsafe with no known legitimate use.
+* Removed deprecated option `ColumnFamilyOptions::ignore_max_compaction_bytes_for_input`
+* `sst_dump --command=check` now compares the number of records in a table with `num_entries` in table property, and reports corruption if there is a mismatch. API `SstFileDumper::ReadSequential()` is updated to optionally do this verification. (#12322)
+
+### Behavior Changes
+* format\_version=6 is the new default setting in BlockBasedTableOptions, for more robust data integrity checking. DBs and SST files written with this setting cannot be read by RocksDB versions before 8.6.0.
+* Compactions can be scheduled in parallel in an additional scenario: multiple files are marked for compaction within a single column family
+* For leveled compaction, RocksDB will try to do intra-L0 compaction if the total L0 size is small compared to Lbase (#12214). Users with atomic_flush=true are more likely to see the impact of this change.
+
+### Bug Fixes
+* Fixed a data race in `DBImpl::RenameTempFileToOptionsFile`.
+* Fix some perf context statistics error in write steps. which include missing write_memtable_time in unordered_write. missing write_memtable_time in PipelineWrite when Writer stat is STATE_PARALLEL_MEMTABLE_WRITER. missing write_delay_time when calling DelayWrite in WriteImplWALOnly function.
+* Fixed a bug that can, under rare circumstances, cause MultiGet to return an incorrect result for a duplicate key in a MultiGet batch.
 * Fix a bug where older data of an ingested key can be returned for read when universal compaction is used
-* Apply appropriate rate limiting and priorities in more places.

 ## 8.11.0 (01/19/2024)
 ### New Features
@@ -126,7 +126,6 @@ most processors made since roughly 2013.
        * Update XCode:  run `xcode-select --install` (or install it from XCode App's settting).
        * Install via [homebrew](http://brew.sh/).
            * If you're first time developer in MacOS, you still need to run: `xcode-select --install` in your command line.
-            * run `brew tap homebrew/versions; brew install gcc7 --use-llvm` to install gcc 7 (or higher).
    * run `brew install rocksdb`

 * **FreeBSD** (11.01):
@@ -169,21 +168,26 @@ most processors made since roughly 2013.

    * Install the dependencies for RocksDB:

-        pkg_add gmake gflags snappy bzip2 lz4 zstd git jdk bash findutils gnuwatch
+      `pkg_add gmake gflags snappy bzip2 lz4 zstd git bash findutils gnuwatch`

    * Build RocksDB from source:

+        ```bash
        cd ~
        git clone https://github.com/facebook/rocksdb.git
        cd rocksdb
        gmake static_lib
+        ```

    * Build RocksJava from source (optional):
-
+        * In OpenBSD, JDK depends on XWindows system, so please check that you installed OpenBSD with `xbase` package.
+        * Install dependencies : `pkg_add -v jdk%1.8`
+        ```bash
        cd rocksdb
        export JAVA_HOME=/usr/local/jdk-1.8.0
        export PATH=$PATH:/usr/local/jdk-1.8.0/bin
-        gmake rocksdbjava
+        gmake rocksdbjava SHA256_CMD='sha256 -q'
+        ```

 * **iOS**:
  * Run: `TARGET_OS=IOS make static_lib`. When building the project which uses rocksdb iOS library, make sure to define an important pre-processing macros: `IOS_CROSS_COMPILE`.
@@ -539,7 +539,8 @@ endif

 ifdef USE_CLANG
 	# Used by some teams in Facebook
-	WARNING_FLAGS += -Wshift-sign-overflow -Wambiguous-reversed-operator -Wimplicit-fallthrough
+	WARNING_FLAGS += -Wshift-sign-overflow -Wambiguous-reversed-operator \
+	  -Wimplicit-fallthrough -Wreinterpret-base-class -Wundefined-reinterpret-cast
 endif

 ifeq ($(PLATFORM), OS_OPENBSD)
@@ -1639,6 +1640,9 @@ wal_edit_test: $(OBJ_DIR)/db/wal_edit_test.o $(TEST_LIBRARY) $(LIBRARY)
 dbformat_test: $(OBJ_DIR)/db/dbformat_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)

+multi_cf_iterator_test: $(OBJ_DIR)/db/multi_cf_iterator_test.o $(TEST_LIBRARY) $(LIBRARY)
+	$(AM_LINK)
+
 env_basic_test: $(OBJ_DIR)/env/env_basic_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)

@@ -2331,47 +2335,47 @@ rocksdbjavastaticreleasedocker: rocksdbjavastaticosx rocksdbjavastaticdockerx86

 rocksdbjavastaticdockerx86:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_x86-be --platform linux/386 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos6_x86-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh
+	docker run --rm --name rocksdb_linux_x86-be --platform linux/386 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:centos6_x86-be /rocksdb-host/java/crossbuild/docker-build-linux.sh

 rocksdbjavastaticdockerx86_64:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_x64-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos6_x64-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh
+	docker run --rm --name rocksdb_linux_x64-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:centos6_x64-be /rocksdb-host/java/crossbuild/docker-build-linux.sh

 rocksdbjavastaticdockerppc64le:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_ppc64le-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos7_ppc64le-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh
+	docker run --rm --name rocksdb_linux_ppc64le-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:centos7_ppc64le-be /rocksdb-host/java/crossbuild/docker-build-linux.sh

 rocksdbjavastaticdockerarm64v8:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_arm64v8-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos7_arm64v8-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh
+	docker run --rm --name rocksdb_linux_arm64v8-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:centos7_arm64v8-be /rocksdb-host/java/crossbuild/docker-build-linux.sh

 rocksdbjavastaticdockers390x:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_s390x-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:ubuntu18_s390x-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh
+	docker run --rm --name rocksdb_linux_s390x-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:ubuntu18_s390x-be /rocksdb-host/java/crossbuild/docker-build-linux.sh

 rocksdbjavastaticdockerriscv64:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_riscv64-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:ubuntu20_riscv64-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh
+	docker run --rm --name rocksdb_linux_riscv64-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:ubuntu20_riscv64-be /rocksdb-host/java/crossbuild/docker-build-linux.sh

 rocksdbjavastaticdockerx86musl:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_x86-musl-be --platform linux/386 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:alpine3_x86-be /rocksdb-host/java/crossbuild/docker-build-linux-alpine.sh
+	docker run --rm --name rocksdb_linux_x86-musl-be --platform linux/386 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_x86-be /rocksdb-host/java/crossbuild/docker-build-linux.sh

 rocksdbjavastaticdockerx86_64musl:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_x64-musl-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:alpine3_x64-be /rocksdb-host/java/crossbuild/docker-build-linux-alpine.sh
+	docker run --rm --name rocksdb_linux_x64-musl-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_x64-be /rocksdb-host/java/crossbuild/docker-build-linux.sh

 rocksdbjavastaticdockerppc64lemusl:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_ppc64le-musl-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:alpine3_ppc64le-be /rocksdb-host/java/crossbuild/docker-build-linux-alpine.sh
+	docker run --rm --name rocksdb_linux_ppc64le-musl-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_ppc64le-be /rocksdb-host/java/crossbuild/docker-build-linux.sh

 rocksdbjavastaticdockerarm64v8musl:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_arm64v8-musl-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:alpine3_arm64v8-be /rocksdb-host/java/crossbuild/docker-build-linux-alpine.sh
+	docker run --rm --name rocksdb_linux_arm64v8-musl-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_arm64v8-be /rocksdb-host/java/crossbuild/docker-build-linux.sh

 rocksdbjavastaticdockers390xmusl:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_s390x-musl-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:alpine3_s390x-be /rocksdb-host/java/crossbuild/docker-build-linux-alpine.sh
+	docker run --rm --name rocksdb_linux_s390x-musl-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_s390x-be /rocksdb-host/java/crossbuild/docker-build-linux.sh

 rocksdbjavastaticpublish: rocksdbjavastaticrelease rocksdbjavastaticpublishcentral

@@ -83,6 +83,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
        "db/memtable_list.cc",
        "db/merge_helper.cc",
        "db/merge_operator.cc",
+        "db/multi_cf_iterator.cc",
        "db/output_validator.cc",
        "db/periodic_task_scheduler.cc",
        "db/range_del_aggregator.cc",
@@ -5226,6 +5227,12 @@ cpp_unittest_wrapper(name="mock_env_test",
            extra_compiler_flags=[])


+cpp_unittest_wrapper(name="multi_cf_iterator_test",
+            srcs=["db/multi_cf_iterator_test.cc"],
+            deps=[":rocksdb_test_lib"],
+            extra_compiler_flags=[])
+
+
 cpp_unittest_wrapper(name="object_registry_test",
            srcs=["utilities/object_registry_test.cc"],
            deps=[":rocksdb_test_lib"],
@@ -163,24 +163,6 @@ case "$TARGET_OS" in
            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic"
        fi
        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt -ldl"
-        if test -z "$ROCKSDB_USE_IO_URING"; then
-            ROCKSDB_USE_IO_URING=1
-        fi
-        if test "$ROCKSDB_USE_IO_URING" -ne 0; then
-            # check for liburing
-            $CXX $PLATFORM_CXXFLAGS -x c++ - -luring -o test.o 2>/dev/null  <<EOF
-              #include <liburing.h>
-              int main() {
-                struct io_uring ring;
-                io_uring_queue_init(1, &ring, 0);
-                return 0;
-              }
-EOF
-            if [ "$?" = 0 ]; then
-                PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -luring"
-                COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_IOURING_PRESENT"
-            fi
-        fi
        # PORT_FILES=port/linux/linux_specific.cc
        ;;
    SunOS)
@@ -614,11 +596,29 @@ EOF
        fi
    fi

+    if test -z "$ROCKSDB_USE_IO_URING"; then
+        ROCKSDB_USE_IO_URING=1
+    fi
+    if [ "$ROCKSDB_USE_IO_URING" -ne 0 -a "$PLATFORM" = OS_LINUX ]; then
+        # check for liburing
+        $CXX $PLATFORM_CXXFLAGS -x c++ - -luring -o test.o 2>/dev/null  <<EOF
+          #include <liburing.h>
+          int main() {
+            struct io_uring ring;
+            io_uring_queue_init(1, &ring, 0);
+            return 0;
+          }
+EOF
+        if [ "$?" = 0 ]; then
+            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -luring"
+            COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_IOURING_PRESENT"
+        fi
+    fi
 fi

 # TODO(tec): Fix -Wshorten-64-to-32 errors on FreeBSD and enable the warning.
 # -Wshorten-64-to-32 breaks compilation on FreeBSD aarch64 and i386
-if ! { [ "$TARGET_OS" = FreeBSD ] && [ "$TARGET_ARCHITECTURE" = arm64 -o "$TARGET_ARCHITECTURE" = i386 ]; }; then
+if ! { [ "$TARGET_OS" = FreeBSD -o "$TARGET_OS" = OpenBSD ] && [ "$TARGET_ARCHITECTURE" = arm64 -o "$TARGET_ARCHITECTURE" = i386 ]; }; then
  # Test whether -Wshorten-64-to-32 is available
  $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o -Wshorten-64-to-32 2>/dev/null  <<EOF
    int main() {}
@@ -1,22 +1,22 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 # The file is generated using update_dependencies.sh.
-GCC_BASE=/mnt/gvfs/third-party2/gcc/e40bde78650fa91b8405a857e3f10bf336633fb0/11.x/centos7-native/886b5eb
-CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/2043340983c032915adbb6f78903dc855b65aee8/12/platform010/9520e0f
-LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/c00dcc6a3e4125c7e8b248e9a79c14b78ac9e0ca/11.x/platform010/5684a5a
-GLIBC_BASE=/mnt/gvfs/third-party2/glibc/0b9c8e4b060eda62f3bc1c6127bbe1256697569b/2.34/platform010/f259413
-SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/bc9647f7912b131315827d65cb6189c21f381d05/1.1.3/platform010/76ebdda
-ZLIB_BASE=/mnt/gvfs/third-party2/zlib/a6f5f3f1d063d2d00cd02fc12f0f05fc3ab3a994/1.2.11/platform010/76ebdda
+GCC_BASE=/mnt/gvfs/third-party2/gcc/62de5a92e5f23c661c3d4b9f322e04eb14e7a5bd/11.x/centos8-native/886b5eb
+CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/1f6edd1ff15c99c861afc8f3cd69054cd974dd64/15/platform010/72a2ff8
+LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/d1129753c8361ac8e9453c0f4291337a4507ebe6/11.x/platform010/5684a5a
+GLIBC_BASE=/mnt/gvfs/third-party2/glibc/fed6e93d87571fb162734c86636119d45a398963/2.34/platform010/f259413
+SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/31a346126a1f3b64812c362511cb04cc1bd40855/1.1.8/platform010/76ebdda
+ZLIB_BASE=/mnt/gvfs/third-party2/zlib/0c65c05468b5a38cef1a106a1f526463e120c8dd/1.2.8/platform010/76ebdda
 BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/09703139cfc376bd8a82642385a0e97726b28287/1.0.6/platform010/76ebdda
-LZ4_BASE=/mnt/gvfs/third-party2/lz4/60220d6a5bf7722b9cc239a1368c596619b12060/1.9.1/platform010/76ebdda
-ZSTD_BASE=/mnt/gvfs/third-party2/zstd/50eace8143eaaea9473deae1f3283e0049e05633/1.4.x/platform010/64091f4
-GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/5d27e5919771603da06000a027b12f799e58a4f7/2.2.0/platform010/76ebdda
-JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/b62912d333ef33f9760efa6219dbe3fe6abb3b0e/master/platform010/f57cc4a
-NUMA_BASE=/mnt/gvfs/third-party2/numa/6b412770957aa3c8a87e5e0dcd8cc2f45f393bc0/2.0.11/platform010/76ebdda
-LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/52f69816e936e147664ad717eb71a1a0e9dc973a/1.4/platform010/5074a48
-TBB_BASE=/mnt/gvfs/third-party2/tbb/c9cc192099fa84c0dcd0ffeedd44a373ad6e4925/2018_U5/platform010/76ebdda
+LZ4_BASE=/mnt/gvfs/third-party2/lz4/ff23d17b932725cc1734a14896a8b67c518ba169/1.9.4/platform010/76ebdda
+ZSTD_BASE=/mnt/gvfs/third-party2/zstd/576397d8b1d9cea7306ad1e454d5e55caaa2ff1c/1.4.x/platform010/64091f4
+GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/fecac07861cb829f5e60dbeff0503d3272db73c0/2.2.0/platform010/76ebdda
+JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/0bb3f5756788ce26e2e16a1cb2f2af2c59b51abe/master/platform010/f57cc4a
+NUMA_BASE=/mnt/gvfs/third-party2/numa/5b602edd46fda54cdd7ea45f77dbe4061206e174/2.0.11/platform010/76ebdda
+LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/97cac22a149c2e202917e05d44e87e516b68216f/1.4/platform010/5074a48
+TBB_BASE=/mnt/gvfs/third-party2/tbb/53953ebc4e3eda85ad6fc3e429ba146035e97b90/2018_U5/platform010/76ebdda
 LIBURING_BASE=/mnt/gvfs/third-party2/liburing/a98e2d137007e3ebf7f33bd6f99c2c56bdaf8488/20210212/platform010/76ebdda
 BENCHMARK_BASE=/mnt/gvfs/third-party2/benchmark/780c7a0f9cf0967961e69ad08e61cddd85d61821/trunk/platform010/76ebdda
-KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/02d9f76aaaba580611cf75e741753c800c7fdc12/fb/platform010/da39a3e
-BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/938dc3f064ef3a48c0446f5b11d788d50b3eb5ee/2.37/centos7-native/da39a3e
-VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/429a6b3203eb415f1599bd15183659153129188e/3.15.0/platform010/76ebdda
+KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/624a2f8f6c93c3c1df8aa4a6255d8202631a6c80/fb/platform010/da39a3e
+BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/39579e8603b48b3540f8b0633f43adf29acccb8b/2.37/centos8-native/da39a3e
+VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/cd9cc656d49ecb53797ce4d055e49fde29fd57ff/3.19.0/platform010/76ebdda
 LUA_BASE=/mnt/gvfs/third-party2/lua/363787fa5cac2a8aa20638909210443278fa138e/5.3.4/platform010/9079c97
@@ -137,11 +137,11 @@ then
  # should be relevant for formatting fixes.
  FORMAT_UPSTREAM_MERGE_BASE="$(git merge-base "$FORMAT_UPSTREAM" HEAD)"
  # Get the differences
-  diffs=$(git diff -U0 "$FORMAT_UPSTREAM_MERGE_BASE" | $CLANG_FORMAT_DIFF -p 1)
+  diffs=$(git diff -U0 "$FORMAT_UPSTREAM_MERGE_BASE" | $CLANG_FORMAT_DIFF -p 1) || true
  echo "Checking format of changes not yet in $FORMAT_UPSTREAM..."
 else
  # Check the format of uncommitted lines,
-  diffs=$(git diff -U0 HEAD | $CLANG_FORMAT_DIFF -p 1)
+  diffs=$(git diff -U0 HEAD | $CLANG_FORMAT_DIFF -p 1) || true
  echo "Checking format of uncommitted changes..."
 fi

@@ -149,6 +149,9 @@ if [ -z "$diffs" ]
 then
  echo "Nothing needs to be reformatted!"
  exit 0
+elif [ $? -ne 1 ]; then
+  # CLANG_FORMAT_DIFF will exit on 1 while there is suggested changes.
+  exit $?
 elif [ $CHECK_ONLY ]
 then
  echo "Your change has unformatted code. Please run make format!"
@@ -75,8 +75,8 @@ touch "$OUTPUT"
 echo "Writing dependencies to $OUTPUT"

 # Compilers locations
-GCC_BASE=`readlink -f $TP2_LATEST/gcc/11.x/centos7-native/*/`
-CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/12/platform010/*/`
+GCC_BASE=`readlink -f $TP2_LATEST/gcc/11.x/centos8-native/*/`
+CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/15/platform010/*/`

 log_header
 log_variable GCC_BASE
@@ -86,7 +86,7 @@ log_variable CLANG_BASE
 get_lib_base libgcc     11.x    platform010
 get_lib_base glibc      2.34    platform010
 get_lib_base snappy     LATEST  platform010
-get_lib_base zlib       LATEST  platform010
+get_lib_base zlib       1.2.8   platform010
 get_lib_base bzip2      LATEST  platform010
 get_lib_base lz4        LATEST  platform010
 get_lib_base zstd       LATEST  platform010
@@ -99,7 +99,7 @@ get_lib_base liburing   LATEST  platform010
 get_lib_base benchmark  LATEST  platform010

 get_lib_base kernel-headers fb platform010
-get_lib_base binutils   LATEST centos7-native
+get_lib_base binutils   LATEST centos8-native
 get_lib_base valgrind   LATEST platform010
 get_lib_base lua        5.3.4  platform010

@@ -194,7 +194,7 @@ class SharedState {
      : cv_(&mu_),
        cache_bench_(cache_bench) {}

-  ~SharedState() {}
+  ~SharedState() = default;

  port::Mutex* GetMutex() { return &mu_; }

@@ -425,7 +425,7 @@ class CacheBench {
    }
  }

-  ~CacheBench() {}
+  ~CacheBench() = default;

  void PopulateCache() {
    Random64 rnd(FLAGS_seed);
@@ -129,7 +129,6 @@ TEST_F(CacheReservationManagerTest,

 TEST(CacheReservationManagerIncreaseReservcationOnFullCacheTest,
     IncreaseCacheReservationOnFullCache) {
-  ;
  constexpr std::size_t kSizeDummyEntry =
      CacheReservationManagerImpl<CacheEntryRole::kMisc>::GetDummyEntrySize();
  constexpr std::size_t kSmallCacheCapacity = 4 * kSizeDummyEntry;
@@ -106,7 +106,7 @@ class CacheTest : public testing::Test,
    type_ = GetParam();
  }

-  ~CacheTest() override {}
+  ~CacheTest() override = default;

  // These functions encode/decode keys in tests cases that use
  // int keys.
@@ -766,7 +766,9 @@ TEST_P(CacheTest, OverCapacity) {
    std::string key = EncodeKey(i + 1);
    auto h = cache.Lookup(key);
    ASSERT_TRUE(h != nullptr);
-    if (h) cache.Release(h);
+    if (h) {
+      cache.Release(h);
+    }
  }

  // the cache is over capacity since nothing could be evicted
@@ -777,7 +779,7 @@ TEST_P(CacheTest, OverCapacity) {

  if (IsHyperClock()) {
    // Make sure eviction is triggered.
-    ASSERT_OK(cache.Insert(EncodeKey(-1), nullptr, 1, &handles[0]));
+    ASSERT_OK(cache.Insert(EncodeKey(-1), nullptr, 1, handles.data()));

    // cache is under capacity now since elements were released
    ASSERT_GE(n, cache.get()->GetUsage());
@@ -560,7 +560,7 @@ void BaseClockTable::TrackAndReleaseEvictedEntry(ClockHandle* h) {
    took_value_ownership =
        eviction_callback_(ClockCacheShard<FixedHyperClockTable>::ReverseHash(
                               h->GetHash(), &unhashed, hash_seed_),
-                           reinterpret_cast<Cache::Handle*>(h),
+                           static_cast<Cache::Handle*>(h),
                           h->meta.LoadRelaxed() & ClockHandle::kHitBitMask);
  }
  if (!took_value_ownership) {
@@ -1428,19 +1428,19 @@ BaseHyperClockCache<Table>::BaseHyperClockCache(

 template <class Table>
 Cache::ObjectPtr BaseHyperClockCache<Table>::Value(Handle* handle) {
-  return reinterpret_cast<const typename Table::HandleImpl*>(handle)->value;
+  return static_cast<const typename Table::HandleImpl*>(handle)->value;
 }

 template <class Table>
 size_t BaseHyperClockCache<Table>::GetCharge(Handle* handle) const {
-  return reinterpret_cast<const typename Table::HandleImpl*>(handle)
+  return static_cast<const typename Table::HandleImpl*>(handle)
      ->GetTotalCharge();
 }

 template <class Table>
 const Cache::CacheItemHelper* BaseHyperClockCache<Table>::GetCacheItemHelper(
    Handle* handle) const {
-  auto h = reinterpret_cast<const typename Table::HandleImpl*>(handle);
+  auto h = static_cast<const typename Table::HandleImpl*>(handle);
  return h->helper;
 }

@@ -298,7 +298,7 @@ class ClockCacheTest;

 // ----------------------------------------------------------------------- //

-struct ClockHandleBasicData {
+struct ClockHandleBasicData : public Cache::Handle {
  Cache::ObjectPtr value = nullptr;
  const Cache::CacheItemHelper* helper = nullptr;
  // A lossless, reversible hash of the fixed-size (16 byte) cache key. This
@@ -26,7 +26,7 @@ CompressedSecondaryCache::CompressedSecondaryCache(
              cache_))),
      disable_cache_(opts.capacity == 0) {}

-CompressedSecondaryCache::~CompressedSecondaryCache() {}
+CompressedSecondaryCache::~CompressedSecondaryCache() = default;

 std::unique_ptr<SecondaryCacheResultHandle> CompressedSecondaryCache::Lookup(
    const Slice& key, const Cache::CacheItemHelper* helper,
@@ -384,7 +384,7 @@ const Cache::CacheItemHelper* CompressedSecondaryCache::GetHelper(
            chunks_head = chunks_head->next;
            tmp_chunk->Free();
            obj = nullptr;
-          };
+          }
        }};
    return &kHelper;
  } else {
@@ -33,7 +33,7 @@ const std::string key3 = "____    ____key3";
 class CompressedSecondaryCacheTestBase : public testing::Test,
                                         public WithCacheType {
 public:
-  CompressedSecondaryCacheTestBase() {}
+  CompressedSecondaryCacheTestBase() = default;
  ~CompressedSecondaryCacheTestBase() override = default;

 protected:
@@ -1062,7 +1062,7 @@ bool CacheUsageWithinBounds(size_t val1, size_t val2, size_t error) {

 TEST_P(CompressedSecCacheTestWithTiered, CacheReservationManager) {
  CompressedSecondaryCache* sec_cache =
-      reinterpret_cast<CompressedSecondaryCache*>(GetSecondaryCache());
+      static_cast<CompressedSecondaryCache*>(GetSecondaryCache());

  // Use EXPECT_PRED3 instead of EXPECT_NEAR to void too many size_t to
  // double explicit casts
@@ -1085,7 +1085,7 @@ TEST_P(CompressedSecCacheTestWithTiered, CacheReservationManager) {
 TEST_P(CompressedSecCacheTestWithTiered,
       CacheReservationManagerMultipleUpdate) {
  CompressedSecondaryCache* sec_cache =
-      reinterpret_cast<CompressedSecondaryCache*>(GetSecondaryCache());
+      static_cast<CompressedSecondaryCache*>(GetSecondaryCache());

  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (30 << 20),
               GetPercent(30 << 20, 1));
@@ -1171,7 +1171,7 @@ TEST_P(CompressedSecCacheTestWithTiered, AdmissionPolicy) {

 TEST_P(CompressedSecCacheTestWithTiered, DynamicUpdate) {
  CompressedSecondaryCache* sec_cache =
-      reinterpret_cast<CompressedSecondaryCache*>(GetSecondaryCache());
+      static_cast<CompressedSecondaryCache*>(GetSecondaryCache());
  std::shared_ptr<Cache> tiered_cache = GetTieredCache();

  // Use EXPECT_PRED3 instead of EXPECT_NEAR to void too many size_t to
@@ -1235,7 +1235,7 @@ TEST_P(CompressedSecCacheTestWithTiered, DynamicUpdate) {

 TEST_P(CompressedSecCacheTestWithTiered, DynamicUpdateWithReservation) {
  CompressedSecondaryCache* sec_cache =
-      reinterpret_cast<CompressedSecondaryCache*>(GetSecondaryCache());
+      static_cast<CompressedSecondaryCache*>(GetSecondaryCache());
  std::shared_ptr<Cache> tiered_cache = GetTieredCache();

  ASSERT_OK(cache_res_mgr()->UpdateCacheReservation(10 << 20));
@@ -1329,7 +1329,7 @@ TEST_P(CompressedSecCacheTestWithTiered, DynamicUpdateWithReservation) {

 TEST_P(CompressedSecCacheTestWithTiered, ReservationOverCapacity) {
  CompressedSecondaryCache* sec_cache =
-      reinterpret_cast<CompressedSecondaryCache*>(GetSecondaryCache());
+      static_cast<CompressedSecondaryCache*>(GetSecondaryCache());
  std::shared_ptr<Cache> tiered_cache = GetTieredCache();

  ASSERT_OK(cache_res_mgr()->UpdateCacheReservation(110 << 20));
@@ -277,8 +277,8 @@ void LRUCacheShard::LRU_Insert(LRUHandle* e) {
    e->SetInHighPriPool(false);
    e->SetInLowPriPool(true);
    low_pri_pool_usage_ += e->total_charge;
-    MaintainPoolSize();
    lru_low_pri_ = e;
+    MaintainPoolSize();
  } else {
    // Insert "e" to the head of bottom-pri pool.
    e->next = lru_bottom_pri_->next;
@@ -301,6 +301,7 @@ void LRUCacheShard::MaintainPoolSize() {
    // Overflow last entry in high-pri pool to low-pri pool.
    lru_low_pri_ = lru_low_pri_->next;
    assert(lru_low_pri_ != &lru_);
+    assert(lru_low_pri_->InHighPriPool());
    lru_low_pri_->SetInHighPriPool(false);
    lru_low_pri_->SetInLowPriPool(true);
    assert(high_pri_pool_usage_ >= lru_low_pri_->total_charge);
@@ -312,6 +313,7 @@ void LRUCacheShard::MaintainPoolSize() {
    // Overflow last entry in low-pri pool to bottom-pri pool.
    lru_bottom_pri_ = lru_bottom_pri_->next;
    assert(lru_bottom_pri_ != &lru_);
+    assert(lru_bottom_pri_->InLowPriPool());
    lru_bottom_pri_->SetInHighPriPool(false);
    lru_bottom_pri_->SetInLowPriPool(false);
    assert(low_pri_pool_usage_ >= lru_bottom_pri_->total_charge);
@@ -339,8 +341,7 @@ void LRUCacheShard::NotifyEvicted(
  MemoryAllocator* alloc = table_.GetAllocator();
  for (LRUHandle* entry : evicted_handles) {
    if (eviction_callback_ &&
-        eviction_callback_(entry->key(),
-                           reinterpret_cast<Cache::Handle*>(entry),
+        eviction_callback_(entry->key(), static_cast<Cache::Handle*>(entry),
                           entry->HasHit())) {
      // Callback took ownership of obj; just free handle
      free(entry);
@@ -506,7 +507,7 @@ bool LRUCacheShard::Release(LRUHandle* e, bool /*useful*/,
    // Only call eviction callback if we're sure no one requested erasure
    // FIXME: disabled because of test churn
    if (false && was_in_cache && !erase_if_last_ref && eviction_callback_ &&
-        eviction_callback_(e->key(), reinterpret_cast<Cache::Handle*>(e),
+        eviction_callback_(e->key(), static_cast<Cache::Handle*>(e),
                           e->HasHit())) {
      // Callback took ownership of obj; just free handle
      free(e);
@@ -661,18 +662,18 @@ LRUCache::LRUCache(const LRUCacheOptions& opts) : ShardedCache(opts) {
 }

 Cache::ObjectPtr LRUCache::Value(Handle* handle) {
-  auto h = reinterpret_cast<const LRUHandle*>(handle);
+  auto h = static_cast<const LRUHandle*>(handle);
  return h->value;
 }

 size_t LRUCache::GetCharge(Handle* handle) const {
-  return reinterpret_cast<const LRUHandle*>(handle)->GetCharge(
+  return static_cast<const LRUHandle*>(handle)->GetCharge(
      GetShard(0).metadata_charge_policy_);
 }

 const Cache::CacheItemHelper* LRUCache::GetCacheItemHelper(
    Handle* handle) const {
-  auto h = reinterpret_cast<const LRUHandle*>(handle);
+  auto h = static_cast<const LRUHandle*>(handle);
  return h->helper;
 }

@@ -47,7 +47,7 @@ namespace lru_cache {
 // LRUCacheShard::Lookup.
 // While refs > 0, public properties like value and deleter must not change.

-struct LRUHandle {
+struct LRUHandle : public Cache::Handle {
  Cache::ObjectPtr value;
  const Cache::CacheItemHelper* helper;
  LRUHandle* next_hash;
@@ -32,7 +32,7 @@ namespace ROCKSDB_NAMESPACE {

 class LRUCacheTest : public testing::Test {
 public:
-  LRUCacheTest() {}
+  LRUCacheTest() = default;
  ~LRUCacheTest() override { DeleteCache(); }

  void DeleteCache() {
@@ -47,7 +47,7 @@ class LRUCacheTest : public testing::Test {
                double low_pri_pool_ratio = 1.0,
                bool use_adaptive_mutex = kDefaultToAdaptiveMutex) {
    DeleteCache();
-    cache_ = reinterpret_cast<LRUCacheShard*>(
+    cache_ = static_cast<LRUCacheShard*>(
        port::cacheline_aligned_alloc(sizeof(LRUCacheShard)));
    new (cache_) LRUCacheShard(capacity, /*strict_capacity_limit=*/false,
                               high_pri_pool_ratio, low_pri_pool_ratio,
@@ -57,10 +57,11 @@ class LRUCacheTest : public testing::Test {
  }

  void Insert(const std::string& key,
-              Cache::Priority priority = Cache::Priority::LOW) {
+              Cache::Priority priority = Cache::Priority::LOW,
+              size_t charge = 1) {
    EXPECT_OK(cache_->Insert(key, 0 /*hash*/, nullptr /*value*/,
-                             &kNoopCacheItemHelper, 1 /*charge*/,
-                             nullptr /*handle*/, priority));
+                             &kNoopCacheItemHelper, charge, nullptr /*handle*/,
+                             priority));
  }

  void Insert(char key, Cache::Priority priority = Cache::Priority::LOW) {
@@ -144,8 +145,10 @@ class LRUCacheTest : public testing::Test {
    ASSERT_EQ(num_bottom_pri_pool_keys, bottom_pri_pool_keys);
  }

- private:
+ protected:
  LRUCacheShard* cache_ = nullptr;
+
+ private:
  Cache::EvictionCallback eviction_callback_;
 };

@@ -378,7 +381,7 @@ class ClockCacheTest : public testing::Test {
  using Table = typename Shard::Table;
  using TableOpts = typename Table::Opts;

-  ClockCacheTest() {}
+  ClockCacheTest() = default;
  ~ClockCacheTest() override { DeleteShard(); }

  void DeleteShard() {
@@ -392,8 +395,7 @@ class ClockCacheTest : public testing::Test {
  void NewShard(size_t capacity, bool strict_capacity_limit = true,
                int eviction_effort_cap = 30) {
    DeleteShard();
-    shard_ =
-        reinterpret_cast<Shard*>(port::cacheline_aligned_alloc(sizeof(Shard)));
+    shard_ = static_cast<Shard*>(port::cacheline_aligned_alloc(sizeof(Shard)));

    TableOpts opts{1 /*value_size*/, eviction_effort_cap};
    new (shard_)
@@ -1977,7 +1979,7 @@ TEST_P(BasicSecondaryCacheTest, BasicWaitAllTest) {
    ah.priority = Cache::Priority::LOW;
    cache->StartAsyncLookup(ah);
  }
-  cache->WaitAll(&async_handles[0], async_handles.size());
+  cache->WaitAll(async_handles.data(), async_handles.size());
  for (size_t i = 0; i < async_handles.size(); ++i) {
    SCOPED_TRACE("i = " + std::to_string(i));
    Cache::Handle* result = async_handles[i].Result();
@@ -2704,6 +2706,23 @@ TEST_P(DBSecondaryCacheTest, TestSecondaryCacheOptionTwoDB) {
  ASSERT_OK(DestroyDB(dbname2, options));
 }

+TEST_F(LRUCacheTest, InsertAfterReducingCapacity) {
+  // Fix a bug in LRU cache where it may try to remove a low pri entry's
+  // charge from high pri pool. It causes
+  // Assertion failed: (high_pri_pool_usage_ >= lru_low_pri_->total_charge),
+  // function MaintainPoolSize, file lru_cache.cc
+  NewCache(/*capacity=*/10, /*high_pri_pool_ratio=*/0.2,
+           /*low_pri_pool_ratio=*/0.8);
+  // high pri pool size and usage are both 2
+  Insert("x", Cache::Priority::HIGH);
+  Insert("y", Cache::Priority::HIGH);
+  cache_->SetCapacity(5);
+  // high_pri_pool_size is 1, the next time we try to maintain pool size,
+  // we will move entries from high pri pool to low pri pool
+  // The bug was deducting this entry's charge from high pri pool usage.
+  Insert("aaa", Cache::Priority::LOW, /*charge=*/3);
+}
+
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -139,7 +139,7 @@ class ShardedCache : public ShardedCacheBase {

  explicit ShardedCache(const ShardedCacheOptions& opts)
      : ShardedCacheBase(opts),
-        shards_(reinterpret_cast<CacheShard*>(port::cacheline_aligned_alloc(
+        shards_(static_cast<CacheShard*>(port::cacheline_aligned_alloc(
            sizeof(CacheShard) * GetNumShards()))),
        destroy_shards_in_dtor_(false) {}

@@ -192,7 +192,7 @@ class ShardedCache : public ShardedCacheBase {
    HashVal hash = CacheShard::ComputeHash(key, hash_seed_);
    HandleImpl* result = GetShard(hash).CreateStandalone(
        key, hash, obj, helper, charge, allow_uncharged);
-    return reinterpret_cast<Handle*>(result);
+    return static_cast<Handle*>(result);
  }

  Handle* Lookup(const Slice& key, const CacheItemHelper* helper = nullptr,
@@ -202,7 +202,7 @@ class ShardedCache : public ShardedCacheBase {
    HashVal hash = CacheShard::ComputeHash(key, hash_seed_);
    HandleImpl* result = GetShard(hash).Lookup(key, hash, helper,
                                               create_context, priority, stats);
-    return reinterpret_cast<Handle*>(result);
+    return static_cast<Handle*>(result);
  }

  void Erase(const Slice& key) override {
@@ -212,11 +212,11 @@ class ShardedCache : public ShardedCacheBase {

  bool Release(Handle* handle, bool useful,
               bool erase_if_last_ref = false) override {
-    auto h = reinterpret_cast<HandleImpl*>(handle);
+    auto h = static_cast<HandleImpl*>(handle);
    return GetShard(h->GetHash()).Release(h, useful, erase_if_last_ref);
  }
  bool Ref(Handle* handle) override {
-    auto h = reinterpret_cast<HandleImpl*>(handle);
+    auto h = static_cast<HandleImpl*>(handle);
    return GetShard(h->GetHash()).Ref(h);
  }
  bool Release(Handle* handle, bool erase_if_last_ref = false) override {
@@ -259,7 +259,7 @@ class ShardedCache : public ShardedCacheBase {
    } while (remaining_work);
  }

-  virtual void EraseUnRefEntries() override {
+  void EraseUnRefEntries() override {
    ForEachShard([](CacheShard* cs) { cs->EraseUnRefEntries(); });
  }

@@ -42,27 +42,25 @@ class TieredSecondaryCache : public SecondaryCacheWrapper {

  // This is a no-op as we currently don't allow demotion (i.e
  // insertion by the upper layer) of evicted blocks.
-  virtual Status Insert(const Slice& /*key*/, Cache::ObjectPtr /*obj*/,
-                        const Cache::CacheItemHelper* /*helper*/,
-                        bool /*force_insert*/) override {
+  Status Insert(const Slice& /*key*/, Cache::ObjectPtr /*obj*/,
+                const Cache::CacheItemHelper* /*helper*/,
+                bool /*force_insert*/) override {
    return Status::OK();
  }

  // Warm up the nvm tier directly
-  virtual Status InsertSaved(
-      const Slice& key, const Slice& saved,
-      CompressionType type = CompressionType::kNoCompression,
-      CacheTier source = CacheTier::kVolatileTier) override {
+  Status InsertSaved(const Slice& key, const Slice& saved,
+                     CompressionType type = CompressionType::kNoCompression,
+                     CacheTier source = CacheTier::kVolatileTier) override {
    return nvm_sec_cache_->InsertSaved(key, saved, type, source);
  }

-  virtual std::unique_ptr<SecondaryCacheResultHandle> Lookup(
+  std::unique_ptr<SecondaryCacheResultHandle> Lookup(
      const Slice& key, const Cache::CacheItemHelper* helper,
      Cache::CreateContext* create_context, bool wait, bool advise_erase,
      Statistics* stats, bool& kept_in_sec_cache) override;

-  virtual void WaitAll(
-      std::vector<SecondaryCacheResultHandle*> handles) override;
+  void WaitAll(std::vector<SecondaryCacheResultHandle*> handles) override;

 private:
  struct CreateContext : public Cache::CreateContext {
@@ -386,7 +386,7 @@ TEST_F(DBTieredSecondaryCacheTest, BasicMultiGetTest) {
  keys.push_back(Key(8));
  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
  ASSERT_EQ(values.size(), keys.size());
-  for (auto value : values) {
+  for (const auto& value : values) {
    ASSERT_EQ(1007, value.size());
  }
  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 3u);
@@ -400,7 +400,7 @@ TEST_F(DBTieredSecondaryCacheTest, BasicMultiGetTest) {
  keys.push_back(Key(20));
  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
  ASSERT_EQ(values.size(), keys.size());
-  for (auto value : values) {
+  for (const auto& value : values) {
    ASSERT_EQ(1007, value.size());
  }
  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u);
@@ -414,7 +414,7 @@ TEST_F(DBTieredSecondaryCacheTest, BasicMultiGetTest) {
  keys.push_back(Key(8));
  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
  ASSERT_EQ(values.size(), keys.size());
-  for (auto value : values) {
+  for (const auto& value : values) {
    ASSERT_EQ(1007, value.size());
  }
  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u);
@@ -428,7 +428,7 @@ TEST_F(DBTieredSecondaryCacheTest, BasicMultiGetTest) {
  keys.push_back(Key(8));
  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
  ASSERT_EQ(values.size(), keys.size());
-  for (auto value : values) {
+  for (const auto& value : values) {
    ASSERT_EQ(1007, value.size());
  }
  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u);
@@ -442,7 +442,7 @@ TEST_F(DBTieredSecondaryCacheTest, BasicMultiGetTest) {
  keys.push_back(Key(8));
  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
  ASSERT_EQ(values.size(), keys.size());
-  for (auto value : values) {
+  for (const auto& value : values) {
    ASSERT_EQ(1007, value.size());
  }
  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u);
@@ -456,7 +456,7 @@ TEST_F(DBTieredSecondaryCacheTest, BasicMultiGetTest) {
  keys.push_back(Key(20));
  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
  ASSERT_EQ(values.size(), keys.size());
-  for (auto value : values) {
+  for (const auto& value : values) {
    ASSERT_EQ(1007, value.size());
  }
  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u);
@@ -470,7 +470,7 @@ TEST_F(DBTieredSecondaryCacheTest, BasicMultiGetTest) {
  keys.push_back(Key(20));
  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
  ASSERT_EQ(values.size(), keys.size());
-  for (auto value : values) {
+  for (const auto& value : values) {
    ASSERT_EQ(1007, value.size());
  }
  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u);
@@ -484,7 +484,7 @@ TEST_F(DBTieredSecondaryCacheTest, BasicMultiGetTest) {
  keys.push_back(Key(20));
  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
  ASSERT_EQ(values.size(), keys.size());
-  for (auto value : values) {
+  for (const auto& value : values) {
    ASSERT_EQ(1007, value.size());
  }
  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u);
@@ -528,7 +528,7 @@ TEST_F(DBTieredSecondaryCacheTest, WaitAllTest) {
  keys.push_back(Key(8));
  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
  ASSERT_EQ(values.size(), keys.size());
-  for (auto value : values) {
+  for (const auto& value : values) {
    ASSERT_EQ(1007, value.size());
  }
  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 3u);
@@ -542,7 +542,7 @@ TEST_F(DBTieredSecondaryCacheTest, WaitAllTest) {
  keys.push_back(Key(20));
  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
  ASSERT_EQ(values.size(), keys.size());
-  for (auto value : values) {
+  for (const auto& value : values) {
    ASSERT_EQ(1007, value.size());
  }
  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u);
@@ -561,7 +561,7 @@ TEST_F(DBTieredSecondaryCacheTest, WaitAllTest) {
  keys.push_back(Key(36));
  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
  ASSERT_EQ(values.size(), keys.size());
-  for (auto value : values) {
+  for (const auto& value : values) {
    ASSERT_EQ(1007, value.size());
  }
  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 10u);
@@ -582,7 +582,7 @@ TEST_F(DBTieredSecondaryCacheTest, WaitAllTest) {
  keys.push_back(Key(8));
  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
  ASSERT_EQ(values.size(), keys.size());
-  for (auto value : values) {
+  for (const auto& value : values) {
    ASSERT_EQ(1007, value.size());
  }
  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 10u);
@@ -629,7 +629,7 @@ TEST_F(DBTieredSecondaryCacheTest, ReadyBeforeWaitAllTest) {
  keys.push_back(Key(8));
  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
  ASSERT_EQ(values.size(), keys.size());
-  for (auto value : values) {
+  for (const auto& value : values) {
    ASSERT_EQ(1007, value.size());
  }
  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 3u);
@@ -644,7 +644,7 @@ TEST_F(DBTieredSecondaryCacheTest, ReadyBeforeWaitAllTest) {
  keys.push_back(Key(20));
  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
  ASSERT_EQ(values.size(), keys.size());
-  for (auto value : values) {
+  for (const auto& value : values) {
    ASSERT_EQ(1007, value.size());
  }
  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u);
@@ -659,7 +659,7 @@ TEST_F(DBTieredSecondaryCacheTest, ReadyBeforeWaitAllTest) {
  keys.push_back(Key(8));
  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
  ASSERT_EQ(values.size(), keys.size());
-  for (auto value : values) {
+  for (const auto& value : values) {
    ASSERT_EQ(1007, value.size());
  }
  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 6u);
@@ -676,7 +676,7 @@ TEST_F(DBTieredSecondaryCacheTest, ReadyBeforeWaitAllTest) {
  keys.push_back(Key(36));
  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
  ASSERT_EQ(values.size(), keys.size());
-  for (auto value : values) {
+  for (const auto& value : values) {
    ASSERT_EQ(1007, value.size());
  }
  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 8u);
@@ -691,7 +691,7 @@ TEST_F(DBTieredSecondaryCacheTest, ReadyBeforeWaitAllTest) {
  keys.push_back(Key(36));
  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
  ASSERT_EQ(values.size(), keys.size());
-  for (auto value : values) {
+  for (const auto& value : values) {
    ASSERT_EQ(1007, value.size());
  }
  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 8u);
@@ -155,7 +155,7 @@ class BasicTypedCacheInterface : public BaseCacheInterface<CachePtr>,
  using BaseCacheInterface<CachePtr>::BaseCacheInterface;
  struct TypedAsyncLookupHandle : public Cache::AsyncLookupHandle {
    TypedHandle* Result() {
-      return reinterpret_cast<TypedHandle*>(Cache::AsyncLookupHandle::Result());
+      return static_cast<TypedHandle*>(Cache::AsyncLookupHandle::Result());
    }
  };

@@ -169,8 +169,7 @@ class BasicTypedCacheInterface : public BaseCacheInterface<CachePtr>,
  }

  inline TypedHandle* Lookup(const Slice& key, Statistics* stats = nullptr) {
-    return reinterpret_cast<TypedHandle*>(
-        this->cache_->BasicLookup(key, stats));
+    return static_cast<TypedHandle*>(this->cache_->BasicLookup(key, stats));
  }

  inline void StartAsyncLookup(TypedAsyncLookupHandle& async_handle) {
@@ -347,7 +346,7 @@ class FullTypedCacheInterface
      Priority priority = Priority::LOW, Statistics* stats = nullptr,
      CacheTier lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier) {
    if (lowest_used_cache_tier > CacheTier::kVolatileTier) {
-      return reinterpret_cast<TypedHandle*>(this->cache_->Lookup(
+      return static_cast<TypedHandle*>(this->cache_->Lookup(
          key, GetFullHelper(), create_context, priority, stats));
    } else {
      return BasicTypedCacheInterface<TValue, kRole, CachePtr>::Lookup(key,
@@ -1,29 +1,29 @@
 # - Find zstd
 # Find the zstd compression library and includes
 #
-# zstd_INCLUDE_DIRS - where to find zstd.h, etc.
-# zstd_LIBRARIES - List of libraries when using zstd.
-# zstd_FOUND - True if zstd found.
+# ZSTD_INCLUDE_DIRS - where to find zstd.h, etc.
+# ZSTD_LIBRARIES - List of libraries when using zstd.
+# ZSTD_FOUND - True if zstd found.

-find_path(zstd_INCLUDE_DIRS
+find_path(ZSTD_INCLUDE_DIRS
  NAMES zstd.h
  HINTS ${zstd_ROOT_DIR}/include)

-find_library(zstd_LIBRARIES
+find_library(ZSTD_LIBRARIES
  NAMES zstd
  HINTS ${zstd_ROOT_DIR}/lib)

 include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(zstd DEFAULT_MSG zstd_LIBRARIES zstd_INCLUDE_DIRS)
+find_package_handle_standard_args(zstd DEFAULT_MSG ZSTD_LIBRARIES ZSTD_INCLUDE_DIRS)

 mark_as_advanced(
-  zstd_LIBRARIES
-  zstd_INCLUDE_DIRS)
+  ZSTD_LIBRARIES
+  ZSTD_INCLUDE_DIRS)

-if(zstd_FOUND AND NOT (TARGET zstd::zstd))
+if(ZSTD_FOUND AND NOT (TARGET zstd::zstd))
  add_library (zstd::zstd UNKNOWN IMPORTED)
  set_target_properties(zstd::zstd
    PROPERTIES
-      IMPORTED_LOCATION ${zstd_LIBRARIES}
-      INTERFACE_INCLUDE_DIRECTORIES ${zstd_INCLUDE_DIRS})
+      IMPORTED_LOCATION ${ZSTD_LIBRARIES}
+      INTERFACE_INCLUDE_DIRECTORIES ${ZSTD_INCLUDE_DIRS})
 endif()
@@ -43,14 +43,13 @@ void ArenaWrappedDBIter::Init(
    Env* env, const ReadOptions& read_options, const ImmutableOptions& ioptions,
    const MutableCFOptions& mutable_cf_options, const Version* version,
    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iteration,
-    uint64_t version_number, ReadCallback* read_callback, DBImpl* db_impl,
-    ColumnFamilyData* cfd, bool expose_blob_index, bool allow_refresh) {
+    uint64_t version_number, ReadCallback* read_callback,
+    ColumnFamilyHandleImpl* cfh, bool expose_blob_index, bool allow_refresh) {
  auto mem = arena_.AllocateAligned(sizeof(DBIter));
-  db_iter_ =
-      new (mem) DBIter(env, read_options, ioptions, mutable_cf_options,
-                       ioptions.user_comparator, /* iter */ nullptr, version,
-                       sequence, true, max_sequential_skip_in_iteration,
-                       read_callback, db_impl, cfd, expose_blob_index);
+  db_iter_ = new (mem) DBIter(
+      env, read_options, ioptions, mutable_cf_options, ioptions.user_comparator,
+      /* iter */ nullptr, version, sequence, true,
+      max_sequential_skip_in_iteration, read_callback, cfh, expose_blob_index);
  sv_number_ = version_number;
  read_options_ = read_options;
  allow_refresh_ = allow_refresh;
@@ -65,40 +64,44 @@ void ArenaWrappedDBIter::Init(
 Status ArenaWrappedDBIter::Refresh() { return Refresh(nullptr); }

 Status ArenaWrappedDBIter::Refresh(const Snapshot* snapshot) {
-  if (cfd_ == nullptr || db_impl_ == nullptr || !allow_refresh_) {
+  if (cfh_ == nullptr || !allow_refresh_) {
    return Status::NotSupported("Creating renew iterator is not allowed.");
  }
  assert(db_iter_ != nullptr);
+  auto cfd = cfh_->cfd();
+  auto db_impl = cfh_->db();
+
  // TODO(yiwu): For last_seq_same_as_publish_seq_==false, this is not the
  // correct behavior. Will be corrected automatically when we take a snapshot
  // here for the case of WritePreparedTxnDB.
-  uint64_t cur_sv_number = cfd_->GetSuperVersionNumber();
+  uint64_t cur_sv_number = cfd->GetSuperVersionNumber();
  // If we recreate a new internal iterator below (NewInternalIterator()),
  // we will pass in read_options_. We need to make sure it
  // has the right snapshot.
  read_options_.snapshot = snapshot;
  TEST_SYNC_POINT("ArenaWrappedDBIter::Refresh:1");
  TEST_SYNC_POINT("ArenaWrappedDBIter::Refresh:2");
+
  auto reinit_internal_iter = [&]() {
    Env* env = db_iter_->env();
    db_iter_->~DBIter();
    arena_.~Arena();
    new (&arena_) Arena();

-    SuperVersion* sv = cfd_->GetReferencedSuperVersion(db_impl_);
+    SuperVersion* sv = cfd->GetReferencedSuperVersion(db_impl);
    assert(sv->version_number >= cur_sv_number);
-    SequenceNumber read_seq = GetSeqNum(db_impl_, snapshot);
+    SequenceNumber read_seq = GetSeqNum(db_impl, snapshot);
    if (read_callback_) {
      read_callback_->Refresh(read_seq);
    }
-    Init(env, read_options_, *(cfd_->ioptions()), sv->mutable_cf_options,
+    Init(env, read_options_, *(cfd->ioptions()), sv->mutable_cf_options,
         sv->current, read_seq,
         sv->mutable_cf_options.max_sequential_skip_in_iterations,
-         sv->version_number, read_callback_, db_impl_, cfd_, expose_blob_index_,
+         sv->version_number, read_callback_, cfh_, expose_blob_index_,
         allow_refresh_);

-    InternalIterator* internal_iter = db_impl_->NewInternalIterator(
-        read_options_, cfd_, sv, &arena_, read_seq,
+    InternalIterator* internal_iter = db_impl->NewInternalIterator(
+        read_options_, cfd, sv, &arena_, read_seq,
        /* allow_unprepared_value */ true, /* db_iter */ this);
    SetIterUnderDBIter(internal_iter);
  };
@@ -107,10 +110,10 @@ Status ArenaWrappedDBIter::Refresh(const Snapshot* snapshot) {
      reinit_internal_iter();
      break;
    } else {
-      SequenceNumber read_seq = GetSeqNum(db_impl_, snapshot);
+      SequenceNumber read_seq = GetSeqNum(db_impl, snapshot);
      // Refresh range-tombstones in MemTable
      if (!read_options_.ignore_range_deletions) {
-        SuperVersion* sv = cfd_->GetThreadLocalSuperVersion(db_impl_);
+        SuperVersion* sv = cfd->GetThreadLocalSuperVersion(db_impl);
        TEST_SYNC_POINT_CALLBACK("ArenaWrappedDBIter::Refresh:SV", nullptr);
        auto t = sv->mem->NewRangeTombstoneIterator(
            read_options_, read_seq, false /* immutable_memtable */);
@@ -123,13 +126,13 @@ Status ArenaWrappedDBIter::Refresh(const Snapshot* snapshot) {
          // will be freed during db_iter destruction there.
          if (memtable_range_tombstone_iter_) {
            assert(!*memtable_range_tombstone_iter_ ||
-                   sv_number_ != cfd_->GetSuperVersionNumber());
+                   sv_number_ != cfd->GetSuperVersionNumber());
          }
          delete t;
        } else {  // current mutable memtable has range tombstones
          if (!memtable_range_tombstone_iter_) {
            delete t;
-            db_impl_->ReturnAndCleanupSuperVersion(cfd_, sv);
+            db_impl->ReturnAndCleanupSuperVersion(cfd, sv);
            // The memtable under DBIter did not have range tombstone before
            // refresh.
            reinit_internal_iter();
@@ -138,13 +141,13 @@ Status ArenaWrappedDBIter::Refresh(const Snapshot* snapshot) {
            delete *memtable_range_tombstone_iter_;
            *memtable_range_tombstone_iter_ = new TruncatedRangeDelIterator(
                std::unique_ptr<FragmentedRangeTombstoneIterator>(t),
-                &cfd_->internal_comparator(), nullptr, nullptr);
+                &cfd->internal_comparator(), nullptr, nullptr);
          }
        }
-        db_impl_->ReturnAndCleanupSuperVersion(cfd_, sv);
+        db_impl->ReturnAndCleanupSuperVersion(cfd, sv);
      }
      // Check again if the latest super version number is changed
-      uint64_t latest_sv_number = cfd_->GetSuperVersionNumber();
+      uint64_t latest_sv_number = cfd->GetSuperVersionNumber();
      if (latest_sv_number != cur_sv_number) {
        // If the super version number is changed after refreshing,
        // fallback to Re-Init the InternalIterator
@@ -163,14 +166,14 @@ ArenaWrappedDBIter* NewArenaWrappedDbIterator(
    Env* env, const ReadOptions& read_options, const ImmutableOptions& ioptions,
    const MutableCFOptions& mutable_cf_options, const Version* version,
    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations,
-    uint64_t version_number, ReadCallback* read_callback, DBImpl* db_impl,
-    ColumnFamilyData* cfd, bool expose_blob_index, bool allow_refresh) {
+    uint64_t version_number, ReadCallback* read_callback,
+    ColumnFamilyHandleImpl* cfh, bool expose_blob_index, bool allow_refresh) {
  ArenaWrappedDBIter* iter = new ArenaWrappedDBIter();
  iter->Init(env, read_options, ioptions, mutable_cf_options, version, sequence,
             max_sequential_skip_in_iterations, version_number, read_callback,
-             db_impl, cfd, expose_blob_index, allow_refresh);
-  if (db_impl != nullptr && cfd != nullptr && allow_refresh) {
-    iter->StoreRefreshInfo(db_impl, cfd, read_callback, expose_blob_index);
+             cfh, expose_blob_index, allow_refresh);
+  if (cfh != nullptr && allow_refresh) {
+    iter->StoreRefreshInfo(cfh, read_callback, expose_blob_index);
  }

  return iter;
@@ -87,15 +87,14 @@ class ArenaWrappedDBIter : public Iterator {
            const MutableCFOptions& mutable_cf_options, const Version* version,
            const SequenceNumber& sequence,
            uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
-            ReadCallback* read_callback, DBImpl* db_impl, ColumnFamilyData* cfd,
+            ReadCallback* read_callback, ColumnFamilyHandleImpl* cfh,
            bool expose_blob_index, bool allow_refresh);

  // Store some parameters so we can refresh the iterator at a later point
  // with these same params
-  void StoreRefreshInfo(DBImpl* db_impl, ColumnFamilyData* cfd,
+  void StoreRefreshInfo(ColumnFamilyHandleImpl* cfh,
                        ReadCallback* read_callback, bool expose_blob_index) {
-    db_impl_ = db_impl;
-    cfd_ = cfd;
+    cfh_ = cfh;
    read_callback_ = read_callback;
    expose_blob_index_ = expose_blob_index;
  }
@@ -104,8 +103,7 @@ class ArenaWrappedDBIter : public Iterator {
  DBIter* db_iter_ = nullptr;
  Arena arena_;
  uint64_t sv_number_;
-  ColumnFamilyData* cfd_ = nullptr;
-  DBImpl* db_impl_ = nullptr;
+  ColumnFamilyHandleImpl* cfh_ = nullptr;
  ReadOptions read_options_;
  ReadCallback* read_callback_;
  bool expose_blob_index_ = false;
@@ -116,13 +114,13 @@ class ArenaWrappedDBIter : public Iterator {
 };

 // Generate the arena wrapped iterator class.
-// `db_impl` and `cfd` are used for reneweal. If left null, renewal will not
+// `cfh` is used for reneweal. If left null, renewal will not
 // be supported.
-extern ArenaWrappedDBIter* NewArenaWrappedDbIterator(
+ArenaWrappedDBIter* NewArenaWrappedDbIterator(
    Env* env, const ReadOptions& read_options, const ImmutableOptions& ioptions,
    const MutableCFOptions& mutable_cf_options, const Version* version,
    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations,
    uint64_t version_number, ReadCallback* read_callback,
-    DBImpl* db_impl = nullptr, ColumnFamilyData* cfd = nullptr,
-    bool expose_blob_index = false, bool allow_refresh = true);
+    ColumnFamilyHandleImpl* cfh = nullptr, bool expose_blob_index = false,
+    bool allow_refresh = true);
 }  // namespace ROCKSDB_NAMESPACE
@@ -405,7 +405,7 @@ TEST_F(BlobFileReaderTest, CreateReaderAndGetBlob) {

    requests_buf[0] =
        BlobReadRequest(key_refs[0], blob_offsets[0], blob_sizes[0],
-                        kNoCompression, nullptr, &statuses_buf[0]);
+                        kNoCompression, nullptr, statuses_buf.data());
    requests_buf[1] =
        BlobReadRequest(key_refs[1], blob_offsets[1], blob_sizes[1] + 1,
                        kNoCompression, nullptr, &statuses_buf[1]);
@@ -168,8 +168,8 @@ TEST_F(BlobSourceTest, GetBlobsFromCache) {

  uint64_t file_size = BlobLogHeader::kSize;
  for (size_t i = 0; i < num_blobs; ++i) {
-    keys.push_back({key_strs[i]});
-    blobs.push_back({blob_strs[i]});
+    keys.emplace_back(key_strs[i]);
+    blobs.emplace_back(blob_strs[i]);
    file_size += BlobLogRecord::kHeaderSize + keys[i].size() + blobs[i].size();
  }
  file_size += BlobLogFooter::kSize;
@@ -482,8 +482,8 @@ TEST_F(BlobSourceTest, GetCompressedBlobs) {
  std::vector<Slice> blobs;

  for (size_t i = 0; i < num_blobs; ++i) {
-    keys.push_back({key_strs[i]});
-    blobs.push_back({blob_strs[i]});
+    keys.emplace_back(key_strs[i]);
+    blobs.emplace_back(blob_strs[i]);
  }

  std::vector<uint64_t> blob_offsets(keys.size());
@@ -610,8 +610,8 @@ TEST_F(BlobSourceTest, MultiGetBlobsFromMultiFiles) {
  uint64_t file_size = BlobLogHeader::kSize;
  uint64_t blob_value_bytes = 0;
  for (size_t i = 0; i < num_blobs; ++i) {
-    keys.push_back({key_strs[i]});
-    blobs.push_back({blob_strs[i]});
+    keys.emplace_back(key_strs[i]);
+    blobs.emplace_back(blob_strs[i]);
    blob_value_bytes += blobs[i].size();
    file_size += BlobLogRecord::kHeaderSize + keys[i].size() + blobs[i].size();
  }
@@ -802,8 +802,8 @@ TEST_F(BlobSourceTest, MultiGetBlobsFromCache) {

  uint64_t file_size = BlobLogHeader::kSize;
  for (size_t i = 0; i < num_blobs; ++i) {
-    keys.push_back({key_strs[i]});
-    blobs.push_back({blob_strs[i]});
+    keys.emplace_back(key_strs[i]);
+    blobs.emplace_back(blob_strs[i]);
    file_size += BlobLogRecord::kHeaderSize + keys[i].size() + blobs[i].size();
  }
  file_size += BlobLogFooter::kSize;
@@ -1164,7 +1164,7 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) {
    ASSERT_OK(blob_source.GetBlob(read_options, keys[0], file_number,
                                  blob_offsets[0], file_size, blob_sizes[0],
                                  kNoCompression, nullptr /* prefetch_buffer */,
-                                  &values[0], nullptr /* bytes_read */));
+                                  values.data(), nullptr /* bytes_read */));
    // Release cache handle
    values[0].Reset();

@@ -1183,7 +1183,7 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) {
    ASSERT_OK(blob_source.GetBlob(read_options, keys[0], file_number,
                                  blob_offsets[0], file_size, blob_sizes[0],
                                  kNoCompression, nullptr /* prefetch_buffer */,
-                                  &values[0], nullptr /* bytes_read */));
+                                  values.data(), nullptr /* bytes_read */));
    ASSERT_EQ(values[0], blobs[0]);
    ASSERT_TRUE(
        blob_source.TEST_BlobInCache(file_number, file_size, blob_offsets[0]));
@@ -1263,7 +1263,7 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) {
      ASSERT_OK(blob_source.GetBlob(
          read_options, keys[0], file_number, blob_offsets[0], file_size,
          blob_sizes[0], kNoCompression, nullptr /* prefetch_buffer */,
-          &values[0], nullptr /* bytes_read */));
+          values.data(), nullptr /* bytes_read */));
      ASSERT_EQ(values[0], blobs[0]);

      // Release cache handle
@@ -1365,8 +1365,8 @@ class BlobSourceCacheReservationTest : public DBTestBase {

    blob_file_size_ = BlobLogHeader::kSize;
    for (size_t i = 0; i < kNumBlobs; ++i) {
-      keys_.push_back({key_strs_[i]});
-      blobs_.push_back({blob_strs_[i]});
+      keys_.emplace_back(key_strs_[i]);
+      blobs_.emplace_back(blob_strs_[i]);
      blob_file_size_ +=
          BlobLogRecord::kHeaderSize + keys_[i].size() + blobs_[i].size();
    }
@@ -418,8 +418,8 @@ TEST_F(DBBlobBasicTest, MultiGetBlobs) {
    std::array<PinnableSlice, num_keys> values;
    std::array<Status, num_keys> statuses;

-    db_->MultiGet(read_options, db_->DefaultColumnFamily(), num_keys, &keys[0],
-                  &values[0], &statuses[0]);
+    db_->MultiGet(read_options, db_->DefaultColumnFamily(), num_keys,
+                  keys.data(), values.data(), statuses.data());

    ASSERT_OK(statuses[0]);
    ASSERT_EQ(values[0], first_value);
@@ -441,8 +441,8 @@ TEST_F(DBBlobBasicTest, MultiGetBlobs) {
    std::array<PinnableSlice, num_keys> values;
    std::array<Status, num_keys> statuses;

-    db_->MultiGet(read_options, db_->DefaultColumnFamily(), num_keys, &keys[0],
-                  &values[0], &statuses[0]);
+    db_->MultiGet(read_options, db_->DefaultColumnFamily(), num_keys,
+                  keys.data(), values.data(), statuses.data());

    ASSERT_OK(statuses[0]);
    ASSERT_EQ(values[0], first_value);
@@ -512,8 +512,8 @@ TEST_F(DBBlobBasicTest, MultiGetBlobsFromCache) {
    std::array<PinnableSlice, num_keys> values;
    std::array<Status, num_keys> statuses;

-    db_->MultiGet(read_options, db_->DefaultColumnFamily(), num_keys, &keys[0],
-                  &values[0], &statuses[0]);
+    db_->MultiGet(read_options, db_->DefaultColumnFamily(), num_keys,
+                  keys.data(), values.data(), statuses.data());

    ASSERT_OK(statuses[0]);
    ASSERT_EQ(values[0], first_value);
@@ -534,8 +534,8 @@ TEST_F(DBBlobBasicTest, MultiGetBlobsFromCache) {
    std::array<PinnableSlice, num_keys> values;
    std::array<Status, num_keys> statuses;

-    db_->MultiGet(read_options, db_->DefaultColumnFamily(), num_keys, &keys[0],
-                  &values[0], &statuses[0]);
+    db_->MultiGet(read_options, db_->DefaultColumnFamily(), num_keys,
+                  keys.data(), values.data(), statuses.data());

    ASSERT_OK(statuses[0]);
    ASSERT_EQ(values[0], first_value);
@@ -553,8 +553,8 @@ TEST_F(DBBlobBasicTest, MultiGetBlobsFromCache) {
    std::array<PinnableSlice, num_keys> values;
    std::array<Status, num_keys> statuses;

-    db_->MultiGet(read_options, db_->DefaultColumnFamily(), num_keys, &keys[0],
-                  &values[0], &statuses[0]);
+    db_->MultiGet(read_options, db_->DefaultColumnFamily(), num_keys,
+                  keys.data(), values.data(), statuses.data());

    ASSERT_OK(statuses[0]);
    ASSERT_EQ(values[0], first_value);
@@ -574,8 +574,8 @@ TEST_F(DBBlobBasicTest, MultiGetBlobsFromCache) {
    std::array<PinnableSlice, num_keys> values;
    std::array<Status, num_keys> statuses;

-    db_->MultiGet(read_options, db_->DefaultColumnFamily(), num_keys, &keys[0],
-                  &values[0], &statuses[0]);
+    db_->MultiGet(read_options, db_->DefaultColumnFamily(), num_keys,
+                  keys.data(), values.data(), statuses.data());

    ASSERT_OK(statuses[0]);
    ASSERT_EQ(values[0], first_value);
@@ -758,8 +758,8 @@ TEST_F(DBBlobBasicTest, MultiGetWithDirectIO) {
    //
    // [offset=0, len=12288]

-    db_->MultiGet(ReadOptions(), db_->DefaultColumnFamily(), num_keys, &keys[0],
-                  &values[0], &statuses[0]);
+    db_->MultiGet(ReadOptions(), db_->DefaultColumnFamily(), num_keys,
+                  keys.data(), values.data(), statuses.data());

    SyncPoint::GetInstance()->DisableProcessing();
    SyncPoint::GetInstance()->ClearAllCallBacks();
@@ -829,8 +829,8 @@ TEST_F(DBBlobBasicTest, MultiGetBlobsFromMultipleFiles) {
  {
    std::array<PinnableSlice, kNumKeys> values;
    std::array<Status, kNumKeys> statuses;
-    db_->MultiGet(read_options, db_->DefaultColumnFamily(), kNumKeys, &keys[0],
-                  &values[0], &statuses[0]);
+    db_->MultiGet(read_options, db_->DefaultColumnFamily(), kNumKeys,
+                  keys.data(), values.data(), statuses.data());

    for (size_t i = 0; i < kNumKeys; ++i) {
      ASSERT_OK(statuses[i]);
@@ -843,8 +843,8 @@ TEST_F(DBBlobBasicTest, MultiGetBlobsFromMultipleFiles) {
  {
    std::array<PinnableSlice, kNumKeys> values;
    std::array<Status, kNumKeys> statuses;
-    db_->MultiGet(read_options, db_->DefaultColumnFamily(), kNumKeys, &keys[0],
-                  &values[0], &statuses[0]);
+    db_->MultiGet(read_options, db_->DefaultColumnFamily(), kNumKeys,
+                  keys.data(), values.data(), statuses.data());

    for (size_t i = 0; i < kNumKeys; ++i) {
      ASSERT_TRUE(statuses[i].IsIncomplete());
@@ -858,8 +858,8 @@ TEST_F(DBBlobBasicTest, MultiGetBlobsFromMultipleFiles) {
  {
    std::array<PinnableSlice, kNumKeys> values;
    std::array<Status, kNumKeys> statuses;
-    db_->MultiGet(read_options, db_->DefaultColumnFamily(), kNumKeys, &keys[0],
-                  &values[0], &statuses[0]);
+    db_->MultiGet(read_options, db_->DefaultColumnFamily(), kNumKeys,
+                  keys.data(), values.data(), statuses.data());

    for (size_t i = 0; i < kNumKeys; ++i) {
      ASSERT_OK(statuses[i]);
@@ -872,8 +872,8 @@ TEST_F(DBBlobBasicTest, MultiGetBlobsFromMultipleFiles) {
  {
    std::array<PinnableSlice, kNumKeys> values;
    std::array<Status, kNumKeys> statuses;
-    db_->MultiGet(read_options, db_->DefaultColumnFamily(), kNumKeys, &keys[0],
-                  &values[0], &statuses[0]);
+    db_->MultiGet(read_options, db_->DefaultColumnFamily(), kNumKeys,
+                  keys.data(), values.data(), statuses.data());

    for (size_t i = 0; i < kNumKeys; ++i) {
      ASSERT_OK(statuses[i]);
@@ -1182,6 +1182,30 @@ TEST_F(DBBlobBasicTest, GetMergeBlobWithPut) {
  ASSERT_EQ(Get("Key1"), "v1,v2,v3");
 }

+TEST_F(DBBlobBasicTest, GetMergeBlobFromMemoryTier) {
+  Options options = GetDefaultOptions();
+  options.merge_operator = MergeOperators::CreateStringAppendOperator();
+  options.enable_blob_files = true;
+  options.min_blob_size = 0;
+
+  Reopen(options);
+
+  ASSERT_OK(Put(Key(0), "v1"));
+  ASSERT_OK(Flush());
+  ASSERT_OK(Merge(Key(0), "v2"));
+  ASSERT_OK(Flush());
+
+  // Regular `Get()` loads data block to cache.
+  std::string value;
+  ASSERT_OK(db_->Get(ReadOptions(), Key(0), &value));
+  ASSERT_EQ("v1,v2", value);
+
+  // Base value blob is still uncached, so an in-memory read will fail.
+  ReadOptions read_options;
+  read_options.read_tier = kBlockCacheTier;
+  ASSERT_TRUE(db_->Get(read_options, Key(0), &value).IsIncomplete());
+}
+
 TEST_F(DBBlobBasicTest, MultiGetMergeBlobWithPut) {
  constexpr size_t num_keys = 3;

@@ -1206,8 +1230,8 @@ TEST_F(DBBlobBasicTest, MultiGetMergeBlobWithPut) {
  std::array<PinnableSlice, num_keys> values;
  std::array<Status, num_keys> statuses;

-  db_->MultiGet(ReadOptions(), db_->DefaultColumnFamily(), num_keys, &keys[0],
-                &values[0], &statuses[0]);
+  db_->MultiGet(ReadOptions(), db_->DefaultColumnFamily(), num_keys,
+                keys.data(), values.data(), statuses.data());

  ASSERT_OK(statuses[0]);
  ASSERT_EQ(values[0], "v0_0,v0_1,v0_2");
@@ -1470,8 +1494,8 @@ TEST_P(DBBlobBasicIOErrorMultiGetTest, MultiGetBlobs_IOError) {
  });
  SyncPoint::GetInstance()->EnableProcessing();

-  db_->MultiGet(ReadOptions(), db_->DefaultColumnFamily(), num_keys, &keys[0],
-                &values[0], &statuses[0]);
+  db_->MultiGet(ReadOptions(), db_->DefaultColumnFamily(), num_keys,
+                keys.data(), values.data(), statuses.data());

  SyncPoint::GetInstance()->DisableProcessing();
  SyncPoint::GetInstance()->ClearAllCallBacks();
@@ -1820,7 +1844,7 @@ TEST_F(DBBlobBasicTest, GetEntityBlob) {
    std::array<Status, num_keys> statuses;

    db_->MultiGetEntity(ReadOptions(), db_->DefaultColumnFamily(), num_keys,
-                        &keys[0], &results[0], &statuses[0]);
+                        keys.data(), results.data(), statuses.data());

    ASSERT_OK(statuses[0]);
    ASSERT_EQ(results[0].columns(), expected_columns);
@@ -1917,8 +1941,8 @@ TEST_F(DBBlobWithTimestampTest, MultiGetBlobs) {
    std::array<PinnableSlice, num_keys> values;
    std::array<Status, num_keys> statuses;

-    db_->MultiGet(read_options, db_->DefaultColumnFamily(), num_keys, &keys[0],
-                  &values[0], &statuses[0]);
+    db_->MultiGet(read_options, db_->DefaultColumnFamily(), num_keys,
+                  keys.data(), values.data(), statuses.data());

    ASSERT_OK(statuses[0]);
    ASSERT_EQ(values[0], first_value);
@@ -2001,8 +2025,8 @@ TEST_F(DBBlobWithTimestampTest, MultiGetMergeBlobWithPut) {
  std::array<PinnableSlice, num_keys> values;
  std::array<Status, num_keys> statuses;

-  db_->MultiGet(read_opts, db_->DefaultColumnFamily(), num_keys, &keys[0],
-                &values[0], &statuses[0]);
+  db_->MultiGet(read_opts, db_->DefaultColumnFamily(), num_keys, keys.data(),
+                values.data(), statuses.data());

  ASSERT_OK(statuses[0]);
  ASSERT_EQ(values[0], "v0_0,v0_1,v0_2");
@@ -45,10 +45,10 @@ class DBBlobIndexTest : public DBTestBase {
  DBBlobIndexTest() : DBTestBase("db_blob_index_test", /*env_do_fsync=*/true) {}

  ColumnFamilyHandle* cfh() { return dbfull()->DefaultColumnFamily(); }
-
-  ColumnFamilyData* cfd() {
-    return static_cast_with_check<ColumnFamilyHandleImpl>(cfh())->cfd();
+  ColumnFamilyHandleImpl* cfh_impl() {
+    return static_cast_with_check<ColumnFamilyHandleImpl>(cfh());
  }
+  ColumnFamilyData* cfd() { return cfh_impl()->cfd(); }

  Status PutBlobIndex(WriteBatch* batch, const Slice& key,
                      const Slice& blob_index) {
@@ -96,11 +96,9 @@ class DBBlobIndexTest : public DBTestBase {
  }

  ArenaWrappedDBIter* GetBlobIterator() {
-    ColumnFamilyData* column_family = cfd();
    DBImpl* db_impl = dbfull();
    return db_impl->NewIteratorImpl(
-        ReadOptions(), column_family,
-        column_family->GetReferencedSuperVersion(db_impl),
+        ReadOptions(), cfh_impl(), cfd()->GetReferencedSuperVersion(db_impl),
        db_impl->GetLatestSequenceNumber(), nullptr /*read_callback*/,
        true /*expose_blob_index*/);
  }
@@ -325,8 +323,7 @@ TEST_F(DBBlobIndexTest, Iterate) {

  auto check_is_blob = [&](bool is_blob) {
    return [is_blob](Iterator* iterator) {
-      ASSERT_EQ(is_blob,
-                reinterpret_cast<ArenaWrappedDBIter*>(iterator)->IsBlob());
+      ASSERT_EQ(is_blob, static_cast<ArenaWrappedDBIter*>(iterator)->IsBlob());
    };
  };

@@ -70,8 +70,8 @@ Status BuildTable(
    bool paranoid_file_checks, InternalStats* internal_stats,
    IOStatus* io_status, const std::shared_ptr<IOTracer>& io_tracer,
    BlobFileCreationReason blob_creation_reason,
-    const SeqnoToTimeMapping& seqno_to_time_mapping, EventLogger* event_logger,
-    int job_id, TableProperties* table_properties,
+    UnownedPtr<const SeqnoToTimeMapping> seqno_to_time_mapping,
+    EventLogger* event_logger, int job_id, TableProperties* table_properties,
    Env::WriteLifeTimeHint write_hint, const std::string* full_history_ts_low,
    BlobFileCompletionCallback* blob_callback, Version* version,
    uint64_t* num_input_entries, uint64_t* memtable_payload_bytes,
@@ -83,11 +83,8 @@ Status BuildTable(
  auto& ioptions = tboptions.ioptions;
  // Reports the IOStats for flush for every following bytes.
  const size_t kReportFlushIOStatsEvery = 1048576;
-  OutputValidator output_validator(
-      tboptions.internal_comparator,
-      /*enable_order_check=*/
-      mutable_cf_options.check_flush_compaction_key_order,
-      /*enable_hash=*/paranoid_file_checks);
+  OutputValidator output_validator(tboptions.internal_comparator,
+                                   /*enable_hash=*/paranoid_file_checks);
  Status s;
  meta->fd.file_size = 0;
  iter->SeekToFirst();
@@ -210,37 +207,61 @@ Status BuildTable(
        /*shutting_down=*/nullptr, db_options.info_log, full_history_ts_low);

    const size_t ts_sz = ucmp->timestamp_size();
-    const bool strip_timestamp =
+    const bool logical_strip_timestamp =
        ts_sz > 0 && !ioptions.persist_user_defined_timestamps;

    std::string key_after_flush_buf;
+    std::string value_buf;
    c_iter.SeekToFirst();
    for (; c_iter.Valid(); c_iter.Next()) {
      const Slice& key = c_iter.key();
      const Slice& value = c_iter.value();
-      const ParsedInternalKey& ikey = c_iter.ikey();
-      Slice key_after_flush = key;
+      ParsedInternalKey ikey = c_iter.ikey();
+      key_after_flush_buf.assign(key.data(), key.size());
+      Slice key_after_flush = key_after_flush_buf;
+      Slice value_after_flush = value;
+
      // If user defined timestamps will be stripped from user key after flush,
      // the in memory version of the key act logically the same as one with a
      // minimum timestamp. We update the timestamp here so file boundary and
      // output validator, block builder all see the effect of the stripping.
-      if (strip_timestamp) {
+      if (logical_strip_timestamp) {
        key_after_flush_buf.clear();
        ReplaceInternalKeyWithMinTimestamp(&key_after_flush_buf, key, ts_sz);
        key_after_flush = key_after_flush_buf;
      }

+      if (ikey.type == kTypeValuePreferredSeqno) {
+        auto [unpacked_value, unix_write_time] =
+            ParsePackedValueWithWriteTime(value);
+        SequenceNumber preferred_seqno =
+            seqno_to_time_mapping
+                ? seqno_to_time_mapping->GetProximalSeqnoBeforeTime(
+                      unix_write_time)
+                : kMaxSequenceNumber;
+        if (preferred_seqno < ikey.sequence) {
+          value_after_flush =
+              PackValueAndSeqno(unpacked_value, preferred_seqno, &value_buf);
+        } else {
+          // Cannot get a useful preferred seqno, convert it to a kTypeValue.
+          UpdateInternalKey(&key_after_flush_buf, ikey.sequence, kTypeValue);
+          ikey = ParsedInternalKey(ikey.user_key, ikey.sequence, kTypeValue);
+          key_after_flush = key_after_flush_buf;
+          value_after_flush = ParsePackedValueForValue(value);
+        }
+      }
+
      //  Generate a rolling 64-bit hash of the key and values
      //  Note :
      //  Here "key" integrates 'sequence_number'+'kType'+'user key'.
-      s = output_validator.Add(key_after_flush, value);
+      s = output_validator.Add(key_after_flush, value_after_flush);
      if (!s.ok()) {
        break;
      }
-      builder->Add(key_after_flush, value);
+      builder->Add(key_after_flush, value_after_flush);

-      s = meta->UpdateBoundaries(key_after_flush, value, ikey.sequence,
-                                 ikey.type);
+      s = meta->UpdateBoundaries(key_after_flush, value_after_flush,
+                                 ikey.sequence, ikey.type);
      if (!s.ok()) {
        break;
      }
@@ -267,9 +288,12 @@ Status BuildTable(
      Slice last_tombstone_start_user_key{};
      for (range_del_it->SeekToFirst(); range_del_it->Valid();
           range_del_it->Next()) {
-        auto tombstone = range_del_it->Tombstone();
-        auto kv = tombstone.Serialize();
-        // TODO(yuzhangyu): handle range deletion for UDT in memtables only.
+        // When user timestamp should not be persisted, we logically strip a
+        // range tombstone's start and end key's timestamp (replace it with min
+        // timestamp) before passing them along to table builder and to update
+        // file boundaries.
+        auto tombstone = range_del_it->Tombstone(logical_strip_timestamp);
+        std::pair<InternalKey, Slice> kv = tombstone.Serialize();
        builder->Add(kv.first.Encode(), kv.second);
        InternalKey tombstone_end = tombstone.SerializeEndKey();
        meta->UpdateBoundariesForRange(kv.first, tombstone_end, tombstone.seq_,
@@ -301,11 +325,13 @@ Status BuildTable(
      builder->Abandon();
    } else {
      SeqnoToTimeMapping relevant_mapping;
-      relevant_mapping.CopyFromSeqnoRange(seqno_to_time_mapping,
-                                          meta->fd.smallest_seqno,
-                                          meta->fd.largest_seqno);
-      relevant_mapping.SetCapacity(kMaxSeqnoTimePairsPerSST);
-      relevant_mapping.Enforce(tboptions.file_creation_time);
+      if (seqno_to_time_mapping) {
+        relevant_mapping.CopyFromSeqnoRange(*seqno_to_time_mapping,
+                                            meta->fd.smallest_seqno,
+                                            meta->fd.largest_seqno);
+        relevant_mapping.SetCapacity(kMaxSeqnoTimePairsPerSST);
+        relevant_mapping.Enforce(tboptions.file_creation_time);
+      }
      builder->SetSeqnoTimeTableProperties(
          relevant_mapping,
          ioptions.compaction_style == CompactionStyle::kCompactionStyleFIFO
@@ -422,7 +448,6 @@ Status BuildTable(
      s = it->status();
      if (s.ok() && paranoid_file_checks) {
        OutputValidator file_validator(tboptions.internal_comparator,
-                                       /*enable_order_check=*/true,
                                       /*enable_hash=*/true);
        for (it->SeekToFirst(); it->Valid(); it->Next()) {
          // Generate a rolling 64-bit hash of the key and values
@@ -23,7 +23,6 @@
 #include "rocksdb/status.h"
 #include "rocksdb/table_properties.h"
 #include "rocksdb/types.h"
-#include "table/scoped_arena_iterator.h"

 namespace ROCKSDB_NAMESPACE {

@@ -50,7 +49,7 @@ TableBuilder* NewTableBuilder(const TableBuilderOptions& tboptions,
 //
 // @param column_family_name Name of the column family that is also identified
 //    by column_family_id, or empty string if unknown.
-extern Status BuildTable(
+Status BuildTable(
    const std::string& dbname, VersionSet* versions,
    const ImmutableDBOptions& db_options, const TableBuilderOptions& tboptions,
    const FileOptions& file_options, TableCache* table_cache,
@@ -64,7 +63,7 @@ extern Status BuildTable(
    bool paranoid_file_checks, InternalStats* internal_stats,
    IOStatus* io_status, const std::shared_ptr<IOTracer>& io_tracer,
    BlobFileCreationReason blob_creation_reason,
-    const SeqnoToTimeMapping& seqno_to_time_mapping,
+    UnownedPtr<const SeqnoToTimeMapping> seqno_to_time_mapping,
    EventLogger* event_logger = nullptr, int job_id = 0,
    TableProperties* table_properties = nullptr,
    Env::WriteLifeTimeHint write_hint = Env::WLTH_NOT_SET,
@@ -46,6 +46,7 @@
 #include "rocksdb/utilities/write_batch_with_index.h"
 #include "rocksdb/write_batch.h"
 #include "rocksdb/write_buffer_manager.h"
+#include "util/stderr_logger.h"
 #include "utilities/merge_operators.h"

 using ROCKSDB_NAMESPACE::BackupEngine;
@@ -115,6 +116,7 @@ using ROCKSDB_NAMESPACE::Snapshot;
 using ROCKSDB_NAMESPACE::SstFileMetaData;
 using ROCKSDB_NAMESPACE::SstFileWriter;
 using ROCKSDB_NAMESPACE::Status;
+using ROCKSDB_NAMESPACE::StderrLogger;
 using ROCKSDB_NAMESPACE::TablePropertiesCollectorFactory;
 using ROCKSDB_NAMESPACE::Transaction;
 using ROCKSDB_NAMESPACE::TransactionDB;
@@ -444,7 +446,7 @@ struct rocksdb_mergeoperator_t : public MergeOperator {
    size_t new_value_len;
    char* tmp_new_value = (*full_merge_)(
        state_, merge_in.key.data(), merge_in.key.size(), existing_value_data,
-        existing_value_len, &operand_pointers[0], &operand_sizes[0],
+        existing_value_len, operand_pointers.data(), operand_sizes.data(),
        static_cast<int>(n), &success, &new_value_len);
    merge_out->new_value.assign(tmp_new_value, new_value_len);

@@ -473,8 +475,9 @@ struct rocksdb_mergeoperator_t : public MergeOperator {
    unsigned char success;
    size_t new_value_len;
    char* tmp_new_value = (*partial_merge_)(
-        state_, key.data(), key.size(), &operand_pointers[0], &operand_sizes[0],
-        static_cast<int>(operand_count), &success, &new_value_len);
+        state_, key.data(), key.size(), operand_pointers.data(),
+        operand_sizes.data(), static_cast<int>(operand_count), &success,
+        &new_value_len);
    new_value->assign(tmp_new_value, new_value_len);

    if (delete_value_ != nullptr) {
@@ -884,9 +887,9 @@ rocksdb_t* rocksdb_open_and_trim_history(
    size_t trim_tslen, char** errptr) {
  std::vector<ColumnFamilyDescriptor> column_families;
  for (int i = 0; i < num_column_families; i++) {
-    column_families.push_back(ColumnFamilyDescriptor(
+    column_families.emplace_back(
        std::string(column_family_names[i]),
-        ColumnFamilyOptions(column_family_options[i]->rep)));
+        ColumnFamilyOptions(column_family_options[i]->rep));
  }

  std::string trim_ts_(trim_ts, trim_tslen);
@@ -917,9 +920,9 @@ rocksdb_t* rocksdb_open_column_families(
    rocksdb_column_family_handle_t** column_family_handles, char** errptr) {
  std::vector<ColumnFamilyDescriptor> column_families;
  for (int i = 0; i < num_column_families; i++) {
-    column_families.push_back(ColumnFamilyDescriptor(
+    column_families.emplace_back(
        std::string(column_family_names[i]),
-        ColumnFamilyOptions(column_family_options[i]->rep)));
+        ColumnFamilyOptions(column_family_options[i]->rep));
  }

  DB* db;
@@ -951,9 +954,9 @@ rocksdb_t* rocksdb_open_column_families_with_ttl(
  for (int i = 0; i < num_column_families; i++) {
    ttls_vec.push_back(ttls[i]);

-    column_families.push_back(ColumnFamilyDescriptor(
+    column_families.emplace_back(
        std::string(column_family_names[i]),
-        ColumnFamilyOptions(column_family_options[i]->rep)));
+        ColumnFamilyOptions(column_family_options[i]->rep));
  }

  ROCKSDB_NAMESPACE::DBWithTTL* db;
@@ -983,9 +986,9 @@ rocksdb_t* rocksdb_open_for_read_only_column_families(
    unsigned char error_if_wal_file_exists, char** errptr) {
  std::vector<ColumnFamilyDescriptor> column_families;
  for (int i = 0; i < num_column_families; i++) {
-    column_families.push_back(ColumnFamilyDescriptor(
+    column_families.emplace_back(
        std::string(column_family_names[i]),
-        ColumnFamilyOptions(column_family_options[i]->rep)));
+        ColumnFamilyOptions(column_family_options[i]->rep));
  }

  DB* db;
@@ -1079,7 +1082,7 @@ rocksdb_column_family_handle_t** rocksdb_create_column_families(
  std::vector<ColumnFamilyHandle*> handles;
  std::vector<std::string> names;
  for (int i = 0; i != num_column_families; ++i) {
-    names.push_back(std::string(column_family_names[i]));
+    names.emplace_back(column_family_names[i]);
  }
  SaveError(errptr, db->rep->CreateColumnFamilies(
                        ColumnFamilyOptions(column_family_options->rep), names,
@@ -1685,6 +1688,11 @@ void rocksdb_release_snapshot(rocksdb_t* db,
  delete snapshot;
 }

+uint64_t rocksdb_snapshot_get_sequence_number(
+    const rocksdb_snapshot_t* snapshot) {
+  return snapshot->rep->GetSequenceNumber();
+}
+
 char* rocksdb_property_value(rocksdb_t* db, const char* propname) {
  std::string tmp;
  if (db->rep->GetProperty(Slice(propname), &tmp)) {
@@ -1872,9 +1880,8 @@ void rocksdb_disable_file_deletions(rocksdb_t* db, char** errptr) {
  SaveError(errptr, db->rep->DisableFileDeletions());
 }

-void rocksdb_enable_file_deletions(rocksdb_t* db, unsigned char force,
-                                   char** errptr) {
-  SaveError(errptr, db->rep->EnableFileDeletions(force));
+void rocksdb_enable_file_deletions(rocksdb_t* db, char** errptr) {
+  SaveError(errptr, db->rep->EnableFileDeletions());
 }

 void rocksdb_destroy_db(const rocksdb_options_t* options, const char* name,
@@ -2782,7 +2789,9 @@ void rocksdb_options_set_cuckoo_table_factory(
 void rocksdb_set_options(rocksdb_t* db, int count, const char* const keys[],
                         const char* const values[], char** errptr) {
  std::unordered_map<std::string, std::string> options_map;
-  for (int i = 0; i < count; i++) options_map[keys[i]] = values[i];
+  for (int i = 0; i < count; i++) {
+    options_map[keys[i]] = values[i];
+  }
  SaveError(errptr, db->rep->SetOptions(options_map));
 }

@@ -2791,7 +2800,9 @@ void rocksdb_set_options_cf(rocksdb_t* db,
                            const char* const keys[],
                            const char* const values[], char** errptr) {
  std::unordered_map<std::string, std::string> options_map;
-  for (int i = 0; i < count; i++) options_map[keys[i]] = values[i];
+  for (int i = 0; i < count; i++) {
+    options_map[keys[i]] = values[i];
+  }
  SaveError(errptr, db->rep->SetOptions(handle->rep, options_map));
 }

@@ -2919,6 +2930,23 @@ void rocksdb_options_set_cf_paths(rocksdb_options_t* opt,
  opt->rep.cf_paths = cf_paths;
 }

+rocksdb_logger_t* rocksdb_logger_create_stderr_logger(int log_level,
+                                                      const char* prefix) {
+  rocksdb_logger_t* logger = new rocksdb_logger_t;
+
+  if (prefix) {
+    logger->rep = std::make_shared<StderrLogger>(
+        static_cast<InfoLogLevel>(log_level), prefix);
+  } else {
+    logger->rep =
+        std::make_shared<StderrLogger>(static_cast<InfoLogLevel>(log_level));
+  }
+
+  return logger;
+}
+
+void rocksdb_logger_destroy(rocksdb_logger_t* logger) { delete logger; }
+
 void rocksdb_options_set_env(rocksdb_options_t* opt, rocksdb_env_t* env) {
  opt->rep.env = (env ? env->rep : nullptr);
 }
@@ -2929,6 +2957,12 @@ void rocksdb_options_set_info_log(rocksdb_options_t* opt, rocksdb_logger_t* l) {
  }
 }

+rocksdb_logger_t* rocksdb_options_get_info_log(rocksdb_options_t* opt) {
+  rocksdb_logger_t* info_log = new rocksdb_logger_t;
+  info_log->rep = opt->rep.info_log;
+  return info_log;
+}
+
 void rocksdb_options_set_info_log_level(rocksdb_options_t* opt, int v) {
  opt->rep.info_log_level = static_cast<InfoLogLevel>(v);
 }
@@ -3119,7 +3153,7 @@ void rocksdb_options_set_enable_blob_files(rocksdb_options_t* opt,
                                           unsigned char val) {
  opt->rep.enable_blob_files = val;
 }
-extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_enable_blob_files(
+ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_enable_blob_files(
    rocksdb_options_t* opt) {
  return opt->rep.enable_blob_files;
 }
@@ -3491,35 +3525,6 @@ unsigned char rocksdb_options_get_advise_random_on_open(
  return opt->rep.advise_random_on_open;
 }

-void rocksdb_options_set_access_hint_on_compaction_start(rocksdb_options_t* opt,
-                                                         int v) {
-  switch (v) {
-    case 0:
-      opt->rep.access_hint_on_compaction_start =
-          ROCKSDB_NAMESPACE::Options::NONE;
-      break;
-    case 1:
-      opt->rep.access_hint_on_compaction_start =
-          ROCKSDB_NAMESPACE::Options::NORMAL;
-      break;
-    case 2:
-      opt->rep.access_hint_on_compaction_start =
-          ROCKSDB_NAMESPACE::Options::SEQUENTIAL;
-      break;
-    case 3:
-      opt->rep.access_hint_on_compaction_start =
-          ROCKSDB_NAMESPACE::Options::WILLNEED;
-      break;
-    default:
-      assert(0);
-  }
-}
-
-int rocksdb_options_get_access_hint_on_compaction_start(
-    rocksdb_options_t* opt) {
-  return opt->rep.access_hint_on_compaction_start;
-}
-
 void rocksdb_options_set_use_adaptive_mutex(rocksdb_options_t* opt,
                                            unsigned char v) {
  opt->rep.use_adaptive_mutex = v;
@@ -4005,6 +4010,16 @@ rocksdb_ratelimiter_t* rocksdb_ratelimiter_create_auto_tuned(
  return rate_limiter;
 }

+rocksdb_ratelimiter_t* rocksdb_ratelimiter_create_with_mode(
+    int64_t rate_bytes_per_sec, int64_t refill_period_us, int32_t fairness,
+    int mode, bool auto_tuned) {
+  rocksdb_ratelimiter_t* rate_limiter = new rocksdb_ratelimiter_t;
+  rate_limiter->rep.reset(
+      NewGenericRateLimiter(rate_bytes_per_sec, refill_period_us, fairness,
+                            static_cast<RateLimiter::Mode>(mode), auto_tuned));
+  return rate_limiter;
+}
+
 void rocksdb_ratelimiter_destroy(rocksdb_ratelimiter_t* limiter) {
  delete limiter;
 }
@@ -4601,7 +4616,7 @@ void rocksdb_readoptions_set_io_timeout(rocksdb_readoptions_t* opt,
  opt->rep.io_timeout = std::chrono::microseconds(microseconds);
 }

-extern ROCKSDB_LIBRARY_API uint64_t
+ROCKSDB_LIBRARY_API uint64_t
 rocksdb_readoptions_get_io_timeout(rocksdb_readoptions_t* opt) {
  return opt->rep.io_timeout.count();
 }
@@ -5050,7 +5065,9 @@ void rocksdb_env_lower_high_priority_thread_pool_cpu_priority(
 }

 void rocksdb_env_destroy(rocksdb_env_t* env) {
-  if (!env->is_default) delete env->rep;
+  if (!env->is_default) {
+    delete env->rep;
+  }
  delete env;
 }

@@ -5434,9 +5451,7 @@ uint64_t rocksdb_livefiles_deletions(const rocksdb_livefiles_t* lf, int index) {
  return lf->rep[index].num_deletions;
 }

-extern void rocksdb_livefiles_destroy(const rocksdb_livefiles_t* lf) {
-  delete lf;
-}
+void rocksdb_livefiles_destroy(const rocksdb_livefiles_t* lf) { delete lf; }

 void rocksdb_get_options_from_string(const rocksdb_options_t* base_options,
                                     const char* opts_str,
@@ -5516,7 +5531,7 @@ size_t rocksdb_column_family_metadata_get_level_count(
 rocksdb_level_metadata_t* rocksdb_column_family_metadata_get_level_metadata(
    rocksdb_column_family_metadata_t* cf_meta, size_t i) {
  if (i >= cf_meta->rep.levels.size()) {
-    return NULL;
+    return nullptr;
  }
  rocksdb_level_metadata_t* level_meta =
      (rocksdb_level_metadata_t*)malloc(sizeof(rocksdb_level_metadata_t));
@@ -5731,9 +5746,9 @@ rocksdb_transactiondb_t* rocksdb_transactiondb_open_column_families(
    rocksdb_column_family_handle_t** column_family_handles, char** errptr) {
  std::vector<ColumnFamilyDescriptor> column_families;
  for (int i = 0; i < num_column_families; i++) {
-    column_families.push_back(ColumnFamilyDescriptor(
+    column_families.emplace_back(
        std::string(column_family_names[i]),
-        ColumnFamilyOptions(column_family_options[i]->rep)));
+        ColumnFamilyOptions(column_family_options[i]->rep));
  }

  TransactionDB* txn_db;
@@ -6525,9 +6540,9 @@ rocksdb_optimistictransactiondb_open_column_families(
    rocksdb_column_family_handle_t** column_family_handles, char** errptr) {
  std::vector<ColumnFamilyDescriptor> column_families;
  for (int i = 0; i < num_column_families; i++) {
-    column_families.push_back(ColumnFamilyDescriptor(
+    column_families.emplace_back(
        std::string(column_family_names[i]),
-        ColumnFamilyOptions(column_family_options[i]->rep)));
+        ColumnFamilyOptions(column_family_options[i]->rep));
  }

  OptimisticTransactionDB* otxn_db;
@@ -50,12 +50,13 @@ static void StartPhase(const char* name) {
 #endif
 static const char* GetTempDir(void) {
  const char* ret = getenv("TEST_TMPDIR");
-  if (ret == NULL || ret[0] == '\0')
+  if (ret == NULL || ret[0] == '\0') {
 #ifdef OS_WIN
    ret = getenv("TEMP");
 #else
    ret = "/tmp";
 #endif
+  }
  return ret;
 }
 #ifdef _MSC_VER
@@ -206,10 +207,11 @@ static int CmpCompare(void* arg, const char* a, size_t alen, const char* b,
  size_t n = (alen < blen) ? alen : blen;
  int r = memcmp(a, b, n);
  if (r == 0) {
-    if (alen < blen)
+    if (alen < blen) {
      r = -1;
-    else if (alen > blen)
+    } else if (alen > blen) {
      r = +1;
+    }
  }
  return r;
 }
@@ -718,6 +720,11 @@ int main(int argc, char** argv) {
  rocksdb_options_set_ratelimiter(options, rate_limiter);
  rocksdb_ratelimiter_destroy(rate_limiter);

+  rate_limiter = rocksdb_ratelimiter_create_with_mode(1000 * 1024 * 1024,
+                                                      100 * 1000, 10, 0, true);
+  rocksdb_options_set_ratelimiter(options, rate_limiter);
+  rocksdb_ratelimiter_destroy(rate_limiter);
+
  roptions = rocksdb_readoptions_create();
  rocksdb_readoptions_set_verify_checksums(roptions, 1);
  rocksdb_readoptions_set_fill_cache(roptions, 1);
@@ -878,9 +885,8 @@ int main(int argc, char** argv) {
  StartPhase("addfile");
  {
    rocksdb_envoptions_t* env_opt = rocksdb_envoptions_create();
-    rocksdb_options_t* io_options = rocksdb_options_create();
    rocksdb_sstfilewriter_t* writer =
-        rocksdb_sstfilewriter_create(env_opt, io_options);
+        rocksdb_sstfilewriter_create(env_opt, options);

    remove(sstfilename);
    rocksdb_sstfilewriter_open(writer, sstfilename, &err);
@@ -939,7 +945,6 @@ int main(int argc, char** argv) {

    rocksdb_ingestexternalfileoptions_destroy(ing_opt);
    rocksdb_sstfilewriter_destroy(writer);
-    rocksdb_options_destroy(io_options);
    rocksdb_envoptions_destroy(env_opt);

    // Delete all keys we just ingested
@@ -2034,8 +2039,6 @@ int main(int argc, char** argv) {
    rocksdb_options_set_advise_random_on_open(o, 1);
    CheckCondition(1 == rocksdb_options_get_advise_random_on_open(o));

-    rocksdb_options_set_access_hint_on_compaction_start(o, 3);
-    CheckCondition(3 == rocksdb_options_get_access_hint_on_compaction_start(o));

    rocksdb_options_set_use_adaptive_mutex(o, 1);
    CheckCondition(1 == rocksdb_options_get_use_adaptive_mutex(o));
@@ -2231,8 +2234,6 @@ int main(int argc, char** argv) {
    CheckCondition(18 == rocksdb_options_get_stats_dump_period_sec(copy));
    CheckCondition(5 == rocksdb_options_get_stats_persist_period_sec(copy));
    CheckCondition(1 == rocksdb_options_get_advise_random_on_open(copy));
-    CheckCondition(3 ==
-                   rocksdb_options_get_access_hint_on_compaction_start(copy));
    CheckCondition(1 == rocksdb_options_get_use_adaptive_mutex(copy));
    CheckCondition(19 == rocksdb_options_get_bytes_per_sync(copy));
    CheckCondition(20 == rocksdb_options_get_wal_bytes_per_sync(copy));
@@ -2515,11 +2516,6 @@ int main(int argc, char** argv) {
    CheckCondition(0 == rocksdb_options_get_advise_random_on_open(copy));
    CheckCondition(1 == rocksdb_options_get_advise_random_on_open(o));

-    rocksdb_options_set_access_hint_on_compaction_start(copy, 2);
-    CheckCondition(2 ==
-                   rocksdb_options_get_access_hint_on_compaction_start(copy));
-    CheckCondition(3 == rocksdb_options_get_access_hint_on_compaction_start(o));
-
    rocksdb_options_set_use_adaptive_mutex(copy, 0);
    CheckCondition(0 == rocksdb_options_get_use_adaptive_mutex(copy));
    CheckCondition(1 == rocksdb_options_get_use_adaptive_mutex(o));
@@ -2788,6 +2784,30 @@ int main(int argc, char** argv) {
    rocksdb_memory_allocator_destroy(allocator);
  }

+  StartPhase("stderr_logger");
+  {
+    rocksdb_options_t* o_no_prefix = rocksdb_options_create();
+    rocksdb_logger_t* no_prefix_logger =
+        rocksdb_logger_create_stderr_logger(3, NULL);
+    rocksdb_options_set_info_log(o_no_prefix, no_prefix_logger);
+    rocksdb_logger_t* no_prefix_info_log =
+        rocksdb_options_get_info_log(o_no_prefix);
+    CheckCondition(no_prefix_info_log != NULL);
+    rocksdb_logger_destroy(no_prefix_logger);
+    rocksdb_logger_destroy(no_prefix_info_log);
+    rocksdb_options_destroy(o_no_prefix);
+
+    rocksdb_options_t* o_prefix = rocksdb_options_create();
+    rocksdb_logger_t* prefix_logger =
+        rocksdb_logger_create_stderr_logger(3, "some prefix");
+    rocksdb_options_set_info_log(o_prefix, prefix_logger);
+    rocksdb_logger_t* prefix_info_log = rocksdb_options_get_info_log(o_prefix);
+    CheckCondition(prefix_info_log != NULL);
+    rocksdb_logger_destroy(prefix_logger);
+    rocksdb_logger_destroy(prefix_info_log);
+    rocksdb_options_destroy(o_prefix);
+  }
+
  StartPhase("env");
  {
    rocksdb_env_t* e;
@@ -3694,7 +3714,7 @@ int main(int argc, char** argv) {

  StartPhase("statistics");
  {
-    const uint32_t BYTES_WRITTEN_TICKER = 40;
+    const uint32_t BYTES_WRITTEN_TICKER = 60;
    const uint32_t DB_WRITE_HIST = 1;

    rocksdb_statistics_histogram_data_t* hist =
@@ -96,16 +96,16 @@ const Comparator* ColumnFamilyHandleImpl::GetComparator() const {
  return cfd()->user_comparator();
 }

-void GetIntTblPropCollectorFactory(
+void GetInternalTblPropCollFactory(
    const ImmutableCFOptions& ioptions,
-    IntTblPropCollectorFactories* int_tbl_prop_collector_factories) {
-  assert(int_tbl_prop_collector_factories);
+    InternalTblPropCollFactories* internal_tbl_prop_coll_factories) {
+  assert(internal_tbl_prop_coll_factories);

  auto& collector_factories = ioptions.table_properties_collector_factories;
  for (size_t i = 0; i < ioptions.table_properties_collector_factories.size();
       ++i) {
    assert(collector_factories[i]);
-    int_tbl_prop_collector_factories->emplace_back(
+    internal_tbl_prop_coll_factories->emplace_back(
        new UserKeyTablePropertiesCollectorFactory(collector_factories[i]));
  }
 }
@@ -411,6 +411,13 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
          "periodic_compaction_seconds does not support FIFO compaction. You"
          "may want to set option TTL instead.");
    }
+    if (result.last_level_temperature != Temperature::kUnknown) {
+      ROCKS_LOG_WARN(
+          db_options.info_log.get(),
+          "last_level_temperature is ignored with FIFO compaction. Consider "
+          "CompactionOptionsFIFO::file_temperature_age_thresholds.");
+      result.last_level_temperature = Temperature::kUnknown;
+    }
  }

  // For universal compaction, `ttl` and `periodic_compaction_seconds` mean the
@@ -470,13 +477,16 @@ void SuperVersion::Cleanup() {
  cfd->UnrefAndTryDelete();
 }

-void SuperVersion::Init(ColumnFamilyData* new_cfd, MemTable* new_mem,
-                        MemTableListVersion* new_imm, Version* new_current) {
+void SuperVersion::Init(
+    ColumnFamilyData* new_cfd, MemTable* new_mem, MemTableListVersion* new_imm,
+    Version* new_current,
+    std::shared_ptr<const SeqnoToTimeMapping> new_seqno_to_time_mapping) {
  cfd = new_cfd;
  mem = new_mem;
  imm = new_imm;
  current = new_current;
  full_history_ts_low = cfd->GetFullHistoryTsLow();
+  seqno_to_time_mapping = std::move(new_seqno_to_time_mapping);
  cfd->Ref();
  mem->Ref();
  imm->Ref();
@@ -572,7 +582,7 @@ ColumnFamilyData::ColumnFamilyData(
  Ref();

  // Convert user defined table properties collector factories to internal ones.
-  GetIntTblPropCollectorFactory(ioptions_, &int_tbl_prop_collector_factories_);
+  GetInternalTblPropCollFactory(ioptions_, &internal_tbl_prop_coll_factories_);

  // if _dummy_versions is nullptr, then this is a dummy column family.
  if (_dummy_versions != nullptr) {
@@ -893,6 +903,14 @@ uint64_t GetPendingCompactionBytesForCompactionSpeedup(
  uint64_t size_threshold = bottommost_files_size / kBottommostSizeDivisor;
  return std::min(size_threshold, slowdown_threshold);
 }
+
+uint64_t GetMarkedFileCountForCompactionSpeedup() {
+  // When just one file is marked, it is not clear that parallel compaction will
+  // help the compaction that the user nicely requested to happen sooner. When
+  // multiple files are marked, however, it is pretty clearly helpful, except
+  // for the rare case in which a single compaction grabs all the marked files.
+  return 2;
+}
 }  // anonymous namespace

 std::pair<WriteStallCondition, WriteStallCause>
@@ -1074,6 +1092,16 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
            "compaction "
            "bytes %" PRIu64,
            name_.c_str(), vstorage->estimated_compaction_needed_bytes());
+      } else if (uint64_t(vstorage->FilesMarkedForCompaction().size()) >=
+                 GetMarkedFileCountForCompactionSpeedup()) {
+        write_controller_token_ =
+            write_controller->GetCompactionPressureToken();
+        ROCKS_LOG_INFO(
+            ioptions_.logger,
+            "[%s] Increasing compaction threads because we have %" PRIu64
+            " files marked for compaction",
+            name_.c_str(),
+            uint64_t(vstorage->FilesMarkedForCompaction().size()));
      } else {
        write_controller_token_.reset();
      }
@@ -1161,7 +1189,7 @@ bool ColumnFamilyData::RangeOverlapWithCompaction(
 }

 Status ColumnFamilyData::RangesOverlapWithMemtables(
-    const autovector<Range>& ranges, SuperVersion* super_version,
+    const autovector<UserKeyRange>& ranges, SuperVersion* super_version,
    bool allow_data_in_errors, bool* overlap) {
  assert(overlap != nullptr);
  *overlap = false;
@@ -1171,11 +1199,12 @@ Status ColumnFamilyData::RangesOverlapWithMemtables(
  ReadOptions read_opts;
  read_opts.total_order_seek = true;
  MergeIteratorBuilder merge_iter_builder(&internal_comparator_, &arena);
-  merge_iter_builder.AddIterator(
-      super_version->mem->NewIterator(read_opts, &arena));
-  super_version->imm->AddIterators(read_opts, &merge_iter_builder,
+  merge_iter_builder.AddIterator(super_version->mem->NewIterator(
+      read_opts, /*seqno_to_time_mapping=*/nullptr, &arena));
+  super_version->imm->AddIterators(read_opts, /*seqno_to_time_mapping=*/nullptr,
+                                   &merge_iter_builder,
                                   false /* add_range_tombstone_iter */);
-  ScopedArenaIterator memtable_iter(merge_iter_builder.Finish());
+  ScopedArenaPtr<InternalIterator> memtable_iter(merge_iter_builder.Finish());

  auto read_seq = super_version->current->version_set()->LastSequence();
  ReadRangeDelAggregator range_del_agg(&internal_comparator_, read_seq);
@@ -1205,7 +1234,8 @@ Status ColumnFamilyData::RangesOverlapWithMemtables(

    if (status.ok()) {
      if (memtable_iter->Valid() &&
-          ucmp->Compare(seek_result.user_key, ranges[i].limit) <= 0) {
+          ucmp->CompareWithoutTimestamp(seek_result.user_key,
+                                        ranges[i].limit) <= 0) {
        *overlap = true;
      } else if (range_del_agg.IsRangeOverlapped(ranges[i].start,
                                                 ranges[i].limit)) {
@@ -1310,7 +1340,12 @@ void ColumnFamilyData::InstallSuperVersion(
    const MutableCFOptions& mutable_cf_options) {
  SuperVersion* new_superversion = sv_context->new_superversion.release();
  new_superversion->mutable_cf_options = mutable_cf_options;
-  new_superversion->Init(this, mem_, imm_.current(), current_);
+  new_superversion->Init(this, mem_, imm_.current(), current_,
+                         sv_context->new_seqno_to_time_mapping
+                             ? std::move(sv_context->new_seqno_to_time_mapping)
+                         : super_version_
+                             ? super_version_->ShareSeqnoToTimeMapping()
+                             : nullptr);
  SuperVersion* old_superversion = super_version_;
  super_version_ = new_superversion;
  if (old_superversion == nullptr || old_superversion->current != current() ||
@@ -26,6 +26,7 @@
 #include "rocksdb/env.h"
 #include "rocksdb/options.h"
 #include "trace_replay/block_cache_tracer.h"
+#include "util/cast_util.h"
 #include "util/hash_containers.h"
 #include "util/thread_local.h"

@@ -168,11 +169,12 @@ class ColumnFamilyHandleImpl : public ColumnFamilyHandle {
  // destroy without mutex
  virtual ~ColumnFamilyHandleImpl();
  virtual ColumnFamilyData* cfd() const { return cfd_; }
+  virtual DBImpl* db() const { return db_; }

-  virtual uint32_t GetID() const override;
-  virtual const std::string& GetName() const override;
-  virtual Status GetDescriptor(ColumnFamilyDescriptor* desc) override;
-  virtual const Comparator* GetComparator() const override;
+  uint32_t GetID() const override;
+  const std::string& GetName() const override;
+  Status GetDescriptor(ColumnFamilyDescriptor* desc) override;
+  const Comparator* GetComparator() const override;

 private:
  ColumnFamilyData* cfd_;
@@ -193,7 +195,7 @@ class ColumnFamilyHandleInternal : public ColumnFamilyHandleImpl {
        internal_cfd_(nullptr) {}

  void SetCFD(ColumnFamilyData* _cfd) { internal_cfd_ = _cfd; }
-  virtual ColumnFamilyData* cfd() const override { return internal_cfd_; }
+  ColumnFamilyData* cfd() const override { return internal_cfd_; }

 private:
  ColumnFamilyData* internal_cfd_;
@@ -218,6 +220,9 @@ struct SuperVersion {
  // enable UDT feature, this is an empty string.
  std::string full_history_ts_low;

+  // A shared copy of the DB's seqno to time mapping.
+  std::shared_ptr<const SeqnoToTimeMapping> seqno_to_time_mapping{nullptr};
+
  // should be called outside the mutex
  SuperVersion() = default;
  ~SuperVersion();
@@ -231,8 +236,23 @@ struct SuperVersion {
  // that needs to be deleted in to_delete vector. Unrefing those
  // objects needs to be done in the mutex
  void Cleanup();
-  void Init(ColumnFamilyData* new_cfd, MemTable* new_mem,
-            MemTableListVersion* new_imm, Version* new_current);
+  void Init(
+      ColumnFamilyData* new_cfd, MemTable* new_mem,
+      MemTableListVersion* new_imm, Version* new_current,
+      std::shared_ptr<const SeqnoToTimeMapping> new_seqno_to_time_mapping);
+
+  // Share the ownership of the seqno to time mapping object referred to in this
+  // SuperVersion. To be used by the new SuperVersion to be installed after this
+  // one if seqno to time mapping does not change in between these two
+  // SuperVersions. Or to share the ownership of the mapping with a FlushJob.
+  std::shared_ptr<const SeqnoToTimeMapping> ShareSeqnoToTimeMapping() {
+    return seqno_to_time_mapping;
+  }
+
+  // Access the seqno to time mapping object in this SuperVersion.
+  UnownedPtr<const SeqnoToTimeMapping> GetSeqnoToTimeMapping() const {
+    return seqno_to_time_mapping.get();
+  }

  // The value of dummy is not actually used. kSVInUse takes its address as a
  // mark in the thread local storage to indicate the SuperVersion is in use
@@ -251,22 +271,21 @@ struct SuperVersion {
  autovector<MemTable*> to_delete;
 };

-extern Status CheckCompressionSupported(const ColumnFamilyOptions& cf_options);
+Status CheckCompressionSupported(const ColumnFamilyOptions& cf_options);

-extern Status CheckConcurrentWritesSupported(
-    const ColumnFamilyOptions& cf_options);
+Status CheckConcurrentWritesSupported(const ColumnFamilyOptions& cf_options);

-extern Status CheckCFPathsSupported(const DBOptions& db_options,
-                                    const ColumnFamilyOptions& cf_options);
+Status CheckCFPathsSupported(const DBOptions& db_options,
+                             const ColumnFamilyOptions& cf_options);

-extern ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
-                                           const ColumnFamilyOptions& src);
+ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
+                                    const ColumnFamilyOptions& src);
 // Wrap user defined table properties collector factories `from cf_options`
-// into internal ones in int_tbl_prop_collector_factories. Add a system internal
+// into internal ones in internal_tbl_prop_coll_factories. Add a system internal
 // one too.
-extern void GetIntTblPropCollectorFactory(
+void GetInternalTblPropCollFactory(
    const ImmutableCFOptions& ioptions,
-    IntTblPropCollectorFactories* int_tbl_prop_collector_factories);
+    InternalTblPropCollFactories* internal_tbl_prop_coll_factories);

 class ColumnFamilySet;

@@ -401,7 +420,7 @@ class ColumnFamilyData {
  //    duration of this function.
  //
  // Thread-safe
-  Status RangesOverlapWithMemtables(const autovector<Range>& ranges,
+  Status RangesOverlapWithMemtables(const autovector<UserKeyRange>& ranges,
                                    SuperVersion* super_version,
                                    bool allow_data_in_errors, bool* overlap);

@@ -430,8 +449,8 @@ class ColumnFamilyData {
    return internal_comparator_;
  }

-  const IntTblPropCollectorFactories* int_tbl_prop_collector_factories() const {
-    return &int_tbl_prop_collector_factories_;
+  const InternalTblPropCollFactories* internal_tbl_prop_coll_factories() const {
+    return &internal_tbl_prop_coll_factories_;
  }

  SuperVersion* GetSuperVersion() { return super_version_; }
@@ -574,7 +593,7 @@ class ColumnFamilyData {
  std::atomic<bool> dropped_;  // true if client dropped it

  const InternalKeyComparator internal_comparator_;
-  IntTblPropCollectorFactories int_tbl_prop_collector_factories_;
+  InternalTblPropCollFactories internal_tbl_prop_coll_factories_;

  const ColumnFamilyOptions initial_cf_options_;
  const ImmutableOptions ioptions_;
@@ -854,17 +873,17 @@ class ColumnFamilyMemTablesImpl : public ColumnFamilyMemTables {
  // REQUIRES: Seek() called first
  // REQUIRES: use this function of DBImpl::column_family_memtables_ should be
  //           under a DB mutex OR from a write thread
-  virtual MemTable* GetMemTable() const override;
+  MemTable* GetMemTable() const override;

  // Returns column family handle for the selected column family
  // REQUIRES: use this function of DBImpl::column_family_memtables_ should be
  //           under a DB mutex OR from a write thread
-  virtual ColumnFamilyHandle* GetColumnFamilyHandle() override;
+  ColumnFamilyHandle* GetColumnFamilyHandle() override;

  // Cannot be called while another thread is calling Seek().
  // REQUIRES: use this function of DBImpl::column_family_memtables_ should be
  //           under a DB mutex OR from a write thread
-  virtual ColumnFamilyData* current() override { return current_; }
+  ColumnFamilyData* current() override { return current_; }

 private:
  ColumnFamilySet* column_family_set_;
@@ -872,12 +891,11 @@ class ColumnFamilyMemTablesImpl : public ColumnFamilyMemTables {
  ColumnFamilyHandleInternal handle_;
 };

-extern uint32_t GetColumnFamilyID(ColumnFamilyHandle* column_family);
+uint32_t GetColumnFamilyID(ColumnFamilyHandle* column_family);

-extern const Comparator* GetColumnFamilyUserComparator(
+const Comparator* GetColumnFamilyUserComparator(
    ColumnFamilyHandle* column_family);

-extern const ImmutableOptions& GetImmutableOptions(
-    ColumnFamilyHandle* column_family);
+const ImmutableOptions& GetImmutableOptions(ColumnFamilyHandle* column_family);

 }  // namespace ROCKSDB_NAMESPACE
@@ -270,7 +270,7 @@ class ColumnFamilyTestBase : public testing::Test {

  void Reopen(const std::vector<ColumnFamilyOptions> options = {}) {
    std::vector<std::string> names;
-    for (auto name : names_) {
+    for (const auto& name : names_) {
      if (name != "") {
        names.push_back(name);
      }
@@ -607,7 +607,7 @@ TEST_P(FlushEmptyCFTestWithParam, FlushEmptyCFTest) {
  // Preserve file system state up to here to simulate a crash condition.
  fault_env->SetFilesystemActive(false);
  std::vector<std::string> names;
-  for (auto name : names_) {
+  for (const auto& name : names_) {
    if (name != "") {
      names.push_back(name);
    }
@@ -669,7 +669,7 @@ TEST_P(FlushEmptyCFTestWithParam, FlushEmptyCFTest2) {
  // Preserve file system state up to here to simulate a crash condition.
  fault_env->SetFilesystemActive(false);
  std::vector<std::string> names;
-  for (auto name : names_) {
+  for (const auto& name : names_) {
    if (name != "") {
      names.push_back(name);
    }
@@ -1034,7 +1034,7 @@ TEST_P(ColumnFamilyTest, CrashAfterFlush) {
  fault_env->SetFilesystemActive(false);

  std::vector<std::string> names;
-  for (auto name : names_) {
+  for (const auto& name : names_) {
    if (name != "") {
      names.push_back(name);
    }
@@ -2604,90 +2604,92 @@ TEST_P(ColumnFamilyTest, WriteStallSingleColumnFamily) {
  mutable_cf_options.hard_pending_compaction_bytes_limit = 2000;
  mutable_cf_options.disable_auto_compactions = false;

-  vstorage->TEST_set_estimated_compaction_needed_bytes(50);
+  auto dbmu = dbfull()->TEST_Mutex();
+
+  vstorage->TEST_set_estimated_compaction_needed_bytes(50, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_TRUE(!IsDbWriteStopped());
  ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());

-  vstorage->TEST_set_estimated_compaction_needed_bytes(201);
+  vstorage->TEST_set_estimated_compaction_needed_bytes(201, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_TRUE(!IsDbWriteStopped());
  ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
  ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate());
  ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed());

-  vstorage->TEST_set_estimated_compaction_needed_bytes(400);
+  vstorage->TEST_set_estimated_compaction_needed_bytes(400, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_TRUE(!IsDbWriteStopped());
  ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
  ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate());
  ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed());

-  vstorage->TEST_set_estimated_compaction_needed_bytes(500);
+  vstorage->TEST_set_estimated_compaction_needed_bytes(500, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_TRUE(!IsDbWriteStopped());
  ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
  ASSERT_EQ(kBaseRate / 1.25 / 1.25, GetDbDelayedWriteRate());

-  vstorage->TEST_set_estimated_compaction_needed_bytes(450);
+  vstorage->TEST_set_estimated_compaction_needed_bytes(450, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_TRUE(!IsDbWriteStopped());
  ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
  ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate());

-  vstorage->TEST_set_estimated_compaction_needed_bytes(205);
+  vstorage->TEST_set_estimated_compaction_needed_bytes(205, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_TRUE(!IsDbWriteStopped());
  ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
  ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate());

-  vstorage->TEST_set_estimated_compaction_needed_bytes(202);
+  vstorage->TEST_set_estimated_compaction_needed_bytes(202, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_TRUE(!IsDbWriteStopped());
  ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
  ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate());

-  vstorage->TEST_set_estimated_compaction_needed_bytes(201);
+  vstorage->TEST_set_estimated_compaction_needed_bytes(201, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_TRUE(!IsDbWriteStopped());
  ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
  ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate());

-  vstorage->TEST_set_estimated_compaction_needed_bytes(198);
+  vstorage->TEST_set_estimated_compaction_needed_bytes(198, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_TRUE(!IsDbWriteStopped());
  ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());

-  vstorage->TEST_set_estimated_compaction_needed_bytes(399);
+  vstorage->TEST_set_estimated_compaction_needed_bytes(399, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_TRUE(!IsDbWriteStopped());
  ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
  ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate());

-  vstorage->TEST_set_estimated_compaction_needed_bytes(599);
+  vstorage->TEST_set_estimated_compaction_needed_bytes(599, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_TRUE(!IsDbWriteStopped());
  ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
  ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate());

-  vstorage->TEST_set_estimated_compaction_needed_bytes(2001);
+  vstorage->TEST_set_estimated_compaction_needed_bytes(2001, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_TRUE(IsDbWriteStopped());
  ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
  ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed());

-  vstorage->TEST_set_estimated_compaction_needed_bytes(3001);
+  vstorage->TEST_set_estimated_compaction_needed_bytes(3001, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_TRUE(IsDbWriteStopped());
  ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());

-  vstorage->TEST_set_estimated_compaction_needed_bytes(390);
+  vstorage->TEST_set_estimated_compaction_needed_bytes(390, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_TRUE(!IsDbWriteStopped());
  ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
  ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate());

-  vstorage->TEST_set_estimated_compaction_needed_bytes(100);
+  vstorage->TEST_set_estimated_compaction_needed_bytes(100, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_TRUE(!IsDbWriteStopped());
  ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
@@ -2706,7 +2708,7 @@ TEST_P(ColumnFamilyTest, WriteStallSingleColumnFamily) {
  ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate());

  vstorage->set_l0_delay_trigger_count(0);
-  vstorage->TEST_set_estimated_compaction_needed_bytes(300);
+  vstorage->TEST_set_estimated_compaction_needed_bytes(300, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_TRUE(!IsDbWriteStopped());
  ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
@@ -2718,14 +2720,14 @@ TEST_P(ColumnFamilyTest, WriteStallSingleColumnFamily) {
  ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
  ASSERT_EQ(kBaseRate / 1.25 / 1.25 / 1.25, GetDbDelayedWriteRate());

-  vstorage->TEST_set_estimated_compaction_needed_bytes(200);
+  vstorage->TEST_set_estimated_compaction_needed_bytes(200, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_TRUE(!IsDbWriteStopped());
  ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
  ASSERT_EQ(kBaseRate / 1.25 / 1.25, GetDbDelayedWriteRate());

  vstorage->set_l0_delay_trigger_count(0);
-  vstorage->TEST_set_estimated_compaction_needed_bytes(0);
+  vstorage->TEST_set_estimated_compaction_needed_bytes(0, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_TRUE(!IsDbWriteStopped());
  ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
@@ -2744,7 +2746,7 @@ TEST_P(ColumnFamilyTest, WriteStallSingleColumnFamily) {
  ASSERT_EQ(kBaseRate, dbfull()->TEST_write_controler().delayed_write_rate());

  vstorage->set_l0_delay_trigger_count(60);
-  vstorage->TEST_set_estimated_compaction_needed_bytes(300);
+  vstorage->TEST_set_estimated_compaction_needed_bytes(300, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_TRUE(!IsDbWriteStopped());
  ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
@@ -2753,14 +2755,14 @@ TEST_P(ColumnFamilyTest, WriteStallSingleColumnFamily) {

  mutable_cf_options.disable_auto_compactions = false;
  vstorage->set_l0_delay_trigger_count(70);
-  vstorage->TEST_set_estimated_compaction_needed_bytes(500);
+  vstorage->TEST_set_estimated_compaction_needed_bytes(500, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_TRUE(!IsDbWriteStopped());
  ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
  ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate());

  vstorage->set_l0_delay_trigger_count(71);
-  vstorage->TEST_set_estimated_compaction_needed_bytes(501);
+  vstorage->TEST_set_estimated_compaction_needed_bytes(501, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_TRUE(!IsDbWriteStopped());
  ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
@@ -2785,19 +2787,21 @@ TEST_P(ColumnFamilyTest, CompactionSpeedupSingleColumnFamily) {
  mutable_cf_options.soft_pending_compaction_bytes_limit = 200;
  mutable_cf_options.hard_pending_compaction_bytes_limit = 2000;

-  vstorage->TEST_set_estimated_compaction_needed_bytes(40);
+  auto dbmu = dbfull()->TEST_Mutex();
+
+  vstorage->TEST_set_estimated_compaction_needed_bytes(40, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());

-  vstorage->TEST_set_estimated_compaction_needed_bytes(50);
+  vstorage->TEST_set_estimated_compaction_needed_bytes(50, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed());

-  vstorage->TEST_set_estimated_compaction_needed_bytes(300);
+  vstorage->TEST_set_estimated_compaction_needed_bytes(300, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed());

-  vstorage->TEST_set_estimated_compaction_needed_bytes(45);
+  vstorage->TEST_set_estimated_compaction_needed_bytes(45, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());

@@ -2853,53 +2857,55 @@ TEST_P(ColumnFamilyTest, WriteStallTwoColumnFamilies) {
  MutableCFOptions mutable_cf_options1 = mutable_cf_options;
  mutable_cf_options1.soft_pending_compaction_bytes_limit = 500;

-  vstorage->TEST_set_estimated_compaction_needed_bytes(50);
+  auto dbmu = dbfull()->TEST_Mutex();
+
+  vstorage->TEST_set_estimated_compaction_needed_bytes(50, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_TRUE(!IsDbWriteStopped());
  ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());

-  vstorage1->TEST_set_estimated_compaction_needed_bytes(201);
+  vstorage1->TEST_set_estimated_compaction_needed_bytes(201, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_TRUE(!IsDbWriteStopped());
  ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());

-  vstorage1->TEST_set_estimated_compaction_needed_bytes(600);
+  vstorage1->TEST_set_estimated_compaction_needed_bytes(600, dbmu);
  RecalculateWriteStallConditions(cfd1, mutable_cf_options);
  ASSERT_TRUE(!IsDbWriteStopped());
  ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
  ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate());

-  vstorage->TEST_set_estimated_compaction_needed_bytes(70);
+  vstorage->TEST_set_estimated_compaction_needed_bytes(70, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_TRUE(!IsDbWriteStopped());
  ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
  ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate());

-  vstorage1->TEST_set_estimated_compaction_needed_bytes(800);
+  vstorage1->TEST_set_estimated_compaction_needed_bytes(800, dbmu);
  RecalculateWriteStallConditions(cfd1, mutable_cf_options);
  ASSERT_TRUE(!IsDbWriteStopped());
  ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
  ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate());

-  vstorage->TEST_set_estimated_compaction_needed_bytes(300);
+  vstorage->TEST_set_estimated_compaction_needed_bytes(300, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_TRUE(!IsDbWriteStopped());
  ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
  ASSERT_EQ(kBaseRate / 1.25 / 1.25, GetDbDelayedWriteRate());

-  vstorage1->TEST_set_estimated_compaction_needed_bytes(700);
+  vstorage1->TEST_set_estimated_compaction_needed_bytes(700, dbmu);
  RecalculateWriteStallConditions(cfd1, mutable_cf_options);
  ASSERT_TRUE(!IsDbWriteStopped());
  ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
  ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate());

-  vstorage->TEST_set_estimated_compaction_needed_bytes(500);
+  vstorage->TEST_set_estimated_compaction_needed_bytes(500, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_TRUE(!IsDbWriteStopped());
  ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
  ASSERT_EQ(kBaseRate / 1.25 / 1.25, GetDbDelayedWriteRate());

-  vstorage1->TEST_set_estimated_compaction_needed_bytes(600);
+  vstorage1->TEST_set_estimated_compaction_needed_bytes(600, dbmu);
  RecalculateWriteStallConditions(cfd1, mutable_cf_options);
  ASSERT_TRUE(!IsDbWriteStopped());
  ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
@@ -2932,29 +2938,31 @@ TEST_P(ColumnFamilyTest, CompactionSpeedupTwoColumnFamilies) {
  MutableCFOptions mutable_cf_options1 = mutable_cf_options;
  mutable_cf_options1.level0_slowdown_writes_trigger = 16;

-  vstorage->TEST_set_estimated_compaction_needed_bytes(40);
+  auto dbmu = dbfull()->TEST_Mutex();
+
+  vstorage->TEST_set_estimated_compaction_needed_bytes(40, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());

-  vstorage->TEST_set_estimated_compaction_needed_bytes(60);
+  vstorage->TEST_set_estimated_compaction_needed_bytes(60, dbmu);
  RecalculateWriteStallConditions(cfd1, mutable_cf_options);
  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed());

-  vstorage1->TEST_set_estimated_compaction_needed_bytes(30);
+  vstorage1->TEST_set_estimated_compaction_needed_bytes(30, dbmu);
  RecalculateWriteStallConditions(cfd1, mutable_cf_options);
  ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed());

-  vstorage1->TEST_set_estimated_compaction_needed_bytes(70);
+  vstorage1->TEST_set_estimated_compaction_needed_bytes(70, dbmu);
  RecalculateWriteStallConditions(cfd1, mutable_cf_options);
  ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed());

-  vstorage->TEST_set_estimated_compaction_needed_bytes(20);
+  vstorage->TEST_set_estimated_compaction_needed_bytes(20, dbmu);
  RecalculateWriteStallConditions(cfd, mutable_cf_options);
  ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed());

-  vstorage1->TEST_set_estimated_compaction_needed_bytes(3);
+  vstorage1->TEST_set_estimated_compaction_needed_bytes(3, dbmu);
  RecalculateWriteStallConditions(cfd1, mutable_cf_options);
  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());

@@ -2980,10 +2988,13 @@ TEST_P(ColumnFamilyTest, CompactionSpeedupForCompactionDebt) {
  mutable_cf_options.soft_pending_compaction_bytes_limit =
      std::numeric_limits<uint64_t>::max();

+  auto dbmu = dbfull()->TEST_Mutex();
+
  {
    // No bottommost data, so debt ratio cannot trigger speedup.
    VersionStorageInfo* vstorage = cfd->current()->storage_info();
-    vstorage->TEST_set_estimated_compaction_needed_bytes(1048576 /* 1MB */);
+    vstorage->TEST_set_estimated_compaction_needed_bytes(1048576 /* 1MB */,
+                                                         dbmu);
    RecalculateWriteStallConditions(cfd, mutable_cf_options);
    ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
  }
@@ -2996,18 +3007,88 @@ TEST_P(ColumnFamilyTest, CompactionSpeedupForCompactionDebt) {
    // 1MB debt is way bigger than bottommost data so definitely triggers
    // speedup.
    VersionStorageInfo* vstorage = cfd->current()->storage_info();
-    vstorage->TEST_set_estimated_compaction_needed_bytes(1048576 /* 1MB */);
+    vstorage->TEST_set_estimated_compaction_needed_bytes(1048576 /* 1MB */,
+                                                         dbmu);
    RecalculateWriteStallConditions(cfd, mutable_cf_options);
    ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed());

    // Eight bytes is way smaller than bottommost data so definitely does not
    // trigger speedup.
-    vstorage->TEST_set_estimated_compaction_needed_bytes(8);
+    vstorage->TEST_set_estimated_compaction_needed_bytes(8, dbmu);
    RecalculateWriteStallConditions(cfd, mutable_cf_options);
    ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
  }
 }

+TEST_P(ColumnFamilyTest, CompactionSpeedupForMarkedFiles) {
+  const int kParallelismLimit = 3;
+  class AlwaysCompactTpc : public TablePropertiesCollector {
+   public:
+    Status Finish(UserCollectedProperties* /* properties */) override {
+      return Status::OK();
+    }
+
+    UserCollectedProperties GetReadableProperties() const override {
+      return UserCollectedProperties{};
+    }
+
+    const char* Name() const override { return "AlwaysCompactTpc"; }
+
+    bool NeedCompact() const override { return true; }
+  };
+
+  class AlwaysCompactTpcf : public TablePropertiesCollectorFactory {
+   public:
+    TablePropertiesCollector* CreateTablePropertiesCollector(
+        TablePropertiesCollectorFactory::Context /* context */) override {
+      return new AlwaysCompactTpc();
+    }
+
+    const char* Name() const override { return "AlwaysCompactTpcf"; }
+  };
+
+  column_family_options_.num_levels = 2;
+  column_family_options_.table_properties_collector_factories.emplace_back(
+      std::make_shared<AlwaysCompactTpcf>());
+  db_options_.max_background_compactions = kParallelismLimit;
+  Open();
+
+  // Make a nonempty last level. Only marked files in upper levels count.
+  ASSERT_OK(db_->Put(WriteOptions(), "foo", "bar"));
+  ASSERT_OK(db_->Flush(FlushOptions()));
+  WaitForCompaction();
+  AssertFilesPerLevel("0,1", 0 /* cf */);
+
+  // Block the compaction thread pool so marked files accumulate in L0.
+  test::SleepingBackgroundTask sleeping_tasks[kParallelismLimit];
+  for (int i = 0; i < kParallelismLimit; i++) {
+    env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
+                   &sleeping_tasks[i], Env::Priority::LOW);
+    sleeping_tasks[i].WaitUntilSleeping();
+  }
+
+  // Zero marked upper-level files. No speedup.
+  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
+  AssertFilesPerLevel("0,1", 0 /* cf */);
+
+  // One marked upper-level file. No speedup.
+  ASSERT_OK(db_->Put(WriteOptions(), "foo", "bar"));
+  ASSERT_OK(db_->Flush(FlushOptions()));
+  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
+  AssertFilesPerLevel("1,1", 0 /* cf */);
+
+  // Two marked upper-level files. Speedup.
+  ASSERT_OK(db_->Put(WriteOptions(), "foo", "bar"));
+  ASSERT_OK(db_->Flush(FlushOptions()));
+  ASSERT_EQ(kParallelismLimit, dbfull()->TEST_BGCompactionsAllowed());
+  AssertFilesPerLevel("2,1", 0 /* cf */);
+
+  for (int i = 0; i < kParallelismLimit; i++) {
+    sleeping_tasks[i].WakeUp();
+    sleeping_tasks[i].WaitUntilDone();
+  }
+}
+
 TEST_P(ColumnFamilyTest, CreateAndDestroyOptions) {
  std::unique_ptr<ColumnFamilyOptions> cfo(new ColumnFamilyOptions());
  ColumnFamilyHandle* cfh;
@@ -3326,9 +3407,13 @@ TEST_P(ColumnFamilyTest, DISABLED_LogTruncationTest) {
  for (size_t i = 0; i < filenames.size(); i++) {
    uint64_t number;
    FileType type;
-    if (!(ParseFileName(filenames[i], &number, &type))) continue;
+    if (!(ParseFileName(filenames[i], &number, &type))) {
+      continue;
+    }

-    if (type != kWalFile) continue;
+    if (type != kWalFile) {
+      continue;
+    }

    logfs.push_back(filenames[i]);
  }
@@ -34,8 +34,8 @@ class CompactFilesTest : public testing::Test {
 // A class which remembers the name of each flushed file.
 class FlushedFileCollector : public EventListener {
 public:
-  FlushedFileCollector() {}
-  ~FlushedFileCollector() override {}
+  FlushedFileCollector() = default;
+  ~FlushedFileCollector() override = default;

  void OnFlushCompleted(DB* /*db*/, const FlushJobInfo& info) override {
    std::lock_guard<std::mutex> lock(mutex_);
@@ -45,7 +45,7 @@ class FlushedFileCollector : public EventListener {
  std::vector<std::string> GetFlushedFiles() {
    std::lock_guard<std::mutex> lock(mutex_);
    std::vector<std::string> result;
-    for (auto fname : flushed_files_) {
+    for (const auto& fname : flushed_files_) {
      result.push_back(fname);
    }
    return result;
@@ -159,7 +159,9 @@ TEST_F(CompactFilesTest, MultipleLevel) {
  // Compact files except the file in L3
  std::vector<std::string> files;
  for (int i = 0; i < 6; ++i) {
-    if (i == 3) continue;
+    if (i == 3) {
+      continue;
+    }
    for (auto& file : meta.levels[i].files) {
      files.push_back(file.db_path + "/" + file.name);
    }
@@ -228,7 +230,7 @@ TEST_F(CompactFilesTest, ObsoleteFiles) {
  ASSERT_OK(static_cast_with_check<DBImpl>(db)->TEST_WaitForCompact());

  // verify all compaction input files are deleted
-  for (auto fname : l0_files) {
+  for (const auto& fname : l0_files) {
    ASSERT_EQ(Status::NotFound(), env_->FileExists(fname));
  }
  delete db;
@@ -492,4 +494,3 @@ int main(int argc, char** argv) {
  ::testing::InitGoogleTest(&argc, argv);
  return RUN_ALL_TESTS();
 }
-
@@ -13,6 +13,7 @@
 #include <vector>

 #include "db/column_family.h"
+#include "db/dbformat.h"
 #include "logging/logging.h"
 #include "rocksdb/compaction_filter.h"
 #include "rocksdb/sst_partitioner.h"
@@ -21,9 +22,6 @@

 namespace ROCKSDB_NAMESPACE {

-const uint64_t kRangeTombstoneSentinel =
-    PackSequenceAndType(kMaxSequenceNumber, kTypeRangeDeletion);
-
 int sstableKeyCompare(const Comparator* uc, const Slice& a, const Slice& b) {
  auto c = uc->CompareWithoutTimestamp(ExtractUserKey(a), ExtractUserKey(b));
  if (c != 0) {
@@ -162,7 +160,9 @@ std::vector<CompactionInputFiles> Compaction::PopulateWithAtomicBoundaries(
    AtomicCompactionUnitBoundary cur_boundary;
    size_t first_atomic_idx = 0;
    auto add_unit_boundary = [&](size_t to) {
-      if (first_atomic_idx == to) return;
+      if (first_atomic_idx == to) {
+        return;
+      }
      for (size_t k = first_atomic_idx; k < to; k++) {
        inputs[i].atomic_compaction_unit_boundaries.push_back(cur_boundary);
      }
@@ -352,11 +352,9 @@ Compaction::Compaction(

  // for the non-bottommost levels, it tries to build files match the target
  // file size, but not guaranteed. It could be 2x the size of the target size.
-  max_output_file_size_ =
-      bottommost_level_ || grandparents_.empty() ||
-              !_immutable_options.level_compaction_dynamic_file_size
-          ? target_output_file_size_
-          : 2 * target_output_file_size_;
+  max_output_file_size_ = bottommost_level_ || grandparents_.empty()
+                              ? target_output_file_size_
+                              : 2 * target_output_file_size_;

 #ifndef NDEBUG
  for (size_t i = 1; i < inputs_.size(); ++i) {
@@ -757,7 +755,9 @@ int InputSummary(const std::vector<FileMetaData*>& files, char* output,
    AppendHumanBytes(files.at(i)->fd.GetFileSize(), sztxt, 16);
    ret = snprintf(output + write, sz, "%" PRIu64 "(%s) ",
                   files.at(i)->fd.GetNumber(), sztxt);
-    if (ret < 0 || ret >= sz) break;
+    if (ret < 0 || ret >= sz) {
+      break;
+    }
    write += ret;
  }
  // if files.size() is non-zero, overwrite the last space
@@ -595,6 +595,6 @@ struct PerKeyPlacementContext {
 #endif /* !NDEBUG */

 // Return sum of sizes of all files in `files`.
-extern uint64_t TotalFileSize(const std::vector<FileMetaData*>& files);
+uint64_t TotalFileSize(const std::vector<FileMetaData*>& files);

 }  // namespace ROCKSDB_NAMESPACE
@@ -44,6 +44,13 @@ struct CompactionIterationStats {
  uint64_t total_blob_bytes_read = 0;
  uint64_t num_blobs_relocated = 0;
  uint64_t total_blob_bytes_relocated = 0;
+
+  // TimedPut diagnostics
+  // Total number of kTypeValuePreferredSeqno records encountered.
+  uint64_t num_input_timed_put_records = 0;
+  // Number of kTypeValuePreferredSeqno records we ended up swapping in
+  // preferred seqno.
+  uint64_t num_timed_put_swap_preferred_seqno = 0;
 };

 }  // namespace ROCKSDB_NAMESPACE
@@ -491,6 +491,8 @@ void CompactionIterator::NextFromInput() {
    if (ikey_.type == kTypeDeletion || ikey_.type == kTypeSingleDeletion ||
        ikey_.type == kTypeDeletionWithTimestamp) {
      iter_stats_.num_input_deletion_records++;
+    } else if (ikey_.type == kTypeValuePreferredSeqno) {
+      iter_stats_.num_input_timed_put_records++;
    }
    iter_stats_.total_input_raw_key_bytes += key_.size();
    iter_stats_.total_input_raw_value_bytes += value_.size();
@@ -618,7 +620,8 @@ void CompactionIterator::NextFromInput() {
      // not compact out.  We will keep this Put, but can drop it's data.
      // (See Optimization 3, below.)
      if (ikey_.type != kTypeValue && ikey_.type != kTypeBlobIndex &&
-          ikey_.type != kTypeWideColumnEntity) {
+          ikey_.type != kTypeWideColumnEntity &&
+          ikey_.type != kTypeValuePreferredSeqno) {
        ROCKS_LOG_FATAL(info_log_, "Unexpected key %s for compaction output",
                        ikey_.DebugString(allow_data_in_errors_, true).c_str());
        assert(false);
@@ -632,7 +635,8 @@ void CompactionIterator::NextFromInput() {
        assert(false);
      }

-      if (ikey_.type == kTypeBlobIndex || ikey_.type == kTypeWideColumnEntity) {
+      if (ikey_.type == kTypeBlobIndex || ikey_.type == kTypeWideColumnEntity ||
+          ikey_.type == kTypeValuePreferredSeqno) {
        ikey_.type = kTypeValue;
        current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type);
      }
@@ -798,7 +802,8 @@ void CompactionIterator::NextFromInput() {
            // happened
            if (next_ikey.type != kTypeValue &&
                next_ikey.type != kTypeBlobIndex &&
-                next_ikey.type != kTypeWideColumnEntity) {
+                next_ikey.type != kTypeWideColumnEntity &&
+                next_ikey.type != kTypeValuePreferredSeqno) {
              ++iter_stats_.num_single_del_mismatch;
            }

@@ -968,6 +973,50 @@ void CompactionIterator::NextFromInput() {
        validity_info_.SetValid(ValidContext::kKeepDel);
        at_next_ = true;
      }
+    } else if (ikey_.type == kTypeValuePreferredSeqno &&
+               DefinitelyInSnapshot(ikey_.sequence, earliest_snapshot_) &&
+               (bottommost_level_ ||
+                (compaction_ != nullptr &&
+                 compaction_->KeyNotExistsBeyondOutputLevel(ikey_.user_key,
+                                                            &level_ptrs_)))) {
+      // This section that attempts to swap preferred sequence number will not
+      // be invoked if this is a CompactionIterator created for flush, since
+      // `compaction_` will be nullptr and it's not bottommost either.
+      //
+      // The entries with the same user key and smaller sequence numbers are
+      // all in this earliest snapshot range to be iterated. Since those entries
+      // will be hidden by this entry [rule A], it's safe to swap in the
+      // preferred seqno now.
+      //
+      // It's otherwise not safe to swap in the preferred seqno since it's
+      // possible for entries in earlier snapshots to have sequence number that
+      // is smaller than this entry's sequence number but bigger than this
+      // entry's preferred sequence number. Swapping in the preferred sequence
+      // number will break the internal key ordering invariant for this key.
+      //
+      // A special case involving range deletion is handled separately below.
+      auto [unpacked_value, preferred_seqno] =
+          ParsePackedValueWithSeqno(value_);
+      assert(preferred_seqno < ikey_.sequence);
+      InternalKey ikey_after_swap(ikey_.user_key, preferred_seqno, kTypeValue);
+      Slice ikey_after_swap_slice(*ikey_after_swap.rep());
+      if (range_del_agg_->ShouldDelete(
+              ikey_after_swap_slice,
+              RangeDelPositioningMode::kForwardTraversal)) {
+        // A range tombstone that doesn't cover this kTypeValuePreferredSeqno
+        // entry may end up covering the entry, so it's not safe to swap
+        // preferred sequence number. In this case, we output the entry as is.
+        validity_info_.SetValid(ValidContext::kNewUserKey);
+      } else {
+        iter_stats_.num_timed_put_swap_preferred_seqno++;
+        ikey_.sequence = preferred_seqno;
+        ikey_.type = kTypeValue;
+        current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type);
+        key_ = current_key_.GetInternalKey();
+        ikey_.user_key = current_key_.GetUserKey();
+        value_ = unpacked_value;
+        validity_info_.SetValid(ValidContext::kSwapPreferredSeqno);
+      }
    } else if (ikey_.type == kTypeMerge) {
      if (!merge_helper_->HasOperator()) {
        status_ = Status::InvalidArgument(
@@ -410,6 +410,7 @@ class CompactionIterator {
    kKeepDel = 9,
    kNewUserKey = 10,
    kRangeDeletion = 11,
+    kSwapPreferredSeqno = 12,
  };

  struct ValidityInfo {
@@ -17,6 +17,14 @@
 #include "utilities/merge_operators.h"

 namespace ROCKSDB_NAMESPACE {
+namespace {
+std::string ValueWithPreferredSeqno(std::string val,
+                                    SequenceNumber preferred_seqno = 0) {
+  std::string result = val;
+  PutFixed64(&result, preferred_seqno);
+  return result;
+}
+}  // namespace

 // Expects no merging attempts.
 class NoMergingMergeOp : public MergeOperator {
@@ -392,6 +400,17 @@ TEST_P(CompactionIteratorTest, CorruptionAfterSingleDeletion) {
  ASSERT_FALSE(c_iter_->Valid());
 }

+// Tests compatibility of TimedPut and SingleDelete. TimedPut should act as if
+// it's a Put.
+TEST_P(CompactionIteratorTest, TimedPutAndSingleDelete) {
+  InitIterators({test::KeyStr("a", 5, kTypeSingleDeletion),
+                 test::KeyStr("a", 3, kTypeValuePreferredSeqno)},
+                {"", "val"}, {}, {}, 5);
+  c_iter_->SeekToFirst();
+  ASSERT_OK(c_iter_->status());
+  ASSERT_FALSE(c_iter_->Valid());
+}
+
 TEST_P(CompactionIteratorTest, SimpleRangeDeletion) {
  InitIterators({test::KeyStr("morning", 5, kTypeValue),
                 test::KeyStr("morning", 2, kTypeValue),
@@ -431,6 +450,31 @@ TEST_P(CompactionIteratorTest, RangeDeletionWithSnapshots) {
  ASSERT_FALSE(c_iter_->Valid());
 }

+// Tests compatibility of TimedPut and Range delete. TimedPut should act as if
+// it's a Put.
+TEST_P(CompactionIteratorTest, TimedPutAndRangeDeletion) {
+  InitIterators(
+      {test::KeyStr("morning", 5, kTypeValuePreferredSeqno),
+       test::KeyStr("morning", 2, kTypeValuePreferredSeqno),
+       test::KeyStr("night", 3, kTypeValuePreferredSeqno)},
+      {ValueWithPreferredSeqno("zao5"), ValueWithPreferredSeqno("zao2"),
+       ValueWithPreferredSeqno("wan")},
+      {test::KeyStr("ma", 4, kTypeRangeDeletion)}, {"mz"}, 5);
+  c_iter_->SeekToFirst();
+  ASSERT_TRUE(c_iter_->Valid());
+  ASSERT_EQ(test::KeyStr("morning", 5, kTypeValuePreferredSeqno),
+            c_iter_->key().ToString());
+  ASSERT_EQ(ValueWithPreferredSeqno("zao5"), c_iter_->value().ToString());
+  c_iter_->Next();
+  ASSERT_TRUE(c_iter_->Valid());
+  ASSERT_EQ(test::KeyStr("night", 3, kTypeValuePreferredSeqno),
+            c_iter_->key().ToString());
+  ASSERT_EQ(ValueWithPreferredSeqno("wan"), c_iter_->value().ToString());
+  c_iter_->Next();
+  ASSERT_OK(c_iter_->status());
+  ASSERT_FALSE(c_iter_->Valid());
+}
+
 TEST_P(CompactionIteratorTest, CompactionFilterSkipUntil) {
  class Filter : public CompactionFilter {
    Decision FilterV2(int /*level*/, const Slice& key, ValueType t,
@@ -502,9 +546,11 @@ TEST_P(CompactionIteratorTest, CompactionFilterSkipUntil) {
       test::KeyStr("f", 25, kTypeValue), test::KeyStr("g", 90, kTypeValue),
       test::KeyStr("h", 91, kTypeValue),  // keep
       test::KeyStr("i", 95, kTypeMerge),  // skip to "z"
-       test::KeyStr("j", 99, kTypeValue)},
+       test::KeyStr("j", 99, kTypeValue),
+       test::KeyStr("k", 100, kTypeValuePreferredSeqno)},
      {"av50", "am45", "bv60", "bv40", "cv35", "dm70", "em71", "fm65", "fm30",
-       "fv25", "gv90", "hv91", "im95", "jv99"},
+       "fv25", "gv90", "hv91", "im95", "jv99",
+       ValueWithPreferredSeqno("kv100")},
      {}, {}, kMaxSequenceNumber, kMaxSequenceNumber, &merge_op, &filter);

  // Compaction should output just "a", "e" and "h" keys.
@@ -614,87 +660,87 @@ TEST_P(CompactionIteratorTest, ShuttingDownInMerge) {
  EXPECT_EQ(2, filter.last_seen.load());
 }

-TEST_P(CompactionIteratorTest, SingleMergeOperand) {
-  class Filter : public CompactionFilter {
-    Decision FilterV2(int /*level*/, const Slice& key, ValueType t,
-                      const Slice& existing_value, std::string* /*new_value*/,
-                      std::string* /*skip_until*/) const override {
-      std::string k = key.ToString();
-      std::string v = existing_value.ToString();
+class Filter : public CompactionFilter {
+  Decision FilterV2(int /*level*/, const Slice& key, ValueType t,
+                    const Slice& existing_value, std::string* /*new_value*/,
+                    std::string* /*skip_until*/) const override {
+    std::string k = key.ToString();
+    std::string v = existing_value.ToString();

-      // See InitIterators() call below for the sequence of keys and their
-      // filtering decisions. Here we closely assert that compaction filter is
-      // called with the expected keys and only them, and with the right values.
-      if (k == "a") {
-        EXPECT_EQ(ValueType::kMergeOperand, t);
-        EXPECT_EQ("av1", v);
-        return Decision::kKeep;
-      } else if (k == "b") {
-        EXPECT_EQ(ValueType::kMergeOperand, t);
-        return Decision::kKeep;
-      } else if (k == "c") {
-        return Decision::kKeep;
-      }
-
-      ADD_FAILURE();
+    // See InitIterators() call below for the sequence of keys and their
+    // filtering decisions. Here we closely assert that compaction filter is
+    // called with the expected keys and only them, and with the right values.
+    if (k == "a") {
+      EXPECT_EQ(ValueType::kMergeOperand, t);
+      EXPECT_EQ("av1", v);
+      return Decision::kKeep;
+    } else if (k == "b") {
+      EXPECT_EQ(ValueType::kMergeOperand, t);
+      return Decision::kKeep;
+    } else if (k == "c") {
      return Decision::kKeep;
    }

-    const char* Name() const override {
-      return "CompactionIteratorTest.SingleMergeOperand::Filter";
-    }
-  };
+    ADD_FAILURE();
+    return Decision::kKeep;
+  }

-  class SingleMergeOp : public MergeOperator {
-   public:
-    bool FullMergeV2(const MergeOperationInput& merge_in,
-                     MergeOperationOutput* merge_out) const override {
-      // See InitIterators() call below for why "c" is the only key for which
-      // FullMergeV2 should be called.
-      EXPECT_EQ("c", merge_in.key.ToString());
+  const char* Name() const override {
+    return "CompactionIteratorTest.SingleMergeOperand::Filter";
+  }
+};

-      std::string temp_value;
-      if (merge_in.existing_value != nullptr) {
-        temp_value = merge_in.existing_value->ToString();
-      }
+class SingleMergeOp : public MergeOperator {
+ public:
+  bool FullMergeV2(const MergeOperationInput& merge_in,
+                   MergeOperationOutput* merge_out) const override {
+    // See InitIterators() call below for why "c" is the only key for which
+    // FullMergeV2 should be called.
+    EXPECT_EQ("c", merge_in.key.ToString());

-      for (auto& operand : merge_in.operand_list) {
-        temp_value.append(operand.ToString());
-      }
-      merge_out->new_value = temp_value;
-
-      return true;
+    std::string temp_value;
+    if (merge_in.existing_value != nullptr) {
+      temp_value = merge_in.existing_value->ToString();
    }

-    bool PartialMergeMulti(const Slice& key,
-                           const std::deque<Slice>& operand_list,
-                           std::string* new_value,
-                           Logger* /*logger*/) const override {
-      std::string string_key = key.ToString();
-      EXPECT_TRUE(string_key == "a" || string_key == "b");
+    for (auto& operand : merge_in.operand_list) {
+      temp_value.append(operand.ToString());
+    }
+    merge_out->new_value = temp_value;

-      if (string_key == "a") {
-        EXPECT_EQ(1, operand_list.size());
-      } else if (string_key == "b") {
-        EXPECT_EQ(2, operand_list.size());
-      }
+    return true;
+  }

-      std::string temp_value;
-      for (auto& operand : operand_list) {
-        temp_value.append(operand.ToString());
-      }
-      swap(temp_value, *new_value);
+  bool PartialMergeMulti(const Slice& key,
+                         const std::deque<Slice>& operand_list,
+                         std::string* new_value,
+                         Logger* /*logger*/) const override {
+    std::string string_key = key.ToString();
+    EXPECT_TRUE(string_key == "a" || string_key == "b");

-      return true;
+    if (string_key == "a") {
+      EXPECT_EQ(1, operand_list.size());
+    } else if (string_key == "b") {
+      EXPECT_EQ(2, operand_list.size());
    }

-    const char* Name() const override {
-      return "CompactionIteratorTest SingleMergeOp";
+    std::string temp_value;
+    for (auto& operand : operand_list) {
+      temp_value.append(operand.ToString());
    }
+    swap(temp_value, *new_value);

-    bool AllowSingleOperand() const override { return true; }
-  };
+    return true;
+  }

+  const char* Name() const override {
+    return "CompactionIteratorTest SingleMergeOp";
+  }
+
+  bool AllowSingleOperand() const override { return true; }
+};
+
+TEST_P(CompactionIteratorTest, SingleMergeOperand) {
  SingleMergeOp merge_op;
  Filter filter;
  InitIterators(
@@ -719,6 +765,24 @@ TEST_P(CompactionIteratorTest, SingleMergeOperand) {
  ASSERT_EQ("cv1cv2", c_iter_->value().ToString());
 }

+// Tests compatibility of TimedPut and Merge operation. When a TimedPut is
+// merged with some merge operand in compaction, it will become a regular Put
+// and lose its preferred sequence number.
+TEST_P(CompactionIteratorTest, TimedPutAndMerge) {
+  SingleMergeOp merge_op;
+  Filter filter;
+  InitIterators({test::KeyStr("c", 90, kTypeMerge),
+                 test::KeyStr("c", 80, kTypeValuePreferredSeqno)},
+                {"cv2", ValueWithPreferredSeqno("cv1")}, {}, {},
+                kMaxSequenceNumber, kMaxSequenceNumber, &merge_op, &filter);
+
+  c_iter_->SeekToFirst();
+  ASSERT_TRUE(c_iter_->Valid());
+  ASSERT_EQ(test::KeyStr("c", 90, kTypeValue), c_iter_->key().ToString());
+  ASSERT_OK(c_iter_->status());
+  ASSERT_EQ("cv1cv2", c_iter_->value().ToString());
+}
+
 // In bottommost level, values earlier than earliest snapshot can be output
 // with sequence = 0.
 TEST_P(CompactionIteratorTest, ZeroOutSequenceAtBottomLevel) {
@@ -963,6 +1027,22 @@ TEST_F(CompactionIteratorWithSnapshotCheckerTest, DedupSameSnapshot_Value) {
      {"v4", "v3", "v1"}, 3 /*last_committed_seq*/);
 }

+TEST_F(CompactionIteratorWithSnapshotCheckerTest, DedupSameSnapshot_TimedPut) {
+  AddSnapshot(2, 1);
+  RunTest({test::KeyStr("foo", 4, kTypeValuePreferredSeqno),
+           test::KeyStr("foo", 3, kTypeValuePreferredSeqno),
+           test::KeyStr("foo", 2, kTypeValuePreferredSeqno),
+           test::KeyStr("foo", 1, kTypeValuePreferredSeqno)},
+          {ValueWithPreferredSeqno("v4"), ValueWithPreferredSeqno("v3"),
+           ValueWithPreferredSeqno("v2"), ValueWithPreferredSeqno("v1")},
+          {test::KeyStr("foo", 4, kTypeValuePreferredSeqno),
+           test::KeyStr("foo", 3, kTypeValuePreferredSeqno),
+           test::KeyStr("foo", 1, kTypeValuePreferredSeqno)},
+          {ValueWithPreferredSeqno("v4"), ValueWithPreferredSeqno("v3"),
+           ValueWithPreferredSeqno("v1")},
+          3 /*last_committed_seq*/);
+}
+
 TEST_F(CompactionIteratorWithSnapshotCheckerTest, DedupSameSnapshot_Deletion) {
  AddSnapshot(2, 1);
  RunTest(
@@ -1128,6 +1208,114 @@ TEST_F(CompactionIteratorWithSnapshotCheckerTest,
          2 /* earliest_write_conflict_snapshot */);
 }

+// Same as above but with a value with preferred seqno entry. In addition to the
+// value getting trimmed, the type of the KV is changed to kTypeValue.
+TEST_F(CompactionIteratorWithSnapshotCheckerTest,
+       KeepSingleDeletionForWriteConflictChecking_TimedPut) {
+  AddSnapshot(2, 0);
+  RunTest({test::KeyStr("a", 2, kTypeSingleDeletion),
+           test::KeyStr("a", 1, kTypeValuePreferredSeqno)},
+          {"", ValueWithPreferredSeqno("v1")},
+          {test::KeyStr("a", 2, kTypeSingleDeletion),
+           test::KeyStr("a", 1, kTypeValue)},
+          {"", ""}, 2 /* last_committed_seq */, nullptr /* merge_operator */,
+          nullptr /* compaction_filter */, false /* bottommost_level */,
+          2 /* earliest_write_conflict_snapshot */);
+}
+
+// Tests when a kTypeValuePreferredSeqno entry can have its preferred sequence
+// number swapped in. The required and sufficient conditions for an entry's
+// preferred sequence number to get swapped in are:
+// 1) The entry is visible to the earliest snapshot, AND
+// 2) No more entries with the same user key on lower levels, AND
+//    This is either because:
+//    2a) This is a compaction to the bottommost level, OR
+//    2b) Keys do not exist beyond output level
+// 3) The entry will not resurface a range deletion entry after swapping in the
+//    preferred sequence number.
+TEST_F(CompactionIteratorWithSnapshotCheckerTest,
+       TimedPut_NotVisibleToEarliestSnapshot_NoSwapPreferredSeqno) {
+  AddSnapshot(3);
+  RunTest({test::KeyStr("bar", 5, kTypeValuePreferredSeqno)},
+          {ValueWithPreferredSeqno("bv2", 2)},
+          {test::KeyStr("bar", 5, kTypeValuePreferredSeqno)},
+          {ValueWithPreferredSeqno("bv2", 2), "bv1"}, 5 /*last_committed_seq*/,
+          nullptr /*merge_operator*/, nullptr /*compaction_filter*/,
+          true /*bottommost_level*/,
+          kMaxSequenceNumber /*earliest_write_conflict_snapshot*/,
+          true /*key_not_exists_beyond_output_level*/);
+}
+
+TEST_F(CompactionIteratorWithSnapshotCheckerTest,
+       TimedPut_MoreEntriesInLowerLevels_NoSwapPreferredSeqno) {
+  // This tests mimics more entries in lower levels with `bottommost_level` and
+  // `key_not_exists_beyond_output_level` set to false.
+  RunTest({test::KeyStr("bar", 5, kTypeValuePreferredSeqno)},
+          {ValueWithPreferredSeqno("bv2", 2)},
+          {test::KeyStr("bar", 5, kTypeValuePreferredSeqno)},
+          {ValueWithPreferredSeqno("bv2", 2)}, 5 /*last_committed_seq*/,
+          nullptr /*merge_operator*/, nullptr /*compaction_filter*/,
+          false /*bottommost_level*/,
+          kMaxSequenceNumber /*earliest_write_conflict_snapshot*/,
+          false /*key_not_exists_beyond_output_level*/);
+}
+
+TEST_F(CompactionIteratorWithSnapshotCheckerTest,
+       TimedPut_WillBeHiddenByRangeDeletionAfterSwap_NoSwap) {
+  InitIterators({test::KeyStr("morning", 5, kTypeValuePreferredSeqno),
+                 test::KeyStr("night", 6, kTypeValue)},
+                {ValueWithPreferredSeqno("zao", 3), "wan"},
+                {test::KeyStr("ma", 4, kTypeRangeDeletion)}, {"mz"}, 6,
+                kMaxSequenceNumber /*last_committed_sequence*/,
+                nullptr /*merge_op*/, nullptr /*filter*/,
+                false /*bottommost_level*/,
+                kMaxSequenceNumber /*earliest_write_conflict_snapshot*/,
+                true /*key_not_exists_beyond_output_level*/);
+  c_iter_->SeekToFirst();
+  ASSERT_TRUE(c_iter_->Valid());
+  ASSERT_EQ(test::KeyStr("morning", 5, kTypeValuePreferredSeqno),
+            c_iter_->key().ToString());
+  ASSERT_EQ(ValueWithPreferredSeqno("zao", 3), c_iter_->value().ToString());
+  c_iter_->Next();
+  ASSERT_TRUE(c_iter_->Valid());
+  ASSERT_EQ(test::KeyStr("night", 6, kTypeValue), c_iter_->key().ToString());
+  ASSERT_EQ("wan", c_iter_->value().ToString());
+  c_iter_->Next();
+  ASSERT_FALSE(c_iter_->Valid());
+  ASSERT_OK(c_iter_->status());
+}
+
+TEST_F(CompactionIteratorWithSnapshotCheckerTest,
+       TimedPut_BottomMostLevelVisibleToEarliestSnapshot_SwapPreferredSeqno) {
+  // Preferred seqno got swapped in and also zeroed out as a bottommost level
+  // optimization.
+  RunTest(
+      {test::KeyStr("bar", 5, kTypeValuePreferredSeqno),
+       test::KeyStr("bar", 4, kTypeValuePreferredSeqno),
+       test::KeyStr("foo", 6, kTypeValue)},
+      {ValueWithPreferredSeqno("bv2", 2), ValueWithPreferredSeqno("bv1", 1),
+       "fv1"},
+      {test::KeyStr("bar", 0, kTypeValue), test::KeyStr("foo", 0, kTypeValue)},
+      {"bv2", "fv1"}, 6 /*last_committed_seq*/, nullptr /*merge_operator*/,
+      nullptr /*compaction_filter*/, true /*bottommost_level*/);
+}
+
+TEST_F(
+    CompactionIteratorWithSnapshotCheckerTest,
+    TimedPut_NonBottomMostLevelVisibleToEarliestSnapshot_SwapPreferredSeqno) {
+  RunTest(
+      {test::KeyStr("bar", 5, kTypeValuePreferredSeqno),
+       test::KeyStr("bar", 4, kTypeValuePreferredSeqno),
+       test::KeyStr("foo", 6, kTypeValue)},
+      {ValueWithPreferredSeqno("bv2", 2), ValueWithPreferredSeqno("bv1", 1),
+       "fv1"},
+      {test::KeyStr("bar", 2, kTypeValue), test::KeyStr("foo", 6, kTypeValue)},
+      {"bv2", "fv1"}, 6 /*last_committed_seq*/, nullptr /*merge_operator*/,
+      nullptr /*compaction_filter*/, false /*bottommost_level*/,
+      kMaxSequenceNumber /*earliest_write_conflict_snapshot*/,
+      true /*key_not_exists_beyond_output_level*/);
+}
+
 // Compaction filter should keep uncommitted key as-is, and
 //   * Convert the latest value to deletion, and/or
 //   * if latest value is a merge, apply filter to all subsequent merges.
@@ -1145,6 +1333,22 @@ TEST_F(CompactionIteratorWithSnapshotCheckerTest, CompactionFilter_Value) {
      nullptr /*merge_operator*/, compaction_filter.get());
 }

+TEST_F(CompactionIteratorWithSnapshotCheckerTest, CompactionFilter_TimedPut) {
+  // TODO(yuzhangyu): Add support for this type in compaction filter.
+  // Type kTypeValuePreferredSeqno is not explicitly exposed in the compaction
+  // filter API, so users can not operate on it through compaction filter API
+  // to remove/purge/change value etc. But this type of entry can be impacted by
+  // other entries' filter result, currently only kRemoveAndSkip type of result
+  // can affect it.
+  std::unique_ptr<CompactionFilter> compaction_filter(
+      new FilterAllKeysCompactionFilter());
+  RunTest({test::KeyStr("a", 2, kTypeValuePreferredSeqno)},
+          {ValueWithPreferredSeqno("v1")},
+          {test::KeyStr("a", 2, kTypeValuePreferredSeqno)},
+          {ValueWithPreferredSeqno("v1")}, 2 /*last_committed_seq*/,
+          nullptr /*merge_operator*/, compaction_filter.get());
+}
+
 TEST_F(CompactionIteratorWithSnapshotCheckerTest, CompactionFilter_Deletion) {
  std::unique_ptr<CompactionFilter> compaction_filter(
      new FilterAllKeysCompactionFilter());
@@ -404,7 +404,9 @@ void CompactionJob::AcquireSubcompactionResources(

 void CompactionJob::ShrinkSubcompactionResources(uint64_t num_extra_resources) {
  // Do nothing when we have zero resources to shrink
-  if (num_extra_resources == 0) return;
+  if (num_extra_resources == 0) {
+    return;
+  }
  db_mutex_->Lock();
  // We cannot release threads more than what we reserved before
  int extra_num_subcompaction_threads_released = env_->ReleaseThreads(
@@ -450,14 +452,6 @@ void CompactionJob::ReleaseSubcompactionResources() {
  ShrinkSubcompactionResources(extra_num_subcompaction_threads_reserved_);
 }

-struct RangeWithSize {
-  Range range;
-  uint64_t size;
-
-  RangeWithSize(const Slice& a, const Slice& b, uint64_t s = 0)
-      : range(a, b), size(s) {}
-};
-
 void CompactionJob::GenSubcompactionBoundaries() {
  // The goal is to find some boundary keys so that we can evenly partition
  // the compaction input data into max_subcompactions ranges.
@@ -592,7 +586,9 @@ void CompactionJob::GenSubcompactionBoundaries() {

  TEST_SYNC_POINT_CALLBACK("CompactionJob::GenSubcompactionBoundaries:0",
                           &num_planned_subcompactions);
-  if (num_planned_subcompactions == 1) return;
+  if (num_planned_subcompactions == 1) {
+    return;
+  }

  // Group the ranges into subcompactions
  uint64_t target_range_size = std::max(
@@ -649,7 +645,7 @@ Status CompactionJob::Run() {

  // Always schedule the first subcompaction (whether or not there are also
  // others) in the current thread to be efficient with resources
-  ProcessKeyValueCompaction(&compact_->sub_compact_states[0]);
+  ProcessKeyValueCompaction(compact_->sub_compact_states.data());

  // Wait for all other threads (if there are any) to finish execution
  for (auto& thread : thread_pool) {
@@ -760,7 +756,6 @@ Status CompactionJob::Run() {

        if (s.ok() && paranoid_file_checks_) {
          OutputValidator validator(cfd->internal_comparator(),
-                                    /*_enable_order_check=*/true,
                                    /*_enable_hash=*/true);
          for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
            s = validator.Add(iter->key(), iter->value());
@@ -831,24 +826,27 @@ Status CompactionJob::Run() {
    // input keys. So the number of keys it processed is not suitable for
    // verification here.
    // TODO: support verification when trim_ts_ is non-empty.
-    if (!(ts_sz > 0 && !trim_ts_.empty()) &&
-        db_options_.compaction_verify_record_count) {
+    if (!(ts_sz > 0 && !trim_ts_.empty())) {
      assert(compaction_stats_.stats.num_input_records > 0);
      // TODO: verify the number of range deletion entries.
      uint64_t expected =
          compaction_stats_.stats.num_input_records - num_input_range_del;
      uint64_t actual = compaction_job_stats_->num_input_records;
      if (expected != actual) {
+        char scratch[2345];
+        compact_->compaction->Summary(scratch, sizeof(scratch));
        std::string msg =
-            "Total number of input records: " + std::to_string(expected) +
-            ", but processed " + std::to_string(actual) + " records.";
+            "Compaction number of input keys does not match "
+            "number of keys processed. Expected " +
+            std::to_string(expected) + " but processed " +
+            std::to_string(actual) + ". Compaction summary: " + scratch;
        ROCKS_LOG_WARN(
-            db_options_.info_log, "[%s] [JOB %d] Compaction %s",
+            db_options_.info_log, "[%s] [JOB %d] Compaction with status: %s",
            compact_->compaction->column_family_data()->GetName().c_str(),
            job_context_->job_id, msg.c_str());
-        status = Status::Corruption(
-            "Compaction number of input keys does not match number of keys "
-            "processed.");
+        if (db_options_.compaction_verify_record_count) {
+          status = Status::Corruption(msg);
+        }
      }
    }
  }
@@ -1285,10 +1283,9 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
          : nullptr);

  TEST_SYNC_POINT("CompactionJob::Run():Inprogress");
-  TEST_SYNC_POINT_CALLBACK(
-      "CompactionJob::Run():PausingManualCompaction:1",
-      reinterpret_cast<void*>(
-          const_cast<std::atomic<bool>*>(&manual_compaction_canceled_)));
+  TEST_SYNC_POINT_CALLBACK("CompactionJob::Run():PausingManualCompaction:1",
+                           static_cast<void*>(const_cast<std::atomic<bool>*>(
+                               &manual_compaction_canceled_)));

  const std::string* const full_history_ts_low =
      full_history_ts_low_.empty() ? nullptr : &full_history_ts_low_;
@@ -1336,8 +1333,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
  Status status;
  TEST_SYNC_POINT_CALLBACK(
      "CompactionJob::ProcessKeyValueCompaction()::Processing",
-      reinterpret_cast<void*>(
-          const_cast<Compaction*>(sub_compact->compaction)));
+      static_cast<void*>(const_cast<Compaction*>(sub_compact->compaction)));
  uint64_t last_cpu_micros = prev_cpu_micros;
  while (status.ok() && !cfd->IsDropped() && c_iter->Valid()) {
    // Invariant: c_iter.status() is guaranteed to be OK if c_iter->Valid()
@@ -1368,10 +1364,9 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
      break;
    }

-    TEST_SYNC_POINT_CALLBACK(
-        "CompactionJob::Run():PausingManualCompaction:2",
-        reinterpret_cast<void*>(
-            const_cast<std::atomic<bool>*>(&manual_compaction_canceled_)));
+    TEST_SYNC_POINT_CALLBACK("CompactionJob::Run():PausingManualCompaction:2",
+                             static_cast<void*>(const_cast<std::atomic<bool>*>(
+                                 &manual_compaction_canceled_)));
    c_iter->Next();
    if (c_iter->status().IsManualCompactionPaused()) {
      break;
@@ -1859,13 +1854,14 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
  // Pass temperature of the last level files to FileSystem.
  FileOptions fo_copy = file_options_;
  Temperature temperature = sub_compact->compaction->output_temperature();
-  // only set for the last level compaction and also it's not output to
-  // penultimate level (when preclude_last_level feature is enabled)
-  if (temperature == Temperature::kUnknown &&
+  Temperature last_level_temp =
+      sub_compact->compaction->mutable_cf_options()->last_level_temperature;
+  // Here last_level_temperature supersedes default_write_temperature, when
+  // enabled and applicable
+  if (last_level_temp != Temperature::kUnknown &&
      sub_compact->compaction->is_last_level() &&
      !sub_compact->IsCurrentPenultimateLevel()) {
-    temperature =
-        sub_compact->compaction->mutable_cf_options()->last_level_temperature;
+    temperature = last_level_temp;
  }
  fo_copy.temperature = temperature;

@@ -1945,8 +1941,6 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
    }

    outputs.AddOutput(std::move(meta), cfd->internal_comparator(),
-                      sub_compact->compaction->mutable_cf_options()
-                          ->check_flush_compaction_key_order,
                      paranoid_file_checks_);
  }

@@ -1969,7 +1963,7 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
  TableBuilderOptions tboptions(
      *cfd->ioptions(), *(sub_compact->compaction->mutable_cf_options()),
      read_options, write_options, cfd->internal_comparator(),
-      cfd->int_tbl_prop_collector_factories(),
+      cfd->internal_tbl_prop_coll_factories(),
      sub_compact->compaction->output_compression(),
      sub_compact->compaction->output_compression_opts(), cfd->GetID(),
      cfd->GetName(), sub_compact->compaction->output_level(),
@@ -41,7 +41,6 @@
 #include "rocksdb/env.h"
 #include "rocksdb/memtablerep.h"
 #include "rocksdb/transaction_log.h"
-#include "table/scoped_arena_iterator.h"
 #include "util/autovector.h"
 #include "util/stop_watch.h"
 #include "util/thread_local.h"
@@ -46,7 +46,6 @@
 #include "table/block_based/block_based_table_factory.h"
 #include "table/mock_table.h"
 #include "table/plain/plain_table_factory.h"
-#include "table/scoped_arena_iterator.h"
 #include "test_util/sync_point.h"
 #include "test_util/testharness.h"
 #include "test_util/testutil.h"
@@ -131,7 +130,7 @@ class CompactionJobStatsTest : public testing::Test,
    ColumnFamilyOptions cf_opts(options);
    size_t cfi = handles_.size();
    handles_.resize(cfi + cfs.size());
-    for (auto cf : cfs) {
+    for (const auto& cf : cfs) {
      ASSERT_OK(db_->CreateColumnFamily(cf_opts, cf, &handles_[cfi++]));
    }
  }
@@ -160,7 +159,7 @@ class CompactionJobStatsTest : public testing::Test,
    EXPECT_EQ(cfs.size(), options.size());
    std::vector<ColumnFamilyDescriptor> column_families;
    for (size_t i = 0; i < cfs.size(); ++i) {
-      column_families.push_back(ColumnFamilyDescriptor(cfs[i], options[i]));
+      column_families.emplace_back(cfs[i], options[i]);
    }
    DBOptions db_opts = DBOptions(options[0]);
    return DB::Open(db_opts, dbname_, column_families, &handles_, &db_);
@@ -302,13 +302,13 @@ class CompactionJobTestBase : public testing::Test {
            TableBuilderOptions(*cfd_->ioptions(), mutable_cf_options_,
                                read_options, write_options,
                                cfd_->internal_comparator(),
-                                cfd_->int_tbl_prop_collector_factories(),
+                                cfd_->internal_tbl_prop_coll_factories(),
                                CompressionType::kNoCompression,
                                CompressionOptions(), 0 /* column_family_id */,
                                kDefaultColumnFamilyName, -1 /* level */),
            file_writer.get()));
    // Build table.
-    for (auto kv : contents) {
+    for (const auto& kv : contents) {
      std::string key;
      std::string value;
      std::tie(key, value) = kv;
@@ -327,7 +327,7 @@ class CompactionJobTestBase : public testing::Test {
    SequenceNumber smallest_seqno = kMaxSequenceNumber;
    SequenceNumber largest_seqno = 0;
    uint64_t oldest_blob_file_number = kInvalidBlobFileNumber;
-    for (auto kv : contents) {
+    for (const auto& kv : contents) {
      ParsedInternalKey key;
      std::string skey;
      std::string value;
@@ -1473,7 +1473,7 @@ TEST_F(CompactionJobTest, OldestBlobFileNumber) {
 }

 TEST_F(CompactionJobTest, VerifyPenultimateLevelOutput) {
-  cf_options_.bottommost_temperature = Temperature::kCold;
+  cf_options_.last_level_temperature = Temperature::kCold;
  SyncPoint::GetInstance()->SetCallBack(
      "Compaction::SupportsPerKeyPlacement:Enabled", [&](void* arg) {
        auto supports_per_key_placement = static_cast<bool*>(arg);
@@ -1752,23 +1752,9 @@ TEST_F(CompactionJobTest, ResultSerialization) {
  }
 }

-class CompactionJobDynamicFileSizeTest
-    : public CompactionJobTestBase,
-      public ::testing::WithParamInterface<bool> {
- public:
-  CompactionJobDynamicFileSizeTest()
-      : CompactionJobTestBase(
-            test::PerThreadDBPath("compaction_job_dynamic_file_size_test"),
-            BytewiseComparator(), [](uint64_t /*ts*/) { return ""; },
-            /*test_io_priority=*/false, TableTypeForTest::kMockTable) {}
-};
-
-TEST_P(CompactionJobDynamicFileSizeTest, CutForMaxCompactionBytes) {
+TEST_F(CompactionJobTest, CutForMaxCompactionBytes) {
  // dynamic_file_size option should have no impact on cutting for max
  // compaction bytes.
-  bool enable_dyanmic_file_size = GetParam();
-  cf_options_.level_compaction_dynamic_file_size = enable_dyanmic_file_size;
-
  NewDB();
  mutable_cf_options_.target_file_size_base = 80;
  mutable_cf_options_.max_compaction_bytes = 21;
@@ -1842,10 +1828,7 @@ TEST_P(CompactionJobDynamicFileSizeTest, CutForMaxCompactionBytes) {
                {expected_file1, expected_file2});
 }

-TEST_P(CompactionJobDynamicFileSizeTest, CutToSkipGrandparentFile) {
-  bool enable_dyanmic_file_size = GetParam();
-  cf_options_.level_compaction_dynamic_file_size = enable_dyanmic_file_size;
-
+TEST_F(CompactionJobTest, CutToSkipGrandparentFile) {
  NewDB();
  // Make sure the grandparent level file size (10) qualifies skipping.
  // Currently, it has to be > 1/8 of target file size.
@@ -1880,28 +1863,15 @@ TEST_P(CompactionJobDynamicFileSizeTest, CutToSkipGrandparentFile) {
      mock::MakeMockFile({{KeyStr("x", 4U, kTypeValue), "val"},
                          {KeyStr("z", 6U, kTypeValue), "val3"}});

-  auto expected_file_disable_dynamic_file_size =
-      mock::MakeMockFile({{KeyStr("a", 5U, kTypeValue), "val2"},
-                          {KeyStr("c", 3U, kTypeValue), "val"},
-                          {KeyStr("x", 4U, kTypeValue), "val"},
-                          {KeyStr("z", 6U, kTypeValue), "val3"}});
-
  SetLastSequence(6U);
  const std::vector<int> input_levels = {0, 1};
  auto lvl0_files = cfd_->current()->storage_info()->LevelFiles(0);
  auto lvl1_files = cfd_->current()->storage_info()->LevelFiles(1);
-  if (enable_dyanmic_file_size) {
    RunCompaction({lvl0_files, lvl1_files}, input_levels,
                  {expected_file1, expected_file2});
-  } else {
-    RunCompaction({lvl0_files, lvl1_files}, input_levels,
-                  {expected_file_disable_dynamic_file_size});
-  }
 }

-TEST_P(CompactionJobDynamicFileSizeTest, CutToAlignGrandparentBoundary) {
-  bool enable_dyanmic_file_size = GetParam();
-  cf_options_.level_compaction_dynamic_file_size = enable_dyanmic_file_size;
+TEST_F(CompactionJobTest, CutToAlignGrandparentBoundary) {
  NewDB();

  // MockTable has 1 byte per entry by default and each file is 10 bytes.
@@ -1968,40 +1938,15 @@ TEST_P(CompactionJobDynamicFileSizeTest, CutToAlignGrandparentBoundary) {
  }
  expected_file2.emplace_back(KeyStr("s", 4U, kTypeValue), "val");

-  mock::KVVector expected_file_disable_dynamic_file_size1;
-  for (char i = 0; i < 10; i++) {
-    expected_file_disable_dynamic_file_size1.emplace_back(
-        KeyStr(std::string(1, ch + i), i + 10, kTypeValue),
-        "val" + std::to_string(i));
-  }
-
-  mock::KVVector expected_file_disable_dynamic_file_size2;
-  for (char i = 10; i < 12; i++) {
-    expected_file_disable_dynamic_file_size2.emplace_back(
-        KeyStr(std::string(1, ch + i), i + 10, kTypeValue),
-        "val" + std::to_string(i));
-  }
-
-  expected_file_disable_dynamic_file_size2.emplace_back(
-      KeyStr("s", 4U, kTypeValue), "val");
-
  SetLastSequence(22U);
  const std::vector<int> input_levels = {0, 1};
  auto lvl0_files = cfd_->current()->storage_info()->LevelFiles(0);
  auto lvl1_files = cfd_->current()->storage_info()->LevelFiles(1);
-  if (enable_dyanmic_file_size) {
    RunCompaction({lvl0_files, lvl1_files}, input_levels,
                  {expected_file1, expected_file2});
-  } else {
-    RunCompaction({lvl0_files, lvl1_files}, input_levels,
-                  {expected_file_disable_dynamic_file_size1,
-                   expected_file_disable_dynamic_file_size2});
-  }
 }

-TEST_P(CompactionJobDynamicFileSizeTest, CutToAlignGrandparentBoundarySameKey) {
-  bool enable_dyanmic_file_size = GetParam();
-  cf_options_.level_compaction_dynamic_file_size = enable_dyanmic_file_size;
+TEST_F(CompactionJobTest, CutToAlignGrandparentBoundarySameKey) {
  NewDB();

  // MockTable has 1 byte per entry by default and each file is 10 bytes.
@@ -2038,13 +1983,9 @@ TEST_P(CompactionJobDynamicFileSizeTest, CutToAlignGrandparentBoundarySameKey) {
  AddMockFile(file5, 2);

  mock::KVVector expected_file1;
-  mock::KVVector expected_file_disable_dynamic_file_size;
-
  for (int i = 0; i < 8; i++) {
    expected_file1.emplace_back(KeyStr("a", 100 - i, kTypeValue),
                                "val" + std::to_string(100 - i));
-    expected_file_disable_dynamic_file_size.emplace_back(
-        KeyStr("a", 100 - i, kTypeValue), "val" + std::to_string(100 - i));
  }

  // make sure `b` is cut in a separated file (so internally it's not using
@@ -2053,9 +1994,6 @@ TEST_P(CompactionJobDynamicFileSizeTest, CutToAlignGrandparentBoundarySameKey) {
  auto expected_file2 =
      mock::MakeMockFile({{KeyStr("b", 90U, kTypeValue), "valb"}});

-  expected_file_disable_dynamic_file_size.emplace_back(
-      KeyStr("b", 90U, kTypeValue), "valb");
-
  SetLastSequence(122U);
  const std::vector<int> input_levels = {0, 1};
  auto lvl0_files = cfd_->current()->storage_info()->LevelFiles(0);
@@ -2066,20 +2004,13 @@ TEST_P(CompactionJobDynamicFileSizeTest, CutToAlignGrandparentBoundarySameKey) {
  for (int i = 80; i <= 100; i++) {
    snapshots.emplace_back(i);
  }
-  if (enable_dyanmic_file_size) {
    RunCompaction({lvl0_files, lvl1_files}, input_levels,
                  {expected_file1, expected_file2}, snapshots);
-  } else {
-    RunCompaction({lvl0_files, lvl1_files}, input_levels,
-                  {expected_file_disable_dynamic_file_size}, snapshots);
-  }
 }

-TEST_P(CompactionJobDynamicFileSizeTest, CutForMaxCompactionBytesSameKey) {
+TEST_F(CompactionJobTest, CutForMaxCompactionBytesSameKey) {
  // dynamic_file_size option should have no impact on cutting for max
  // compaction bytes.
-  bool enable_dyanmic_file_size = GetParam();
-  cf_options_.level_compaction_dynamic_file_size = enable_dyanmic_file_size;

  NewDB();
  mutable_cf_options_.target_file_size_base = 80;
@@ -2136,9 +2067,6 @@ TEST_P(CompactionJobDynamicFileSizeTest, CutForMaxCompactionBytesSameKey) {
                {expected_file1, expected_file2, expected_file3}, snapshots);
 }

-INSTANTIATE_TEST_CASE_P(CompactionJobDynamicFileSizeTest,
-                        CompactionJobDynamicFileSizeTest, testing::Bool());
-
 class CompactionJobTimestampTest : public CompactionJobTestBase {
 public:
  CompactionJobTimestampTest()
@@ -320,7 +320,6 @@ bool CompactionOutputs::ShouldStopBefore(const CompactionIterator& c_iter) {
        being_grandparent_gap_ ? 2 : 3;
    if (compaction_->immutable_options()->compaction_style ==
            kCompactionStyleLevel &&
-        compaction_->immutable_options()->level_compaction_dynamic_file_size &&
        num_grandparent_boundaries_crossed >=
            num_skippable_boundaries_crossed &&
        grandparent_overlapped_bytes_ - previous_overlapped_bytes >
@@ -342,7 +341,6 @@ bool CompactionOutputs::ShouldStopBefore(const CompactionIterator& c_iter) {
    // improvement.
    if (compaction_->immutable_options()->compaction_style ==
            kCompactionStyleLevel &&
-        compaction_->immutable_options()->level_compaction_dynamic_file_size &&
        current_output_file_size_ >=
            ((compaction_->target_output_file_size() + 99) / 100) *
                (50 + std::min(grandparent_boundary_switched_num_ * 5,
@@ -30,11 +30,9 @@ class CompactionOutputs {
  // compaction output file
  struct Output {
    Output(FileMetaData&& _meta, const InternalKeyComparator& _icmp,
-           bool _enable_order_check, bool _enable_hash, bool _finished,
-           uint64_t precalculated_hash)
+           bool _enable_hash, bool _finished, uint64_t precalculated_hash)
        : meta(std::move(_meta)),
-          validator(_icmp, _enable_order_check, _enable_hash,
-                    precalculated_hash),
+          validator(_icmp, _enable_hash, precalculated_hash),
          finished(_finished) {}
    FileMetaData meta;
    OutputValidator validator;
@@ -49,10 +47,10 @@ class CompactionOutputs {

  // Add generated output to the list
  void AddOutput(FileMetaData&& meta, const InternalKeyComparator& icmp,
-                 bool enable_order_check, bool enable_hash,
-                 bool finished = false, uint64_t precalculated_hash = 0) {
-    outputs_.emplace_back(std::move(meta), icmp, enable_order_check,
-                          enable_hash, finished, precalculated_hash);
+                 bool enable_hash, bool finished = false,
+                 uint64_t precalculated_hash = 0) {
+    outputs_.emplace_back(std::move(meta), icmp, enable_hash, finished,
+                          precalculated_hash);
  }

  // Set new table builder for the current output
@@ -130,7 +130,7 @@ CompactionPicker::CompactionPicker(const ImmutableOptions& ioptions,
                                   const InternalKeyComparator* icmp)
    : ioptions_(ioptions), icmp_(icmp) {}

-CompactionPicker::~CompactionPicker() {}
+CompactionPicker::~CompactionPicker() = default;

 // Delete this compaction from the list of running compactions.
 void CompactionPicker::ReleaseCompactionFiles(Compaction* c, Status status) {
@@ -377,7 +377,8 @@ Compaction* CompactionPicker::CompactFiles(
      output_level, compact_options.output_file_size_limit,
      mutable_cf_options.max_compaction_bytes, output_path_id, compression_type,
      GetCompressionOptions(mutable_cf_options, vstorage, output_level),
-      Temperature::kUnknown, compact_options.max_subcompactions,
+      mutable_cf_options.default_write_temperature,
+      compact_options.max_subcompactions,
      /* grandparents */ {}, true);
  RegisterCompaction(c);
  return c;
@@ -456,10 +457,9 @@ bool CompactionPicker::IsRangeInCompaction(VersionStorageInfo* vstorage,
 // Returns false if files on parent level are currently in compaction, which
 // means that we can't compact them
 bool CompactionPicker::SetupOtherInputs(
-    const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-    VersionStorageInfo* vstorage, CompactionInputFiles* inputs,
-    CompactionInputFiles* output_level_inputs, int* parent_index,
-    int base_index, bool only_expand_towards_right) {
+    const std::string& cf_name, VersionStorageInfo* vstorage,
+    CompactionInputFiles* inputs, CompactionInputFiles* output_level_inputs,
+    int* parent_index, int base_index, bool only_expand_towards_right) {
  assert(!inputs->empty());
  assert(output_level_inputs->empty());
  const int input_level = inputs->level;
@@ -500,7 +500,6 @@ bool CompactionPicker::SetupOtherInputs(
  // user key, while excluding other entries for the same user key. This
  // can happen when one user key spans multiple files.
  if (!output_level_inputs->empty()) {
-    const uint64_t limit = mutable_cf_options.max_compaction_bytes;
    const uint64_t output_level_inputs_size =
        TotalFileSize(output_level_inputs->files);
    const uint64_t inputs_size = TotalFileSize(inputs->files);
@@ -527,8 +526,6 @@ bool CompactionPicker::SetupOtherInputs(
      try_overlapping_inputs = false;
    }
    if (try_overlapping_inputs && expanded_inputs.size() > inputs->size() &&
-        (mutable_cf_options.ignore_max_compaction_bytes_for_input ||
-         output_level_inputs_size + expanded_inputs_size < limit) &&
        !AreFilesInCompaction(expanded_inputs.files)) {
      InternalKey new_start, new_limit;
      GetRange(expanded_inputs, &new_start, &new_limit);
@@ -551,8 +548,6 @@ bool CompactionPicker::SetupOtherInputs(
                                             base_index, nullptr);
      expanded_inputs_size = TotalFileSize(expanded_inputs.files);
      if (expanded_inputs.size() > inputs->size() &&
-          (mutable_cf_options.ignore_max_compaction_bytes_for_input ||
-           output_level_inputs_size + expanded_inputs_size < limit) &&
          !AreFilesInCompaction(expanded_inputs.files)) {
        expand_inputs = true;
      }
@@ -670,7 +665,8 @@ Compaction* CompactionPicker::CompactRange(
        compact_range_options.target_path_id,
        GetCompressionType(vstorage, mutable_cf_options, output_level, 1),
        GetCompressionOptions(mutable_cf_options, vstorage, output_level),
-        Temperature::kUnknown, compact_range_options.max_subcompactions,
+        mutable_cf_options.default_write_temperature,
+        compact_range_options.max_subcompactions,
        /* grandparents */ {}, /* is manual */ true, trim_ts, /* score */ -1,
        /* deletion_compaction */ false, /* l0_files_might_overlap */ true,
        CompactionReason::kUnknown,
@@ -812,8 +808,8 @@ Compaction* CompactionPicker::CompactRange(
  output_level_inputs.level = output_level;
  if (input_level != output_level) {
    int parent_index = -1;
-    if (!SetupOtherInputs(cf_name, mutable_cf_options, vstorage, &inputs,
-                          &output_level_inputs, &parent_index, -1)) {
+    if (!SetupOtherInputs(cf_name, vstorage, &inputs, &output_level_inputs,
+                          &parent_index, -1)) {
      // manual compaction is now multi-threaded, so it can
      // happen that SetupOtherInputs fails
      // we handle it higher in RunManualCompaction
@@ -858,8 +854,9 @@ Compaction* CompactionPicker::CompactRange(
      GetCompressionType(vstorage, mutable_cf_options, output_level,
                         vstorage->base_level()),
      GetCompressionOptions(mutable_cf_options, vstorage, output_level),
-      Temperature::kUnknown, compact_range_options.max_subcompactions,
-      std::move(grandparents), /* is manual */ true, trim_ts, /* score */ -1,
+      mutable_cf_options.default_write_temperature,
+      compact_range_options.max_subcompactions, std::move(grandparents),
+      /* is manual */ true, trim_ts, /* score */ -1,
      /* deletion_compaction */ false, /* l0_files_might_overlap */ true,
      CompactionReason::kUnknown,
      compact_range_options.blob_garbage_collection_policy,
@@ -186,7 +186,6 @@ class CompactionPicker {
      int penultimate_level) const;

  bool SetupOtherInputs(const std::string& cf_name,
-                        const MutableCFOptions& mutable_cf_options,
                        VersionStorageInfo* vstorage,
                        CompactionInputFiles* inputs,
                        CompactionInputFiles* output_level_inputs,
@@ -277,8 +276,7 @@ class NullCompactionPicker : public CompactionPicker {
  }

  // Always returns false.
-  virtual bool NeedsCompaction(
-      const VersionStorageInfo* /*vstorage*/) const override {
+  bool NeedsCompaction(const VersionStorageInfo* /*vstorage*/) const override {
    return false;
  }
 };
@@ -116,7 +116,8 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction(
  Compaction* c = new Compaction(
      vstorage, ioptions_, mutable_cf_options, mutable_db_options,
      std::move(inputs), 0, 0, 0, 0, kNoCompression,
-      mutable_cf_options.compression_opts, Temperature::kUnknown,
+      mutable_cf_options.compression_opts,
+      mutable_cf_options.default_write_temperature,
      /* max_subcompactions */ 0, {}, /* is manual */ false,
      /* trim_ts */ "", vstorage->CompactionScore(0),
      /* is deletion compaction */ true, /* l0_files_might_overlap */ true,
@@ -185,7 +186,8 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction(
            {comp_inputs}, 0, 16 * 1024 * 1024 /* output file size limit */,
            0 /* max compaction bytes, not applicable */,
            0 /* output path ID */, mutable_cf_options.compression,
-            mutable_cf_options.compression_opts, Temperature::kUnknown,
+            mutable_cf_options.compression_opts,
+            mutable_cf_options.default_write_temperature,
            0 /* max_subcompactions */, {}, /* is manual */ false,
            /* trim_ts */ "", vstorage->CompactionScore(0),
            /* is deletion compaction */ false,
@@ -280,7 +282,8 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction(
      /* target_file_size */ 0,
      /* max_compaction_bytes */ 0,
      /* output_path_id */ 0, kNoCompression,
-      mutable_cf_options.compression_opts, Temperature::kUnknown,
+      mutable_cf_options.compression_opts,
+      mutable_cf_options.default_write_temperature,
      /* max_subcompactions */ 0, {}, /* is manual */ false,
      /* trim_ts */ "", vstorage->CompactionScore(0),
      /* is deletion compaction */ true,
@@ -414,6 +417,7 @@ Compaction* FIFOCompactionPicker::PickTemperatureChangeCompaction(
  if (inputs[0].files.empty()) {
    return nullptr;
  }
+  assert(compaction_target_temp != Temperature::kLastTemperature);

  Compaction* c = new Compaction(
      vstorage, ioptions_, mutable_cf_options, mutable_db_options,
@@ -18,26 +18,27 @@ class FIFOCompactionPicker : public CompactionPicker {
                       const InternalKeyComparator* icmp)
      : CompactionPicker(ioptions, icmp) {}

-  virtual Compaction* PickCompaction(const std::string& cf_name,
-                                     const MutableCFOptions& mutable_cf_options,
-                                     const MutableDBOptions& mutable_db_options,
-                                     VersionStorageInfo* version,
-                                     LogBuffer* log_buffer) override;
+  Compaction* PickCompaction(const std::string& cf_name,
+                             const MutableCFOptions& mutable_cf_options,
+                             const MutableDBOptions& mutable_db_options,
+                             VersionStorageInfo* version,
+                             LogBuffer* log_buffer) override;

-  virtual Compaction* CompactRange(
-      const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-      const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage,
-      int input_level, int output_level,
-      const CompactRangeOptions& compact_range_options,
-      const InternalKey* begin, const InternalKey* end,
-      InternalKey** compaction_end, bool* manual_conflict,
-      uint64_t max_file_num_to_ignore, const std::string& trim_ts) override;
+  Compaction* CompactRange(const std::string& cf_name,
+                           const MutableCFOptions& mutable_cf_options,
+                           const MutableDBOptions& mutable_db_options,
+                           VersionStorageInfo* vstorage, int input_level,
+                           int output_level,
+                           const CompactRangeOptions& compact_range_options,
+                           const InternalKey* begin, const InternalKey* end,
+                           InternalKey** compaction_end, bool* manual_conflict,
+                           uint64_t max_file_num_to_ignore,
+                           const std::string& trim_ts) override;

  // The maximum allowed output level.  Always returns 0.
-  virtual int MaxOutputLevel() const override { return 0; }
+  int MaxOutputLevel() const override { return 0; }

-  virtual bool NeedsCompaction(
-      const VersionStorageInfo* vstorage) const override;
+  bool NeedsCompaction(const VersionStorageInfo* vstorage) const override;

 private:
  Compaction* PickTTLCompaction(const std::string& cf_name,
@@ -113,6 +113,15 @@ class LevelCompactionBuilder {
  // otherwise, returns false.
  bool PickIntraL0Compaction();

+  // When total L0 size is small compared to Lbase, try to pick intra-L0
+  // compaction starting from the newest L0 file. This helps to prevent
+  // L0->Lbase compaction with large write-amp.
+  //
+  // Returns true iff an intra-L0 compaction is picked.
+  // `start_level_inputs_` and `output_level_` will be updated accordingly if
+  // a compaction is picked.
+  bool PickSizeBasedIntraL0Compaction();
+
  // Return true if TrivialMove is extended. `start_index` is the index of
  // the initial file picked, which should already be in `start_level_inputs_`.
  bool TryExtendNonL0TrivialMove(int start_index,
@@ -346,7 +355,9 @@ void LevelCompactionBuilder::SetupOtherFilesWithRoundRobinExpansion() {
  TEST_SYNC_POINT("LevelCompactionPicker::RoundRobin");

  // Only expand the inputs when we have selected a file in start_level_inputs_
-  if (start_level_inputs_.size() == 0) return;
+  if (start_level_inputs_.size() == 0) {
+    return;
+  }

  uint64_t start_lvl_bytes_no_compacting = 0;
  uint64_t curr_bytes_to_compact = 0;
@@ -456,9 +467,8 @@ bool LevelCompactionBuilder::SetupOtherInputsIfNeeded() {
    }
    if (!is_l0_trivial_move_ &&
        !compaction_picker_->SetupOtherInputs(
-            cf_name_, mutable_cf_options_, vstorage_, &start_level_inputs_,
-            &output_level_inputs_, &parent_index_, base_index_,
-            round_robin_expanding)) {
+            cf_name_, vstorage_, &start_level_inputs_, &output_level_inputs_,
+            &parent_index_, base_index_, round_robin_expanding)) {
      return false;
    }

@@ -542,7 +552,7 @@ Compaction* LevelCompactionBuilder::GetCompaction() {
      GetCompressionType(vstorage_, mutable_cf_options_, output_level_,
                         vstorage_->base_level()),
      GetCompressionOptions(mutable_cf_options_, vstorage_, output_level_),
-      Temperature::kUnknown,
+      mutable_cf_options_.default_write_temperature,
      /* max_subcompactions */ 0, std::move(grandparents_), is_manual_,
      /* trim_ts */ "", start_level_score_, false /* deletion_compaction */,
      l0_files_might_overlap, compaction_reason_);
@@ -776,6 +786,9 @@ bool LevelCompactionBuilder::PickFileToCompact() {
  // being compacted at level 0.
  if (start_level_ == 0 &&
      !compaction_picker_->level0_compactions_in_progress()->empty()) {
+    if (PickSizeBasedIntraL0Compaction()) {
+      return true;
+    }
    TEST_SYNC_POINT("LevelCompactionPicker::PickCompactionBySize:0");
    return false;
  }
@@ -788,6 +801,9 @@ bool LevelCompactionBuilder::PickFileToCompact() {
  if (TryPickL0TrivialMove()) {
    return true;
  }
+  if (start_level_ == 0 && PickSizeBasedIntraL0Compaction()) {
+    return true;
+  }

  const std::vector<FileMetaData*>& level_files =
      vstorage_->LevelFiles(start_level_);
@@ -892,6 +908,56 @@ bool LevelCompactionBuilder::PickIntraL0Compaction() {
                               mutable_cf_options_.max_compaction_bytes,
                               &start_level_inputs_);
 }
+
+bool LevelCompactionBuilder::PickSizeBasedIntraL0Compaction() {
+  assert(start_level_ == 0);
+  int base_level = vstorage_->base_level();
+  if (base_level <= 0) {
+    return false;
+  }
+  const std::vector<FileMetaData*>& l0_files =
+      vstorage_->LevelFiles(/*level=*/0);
+  size_t min_num_file =
+      std::max(2, mutable_cf_options_.level0_file_num_compaction_trigger);
+  if (l0_files.size() < min_num_file) {
+    return false;
+  }
+  uint64_t l0_size = 0;
+  for (const auto& file : l0_files) {
+    l0_size += file->fd.GetFileSize();
+  }
+  const uint64_t min_lbase_size =
+      l0_size * static_cast<uint64_t>(std::max(
+                    10.0, mutable_cf_options_.max_bytes_for_level_multiplier));
+  assert(min_lbase_size >= l0_size);
+  const std::vector<FileMetaData*>& lbase_files =
+      vstorage_->LevelFiles(/*level=*/base_level);
+  uint64_t lbase_size = 0;
+  for (const auto& file : lbase_files) {
+    lbase_size += file->fd.GetFileSize();
+    if (lbase_size > min_lbase_size) {
+      break;
+    }
+  }
+  if (lbase_size <= min_lbase_size) {
+    return false;
+  }
+
+  start_level_inputs_.clear();
+  start_level_inputs_.level = 0;
+  for (const auto& file : l0_files) {
+    if (file->being_compacted) {
+      break;
+    }
+    start_level_inputs_.files.push_back(file);
+  }
+  if (start_level_inputs_.files.size() < min_num_file) {
+    start_level_inputs_.clear();
+    return false;
+  }
+  output_level_ = 0;
+  return true /* picked an intra-L0 compaction */;
+}
 }  // namespace

 Compaction* LevelCompactionPicker::PickCompaction(
@@ -20,14 +20,13 @@ class LevelCompactionPicker : public CompactionPicker {
  LevelCompactionPicker(const ImmutableOptions& ioptions,
                        const InternalKeyComparator* icmp)
      : CompactionPicker(ioptions, icmp) {}
-  virtual Compaction* PickCompaction(const std::string& cf_name,
-                                     const MutableCFOptions& mutable_cf_options,
-                                     const MutableDBOptions& mutable_db_options,
-                                     VersionStorageInfo* vstorage,
-                                     LogBuffer* log_buffer) override;
+  Compaction* PickCompaction(const std::string& cf_name,
+                             const MutableCFOptions& mutable_cf_options,
+                             const MutableDBOptions& mutable_db_options,
+                             VersionStorageInfo* vstorage,
+                             LogBuffer* log_buffer) override;

-  virtual bool NeedsCompaction(
-      const VersionStorageInfo* vstorage) const override;
+  bool NeedsCompaction(const VersionStorageInfo* vstorage) const override;
 };

 }  // namespace ROCKSDB_NAMESPACE
@@ -77,7 +77,7 @@ class CompactionPickerTestBase : public testing::Test {
    ioptions_.level_compaction_dynamic_level_bytes = false;
  }

-  ~CompactionPickerTestBase() override {}
+  ~CompactionPickerTestBase() override = default;

  void NewVersionStorage(int num_levels, CompactionStyle style) {
    DeleteVersionStorage();
@@ -214,7 +214,7 @@ class CompactionPickerTest : public CompactionPickerTestBase {
  explicit CompactionPickerTest()
      : CompactionPickerTestBase(BytewiseComparator()) {}

-  ~CompactionPickerTest() override {}
+  ~CompactionPickerTest() override = default;
 };

 class CompactionPickerU64TsTest : public CompactionPickerTestBase {
@@ -222,7 +222,7 @@ class CompactionPickerU64TsTest : public CompactionPickerTestBase {
  explicit CompactionPickerU64TsTest()
      : CompactionPickerTestBase(test::BytewiseComparatorWithU64TsWrapper()) {}

-  ~CompactionPickerU64TsTest() override {}
+  ~CompactionPickerU64TsTest() override = default;
 };

 TEST_F(CompactionPickerTest, Empty) {
@@ -1443,15 +1443,12 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping4) {
  ioptions_.compaction_pri = kMinOverlappingRatio;
  mutable_cf_options_.max_bytes_for_level_base = 10000000;
  mutable_cf_options_.max_bytes_for_level_multiplier = 10;
-  mutable_cf_options_.ignore_max_compaction_bytes_for_input = false;

-  // file 7 and 8 over lap with the same file, but file 8 is smaller so
-  // it will be picked.
-  // Overlaps with file 26, 27. And the file is compensated so will be
-  // picked up.
+  // Overlaps with file 26, 27, ratio is
+  // (60000000U + 60000000U / 180000000U) = 0.67
  Add(2, 6U, "150", "167", 60000000U, 0, 100, 100, 180000000U);
-  Add(2, 7U, "168", "169", 60000000U);  // Overlaps with file 27
-  Add(2, 8U, "201", "300", 61000000U);  // Overlaps with file 28
+  Add(2, 7U, "178", "189", 60000000U);  // Overlaps with file 28
+  Add(2, 8U, "401", "500", 61000000U);  // Overlaps with file 29

  Add(3, 26U, "160", "165", 60000000U);
  // Boosted file size in output level is not considered.
@@ -1465,7 +1462,7 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping4) {
      &log_buffer_));
  ASSERT_TRUE(compaction.get() != nullptr);
  ASSERT_EQ(1U, compaction->num_input_files(0));
-  // Picking file 8 because overlapping ratio is the biggest.
+  // Picking file 6 because overlapping ratio is the biggest.
  ASSERT_EQ(6U, compaction->input(0, 0)->fd.GetNumber());
 }

@@ -2468,20 +2465,23 @@ TEST_F(CompactionPickerTest, IsBottommostLevelTest) {
  DeleteVersionStorage();
 }

-TEST_F(CompactionPickerTest, MaxCompactionBytesHit) {
+TEST_F(CompactionPickerTest, IgnoreCompactionLimitWhenAddFileFromInputLevel) {
  mutable_cf_options_.max_bytes_for_level_base = 1000000u;
  mutable_cf_options_.max_compaction_bytes = 800000u;
-  mutable_cf_options_.ignore_max_compaction_bytes_for_input = false;
  ioptions_.level_compaction_dynamic_level_bytes = false;
  NewVersionStorage(6, kCompactionStyleLevel);
  // A compaction should be triggered and pick file 2 and 5.
-  // It can expand because adding file 1 and 3, the compaction size will
-  // exceed mutable_cf_options_.max_bytes_for_level_base.
+  // It pulls in other compaction input file from the input level L1
+  // without pulling in more output level files.
+  // Files 1, 3, 4 will be included in the compaction.
+  // File 6 is excluded since it overlaps with file 7.
  Add(1, 1U, "100", "150", 300000U);
  Add(1, 2U, "151", "200", 300001U, 0, 0);
  Add(1, 3U, "201", "250", 300000U, 0, 0);
  Add(1, 4U, "251", "300", 300000U, 0, 0);
-  Add(2, 5U, "100", "256", 1U);
+  Add(1, 6U, "325", "400", 300000U, 0, 0);
+  Add(2, 5U, "100", "350", 1U);
+  Add(2, 7U, "375", "425", 1U);
  UpdateVersionStorageInfo();

  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
@@ -2489,37 +2489,12 @@ TEST_F(CompactionPickerTest, MaxCompactionBytesHit) {
      &log_buffer_));
  ASSERT_TRUE(compaction.get() != nullptr);
  ASSERT_EQ(2U, compaction->num_input_levels());
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(1U, compaction->num_input_files(1));
-  ASSERT_EQ(2U, compaction->input(0, 0)->fd.GetNumber());
-  ASSERT_EQ(5U, compaction->input(1, 0)->fd.GetNumber());
-}
-
-TEST_F(CompactionPickerTest, MaxCompactionBytesNotHit) {
-  mutable_cf_options_.max_bytes_for_level_base = 800000u;
-  mutable_cf_options_.max_compaction_bytes = 1000000u;
-  mutable_cf_options_.ignore_max_compaction_bytes_for_input = false;
-  ioptions_.level_compaction_dynamic_level_bytes = false;
-  NewVersionStorage(6, kCompactionStyleLevel);
-  // A compaction should be triggered and pick file 2 and 5.
-  // and it expands to file 1 and 3 too.
-  Add(1, 1U, "100", "150", 300000U);
-  Add(1, 2U, "151", "200", 300001U, 0, 0);
-  Add(1, 3U, "201", "250", 300000U, 0, 0);
-  Add(1, 4U, "251", "300", 300000U, 0, 0);
-  Add(2, 5U, "000", "251", 1U);
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(),
-      &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(2U, compaction->num_input_levels());
-  ASSERT_EQ(3U, compaction->num_input_files(0));
+  ASSERT_EQ(4U, compaction->num_input_files(0));
  ASSERT_EQ(1U, compaction->num_input_files(1));
  ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
  ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber());
  ASSERT_EQ(3U, compaction->input(0, 2)->fd.GetNumber());
+  ASSERT_EQ(4U, compaction->input(0, 3)->fd.GetNumber());
  ASSERT_EQ(5U, compaction->input(1, 0)->fd.GetNumber());
 }

@@ -4268,6 +4243,58 @@ TEST_F(CompactionPickerTest,
 INSTANTIATE_TEST_CASE_P(PerKeyPlacementCompactionPickerTest,
                        PerKeyPlacementCompactionPickerTest, ::testing::Bool());

+TEST_F(CompactionPickerTest, IntraL0WhenL0IsSmall) {
+  mutable_cf_options_.level0_file_num_compaction_trigger = 4;
+  mutable_cf_options_.max_bytes_for_level_multiplier = 10;
+  for (const uint64_t lbase_size_multiplier : {1, 10, 11, 40}) {
+    SCOPED_TRACE("lbase_size_multiplier=" +
+                 std::to_string(lbase_size_multiplier));
+    NewVersionStorage(6, kCompactionStyleLevel);
+    // When L0 size is <= Lbase size / max_bytes_for_level_multiplier,
+    // intra-L0 compaction is picked. Otherwise, L0->L1
+    // compaction is picked.
+    Add(/*level=*/0, /*file_number=*/1U, /*smallest=*/"100",
+        /*largest=*/"200", /*file_size=*/1000, /*path_id=*/0,
+        /*smallest_seq=*/10, /*largest_seq=*/11,
+        /*compensated_file_size=*/1000);
+    Add(/*level=*/0, /*file_number=*/2U, /*smallest=*/"100",
+        /*largest=*/"100", /*file_size=*/1000, /*path_id=*/0,
+        /*smallest_seq=*/20, /*largest_seq=*/21,
+        /*compensated_file_size=*/1000);
+    Add(/*level=*/0, /*file_number=*/3U, /*smallest=*/"100",
+        /*largest=*/"200", /*file_size=*/1000, /*path_id=*/0,
+        /*smallest_seq=*/30, /*largest_seq=*/31,
+        /*compensated_file_size=*/1000);
+    Add(/*level=*/0, /*file_number=*/4U, /*smallest=*/"100",
+        /*largest=*/"200", /*file_size=*/1000, /*path_id=*/0,
+        /*smallest_seq=*/40, /*largest_seq=*/41,
+        /*compensated_file_size=*/1000);
+    const uint64_t l0_size = 4000;
+    const uint64_t lbase_size = l0_size * lbase_size_multiplier;
+    Add(/*level=*/1, /*file_number=*/5U, /*smallest=*/"100",
+        /*largest=*/"200", /*file_size=*/lbase_size, /*path_id=*/0,
+        /*smallest_seq=*/0, /*largest_seq=*/0,
+        /*compensated_file_size=*/lbase_size);
+    UpdateVersionStorageInfo();
+
+    LevelCompactionPicker compaction_picker(ioptions_, &icmp_);
+    std::unique_ptr<Compaction> compaction(compaction_picker.PickCompaction(
+        cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(),
+        &log_buffer_));
+    ASSERT_TRUE(compaction.get() != nullptr);
+    ASSERT_EQ(CompactionReason::kLevelL0FilesNum,
+              compaction->compaction_reason());
+    ASSERT_EQ(4U, compaction->num_input_files(0));
+    if (lbase_size_multiplier >
+        mutable_cf_options_.max_bytes_for_level_multiplier) {
+      ASSERT_EQ(1U, compaction->num_input_levels());
+      ASSERT_EQ(0, compaction->output_level());
+    } else {
+      ASSERT_EQ(2U, compaction->num_input_levels());
+      ASSERT_EQ(1, compaction->output_level());
+    }
+  }
+}

 }  // namespace ROCKSDB_NAMESPACE

@@ -870,7 +870,7 @@ Compaction* UniversalCompactionBuilder::PickCompactionToReduceSortedRuns(
                                           output_level, 1, enable_compression),
                        GetCompressionOptions(mutable_cf_options_, vstorage_,
                                              output_level, enable_compression),
-                        Temperature::kUnknown,
+                        mutable_cf_options_.default_write_temperature,
                        /* max_subcompactions */ 0, grandparents,
                        /* is manual */ false, /* trim_ts */ "", score_,
                        false /* deletion_compaction */,
@@ -1139,8 +1139,7 @@ Compaction* UniversalCompactionBuilder::PickIncrementalForReduceSizeAmp(
  // from bottom_start_idx and bottom_end_idx, but for now, we use
  // SetupOtherInputs() for simplicity.
  int parent_index = -1;  // Create and use bottom_start_idx?
-  if (!picker_->SetupOtherInputs(cf_name_, mutable_cf_options_, vstorage_,
-                                 &second_last_level_inputs,
+  if (!picker_->SetupOtherInputs(cf_name_, vstorage_, &second_last_level_inputs,
                                 &bottom_level_inputs, &parent_index,
                                 /*base_index=*/-1)) {
    return nullptr;
@@ -1205,7 +1204,7 @@ Compaction* UniversalCompactionBuilder::PickIncrementalForReduceSizeAmp(
                         true /* enable_compression */),
      GetCompressionOptions(mutable_cf_options_, vstorage_, output_level,
                            true /* enable_compression */),
-      Temperature::kUnknown,
+      mutable_cf_options_.default_write_temperature,
      /* max_subcompactions */ 0, /* grandparents */ {}, /* is manual */ false,
      /* trim_ts */ "", score_, false /* deletion_compaction */,
      /* l0_files_might_overlap */ true,
@@ -1311,9 +1310,8 @@ Compaction* UniversalCompactionBuilder::PickDeleteTriggeredCompaction() {
      int parent_index = -1;

      output_level_inputs.level = output_level;
-      if (!picker_->SetupOtherInputs(cf_name_, mutable_cf_options_, vstorage_,
-                                     &start_level_inputs, &output_level_inputs,
-                                     &parent_index, -1)) {
+      if (!picker_->SetupOtherInputs(cf_name_, vstorage_, &start_level_inputs,
+                                     &output_level_inputs, &parent_index, -1)) {
        return nullptr;
      }
      inputs.push_back(start_level_inputs);
@@ -1349,7 +1347,7 @@ Compaction* UniversalCompactionBuilder::PickDeleteTriggeredCompaction() {
      /* max_grandparent_overlap_bytes */ GetMaxOverlappingBytes(), path_id,
      GetCompressionType(vstorage_, mutable_cf_options_, output_level, 1),
      GetCompressionOptions(mutable_cf_options_, vstorage_, output_level),
-      Temperature::kUnknown,
+      mutable_cf_options_.default_write_temperature,
      /* max_subcompactions */ 0, grandparents, /* is manual */ false,
      /* trim_ts */ "", score_, false /* deletion_compaction */,
      /* l0_files_might_overlap */ true,
@@ -1442,7 +1440,7 @@ Compaction* UniversalCompactionBuilder::PickCompactionWithSortedRunRange(
                         true /* enable_compression */),
      GetCompressionOptions(mutable_cf_options_, vstorage_, output_level,
                            true /* enable_compression */),
-      Temperature::kUnknown,
+      mutable_cf_options_.default_write_temperature,
      /* max_subcompactions */ 0, /* grandparents */ {}, /* is manual */ false,
      /* trim_ts */ "", score_, false /* deletion_compaction */,
      /* l0_files_might_overlap */ true, compaction_reason);
@@ -17,14 +17,13 @@ class UniversalCompactionPicker : public CompactionPicker {
  UniversalCompactionPicker(const ImmutableOptions& ioptions,
                            const InternalKeyComparator* icmp)
      : CompactionPicker(ioptions, icmp) {}
-  virtual Compaction* PickCompaction(const std::string& cf_name,
-                                     const MutableCFOptions& mutable_cf_options,
-                                     const MutableDBOptions& mutable_db_options,
-                                     VersionStorageInfo* vstorage,
-                                     LogBuffer* log_buffer) override;
-  virtual int MaxOutputLevel() const override { return NumberLevels() - 1; }
+  Compaction* PickCompaction(const std::string& cf_name,
+                             const MutableCFOptions& mutable_cf_options,
+                             const MutableDBOptions& mutable_db_options,
+                             VersionStorageInfo* vstorage,
+                             LogBuffer* log_buffer) override;
+  int MaxOutputLevel() const override { return NumberLevels() - 1; }

-  virtual bool NeedsCompaction(
-      const VersionStorageInfo* vstorage) const override;
+  bool NeedsCompaction(const VersionStorageInfo* vstorage) const override;
 };
 }  // namespace ROCKSDB_NAMESPACE
@@ -74,24 +74,24 @@ CompactionJob::ProcessKeyValueCompactionWithCompactionService(
      compaction_input.output_level, input_files_oss.str().c_str());
  CompactionServiceJobInfo info(dbname_, db_id_, db_session_id_,
                                GetCompactionId(sub_compact), thread_pri_);
-  CompactionServiceJobStatus compaction_status =
-      db_options_.compaction_service->StartV2(info, compaction_input_binary);
-  switch (compaction_status) {
+  CompactionServiceScheduleResponse response =
+      db_options_.compaction_service->Schedule(info, compaction_input_binary);
+  switch (response.status) {
    case CompactionServiceJobStatus::kSuccess:
      break;
    case CompactionServiceJobStatus::kFailure:
      sub_compact->status = Status::Incomplete(
-          "CompactionService failed to start compaction job.");
+          "CompactionService failed to schedule a remote compaction job.");
      ROCKS_LOG_WARN(db_options_.info_log,
                     "[%s] [JOB %d] Remote compaction failed to start.",
                     compaction_input.column_family.name.c_str(), job_id_);
-      return compaction_status;
+      return response.status;
    case CompactionServiceJobStatus::kUseLocal:
      ROCKS_LOG_INFO(
          db_options_.info_log,
-          "[%s] [JOB %d] Remote compaction fallback to local by API Start.",
+          "[%s] [JOB %d] Remote compaction fallback to local by API (Schedule)",
          compaction_input.column_family.name.c_str(), job_id_);
-      return compaction_status;
+      return response.status;
    default:
      assert(false);  // unknown status
      break;
@@ -101,14 +101,15 @@ CompactionJob::ProcessKeyValueCompactionWithCompactionService(
                 "[%s] [JOB %d] Waiting for remote compaction...",
                 compaction_input.column_family.name.c_str(), job_id_);
  std::string compaction_result_binary;
-  compaction_status = db_options_.compaction_service->WaitForCompleteV2(
-      info, &compaction_result_binary);
+  CompactionServiceJobStatus compaction_status =
+      db_options_.compaction_service->Wait(response.scheduled_job_id,
+                                           &compaction_result_binary);

  if (compaction_status == CompactionServiceJobStatus::kUseLocal) {
-    ROCKS_LOG_INFO(db_options_.info_log,
-                   "[%s] [JOB %d] Remote compaction fallback to local by API "
-                   "WaitForComplete.",
-                   compaction_input.column_family.name.c_str(), job_id_);
+    ROCKS_LOG_INFO(
+        db_options_.info_log,
+        "[%s] [JOB %d] Remote compaction fallback to local by API (Wait)",
+        compaction_input.column_family.name.c_str(), job_id_);
    return compaction_status;
  }

@@ -195,8 +196,8 @@ CompactionJob::ProcessKeyValueCompactionWithCompactionService(

    auto cfd = compaction->column_family_data();
    sub_compact->Current().AddOutput(std::move(meta),
-                                     cfd->internal_comparator(), false, false,
-                                     true, file.paranoid_hash);
+                                     cfd->internal_comparator(), false, true,
+                                     file.paranoid_hash);
  }
  sub_compact->compaction_job_stats = compaction_result.stats;
  sub_compact->Current().SetNumOutputRecords(
@@ -830,4 +831,3 @@ bool CompactionServiceInput::TEST_Equals(CompactionServiceInput* other,
 }
 #endif  // NDEBUG
 }  // namespace ROCKSDB_NAMESPACE
-
@@ -31,40 +31,44 @@ class MyTestCompactionService : public CompactionService {

  const char* Name() const override { return kClassName(); }

-  CompactionServiceJobStatus StartV2(
+  CompactionServiceScheduleResponse Schedule(
      const CompactionServiceJobInfo& info,
      const std::string& compaction_service_input) override {
    InstrumentedMutexLock l(&mutex_);
    start_info_ = info;
    assert(info.db_name == db_path_);
-    jobs_.emplace(info.job_id, compaction_service_input);
-    CompactionServiceJobStatus s = CompactionServiceJobStatus::kSuccess;
-    if (is_override_start_status_) {
-      return override_start_status_;
-    }
-    return s;
+    std::string unique_id = Env::Default()->GenerateUniqueId();
+    jobs_.emplace(unique_id, compaction_service_input);
+    infos_.emplace(unique_id, info);
+    CompactionServiceScheduleResponse response(
+        unique_id, is_override_start_status_
+                       ? override_start_status_
+                       : CompactionServiceJobStatus::kSuccess);
+    return response;
  }

-  CompactionServiceJobStatus WaitForCompleteV2(
-      const CompactionServiceJobInfo& info,
-      std::string* compaction_service_result) override {
+  CompactionServiceJobStatus Wait(const std::string& scheduled_job_id,
+                                  std::string* result) override {
    std::string compaction_input;
-    assert(info.db_name == db_path_);
    {
      InstrumentedMutexLock l(&mutex_);
-      wait_info_ = info;
-      auto i = jobs_.find(info.job_id);
-      if (i == jobs_.end()) {
+      auto job_index = jobs_.find(scheduled_job_id);
+      if (job_index == jobs_.end()) {
        return CompactionServiceJobStatus::kFailure;
      }
-      compaction_input = std::move(i->second);
-      jobs_.erase(i);
-    }
+      compaction_input = std::move(job_index->second);
+      jobs_.erase(job_index);

+      auto info_index = infos_.find(scheduled_job_id);
+      if (info_index == infos_.end()) {
+        return CompactionServiceJobStatus::kFailure;
+      }
+      wait_info_ = std::move(info_index->second);
+      infos_.erase(info_index);
+    }
    if (is_override_wait_status_) {
      return override_wait_status_;
    }
-
    CompactionServiceOptionsOverride options_override;
    options_override.env = options_.env;
    options_override.file_checksum_gen_factory =
@@ -90,11 +94,11 @@ class MyTestCompactionService : public CompactionService {
    OpenAndCompactOptions options;
    options.canceled = &canceled_;

-    Status s = DB::OpenAndCompact(
-        options, db_path_, db_path_ + "/" + std::to_string(info.job_id),
-        compaction_input, compaction_service_result, options_override);
+    Status s =
+        DB::OpenAndCompact(options, db_path_, db_path_ + "/" + scheduled_job_id,
+                           compaction_input, result, options_override);
    if (is_override_wait_result_) {
-      *compaction_service_result = override_wait_result_;
+      *result = override_wait_result_;
    }
    compaction_num_.fetch_add(1);
    if (s.ok()) {
@@ -135,7 +139,8 @@ class MyTestCompactionService : public CompactionService {
 private:
  InstrumentedMutex mutex_;
  std::atomic_int compaction_num_{0};
-  std::map<uint64_t, std::string> jobs_;
+  std::map<std::string, std::string> jobs_;
+  std::map<std::string, CompactionServiceJobInfo> infos_;
  const std::string db_path_;
  Options options_;
  std::shared_ptr<Statistics> statistics_;
@@ -172,6 +177,7 @@ class CompactionServiceTest : public DBTestBase {
        remote_table_properties_collector_factories);
    options->compaction_service = compaction_service_;
    DestroyAndReopen(*options);
+    CreateAndReopenWithCF({"cf_1", "cf_2", "cf_3"}, *options);
  }

  Statistics* GetCompactorStatistics() { return compactor_statistics_.get(); }
@@ -183,36 +189,45 @@ class CompactionServiceTest : public DBTestBase {
    return static_cast_with_check<MyTestCompactionService>(cs);
  }

-  void GenerateTestData() {
-    // Generate 20 files @ L2
-    for (int i = 0; i < 20; i++) {
-      for (int j = 0; j < 10; j++) {
-        int key_id = i * 10 + j;
-        ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id)));
+  void GenerateTestData(bool move_files_manually = false) {
+    // Generate 20 files @ L2 Per CF
+    for (int cf_id = 0; cf_id < static_cast<int>(handles_.size()); cf_id++) {
+      for (int i = 0; i < 20; i++) {
+        for (int j = 0; j < 10; j++) {
+          int key_id = i * 10 + j;
+          ASSERT_OK(Put(cf_id, Key(key_id), "value" + std::to_string(key_id)));
+        }
+        ASSERT_OK(Flush(cf_id));
+      }
+      if (move_files_manually) {
+        MoveFilesToLevel(2, cf_id);
      }
-      ASSERT_OK(Flush());
-    }
-    MoveFilesToLevel(2);

-    // Generate 10 files @ L1 overlap with all 20 files @ L2
-    for (int i = 0; i < 10; i++) {
-      for (int j = 0; j < 10; j++) {
-        int key_id = i * 20 + j * 2;
-        ASSERT_OK(Put(Key(key_id), "value_new" + std::to_string(key_id)));
+      // Generate 10 files @ L1 overlap with all 20 files @ L2
+      for (int i = 0; i < 10; i++) {
+        for (int j = 0; j < 10; j++) {
+          int key_id = i * 20 + j * 2;
+          ASSERT_OK(
+              Put(cf_id, Key(key_id), "value_new" + std::to_string(key_id)));
+        }
+        ASSERT_OK(Flush(cf_id));
+      }
+      if (move_files_manually) {
+        MoveFilesToLevel(1, cf_id);
+        ASSERT_EQ(FilesPerLevel(cf_id), "0,10,20");
      }
-      ASSERT_OK(Flush());
    }
-    MoveFilesToLevel(1);
-    ASSERT_EQ(FilesPerLevel(), "0,10,20");
  }

  void VerifyTestData() {
-    for (int i = 0; i < 200; i++) {
-      auto result = Get(Key(i));
-      if (i % 2) {
-        ASSERT_EQ(result, "value" + std::to_string(i));
-      } else {
-        ASSERT_EQ(result, "value_new" + std::to_string(i));
+    for (int cf_id = 0; cf_id < static_cast<int>(handles_.size()); cf_id++) {
+      for (int i = 0; i < 200; i++) {
+        auto result = Get(cf_id, Key(i));
+        if (i % 2) {
+          ASSERT_EQ(result, "value" + std::to_string(i));
+        } else {
+          ASSERT_EQ(result, "value_new" + std::to_string(i));
+        }
      }
    }
  }
@@ -234,32 +249,10 @@ TEST_F(CompactionServiceTest, BasicCompactions) {
  Statistics* primary_statistics = GetPrimaryStatistics();
  Statistics* compactor_statistics = GetCompactorStatistics();

-  for (int i = 0; i < 20; i++) {
-    for (int j = 0; j < 10; j++) {
-      int key_id = i * 10 + j;
-      ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id)));
-    }
-    ASSERT_OK(Flush());
-  }
-
-  for (int i = 0; i < 10; i++) {
-    for (int j = 0; j < 10; j++) {
-      int key_id = i * 20 + j * 2;
-      ASSERT_OK(Put(Key(key_id), "value_new" + std::to_string(key_id)));
-    }
-    ASSERT_OK(Flush());
-  }
+  GenerateTestData();
  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+  VerifyTestData();

-  // verify result
-  for (int i = 0; i < 200; i++) {
-    auto result = Get(Key(i));
-    if (i % 2) {
-      ASSERT_EQ(result, "value" + std::to_string(i));
-    } else {
-      ASSERT_EQ(result, "value_new" + std::to_string(i));
-    }
-  }
  auto my_cs = GetCompactionService();
  ASSERT_GE(my_cs->GetCompactionNum(), 1);

@@ -322,7 +315,8 @@ TEST_F(CompactionServiceTest, BasicCompactions) {
        assert(*id != kNullUniqueId64x2);
        verify_passed++;
      });
-  Reopen(options);
+  ReopenWithColumnFamilies({kDefaultColumnFamilyName, "cf_1", "cf_2", "cf_3"},
+                           options);
  ASSERT_GT(verify_passed, 0);
  Close();
 }
@@ -490,26 +484,9 @@ TEST_F(CompactionServiceTest, CompactionFilter) {
      new PartialDeleteCompactionFilter());
  options.compaction_filter = delete_comp_filter.get();
  ReopenWithCompactionService(&options);
-
-  for (int i = 0; i < 20; i++) {
-    for (int j = 0; j < 10; j++) {
-      int key_id = i * 10 + j;
-      ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id)));
-    }
-    ASSERT_OK(Flush());
-  }
-
-  for (int i = 0; i < 10; i++) {
-    for (int j = 0; j < 10; j++) {
-      int key_id = i * 20 + j * 2;
-      ASSERT_OK(Put(Key(key_id), "value_new" + std::to_string(key_id)));
-    }
-    ASSERT_OK(Flush());
-  }
+  GenerateTestData();
  ASSERT_OK(dbfull()->TEST_WaitForCompact());
-
  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
-
  // verify result
  for (int i = 0; i < 200; i++) {
    auto result = Get(Key(i));
@@ -551,17 +528,17 @@ TEST_F(CompactionServiceTest, ConcurrentCompaction) {
  options.level0_file_num_compaction_trigger = 100;
  options.max_background_jobs = 20;
  ReopenWithCompactionService(&options);
-  GenerateTestData();
+  GenerateTestData(true);

  ColumnFamilyMetaData meta;
  db_->GetColumnFamilyMetaData(&meta);

  std::vector<std::thread> threads;
  for (const auto& file : meta.levels[1].files) {
-    threads.emplace_back(std::thread([&]() {
+    threads.emplace_back([&]() {
      std::string fname = file.db_path + "/" + file.name;
      ASSERT_OK(db_->CompactFiles(CompactionOptions(), {fname}, 2));
-    }));
+    });
  }

  for (auto& thread : threads) {
@@ -570,14 +547,7 @@ TEST_F(CompactionServiceTest, ConcurrentCompaction) {
  ASSERT_OK(dbfull()->TEST_WaitForCompact());

  // verify result
-  for (int i = 0; i < 200; i++) {
-    auto result = Get(Key(i));
-    if (i % 2) {
-      ASSERT_EQ(result, "value" + std::to_string(i));
-    } else {
-      ASSERT_EQ(result, "value_new" + std::to_string(i));
-    }
-  }
+  VerifyTestData();
  auto my_cs = GetCompactionService();
  ASSERT_EQ(my_cs->GetCompactionNum(), 10);
  ASSERT_EQ(FilesPerLevel(), "0,0,10");
@@ -587,21 +557,7 @@ TEST_F(CompactionServiceTest, CompactionInfo) {
  Options options = CurrentOptions();
  ReopenWithCompactionService(&options);

-  for (int i = 0; i < 20; i++) {
-    for (int j = 0; j < 10; j++) {
-      int key_id = i * 10 + j;
-      ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id)));
-    }
-    ASSERT_OK(Flush());
-  }
-
-  for (int i = 0; i < 10; i++) {
-    for (int j = 0; j < 10; j++) {
-      int key_id = i * 20 + j * 2;
-      ASSERT_OK(Put(Key(key_id), "value_new" + std::to_string(key_id)));
-    }
-    ASSERT_OK(Flush());
-  }
+  GenerateTestData();
  ASSERT_OK(dbfull()->TEST_WaitForCompact());
  auto my_cs =
      static_cast_with_check<MyTestCompactionService>(GetCompactionService());
@@ -676,32 +632,9 @@ TEST_F(CompactionServiceTest, FallbackLocalAuto) {

  my_cs->OverrideStartStatus(CompactionServiceJobStatus::kUseLocal);

-  for (int i = 0; i < 20; i++) {
-    for (int j = 0; j < 10; j++) {
-      int key_id = i * 10 + j;
-      ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id)));
-    }
-    ASSERT_OK(Flush());
-  }
-
-  for (int i = 0; i < 10; i++) {
-    for (int j = 0; j < 10; j++) {
-      int key_id = i * 20 + j * 2;
-      ASSERT_OK(Put(Key(key_id), "value_new" + std::to_string(key_id)));
-    }
-    ASSERT_OK(Flush());
-  }
+  GenerateTestData();
  ASSERT_OK(dbfull()->TEST_WaitForCompact());
-
-  // verify result
-  for (int i = 0; i < 200; i++) {
-    auto result = Get(Key(i));
-    if (i % 2) {
-      ASSERT_EQ(result, "value" + std::to_string(i));
-    } else {
-      ASSERT_EQ(result, "value_new" + std::to_string(i));
-    }
-  }
+  VerifyTestData();

  ASSERT_EQ(my_cs->GetCompactionNum(), 0);

@@ -14,12 +14,11 @@
 #include "rocksdb/listener.h"
 #include "rocksdb/utilities/debug.h"
 #include "test_util/mock_time_env.h"
+#include "utilities/merge_operators.h"

 namespace ROCKSDB_NAMESPACE {

-
-class TieredCompactionTest : public DBTestBase,
-                             public testing::WithParamInterface<bool> {
+class TieredCompactionTest : public DBTestBase {
 public:
  TieredCompactionTest()
      : DBTestBase("tiered_compaction_test", /*env_do_fsync=*/true),
@@ -123,14 +122,8 @@ class TieredCompactionTest : public DBTestBase,
    pl_stats.Clear();
  }

-  // bottommost_temperature is renaming to last_level_temperature, set either
-  // of them should have the same effect.
  void SetColdTemperature(Options& options) {
-    if (GetParam()) {
-      options.bottommost_temperature = Temperature::kCold;
-    } else {
-      options.last_level_temperature = Temperature::kCold;
-    }
+    options.last_level_temperature = Temperature::kCold;
  }

 private:
@@ -172,7 +165,7 @@ class TieredCompactionTest : public DBTestBase,
  }
 };

-TEST_P(TieredCompactionTest, SequenceBasedTieredStorageUniversal) {
+TEST_F(TieredCompactionTest, SequenceBasedTieredStorageUniversal) {
  const int kNumTrigger = 4;
  const int kNumLevels = 7;
  const int kNumKeys = 100;
@@ -334,7 +327,7 @@ TEST_P(TieredCompactionTest, SequenceBasedTieredStorageUniversal) {
  ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
 }

-TEST_P(TieredCompactionTest, RangeBasedTieredStorageUniversal) {
+TEST_F(TieredCompactionTest, RangeBasedTieredStorageUniversal) {
  const int kNumTrigger = 4;
  const int kNumLevels = 7;
  const int kNumKeys = 100;
@@ -508,7 +501,7 @@ TEST_P(TieredCompactionTest, RangeBasedTieredStorageUniversal) {
      1);
 }

-TEST_P(TieredCompactionTest, LevelColdRangeDelete) {
+TEST_F(TieredCompactionTest, LevelColdRangeDelete) {
  const int kNumTrigger = 4;
  const int kNumLevels = 7;
  const int kNumKeys = 100;
@@ -614,7 +607,7 @@ class SingleKeySstPartitionerFactory : public SstPartitionerFactory {
  }
 };

-TEST_P(TieredCompactionTest, LevelOutofBoundaryRangeDelete) {
+TEST_F(TieredCompactionTest, LevelOutofBoundaryRangeDelete) {
  const int kNumTrigger = 4;
  const int kNumLevels = 3;
  const int kNumKeys = 10;
@@ -743,7 +736,7 @@ TEST_P(TieredCompactionTest, LevelOutofBoundaryRangeDelete) {
  ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
 }

-TEST_P(TieredCompactionTest, UniversalRangeDelete) {
+TEST_F(TieredCompactionTest, UniversalRangeDelete) {
  const int kNumTrigger = 4;
  const int kNumLevels = 7;
  const int kNumKeys = 10;
@@ -875,7 +868,7 @@ TEST_P(TieredCompactionTest, UniversalRangeDelete) {
  ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
 }

-TEST_P(TieredCompactionTest, SequenceBasedTieredStorageLevel) {
+TEST_F(TieredCompactionTest, SequenceBasedTieredStorageLevel) {
  const int kNumTrigger = 4;
  const int kNumLevels = 7;
  const int kNumKeys = 100;
@@ -1099,7 +1092,7 @@ TEST_P(TieredCompactionTest, SequenceBasedTieredStorageLevel) {
  ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
 }

-TEST_P(TieredCompactionTest, RangeBasedTieredStorageLevel) {
+TEST_F(TieredCompactionTest, RangeBasedTieredStorageLevel) {
  const int kNumTrigger = 4;
  const int kNumLevels = 7;
  const int kNumKeys = 100;
@@ -1240,10 +1233,7 @@ TEST_P(TieredCompactionTest, RangeBasedTieredStorageLevel) {
  db_->ReleaseSnapshot(temp_snap);
 }

-INSTANTIATE_TEST_CASE_P(TieredCompactionTest, TieredCompactionTest,
-                        testing::Bool());
-
-TEST_P(TieredCompactionTest, CheckInternalKeyRange) {
+TEST_F(TieredCompactionTest, CheckInternalKeyRange) {
  // When compacting keys from the last level to penultimate level,
  // output to penultimate level should be within internal key range
  // of input files from penultimate level.
@@ -1318,8 +1308,8 @@ TEST_P(TieredCompactionTest, CheckInternalKeyRange) {

 class PrecludeLastLevelTest : public DBTestBase {
 public:
-  PrecludeLastLevelTest()
-      : DBTestBase("preclude_last_level_test", /*env_do_fsync=*/false) {
+  PrecludeLastLevelTest(std::string test_name = "preclude_last_level_test")
+      : DBTestBase(test_name, /*env_do_fsync=*/false) {
    mock_clock_ = std::make_shared<MockSystemClock>(env_->GetSystemClock());
    mock_clock_->SetCurrentTime(kMockStartTime);
    mock_env_ = std::make_unique<CompositeEnvWrapper>(env_, mock_clock_);
@@ -1338,7 +1328,7 @@ class PrecludeLastLevelTest : public DBTestBase {
    SyncPoint::GetInstance()->SetCallBack(
        "DBImpl::StartPeriodicTaskScheduler:Init", [&](void* arg) {
          auto periodic_task_scheduler_ptr =
-              reinterpret_cast<PeriodicTaskScheduler*>(arg);
+              static_cast<PeriodicTaskScheduler*>(arg);
          periodic_task_scheduler_ptr->TEST_OverrideTimer(mock_clock_.get());
        });
    mock_clock_->SetCurrentTime(kMockStartTime);
@@ -1594,6 +1584,62 @@ TEST_F(PrecludeLastLevelTest, SmallPrecludeTime) {
  Close();
 }

+TEST_F(PrecludeLastLevelTest, FastTrackTimedPutToLastLevel) {
+  const int kNumTrigger = 4;
+  const int kNumLevels = 7;
+  const int kNumKeys = 100;
+
+  Options options = CurrentOptions();
+  options.compaction_style = kCompactionStyleUniversal;
+  options.preclude_last_level_data_seconds = 60;
+  options.preserve_internal_time_seconds = 0;
+  options.env = mock_env_.get();
+  options.level0_file_num_compaction_trigger = kNumTrigger;
+  options.num_levels = kNumLevels;
+  options.last_level_temperature = Temperature::kCold;
+  DestroyAndReopen(options);
+
+  Random rnd(301);
+
+  dbfull()->TEST_WaitForPeriodicTaskRun([&] {
+    mock_clock_->MockSleepForSeconds(static_cast<int>(rnd.Uniform(10) + 1));
+  });
+
+  for (int i = 0; i < kNumKeys / 2; i++) {
+    ASSERT_OK(Put(Key(i), rnd.RandomString(100)));
+    dbfull()->TEST_WaitForPeriodicTaskRun([&] {
+      mock_clock_->MockSleepForSeconds(static_cast<int>(rnd.Uniform(2)));
+    });
+  }
+  // Create one file with regular Put.
+  ASSERT_OK(Flush());
+
+  // Create one file with TimedPut.
+  // With above mock clock operations, write_unix_time 50 should be before
+  // current_time - preclude_last_level_seconds.
+  // These data are eligible to be put on the last level once written to db
+  // and compaction will fast track them to the last level.
+  for (int i = kNumKeys / 2; i < kNumKeys; i++) {
+    ASSERT_OK(TimedPut(0, Key(i), rnd.RandomString(100), 50));
+  }
+  ASSERT_OK(Flush());
+
+  // TimedPut file moved to the last level immediately.
+  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+  ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel());
+
+  // Wait more than preclude_last_level time, Put file eventually moved to the
+  // last level.
+  mock_clock_->MockSleepForSeconds(100);
+
+  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+  ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel());
+  ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0);
+  ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
+
+  Close();
+}
+
 TEST_F(PrecludeLastLevelTest, LastLevelOnlyCompactionPartial) {
  const int kNumTrigger = 4;
  const int kNumLevels = 7;
@@ -1823,7 +1869,6 @@ TEST_P(PrecludeLastLevelTestWithParms, PeriodicCompactionToPenultimateLevel) {
  options.env = mock_env_.get();
  options.level0_file_num_compaction_trigger = kNumTrigger;
  options.num_levels = kNumLevels;
-  options.ignore_max_compaction_bytes_for_input = false;
  options.periodic_compaction_seconds = 10000;
  DestroyAndReopen(options);

@@ -2212,6 +2257,309 @@ TEST_F(PrecludeLastLevelTest, RangeDelsCauseFileEndpointsToOverlap) {
  Close();
 }

+// Tests DBIter::GetProperty("rocksdb.iterator.write-time") return a data's
+// approximate write unix time.
+// Test Param:
+// 1) use tailing iterator or regular iterator (when it applies)
+class IteratorWriteTimeTest : public PrecludeLastLevelTest,
+                              public testing::WithParamInterface<bool> {
+ public:
+  IteratorWriteTimeTest() : PrecludeLastLevelTest("iterator_write_time_test") {}
+
+  uint64_t VerifyKeyAndGetWriteTime(Iterator* iter,
+                                    const std::string& expected_key) {
+    std::string prop;
+    uint64_t write_time = 0;
+    EXPECT_TRUE(iter->Valid());
+    EXPECT_EQ(expected_key, iter->key());
+    EXPECT_OK(iter->GetProperty("rocksdb.iterator.write-time", &prop));
+    Slice prop_slice = prop;
+    EXPECT_TRUE(GetFixed64(&prop_slice, &write_time));
+    return write_time;
+  }
+
+  void VerifyKeyAndWriteTime(Iterator* iter, const std::string& expected_key,
+                             uint64_t expected_write_time) {
+    std::string prop;
+    uint64_t write_time = 0;
+    EXPECT_TRUE(iter->Valid());
+    EXPECT_EQ(expected_key, iter->key());
+    EXPECT_OK(iter->GetProperty("rocksdb.iterator.write-time", &prop));
+    Slice prop_slice = prop;
+    EXPECT_TRUE(GetFixed64(&prop_slice, &write_time));
+    EXPECT_EQ(expected_write_time, write_time);
+  }
+};
+
+TEST_P(IteratorWriteTimeTest, ReadFromMemtables) {
+  const int kNumTrigger = 4;
+  const int kNumLevels = 7;
+  const int kNumKeys = 100;
+  const int kSecondsPerRecording = 101;
+  const int kKeyWithWriteTime = 25;
+  const uint64_t kUserSpecifiedWriteTime =
+      kMockStartTime + kSecondsPerRecording * 15;
+
+  Options options = CurrentOptions();
+  options.compaction_style = kCompactionStyleUniversal;
+  options.env = mock_env_.get();
+  options.level0_file_num_compaction_trigger = kNumTrigger;
+  options.preserve_internal_time_seconds = 10000;
+  options.num_levels = kNumLevels;
+  DestroyAndReopen(options);
+
+  Random rnd(301);
+  for (int i = 0; i < kNumKeys; i++) {
+    dbfull()->TEST_WaitForPeriodicTaskRun(
+        [&] { mock_clock_->MockSleepForSeconds(kSecondsPerRecording); });
+    if (i == kKeyWithWriteTime) {
+      ASSERT_OK(
+          TimedPut(Key(i), rnd.RandomString(100), kUserSpecifiedWriteTime));
+    } else {
+      ASSERT_OK(Put(Key(i), rnd.RandomString(100)));
+    }
+  }
+
+  ReadOptions ropts;
+  ropts.tailing = GetParam();
+  int i;
+
+  // Forward iteration
+  uint64_t start_time = 0;
+  {
+    std::unique_ptr<Iterator> iter(dbfull()->NewIterator(ropts));
+    for (iter->SeekToFirst(), i = 0; iter->Valid(); iter->Next(), i++) {
+      if (start_time == 0) {
+        start_time = VerifyKeyAndGetWriteTime(iter.get(), Key(i));
+      } else if (i == kKeyWithWriteTime) {
+        VerifyKeyAndWriteTime(iter.get(), Key(i), kUserSpecifiedWriteTime);
+      } else {
+        VerifyKeyAndWriteTime(iter.get(), Key(i),
+                              start_time + kSecondsPerRecording * (i + 1));
+      }
+    }
+    ASSERT_OK(iter->status());
+  }
+
+  // Backward iteration
+  {
+    ropts.tailing = false;
+    std::unique_ptr<Iterator> iter(dbfull()->NewIterator(ropts));
+    for (iter->SeekToLast(), i = kNumKeys - 1; iter->Valid();
+         iter->Prev(), i--) {
+      if (i == 0) {
+        VerifyKeyAndWriteTime(iter.get(), Key(i), start_time);
+      } else if (i == kKeyWithWriteTime) {
+        VerifyKeyAndWriteTime(iter.get(), Key(i), kUserSpecifiedWriteTime);
+      } else {
+        VerifyKeyAndWriteTime(iter.get(), Key(i),
+                              start_time + kSecondsPerRecording * (i + 1));
+      }
+    }
+    ASSERT_OK(iter->status());
+  }
+
+  // Reopen the DB and disable the seqno to time recording, data with user
+  // specified write time can still get a write time before it's flushed.
+  options.preserve_internal_time_seconds = 0;
+  DestroyAndReopen(options);
+  ASSERT_OK(TimedPut(Key(kKeyWithWriteTime), rnd.RandomString(100),
+                     kUserSpecifiedWriteTime));
+  {
+    std::unique_ptr<Iterator> iter(dbfull()->NewIterator(ropts));
+    iter->Seek(Key(kKeyWithWriteTime));
+    VerifyKeyAndWriteTime(iter.get(), Key(kKeyWithWriteTime),
+                          kUserSpecifiedWriteTime);
+    ASSERT_OK(iter->status());
+  }
+
+  ASSERT_OK(Flush());
+  {
+    std::unique_ptr<Iterator> iter(dbfull()->NewIterator(ropts));
+    iter->Seek(Key(kKeyWithWriteTime));
+    VerifyKeyAndWriteTime(iter.get(), Key(kKeyWithWriteTime),
+                          std::numeric_limits<uint64_t>::max());
+    ASSERT_OK(iter->status());
+  }
+
+  Close();
+}
+
+TEST_P(IteratorWriteTimeTest, ReadFromSstFile) {
+  const int kNumTrigger = 4;
+  const int kNumLevels = 7;
+  const int kNumKeys = 100;
+  const int kSecondsPerRecording = 101;
+  const int kKeyWithWriteTime = 25;
+  const uint64_t kUserSpecifiedWriteTime =
+      kMockStartTime + kSecondsPerRecording * 15;
+
+  Options options = CurrentOptions();
+  options.compaction_style = kCompactionStyleUniversal;
+  options.env = mock_env_.get();
+  options.level0_file_num_compaction_trigger = kNumTrigger;
+  options.preserve_internal_time_seconds = 10000;
+  options.num_levels = kNumLevels;
+  DestroyAndReopen(options);
+
+  Random rnd(301);
+  for (int i = 0; i < kNumKeys; i++) {
+    dbfull()->TEST_WaitForPeriodicTaskRun(
+        [&] { mock_clock_->MockSleepForSeconds(kSecondsPerRecording); });
+    if (i == kKeyWithWriteTime) {
+      ASSERT_OK(
+          TimedPut(Key(i), rnd.RandomString(100), kUserSpecifiedWriteTime));
+    } else {
+      ASSERT_OK(Put(Key(i), rnd.RandomString(100)));
+    }
+  }
+
+  ASSERT_OK(Flush());
+  ReadOptions ropts;
+  ropts.tailing = GetParam();
+  std::string prop;
+  int i;
+
+  // Forward iteration
+  uint64_t start_time = 0;
+  {
+    std::unique_ptr<Iterator> iter(dbfull()->NewIterator(ropts));
+    for (iter->SeekToFirst(), i = 0; iter->Valid(); iter->Next(), i++) {
+      if (start_time == 0) {
+        start_time = VerifyKeyAndGetWriteTime(iter.get(), Key(i));
+      } else if (i == kKeyWithWriteTime) {
+        // It's not precisely kUserSpecifiedWriteTime, instead it has a margin
+        // of error that is one recording apart while we convert write time to
+        // sequence number, and then back to write time.
+        VerifyKeyAndWriteTime(iter.get(), Key(i),
+                              kUserSpecifiedWriteTime - kSecondsPerRecording);
+      } else {
+        VerifyKeyAndWriteTime(iter.get(), Key(i),
+                              start_time + kSecondsPerRecording * (i + 1));
+      }
+    }
+    ASSERT_OK(iter->status());
+  }
+
+  // Backward iteration
+  {
+    ropts.tailing = false;
+    std::unique_ptr<Iterator> iter(dbfull()->NewIterator(ropts));
+    for (iter->SeekToLast(), i = kNumKeys - 1; iter->Valid();
+         iter->Prev(), i--) {
+      if (i == 0) {
+        VerifyKeyAndWriteTime(iter.get(), Key(i), start_time);
+      } else if (i == kKeyWithWriteTime) {
+        VerifyKeyAndWriteTime(iter.get(), Key(i),
+                              kUserSpecifiedWriteTime - kSecondsPerRecording);
+      } else {
+        VerifyKeyAndWriteTime(iter.get(), Key(i),
+                              start_time + kSecondsPerRecording * (i + 1));
+      }
+    }
+    ASSERT_OK(iter->status());
+  }
+
+  // Reopen the DB and disable the seqno to time recording. Data retrieved from
+  // SST files still have write time available.
+  options.preserve_internal_time_seconds = 0;
+  DestroyAndReopen(options);
+
+  dbfull()->TEST_WaitForPeriodicTaskRun(
+      [&] { mock_clock_->MockSleepForSeconds(kSecondsPerRecording); });
+  ASSERT_OK(Put("a", "val"));
+  ASSERT_TRUE(dbfull()->TEST_GetSeqnoToTimeMapping().Empty());
+
+  {
+    std::unique_ptr<Iterator> iter(dbfull()->NewIterator(ropts));
+    iter->SeekToFirst();
+    ASSERT_TRUE(iter->Valid());
+    // "a" is retrieved from memtable, its write time is unknown because the
+    // seqno to time mapping recording is not available.
+    VerifyKeyAndWriteTime(iter.get(), "a",
+                          std::numeric_limits<uint64_t>::max());
+    for (iter->Next(), i = 0; iter->Valid(); iter->Next(), i++) {
+      if (i == 0) {
+        VerifyKeyAndWriteTime(iter.get(), Key(i), start_time);
+      } else if (i == kKeyWithWriteTime) {
+        VerifyKeyAndWriteTime(iter.get(), Key(i),
+                              kUserSpecifiedWriteTime - kSecondsPerRecording);
+      } else {
+        VerifyKeyAndWriteTime(iter.get(), Key(i),
+                              start_time + kSecondsPerRecording * (i + 1));
+      }
+    }
+    ASSERT_OK(iter->status());
+  }
+
+  // There is no write time info for "a" after it's flushed to SST file either.
+  ASSERT_OK(Flush());
+  {
+    std::unique_ptr<Iterator> iter(dbfull()->NewIterator(ropts));
+    iter->SeekToFirst();
+    ASSERT_TRUE(iter->Valid());
+    VerifyKeyAndWriteTime(iter.get(), "a",
+                          std::numeric_limits<uint64_t>::max());
+  }
+
+  // Sequence number zeroed out after compacted to the last level, write time
+  // all becomes zero.
+  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+  {
+    std::unique_ptr<Iterator> iter(dbfull()->NewIterator(ropts));
+    iter->SeekToFirst();
+    for (iter->Next(), i = 0; iter->Valid(); iter->Next(), i++) {
+      VerifyKeyAndWriteTime(iter.get(), Key(i), 0);
+    }
+    ASSERT_OK(iter->status());
+  }
+  Close();
+}
+
+TEST_P(IteratorWriteTimeTest, MergeReturnsBaseValueWriteTime) {
+  const int kNumTrigger = 4;
+  const int kNumLevels = 7;
+  const int kSecondsPerRecording = 101;
+
+  Options options = CurrentOptions();
+  options.compaction_style = kCompactionStyleUniversal;
+  options.env = mock_env_.get();
+  options.level0_file_num_compaction_trigger = kNumTrigger;
+  options.preserve_internal_time_seconds = 10000;
+  options.num_levels = kNumLevels;
+  options.merge_operator = MergeOperators::CreateStringAppendOperator();
+  DestroyAndReopen(options);
+
+  dbfull()->TEST_WaitForPeriodicTaskRun(
+      [&] { mock_clock_->MockSleepForSeconds(kSecondsPerRecording); });
+  ASSERT_OK(Put("foo", "fv1"));
+
+  dbfull()->TEST_WaitForPeriodicTaskRun(
+      [&] { mock_clock_->MockSleepForSeconds(kSecondsPerRecording); });
+  ASSERT_OK(Put("bar", "bv1"));
+  ASSERT_OK(Merge("foo", "bv1"));
+
+  ReadOptions ropts;
+  ropts.tailing = GetParam();
+  {
+    std::unique_ptr<Iterator> iter(dbfull()->NewIterator(ropts));
+    iter->SeekToFirst();
+    uint64_t bar_time = VerifyKeyAndGetWriteTime(iter.get(), "bar");
+    iter->Next();
+    uint64_t foo_time = VerifyKeyAndGetWriteTime(iter.get(), "foo");
+    // "foo" has an older write time because its base value's write time is used
+    ASSERT_GT(bar_time, foo_time);
+    iter->Next();
+    ASSERT_FALSE(iter->Valid());
+    ASSERT_OK(iter->status());
+  }
+
+  Close();
+}
+
+INSTANTIATE_TEST_CASE_P(IteratorWriteTimeTest, IteratorWriteTimeTest,
+                        testing::Bool());
+
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -170,7 +170,7 @@ void DoRandomIteraratorTest(DB* db, std::vector<std::string> source_strings,

 class DoubleComparator : public Comparator {
 public:
-  DoubleComparator() {}
+  DoubleComparator() = default;

  const char* Name() const override { return "DoubleComparator"; }

@@ -198,7 +198,7 @@ class DoubleComparator : public Comparator {

 class HashComparator : public Comparator {
 public:
-  HashComparator() {}
+  HashComparator() = default;

  const char* Name() const override { return "HashComparator"; }

@@ -221,7 +221,7 @@ class HashComparator : public Comparator {

 class TwoStrComparator : public Comparator {
 public:
-  TwoStrComparator() {}
+  TwoStrComparator() = default;

  const char* Name() const override { return "TwoStrComparator"; }

@@ -372,7 +372,7 @@ TEST_P(ComparatorDBTest, Uint64Comparator) {
      uint64_t r = rnd64.Next();
      std::string str;
      str.resize(8);
-      memcpy(&str[0], static_cast<void*>(&r), 8);
+      memcpy(str.data(), static_cast<void*>(&r), 8);
      source_strings.push_back(str);
    }

@@ -53,10 +53,9 @@ class ErrorFS : public FileSystemWrapper {
        num_writable_file_errors_(0) {}
  const char* Name() const override { return "ErrorEnv"; }

-  virtual IOStatus NewWritableFile(const std::string& fname,
-                                   const FileOptions& opts,
-                                   std::unique_ptr<FSWritableFile>* result,
-                                   IODebugContext* dbg) override {
+  IOStatus NewWritableFile(const std::string& fname, const FileOptions& opts,
+                           std::unique_ptr<FSWritableFile>* result,
+                           IODebugContext* dbg) override {
    result->reset();
    if (writable_file_error_) {
      ++num_writable_file_errors_;
@@ -823,7 +822,6 @@ TEST_F(CorruptionTest, ParanoidFileChecksOnFlush) {
  Options options;
  options.level_compaction_dynamic_level_bytes = false;
  options.env = env_.get();
-  options.check_flush_compaction_key_order = false;
  options.paranoid_file_checks = true;
  options.create_if_missing = true;
  Status s;
@@ -854,7 +852,6 @@ TEST_F(CorruptionTest, ParanoidFileChecksOnCompact) {
  options.env = env_.get();
  options.paranoid_file_checks = true;
  options.create_if_missing = true;
-  options.check_flush_compaction_key_order = false;
  Status s;
  for (const auto& mode : corruption_modes) {
    delete db_;
@@ -886,7 +883,6 @@ TEST_F(CorruptionTest, ParanoidFileChecksWithDeleteRangeFirst) {
  Options options;
  options.level_compaction_dynamic_level_bytes = false;
  options.env = env_.get();
-  options.check_flush_compaction_key_order = false;
  options.paranoid_file_checks = true;
  options.create_if_missing = true;
  for (bool do_flush : {true, false}) {
@@ -923,7 +919,6 @@ TEST_F(CorruptionTest, ParanoidFileChecksWithDeleteRange) {
  Options options;
  options.level_compaction_dynamic_level_bytes = false;
  options.env = env_.get();
-  options.check_flush_compaction_key_order = false;
  options.paranoid_file_checks = true;
  options.create_if_missing = true;
  for (bool do_flush : {true, false}) {
@@ -963,7 +958,6 @@ TEST_F(CorruptionTest, ParanoidFileChecksWithDeleteRangeLast) {
  Options options;
  options.level_compaction_dynamic_level_bytes = false;
  options.env = env_.get();
-  options.check_flush_compaction_key_order = false;
  options.paranoid_file_checks = true;
  options.create_if_missing = true;
  for (bool do_flush : {true, false}) {
@@ -1032,7 +1026,6 @@ TEST_F(CorruptionTest, CompactionKeyOrderCheck) {
  options.env = env_.get();
  options.paranoid_file_checks = false;
  options.create_if_missing = true;
-  options.check_flush_compaction_key_order = false;
  delete db_;
  db_ = nullptr;
  ASSERT_OK(DestroyDB(dbname_, options));
@@ -1047,7 +1040,6 @@ TEST_F(CorruptionTest, CompactionKeyOrderCheck) {
  ASSERT_OK(dbi->TEST_FlushMemTable());

  mock->SetCorruptionMode(mock::MockTableFactory::kCorruptNone);
-  ASSERT_OK(db_->SetOptions({{"check_flush_compaction_key_order", "true"}}));
  CompactRangeOptions cro;
  cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
  ASSERT_NOK(
@@ -1060,7 +1052,6 @@ TEST_F(CorruptionTest, FlushKeyOrderCheck) {
  options.env = env_.get();
  options.paranoid_file_checks = false;
  options.create_if_missing = true;
-  ASSERT_OK(db_->SetOptions({{"check_flush_compaction_key_order", "true"}}));

  ASSERT_OK(db_->Put(WriteOptions(), "foo1", "v1"));
  ASSERT_OK(db_->Put(WriteOptions(), "foo2", "v1"));
@@ -1085,28 +1076,6 @@ TEST_F(CorruptionTest, FlushKeyOrderCheck) {
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
 }

-TEST_F(CorruptionTest, DisableKeyOrderCheck) {
-  ASSERT_OK(db_->SetOptions({{"check_flush_compaction_key_order", "false"}}));
-  DBImpl* dbi = static_cast_with_check<DBImpl>(db_);
-
-  SyncPoint::GetInstance()->SetCallBack(
-      "OutputValidator::Add:order_check",
-      [&](void* /*arg*/) { ASSERT_TRUE(false); });
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
-  ASSERT_OK(db_->Put(WriteOptions(), "foo1", "v1"));
-  ASSERT_OK(db_->Put(WriteOptions(), "foo3", "v1"));
-  ASSERT_OK(dbi->TEST_FlushMemTable());
-  ASSERT_OK(db_->Put(WriteOptions(), "foo2", "v1"));
-  ASSERT_OK(db_->Put(WriteOptions(), "foo4", "v1"));
-  ASSERT_OK(dbi->TEST_FlushMemTable());
-  CompactRangeOptions cro;
-  cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
-  ASSERT_OK(
-      dbi->CompactRange(cro, dbi->DefaultColumnFamily(), nullptr, nullptr));
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
-}
-
 TEST_F(CorruptionTest, VerifyWholeTableChecksum) {
  CloseDb();
  Options options;
@@ -1133,7 +1102,7 @@ TEST_F(CorruptionTest, VerifyWholeTableChecksum) {
  int count{0};
  SyncPoint::GetInstance()->SetCallBack(
      "DBImpl::VerifyFullFileChecksum:mismatch", [&](void* arg) {
-        auto* s = reinterpret_cast<Status*>(arg);
+        auto* s = static_cast<Status*>(arg);
        ASSERT_NE(s, nullptr);
        ++count;
        ASSERT_NOK(*s);
@@ -1278,7 +1247,7 @@ TEST_P(CrashDuringRecoveryWithCorruptionTest, CrashDuringRecovery) {
    SyncPoint::GetInstance()->ClearAllCallBacks();
    SyncPoint::GetInstance()->SetCallBack(
        "DBImpl::GetLogSizeAndMaybeTruncate:0", [&](void* arg) {
-          auto* tmp_s = reinterpret_cast<Status*>(arg);
+          auto* tmp_s = static_cast<Status*>(arg);
          assert(tmp_s);
          *tmp_s = Status::IOError("Injected");
        });
@@ -1460,7 +1429,7 @@ TEST_P(CrashDuringRecoveryWithCorruptionTest, TxnDbCrashDuringRecovery) {
    SyncPoint::GetInstance()->ClearAllCallBacks();
    SyncPoint::GetInstance()->SetCallBack(
        "DBImpl::Open::BeforeSyncWAL", [&](void* arg) {
-          auto* tmp_s = reinterpret_cast<Status*>(arg);
+          auto* tmp_s = static_cast<Status*>(arg);
          assert(tmp_s);
          *tmp_s = Status::IOError("Injected");
        });
@@ -1628,7 +1597,7 @@ TEST_P(CrashDuringRecoveryWithCorruptionTest, CrashDuringRecoveryWithFlush) {
    SyncPoint::GetInstance()->ClearAllCallBacks();
    SyncPoint::GetInstance()->SetCallBack(
        "DBImpl::GetLogSizeAndMaybeTruncate:0", [&](void* arg) {
-          auto* tmp_s = reinterpret_cast<Status*>(arg);
+          auto* tmp_s = static_cast<Status*>(arg);
          assert(tmp_s);
          *tmp_s = Status::IOError("Injected");
        });
@@ -131,7 +131,7 @@ TEST_F(CuckooTableDBTest, Flush) {
  ASSERT_OK(dbfull()->TEST_FlushMemTable());

  TablePropertiesCollection ptc;
-  ASSERT_OK(reinterpret_cast<DB*>(dbfull())->GetPropertiesOfAllTables(&ptc));
+  ASSERT_OK(static_cast<DB*>(dbfull())->GetPropertiesOfAllTables(&ptc));
  VerifySstUniqueIds(ptc);
  ASSERT_EQ(1U, ptc.size());
  ASSERT_EQ(3U, ptc.begin()->second->num_entries);
@@ -148,7 +148,7 @@ TEST_F(CuckooTableDBTest, Flush) {
  ASSERT_OK(Put("key6", "v6"));
  ASSERT_OK(dbfull()->TEST_FlushMemTable());

-  ASSERT_OK(reinterpret_cast<DB*>(dbfull())->GetPropertiesOfAllTables(&ptc));
+  ASSERT_OK(static_cast<DB*>(dbfull())->GetPropertiesOfAllTables(&ptc));
  VerifySstUniqueIds(ptc);
  ASSERT_EQ(2U, ptc.size());
  auto row = ptc.begin();
@@ -166,7 +166,7 @@ TEST_F(CuckooTableDBTest, Flush) {
  ASSERT_OK(Delete("key5"));
  ASSERT_OK(Delete("key4"));
  ASSERT_OK(dbfull()->TEST_FlushMemTable());
-  ASSERT_OK(reinterpret_cast<DB*>(dbfull())->GetPropertiesOfAllTables(&ptc));
+  ASSERT_OK(static_cast<DB*>(dbfull())->GetPropertiesOfAllTables(&ptc));
  VerifySstUniqueIds(ptc);
  ASSERT_EQ(3U, ptc.size());
  row = ptc.begin();
@@ -191,7 +191,7 @@ TEST_F(CuckooTableDBTest, FlushWithDuplicateKeys) {
  ASSERT_OK(dbfull()->TEST_FlushMemTable());

  TablePropertiesCollection ptc;
-  ASSERT_OK(reinterpret_cast<DB*>(dbfull())->GetPropertiesOfAllTables(&ptc));
+  ASSERT_OK(static_cast<DB*>(dbfull())->GetPropertiesOfAllTables(&ptc));
  VerifySstUniqueIds(ptc);
  ASSERT_EQ(1U, ptc.size());
  ASSERT_EQ(2U, ptc.begin()->second->num_entries);
@@ -209,7 +209,7 @@ static std::string Key(int i) {
 static std::string Uint64Key(uint64_t i) {
  std::string str;
  str.resize(8);
-  memcpy(&str[0], static_cast<void*>(&i), 8);
+  memcpy(str.data(), static_cast<void*>(&i), 8);
  return str;
 }
 }  // namespace.
@@ -25,10 +25,60 @@
 #include "util/random.h"
 #include "utilities/counted_fs.h"
 #include "utilities/fault_injection_env.h"
+#include "utilities/fault_injection_fs.h"
 #include "utilities/merge_operators.h"
 #include "utilities/merge_operators/string_append/stringappend.h"

 namespace ROCKSDB_NAMESPACE {
+namespace {
+class MyFlushBlockPolicy : public FlushBlockPolicy {
+ public:
+  explicit MyFlushBlockPolicy(const int num_keys_in_block,
+                              const BlockBuilder& data_block_builder)
+      : num_keys_in_block_(num_keys_in_block),
+        num_keys_(0),
+        data_block_builder_(data_block_builder) {}
+
+  bool Update(const Slice& /*key*/, const Slice& /*value*/) override {
+    if (data_block_builder_.empty()) {
+      // First key in this block
+      num_keys_ = 1;
+      return false;
+    }
+    // Flush every 10 keys
+    if (num_keys_ == num_keys_in_block_) {
+      num_keys_ = 1;
+      return true;
+    }
+    num_keys_++;
+    return false;
+  }
+
+ private:
+  const int num_keys_in_block_;
+  int num_keys_;
+  const BlockBuilder& data_block_builder_;
+};
+
+class MyFlushBlockPolicyFactory : public FlushBlockPolicyFactory {
+ public:
+  explicit MyFlushBlockPolicyFactory(const int num_keys_in_block)
+      : num_keys_in_block_(num_keys_in_block) {}
+
+  virtual const char* Name() const override {
+    return "MyFlushBlockPolicyFactory";
+  }
+
+  virtual FlushBlockPolicy* NewFlushBlockPolicy(
+      const BlockBasedTableOptions& /*table_options*/,
+      const BlockBuilder& data_block_builder) const override {
+    return new MyFlushBlockPolicy(num_keys_in_block_, data_block_builder);
+  }
+
+ private:
+  const int num_keys_in_block_;
+};
+}  // namespace

 static bool enable_io_uring = true;
 extern "C" bool RocksDbIOUringEnable() { return enable_io_uring; }
@@ -392,6 +442,41 @@ TEST_F(DBBasicTest, PutSingleDeleteGet) {
                         kSkipMergePut));
 }

+TEST_F(DBBasicTest, TimedPutBasic) {
+  do {
+    Options options = CurrentOptions();
+    options.merge_operator = MergeOperators::CreateStringAppendOperator();
+    CreateAndReopenWithCF({"pikachu"}, options);
+    ASSERT_OK(TimedPut(1, "foo", "v1", /*write_unix_time=*/0));
+    // Read from memtable
+    ASSERT_EQ("v1", Get(1, "foo"));
+    ASSERT_OK(TimedPut(1, "foo", "v2.1", /*write_unix_time=*/3));
+    ASSERT_EQ("v2.1", Get(1, "foo"));
+
+    // Read from sst file
+    ASSERT_OK(db_->Flush(FlushOptions(), handles_[1]));
+    ASSERT_OK(Merge(1, "foo", "v2.2"));
+    ASSERT_EQ("v2.1,v2.2", Get(1, "foo"));
+    ASSERT_OK(Delete(1, "foo"));
+    ASSERT_EQ("NOT_FOUND", Get(1, "foo"));
+
+    ASSERT_OK(TimedPut(1, "bar", "bv1", /*write_unix_time=*/0));
+    ASSERT_EQ("bv1", Get(1, "bar"));
+    ASSERT_OK(TimedPut(1, "baz", "bzv1", /*write_unix_time=*/0));
+    ASSERT_EQ("bzv1", Get(1, "baz"));
+    std::string range_del_begin = "b";
+    std::string range_del_end = "baz";
+    Slice begin_rdel = range_del_begin, end_rdel = range_del_end;
+    ASSERT_OK(
+        db_->DeleteRange(WriteOptions(), handles_[1], begin_rdel, end_rdel));
+    ASSERT_EQ("NOT_FOUND", Get(1, "bar"));
+
+    ASSERT_EQ("bzv1", Get(1, "baz"));
+    ASSERT_OK(SingleDelete(1, "baz"));
+    ASSERT_EQ("NOT_FOUND", Get(1, "baz"));
+  } while (ChangeOptions(kSkipPlainTable));
+}
+
 TEST_F(DBBasicTest, EmptyFlush) {
  // It is possible to produce empty flushes when using single deletes. Tests
  // whether empty flushes cause issues.
@@ -1318,9 +1403,9 @@ TEST_P(DBMultiGetTestWithParam, MultiGetMultiCF) {
  for (int i = 0; i < num_keys; ++i) {
    int cf = i / 3;
    int cf_key = 1 % 3;
-    cf_kv_vec.emplace_back(std::make_tuple(
+    cf_kv_vec.emplace_back(
        cf, "cf" + std::to_string(cf) + "_key_" + std::to_string(cf_key),
-        "cf" + std::to_string(cf) + "_val_" + std::to_string(cf_key)));
+        "cf" + std::to_string(cf) + "_val_" + std::to_string(cf_key));
    ASSERT_OK(Put(std::get<0>(cf_kv_vec[i]), std::get<1>(cf_kv_vec[i]),
                  std::get<2>(cf_kv_vec[i])));
  }
@@ -1853,7 +1938,7 @@ TEST_P(DBMultiGetTestWithParam, MultiGetBatchedMultiLevel) {
  }
 }

-TEST_P(DBMultiGetTestWithParam, MultiGetBatchedEmptyLevel) {
+TEST_P(DBMultiGetTestWithParam, MultiGetDuplicatesEmptyLevel) {
 #ifndef USE_COROUTINES
  if (std::get<1>(GetParam())) {
    ROCKSDB_GTEST_BYPASS("This test requires coroutine support");
@@ -1865,12 +1950,37 @@ TEST_P(DBMultiGetTestWithParam, MultiGetBatchedEmptyLevel) {
    ROCKSDB_GTEST_BYPASS("This test is only for batched MultiGet");
    return;
  }
+  std::shared_ptr<FaultInjectionTestFS> fault_fs(
+      new FaultInjectionTestFS(env_->GetFileSystem()));
+  std::unique_ptr<Env> env(new CompositeEnvWrapper(env_, fault_fs));
  Options options = CurrentOptions();
+  options.env = env.get();
  options.disable_auto_compactions = true;
  options.merge_operator = MergeOperators::CreateStringAppendOperator();
+
+  LRUCacheOptions cache_opts;
+  cache_opts.capacity = 1 << 20;
+
+  BlockBasedTableOptions table_opts;
+  table_opts.metadata_cache_options.top_level_index_pinning = PinningTier::kAll;
+  table_opts.metadata_cache_options.partition_pinning = PinningTier::kNone;
+  table_opts.index_type =
+      BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
+  table_opts.cache_index_and_filter_blocks = true;
+  table_opts.block_cache = cache_opts.MakeSharedCache();
+  table_opts.flush_block_policy_factory.reset(new MyFlushBlockPolicyFactory(1));
+
+  options.table_factory.reset(new BlockBasedTableFactory(table_opts));
  Reopen(options);
  int key;

+  // Setup the LSM so that the following search bounds are generated for
+  // key 9 for each level -
+  // Level 1 - lb = 0, rb = max
+  // Level 2 - lb = 0, rb = 0
+  // Level 3 - lb = 0, rb = -1
+  // Level 4 - lb = 0, rb = 0
+
  key = 9;
  ASSERT_OK(Put("key_" + std::to_string(key), "val_l2_" + std::to_string(key)));
  ASSERT_OK(Flush());
@@ -1881,6 +1991,9 @@ TEST_P(DBMultiGetTestWithParam, MultiGetBatchedEmptyLevel) {
  key = 9;
  ASSERT_OK(
      Merge("key_" + std::to_string(key), "val_l2_" + std::to_string(key)));
+  const Snapshot* snap = dbfull()->GetSnapshot();
+  ASSERT_OK(
+      Merge("key_" + std::to_string(key), "val_l2_ext_" + std::to_string(key)));
  ASSERT_OK(Flush());
  // Leave level 3 empty
  MoveFilesToLevel(2);
@@ -1899,8 +2012,138 @@ TEST_P(DBMultiGetTestWithParam, MultiGetBatchedEmptyLevel) {
  keys.push_back("key_" + std::to_string(9));
  keys.push_back("key_" + std::to_string(9));

+  int num_reads = 0;
+  SyncPoint::GetInstance()->SetCallBack(
+      "FaultInjectionTestFS::RandomRead", [&](void*) {
+        ++num_reads;
+        // Fail on the 2nd read. First read is index partition,
+        // second read is data block in level 1
+        if (num_reads == 2) {
+          fault_fs->SetFilesystemActive(false);
+        } else {
+          fault_fs->SetFilesystemActive(true);
+        }
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+  size_t capacity = table_opts.block_cache->GetCapacity();
+  table_opts.block_cache->SetCapacity(0);
+  table_opts.block_cache->SetCapacity(capacity);
+
  values = MultiGet(keys, nullptr, std::get<1>(GetParam()));
  ASSERT_EQ(values.size(), 2);
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  dbfull()->ReleaseSnapshot(snap);
+  Destroy(options);
+}
+
+TEST_P(DBMultiGetTestWithParam, MultiGetDuplicatesNonEmptyLevel) {
+#ifndef USE_COROUTINES
+  if (std::get<1>(GetParam())) {
+    ROCKSDB_GTEST_BYPASS("This test requires coroutine support");
+    return;
+  }
+#endif  // USE_COROUTINES
+  // Skip for unbatched MultiGet
+  if (!std::get<0>(GetParam())) {
+    ROCKSDB_GTEST_BYPASS("This test is only for batched MultiGet");
+    return;
+  }
+  std::shared_ptr<FaultInjectionTestFS> fault_fs(
+      new FaultInjectionTestFS(env_->GetFileSystem()));
+  std::unique_ptr<Env> env(new CompositeEnvWrapper(env_, fault_fs));
+  Options options = CurrentOptions();
+  options.env = env.get();
+  options.disable_auto_compactions = true;
+  options.merge_operator = MergeOperators::CreateStringAppendOperator();
+
+  LRUCacheOptions cache_opts;
+  cache_opts.capacity = 1 << 20;
+
+  BlockBasedTableOptions table_opts;
+  table_opts.metadata_cache_options.top_level_index_pinning = PinningTier::kAll;
+  table_opts.metadata_cache_options.partition_pinning = PinningTier::kNone;
+  table_opts.index_type =
+      BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
+  table_opts.cache_index_and_filter_blocks = true;
+  table_opts.block_cache = cache_opts.MakeSharedCache();
+  table_opts.flush_block_policy_factory.reset(new MyFlushBlockPolicyFactory(1));
+
+  options.table_factory.reset(new BlockBasedTableFactory(table_opts));
+  Reopen(options);
+  int key;
+
+  // Setup the LSM so that the following search bounds are generated for
+  // key 9 for each level -
+  // Level 1 - lb = 0, rb = max
+  // Level 2 - lb = 0, rb = 0
+  // Level 3 - lb = 0, rb = 1
+  // Level 4 - N/A
+
+  key = 8;
+  ASSERT_OK(Put("key_" + std::to_string(key), "val_l2_" + std::to_string(key)));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(4);
+
+  key = 7;
+  ASSERT_OK(Put("key_" + std::to_string(key), "val_l2_" + std::to_string(key)));
+  ASSERT_OK(Flush());
+
+  key = 9;
+  ASSERT_OK(Put("key_" + std::to_string(key), "val_l2_" + std::to_string(key)));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(3);
+
+  key = 5;
+  ASSERT_OK(Put("key_" + std::to_string(key), "val_l2_" + std::to_string(key)));
+  key = 9;
+  ASSERT_OK(
+      Merge("key_" + std::to_string(key), "merge1_l2_" + std::to_string(key)));
+  const Snapshot* snap = dbfull()->GetSnapshot();
+  ASSERT_OK(
+      Merge("key_" + std::to_string(key), "merge2_l2_" + std::to_string(key)));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(2);
+
+  key = 2;
+  ASSERT_OK(Put("key_" + std::to_string(key), "val_l2_" + std::to_string(key)));
+  key = 6;
+  ASSERT_OK(
+      Merge("key_" + std::to_string(key), "val_l2_" + std::to_string(key)));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(1);
+
+  std::vector<std::string> keys;
+  std::vector<std::string> values;
+
+  keys.push_back("key_" + std::to_string(9));
+  keys.push_back("key_" + std::to_string(9));
+
+  int num_reads = 0;
+  SyncPoint::GetInstance()->SetCallBack(
+      "FaultInjectionTestFS::RandomRead", [&](void*) {
+        ++num_reads;
+        // Fail on the 2nd read. First read is index partition,
+        // second read is data block in level 1
+        if (num_reads == 2) {
+          fault_fs->SetFilesystemActive(false);
+        } else {
+          fault_fs->SetFilesystemActive(true);
+        }
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+  size_t capacity = table_opts.block_cache->GetCapacity();
+  table_opts.block_cache->SetCapacity(0);
+  table_opts.block_cache->SetCapacity(capacity);
+
+  values = MultiGet(keys, nullptr, std::get<1>(GetParam()));
+  ASSERT_EQ(values.size(), 2);
+  ASSERT_EQ(values[0], "Corruption: Not active");
+  ASSERT_EQ(values[1], "val_l2_9,merge1_l2_9,merge2_l2_9");
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  dbfull()->ReleaseSnapshot(snap);
+  Destroy(options);
 }

 TEST_P(DBMultiGetTestWithParam, MultiGetBatchedMultiLevelMerge) {
@@ -2399,9 +2642,9 @@ TEST_P(DBMultiGetAsyncIOTest, GetFromL1) {
  key_strs.push_back(Key(33));
  key_strs.push_back(Key(54));
  key_strs.push_back(Key(102));
-  keys.push_back(key_strs[0]);
-  keys.push_back(key_strs[1]);
-  keys.push_back(key_strs[2]);
+  keys.emplace_back(key_strs[0]);
+  keys.emplace_back(key_strs[1]);
+  keys.emplace_back(key_strs[2]);
  values.resize(keys.size());
  statuses.resize(keys.size());

@@ -2444,9 +2687,9 @@ TEST_P(DBMultiGetAsyncIOTest, GetFromL1Error) {
  key_strs.push_back(Key(33));
  key_strs.push_back(Key(54));
  key_strs.push_back(Key(102));
-  keys.push_back(key_strs[0]);
-  keys.push_back(key_strs[1]);
-  keys.push_back(key_strs[2]);
+  keys.emplace_back(key_strs[0]);
+  keys.emplace_back(key_strs[1]);
+  keys.emplace_back(key_strs[2]);
  values.resize(keys.size());
  statuses.resize(keys.size());

@@ -2509,9 +2752,9 @@ TEST_P(DBMultiGetAsyncIOTest, LastKeyInFile) {
  key_strs.push_back(Key(21));
  key_strs.push_back(Key(54));
  key_strs.push_back(Key(102));
-  keys.push_back(key_strs[0]);
-  keys.push_back(key_strs[1]);
-  keys.push_back(key_strs[2]);
+  keys.emplace_back(key_strs[0]);
+  keys.emplace_back(key_strs[1]);
+  keys.emplace_back(key_strs[2]);
  values.resize(keys.size());
  statuses.resize(keys.size());

@@ -2554,9 +2797,9 @@ TEST_P(DBMultiGetAsyncIOTest, GetFromL1AndL2) {
  key_strs.push_back(Key(33));
  key_strs.push_back(Key(56));
  key_strs.push_back(Key(102));
-  keys.push_back(key_strs[0]);
-  keys.push_back(key_strs[1]);
-  keys.push_back(key_strs[2]);
+  keys.emplace_back(key_strs[0]);
+  keys.emplace_back(key_strs[1]);
+  keys.emplace_back(key_strs[2]);
  values.resize(keys.size());
  statuses.resize(keys.size());

@@ -2597,8 +2840,8 @@ TEST_P(DBMultiGetAsyncIOTest, GetFromL2WithRangeOverlapL0L1) {
  // 19 and 26 are in L2, but overlap with L0 and L1 file ranges
  key_strs.push_back(Key(19));
  key_strs.push_back(Key(26));
-  keys.push_back(key_strs[0]);
-  keys.push_back(key_strs[1]);
+  keys.emplace_back(key_strs[0]);
+  keys.emplace_back(key_strs[1]);
  values.resize(keys.size());
  statuses.resize(keys.size());

@@ -2633,8 +2876,8 @@ TEST_P(DBMultiGetAsyncIOTest, GetFromL2WithRangeDelInL1) {
  // 139 and 163 are in L2, but overlap with a range deletes in L1
  key_strs.push_back(Key(139));
  key_strs.push_back(Key(163));
-  keys.push_back(key_strs[0]);
-  keys.push_back(key_strs[1]);
+  keys.emplace_back(key_strs[0]);
+  keys.emplace_back(key_strs[1]);
  values.resize(keys.size());
  statuses.resize(keys.size());

@@ -2663,9 +2906,9 @@ TEST_P(DBMultiGetAsyncIOTest, GetFromL1AndL2WithRangeDelInL1) {
  key_strs.push_back(Key(139));
  key_strs.push_back(Key(144));
  key_strs.push_back(Key(163));
-  keys.push_back(key_strs[0]);
-  keys.push_back(key_strs[1]);
-  keys.push_back(key_strs[2]);
+  keys.emplace_back(key_strs[0]);
+  keys.emplace_back(key_strs[1]);
+  keys.emplace_back(key_strs[2]);
  values.resize(keys.size());
  statuses.resize(keys.size());

@@ -2696,9 +2939,9 @@ TEST_P(DBMultiGetAsyncIOTest, GetNoIOUring) {
  key_strs.push_back(Key(33));
  key_strs.push_back(Key(54));
  key_strs.push_back(Key(102));
-  keys.push_back(key_strs[0]);
-  keys.push_back(key_strs[1]);
-  keys.push_back(key_strs[2]);
+  keys.emplace_back(key_strs[0]);
+  keys.emplace_back(key_strs[1]);
+  keys.emplace_back(key_strs[2]);
  values.resize(keys.size());
  statuses.resize(keys.size());

@@ -3077,9 +3320,9 @@ TEST_F(DBBasicTest, MultiGetIOBufferOverrun) {

  // Warm up the cache first
  key_data.emplace_back(Key(0));
-  keys.emplace_back(Slice(key_data.back()));
+  keys.emplace_back(key_data.back());
  key_data.emplace_back(Key(50));
-  keys.emplace_back(Slice(key_data.back()));
+  keys.emplace_back(key_data.back());
  statuses.resize(keys.size());

  dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(),
@@ -3134,8 +3377,7 @@ TEST_F(DBBasicTest, BestEffortsRecoveryWithVersionBuildingFailure) {
  SyncPoint::GetInstance()->SetCallBack(
      "VersionBuilder::CheckConsistencyBeforeReturn", [&](void* arg) {
        ASSERT_NE(nullptr, arg);
-        *(reinterpret_cast<Status*>(arg)) =
-            Status::Corruption("Inject corruption");
+        *(static_cast<Status*>(arg)) = Status::Corruption("Inject corruption");
      });
  SyncPoint::GetInstance()->EnableProcessing();

@@ -3454,10 +3696,10 @@ TEST_F(DBBasicTest, ConcurrentlyCloseDB) {
  DestroyAndReopen(options);
  std::vector<std::thread> workers;
  for (int i = 0; i < 10; i++) {
-    workers.push_back(std::thread([&]() {
+    workers.emplace_back([&]() {
      auto s = db_->Close();
      ASSERT_OK(s);
-    }));
+    });
  }
  for (auto& w : workers) {
    w.join();
@@ -3566,7 +3808,7 @@ class DBBasicTestMultiGet : public DBTestBase {
      table_options.pin_l0_filter_and_index_blocks_in_cache = true;
    }
    table_options.flush_block_policy_factory.reset(
-        new MyFlushBlockPolicyFactory());
+        new MyFlushBlockPolicyFactory(10));
    options.table_factory.reset(NewBlockBasedTableFactory(table_options));
    if (!compression_enabled_) {
      options.compression = kNoCompression;
@@ -3654,46 +3896,6 @@ class DBBasicTestMultiGet : public DBTestBase {
  static void TearDownTestCase() {}

 protected:
-  class MyFlushBlockPolicyFactory : public FlushBlockPolicyFactory {
-   public:
-    MyFlushBlockPolicyFactory() {}
-
-    virtual const char* Name() const override {
-      return "MyFlushBlockPolicyFactory";
-    }
-
-    virtual FlushBlockPolicy* NewFlushBlockPolicy(
-        const BlockBasedTableOptions& /*table_options*/,
-        const BlockBuilder& data_block_builder) const override {
-      return new MyFlushBlockPolicy(data_block_builder);
-    }
-  };
-
-  class MyFlushBlockPolicy : public FlushBlockPolicy {
-   public:
-    explicit MyFlushBlockPolicy(const BlockBuilder& data_block_builder)
-        : num_keys_(0), data_block_builder_(data_block_builder) {}
-
-    bool Update(const Slice& /*key*/, const Slice& /*value*/) override {
-      if (data_block_builder_.empty()) {
-        // First key in this block
-        num_keys_ = 1;
-        return false;
-      }
-      // Flush every 10 keys
-      if (num_keys_ == 10) {
-        num_keys_ = 1;
-        return true;
-      }
-      num_keys_++;
-      return false;
-    }
-
-   private:
-    int num_keys_;
-    const BlockBuilder& data_block_builder_;
-  };
-
  class MyBlockCache : public CacheWrapper {
   public:
    explicit MyBlockCache(std::shared_ptr<Cache> target)
@@ -3771,9 +3973,9 @@ TEST_P(DBBasicTestWithParallelIO, MultiGet) {

  // Warm up the cache first
  key_data.emplace_back(Key(0));
-  keys.emplace_back(Slice(key_data.back()));
+  keys.emplace_back(key_data.back());
  key_data.emplace_back(Key(50));
-  keys.emplace_back(Slice(key_data.back()));
+  keys.emplace_back(key_data.back());
  statuses.resize(keys.size());

  dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(),
@@ -3952,9 +4154,9 @@ TEST_P(DBBasicTestWithParallelIO, MultiGetDirectIO) {

  // Warm up the cache first
  key_data.emplace_back(Key(0));
-  keys.emplace_back(Slice(key_data.back()));
+  keys.emplace_back(key_data.back());
  key_data.emplace_back(Key(50));
-  keys.emplace_back(Slice(key_data.back()));
+  keys.emplace_back(key_data.back());
  statuses.resize(keys.size());

  dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(),
@@ -4022,9 +4224,9 @@ TEST_P(DBBasicTestWithParallelIO, MultiGetWithChecksumMismatch) {

  // Warm up the cache first
  key_data.emplace_back(Key(0));
-  keys.emplace_back(Slice(key_data.back()));
+  keys.emplace_back(key_data.back());
  key_data.emplace_back(Key(50));
-  keys.emplace_back(Slice(key_data.back()));
+  keys.emplace_back(key_data.back());
  statuses.resize(keys.size());

  dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(),
@@ -4070,9 +4272,9 @@ TEST_P(DBBasicTestWithParallelIO, MultiGetWithMissingFile) {

  // Warm up the cache first
  key_data.emplace_back(Key(0));
-  keys.emplace_back(Slice(key_data.back()));
+  keys.emplace_back(key_data.back());
  key_data.emplace_back(Key(50));
-  keys.emplace_back(Slice(key_data.back()));
+  keys.emplace_back(key_data.back());
  statuses.resize(keys.size());

  dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(),
@@ -4110,7 +4312,7 @@ class DeadlineRandomAccessFile : public FSRandomAccessFileOwnerWrapper {
                     const IOOptions& options, IODebugContext* dbg) override;

  IOStatus ReadAsync(FSReadRequest& req, const IOOptions& opts,
-                     std::function<void(const FSReadRequest&, void*)> cb,
+                     std::function<void(FSReadRequest&, void*)> cb,
                     void* cb_arg, void** io_handle, IOHandleDeleter* del_fn,
                     IODebugContext* dbg) override;

@@ -4256,7 +4458,7 @@ IOStatus DeadlineRandomAccessFile::Read(uint64_t offset, size_t len,

 IOStatus DeadlineRandomAccessFile::ReadAsync(
    FSReadRequest& req, const IOOptions& opts,
-    std::function<void(const FSReadRequest&, void*)> cb, void* cb_arg,
+    std::function<void(FSReadRequest&, void*)> cb, void* cb_arg,
    void** io_handle, IOHandleDeleter* del_fn, IODebugContext* dbg) {
  const std::chrono::microseconds deadline = fs_.GetDeadline();
  const std::chrono::microseconds io_timeout = fs_.GetIOTimeout();
@@ -4337,68 +4539,35 @@ TEST_P(DBBasicTestMultiGetDeadline, MultiGetDeadlineExceeded) {
  SetTimeElapseOnlySleepOnReopen(&options);
  ReopenWithColumnFamilies(GetCFNames(), options);

-  // Test the non-batched version of MultiGet with multiple column
-  // families
+  // Test batched MultiGet with an IO delay in the first data block read.
+  // Both keys in the first CF should succeed as they're in the same data
+  // block and would form one batch, and we check for deadline between
+  // batches.
  std::vector<std::string> key_str;
  size_t i;
-  for (i = 0; i < 5; ++i) {
+  for (i = 0; i < 10; ++i) {
    key_str.emplace_back(Key(static_cast<int>(i)));
  }
  std::vector<ColumnFamilyHandle*> cfs(key_str.size());
-  ;
  std::vector<Slice> keys(key_str.size());
-  std::vector<std::string> values(key_str.size());
+  std::vector<PinnableSlice> pin_values(keys.size());
+
  for (i = 0; i < key_str.size(); ++i) {
-    cfs[i] = handles_[i];
+    // 2 keys per CF
+    cfs[i] = handles_[i / 2];
    keys[i] = Slice(key_str[i].data(), key_str[i].size());
  }
-
  ReadOptions ro;
  ro.deadline = std::chrono::microseconds{env->NowMicros() + 10000};
  ro.async_io = GetParam();
  // Delay the first IO
  fs->SetDelayTrigger(ro.deadline, ro.io_timeout, 0);

-  std::vector<Status> statuses = dbfull()->MultiGet(ro, cfs, keys, &values);
-  // The first key is successful because we check after the lookup, but
-  // subsequent keys fail due to deadline exceeded
-  CheckStatus(statuses, 1);
-
-  // Clear the cache
-  cache->SetCapacity(0);
-  cache->SetCapacity(1048576);
-  // Test non-batched Multiget with multiple column families and
-  // introducing an IO delay in one of the middle CFs
-  key_str.clear();
-  for (i = 0; i < 10; ++i) {
-    key_str.emplace_back(Key(static_cast<int>(i)));
-  }
-  cfs.resize(key_str.size());
-  keys.resize(key_str.size());
-  values.resize(key_str.size());
-  for (i = 0; i < key_str.size(); ++i) {
-    // 2 keys per CF
-    cfs[i] = handles_[i / 2];
-    keys[i] = Slice(key_str[i].data(), key_str[i].size());
-  }
-  ro.deadline = std::chrono::microseconds{env->NowMicros() + 10000};
-  fs->SetDelayTrigger(ro.deadline, ro.io_timeout, 1);
-  statuses = dbfull()->MultiGet(ro, cfs, keys, &values);
-  CheckStatus(statuses, 3);
-
-  // Test batched MultiGet with an IO delay in the first data block read.
-  // Both keys in the first CF should succeed as they're in the same data
-  // block and would form one batch, and we check for deadline between
-  // batches.
-  std::vector<PinnableSlice> pin_values(keys.size());
-  cache->SetCapacity(0);
-  cache->SetCapacity(1048576);
-  statuses.clear();
-  statuses.resize(keys.size());
-  ro.deadline = std::chrono::microseconds{env->NowMicros() + 10000};
-  fs->SetDelayTrigger(ro.deadline, ro.io_timeout, 0);
+  std::vector<Status> statuses(key_str.size());
  dbfull()->MultiGet(ro, keys.size(), cfs.data(), keys.data(),
                     pin_values.data(), statuses.data());
+  // The first key is successful because we check after the lookup, but
+  // subsequent keys fail due to deadline exceeded
  CheckStatus(statuses, 2);

  // Similar to the previous one, but an IO delay in the third CF data block
@@ -4476,7 +4645,7 @@ TEST_F(DBBasicTest, ManifestWriteFailure) {
  SyncPoint::GetInstance()->SetCallBack(
      "VersionSet::ProcessManifestWrites:AfterSyncManifest", [&](void* arg) {
        ASSERT_NE(nullptr, arg);
-        auto* s = reinterpret_cast<Status*>(arg);
+        auto* s = static_cast<Status*>(arg);
        ASSERT_OK(*s);
        // Manually overwrite return status
        *s = Status::IOError();
@@ -4531,7 +4700,7 @@ TEST_F(DBBasicTest, FailOpenIfLoggerCreationFail) {
  SyncPoint::GetInstance()->ClearAllCallBacks();
  SyncPoint::GetInstance()->SetCallBack(
      "rocksdb::CreateLoggerFromOptions:AfterGetPath", [&](void* arg) {
-        auto* s = reinterpret_cast<Status*>(arg);
+        auto* s = static_cast<Status*>(arg);
        assert(s);
        *s = Status::IOError("Injected");
      });
@@ -4609,7 +4778,7 @@ TEST_F(DBBasicTest, VerifyFileChecksumsReadahead) {
  uint64_t number;
  FileType type;
  ASSERT_OK(env_->GetChildren(dbname_, &filenames));
-  for (auto name : filenames) {
+  for (const auto& name : filenames) {
    if (ParseFileName(name, &number, &type)) {
      if (type == kTableFile) {
        sst_cnt++;
@@ -744,7 +744,7 @@ TEST_F(DBBlockCacheTest, AddRedundantStats) {
  const size_t capacity = size_t{1} << 25;
  const int num_shard_bits = 0;  // 1 shard
  int iterations_tested = 0;
-  for (std::shared_ptr<Cache> base_cache :
+  for (const std::shared_ptr<Cache>& base_cache :
       {NewLRUCache(capacity, num_shard_bits),
        // FixedHyperClockCache
        HyperClockCacheOptions(
@@ -990,7 +990,7 @@ TEST_F(DBBlockCacheTest, CacheEntryRoleStats) {
  int iterations_tested = 0;
  for (bool partition : {false, true}) {
    SCOPED_TRACE("Partition? " + std::to_string(partition));
-    for (std::shared_ptr<Cache> cache :
+    for (const std::shared_ptr<Cache>& cache :
         {NewLRUCache(capacity),
          HyperClockCacheOptions(
              capacity,
@@ -1251,7 +1251,7 @@ void DummyFillCache(Cache& cache, size_t entry_size,

 class CountingLogger : public Logger {
 public:
-  ~CountingLogger() override {}
+  ~CountingLogger() override = default;
  using Logger::Logv;
  void Logv(const InfoLogLevel log_level, const char* format,
            va_list /*ap*/) override {
@@ -1373,7 +1373,7 @@ class StableCacheKeyTestFS : public FaultInjectionTestFS {
    SetFailGetUniqueId(true);
  }

-  virtual ~StableCacheKeyTestFS() override {}
+  ~StableCacheKeyTestFS() override = default;

  IOStatus LinkFile(const std::string&, const std::string&, const IOOptions&,
                    IODebugContext*) override {
@@ -1424,7 +1424,7 @@ TEST_P(DBBlockCacheKeyTest, StableCacheKeys) {
    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
        "BlockBasedTableBuilder::BlockBasedTableBuilder:PreSetupBaseCacheKey",
        [&](void* arg) {
-          TableProperties* props = reinterpret_cast<TableProperties*>(arg);
+          TableProperties* props = static_cast<TableProperties*>(arg);
          props->orig_file_number = 0;
        });
    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
@@ -1566,7 +1566,7 @@ class CacheKeyTest : public testing::Test {
    tp_.db_id = std::to_string(db_id_);
    tp_.orig_file_number = file_number;
    bool is_stable;
-    std::string cur_session_id = "";  // ignored
+    std::string cur_session_id;       // ignored
    uint64_t cur_file_number = 42;    // ignored
    OffsetableCacheKey rv;
    BlockBasedTable::SetupBaseCacheKey(&tp_, cur_session_id, cur_file_number,
@@ -1834,6 +1834,7 @@ class DBBlockCachePinningTest
  PinningTier unpartitioned_pinning_;
 };

+#ifdef LZ4
 TEST_P(DBBlockCachePinningTest, TwoLevelDB) {
  // Creates one file in L0 and one file in L1. Both files have enough data that
  // their index and filter blocks are partitioned. The L1 file will also have
@@ -1845,10 +1846,7 @@ TEST_P(DBBlockCachePinningTest, TwoLevelDB) {
  const int kNumKeysPerFile = kBlockSize * kNumBlocksPerFile / kKeySize;

  Options options = CurrentOptions();
-  // `kNoCompression` makes the unit test more portable. But it relies on the
-  // current behavior of persisting/accessing dictionary even when there's no
-  // (de)compression happening, which seems fairly likely to change over time.
-  options.compression = kNoCompression;
+  options.compression = kLZ4Compression;
  options.compression_opts.max_dict_bytes = 4 << 10;
  options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
  BlockBasedTableOptions table_options;
@@ -1961,6 +1959,7 @@ TEST_P(DBBlockCachePinningTest, TwoLevelDB) {
  ASSERT_EQ(expected_compression_dict_misses,
            TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_MISS));
 }
+#endif

 INSTANTIATE_TEST_CASE_P(
    DBBlockCachePinningTest, DBBlockCachePinningTest,
@@ -19,6 +19,7 @@
 #include "port/stack_trace.h"
 #include "rocksdb/advanced_options.h"
 #include "rocksdb/convenience.h"
+#include "rocksdb/experimental.h"
 #include "rocksdb/filter_policy.h"
 #include "rocksdb/perf_context.h"
 #include "rocksdb/statistics.h"
@@ -77,7 +78,7 @@ class DBBloomFilterTestWithParam
  DBBloomFilterTestWithParam()
      : DBTestBase("db_bloom_filter_tests", /*env_do_fsync=*/true) {}

-  ~DBBloomFilterTestWithParam() override {}
+  ~DBBloomFilterTestWithParam() override = default;

  void SetUp() override {
    bfp_impl_ = std::get<0>(GetParam());
@@ -2050,7 +2051,7 @@ class DBBloomFilterTestVaryPrefixAndFormatVer
  DBBloomFilterTestVaryPrefixAndFormatVer()
      : DBTestBase("db_bloom_filter_tests", /*env_do_fsync=*/true) {}

-  ~DBBloomFilterTestVaryPrefixAndFormatVer() override {}
+  ~DBBloomFilterTestVaryPrefixAndFormatVer() override = default;

  void SetUp() override {
    use_prefix_ = std::get<0>(GetParam());
@@ -2125,8 +2126,9 @@ TEST_P(DBBloomFilterTestVaryPrefixAndFormatVer, PartitionedMultiGet) {
      values[i] = PinnableSlice();
    }

-    db_->MultiGet(ropts, Q, &column_families[0], &key_slices[0], &values[0],
-                  /*timestamps=*/nullptr, &statuses[0], true);
+    db_->MultiGet(ropts, Q, column_families.data(), key_slices.data(),
+                  values.data(),
+                  /*timestamps=*/nullptr, statuses.data(), true);

    // Confirm correct status results
    uint32_t number_not_found = 0;
@@ -2176,8 +2178,9 @@ TEST_P(DBBloomFilterTestVaryPrefixAndFormatVer, PartitionedMultiGet) {
      values[i] = PinnableSlice();
    }

-    db_->MultiGet(ropts, Q, &column_families[0], &key_slices[0], &values[0],
-                  /*timestamps=*/nullptr, &statuses[0], true);
+    db_->MultiGet(ropts, Q, column_families.data(), key_slices.data(),
+                  values.data(),
+                  /*timestamps=*/nullptr, statuses.data(), true);

    // Confirm correct status results
    uint32_t number_not_found = 0;
@@ -3562,6 +3565,277 @@ TEST_F(DBBloomFilterTest, WeirdPrefixExtractorWithFilter3) {
  }
 }

+TEST_F(DBBloomFilterTest, SstQueryFilter) {
+  using experimental::KeySegmentsExtractor;
+  using experimental::MakeSharedBytewiseMinMaxSQFC;
+  using experimental::SelectKeySegment;
+  using experimental::SstQueryFilterConfigs;
+  using experimental::SstQueryFilterConfigsManager;
+  using KeyCategorySet = KeySegmentsExtractor::KeyCategorySet;
+
+  struct MySegmentExtractor : public KeySegmentsExtractor {
+    char min_first_char;
+    char max_first_char;
+    char delim_char;
+    MySegmentExtractor(char _min_first_char, char _max_first_char,
+                       char _delim_char)
+        : min_first_char(_min_first_char),
+          max_first_char(_max_first_char),
+          delim_char(_delim_char) {}
+
+    const char* Name() const override { return "MySegmentExtractor"; }
+
+    std::string GetId() const override {
+      return std::string("MySegmentExtractor+") + min_first_char +
+             max_first_char + delim_char;
+    }
+
+    void Extract(const Slice& key_or_bound, KeyKind /*kind*/,
+                 Result* result) const override {
+      size_t len = key_or_bound.size();
+      if (len == 0) {
+        result->category = KeySegmentsExtractor::kReservedLowCategory;
+      } else if (static_cast<unsigned char>(key_or_bound[0]) <
+                 static_cast<unsigned char>(min_first_char)) {
+        result->category = KeySegmentsExtractor::kReservedLowCategory;
+      } else if (static_cast<unsigned char>(key_or_bound[0]) >
+                 static_cast<unsigned char>(max_first_char)) {
+        result->category = KeySegmentsExtractor::kReservedHighCategory;
+      }
+      for (uint32_t i = 0; i < len; ++i) {
+        if (key_or_bound[i] == delim_char || i + 1 == key_or_bound.size()) {
+          result->segment_ends.push_back(i + 1);
+        }
+      }
+    }
+  };
+
+  // Use '_' as delimiter, but different spans for default category
+  auto extractor_to_c = std::make_shared<MySegmentExtractor>('a', 'c', '_');
+  auto extractor_to_z = std::make_shared<MySegmentExtractor>('a', 'z', '_');
+  auto extractor_alt = std::make_shared<MySegmentExtractor>('0', '9', '_');
+
+  // Filter on 2nd field, only for default category
+  auto filter1_def = MakeSharedBytewiseMinMaxSQFC(
+      experimental::SelectKeySegment(1),
+      KeyCategorySet{KeySegmentsExtractor::kDefaultCategory});
+
+  // Also filter on 3rd field regardless of category
+  auto filter2_all =
+      MakeSharedBytewiseMinMaxSQFC(experimental::SelectKeySegment(2));
+
+  SstQueryFilterConfigs configs1 = {{filter1_def, filter2_all}, extractor_to_c};
+  SstQueryFilterConfigs configs2 = {{filter1_def, filter2_all}, extractor_to_z};
+  SstQueryFilterConfigs configs3 = {{filter2_all}, extractor_alt};
+
+  SstQueryFilterConfigsManager::Data data = {
+      {42, {{"foo", configs1}}}, {43, {{"foo", configs2}, {"bar", configs3}}}};
+
+  std::shared_ptr<SstQueryFilterConfigsManager> configs_manager;
+  ASSERT_OK(SstQueryFilterConfigsManager::MakeShared(data, &configs_manager));
+
+  // Test manager behaviors
+  auto MakeFactory = [configs_manager](
+                         const std::string& configs_name,
+                         SstQueryFilterConfigsManager::FilteringVersion ver)
+      -> std::shared_ptr<SstQueryFilterConfigsManager::Factory> {
+    std::shared_ptr<SstQueryFilterConfigsManager::Factory> factory;
+    Status s = configs_manager->MakeSharedFactory(configs_name, ver, &factory);
+    assert(s.ok());
+    return factory;
+  };
+  std::shared_ptr<SstQueryFilterConfigsManager::Factory> factory;
+
+  // Version 0 is always OK, returning empty/not found configs
+  ASSERT_TRUE(MakeFactory("blah", 0)->GetConfigs().IsEmptyNotFound());
+  ASSERT_TRUE(MakeFactory("foo", 0)->GetConfigs().IsEmptyNotFound());
+  ASSERT_TRUE(MakeFactory("bar", 0)->GetConfigs().IsEmptyNotFound());
+
+  // We can't be sure about the proper configuration for versions outside the
+  // known range (and reserved version 0).
+  ASSERT_TRUE(configs_manager->MakeSharedFactory("foo", 1, &factory)
+                  .IsInvalidArgument());
+  ASSERT_TRUE(configs_manager->MakeSharedFactory("foo", 41, &factory)
+                  .IsInvalidArgument());
+  ASSERT_TRUE(configs_manager->MakeSharedFactory("foo", 44, &factory)
+                  .IsInvalidArgument());
+  ASSERT_TRUE(configs_manager->MakeSharedFactory("foo", 500, &factory)
+                  .IsInvalidArgument());
+
+  ASSERT_TRUE(MakeFactory("blah", 42)->GetConfigs().IsEmptyNotFound());
+  ASSERT_TRUE(MakeFactory("blah", 43)->GetConfigs().IsEmptyNotFound());
+  ASSERT_FALSE(MakeFactory("foo", 42)->GetConfigs().IsEmptyNotFound());
+  ASSERT_FALSE(MakeFactory("foo", 43)->GetConfigs().IsEmptyNotFound());
+  ASSERT_TRUE(MakeFactory("bar", 42)->GetConfigs().IsEmptyNotFound());
+  ASSERT_FALSE(MakeFactory("bar", 43)->GetConfigs().IsEmptyNotFound());
+
+  ASSERT_OK(configs_manager->MakeSharedFactory("foo", 42, &factory));
+  ASSERT_EQ(factory->GetConfigsName(), "foo");
+  ASSERT_EQ(factory->GetConfigs().IsEmptyNotFound(), false);
+
+  Options options = CurrentOptions();
+  options.statistics = CreateDBStatistics();
+  options.table_properties_collector_factories.push_back(factory);
+
+  DestroyAndReopen(options);
+
+  // For lower level file
+  ASSERT_OK(Put("   ", "val0"));
+  ASSERT_OK(Put("   _345_678", "val0"));
+  ASSERT_OK(Put("aaa", "val0"));
+  ASSERT_OK(Put("abc_123", "val1"));
+  ASSERT_OK(Put("abc_13", "val2"));
+  ASSERT_OK(Put("abc_156_987", "val3"));
+  ASSERT_OK(Put("bcd_1722", "val4"));
+  ASSERT_OK(Put("xyz_145", "val5"));
+  ASSERT_OK(Put("xyz_167", "val6"));
+  ASSERT_OK(Put("xyz_178", "val7"));
+  ASSERT_OK(Put("zzz", "val0"));
+  ASSERT_OK(Put("~~~", "val0"));
+  ASSERT_OK(Put("~~~_456_789", "val0"));
+
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(1);
+
+  ASSERT_EQ(factory->GetFilteringVersion(), 42U);
+  ASSERT_NOK(factory->SetFilteringVersion(41));
+  ASSERT_NOK(factory->SetFilteringVersion(44));
+  ASSERT_EQ(factory->GetFilteringVersion(), 42U);
+  ASSERT_OK(factory->SetFilteringVersion(43));
+  ASSERT_EQ(factory->GetFilteringVersion(), 43U);
+
+  // For higher level file
+  ASSERT_OK(Put("   ", "val0"));
+  ASSERT_OK(Put("   _345_680", "val0"));
+  ASSERT_OK(Put("aaa", "val9"));
+  ASSERT_OK(Put("abc_234", "val1"));
+  ASSERT_OK(Put("abc_245_567", "val2"));
+  ASSERT_OK(Put("abc_25", "val3"));
+  ASSERT_OK(Put("xyz_180", "val4"));
+  ASSERT_OK(Put("xyz_191", "val4"));
+  ASSERT_OK(Put("xyz_260", "val4"));
+  ASSERT_OK(Put("zzz", "val9"));
+  ASSERT_OK(Put("~~~", "val0"));
+  ASSERT_OK(Put("~~~_456_790", "val0"));
+
+  ASSERT_OK(Flush());
+
+  using Keys = std::vector<std::string>;
+  auto RangeQueryKeys =
+      [factory, db = db_](
+          std::string lb, std::string ub,
+          std::shared_ptr<SstQueryFilterConfigsManager::Factory> alt_factory =
+              nullptr) {
+        Slice lb_slice = lb;
+        Slice ub_slice = ub;
+
+        ReadOptions ro;
+        ro.iterate_lower_bound = &lb_slice;
+        ro.iterate_upper_bound = &ub_slice;
+        ro.table_filter = (alt_factory ? alt_factory : factory)
+                              ->GetTableFilterForRangeQuery(lb_slice, ub_slice);
+        auto it = db->NewIterator(ro);
+        Keys ret;
+        for (it->Seek(lb_slice); it->Valid(); it->Next()) {
+          ret.push_back(it->key().ToString());
+        }
+        EXPECT_OK(it->status());
+        delete it;
+        return ret;
+      };
+
+  // Control 1: range is not filtered but min/max filter is checked
+  // because of common prefix leading up to 2nd segment
+  // TODO/future: statistics for when filter is checked vs. not applicable
+  EXPECT_EQ(RangeQueryKeys("abc_150", "abc_249"),
+            Keys({"abc_156_987", "abc_234", "abc_245_567"}));
+  EXPECT_EQ(TestGetAndResetTickerCount(options, NON_LAST_LEVEL_SEEK_DATA), 2);
+
+  // Test 1: range is filtered to just lowest level, fully containing the
+  // segments in that category
+  EXPECT_EQ(RangeQueryKeys("abc_100", "abc_179"),
+            Keys({"abc_123", "abc_13", "abc_156_987"}));
+  EXPECT_EQ(TestGetAndResetTickerCount(options, NON_LAST_LEVEL_SEEK_DATA), 1);
+
+  // Test 2: range is filtered to just lowest level, partial overlap
+  EXPECT_EQ(RangeQueryKeys("abc_1500_x_y", "abc_16QQ"), Keys({"abc_156_987"}));
+  EXPECT_EQ(TestGetAndResetTickerCount(options, NON_LAST_LEVEL_SEEK_DATA), 1);
+
+  // Test 3: range is filtered to just highest level, fully containing the
+  // segments in that category but would be overlapping the range for the other
+  // file if the filter included all categories
+  EXPECT_EQ(RangeQueryKeys("abc_200", "abc_300"),
+            Keys({"abc_234", "abc_245_567", "abc_25"}));
+  EXPECT_EQ(TestGetAndResetTickerCount(options, NON_LAST_LEVEL_SEEK_DATA), 1);
+
+  // Test 4: range is filtered to just highest level, partial overlap (etc.)
+  EXPECT_EQ(RangeQueryKeys("abc_200", "abc_249"),
+            Keys({"abc_234", "abc_245_567"}));
+  EXPECT_EQ(TestGetAndResetTickerCount(options, NON_LAST_LEVEL_SEEK_DATA), 1);
+
+  // Test 5: range is filtered from both levels, because of category scope
+  EXPECT_EQ(RangeQueryKeys("abc_300", "abc_400"), Keys({}));
+  EXPECT_EQ(TestGetAndResetTickerCount(options, NON_LAST_LEVEL_SEEK_DATA), 0);
+
+  // Control 2: range is not filtered because association between 1st and
+  // 2nd segment is not represented
+  EXPECT_EQ(RangeQueryKeys("abc_170", "abc_190"), Keys({}));
+  EXPECT_EQ(TestGetAndResetTickerCount(options, NON_LAST_LEVEL_SEEK_DATA), 2);
+
+  // Control 3: range is not filtered because there's no (bloom) filter on
+  // 1st segment (like prefix filtering)
+  EXPECT_EQ(RangeQueryKeys("baa_170", "baa_190"), Keys({}));
+  EXPECT_EQ(TestGetAndResetTickerCount(options, NON_LAST_LEVEL_SEEK_DATA), 2);
+
+  // Control 4: range is not filtered because difference in segments leading
+  // up to 2nd segment
+  EXPECT_EQ(RangeQueryKeys("abc_500", "abd_501"), Keys({}));
+  EXPECT_EQ(TestGetAndResetTickerCount(options, NON_LAST_LEVEL_SEEK_DATA), 2);
+
+  // TODO: exclusive upper bound tests
+
+  // ======= Testing 3rd segment (cross-category filter) =======
+  // Control 5: not filtered because of segment range overlap
+  EXPECT_EQ(RangeQueryKeys(" z__700", " z__750"), Keys({}));
+  EXPECT_EQ(TestGetAndResetTickerCount(options, NON_LAST_LEVEL_SEEK_DATA), 2);
+
+  // Test 6: filtered on both levels
+  EXPECT_EQ(RangeQueryKeys(" z__100", " z__300"), Keys({}));
+  EXPECT_EQ(TestGetAndResetTickerCount(options, NON_LAST_LEVEL_SEEK_DATA), 0);
+
+  // Control 6: finding something, with 2nd segment filter helping
+  EXPECT_EQ(RangeQueryKeys("abc_156_9", "abc_156_99"), Keys({"abc_156_987"}));
+  EXPECT_EQ(TestGetAndResetTickerCount(options, NON_LAST_LEVEL_SEEK_DATA), 1);
+
+  EXPECT_EQ(RangeQueryKeys("abc_245_56", "abc_245_57"), Keys({"abc_245_567"}));
+  EXPECT_EQ(TestGetAndResetTickerCount(options, NON_LAST_LEVEL_SEEK_DATA), 1);
+
+  // Test 6: filtered on both levels, for different segments
+  EXPECT_EQ(RangeQueryKeys("abc_245_900", "abc_245_999"), Keys({}));
+  EXPECT_EQ(TestGetAndResetTickerCount(options, NON_LAST_LEVEL_SEEK_DATA), 0);
+
+  // ======= Testing extractor read portability =======
+  EXPECT_EQ(RangeQueryKeys("abc_300", "abc_400"), Keys({}));
+  EXPECT_EQ(TestGetAndResetTickerCount(options, NON_LAST_LEVEL_SEEK_DATA), 0);
+
+  // Only modifies how filters are written
+  ASSERT_OK(factory->SetFilteringVersion(0));
+  ASSERT_EQ(factory->GetFilteringVersion(), 0U);
+  ASSERT_EQ(factory->GetConfigs().IsEmptyNotFound(), true);
+
+  EXPECT_EQ(RangeQueryKeys("abc_300", "abc_400"), Keys({}));
+  EXPECT_EQ(TestGetAndResetTickerCount(options, NON_LAST_LEVEL_SEEK_DATA), 0);
+
+  // Even a different config name with different extractor can read
+  EXPECT_EQ(RangeQueryKeys("abc_300", "abc_400", MakeFactory("bar", 43)),
+            Keys({}));
+  EXPECT_EQ(TestGetAndResetTickerCount(options, NON_LAST_LEVEL_SEEK_DATA), 0);
+
+  // Or a "not found" config name
+  EXPECT_EQ(RangeQueryKeys("abc_300", "abc_400", MakeFactory("blah", 43)),
+            Keys({}));
+  EXPECT_EQ(TestGetAndResetTickerCount(options, NON_LAST_LEVEL_SEEK_DATA), 0);
+}

 }  // namespace ROCKSDB_NAMESPACE

@@ -150,7 +150,7 @@ class ConditionalFilter : public CompactionFilter {

 class ChangeFilter : public CompactionFilter {
 public:
-  explicit ChangeFilter() {}
+  explicit ChangeFilter() = default;

  bool Filter(int /*level*/, const Slice& /*key*/, const Slice& /*value*/,
              std::string* new_value, bool* value_changed) const override {
@@ -289,7 +289,7 @@ class ConditionalFilterFactory : public CompactionFilterFactory {

 class ChangeFilterFactory : public CompactionFilterFactory {
 public:
-  explicit ChangeFilterFactory() {}
+  explicit ChangeFilterFactory() = default;

  std::unique_ptr<CompactionFilter> CreateCompactionFilter(
      const CompactionFilter::Context& /*context*/) override {
@@ -342,7 +342,7 @@ TEST_F(DBTestCompactionFilter, CompactionFilter) {
  {
    InternalKeyComparator icmp(options.comparator);
    ReadOptions read_options;
-    ScopedArenaIterator iter(dbfull()->NewInternalIterator(
+    ScopedArenaPtr<InternalIterator> iter(dbfull()->NewInternalIterator(
        read_options, &arena, kMaxSequenceNumber, handles_[1]));
    iter->SeekToFirst();
    ASSERT_OK(iter->status());
@@ -434,7 +434,7 @@ TEST_F(DBTestCompactionFilter, CompactionFilter) {
  {
    InternalKeyComparator icmp(options.comparator);
    ReadOptions read_options;
-    ScopedArenaIterator iter(dbfull()->NewInternalIterator(
+    ScopedArenaPtr<InternalIterator> iter(dbfull()->NewInternalIterator(
        read_options, &arena, kMaxSequenceNumber, handles_[1]));
    iter->SeekToFirst();
    ASSERT_OK(iter->status());
@@ -717,8 +717,8 @@ TEST_F(DBTestCompactionFilter, CompactionFilterContextManual) {
    Arena arena;
    InternalKeyComparator icmp(options.comparator);
    ReadOptions read_options;
-    ScopedArenaIterator iter(dbfull()->NewInternalIterator(read_options, &arena,
-                                                           kMaxSequenceNumber));
+    ScopedArenaPtr<InternalIterator> iter(dbfull()->NewInternalIterator(
+        read_options, &arena, kMaxSequenceNumber));
    iter->SeekToFirst();
    ASSERT_OK(iter->status());
    while (iter->Valid()) {
@@ -41,7 +41,7 @@ class CompactionStatsCollector : public EventListener {
    }
  }

-  ~CompactionStatsCollector() override {}
+  ~CompactionStatsCollector() override = default;

  void OnCompactionCompleted(DB* /* db */,
                             const CompactionJobInfo& info) override {
@@ -241,8 +241,8 @@ class RoundRobinSubcompactionsAgainstResources
 namespace {
 class FlushedFileCollector : public EventListener {
 public:
-  FlushedFileCollector() {}
-  ~FlushedFileCollector() override {}
+  FlushedFileCollector() = default;
+  ~FlushedFileCollector() override = default;

  void OnFlushCompleted(DB* /*db*/, const FlushJobInfo& info) override {
    std::lock_guard<std::mutex> lock(mutex_);
@@ -252,7 +252,7 @@ class FlushedFileCollector : public EventListener {
  std::vector<std::string> GetFlushedFiles() {
    std::lock_guard<std::mutex> lock(mutex_);
    std::vector<std::string> result;
-    for (auto fname : flushed_files_) {
+    for (const auto& fname : flushed_files_) {
      result.push_back(fname);
    }
    return result;
@@ -505,7 +505,7 @@ TEST_F(DBCompactionTest, TestTableReaderForCompaction) {
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "TableCache::FindTable:0", [&](void* arg) {
        assert(arg != nullptr);
-        bool no_io = *(reinterpret_cast<bool*>(arg));
+        bool no_io = *(static_cast<bool*>(arg));
        if (!no_io) {
          // filter out cases for table properties queries.
          num_table_cache_lookup++;
@@ -681,7 +681,7 @@ TEST_F(DBCompactionTest, CompactRangeBottomPri) {
  int bottom_pri_count = 0;
  SyncPoint::GetInstance()->SetCallBack(
      "ThreadPoolImpl::Impl::BGThread:BeforeRun", [&](void* arg) {
-        Env::Priority* pri = reinterpret_cast<Env::Priority*>(arg);
+        Env::Priority* pri = static_cast<Env::Priority*>(arg);
        // First time is low pri pool in the test case.
        if (low_pri_count == 0 && bottom_pri_count == 0) {
          ASSERT_EQ(Env::Priority::LOW, *pri);
@@ -1870,13 +1870,60 @@ TEST_F(DBCompactionTest, ManualCompactionWithUnorderedWrite) {
  ASSERT_EQ(Get("foo"), "v2");
 }

-TEST_F(DBCompactionTest, DeleteFileRange) {
+// Test params:
+// 1) whether to enable user-defined timestamps.
+class DBDeleteFileRangeTest : public DBTestBase,
+                              public testing::WithParamInterface<bool> {
+ public:
+  DBDeleteFileRangeTest()
+      : DBTestBase("db_delete_file_range_test", /*env_do_fsync=*/true) {}
+
+  void SetUp() override { enable_udt_ = GetParam(); }
+
+ protected:
+  void PutKeyValue(const Slice& key, const Slice& value) {
+    if (enable_udt_) {
+      EXPECT_OK(db_->Put(WriteOptions(), key, min_ts_, value));
+    } else {
+      EXPECT_OK(Put(key, value));
+    }
+  }
+
+  std::string GetValue(const std::string& key) {
+    ReadOptions roptions;
+    std::string result;
+    if (enable_udt_) {
+      roptions.timestamp = &min_ts_;
+    }
+    Status s = db_->Get(roptions, key, &result);
+    EXPECT_TRUE(s.ok());
+    return result;
+  }
+
+  Status MaybeGetValue(const std::string& key, std::string* result) {
+    ReadOptions roptions;
+    if (enable_udt_) {
+      roptions.timestamp = &min_ts_;
+    }
+    Status s = db_->Get(roptions, key, result);
+    EXPECT_TRUE(s.IsNotFound() || s.ok());
+    return s;
+  }
+
+  bool enable_udt_ = false;
+  Slice min_ts_ = MinU64Ts();
+};
+
+TEST_P(DBDeleteFileRangeTest, DeleteFileRange) {
  Options options = CurrentOptions();
  options.write_buffer_size = 10 * 1024 * 1024;
  options.max_bytes_for_level_multiplier = 2;
  options.num_levels = 4;
  options.level0_file_num_compaction_trigger = 3;
  options.max_background_compactions = 3;
+  if (enable_udt_) {
+    options.comparator = test::BytewiseComparatorWithU64TsWrapper();
+  }

  DestroyAndReopen(options);
  int32_t value_size = 10 * 1024;  // 10 KB
@@ -1888,14 +1935,14 @@ TEST_F(DBCompactionTest, DeleteFileRange) {
  // file 1 [0 => 100]
  for (int32_t i = 0; i < 100; i++) {
    values[i] = rnd.RandomString(value_size);
-    ASSERT_OK(Put(Key(i), values[i]));
+    PutKeyValue(Key(i), values[i]);
  }
  ASSERT_OK(Flush());

  // file 2 [100 => 300]
  for (int32_t i = 100; i < 300; i++) {
    values[i] = rnd.RandomString(value_size);
-    ASSERT_OK(Put(Key(i), values[i]));
+    PutKeyValue(Key(i), values[i]);
  }
  ASSERT_OK(Flush());

@@ -1911,7 +1958,7 @@ TEST_F(DBCompactionTest, DeleteFileRange) {
  // file 3 [ 0 => 200]
  for (int32_t i = 0; i < 200; i++) {
    values[i] = rnd.RandomString(value_size);
-    ASSERT_OK(Put(Key(i), values[i]));
+    PutKeyValue(Key(i), values[i]);
  }
  ASSERT_OK(Flush());

@@ -1923,7 +1970,7 @@ TEST_F(DBCompactionTest, DeleteFileRange) {
        ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
      }
      values[j] = rnd.RandomString(value_size);
-      ASSERT_OK(Put(Key(j), values[j]));
+      PutKeyValue(Key(j), values[j]);
    }
  }
  ASSERT_OK(Flush());
@@ -1948,11 +1995,10 @@ TEST_F(DBCompactionTest, DeleteFileRange) {
  int32_t deleted_count = 0;
  for (int32_t i = 0; i < 4300; i++) {
    if (i < 1000 || i > 2000) {
-      ASSERT_EQ(Get(Key(i)), values[i]);
+      ASSERT_EQ(GetValue(Key(i)), values[i]);
    } else {
-      ReadOptions roptions;
      std::string result;
-      Status s = db_->Get(roptions, Key(i), &result);
+      Status s = MaybeGetValue(Key(i), &result);
      ASSERT_TRUE(s.IsNotFound() || s.ok());
      if (s.IsNotFound()) {
        deleted_count++;
@@ -1982,7 +2028,7 @@ TEST_F(DBCompactionTest, DeleteFileRange) {
  for (int32_t i = 0; i < 4300; i++) {
    ReadOptions roptions;
    std::string result;
-    ASSERT_TRUE(db_->Get(roptions, Key(i), &result).IsNotFound());
+    ASSERT_TRUE(MaybeGetValue(Key(i), &result).IsNotFound());
    deleted_count2++;
  }
  ASSERT_GT(deleted_count2, deleted_count);
@@ -1990,13 +2036,16 @@ TEST_F(DBCompactionTest, DeleteFileRange) {
  ASSERT_GT(old_num_files, new_num_files);
 }

-TEST_F(DBCompactionTest, DeleteFilesInRanges) {
+TEST_P(DBDeleteFileRangeTest, DeleteFilesInRanges) {
  Options options = CurrentOptions();
  options.write_buffer_size = 10 * 1024 * 1024;
  options.max_bytes_for_level_multiplier = 2;
  options.num_levels = 4;
  options.max_background_compactions = 3;
  options.disable_auto_compactions = true;
+  if (enable_udt_) {
+    options.comparator = test::BytewiseComparatorWithU64TsWrapper();
+  }

  DestroyAndReopen(options);
  int32_t value_size = 10 * 1024;  // 10 KB
@@ -2009,7 +2058,7 @@ TEST_F(DBCompactionTest, DeleteFilesInRanges) {
    for (auto j = 0; j < 100; j++) {
      auto k = i * 100 + j;
      values[k] = rnd.RandomString(value_size);
-      ASSERT_OK(Put(Key(k), values[k]));
+      PutKeyValue(Key(k), values[k]);
    }
    ASSERT_OK(Flush());
  }
@@ -2024,7 +2073,7 @@ TEST_F(DBCompactionTest, DeleteFilesInRanges) {
  for (auto i = 0; i < 10; i += 2) {
    for (auto j = 0; j < 100; j++) {
      auto k = i * 100 + j;
-      ASSERT_OK(Put(Key(k), values[k]));
+      PutKeyValue(Key(k), values[k]);
    }
    ASSERT_OK(Flush());
  }
@@ -2041,22 +2090,21 @@ TEST_F(DBCompactionTest, DeleteFilesInRanges) {
    Slice begin2(begin_str2), end2(end_str2);
    Slice begin3(begin_str3), end3(end_str3);
    std::vector<RangePtr> ranges;
-    ranges.push_back(RangePtr(&begin1, &end1));
-    ranges.push_back(RangePtr(&begin2, &end2));
-    ranges.push_back(RangePtr(&begin3, &end3));
+    ranges.emplace_back(&begin1, &end1);
+    ranges.emplace_back(&begin2, &end2);
+    ranges.emplace_back(&begin3, &end3);
    ASSERT_OK(DeleteFilesInRanges(db_, db_->DefaultColumnFamily(),
                                  ranges.data(), ranges.size()));
    ASSERT_EQ("0,3,7", FilesPerLevel(0));

    // Keys [0, 300) should not exist.
    for (auto i = 0; i < 300; i++) {
-      ReadOptions ropts;
      std::string result;
-      auto s = db_->Get(ropts, Key(i), &result);
+      auto s = MaybeGetValue(Key(i), &result);
      ASSERT_TRUE(s.IsNotFound());
    }
    for (auto i = 300; i < 1000; i++) {
-      ASSERT_EQ(Get(Key(i)), values[i]);
+      ASSERT_EQ(GetValue(Key(i)), values[i]);
    }
  }

@@ -2069,25 +2117,24 @@ TEST_F(DBCompactionTest, DeleteFilesInRanges) {
    Slice begin2(begin_str2), end2(end_str2);
    Slice begin3(begin_str3), end3(end_str3);
    std::vector<RangePtr> ranges;
-    ranges.push_back(RangePtr(&begin1, &end1));
-    ranges.push_back(RangePtr(&begin2, &end2));
-    ranges.push_back(RangePtr(&begin3, &end3));
+    ranges.emplace_back(&begin1, &end1);
+    ranges.emplace_back(&begin2, &end2);
+    ranges.emplace_back(&begin3, &end3);
    ASSERT_OK(DeleteFilesInRanges(db_, db_->DefaultColumnFamily(),
                                  ranges.data(), ranges.size(), false));
    ASSERT_EQ("0,1,4", FilesPerLevel(0));

    // Keys [600, 900) should not exist.
    for (auto i = 600; i < 900; i++) {
-      ReadOptions ropts;
      std::string result;
-      auto s = db_->Get(ropts, Key(i), &result);
+      auto s = MaybeGetValue(Key(i), &result);
      ASSERT_TRUE(s.IsNotFound());
    }
    for (auto i = 300; i < 600; i++) {
-      ASSERT_EQ(Get(Key(i)), values[i]);
+      ASSERT_EQ(GetValue(Key(i)), values[i]);
    }
    for (auto i = 900; i < 1000; i++) {
-      ASSERT_EQ(Get(Key(i)), values[i]);
+      ASSERT_EQ(GetValue(Key(i)), values[i]);
    }
  }

@@ -2098,15 +2145,14 @@ TEST_F(DBCompactionTest, DeleteFilesInRanges) {
    ASSERT_EQ("", FilesPerLevel(0));

    for (auto i = 0; i < 1000; i++) {
-      ReadOptions ropts;
      std::string result;
-      auto s = db_->Get(ropts, Key(i), &result);
+      auto s = MaybeGetValue(Key(i), &result);
      ASSERT_TRUE(s.IsNotFound());
    }
  }
 }

-TEST_F(DBCompactionTest, DeleteFileRangeFileEndpointsOverlapBug) {
+TEST_P(DBDeleteFileRangeTest, DeleteFileRangeFileEndpointsOverlapBug) {
  // regression test for #2833: groups of files whose user-keys overlap at the
  // endpoints could be split by `DeleteFilesInRange`. This caused old data to
  // reappear, either because a new version of the key was removed, or a range
@@ -2118,6 +2164,9 @@ TEST_F(DBCompactionTest, DeleteFileRangeFileEndpointsOverlapBug) {
  Options options = CurrentOptions();
  options.level0_file_num_compaction_trigger = kNumL0Files;
  options.target_file_size_base = 1 << 10;  // 1KB
+  if (enable_udt_) {
+    options.comparator = test::BytewiseComparatorWithU64TsWrapper();
+  }
  DestroyAndReopen(options);

  // The snapshot prevents key 1 from having its old version dropped. The low
@@ -2141,8 +2190,8 @@ TEST_F(DBCompactionTest, DeleteFileRangeFileEndpointsOverlapBug) {
  std::string vals[kNumL0Files];
  for (int i = 0; i < kNumL0Files; ++i) {
    vals[i] = rnd.RandomString(kValSize);
-    ASSERT_OK(Put(Key(i), vals[i]));
-    ASSERT_OK(Put(Key(i + 1), vals[i]));
+    PutKeyValue(Key(i), vals[i]);
+    PutKeyValue(Key(i + 1), vals[i]);
    ASSERT_OK(Flush());
    if (i == 0) {
      snapshot = db_->GetSnapshot();
@@ -2155,11 +2204,14 @@ TEST_F(DBCompactionTest, DeleteFileRangeFileEndpointsOverlapBug) {
  std::string begin_str = Key(0), end_str = Key(1);
  Slice begin = begin_str, end = end_str;
  ASSERT_OK(DeleteFilesInRange(db_, db_->DefaultColumnFamily(), &begin, &end));
-  ASSERT_EQ(vals[1], Get(Key(1)));
+  ASSERT_EQ(vals[1], GetValue(Key(1)));

  db_->ReleaseSnapshot(snapshot);
 }

+INSTANTIATE_TEST_CASE_P(DBDeleteFileRangeTest, DBDeleteFileRangeTest,
+                        ::testing::Bool());
+
 TEST_P(DBCompactionTestWithParam, TrivialMoveToLastLevelWithFiles) {
  int32_t trivial_move = 0;
  int32_t non_trivial_move = 0;
@@ -3945,7 +3997,7 @@ TEST_P(DBCompactionTestWithParam, FullCompactionInBottomPriThreadPool) {
  Env::Default()->SetBackgroundThreads(0, Env::Priority::BOTTOM);
 }

-TEST_F(DBCompactionTest, CancelCompactionWaitingOnConflict) {
+TEST_F(DBCompactionTest, CancelCompactionWaitingOnRunningConflict) {
  // This test verifies cancellation of a compaction waiting to be scheduled due
  // to conflict with a running compaction.
  //
@@ -3984,7 +4036,7 @@ TEST_F(DBCompactionTest, CancelCompactionWaitingOnConflict) {
  // Make sure the manual compaction has seen the conflict before being canceled
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
      {{"ColumnFamilyData::CompactRange:Return",
-        "DBCompactionTest::CancelCompactionWaitingOnConflict:"
+        "DBCompactionTest::CancelCompactionWaitingOnRunningConflict:"
        "PreDisableManualCompaction"}});
  auto manual_compaction_thread = port::Thread([this]() {
    ASSERT_TRUE(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)
@@ -3995,12 +4047,73 @@ TEST_F(DBCompactionTest, CancelCompactionWaitingOnConflict) {
  // despite finding a conflict with an automatic compaction that is still
  // running
  TEST_SYNC_POINT(
-      "DBCompactionTest::CancelCompactionWaitingOnConflict:"
+      "DBCompactionTest::CancelCompactionWaitingOnRunningConflict:"
      "PreDisableManualCompaction");
  db_->DisableManualCompaction();
  manual_compaction_thread.join();
 }

+TEST_F(DBCompactionTest, CancelCompactionWaitingOnScheduledConflict) {
+  // This test verifies cancellation of a compaction waiting to be scheduled due
+  // to conflict with a scheduled (but not running) compaction.
+  //
+  // A `CompactRange()` in universal compacts all files, waiting for files to
+  // become available if they are locked for another compaction. This test
+  // blocks the compaction thread pool and then calls `CompactRange()` twice.
+  // The first call to `CompactRange()` schedules a compaction that is queued
+  // in the thread pool. The second call to `CompactRange()` blocks on the first
+  // call due to the conflict in file picking. The test verifies that
+  // `DisableManualCompaction()` can cancel both while the thread pool remains
+  // blocked.
+  const int kNumSortedRuns = 4;
+
+  Options options = CurrentOptions();
+  options.compaction_style = kCompactionStyleUniversal;
+  options.disable_auto_compactions = true;
+  options.memtable_factory.reset(
+      test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
+  Reopen(options);
+
+  test::SleepingBackgroundTask sleeping_task_low;
+  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
+                 Env::Priority::LOW);
+
+  // Fill overlapping files in L0
+  Random rnd(301);
+  for (int i = 0; i < kNumSortedRuns; ++i) {
+    int key_idx = 0;
+    GenerateNewFile(&rnd, &key_idx, false /* nowait */);
+  }
+
+  std::atomic<int> num_compact_range_calls{0};
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "ColumnFamilyData::CompactRange:Return",
+      [&](void* /* arg */) { num_compact_range_calls++; });
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  const int kNumManualCompactions = 2;
+  port::Thread manual_compaction_threads[kNumManualCompactions];
+  for (int i = 0; i < kNumManualCompactions; i++) {
+    manual_compaction_threads[i] = port::Thread([this]() {
+      ASSERT_TRUE(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)
+                      .IsIncomplete());
+    });
+  }
+  while (num_compact_range_calls < kNumManualCompactions) {
+  }
+
+  // Cancel it. Threads should be joinable, i.e., both the scheduled and blocked
+  // manual compactions were canceled despite no compaction could have ever run.
+  db_->DisableManualCompaction();
+  for (int i = 0; i < kNumManualCompactions; i++) {
+    manual_compaction_threads[i].join();
+  }
+
+  sleeping_task_low.WakeUp();
+  sleeping_task_low.WaitUntilDone();
+}
+
 TEST_F(DBCompactionTest, OptimizedDeletionObsoleting) {
  // Deletions can be dropped when compacted to non-last level if they fall
  // outside the lower-level files' key-ranges.
@@ -4192,7 +4305,7 @@ TEST_F(DBCompactionTest, CompactBottomLevelFilesWithDeletions) {
  ASSERT_NE(kMaxSequenceNumber, dbfull()->bottommost_files_mark_threshold_);
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) {
-        Compaction* compaction = reinterpret_cast<Compaction*>(arg);
+        Compaction* compaction = static_cast<Compaction*>(arg);
        ASSERT_TRUE(compaction->compaction_reason() ==
                    CompactionReason::kBottommostFiles);
      });
@@ -4248,7 +4361,7 @@ TEST_F(DBCompactionTest, DelayCompactBottomLevelFilesWithDeletions) {
  std::atomic_int compaction_count = 0;
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) {
-        Compaction* compaction = reinterpret_cast<Compaction*>(arg);
+        Compaction* compaction = static_cast<Compaction*>(arg);
        ASSERT_TRUE(compaction->compaction_reason() ==
                    CompactionReason::kBottommostFiles);
        compaction_count++;
@@ -4379,7 +4492,7 @@ TEST_F(DBCompactionTest, RoundRobinTtlCompactionNormal) {

  SyncPoint::GetInstance()->SetCallBack(
      "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) {
-        Compaction* compaction = reinterpret_cast<Compaction*>(arg);
+        Compaction* compaction = static_cast<Compaction*>(arg);
        auto compaction_reason = compaction->compaction_reason();
        if (compaction_reason == CompactionReason::kTtl) {
          ttl_compactions++;
@@ -4529,7 +4642,7 @@ TEST_F(DBCompactionTest, RoundRobinTtlCompactionUnsortedTime) {

  SyncPoint::GetInstance()->SetCallBack(
      "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) {
-        Compaction* compaction = reinterpret_cast<Compaction*>(arg);
+        Compaction* compaction = static_cast<Compaction*>(arg);
        auto compaction_reason = compaction->compaction_reason();
        if (compaction_reason == CompactionReason::kTtl) {
          ttl_compactions++;
@@ -4645,7 +4758,7 @@ TEST_F(DBCompactionTest, LevelCompactExpiredTtlFiles) {
  ASSERT_OK(Flush());
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) {
-        Compaction* compaction = reinterpret_cast<Compaction*>(arg);
+        Compaction* compaction = static_cast<Compaction*>(arg);
        ASSERT_TRUE(compaction->compaction_reason() == CompactionReason::kTtl);
      });
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
@@ -4693,7 +4806,7 @@ TEST_F(DBCompactionTest, LevelCompactExpiredTtlFiles) {

  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) {
-        Compaction* compaction = reinterpret_cast<Compaction*>(arg);
+        Compaction* compaction = static_cast<Compaction*>(arg);
        ASSERT_TRUE(compaction->compaction_reason() == CompactionReason::kTtl);
      });
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
@@ -4720,7 +4833,6 @@ TEST_F(DBCompactionTest, LevelTtlCompactionOutputCuttingIteractingWithOther) {
  options.env = env_;
  options.target_file_size_base = 4 << 10;
  options.disable_auto_compactions = true;
-  options.level_compaction_dynamic_file_size = false;

  DestroyAndReopen(options);
  Random rnd(301);
@@ -4821,7 +4933,7 @@ TEST_F(DBCompactionTest, LevelTtlCascadingCompactions) {
      int ttl_compactions = 0;
      ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
          "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) {
-            Compaction* compaction = reinterpret_cast<Compaction*>(arg);
+            Compaction* compaction = static_cast<Compaction*>(arg);
            auto compaction_reason = compaction->compaction_reason();
            if (compaction_reason == CompactionReason::kTtl) {
              ttl_compactions++;
@@ -4969,7 +5081,7 @@ TEST_F(DBCompactionTest, LevelPeriodicCompaction) {
      int periodic_compactions = 0;
      ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
          "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) {
-            Compaction* compaction = reinterpret_cast<Compaction*>(arg);
+            Compaction* compaction = static_cast<Compaction*>(arg);
            auto compaction_reason = compaction->compaction_reason();
            if (compaction_reason == CompactionReason::kPeriodicCompaction) {
              periodic_compactions++;
@@ -5153,7 +5265,7 @@ TEST_F(DBCompactionTest, LevelPeriodicCompactionWithOldDB) {
  bool set_creation_time_to_zero = true;
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) {
-        Compaction* compaction = reinterpret_cast<Compaction*>(arg);
+        Compaction* compaction = static_cast<Compaction*>(arg);
        auto compaction_reason = compaction->compaction_reason();
        if (compaction_reason == CompactionReason::kPeriodicCompaction) {
          periodic_compactions++;
@@ -5161,7 +5273,7 @@ TEST_F(DBCompactionTest, LevelPeriodicCompactionWithOldDB) {
      });
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "PropertyBlockBuilder::AddTableProperty:Start", [&](void* arg) {
-        TableProperties* props = reinterpret_cast<TableProperties*>(arg);
+        TableProperties* props = static_cast<TableProperties*>(arg);
        if (set_file_creation_time_to_zero) {
          props->file_creation_time = 0;
        }
@@ -5225,7 +5337,7 @@ TEST_F(DBCompactionTest, LevelPeriodicAndTtlCompaction) {
  int ttl_compactions = 0;
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) {
-        Compaction* compaction = reinterpret_cast<Compaction*>(arg);
+        Compaction* compaction = static_cast<Compaction*>(arg);
        auto compaction_reason = compaction->compaction_reason();
        if (compaction_reason == CompactionReason::kPeriodicCompaction) {
          periodic_compactions++;
@@ -5408,7 +5520,7 @@ TEST_F(DBCompactionTest, LevelPeriodicCompactionWithCompactionFilters) {
    int periodic_compactions = 0;
    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
        "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) {
-          Compaction* compaction = reinterpret_cast<Compaction*>(arg);
+          Compaction* compaction = static_cast<Compaction*>(arg);
          auto compaction_reason = compaction->compaction_reason();
          if (compaction_reason == CompactionReason::kPeriodicCompaction) {
            periodic_compactions++;
@@ -6590,7 +6702,7 @@ TEST_F(DBCompactionTest, RoundRobinCutOutputAtCompactCursor) {

 class NoopMergeOperator : public MergeOperator {
 public:
-  NoopMergeOperator() {}
+  NoopMergeOperator() = default;

  bool FullMergeV2(const MergeOperationInput& /*merge_in*/,
                   MergeOperationOutput* merge_out) const override {
@@ -7146,8 +7258,7 @@ TEST_F(DBCompactionTest, ConsistencyFailTest) {

  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "VersionBuilder::CheckConsistency0", [&](void* arg) {
-        auto p =
-            reinterpret_cast<std::pair<FileMetaData**, FileMetaData**>*>(arg);
+        auto p = static_cast<std::pair<FileMetaData**, FileMetaData**>*>(arg);
        // just swap the two FileMetaData so that we hit error
        // in CheckConsistency funcion
        FileMetaData* temp = *(p->first);
@@ -7184,8 +7295,7 @@ TEST_F(DBCompactionTest, ConsistencyFailTest2) {

  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "VersionBuilder::CheckConsistency1", [&](void* arg) {
-        auto p =
-            reinterpret_cast<std::pair<FileMetaData**, FileMetaData**>*>(arg);
+        auto p = static_cast<std::pair<FileMetaData**, FileMetaData**>*>(arg);
        // just swap the two FileMetaData so that we hit error
        // in CheckConsistency funcion
        FileMetaData* temp = *(p->first);
@@ -7998,7 +8108,7 @@ TEST_F(DBCompactionTest, UpdateLevelSubCompactionTest) {
  bool has_compaction = false;
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) {
-        Compaction* compaction = reinterpret_cast<Compaction*>(arg);
+        Compaction* compaction = static_cast<Compaction*>(arg);
        ASSERT_TRUE(compaction->max_subcompactions() == 10);
        has_compaction = true;
      });
@@ -8022,7 +8132,7 @@ TEST_F(DBCompactionTest, UpdateLevelSubCompactionTest) {

  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) {
-        Compaction* compaction = reinterpret_cast<Compaction*>(arg);
+        Compaction* compaction = static_cast<Compaction*>(arg);
        ASSERT_TRUE(compaction->max_subcompactions() == 2);
        has_compaction = true;
      });
@@ -8050,7 +8160,7 @@ TEST_F(DBCompactionTest, UpdateUniversalSubCompactionTest) {
  bool has_compaction = false;
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "UniversalCompactionBuilder::PickCompaction:Return", [&](void* arg) {
-        Compaction* compaction = reinterpret_cast<Compaction*>(arg);
+        Compaction* compaction = static_cast<Compaction*>(arg);
        ASSERT_TRUE(compaction->max_subcompactions() == 10);
        has_compaction = true;
      });
@@ -8073,7 +8183,7 @@ TEST_F(DBCompactionTest, UpdateUniversalSubCompactionTest) {

  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "UniversalCompactionBuilder::PickCompaction:Return", [&](void* arg) {
-        Compaction* compaction = reinterpret_cast<Compaction*>(arg);
+        Compaction* compaction = static_cast<Compaction*>(arg);
        ASSERT_TRUE(compaction->max_subcompactions() == 2);
        has_compaction = true;
      });
@@ -9115,66 +9225,104 @@ TEST_F(DBCompactionTest, CompactionWithChecksumHandoffManifest2) {
 }

 TEST_F(DBCompactionTest, FIFOChangeTemperature) {
-  Options options = CurrentOptions();
-  options.compaction_style = kCompactionStyleFIFO;
-  options.num_levels = 1;
-  options.max_open_files = -1;
-  options.level0_file_num_compaction_trigger = 2;
-  options.create_if_missing = true;
-  CompactionOptionsFIFO fifo_options;
-  fifo_options.file_temperature_age_thresholds = {{Temperature::kCold, 1000}};
-  fifo_options.max_table_files_size = 100000000;
-  options.compaction_options_fifo = fifo_options;
-  env_->SetMockSleep();
-  Reopen(options);
+  for (bool write_time_default : {false, true}) {
+    SCOPED_TRACE("write time default? " + std::to_string(write_time_default));

-  int total_cold = 0;
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
-      "NewWritableFile::FileOptions.temperature", [&](void* arg) {
-        Temperature temperature = *(static_cast<Temperature*>(arg));
-        if (temperature == Temperature::kCold) {
-          total_cold++;
-        }
-      });
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+    Options options = CurrentOptions();
+    options.compaction_style = kCompactionStyleFIFO;
+    options.num_levels = 1;
+    options.max_open_files = -1;
+    options.level0_file_num_compaction_trigger = 2;
+    options.create_if_missing = true;
+    CompactionOptionsFIFO fifo_options;
+    fifo_options.file_temperature_age_thresholds = {{Temperature::kCold, 1000}};
+    fifo_options.max_table_files_size = 100000000;
+    options.compaction_options_fifo = fifo_options;
+    env_->SetMockSleep();
+    if (write_time_default) {
+      options.default_write_temperature = Temperature::kWarm;
+    }
+    // Should be ignored (TODO: fail?)
+    options.last_level_temperature = Temperature::kHot;
+    Reopen(options);

-  // The file system does not support checksum handoff. The check
-  // will be ignored.
-  ASSERT_OK(Put(Key(0), "value1"));
-  env_->MockSleepForSeconds(800);
-  ASSERT_OK(Put(Key(2), "value2"));
-  ASSERT_OK(Flush());
+    int total_cold = 0;
+    int total_warm = 0;
+    int total_hot = 0;
+    int total_unknown = 0;
+    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+        "NewWritableFile::FileOptions.temperature", [&](void* arg) {
+          Temperature temperature = *(static_cast<Temperature*>(arg));
+          if (temperature == Temperature::kCold) {
+            total_cold++;
+          } else if (temperature == Temperature::kWarm) {
+            total_warm++;
+          } else if (temperature == Temperature::kHot) {
+            total_hot++;
+          } else {
+            assert(temperature == Temperature::kUnknown);
+            total_unknown++;
+          }
+        });
+    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();

-  ASSERT_OK(Put(Key(0), "value1"));
-  env_->MockSleepForSeconds(800);
-  ASSERT_OK(Put(Key(2), "value2"));
-  ASSERT_OK(Flush());
+    // The file system does not support checksum handoff. The check
+    // will be ignored.
+    ASSERT_OK(Put(Key(0), "value1"));
+    env_->MockSleepForSeconds(800);
+    ASSERT_OK(Put(Key(2), "value2"));
+    ASSERT_OK(Flush());

-  ASSERT_OK(Put(Key(0), "value1"));
-  env_->MockSleepForSeconds(800);
-  ASSERT_OK(Put(Key(2), "value2"));
-  ASSERT_OK(Flush());
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+    ASSERT_OK(Put(Key(0), "value1"));
+    env_->MockSleepForSeconds(800);
+    ASSERT_OK(Put(Key(2), "value2"));
+    ASSERT_OK(Flush());

-  ASSERT_OK(Put(Key(0), "value1"));
-  env_->MockSleepForSeconds(800);
-  ASSERT_OK(Put(Key(2), "value2"));
-  ASSERT_OK(Flush());
+    ASSERT_OK(Put(Key(0), "value1"));
+    env_->MockSleepForSeconds(800);
+    ASSERT_OK(Put(Key(2), "value2"));
+    ASSERT_OK(Flush());
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());

-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+    if (write_time_default) {
+      // Also test dynamic option change
+      ASSERT_OK(db_->SetOptions({{"default_write_temperature", "kHot"}}));
+    }

-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+    ASSERT_OK(Put(Key(0), "value1"));
+    env_->MockSleepForSeconds(800);
+    ASSERT_OK(Put(Key(2), "value2"));
+    ASSERT_OK(Flush());

-  ColumnFamilyMetaData metadata;
-  db_->GetColumnFamilyMetaData(&metadata);
-  ASSERT_EQ(4, metadata.file_count);
-  ASSERT_EQ(Temperature::kUnknown, metadata.levels[0].files[0].temperature);
-  ASSERT_EQ(Temperature::kUnknown, metadata.levels[0].files[1].temperature);
-  ASSERT_EQ(Temperature::kCold, metadata.levels[0].files[2].temperature);
-  ASSERT_EQ(Temperature::kCold, metadata.levels[0].files[3].temperature);
-  ASSERT_EQ(2, total_cold);
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());

-  Destroy(options);
+    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+
+    ColumnFamilyMetaData metadata;
+    db_->GetColumnFamilyMetaData(&metadata);
+    ASSERT_EQ(4, metadata.file_count);
+    if (write_time_default) {
+      ASSERT_EQ(Temperature::kHot, metadata.levels[0].files[0].temperature);
+      ASSERT_EQ(Temperature::kWarm, metadata.levels[0].files[1].temperature);
+      // Includes obsolete/deleted files moved to cold
+      ASSERT_EQ(total_warm, 3);
+      ASSERT_EQ(total_hot, 1);
+      // Includes non-SST DB files
+      ASSERT_GT(total_unknown, 0);
+    } else {
+      ASSERT_EQ(Temperature::kUnknown, metadata.levels[0].files[0].temperature);
+      ASSERT_EQ(Temperature::kUnknown, metadata.levels[0].files[1].temperature);
+      ASSERT_EQ(total_warm, 0);
+      ASSERT_EQ(total_hot, 0);
+      // Includes non-SST DB files
+      ASSERT_GT(total_unknown, 4);
+    }
+    ASSERT_EQ(Temperature::kCold, metadata.levels[0].files[2].temperature);
+    ASSERT_EQ(Temperature::kCold, metadata.levels[0].files[3].temperature);
+    ASSERT_EQ(2, total_cold);
+
+    Destroy(options);
+  }
 }

 TEST_F(DBCompactionTest, DisableMultiManualCompaction) {
@@ -9650,6 +9798,7 @@ TEST_F(DBCompactionTest, BottomPriCompactionCountsTowardConcurrencyLimit) {
  test::SleepingBackgroundTask sleeping_task_low;
  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
                 Env::Priority::LOW);
+  sleeping_task_low.WaitUntilSleeping();

  TEST_SYNC_POINT(
      "DBCompactionTest::BottomPriCompactionCountsTowardConcurrencyLimit:"
@@ -9790,7 +9939,7 @@ TEST_F(DBCompactionTest, TurnOnLevelCompactionDynamicLevelBytesUCToLC) {
  options.compaction_style = CompactionStyle::kCompactionStyleLevel;
  options.level_compaction_dynamic_level_bytes = true;
  ReopenWithColumnFamilies({"default", "pikachu"}, options);
-  std::string expected_lsm = "";
+  std::string expected_lsm;
  for (int i = 0; i < 49; ++i) {
    expected_lsm += "0,";
  }
@@ -10198,8 +10347,7 @@ TEST_F(DBCompactionTest, ErrorWhenReadFileHead) {
    SyncPoint::GetInstance()->SetCallBack(
        "RandomAccessFileReader::Read::BeforeReturn",
        [&count, &error_file](void* pair_ptr) {
-          auto p =
-              reinterpret_cast<std::pair<std::string*, IOStatus*>*>(pair_ptr);
+          auto p = static_cast<std::pair<std::string*, IOStatus*>*>(pair_ptr);
          int cur = ++count;
          if (cur == error_file) {
            IOStatus* io_s = p->second;
@@ -10307,20 +10455,20 @@ TEST_F(DBCompactionTest, ReleaseCompactionDuringManifestWrite) {
  SyncPoint::GetInstance()->EnableProcessing();

  std::vector<std::thread> threads;
-  threads.emplace_back(std::thread([&]() {
+  threads.emplace_back([&]() {
    std::string k1_str = Key(1);
    std::string k2_str = Key(2);
    Slice k1 = k1_str;
    Slice k2 = k2_str;
    ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &k1, &k2));
-  }));
-  threads.emplace_back(std::thread([&]() {
+  });
+  threads.emplace_back([&]() {
    std::string k10_str = Key(10);
    std::string k11_str = Key(11);
    Slice k10 = k10_str;
    Slice k11 = k11_str;
    ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &k10, &k11));
-  }));
+  });
  std::string k100_str = Key(100);
  std::string k101_str = Key(101);
  Slice k100 = k100_str;
@@ -338,7 +338,7 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesCompactRange) {
  std::set<int> output_levels;
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "CompactionPicker::CompactRange:Return", [&](void* arg) {
-        Compaction* compaction = reinterpret_cast<Compaction*>(arg);
+        Compaction* compaction = static_cast<Compaction*>(arg);
        output_levels.insert(compaction->output_level());
      });
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
@@ -121,7 +121,7 @@ Status DBImpl::GetSortedWalFiles(VectorLogPtr& files) {

  // DisableFileDeletions / EnableFileDeletions not supported in read-only DB
  if (deletions_disabled.ok()) {
-    Status s2 = EnableFileDeletions(/*force=*/false);
+    Status s2 = EnableFileDeletions();
    assert(s2.ok());
    s2.PermitUncheckedError();
  } else {
@@ -390,8 +390,11 @@ Status DBImpl::GetLiveFilesStorageInfo(
      info.file_number = live_wal_files[i]->LogNumber();
      info.file_type = kWalFile;
      info.size = live_wal_files[i]->SizeFileBytes();
-      // Only last should need to be trimmed
-      info.trim_to_size = (i + 1 == wal_size);
+      // Trim the log either if its the last one, or log file recycling is
+      // enabled. In the latter case, a hard link doesn't prevent the file
+      // from being renamed and recycled. So we need to copy it instead.
+      info.trim_to_size = (i + 1 == wal_size) ||
+                          (immutable_db_options_.recycle_log_file_num > 0);
      if (opts.include_checksum_info) {
        info.file_checksum_func_name = kUnknownFileChecksumFuncName;
        info.file_checksum = kUnknownFileChecksum;
@@ -272,7 +272,7 @@ TEST_F(DBFlushTest, ScheduleOnlyOneBgThread) {
  SyncPoint::GetInstance()->SetCallBack(
      "DBImpl::MaybeScheduleFlushOrCompaction:AfterSchedule:0", [&](void* arg) {
        ASSERT_NE(nullptr, arg);
-        auto unscheduled_flushes = *reinterpret_cast<int*>(arg);
+        auto unscheduled_flushes = *static_cast<int*>(arg);
        ASSERT_EQ(0, unscheduled_flushes);
        ++called;
      });
@@ -1367,14 +1367,15 @@ TEST_F(DBFlushTest, MemPurgeDeleteAndDeleteRange) {
      ASSERT_OK(iter->status());
      key = (iter->key()).ToString(false);
      value = (iter->value()).ToString(false);
-      if (key.compare(KEY3) == 0)
+      if (key.compare(KEY3) == 0) {
        ASSERT_EQ(value, p_v3b);
-      else if (key.compare(KEY4) == 0)
+      } else if (key.compare(KEY4) == 0) {
        ASSERT_EQ(value, p_v4);
-      else if (key.compare(KEY5) == 0)
+      } else if (key.compare(KEY5) == 0) {
        ASSERT_EQ(value, p_v5);
-      else
+      } else {
        ASSERT_EQ(value, NOT_FOUND);
+      }
      count++;
    }
    ASSERT_OK(iter->status());
@@ -1404,22 +1405,25 @@ TEST_F(DBFlushTest, MemPurgeDeleteAndDeleteRange) {
    ASSERT_OK(iter->status());
    key = (iter->key()).ToString(false);
    value = (iter->value()).ToString(false);
-    if (key.compare(KEY2) == 0)
+    if (key.compare(KEY2) == 0) {
      ASSERT_EQ(value, p_v2);
-    else if (key.compare(KEY3) == 0)
+    } else if (key.compare(KEY3) == 0) {
      ASSERT_EQ(value, p_v3b);
-    else if (key.compare(KEY4) == 0)
+    } else if (key.compare(KEY4) == 0) {
      ASSERT_EQ(value, p_v4);
-    else if (key.compare(KEY5) == 0)
+    } else if (key.compare(KEY5) == 0) {
      ASSERT_EQ(value, p_v5);
-    else
+    } else {
      ASSERT_EQ(value, NOT_FOUND);
+    }
    count++;
  }

  // Expected count here is 4: KEY2, KEY3, KEY4, KEY5.
  ASSERT_EQ(count, EXPECTED_COUNT_END);
-  if (iter) delete iter;
+  if (iter) {
+    delete iter;
+  }

  Close();
 }
@@ -1791,7 +1795,7 @@ TEST_F(DBFlushTest, MemPurgeCorrectLogNumberAndSSTFileCreation) {
  std::atomic<uint64_t> num_memtable_at_first_flush(0);
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "FlushJob::WriteLevel0Table:num_memtables", [&](void* arg) {
-        uint64_t* mems_size = reinterpret_cast<uint64_t*>(arg);
+        uint64_t* mems_size = static_cast<uint64_t*>(arg);
        // atomic_compare_exchange_strong sometimes updates the value
        // of ZERO (the "expected" object), so we make sure ZERO is indeed...
        // zero.
@@ -2038,7 +2042,7 @@ TEST_F(DBFlushTest, FireOnFlushCompletedAfterCommittedResult) {
  SyncPoint::GetInstance()->SetCallBack(
      "FlushJob::WriteLevel0Table", [&listener](void* arg) {
        // Wait for the second flush finished, out of mutex.
-        auto* mems = reinterpret_cast<autovector<MemTable*>*>(arg);
+        auto* mems = static_cast<autovector<MemTable*>*>(arg);
        if (mems->front()->GetEarliestSequenceNumber() == listener->seq1 - 1) {
          TEST_SYNC_POINT(
              "DBFlushTest::FireOnFlushCompletedAfterCommittedResult:"
@@ -2386,7 +2390,7 @@ TEST_F(DBFlushTest, PickRightMemtables) {
      });
  SyncPoint::GetInstance()->SetCallBack(
      "DBImpl::FlushMemTableToOutputFile:AfterPickMemtables", [&](void* arg) {
-        auto* job = reinterpret_cast<FlushJob*>(arg);
+        auto* job = static_cast<FlushJob*>(arg);
        assert(job);
        const auto& mems = job->GetMemTables();
        assert(mems.size() == 1);
@@ -2499,7 +2503,7 @@ TEST_F(DBFlushTest, TombstoneVisibleInSnapshot) {
  class SimpleTestFlushListener : public EventListener {
   public:
    explicit SimpleTestFlushListener(DBFlushTest* _test) : test_(_test) {}
-    ~SimpleTestFlushListener() override {}
+    ~SimpleTestFlushListener() override = default;

    void OnFlushBegin(DB* db, const FlushJobInfo& info) override {
      ASSERT_EQ(static_cast<uint32_t>(0), info.cf_id);
@@ -2634,7 +2638,7 @@ TEST_P(DBAtomicFlushTest, ManualFlushUnder2PC) {
  // it means atomic flush didn't write the min_log_number_to_keep to MANIFEST.
  cfs.push_back(kDefaultColumnFamilyName);
  ASSERT_OK(TryReopenWithColumnFamilies(cfs, options));
-  DBImpl* db_impl = reinterpret_cast<DBImpl*>(db_);
+  DBImpl* db_impl = static_cast<DBImpl*>(db_);
  ASSERT_TRUE(db_impl->allow_2pc());
  ASSERT_NE(db_impl->MinLogNumberToKeep(), 0);
 }
@@ -3171,7 +3175,7 @@ TEST_P(DBAtomicFlushTest, BgThreadNoWaitAfterManifestError) {

  SyncPoint::GetInstance()->SetCallBack(
      "VersionSet::ProcessManifestWrites:AfterSyncManifest", [&](void* arg) {
-        auto* ptr = reinterpret_cast<IOStatus*>(arg);
+        auto* ptr = static_cast<IOStatus*>(arg);
        assert(ptr);
        *ptr = IOStatus::IOError("Injected failure");
      });
@@ -13,10 +13,6 @@

 namespace ROCKSDB_NAMESPACE {

-extern void MarkKeyMayExist(void* arg);
-extern bool SaveValue(void* arg, const ParsedInternalKey& parsed_key,
-                      const Slice& v, bool hit_and_return);
-
 CompactedDBImpl::CompactedDBImpl(const DBOptions& options,
                                 const std::string& dbname)
    : DBImpl(options, dbname, /*seq_per_batch*/ false, +/*batch_per_txn*/ true,
@@ -25,7 +21,7 @@ CompactedDBImpl::CompactedDBImpl(const DBOptions& options,
      version_(nullptr),
      user_comparator_(nullptr) {}

-CompactedDBImpl::~CompactedDBImpl() {}
+CompactedDBImpl::~CompactedDBImpl() = default;

 size_t CompactedDBImpl::FindFile(const Slice& key) {
  size_t right = files_.num_files - 1;
@@ -37,12 +33,6 @@ size_t CompactedDBImpl::FindFile(const Slice& key) {
      files_.files);
 }

-Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*,
-                            const Slice& key, PinnableSlice* value) {
-  return Get(options, /*column_family*/ nullptr, key, value,
-             /*timestamp*/ nullptr);
-}
-
 Status CompactedDBImpl::Get(const ReadOptions& _read_options,
                            ColumnFamilyHandle*, const Slice& key,
                            PinnableSlice* value, std::string* timestamp) {
@@ -112,62 +102,59 @@ Status CompactedDBImpl::Get(const ReadOptions& _read_options,
  return Status::NotFound();
 }

-std::vector<Status> CompactedDBImpl::MultiGet(
-    const ReadOptions& options, const std::vector<ColumnFamilyHandle*>&,
-    const std::vector<Slice>& keys, std::vector<std::string>* values) {
-  return MultiGet(options, keys, values, /*timestamps*/ nullptr);
-}
-
-std::vector<Status> CompactedDBImpl::MultiGet(
-    const ReadOptions& _read_options, const std::vector<ColumnFamilyHandle*>&,
-    const std::vector<Slice>& keys, std::vector<std::string>* values,
-    std::vector<std::string>* timestamps) {
+void CompactedDBImpl::MultiGet(const ReadOptions& _read_options,
+                               size_t num_keys,
+                               ColumnFamilyHandle** /*column_families*/,
+                               const Slice* keys, PinnableSlice* values,
+                               std::string* timestamps, Status* statuses,
+                               const bool /*sorted_input*/) {
  assert(user_comparator_);
-  size_t num_keys = keys.size();
+  Status s;
  if (_read_options.io_activity != Env::IOActivity::kUnknown &&
      _read_options.io_activity != Env::IOActivity::kMultiGet) {
-    Status s = Status::InvalidArgument(
+    s = Status::InvalidArgument(
        "Can only call MultiGet with `ReadOptions::io_activity` is "
        "`Env::IOActivity::kUnknown` or `Env::IOActivity::kMultiGet`");
-    return std::vector<Status>(num_keys, s);
  }

  ReadOptions read_options(_read_options);
-  if (read_options.io_activity == Env::IOActivity::kUnknown) {
-    read_options.io_activity = Env::IOActivity::kMultiGet;
+  if (s.ok()) {
+    if (read_options.io_activity == Env::IOActivity::kUnknown) {
+      read_options.io_activity = Env::IOActivity::kMultiGet;
+    }
+
+    if (read_options.timestamp) {
+      s = FailIfTsMismatchCf(DefaultColumnFamily(), *(read_options.timestamp));
+      if (s.ok()) {
+        if (read_options.timestamp->size() > 0) {
+          s = FailIfReadCollapsedHistory(cfd_, cfd_->GetSuperVersion(),
+                                         *(read_options.timestamp));
+        }
+      }
+    } else {
+      s = FailIfCfHasTs(DefaultColumnFamily());
+    }
  }

-  if (read_options.timestamp) {
-    Status s =
-        FailIfTsMismatchCf(DefaultColumnFamily(), *(read_options.timestamp));
-    if (!s.ok()) {
-      return std::vector<Status>(num_keys, s);
-    }
-    if (read_options.timestamp->size() > 0) {
-      s = FailIfReadCollapsedHistory(cfd_, cfd_->GetSuperVersion(),
-                                     *(read_options.timestamp));
-      if (!s.ok()) {
-        return std::vector<Status>(num_keys, s);
-      }
-    }
-  } else {
-    Status s = FailIfCfHasTs(DefaultColumnFamily());
-    if (!s.ok()) {
-      return std::vector<Status>(num_keys, s);
+  if (!s.ok()) {
+    for (size_t i = 0; i < num_keys; ++i) {
+      statuses[i] = s;
    }
+    return;
  }

  // Clear the timestamps for returning results so that we can distinguish
  // between tombstone or key that has never been written
  if (timestamps) {
-    for (auto& ts : *timestamps) {
-      ts.clear();
+    for (size_t i = 0; i < num_keys; ++i) {
+      timestamps[i].clear();
    }
  }

  GetWithTimestampReadCallback read_cb(kMaxSequenceNumber);
  autovector<TableReader*, 16> reader_list;
-  for (const auto& key : keys) {
+  for (size_t i = 0; i < num_keys; ++i) {
+    const Slice& key = keys[i];
    LookupKey lkey(key, kMaxSequenceNumber, read_options.timestamp);
    const FdWithKeyRange& f = files_.files[FindFile(lkey.user_key())];
    if (user_comparator_->CompareWithoutTimestamp(
@@ -181,30 +168,26 @@ std::vector<Status> CompactedDBImpl::MultiGet(
      reader_list.push_back(f.fd.table_reader);
    }
  }
-  std::vector<Status> statuses(num_keys, Status::NotFound());
-  values->resize(num_keys);
-  if (timestamps) {
-    timestamps->resize(num_keys);
+  for (size_t i = 0; i < num_keys; ++i) {
+    statuses[i] = Status::NotFound();
  }
  int idx = 0;
  for (auto* r : reader_list) {
    if (r != nullptr) {
-      PinnableSlice pinnable_val;
-      std::string& value = (*values)[idx];
+      PinnableSlice& pinnable_val = values[idx];
      LookupKey lkey(keys[idx], kMaxSequenceNumber, read_options.timestamp);
-      std::string* timestamp = timestamps ? &(*timestamps)[idx] : nullptr;
+      std::string* timestamp = timestamps ? &timestamps[idx] : nullptr;
      GetContext get_context(
          user_comparator_, nullptr, nullptr, nullptr, GetContext::kNotFound,
          lkey.user_key(), &pinnable_val, /*columns=*/nullptr,
          user_comparator_->timestamp_size() > 0 ? timestamp : nullptr, nullptr,
          nullptr, true, nullptr, nullptr, nullptr, nullptr, &read_cb);
-      Status s =
+      Status status =
          r->Get(read_options, lkey.internal_key(), &get_context, nullptr);
-      assert(static_cast<size_t>(idx) < statuses.size());
-      if (!s.ok() && !s.IsNotFound()) {
-        statuses[idx] = s;
+      assert(static_cast<size_t>(idx) < num_keys);
+      if (!status.ok() && !status.IsNotFound()) {
+        statuses[idx] = status;
      } else {
-        value.assign(pinnable_val.data(), pinnable_val.size());
        if (get_context.State() == GetContext::kFound) {
          statuses[idx] = Status::OK();
        }
@@ -212,7 +195,6 @@ std::vector<Status> CompactedDBImpl::MultiGet(
    }
    ++idx;
  }
-  return statuses;
 }

 Status CompactedDBImpl::Init(const Options& options) {
@@ -26,33 +26,23 @@ class CompactedDBImpl : public DBImpl {

  // Implementations of the DB interface
  using DB::Get;
-  virtual Status Get(const ReadOptions& options,
-                     ColumnFamilyHandle* column_family, const Slice& key,
-                     PinnableSlice* value) override;
-
-  Status Get(const ReadOptions& _read_options,
-             ColumnFamilyHandle* column_family, const Slice& key,
-             PinnableSlice* value, std::string* timestamp) override;
+  Status Get(const ReadOptions& options, ColumnFamilyHandle* column_family,
+             const Slice& key, PinnableSlice* value,
+             std::string* timestamp) override;

  using DB::MultiGet;
  // Note that CompactedDBImpl::MultiGet is not the optimized version of
  // MultiGet to use.
  // TODO: optimize CompactedDBImpl::MultiGet, see DBImpl::MultiGet for details.
-  virtual std::vector<Status> MultiGet(
-      const ReadOptions& options, const std::vector<ColumnFamilyHandle*>&,
-      const std::vector<Slice>& keys,
-      std::vector<std::string>* values) override;
-
-  std::vector<Status> MultiGet(const ReadOptions& _read_options,
-                               const std::vector<ColumnFamilyHandle*>&,
-                               const std::vector<Slice>& keys,
-                               std::vector<std::string>* values,
-                               std::vector<std::string>* timestamps) override;
+  void MultiGet(const ReadOptions& options, size_t num_keys,
+                ColumnFamilyHandle** column_families, const Slice* keys,
+                PinnableSlice* values, std::string* timestamps,
+                Status* statuses, const bool sorted_input) override;

  using DBImpl::Put;
-  virtual Status Put(const WriteOptions& /*options*/,
-                     ColumnFamilyHandle* /*column_family*/,
-                     const Slice& /*key*/, const Slice& /*value*/) override {
+  Status Put(const WriteOptions& /*options*/,
+             ColumnFamilyHandle* /*column_family*/, const Slice& /*key*/,
+             const Slice& /*value*/) override {
    return Status::NotSupported("Not supported in compacted db mode.");
  }

@@ -65,54 +55,53 @@ class CompactedDBImpl : public DBImpl {
  }

  using DBImpl::Merge;
-  virtual Status Merge(const WriteOptions& /*options*/,
-                       ColumnFamilyHandle* /*column_family*/,
-                       const Slice& /*key*/, const Slice& /*value*/) override {
+  Status Merge(const WriteOptions& /*options*/,
+               ColumnFamilyHandle* /*column_family*/, const Slice& /*key*/,
+               const Slice& /*value*/) override {
    return Status::NotSupported("Not supported in compacted db mode.");
  }

  using DBImpl::Delete;
-  virtual Status Delete(const WriteOptions& /*options*/,
-                        ColumnFamilyHandle* /*column_family*/,
-                        const Slice& /*key*/) override {
+  Status Delete(const WriteOptions& /*options*/,
+                ColumnFamilyHandle* /*column_family*/,
+                const Slice& /*key*/) override {
    return Status::NotSupported("Not supported in compacted db mode.");
  }
-  virtual Status Write(const WriteOptions& /*options*/,
-                       WriteBatch* /*updates*/) override {
+  Status Write(const WriteOptions& /*options*/,
+               WriteBatch* /*updates*/) override {
    return Status::NotSupported("Not supported in compacted db mode.");
  }
  using DBImpl::CompactRange;
-  virtual Status CompactRange(const CompactRangeOptions& /*options*/,
-                              ColumnFamilyHandle* /*column_family*/,
-                              const Slice* /*begin*/,
-                              const Slice* /*end*/) override {
+  Status CompactRange(const CompactRangeOptions& /*options*/,
+                      ColumnFamilyHandle* /*column_family*/,
+                      const Slice* /*begin*/, const Slice* /*end*/) override {
    return Status::NotSupported("Not supported in compacted db mode.");
  }

-  virtual Status DisableFileDeletions() override {
+  Status DisableFileDeletions() override {
    return Status::NotSupported("Not supported in compacted db mode.");
  }
-  virtual Status EnableFileDeletions(bool /*force*/) override {
+  Status EnableFileDeletions() override {
    return Status::NotSupported("Not supported in compacted db mode.");
  }
-  virtual Status GetLiveFiles(std::vector<std::string>& ret,
-                              uint64_t* manifest_file_size,
-                              bool /*flush_memtable*/) override {
+  Status GetLiveFiles(std::vector<std::string>& ret,
+                      uint64_t* manifest_file_size,
+                      bool /*flush_memtable*/) override {
    return DBImpl::GetLiveFiles(ret, manifest_file_size,
                                false /* flush_memtable */);
  }
  using DBImpl::Flush;
-  virtual Status Flush(const FlushOptions& /*options*/,
-                       ColumnFamilyHandle* /*column_family*/) override {
+  Status Flush(const FlushOptions& /*options*/,
+               ColumnFamilyHandle* /*column_family*/) override {
    return Status::NotSupported("Not supported in compacted db mode.");
  }

-  virtual Status SyncWAL() override {
+  Status SyncWAL() override {
    return Status::NotSupported("Not supported in compacted db mode.");
  }

  using DB::IngestExternalFile;
-  virtual Status IngestExternalFile(
+  Status IngestExternalFile(
      ColumnFamilyHandle* /*column_family*/,
      const std::vector<std::string>& /*external_files*/,
      const IngestExternalFileOptions& /*ingestion_options*/) override {
@@ -120,7 +109,7 @@ class CompactedDBImpl : public DBImpl {
  }

  using DB::CreateColumnFamilyWithImport;
-  virtual Status CreateColumnFamilyWithImport(
+  Status CreateColumnFamilyWithImport(
      const ColumnFamilyOptions& /*options*/,
      const std::string& /*column_family_name*/,
      const ImportColumnFamilyOptions& /*import_options*/,
@@ -130,9 +119,9 @@ class CompactedDBImpl : public DBImpl {
  }

  using DB::ClipColumnFamily;
-  virtual Status ClipColumnFamily(ColumnFamilyHandle* /*column_family*/,
-                                  const Slice& /*begin*/,
-                                  const Slice& /*end*/) override {
+  Status ClipColumnFamily(ColumnFamilyHandle* /*column_family*/,
+                          const Slice& /*begin*/,
+                          const Slice& /*end*/) override {
    return Status::NotSupported("Not supported in compacted db mode.");
  }

@@ -8,7 +8,7 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 #include "db/db_impl/db_impl.h"

-#include <stdint.h>
+#include <cstdint>
 #ifdef OS_SOLARIS
 #include <alloca.h>
 #endif
@@ -16,6 +16,7 @@
 #include <cinttypes>
 #include <cstdio>
 #include <map>
+#include <memory>
 #include <set>
 #include <sstream>
 #include <stdexcept>
@@ -44,7 +45,7 @@
 #include "db/memtable.h"
 #include "db/memtable_list.h"
 #include "db/merge_context.h"
-#include "db/merge_helper.h"
+#include "db/multi_cf_iterator.h"
 #include "db/periodic_task_scheduler.h"
 #include "db/range_tombstone_fragmenter.h"
 #include "db/table_cache.h"
@@ -71,6 +72,9 @@
 #include "options/cf_options.h"
 #include "options/options_helper.h"
 #include "options/options_parser.h"
+#ifdef ROCKSDB_JEMALLOC
+#include "port/jemalloc_helper.h"
+#endif
 #include "port/port.h"
 #include "rocksdb/cache.h"
 #include "rocksdb/compaction_filter.h"
@@ -148,6 +152,12 @@ void DumpSupportInfo(Logger* logger) {
                   crc32c::IsFastCrc32Supported().c_str());

  ROCKS_LOG_HEADER(logger, "DMutex implementation: %s", DMutex::kName());
+
+  bool jemalloc_supported = false;
+#ifdef ROCKSDB_JEMALLOC
+  jemalloc_supported = HasJemalloc();
+#endif
+  ROCKS_LOG_HEADER(logger, "Jemalloc supported: %d", jemalloc_supported);
 }
 }  // namespace

@@ -309,7 +319,7 @@ Status DBImpl::Resume() {

  if (error_handler_.IsRecoveryInProgress()) {
    // Don't allow a mix of manual and automatic recovery
-    return Status::Busy();
+    return Status::Busy("Recovery in progress");
  }

  mutex_.Unlock();
@@ -950,7 +960,9 @@ size_t DBImpl::EstimateInMemoryStatsHistorySize() const {
  stats_history_mutex_.AssertHeld();
  size_t size_total =
      sizeof(std::map<uint64_t, std::map<std::string, uint64_t>>);
-  if (stats_history_.size() == 0) return size_total;
+  if (stats_history_.size() == 0) {
+    return size_total;
+  }
  size_t size_per_slice =
      sizeof(uint64_t) + sizeof(std::map<std::string, uint64_t>);
  // non-empty map, stats_history_.begin() guaranteed to exist
@@ -1076,7 +1088,9 @@ bool DBImpl::FindStatsByTime(uint64_t start_time, uint64_t end_time,
                             std::map<std::string, uint64_t>* stats_map) {
  assert(new_time);
  assert(stats_map);
-  if (!new_time || !stats_map) return false;
+  if (!new_time || !stats_map) {
+    return false;
+  }
  // lock when search for start_time
  {
    InstrumentedMutexLock l(&stats_history_mutex_);
@@ -1483,7 +1497,9 @@ int DBImpl::FindMinimumEmptyLevelFitting(
  int minimum_level = level;
  for (int i = level - 1; i > 0; --i) {
    // stop if level i is not empty
-    if (vstorage->NumLevelFiles(i) > 0) break;
+    if (vstorage->NumLevelFiles(i) > 0) {
+      break;
+    }
    // stop if level i is too small (cannot fit the level files)
    if (vstorage->MaxBytesForLevel(i) < vstorage->NumLevelBytes(level)) {
      break;
@@ -1741,7 +1757,11 @@ void DBImpl::MarkLogsSynced(uint64_t up_to, bool synced_dir,
          wal.GetPreSyncSize() > 0) {
        synced_wals->AddWal(wal.number, WalMetadata(wal.GetPreSyncSize()));
      }
-      if (wal.GetPreSyncSize() == wal.writer->file()->GetFlushedSize()) {
+      // Check if the file has been closed, i.e wal.writer->file() == nullptr
+      // which can happen if log recycling is enabled, or if all the data in
+      // the log has been synced
+      if (wal.writer->file() == nullptr ||
+          wal.GetPreSyncSize() == wal.writer->file()->GetFlushedSize()) {
        // Fully synced
        logs_to_free_.push_back(wal.ReleaseWriter());
        it = logs_.erase(it);
@@ -1907,7 +1927,7 @@ struct SuperVersionHandle {
 };

 static void CleanupSuperVersionHandle(void* arg1, void* /*arg2*/) {
-  SuperVersionHandle* sv_handle = reinterpret_cast<SuperVersionHandle*>(arg1);
+  SuperVersionHandle* sv_handle = static_cast<SuperVersionHandle*>(arg1);

  if (sv_handle->super_version->Unref()) {
    // Job id == 0 means that this is not our background process, but rather
@@ -1964,7 +1984,8 @@ InternalIterator* DBImpl::NewInternalIterator(
          super_version->mutable_cf_options.prefix_extractor != nullptr,
      read_options.iterate_upper_bound);
  // Collect iterator for mutable memtable
-  auto mem_iter = super_version->mem->NewIterator(read_options, arena);
+  auto mem_iter = super_version->mem->NewIterator(
+      read_options, super_version->GetSeqnoToTimeMapping(), arena);
  Status s;
  if (!read_options.ignore_range_deletions) {
    TruncatedRangeDelIterator* mem_tombstone_iter = nullptr;
@@ -1986,8 +2007,9 @@ InternalIterator* DBImpl::NewInternalIterator(

  // Collect all needed child iterators for immutable memtables
  if (s.ok()) {
-    super_version->imm->AddIterators(read_options, &merge_iter_builder,
-                                     !read_options.ignore_range_deletions);
+    super_version->imm->AddIterators(
+        read_options, super_version->GetSeqnoToTimeMapping(),
+        &merge_iter_builder, !read_options.ignore_range_deletions);
  }
  TEST_SYNC_POINT_CALLBACK("DBImpl::NewInternalIterator:StatusCallback", &s);
  if (s.ok()) {
@@ -2020,12 +2042,6 @@ ColumnFamilyHandle* DBImpl::PersistentStatsColumnFamily() const {
  return persist_stats_cf_handle_;
 }

-Status DBImpl::Get(const ReadOptions& read_options,
-                   ColumnFamilyHandle* column_family, const Slice& key,
-                   PinnableSlice* value) {
-  return Get(read_options, column_family, key, value, /*timestamp=*/nullptr);
-}
-
 Status DBImpl::GetImpl(const ReadOptions& read_options,
                       ColumnFamilyHandle* column_family, const Slice& key,
                       PinnableSlice* value) {
@@ -2269,7 +2285,7 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key,
      snapshot = get_impl_options.callback->max_visible_seq();
    } else {
      snapshot =
-          reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)->number_;
+          static_cast<const SnapshotImpl*>(read_options.snapshot)->number_;
    }
  } else {
    // Note that the snapshot is assigned AFTER referencing the super
@@ -2314,6 +2330,8 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key,

  // Prepare to store a list of merge operations if merge occurs.
  MergeContext merge_context;
+  merge_context.get_merge_operands_options =
+      get_impl_options.get_merge_operands_options;
  SequenceNumber max_covering_tombstone_seq = 0;

  Status s;
@@ -2497,250 +2515,6 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key,
  return s;
 }

-std::vector<Status> DBImpl::MultiGet(
-    const ReadOptions& read_options,
-    const std::vector<ColumnFamilyHandle*>& column_family,
-    const std::vector<Slice>& keys, std::vector<std::string>* values) {
-  return MultiGet(read_options, column_family, keys, values,
-                  /*timestamps=*/nullptr);
-}
-
-std::vector<Status> DBImpl::MultiGet(
-    const ReadOptions& _read_options,
-    const std::vector<ColumnFamilyHandle*>& column_family,
-    const std::vector<Slice>& keys, std::vector<std::string>* values,
-    std::vector<std::string>* timestamps) {
-  PERF_CPU_TIMER_GUARD(get_cpu_nanos, immutable_db_options_.clock);
-  StopWatch sw(immutable_db_options_.clock, stats_, DB_MULTIGET);
-  PERF_TIMER_GUARD(get_snapshot_time);
-
-  size_t num_keys = keys.size();
-  assert(column_family.size() == num_keys);
-  std::vector<Status> stat_list(num_keys);
-
-  if (_read_options.io_activity != Env::IOActivity::kUnknown &&
-      _read_options.io_activity != Env::IOActivity::kMultiGet) {
-    Status s = Status::InvalidArgument(
-        "Can only call MultiGet with `ReadOptions::io_activity` is "
-        "`Env::IOActivity::kUnknown` or `Env::IOActivity::kMultiGet`");
-
-    for (size_t i = 0; i < num_keys; ++i) {
-      stat_list[i] = s;
-    }
-    return stat_list;
-  }
-
-  ReadOptions read_options(_read_options);
-  if (read_options.io_activity == Env::IOActivity::kUnknown) {
-    read_options.io_activity = Env::IOActivity::kMultiGet;
-  }
-
-  bool should_fail = false;
-  for (size_t i = 0; i < num_keys; ++i) {
-    assert(column_family[i]);
-    if (read_options.timestamp) {
-      stat_list[i] =
-          FailIfTsMismatchCf(column_family[i], *(read_options.timestamp));
-      if (!stat_list[i].ok()) {
-        should_fail = true;
-      }
-    } else {
-      stat_list[i] = FailIfCfHasTs(column_family[i]);
-      if (!stat_list[i].ok()) {
-        should_fail = true;
-      }
-    }
-  }
-
-  if (should_fail) {
-    for (auto& s : stat_list) {
-      if (s.ok()) {
-        s = Status::Incomplete(
-            "DB not queried due to invalid argument(s) in the same MultiGet");
-      }
-    }
-    return stat_list;
-  }
-
-  if (tracer_) {
-    // TODO: This mutex should be removed later, to improve performance when
-    // tracing is enabled.
-    InstrumentedMutexLock lock(&trace_mutex_);
-    if (tracer_) {
-      // TODO: maybe handle the tracing status?
-      tracer_->MultiGet(column_family, keys).PermitUncheckedError();
-    }
-  }
-
-  UnorderedMap<uint32_t, MultiGetColumnFamilyData> multiget_cf_data(
-      column_family.size());
-  for (auto cf : column_family) {
-    auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(cf);
-    auto cfd = cfh->cfd();
-    if (multiget_cf_data.find(cfd->GetID()) == multiget_cf_data.end()) {
-      multiget_cf_data.emplace(cfd->GetID(),
-                               MultiGetColumnFamilyData(cfh, nullptr));
-    }
-  }
-
-  std::function<MultiGetColumnFamilyData*(
-      UnorderedMap<uint32_t, MultiGetColumnFamilyData>::iterator&)>
-      iter_deref_lambda =
-          [](UnorderedMap<uint32_t, MultiGetColumnFamilyData>::iterator&
-                 cf_iter) { return &cf_iter->second; };
-
-  SequenceNumber consistent_seqnum;
-  bool sv_from_thread_local;
-  Status status =
-      MultiCFSnapshot<UnorderedMap<uint32_t, MultiGetColumnFamilyData>>(
-          read_options, nullptr, iter_deref_lambda, &multiget_cf_data,
-          &consistent_seqnum, &sv_from_thread_local);
-
-  if (!status.ok()) {
-    for (auto& s : stat_list) {
-      if (s.ok()) {
-        s = status;
-      }
-    }
-    return stat_list;
-  }
-
-  TEST_SYNC_POINT("DBImpl::MultiGet:AfterGetSeqNum1");
-  TEST_SYNC_POINT("DBImpl::MultiGet:AfterGetSeqNum2");
-
-  // Contain a list of merge operations if merge occurs.
-  MergeContext merge_context;
-
-  // Note: this always resizes the values array
-  values->resize(num_keys);
-  if (timestamps) {
-    timestamps->resize(num_keys);
-  }
-
-  // Keep track of bytes that we read for statistics-recording later
-  uint64_t bytes_read = 0;
-  PERF_TIMER_STOP(get_snapshot_time);
-
-  // For each of the given keys, apply the entire "get" process as follows:
-  // First look in the memtable, then in the immutable memtable (if any).
-  // s is both in/out. When in, s could either be OK or MergeInProgress.
-  // merge_operands will contain the sequence of merges in the latter case.
-  size_t num_found = 0;
-  size_t keys_read;
-  uint64_t curr_value_size = 0;
-
-  GetWithTimestampReadCallback timestamp_read_callback(0);
-  ReadCallback* read_callback = nullptr;
-  if (read_options.timestamp && read_options.timestamp->size() > 0) {
-    timestamp_read_callback.Refresh(consistent_seqnum);
-    read_callback = &timestamp_read_callback;
-  }
-
-  for (keys_read = 0; keys_read < num_keys; ++keys_read) {
-    merge_context.Clear();
-    Status& s = stat_list[keys_read];
-    std::string* value = &(*values)[keys_read];
-    std::string* timestamp = timestamps ? &(*timestamps)[keys_read] : nullptr;
-
-    LookupKey lkey(keys[keys_read], consistent_seqnum, read_options.timestamp);
-    auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(
-        column_family[keys_read]);
-    SequenceNumber max_covering_tombstone_seq = 0;
-    auto mgd_iter = multiget_cf_data.find(cfh->cfd()->GetID());
-    assert(mgd_iter != multiget_cf_data.end());
-    auto mgd = mgd_iter->second;
-    auto super_version = mgd.super_version;
-    bool skip_memtable =
-        (read_options.read_tier == kPersistedTier &&
-         has_unpersisted_data_.load(std::memory_order_relaxed));
-    bool done = false;
-    if (!skip_memtable) {
-      if (super_version->mem->Get(
-              lkey, value, /*columns=*/nullptr, timestamp, &s, &merge_context,
-              &max_covering_tombstone_seq, read_options,
-              false /* immutable_memtable */, read_callback)) {
-        done = true;
-        RecordTick(stats_, MEMTABLE_HIT);
-      } else if (super_version->imm->Get(lkey, value, /*columns=*/nullptr,
-                                         timestamp, &s, &merge_context,
-                                         &max_covering_tombstone_seq,
-                                         read_options, read_callback)) {
-        done = true;
-        RecordTick(stats_, MEMTABLE_HIT);
-      }
-    }
-    if (!done) {
-      PinnableSlice pinnable_val;
-      PERF_TIMER_GUARD(get_from_output_files_time);
-      PinnedIteratorsManager pinned_iters_mgr;
-      super_version->current->Get(read_options, lkey, &pinnable_val,
-                                  /*columns=*/nullptr, timestamp, &s,
-                                  &merge_context, &max_covering_tombstone_seq,
-                                  &pinned_iters_mgr, /*value_found=*/nullptr,
-                                  /*key_exists=*/nullptr,
-                                  /*seq=*/nullptr, read_callback);
-      value->assign(pinnable_val.data(), pinnable_val.size());
-      RecordTick(stats_, MEMTABLE_MISS);
-    }
-
-    if (s.ok()) {
-      const auto& merge_threshold = read_options.merge_operand_count_threshold;
-      if (merge_threshold.has_value() &&
-          merge_context.GetNumOperands() > merge_threshold.value()) {
-        s = Status::OkMergeOperandThresholdExceeded();
-      }
-
-      bytes_read += value->size();
-      num_found++;
-
-      curr_value_size += value->size();
-      if (curr_value_size > read_options.value_size_soft_limit) {
-        while (++keys_read < num_keys) {
-          stat_list[keys_read] = Status::Aborted();
-        }
-        break;
-      }
-    }
-    if (read_options.deadline.count() &&
-        immutable_db_options_.clock->NowMicros() >
-            static_cast<uint64_t>(read_options.deadline.count())) {
-      break;
-    }
-  }
-
-  if (keys_read < num_keys) {
-    // The only reason to break out of the loop is when the deadline is
-    // exceeded
-    assert(immutable_db_options_.clock->NowMicros() >
-           static_cast<uint64_t>(read_options.deadline.count()));
-    for (++keys_read; keys_read < num_keys; ++keys_read) {
-      stat_list[keys_read] = Status::TimedOut();
-    }
-  }
-
-  // Post processing (decrement reference counts and record statistics)
-  PERF_TIMER_GUARD(get_post_process_time);
-
-  for (auto mgd_iter : multiget_cf_data) {
-    auto mgd = mgd_iter.second;
-    if (sv_from_thread_local) {
-      ReturnAndCleanupSuperVersion(mgd.cfd, mgd.super_version);
-    } else {
-      TEST_SYNC_POINT("DBImpl::MultiGet::BeforeLastTryUnRefSV");
-      CleanupSuperVersion(mgd.super_version);
-    }
-  }
-  RecordTick(stats_, NUMBER_MULTIGET_CALLS);
-  RecordTick(stats_, NUMBER_MULTIGET_KEYS_READ, num_keys);
-  RecordTick(stats_, NUMBER_MULTIGET_KEYS_FOUND, num_found);
-  RecordTick(stats_, NUMBER_MULTIGET_BYTES_READ, bytes_read);
-  RecordInHistogram(stats_, BYTES_PER_MULTIGET, bytes_read);
-  PERF_COUNTER_ADD(multiget_read_bytes, bytes_read);
-  PERF_TIMER_STOP(get_post_process_time);
-
-  return stat_list;
-}
-
 template <class T>
 Status DBImpl::MultiCFSnapshot(
    const ReadOptions& read_options, ReadCallback* callback,
@@ -2889,7 +2663,8 @@ Status DBImpl::MultiCFSnapshot(
    }
  }

-  // Keep track of bytes that we read for statistics-recording later
+  TEST_SYNC_POINT("DBImpl::MultiCFSnapshot:AfterGetSeqNum1");
+  TEST_SYNC_POINT("DBImpl::MultiCFSnapshot:AfterGetSeqNum2");
  PERF_TIMER_STOP(get_snapshot_time);
  *sv_from_thread_local = !last_try;
  if (!s.ok()) {
@@ -2898,14 +2673,6 @@ Status DBImpl::MultiCFSnapshot(
  return s;
 }

-void DBImpl::MultiGet(const ReadOptions& read_options, const size_t num_keys,
-                      ColumnFamilyHandle** column_families, const Slice* keys,
-                      PinnableSlice* values, Status* statuses,
-                      const bool sorted_input) {
-  MultiGet(read_options, num_keys, column_families, keys, values,
-           /* timestamps */ nullptr, statuses, sorted_input);
-}
-
 void DBImpl::MultiGet(const ReadOptions& _read_options, const size_t num_keys,
                      ColumnFamilyHandle** column_families, const Slice* keys,
                      PinnableSlice* values, std::string* timestamps,
@@ -3122,38 +2889,23 @@ void DBImpl::PrepareMultiGetKeys(
            CompareKeyContext());
 }

-void DBImpl::MultiGet(const ReadOptions& read_options,
-                      ColumnFamilyHandle* column_family, const size_t num_keys,
-                      const Slice* keys, PinnableSlice* values,
-                      Status* statuses, const bool sorted_input) {
-  MultiGet(read_options, column_family, num_keys, keys, values,
-           /* timestamps */ nullptr, statuses, sorted_input);
-}
-
-void DBImpl::MultiGet(const ReadOptions& _read_options,
-                      ColumnFamilyHandle* column_family, const size_t num_keys,
-                      const Slice* keys, PinnableSlice* values,
-                      std::string* timestamps, Status* statuses,
-                      const bool sorted_input) {
-  if (_read_options.io_activity != Env::IOActivity::kUnknown &&
-      _read_options.io_activity != Env::IOActivity::kMultiGet) {
-    Status s = Status::InvalidArgument(
-        "Can only call MultiGet with `ReadOptions::io_activity` is "
-        "`Env::IOActivity::kUnknown` or `Env::IOActivity::kMultiGet`");
-    for (size_t i = 0; i < num_keys; ++i) {
-      if (statuses[i].ok()) {
-        statuses[i] = s;
-      }
-    }
-    return;
+void DB::MultiGet(const ReadOptions& options, ColumnFamilyHandle* column_family,
+                  const size_t num_keys, const Slice* keys,
+                  PinnableSlice* values, std::string* timestamps,
+                  Status* statuses, const bool sorted_input) {
+  // Use std::array, if possible, to avoid memory allocation overhead
+  if (num_keys > MultiGetContext::MAX_BATCH_SIZE) {
+    std::vector<ColumnFamilyHandle*> column_families(num_keys, column_family);
+    MultiGet(options, num_keys, column_families.data(), keys, values,
+             timestamps, statuses, sorted_input);
+  } else {
+    std::array<ColumnFamilyHandle*, MultiGetContext::MAX_BATCH_SIZE>
+        column_families;
+    std::fill(column_families.begin(), column_families.begin() + num_keys,
+              column_family);
+    MultiGet(options, num_keys, column_families.data(), keys, values,
+             timestamps, statuses, sorted_input);
  }
-
-  ReadOptions read_options(_read_options);
-  if (read_options.io_activity == Env::IOActivity::kUnknown) {
-    read_options.io_activity = Env::IOActivity::kMultiGet;
-  }
-  MultiGetCommon(read_options, column_family, num_keys, keys, values,
-                 /* columns */ nullptr, timestamps, statuses, sorted_input);
 }

 void DBImpl::MultiGetCommon(const ReadOptions& read_options,
@@ -3881,9 +3633,9 @@ Iterator* DBImpl::NewIterator(const ReadOptions& _read_options,
  }

  auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(column_family);
+  assert(cfh != nullptr);
  ColumnFamilyData* cfd = cfh->cfd();
  assert(cfd != nullptr);
-  ReadCallback* read_callback = nullptr;  // No read callback provided.
  SuperVersion* sv = cfd->GetReferencedSuperVersion(this);
  if (read_options.timestamp && read_options.timestamp->size() > 0) {
    const Status s =
@@ -3899,24 +3651,24 @@ Iterator* DBImpl::NewIterator(const ReadOptions& _read_options,
    result = NewDBIterator(
        env_, read_options, *cfd->ioptions(), sv->mutable_cf_options,
        cfd->user_comparator(), iter, sv->current, kMaxSequenceNumber,
-        sv->mutable_cf_options.max_sequential_skip_in_iterations, read_callback,
-        this, cfd);
+        sv->mutable_cf_options.max_sequential_skip_in_iterations,
+        nullptr /* read_callback */, cfh);
  } else {
    // Note: no need to consider the special case of
    // last_seq_same_as_publish_seq_==false since NewIterator is overridden in
    // WritePreparedTxnDB
-    result = NewIteratorImpl(read_options, cfd, sv,
+    result = NewIteratorImpl(read_options, cfh, sv,
                             (read_options.snapshot != nullptr)
                                 ? read_options.snapshot->GetSequenceNumber()
                                 : kMaxSequenceNumber,
-                             read_callback);
+                             nullptr /* read_callback */);
  }
  return result;
 }

 ArenaWrappedDBIter* DBImpl::NewIteratorImpl(
-    const ReadOptions& read_options, ColumnFamilyData* cfd, SuperVersion* sv,
-    SequenceNumber snapshot, ReadCallback* read_callback,
+    const ReadOptions& read_options, ColumnFamilyHandleImpl* cfh,
+    SuperVersion* sv, SequenceNumber snapshot, ReadCallback* read_callback,
    bool expose_blob_index, bool allow_refresh) {
  TEST_SYNC_POINT("DBImpl::NewIterator:1");
  TEST_SYNC_POINT("DBImpl::NewIterator:2");
@@ -3979,19 +3731,44 @@ ArenaWrappedDBIter* DBImpl::NewIteratorImpl(
  // likely that any iterator pointer is close to the iterator it points to so
  // that they are likely to be in the same cache line and/or page.
  ArenaWrappedDBIter* db_iter = NewArenaWrappedDbIterator(
-      env_, read_options, *cfd->ioptions(), sv->mutable_cf_options, sv->current,
-      snapshot, sv->mutable_cf_options.max_sequential_skip_in_iterations,
-      sv->version_number, read_callback, this, cfd, expose_blob_index,
-      allow_refresh);
+      env_, read_options, *cfh->cfd()->ioptions(), sv->mutable_cf_options,
+      sv->current, snapshot,
+      sv->mutable_cf_options.max_sequential_skip_in_iterations,
+      sv->version_number, read_callback, cfh, expose_blob_index, allow_refresh);

  InternalIterator* internal_iter = NewInternalIterator(
-      db_iter->GetReadOptions(), cfd, sv, db_iter->GetArena(), snapshot,
+      db_iter->GetReadOptions(), cfh->cfd(), sv, db_iter->GetArena(), snapshot,
      /* allow_unprepared_value */ true, db_iter);
  db_iter->SetIterUnderDBIter(internal_iter);

  return db_iter;
 }

+std::unique_ptr<Iterator> DBImpl::NewMultiCfIterator(
+    const ReadOptions& _read_options,
+    const std::vector<ColumnFamilyHandle*>& column_families) {
+  if (column_families.size() == 0) {
+    return std::unique_ptr<Iterator>(NewErrorIterator(
+        Status::InvalidArgument("No Column Family was provided")));
+  }
+  const Comparator* first_comparator = column_families[0]->GetComparator();
+  for (size_t i = 1; i < column_families.size(); ++i) {
+    const Comparator* cf_comparator = column_families[i]->GetComparator();
+    if (first_comparator != cf_comparator &&
+        first_comparator->GetId().compare(cf_comparator->GetId()) != 0) {
+      return std::unique_ptr<Iterator>(NewErrorIterator(Status::InvalidArgument(
+          "Different comparators are being used across CFs")));
+    }
+  }
+  std::vector<Iterator*> child_iterators;
+  Status s = NewIterators(_read_options, column_families, &child_iterators);
+  if (s.ok()) {
+    return std::make_unique<MultiCfIterator>(first_comparator, column_families,
+                                             std::move(child_iterators));
+  }
+  return std::unique_ptr<Iterator>(NewErrorIterator(s));
+}
+
 Status DBImpl::NewIterators(
    const ReadOptions& _read_options,
    const std::vector<ColumnFamilyHandle*>& column_families,
@@ -4032,37 +3809,37 @@ Status DBImpl::NewIterators(
    }
  }

-  ReadCallback* read_callback = nullptr;  // No read callback provided.
  iterators->clear();
  iterators->reserve(column_families.size());
-  autovector<std::tuple<ColumnFamilyData*, SuperVersion*>> cfd_to_sv;
+  autovector<std::tuple<ColumnFamilyHandleImpl*, SuperVersion*>> cfh_to_sv;
  const bool check_read_ts =
      read_options.timestamp && read_options.timestamp->size() > 0;
-  for (auto cfh : column_families) {
-    auto cfd = static_cast_with_check<ColumnFamilyHandleImpl>(cfh)->cfd();
+  for (auto cf : column_families) {
+    auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(cf);
+    auto cfd = cfh->cfd();
    SuperVersion* sv = cfd->GetReferencedSuperVersion(this);
-    cfd_to_sv.emplace_back(cfd, sv);
+    cfh_to_sv.emplace_back(cfh, sv);
    if (check_read_ts) {
      const Status s =
          FailIfReadCollapsedHistory(cfd, sv, *(read_options.timestamp));
      if (!s.ok()) {
-        for (auto prev_entry : cfd_to_sv) {
+        for (auto prev_entry : cfh_to_sv) {
          CleanupSuperVersion(std::get<1>(prev_entry));
        }
        return s;
      }
    }
  }
-  assert(cfd_to_sv.size() == column_families.size());
+  assert(cfh_to_sv.size() == column_families.size());
  if (read_options.tailing) {
-    for (auto [cfd, sv] : cfd_to_sv) {
-      auto iter = new ForwardIterator(this, read_options, cfd, sv,
+    for (auto [cfh, sv] : cfh_to_sv) {
+      auto iter = new ForwardIterator(this, read_options, cfh->cfd(), sv,
                                      /* allow_unprepared_value */ true);
      iterators->push_back(NewDBIterator(
-          env_, read_options, *cfd->ioptions(), sv->mutable_cf_options,
-          cfd->user_comparator(), iter, sv->current, kMaxSequenceNumber,
+          env_, read_options, *cfh->cfd()->ioptions(), sv->mutable_cf_options,
+          cfh->cfd()->user_comparator(), iter, sv->current, kMaxSequenceNumber,
          sv->mutable_cf_options.max_sequential_skip_in_iterations,
-          read_callback, this, cfd));
+          nullptr /*read_callback*/, cfh));
    }
  } else {
    // Note: no need to consider the special case of
@@ -4071,9 +3848,9 @@ Status DBImpl::NewIterators(
    auto snapshot = read_options.snapshot != nullptr
                        ? read_options.snapshot->GetSequenceNumber()
                        : versions_->LastSequence();
-    for (auto [cfd, sv] : cfd_to_sv) {
-      iterators->push_back(
-          NewIteratorImpl(read_options, cfd, sv, snapshot, read_callback));
+    for (auto [cfh, sv] : cfh_to_sv) {
+      iterators->push_back(NewIteratorImpl(read_options, cfh, sv, snapshot,
+                                           nullptr /*read_callback*/));
    }
  }

@@ -4285,7 +4062,7 @@ void DBImpl::ReleaseSnapshot(const Snapshot* s) {
    // inplace_update_support enabled.
    return;
  }
-  const SnapshotImpl* casted_s = reinterpret_cast<const SnapshotImpl*>(s);
+  const SnapshotImpl* casted_s = static_cast<const SnapshotImpl*>(s);
  {
    InstrumentedMutexLock l(&mutex_);
    snapshots_.Delete(casted_s);
@@ -4370,7 +4147,25 @@ Status DBImpl::GetPropertiesOfTablesInRange(ColumnFamilyHandle* column_family,

  // TODO: plumb Env::IOActivity, Env::IOPriority
  const ReadOptions read_options;
-  auto s = version->GetPropertiesOfTablesInRange(read_options, range, n, props);
+  const Comparator* const ucmp = cfd->user_comparator();
+  assert(ucmp);
+  size_t ts_sz = ucmp->timestamp_size();
+
+  autovector<UserKeyRange> ukey_ranges;
+  std::vector<std::string> keys;
+  ukey_ranges.reserve(n);
+  keys.reserve(2 * n);
+  // Add timestamp if needed
+  for (size_t i = 0; i < n; i++) {
+    auto [start, limit] = MaybeAddTimestampsToRange(
+        &range[i].start, &range[i].limit, ts_sz, &keys.emplace_back(),
+        &keys.emplace_back(), /*exclusive_end=*/false);
+    assert(start.has_value());
+    assert(limit.has_value());
+    ukey_ranges.emplace_back(start.value(), limit.value());
+  }
+  auto s =
+      version->GetPropertiesOfTablesInRange(read_options, ukey_ranges, props);

  // Decrement the ref count
  mutex_.Lock();
@@ -4860,9 +4655,9 @@ Status DBImpl::DeleteFile(std::string name) {
                                    read_options, write_options, &edit, &mutex_,
                                    directories_.GetDbDir());
    if (status.ok()) {
-      InstallSuperVersionAndScheduleWork(cfd,
-                                         &job_context.superversion_contexts[0],
-                                         *cfd->GetLatestMutableCFOptions());
+      InstallSuperVersionAndScheduleWork(
+          cfd, job_context.superversion_contexts.data(),
+          *cfd->GetLatestMutableCFOptions());
    }
    FindObsoleteFiles(&job_context, false);
  }  // lock released here
@@ -4887,6 +4682,24 @@ Status DBImpl::DeleteFilesInRanges(ColumnFamilyHandle* column_family,
  Status status = Status::OK();
  auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(column_family);
  ColumnFamilyData* cfd = cfh->cfd();
+  const Comparator* ucmp = cfd->user_comparator();
+  assert(ucmp);
+  const size_t ts_sz = ucmp->timestamp_size();
+  autovector<UserKeyRangePtr> ukey_ranges;
+  std::vector<std::string> keys;
+  std::vector<Slice> key_slices;
+  ukey_ranges.reserve(n);
+  keys.reserve(2 * n);
+  key_slices.reserve(2 * n);
+  for (size_t i = 0; i < n; i++) {
+    auto [start, limit] = MaybeAddTimestampsToRange(
+        ranges[i].start, ranges[i].limit, ts_sz, &keys.emplace_back(),
+        &keys.emplace_back(), !include_end);
+    assert((ranges[i].start != nullptr) == start.has_value());
+    assert((ranges[i].limit != nullptr) == limit.has_value());
+    ukey_ranges.emplace_back(start, limit);
+  }
+
  VersionEdit edit;
  std::set<FileMetaData*> deleted_files;
  JobContext job_context(next_job_id_.fetch_add(1), true);
@@ -4895,8 +4708,9 @@ Status DBImpl::DeleteFilesInRanges(ColumnFamilyHandle* column_family,
    Version* input_version = cfd->current();

    auto* vstorage = input_version->storage_info();
-    for (size_t r = 0; r < n; r++) {
-      auto begin = ranges[r].start, end = ranges[r].limit;
+    for (const auto& range : ukey_ranges) {
+      auto begin = range.start.has_value() ? &range.start.value() : nullptr;
+      auto end = range.limit.has_value() ? &range.limit.value() : nullptr;
      for (int i = 1; i < cfd->NumberLevels(); i++) {
        if (vstorage->LevelFiles(i).empty() ||
            !vstorage->OverlapInLevel(i, begin, end)) {
@@ -4930,8 +4744,8 @@ Status DBImpl::DeleteFilesInRanges(ColumnFamilyHandle* column_family,
            continue;
          }
          if (!include_end && end != nullptr &&
-              cfd->user_comparator()->Compare(level_file->largest.user_key(),
-                                              *end) == 0) {
+              (ucmp->CompareWithoutTimestamp(level_file->largest.user_key(),
+                                             *end) == 0)) {
            continue;
          }
          edit.SetColumnFamily(cfd->GetID());
@@ -4954,9 +4768,9 @@ Status DBImpl::DeleteFilesInRanges(ColumnFamilyHandle* column_family,
                                    read_options, write_options, &edit, &mutex_,
                                    directories_.GetDbDir());
    if (status.ok()) {
-      InstallSuperVersionAndScheduleWork(cfd,
-                                         &job_context.superversion_contexts[0],
-                                         *cfd->GetLatestMutableCFOptions());
+      InstallSuperVersionAndScheduleWork(
+          cfd, job_context.superversion_contexts.data(),
+          *cfd->GetLatestMutableCFOptions());
    }
    for (auto* deleted_file : deleted_files) {
      deleted_file->being_compacted = false;
@@ -5191,7 +5005,7 @@ Status DB::DestroyColumnFamilyHandle(ColumnFamilyHandle* column_family) {
  return Status::OK();
 }

-DB::~DB() {}
+DB::~DB() = default;

 Status DBImpl::Close() {
  InstrumentedMutexLock closing_lock_guard(&closing_mutex_);
@@ -5218,7 +5032,7 @@ Status DB::ListColumnFamilies(const DBOptions& db_options,
  return VersionSet::ListColumnFamilies(column_families, name, fs.get());
 }

-Snapshot::~Snapshot() {}
+Snapshot::~Snapshot() = default;

 Status DestroyDB(const std::string& dbname, const Options& options,
                 const std::vector<ColumnFamilyDescriptor>& column_families) {
@@ -5514,16 +5328,23 @@ Status DBImpl::RenameTempFileToOptionsFile(const std::string& file_name) {
      }
    }
  }
+
  if (s.ok()) {
-    InstrumentedMutexLock l(&mutex_);
-    versions_->options_file_number_ = options_file_number;
-    versions_->options_file_size_ = options_file_size;
+    int my_disable_delete_obsolete_files;
+
+    {
+      InstrumentedMutexLock l(&mutex_);
+      versions_->options_file_number_ = options_file_number;
+      versions_->options_file_size_ = options_file_size;
+      my_disable_delete_obsolete_files = disable_delete_obsolete_files_;
+    }
+
+    if (!my_disable_delete_obsolete_files) {
+      // TODO: Should we check for errors here?
+      DeleteObsoleteOptionsFiles().PermitUncheckedError();
+    }
  }

-  if (0 == disable_delete_obsolete_files_) {
-    // TODO: Should we check for errors here?
-    DeleteObsoleteOptionsFiles().PermitUncheckedError();
-  }
  return s;
 }

@@ -5780,10 +5601,18 @@ Status DBImpl::IngestExternalFiles(
  }
  for (const auto& arg : args) {
    const IngestExternalFileOptions& ingest_opts = arg.options;
-    if (ingest_opts.ingest_behind &&
-        !immutable_db_options_.allow_ingest_behind) {
-      return Status::InvalidArgument(
-          "can't ingest_behind file in DB with allow_ingest_behind=false");
+    if (ingest_opts.ingest_behind) {
+      if (!immutable_db_options_.allow_ingest_behind) {
+        return Status::InvalidArgument(
+            "can't ingest_behind file in DB with allow_ingest_behind=false");
+      }
+      auto ucmp = arg.column_family->GetComparator();
+      assert(ucmp);
+      if (ucmp->timestamp_size() > 0) {
+        return Status::NotSupported(
+            "Column family with user-defined "
+            "timestamps enabled doesn't support ingest behind.");
+      }
    }
  }

@@ -6235,8 +6064,8 @@ Status DBImpl::ClipColumnFamily(ColumnFamilyHandle* column_family,
  if (status.ok()) {
    // DeleteFilesInRanges non-overlap files except L0
    std::vector<RangePtr> ranges;
-    ranges.push_back(RangePtr(nullptr, &begin_key));
-    ranges.push_back(RangePtr(&end_key, nullptr));
+    ranges.emplace_back(nullptr, &begin_key);
+    ranges.emplace_back(&end_key, nullptr);
    status = DeleteFilesInRanges(column_family, ranges.data(), ranges.size());
  }

@@ -6484,7 +6313,7 @@ void DBImpl::NotifyOnExternalFileIngested(
    info.internal_file_path = f.internal_file_path;
    info.global_seqno = f.assigned_seqno;
    info.table_properties = f.table_properties;
-    for (auto listener : immutable_db_options_.listeners) {
+    for (const auto& listener : immutable_db_options_.listeners) {
      listener->OnExternalFileIngested(this, info);
    }
  }
@@ -6644,6 +6473,8 @@ void DBImpl::RecordSeqnoToTimeMapping(uint64_t populate_historical_seconds) {
  immutable_db_options_.clock->GetCurrentTime(&unix_time_signed)
      .PermitUncheckedError();  // Ignore error
  uint64_t unix_time = static_cast<uint64_t>(unix_time_signed);
+
+  std::vector<SuperVersionContext> sv_contexts;
  if (populate_historical_seconds > 0) {
    bool success = true;
    {
@@ -6654,6 +6485,7 @@ void DBImpl::RecordSeqnoToTimeMapping(uint64_t populate_historical_seconds) {
        success = seqno_to_time_mapping_.PrePopulate(
            from_seqno, seqno, unix_time - populate_historical_seconds,
            unix_time);
+        InstallSeqnoToTimeMappingInSV(&sv_contexts);
      } else {
        // One of these will fail
        assert(seqno > 1);
@@ -6679,7 +6511,31 @@ void DBImpl::RecordSeqnoToTimeMapping(uint64_t populate_historical_seconds) {
    // FIXME: assert(seqno > 0);
    // Always successful assuming seqno never go backwards
    seqno_to_time_mapping_.Append(seqno, unix_time);
+    InstallSeqnoToTimeMappingInSV(&sv_contexts);
+  }
+
+  // clean up outside db mutex
+  for (SuperVersionContext& sv_context : sv_contexts) {
+    sv_context.Clean();
  }
 }

+void DBImpl::InstallSeqnoToTimeMappingInSV(
+    std::vector<SuperVersionContext>* sv_contexts) {
+  mutex_.AssertHeld();
+  std::shared_ptr<SeqnoToTimeMapping> new_seqno_to_time_mapping =
+      std::make_shared<SeqnoToTimeMapping>();
+  new_seqno_to_time_mapping->CopyFrom(seqno_to_time_mapping_);
+  for (ColumnFamilyData* cfd : *versions_->GetColumnFamilySet()) {
+    if (cfd->IsDropped()) {
+      continue;
+    }
+    sv_contexts->emplace_back(/*create_superversion=*/true);
+    sv_contexts->back().new_seqno_to_time_mapping = new_seqno_to_time_mapping;
+    cfd->InstallSuperVersion(&sv_contexts->back(),
+                             *(cfd->GetLatestMutableCFOptions()));
+  }
+  bg_cv_.SignalAll();
+}
+
 }  // namespace ROCKSDB_NAMESPACE
@@ -59,7 +59,6 @@
 #include "rocksdb/utilities/replayer.h"
 #include "rocksdb/write_buffer_manager.h"
 #include "table/merging_iterator.h"
-#include "table/scoped_arena_iterator.h"
 #include "util/autovector.h"
 #include "util/hash.h"
 #include "util/repeatable_thread.h"
@@ -229,16 +228,12 @@ class DBImpl : public DB {
                     const Slice& end_key, const Slice& ts) override;

  using DB::Write;
-  virtual Status Write(const WriteOptions& options,
-                       WriteBatch* updates) override;
+  Status Write(const WriteOptions& options, WriteBatch* updates) override;

  using DB::Get;
-  virtual Status Get(const ReadOptions& options,
-                     ColumnFamilyHandle* column_family, const Slice& key,
-                     PinnableSlice* value) override;
-  virtual Status Get(const ReadOptions& _read_options,
-                     ColumnFamilyHandle* column_family, const Slice& key,
-                     PinnableSlice* value, std::string* timestamp) override;
+  Status Get(const ReadOptions& _read_options,
+             ColumnFamilyHandle* column_family, const Slice& key,
+             PinnableSlice* value, std::string* timestamp) override;

  using DB::GetEntity;
  Status GetEntity(const ReadOptions& options,
@@ -263,17 +258,6 @@ class DBImpl : public DB {
  }

  using DB::MultiGet;
-  virtual std::vector<Status> MultiGet(
-      const ReadOptions& options,
-      const std::vector<ColumnFamilyHandle*>& column_family,
-      const std::vector<Slice>& keys,
-      std::vector<std::string>* values) override;
-  virtual std::vector<Status> MultiGet(
-      const ReadOptions& _read_options,
-      const std::vector<ColumnFamilyHandle*>& column_family,
-      const std::vector<Slice>& keys, std::vector<std::string>* values,
-      std::vector<std::string>* timestamps) override;
-
  // This MultiGet is a batched version, which may be faster than calling Get
  // multiple times, especially if the keys have some spatial locality that
  // enables them to be queried in the same SST files/set of files. The larger
@@ -281,19 +265,6 @@ class DBImpl : public DB {
  // The values and statuses parameters are arrays with number of elements
  // equal to keys.size(). This allows the storage for those to be alloacted
  // by the caller on the stack for small batches
-  void MultiGet(const ReadOptions& options, ColumnFamilyHandle* column_family,
-                const size_t num_keys, const Slice* keys, PinnableSlice* values,
-                Status* statuses, const bool sorted_input = false) override;
-  void MultiGet(const ReadOptions& _read_options,
-                ColumnFamilyHandle* column_family, const size_t num_keys,
-                const Slice* keys, PinnableSlice* values,
-                std::string* timestamps, Status* statuses,
-                const bool sorted_input = false) override;
-
-  void MultiGet(const ReadOptions& options, const size_t num_keys,
-                ColumnFamilyHandle** column_families, const Slice* keys,
-                PinnableSlice* values, Status* statuses,
-                const bool sorted_input = false) override;
  void MultiGet(const ReadOptions& _read_options, const size_t num_keys,
                ColumnFamilyHandle** column_families, const Slice* keys,
                PinnableSlice* values, std::string* timestamps,
@@ -319,9 +290,9 @@ class DBImpl : public DB {
                      const Slice* keys,
                      PinnableAttributeGroups* results) override;

-  virtual Status CreateColumnFamily(const ColumnFamilyOptions& cf_options,
-                                    const std::string& column_family,
-                                    ColumnFamilyHandle** handle) override {
+  Status CreateColumnFamily(const ColumnFamilyOptions& cf_options,
+                            const std::string& column_family,
+                            ColumnFamilyHandle** handle) override {
    // TODO: plumb Env::IOActivity, Env::IOPriority
    return CreateColumnFamily(ReadOptions(), WriteOptions(), cf_options,
                              column_family, handle);
@@ -331,7 +302,7 @@ class DBImpl : public DB {
                                    const ColumnFamilyOptions& cf_options,
                                    const std::string& column_family,
                                    ColumnFamilyHandle** handle);
-  virtual Status CreateColumnFamilies(
+  Status CreateColumnFamilies(
      const ColumnFamilyOptions& cf_options,
      const std::vector<std::string>& column_family_names,
      std::vector<ColumnFamilyHandle*>* handles) override {
@@ -345,7 +316,7 @@ class DBImpl : public DB {
      const std::vector<std::string>& column_family_names,
      std::vector<ColumnFamilyHandle*>* handles);

-  virtual Status CreateColumnFamilies(
+  Status CreateColumnFamilies(
      const std::vector<ColumnFamilyDescriptor>& column_families,
      std::vector<ColumnFamilyHandle*>* handles) override {
    // TODO: plumb Env::IOActivity, Env::IOPriority
@@ -356,8 +327,8 @@ class DBImpl : public DB {
      const ReadOptions& read_options, const WriteOptions& write_options,
      const std::vector<ColumnFamilyDescriptor>& column_families,
      std::vector<ColumnFamilyHandle*>* handles);
-  virtual Status DropColumnFamily(ColumnFamilyHandle* column_family) override;
-  virtual Status DropColumnFamilies(
+  Status DropColumnFamily(ColumnFamilyHandle* column_family) override;
+  Status DropColumnFamilies(
      const std::vector<ColumnFamilyHandle*>& column_families) override;

  // Returns false if key doesn't exist in the database and true if it may.
@@ -365,21 +336,27 @@ class DBImpl : public DB {
  // memory. On return, if value was found, then value_found will be set to true
  // , otherwise false.
  using DB::KeyMayExist;
-  virtual bool KeyMayExist(const ReadOptions& options,
-                           ColumnFamilyHandle* column_family, const Slice& key,
-                           std::string* value, std::string* timestamp,
-                           bool* value_found = nullptr) override;
+  bool KeyMayExist(const ReadOptions& options,
+                   ColumnFamilyHandle* column_family, const Slice& key,
+                   std::string* value, std::string* timestamp,
+                   bool* value_found = nullptr) override;

  using DB::NewIterator;
-  virtual Iterator* NewIterator(const ReadOptions& _read_options,
-                                ColumnFamilyHandle* column_family) override;
-  virtual Status NewIterators(
-      const ReadOptions& _read_options,
-      const std::vector<ColumnFamilyHandle*>& column_families,
-      std::vector<Iterator*>* iterators) override;
+  Iterator* NewIterator(const ReadOptions& _read_options,
+                        ColumnFamilyHandle* column_family) override;
+  Status NewIterators(const ReadOptions& _read_options,
+                      const std::vector<ColumnFamilyHandle*>& column_families,
+                      std::vector<Iterator*>* iterators) override;
+
+  const Snapshot* GetSnapshot() override;
+  void ReleaseSnapshot(const Snapshot* snapshot) override;
+
+  // UNDER CONSTRUCTION - DO NOT USE
+  // Return a cross-column-family iterator from a consistent database state.
+  std::unique_ptr<Iterator> NewMultiCfIterator(
+      const ReadOptions& options,
+      const std::vector<ColumnFamilyHandle*>& column_families) override;

-  virtual const Snapshot* GetSnapshot() override;
-  virtual void ReleaseSnapshot(const Snapshot* snapshot) override;
  // Create a timestamped snapshot. This snapshot can be shared by multiple
  // readers. If any of them uses it for write conflict checking, then
  // is_write_conflict_boundary is true. For simplicity, set it to true by
@@ -394,35 +371,33 @@ class DBImpl : public DB {
                                     timestamped_snapshots) const;

  using DB::GetProperty;
-  virtual bool GetProperty(ColumnFamilyHandle* column_family,
-                           const Slice& property, std::string* value) override;
+  bool GetProperty(ColumnFamilyHandle* column_family, const Slice& property,
+                   std::string* value) override;
  using DB::GetMapProperty;
-  virtual bool GetMapProperty(
-      ColumnFamilyHandle* column_family, const Slice& property,
-      std::map<std::string, std::string>* value) override;
+  bool GetMapProperty(ColumnFamilyHandle* column_family, const Slice& property,
+                      std::map<std::string, std::string>* value) override;
  using DB::GetIntProperty;
-  virtual bool GetIntProperty(ColumnFamilyHandle* column_family,
-                              const Slice& property, uint64_t* value) override;
+  bool GetIntProperty(ColumnFamilyHandle* column_family, const Slice& property,
+                      uint64_t* value) override;
  using DB::GetAggregatedIntProperty;
-  virtual bool GetAggregatedIntProperty(const Slice& property,
-                                        uint64_t* aggregated_value) override;
+  bool GetAggregatedIntProperty(const Slice& property,
+                                uint64_t* aggregated_value) override;
  using DB::GetApproximateSizes;
-  virtual Status GetApproximateSizes(const SizeApproximationOptions& options,
-                                     ColumnFamilyHandle* column_family,
-                                     const Range* range, int n,
-                                     uint64_t* sizes) override;
+  Status GetApproximateSizes(const SizeApproximationOptions& options,
+                             ColumnFamilyHandle* column_family,
+                             const Range* range, int n,
+                             uint64_t* sizes) override;
  using DB::GetApproximateMemTableStats;
-  virtual void GetApproximateMemTableStats(ColumnFamilyHandle* column_family,
-                                           const Range& range,
-                                           uint64_t* const count,
-                                           uint64_t* const size) override;
+  void GetApproximateMemTableStats(ColumnFamilyHandle* column_family,
+                                   const Range& range, uint64_t* const count,
+                                   uint64_t* const size) override;
  using DB::CompactRange;
-  virtual Status CompactRange(const CompactRangeOptions& options,
-                              ColumnFamilyHandle* column_family,
-                              const Slice* begin, const Slice* end) override;
+  Status CompactRange(const CompactRangeOptions& options,
+                      ColumnFamilyHandle* column_family, const Slice* begin,
+                      const Slice* end) override;

  using DB::CompactFiles;
-  virtual Status CompactFiles(
+  Status CompactFiles(
      const CompactionOptions& compact_options,
      ColumnFamilyHandle* column_family,
      const std::vector<std::string>& input_file_names, const int output_level,
@@ -430,55 +405,54 @@ class DBImpl : public DB {
      std::vector<std::string>* const output_file_names = nullptr,
      CompactionJobInfo* compaction_job_info = nullptr) override;

-  virtual Status PauseBackgroundWork() override;
-  virtual Status ContinueBackgroundWork() override;
+  Status PauseBackgroundWork() override;
+  Status ContinueBackgroundWork() override;

-  virtual Status EnableAutoCompaction(
+  Status EnableAutoCompaction(
      const std::vector<ColumnFamilyHandle*>& column_family_handles) override;

-  virtual void EnableManualCompaction() override;
-  virtual void DisableManualCompaction() override;
+  void EnableManualCompaction() override;
+  void DisableManualCompaction() override;

  using DB::SetOptions;
  Status SetOptions(
      ColumnFamilyHandle* column_family,
      const std::unordered_map<std::string, std::string>& options_map) override;

-  virtual Status SetDBOptions(
+  Status SetDBOptions(
      const std::unordered_map<std::string, std::string>& options_map) override;

  using DB::NumberLevels;
-  virtual int NumberLevels(ColumnFamilyHandle* column_family) override;
+  int NumberLevels(ColumnFamilyHandle* column_family) override;
  using DB::MaxMemCompactionLevel;
-  virtual int MaxMemCompactionLevel(ColumnFamilyHandle* column_family) override;
+  int MaxMemCompactionLevel(ColumnFamilyHandle* column_family) override;
  using DB::Level0StopWriteTrigger;
-  virtual int Level0StopWriteTrigger(
-      ColumnFamilyHandle* column_family) override;
-  virtual const std::string& GetName() const override;
-  virtual Env* GetEnv() const override;
-  virtual FileSystem* GetFileSystem() const override;
+  int Level0StopWriteTrigger(ColumnFamilyHandle* column_family) override;
+  const std::string& GetName() const override;
+  Env* GetEnv() const override;
+  FileSystem* GetFileSystem() const override;
  using DB::GetOptions;
-  virtual Options GetOptions(ColumnFamilyHandle* column_family) const override;
+  Options GetOptions(ColumnFamilyHandle* column_family) const override;
  using DB::GetDBOptions;
-  virtual DBOptions GetDBOptions() const override;
+  DBOptions GetDBOptions() const override;
  using DB::Flush;
-  virtual Status Flush(const FlushOptions& options,
-                       ColumnFamilyHandle* column_family) override;
-  virtual Status Flush(
+  Status Flush(const FlushOptions& options,
+               ColumnFamilyHandle* column_family) override;
+  Status Flush(
      const FlushOptions& options,
      const std::vector<ColumnFamilyHandle*>& column_families) override;
-  virtual Status FlushWAL(bool sync) override {
+  Status FlushWAL(bool sync) override {
    // TODO: plumb Env::IOActivity, Env::IOPriority
    return FlushWAL(WriteOptions(), sync);
  }

  virtual Status FlushWAL(const WriteOptions& write_options, bool sync);
  bool WALBufferIsEmpty();
-  virtual Status SyncWAL() override;
-  virtual Status LockWAL() override;
-  virtual Status UnlockWAL() override;
+  Status SyncWAL() override;
+  Status LockWAL() override;
+  Status UnlockWAL() override;

-  virtual SequenceNumber GetLatestSequenceNumber() const override;
+  SequenceNumber GetLatestSequenceNumber() const override;

  // IncreaseFullHistoryTsLow(ColumnFamilyHandle*, std::string) will acquire
  // and release db_mutex
@@ -490,21 +464,21 @@ class DBImpl : public DB {
  Status GetFullHistoryTsLow(ColumnFamilyHandle* column_family,
                             std::string* ts_low) override;

-  virtual Status GetDbIdentity(std::string& identity) const override;
+  Status GetDbIdentity(std::string& identity) const override;

  virtual Status GetDbIdentityFromIdentityFile(std::string* identity) const;

-  virtual Status GetDbSessionId(std::string& session_id) const override;
+  Status GetDbSessionId(std::string& session_id) const override;

  ColumnFamilyHandle* DefaultColumnFamily() const override;

  ColumnFamilyHandle* PersistentStatsColumnFamily() const;

-  virtual Status Close() override;
+  Status Close() override;

-  virtual Status DisableFileDeletions() override;
+  Status DisableFileDeletions() override;

-  virtual Status EnableFileDeletions(bool force) override;
+  Status EnableFileDeletions() override;

  virtual bool IsFileDeletionsEnabled() const;

@@ -513,40 +487,35 @@ class DBImpl : public DB {
      std::unique_ptr<StatsHistoryIterator>* stats_iterator) override;

  using DB::ResetStats;
-  virtual Status ResetStats() override;
+  Status ResetStats() override;
  // All the returned filenames start with "/"
-  virtual Status GetLiveFiles(std::vector<std::string>&,
-                              uint64_t* manifest_file_size,
-                              bool flush_memtable = true) override;
-  virtual Status GetSortedWalFiles(VectorLogPtr& files) override;
-  virtual Status GetCurrentWalFile(
-      std::unique_ptr<LogFile>* current_log_file) override;
-  virtual Status GetCreationTimeOfOldestFile(
-      uint64_t* creation_time) override;
+  Status GetLiveFiles(std::vector<std::string>&, uint64_t* manifest_file_size,
+                      bool flush_memtable = true) override;
+  Status GetSortedWalFiles(VectorLogPtr& files) override;
+  Status GetCurrentWalFile(std::unique_ptr<LogFile>* current_log_file) override;
+  Status GetCreationTimeOfOldestFile(uint64_t* creation_time) override;

-  virtual Status GetUpdatesSince(
+  Status GetUpdatesSince(
      SequenceNumber seq_number, std::unique_ptr<TransactionLogIterator>* iter,
      const TransactionLogIterator::ReadOptions& read_options =
          TransactionLogIterator::ReadOptions()) override;
-  virtual Status DeleteFile(std::string name) override;
+  Status DeleteFile(std::string name) override;
  Status DeleteFilesInRanges(ColumnFamilyHandle* column_family,
                             const RangePtr* ranges, size_t n,
                             bool include_end = true);

-  virtual void GetLiveFilesMetaData(
-      std::vector<LiveFileMetaData>* metadata) override;
+  void GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata) override;

-  virtual Status GetLiveFilesChecksumInfo(
-      FileChecksumList* checksum_list) override;
+  Status GetLiveFilesChecksumInfo(FileChecksumList* checksum_list) override;

-  virtual Status GetLiveFilesStorageInfo(
+  Status GetLiveFilesStorageInfo(
      const LiveFilesStorageInfoOptions& opts,
      std::vector<LiveFileStorageInfo>* files) override;

  // Obtains the meta data of the specified column family of the DB.
  // TODO(yhchiang): output parameter is placed in the end in this codebase.
-  virtual void GetColumnFamilyMetaData(ColumnFamilyHandle* column_family,
-                                       ColumnFamilyMetaData* metadata) override;
+  void GetColumnFamilyMetaData(ColumnFamilyHandle* column_family,
+                               ColumnFamilyMetaData* metadata) override;

  void GetAllColumnFamilyMetaData(
      std::vector<ColumnFamilyMetaData>* metadata) override;
@@ -558,32 +527,32 @@ class DBImpl : public DB {
                   int target_level) override;

  using DB::IngestExternalFile;
-  virtual Status IngestExternalFile(
+  Status IngestExternalFile(
      ColumnFamilyHandle* column_family,
      const std::vector<std::string>& external_files,
      const IngestExternalFileOptions& ingestion_options) override;

  using DB::IngestExternalFiles;
-  virtual Status IngestExternalFiles(
+  Status IngestExternalFiles(
      const std::vector<IngestExternalFileArg>& args) override;

  using DB::CreateColumnFamilyWithImport;
-  virtual Status CreateColumnFamilyWithImport(
+  Status CreateColumnFamilyWithImport(
      const ColumnFamilyOptions& options, const std::string& column_family_name,
      const ImportColumnFamilyOptions& import_options,
      const std::vector<const ExportImportFilesMetaData*>& metadatas,
      ColumnFamilyHandle** handle) override;

  using DB::ClipColumnFamily;
-  virtual Status ClipColumnFamily(ColumnFamilyHandle* column_family,
-                                  const Slice& begin_key,
-                                  const Slice& end_key) override;
+  Status ClipColumnFamily(ColumnFamilyHandle* column_family,
+                          const Slice& begin_key,
+                          const Slice& end_key) override;

  using DB::VerifyFileChecksums;
  Status VerifyFileChecksums(const ReadOptions& read_options) override;

  using DB::VerifyChecksum;
-  virtual Status VerifyChecksum(const ReadOptions& /*read_options*/) override;
+  Status VerifyChecksum(const ReadOptions& /*read_options*/) override;
  // Verify the checksums of files in db. Currently only tables are checked.
  //
  // read_options: controls file I/O behavior, e.g. read ahead size while
@@ -604,18 +573,16 @@ class DBImpl : public DB {
                                const ReadOptions& read_options);

  using DB::StartTrace;
-  virtual Status StartTrace(
-      const TraceOptions& options,
-      std::unique_ptr<TraceWriter>&& trace_writer) override;
+  Status StartTrace(const TraceOptions& options,
+                    std::unique_ptr<TraceWriter>&& trace_writer) override;

  using DB::EndTrace;
-  virtual Status EndTrace() override;
+  Status EndTrace() override;

  using DB::NewDefaultReplayer;
-  virtual Status NewDefaultReplayer(
-      const std::vector<ColumnFamilyHandle*>& handles,
-      std::unique_ptr<TraceReader>&& reader,
-      std::unique_ptr<Replayer>* replayer) override;
+  Status NewDefaultReplayer(const std::vector<ColumnFamilyHandle*>& handles,
+                            std::unique_ptr<TraceReader>&& reader,
+                            std::unique_ptr<Replayer>* replayer) override;

  using DB::StartBlockCacheTrace;
  Status StartBlockCacheTrace(
@@ -637,14 +604,12 @@ class DBImpl : public DB {
  Status EndIOTrace() override;

  using DB::GetPropertiesOfAllTables;
-  virtual Status GetPropertiesOfAllTables(
-      ColumnFamilyHandle* column_family,
-      TablePropertiesCollection* props) override;
-  virtual Status GetPropertiesOfTablesInRange(
+  Status GetPropertiesOfAllTables(ColumnFamilyHandle* column_family,
+                                  TablePropertiesCollection* props) override;
+  Status GetPropertiesOfTablesInRange(
      ColumnFamilyHandle* column_family, const Range* range, std::size_t n,
      TablePropertiesCollection* props) override;

-
  // ---- End of implementations of the DB interface ----
  SystemClock* GetSystemClock() const;

@@ -687,8 +652,8 @@ class DBImpl : public DB {

  // If `snapshot` == kMaxSequenceNumber, set a recent one inside the file.
  ArenaWrappedDBIter* NewIteratorImpl(const ReadOptions& options,
-                                      ColumnFamilyData* cfd, SuperVersion* sv,
-                                      SequenceNumber snapshot,
+                                      ColumnFamilyHandleImpl* cfh,
+                                      SuperVersion* sv, SequenceNumber snapshot,
                                      ReadCallback* read_callback,
                                      bool expose_blob_index = false,
                                      bool allow_refresh = true);
@@ -1181,6 +1146,8 @@ class DBImpl : public DB {

  void TEST_UnlockMutex();

+  InstrumentedMutex* TEST_Mutex() { return &mutex_; }
+
  void TEST_SignalAllBgCv();

  // REQUIRES: mutex locked
@@ -1258,6 +1225,22 @@ class DBImpl : public DB {
  // populate_historical_seconds, now].
  void RecordSeqnoToTimeMapping(uint64_t populate_historical_seconds);

+  // Everytime DB's seqno to time mapping changed (which already hold the db
+  // mutex), we install a new SuperVersion in each column family with a shared
+  // copy of the new mapping while holding the db mutex.
+  // This is done for all column families even though the column family does not
+  // explicitly enabled the
+  // `preclude_last_level_data_seconds` or `preserve_internal_time_seconds`
+  // features.
+  // This mapping supports iterators to fulfill the
+  // "rocksdb.iterator.write-time" iterator property for entries in memtables.
+  //
+  // Since this new SuperVersion doesn't involve an LSM tree shape change, we
+  // don't schedule work after installing this SuperVersion. It returns the used
+  // `SuperVersionContext` for clean up after release mutex.
+  void InstallSeqnoToTimeMappingInSV(
+      std::vector<SuperVersionContext>* sv_contexts);
+
  // Interface to block and signal the DB in case of stalling writes by
  // WriteBufferManager. Each DBImpl object contains ptr to WBMStallInterface.
  // When DB needs to be blocked or signalled by WriteBufferManager,
@@ -2853,17 +2836,16 @@ class GetWithTimestampReadCallback : public ReadCallback {
  }
 };

-extern Options SanitizeOptions(const std::string& db, const Options& src,
-                               bool read_only = false,
-                               Status* logger_creation_s = nullptr);
+Options SanitizeOptions(const std::string& db, const Options& src,
+                        bool read_only = false,
+                        Status* logger_creation_s = nullptr);

-extern DBOptions SanitizeOptions(const std::string& db, const DBOptions& src,
-                                 bool read_only = false,
-                                 Status* logger_creation_s = nullptr);
+DBOptions SanitizeOptions(const std::string& db, const DBOptions& src,
+                          bool read_only = false,
+                          Status* logger_creation_s = nullptr);

-extern CompressionType GetCompressionFlush(
-    const ImmutableCFOptions& ioptions,
-    const MutableCFOptions& mutable_cf_options);
+CompressionType GetCompressionFlush(const ImmutableCFOptions& ioptions,
+                                    const MutableCFOptions& mutable_cf_options);

 // Return the earliest log file to keep after the memtable flush is
 // finalized.
@@ -2871,13 +2853,13 @@ extern CompressionType GetCompressionFlush(
 // `memtables_to_flush`) will be flushed and thus will not depend on any WAL
 // file.
 // The function is only applicable to 2pc mode.
-extern uint64_t PrecomputeMinLogNumberToKeep2PC(
+uint64_t PrecomputeMinLogNumberToKeep2PC(
    VersionSet* vset, const ColumnFamilyData& cfd_to_flush,
    const autovector<VersionEdit*>& edit_list,
    const autovector<MemTable*>& memtables_to_flush,
    LogsWithPrepTracker* prep_tracker);
 // For atomic flush.
-extern uint64_t PrecomputeMinLogNumberToKeep2PC(
+uint64_t PrecomputeMinLogNumberToKeep2PC(
    VersionSet* vset, const autovector<ColumnFamilyData*>& cfds_to_flush,
    const autovector<autovector<VersionEdit*>>& edit_lists,
    const autovector<const autovector<MemTable*>*>& memtables_to_flush,
@@ -2885,21 +2867,21 @@ extern uint64_t PrecomputeMinLogNumberToKeep2PC(

 // In non-2PC mode, WALs with log number < the returned number can be
 // deleted after the cfd_to_flush column family is flushed successfully.
-extern uint64_t PrecomputeMinLogNumberToKeepNon2PC(
+uint64_t PrecomputeMinLogNumberToKeepNon2PC(
    VersionSet* vset, const ColumnFamilyData& cfd_to_flush,
    const autovector<VersionEdit*>& edit_list);
 // For atomic flush.
-extern uint64_t PrecomputeMinLogNumberToKeepNon2PC(
+uint64_t PrecomputeMinLogNumberToKeepNon2PC(
    VersionSet* vset, const autovector<ColumnFamilyData*>& cfds_to_flush,
    const autovector<autovector<VersionEdit*>>& edit_lists);

 // `cfd_to_flush` is the column family whose memtable will be flushed and thus
 // will not depend on any WAL file. nullptr means no memtable is being flushed.
 // The function is only applicable to 2pc mode.
-extern uint64_t FindMinPrepLogReferencedByMemTable(
+uint64_t FindMinPrepLogReferencedByMemTable(
    VersionSet* vset, const autovector<MemTable*>& memtables_to_flush);
 // For atomic flush.
-extern uint64_t FindMinPrepLogReferencedByMemTable(
+uint64_t FindMinPrepLogReferencedByMemTable(
    VersionSet* vset,
    const autovector<const autovector<MemTable*>*>& memtables_to_flush);

@@ -163,8 +163,10 @@ IOStatus DBImpl::SyncClosedLogs(const WriteOptions& write_options,
        if (error_recovery_in_prog) {
          log->file()->reset_seen_error();
        }
-        // TODO: plumb Env::IOActivity, Env::IOPriority
-        io_s = log->Close(WriteOptions());
+        // Normally the log file is closed when purging obsolete file, but if
+        // log recycling is enabled, the log file is closed here so that it
+        // can be reused.
+        io_s = log->Close(write_options);
        if (!io_s.ok()) {
          break;
        }
@@ -270,8 +272,8 @@ Status DBImpl::FlushMemTableToOutputFile(
      GetCompressionFlush(*cfd->ioptions(), mutable_cf_options), stats_,
      &event_logger_, mutable_cf_options.report_bg_io_stats,
      true /* sync_output_directory */, true /* write_manifest */, thread_pri,
-      io_tracer_, seqno_to_time_mapping_, db_id_, db_session_id_,
-      cfd->GetFullHistoryTsLow(), &blob_callback_);
+      io_tracer_, cfd->GetSuperVersion()->ShareSeqnoToTimeMapping(), db_id_,
+      db_session_id_, cfd->GetFullHistoryTsLow(), &blob_callback_);
  FileMetaData file_meta;

  Status s;
@@ -545,8 +547,9 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
        GetCompressionFlush(*cfd->ioptions(), mutable_cf_options), stats_,
        &event_logger_, mutable_cf_options.report_bg_io_stats,
        false /* sync_output_directory */, false /* write_manifest */,
-        thread_pri, io_tracer_, seqno_to_time_mapping_, db_id_, db_session_id_,
-        cfd->GetFullHistoryTsLow(), &blob_callback_));
+        thread_pri, io_tracer_,
+        cfd->GetSuperVersion()->ShareSeqnoToTimeMapping(), db_id_,
+        db_session_id_, cfd->GetFullHistoryTsLow(), &blob_callback_));
  }

  std::vector<FileMetaData> file_meta(num_cfs);
@@ -970,7 +973,7 @@ void DBImpl::NotifyOnFlushBegin(ColumnFamilyData* cfd, FileMetaData* file_meta,
    info.smallest_seqno = file_meta->fd.smallest_seqno;
    info.largest_seqno = file_meta->fd.largest_seqno;
    info.flush_reason = flush_reason;
-    for (auto listener : immutable_db_options_.listeners) {
+    for (const auto& listener : immutable_db_options_.listeners) {
      listener->OnFlushBegin(this, info);
    }
  }
@@ -1002,7 +1005,7 @@ void DBImpl::NotifyOnFlushCompleted(
    for (auto& info : *flush_jobs_info) {
      info->triggered_writes_slowdown = triggered_writes_slowdown;
      info->triggered_writes_stop = triggered_writes_stop;
-      for (auto listener : immutable_db_options_.listeners) {
+      for (const auto& listener : immutable_db_options_.listeners) {
        listener->OnFlushCompleted(this, *info);
      }
      TEST_SYNC_POINT(
@@ -1140,7 +1143,7 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
    // TODO(ajkr): We could also optimize away the flush in certain cases where
    // one/both sides of the interval are unbounded. But it requires more
    // changes to RangesOverlapWithMemtables.
-    Range range(*begin, *end);
+    UserKeyRange range(*begin, *end);
    SuperVersion* super_version = cfd->GetReferencedSuperVersion(this);
    s = cfd->RangesOverlapWithMemtables(
        {range}, super_version, immutable_db_options_.allow_data_in_errors,
@@ -1419,10 +1422,9 @@ Status DBImpl::CompactFiles(const CompactionOptions& compact_options,

  // Perform CompactFiles
  TEST_SYNC_POINT("TestCompactFiles::IngestExternalFile2");
-  TEST_SYNC_POINT_CALLBACK(
-      "TestCompactFiles:PausingManualCompaction:3",
-      reinterpret_cast<void*>(
-          const_cast<std::atomic<int>*>(&manual_compaction_paused_)));
+  TEST_SYNC_POINT_CALLBACK("TestCompactFiles:PausingManualCompaction:3",
+                           static_cast<void*>(const_cast<std::atomic<int>*>(
+                               &manual_compaction_paused_)));
  {
    InstrumentedMutexLock l(&mutex_);
    auto* current = cfd->current();
@@ -1610,9 +1612,9 @@ Status DBImpl::CompactFilesImpl(
  }
  if (status.ok()) {
    assert(compaction_job.io_status().ok());
-    InstallSuperVersionAndScheduleWork(c->column_family_data(),
-                                       &job_context->superversion_contexts[0],
-                                       *c->mutable_cf_options());
+    InstallSuperVersionAndScheduleWork(
+        c->column_family_data(), job_context->superversion_contexts.data(),
+        *c->mutable_cf_options());
  }
  // status above captures any error during compaction_job.Install, so its ok
  // not check compaction_job.io_status() explicitly if we're not calling
@@ -1695,7 +1697,7 @@ Status DBImpl::PauseBackgroundWork() {
 Status DBImpl::ContinueBackgroundWork() {
  InstrumentedMutexLock guard_lock(&mutex_);
  if (bg_work_paused_ == 0) {
-    return Status::InvalidArgument();
+    return Status::InvalidArgument("Background work already unpaused");
  }
  assert(bg_work_paused_ > 0);
  assert(bg_compaction_paused_ > 0);
@@ -1732,7 +1734,7 @@ void DBImpl::NotifyOnCompactionBegin(ColumnFamilyData* cfd, Compaction* c,
  {
    CompactionJobInfo info{};
    BuildCompactionJobInfo(cfd, c, st, job_stats, job_id, &info);
-    for (auto listener : immutable_db_options_.listeners) {
+    for (const auto& listener : immutable_db_options_.listeners) {
      listener->OnCompactionBegin(this, info);
    }
    info.status.PermitUncheckedError();
@@ -1761,7 +1763,7 @@ void DBImpl::NotifyOnCompactionCompleted(
  {
    CompactionJobInfo info{};
    BuildCompactionJobInfo(cfd, c, st, compaction_job_stats, job_id, &info);
-    for (auto listener : immutable_db_options_.listeners) {
+    for (const auto& listener : immutable_db_options_.listeners) {
      listener->OnCompactionCompleted(this, info);
    }
  }
@@ -1869,7 +1871,8 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
        ,
        LLONG_MAX /* max compaction bytes, not applicable */,
        0 /* output path ID, not applicable */, mutable_cf_options.compression,
-        mutable_cf_options.compression_opts, Temperature::kUnknown,
+        mutable_cf_options.compression_opts,
+        mutable_cf_options.default_write_temperature,
        0 /* max_subcompactions, not applicable */,
        {} /* grandparents, not applicable */, false /* is manual */,
        "" /* trim_ts */, -1 /* score, not applicable */,
@@ -2151,16 +2154,6 @@ Status DBImpl::RunManualCompaction(
               manual.begin, manual.end, &manual.manual_end, &manual_conflict,
               max_file_num_to_ignore, trim_ts)) == nullptr &&
          manual_conflict))) {
-      if (!scheduled) {
-        // There is a conflicting compaction
-        if (manual_compaction_paused_ > 0 || manual.canceled == true) {
-          // Stop waiting since it was canceled. Pretend the error came from
-          // compaction so the below cleanup/error handling code can process it.
-          manual.done = true;
-          manual.status =
-              Status::Incomplete(Status::SubCode::kManualCompactionPaused);
-        }
-      }
      if (!manual.done) {
        bg_cv_.Wait();
      }
@@ -2235,6 +2228,17 @@ Status DBImpl::RunManualCompaction(
        *final_output_level = compaction->output_level();
      }
    }
+    if (!scheduled) {
+      // There is nothing scheduled to wait on, so any cancellation can end the
+      // manual now.
+      if (manual_compaction_paused_ > 0 || manual.canceled == true) {
+        // Stop waiting since it was canceled. Pretend the error came from
+        // compaction so the below cleanup/error handling code can process it.
+        manual.done = true;
+        manual.status =
+            Status::Incomplete(Status::SubCode::kManualCompactionPaused);
+      }
+    }
  }

  log_buffer.FlushBufferToLog();
@@ -3045,8 +3049,8 @@ void DBImpl::SchedulePendingPurge(std::string fname, std::string dir_to_sync,
 }

 void DBImpl::BGWorkFlush(void* arg) {
-  FlushThreadArg fta = *(reinterpret_cast<FlushThreadArg*>(arg));
-  delete reinterpret_cast<FlushThreadArg*>(arg);
+  FlushThreadArg fta = *(static_cast<FlushThreadArg*>(arg));
+  delete static_cast<FlushThreadArg*>(arg);

  IOSTATS_SET_THREAD_POOL_ID(fta.thread_pri_);
  TEST_SYNC_POINT("DBImpl::BGWorkFlush");
@@ -3055,8 +3059,8 @@ void DBImpl::BGWorkFlush(void* arg) {
 }

 void DBImpl::BGWorkCompaction(void* arg) {
-  CompactionArg ca = *(reinterpret_cast<CompactionArg*>(arg));
-  delete reinterpret_cast<CompactionArg*>(arg);
+  CompactionArg ca = *(static_cast<CompactionArg*>(arg));
+  delete static_cast<CompactionArg*>(arg);
  IOSTATS_SET_THREAD_POOL_ID(Env::Priority::LOW);
  TEST_SYNC_POINT("DBImpl::BGWorkCompaction");
  auto prepicked_compaction =
@@ -3080,12 +3084,12 @@ void DBImpl::BGWorkBottomCompaction(void* arg) {
 void DBImpl::BGWorkPurge(void* db) {
  IOSTATS_SET_THREAD_POOL_ID(Env::Priority::HIGH);
  TEST_SYNC_POINT("DBImpl::BGWorkPurge:start");
-  reinterpret_cast<DBImpl*>(db)->BackgroundCallPurge();
+  static_cast<DBImpl*>(db)->BackgroundCallPurge();
  TEST_SYNC_POINT("DBImpl::BGWorkPurge:end");
 }

 void DBImpl::UnscheduleCompactionCallback(void* arg) {
-  CompactionArg* ca_ptr = reinterpret_cast<CompactionArg*>(arg);
+  CompactionArg* ca_ptr = static_cast<CompactionArg*>(arg);
  Env::Priority compaction_pri = ca_ptr->compaction_pri_;
  if (Env::Priority::BOTTOM == compaction_pri) {
    // Decrement bg_bottom_compaction_scheduled_ if priority is BOTTOM
@@ -3095,7 +3099,7 @@ void DBImpl::UnscheduleCompactionCallback(void* arg) {
    ca_ptr->db->bg_compaction_scheduled_--;
  }
  CompactionArg ca = *(ca_ptr);
-  delete reinterpret_cast<CompactionArg*>(arg);
+  delete static_cast<CompactionArg*>(arg);
  if (ca.prepicked_compaction != nullptr) {
    // if it's a manual compaction, set status to ManualCompactionPaused
    if (ca.prepicked_compaction->manual_compaction_state) {
@@ -3115,14 +3119,14 @@ void DBImpl::UnscheduleCompactionCallback(void* arg) {

 void DBImpl::UnscheduleFlushCallback(void* arg) {
  // Decrement bg_flush_scheduled_ in flush callback
-  reinterpret_cast<FlushThreadArg*>(arg)->db_->bg_flush_scheduled_--;
-  Env::Priority flush_pri = reinterpret_cast<FlushThreadArg*>(arg)->thread_pri_;
+  static_cast<FlushThreadArg*>(arg)->db_->bg_flush_scheduled_--;
+  Env::Priority flush_pri = static_cast<FlushThreadArg*>(arg)->thread_pri_;
  if (Env::Priority::LOW == flush_pri) {
    TEST_SYNC_POINT("DBImpl::UnscheduleLowFlushCallback");
  } else if (Env::Priority::HIGH == flush_pri) {
    TEST_SYNC_POINT("DBImpl::UnscheduleHighFlushCallback");
  }
-  delete reinterpret_cast<FlushThreadArg*>(arg);
+  delete static_cast<FlushThreadArg*>(arg);
  TEST_SYNC_POINT("DBImpl::UnscheduleFlushCallback");
 }

@@ -3221,7 +3225,7 @@ Status DBImpl::BackgroundFlush(bool* made_progress, JobContext* job_context,
        column_families_not_to_flush.push_back(cfd);
        continue;
      }
-      superversion_contexts.emplace_back(SuperVersionContext(true));
+      superversion_contexts.emplace_back(true);
      bg_flush_args.emplace_back(cfd, max_memtable_id,
                                 &(superversion_contexts.back()), flush_reason);
    }
@@ -3726,9 +3730,9 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
          compaction_released = true;
        });
    io_s = versions_->io_status();
-    InstallSuperVersionAndScheduleWork(c->column_family_data(),
-                                       &job_context->superversion_contexts[0],
-                                       *c->mutable_cf_options());
+    InstallSuperVersionAndScheduleWork(
+        c->column_family_data(), job_context->superversion_contexts.data(),
+        *c->mutable_cf_options());
    ROCKS_LOG_BUFFER(log_buffer, "[%s] Deleted %d files\n",
                     c->column_family_data()->GetName().c_str(),
                     c->num_input_files(0));
@@ -3801,9 +3805,9 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
        });
    io_s = versions_->io_status();
    // Use latest MutableCFOptions
-    InstallSuperVersionAndScheduleWork(c->column_family_data(),
-                                       &job_context->superversion_contexts[0],
-                                       *c->mutable_cf_options());
+    InstallSuperVersionAndScheduleWork(
+        c->column_family_data(), job_context->superversion_contexts.data(),
+        *c->mutable_cf_options());

    VersionStorageInfo::LevelSummaryStorage tmp;
    c->column_family_data()->internal_stats()->IncBytesMoved(c->output_level(),
@@ -3896,9 +3900,9 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
        compaction_job.Install(*c->mutable_cf_options(), &compaction_released);
    io_s = compaction_job.io_status();
    if (status.ok()) {
-      InstallSuperVersionAndScheduleWork(c->column_family_data(),
-                                         &job_context->superversion_contexts[0],
-                                         *c->mutable_cf_options());
+      InstallSuperVersionAndScheduleWork(
+          c->column_family_data(), job_context->superversion_contexts.data(),
+          *c->mutable_cf_options());
    }
    *made_progress = true;
    TEST_SYNC_POINT_CALLBACK("DBImpl::BackgroundCompaction:AfterCompaction",
@@ -4045,7 +4049,6 @@ void DBImpl::RemoveManualCompaction(DBImpl::ManualCompactionState* m) {
    ++it;
  }
  assert(false);
-  return;
 }

 bool DBImpl::ShouldntRunManualCompaction(ManualCompactionState* m) {
@@ -208,11 +208,11 @@ void DBImpl::TEST_SignalAllBgCv() { bg_cv_.SignalAll(); }
 void* DBImpl::TEST_BeginWrite() {
  auto w = new WriteThread::Writer();
  write_thread_.EnterUnbatched(w, &mutex_);
-  return reinterpret_cast<void*>(w);
+  return static_cast<void*>(w);
 }

 void DBImpl::TEST_EndWrite(void* w) {
-  auto writer = reinterpret_cast<WriteThread::Writer*>(w);
+  auto writer = static_cast<WriteThread::Writer*>(w);
  write_thread_.ExitUnbatched(writer);
  delete writer;
 }
@@ -104,7 +104,9 @@ Status DBImpl::PromoteL0(ColumnFamilyHandle* column_family, int target_level) {
        return status;
      }

-      if (i == 0) continue;
+      if (i == 0) {
+        continue;
+      }
      auto prev_f = l0_files[i - 1];
      if (icmp->Compare(prev_f->largest, f->smallest) >= 0) {
        ROCKS_LOG_INFO(immutable_db_options_.info_log,
@@ -148,9 +150,9 @@ Status DBImpl::PromoteL0(ColumnFamilyHandle* column_family, int target_level) {
                                    read_options, write_options, &edit, &mutex_,
                                    directories_.GetDbDir());
    if (status.ok()) {
-      InstallSuperVersionAndScheduleWork(cfd,
-                                         &job_context.superversion_contexts[0],
-                                         *cfd->GetLatestMutableCFOptions());
+      InstallSuperVersionAndScheduleWork(
+          cfd, job_context.superversion_contexts.data(),
+          *cfd->GetLatestMutableCFOptions());
    }
  }  // lock released here
  LogFlush(immutable_db_options_.info_log);
@@ -52,7 +52,7 @@ Status DBImpl::DisableFileDeletions() {
  if (my_disable_delete_obsolete_files == 1) {
    ROCKS_LOG_INFO(immutable_db_options_.info_log, "File Deletions Disabled");
  } else {
-    ROCKS_LOG_WARN(immutable_db_options_.info_log,
+    ROCKS_LOG_INFO(immutable_db_options_.info_log,
                   "File Deletions Disabled, but already disabled. Counter: %d",
                   my_disable_delete_obsolete_files);
  }
@@ -67,17 +67,14 @@ Status DBImpl::DisableFileDeletionsWithLock() {
  return Status::OK();
 }

-Status DBImpl::EnableFileDeletions(bool force) {
+Status DBImpl::EnableFileDeletions() {
  // Job id == 0 means that this is not our background process, but rather
  // user thread
  JobContext job_context(0);
  int saved_counter;  // initialize on all paths
  {
    InstrumentedMutexLock l(&mutex_);
-    if (force) {
-      // if force, we need to enable file deletions right away
-      disable_delete_obsolete_files_ = 0;
-    } else if (disable_delete_obsolete_files_ > 0) {
+    if (disable_delete_obsolete_files_ > 0) {
      --disable_delete_obsolete_files_;
    }
    saved_counter = disable_delete_obsolete_files_;
@@ -92,7 +89,7 @@ Status DBImpl::EnableFileDeletions(bool force) {
      PurgeObsoleteFiles(job_context);
    }
  } else {
-    ROCKS_LOG_WARN(immutable_db_options_.info_log,
+    ROCKS_LOG_INFO(immutable_db_options_.info_log,
                   "File Deletions Enable, but not really enabled. Counter: %d",
                   saved_counter);
  }
@@ -104,7 +104,6 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src,
  if (result.recycle_log_file_num &&
      (result.wal_recovery_mode ==
           WALRecoveryMode::kTolerateCorruptedTailRecords ||
-       result.wal_recovery_mode == WALRecoveryMode::kPointInTimeRecovery ||
       result.wal_recovery_mode == WALRecoveryMode::kAbsoluteConsistency)) {
    // - kTolerateCorruptedTailRecords is inconsistent with recycle log file
    //   feature. WAL recycling expects recovery success upon encountering a
@@ -281,12 +280,6 @@ Status DBImpl::ValidateOptions(const DBOptions& db_options) {
        "atomic_flush is incompatible with enable_pipelined_write");
  }

-  // TODO remove this restriction
-  if (db_options.atomic_flush && db_options.best_efforts_recovery) {
-    return Status::InvalidArgument(
-        "atomic_flush is currently incompatible with best-efforts recovery");
-  }
-
  if (db_options.use_direct_io_for_flush_and_compaction &&
      0 == db_options.writable_file_max_buffer_size) {
    return Status::InvalidArgument(
@@ -1092,6 +1085,7 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& wal_numbers,
    Logger* info_log;
    const char* fname;
    Status* status;  // nullptr if immutable_db_options_.paranoid_checks==false
+    bool* old_log_record;
    void Corruption(size_t bytes, const Status& s) override {
      ROCKS_LOG_WARN(info_log, "%s%s: dropping %d bytes; %s",
                     (status == nullptr ? "(ignoring error) " : ""), fname,
@@ -1100,10 +1094,19 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& wal_numbers,
        *status = s;
      }
    }
+
+    void OldLogRecord(size_t bytes) override {
+      if (old_log_record != nullptr) {
+        *old_log_record = true;
+      }
+      ROCKS_LOG_WARN(info_log, "%s: dropping %d bytes; possibly recycled",
+                     fname, static_cast<int>(bytes));
+    }
  };

  mutex_.AssertHeld();
  Status status;
+  bool old_log_record = false;
  std::unordered_map<int, VersionEdit> version_edits;
  // no need to refcount because iteration is under mutex
  for (auto cfd : *versions_->GetColumnFamilySet()) {
@@ -1194,6 +1197,7 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& wal_numbers,
    reporter.env = env_;
    reporter.info_log = immutable_db_options_.info_log.get();
    reporter.fname = fname.c_str();
+    reporter.old_log_record = &old_log_record;
    if (!immutable_db_options_.paranoid_checks ||
        immutable_db_options_.wal_recovery_mode ==
            WALRecoveryMode::kSkipAnyCorruptedRecords) {
@@ -1341,7 +1345,7 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& wal_numbers,
      }
    }

-    if (!status.ok()) {
+    if (!status.ok() || old_log_record) {
      if (status.IsNotSupported()) {
        // We should not treat NotSupported as corruption. It is rather a clear
        // sign that we are processing a WAL that is produced by an incompatible
@@ -1366,6 +1370,7 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& wal_numbers,
        }
        // We should ignore the error but not continue replaying
        status = Status::OK();
+        old_log_record = false;
        stop_replay_for_corruption = true;
        corrupted_wal_number = wal_number;
        if (corrupted_wal_found != nullptr) {
@@ -1636,7 +1641,8 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
  Status s;
  TableProperties table_properties;
  {
-    ScopedArenaIterator iter(mem->NewIterator(ro, &arena));
+    ScopedArenaPtr<InternalIterator> iter(
+        mem->NewIterator(ro, /*seqno_to_time_mapping=*/nullptr, &arena));
    ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
                    "[%s] [WriteLevel0TableForRecovery]"
                    " Level-0 table #%" PRIu64 ": started",
@@ -1682,14 +1688,13 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
      const WriteOptions write_option(Env::IO_HIGH, Env::IOActivity::kDBOpen);
      TableBuilderOptions tboptions(
          *cfd->ioptions(), mutable_cf_options, read_option, write_option,
-          cfd->internal_comparator(), cfd->int_tbl_prop_collector_factories(),
+          cfd->internal_comparator(), cfd->internal_tbl_prop_coll_factories(),
          GetCompressionFlush(*cfd->ioptions(), mutable_cf_options),
          mutable_cf_options.compression_opts, cfd->GetID(), cfd->GetName(),
          0 /* level */, false /* is_bottommost */,
          TableFileCreationReason::kRecovery, 0 /* oldest_key_time */,
          0 /* file_creation_time */, db_id_, db_session_id_,
          0 /* target_file_size */, meta.fd.GetNumber());
-      SeqnoToTimeMapping empty_seqno_to_time_mapping;
      Version* version = cfd->current();
      version->Ref();
      uint64_t num_input_entries = 0;
@@ -1700,7 +1705,7 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
          snapshot_seqs, earliest_write_conflict_snapshot, kMaxSequenceNumber,
          snapshot_checker, paranoid_file_checks, cfd->internal_stats(), &io_s,
          io_tracer_, BlobFileCreationReason::kRecovery,
-          empty_seqno_to_time_mapping, &event_logger_, job_id,
+          nullptr /* seqno_to_time_mapping */, &event_logger_, job_id,
          nullptr /* table_properties */, write_hint,
          nullptr /*full_history_ts_low*/, &blob_callback_, version,
          &num_input_entries);
@@ -1799,11 +1804,9 @@ Status DB::Open(const Options& options, const std::string& dbname, DB** dbptr) {
  DBOptions db_options(options);
  ColumnFamilyOptions cf_options(options);
  std::vector<ColumnFamilyDescriptor> column_families;
-  column_families.push_back(
-      ColumnFamilyDescriptor(kDefaultColumnFamilyName, cf_options));
+  column_families.emplace_back(kDefaultColumnFamilyName, cf_options);
  if (db_options.persist_stats_to_disk) {
-    column_families.push_back(
-        ColumnFamilyDescriptor(kPersistentStatsColumnFamilyName, cf_options));
+    column_families.emplace_back(kPersistentStatsColumnFamilyName, cf_options);
  }
  std::vector<ColumnFamilyHandle*> handles;
  Status s = DB::Open(db_options, dbname, column_families, &handles, dbptr);
@@ -1972,7 +1975,7 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
  handles->clear();

  size_t max_write_buffer_size = 0;
-  for (auto cf : column_families) {
+  for (const auto& cf : column_families) {
    max_write_buffer_size =
        std::max(max_write_buffer_size, cf.options.write_buffer_size);
  }
@@ -2044,8 +2047,7 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
    }

    if (s.ok()) {
-      impl->alive_log_files_.push_back(
-          DBImpl::LogFileNumberSize(impl->logfile_number_));
+      impl->alive_log_files_.emplace_back(impl->logfile_number_);
      // In WritePrepared there could be gap in sequence numbers. This breaks
      // the trick we use in kPointInTimeRecovery which assumes the first seq in
      // the log right after the corrupted log is one larger than the last seq
@@ -2093,7 +2095,7 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,

  if (s.ok()) {
    // set column family handles
-    for (auto cf : column_families) {
+    for (const auto& cf : column_families) {
      auto cfd =
          impl->versions_->GetColumnFamilySet()->GetColumnFamily(cf.name);
      if (cfd != nullptr) {
@@ -26,7 +26,7 @@ DBImplReadOnly::DBImplReadOnly(const DBOptions& db_options,
  LogFlush(immutable_db_options_.info_log);
 }

-DBImplReadOnly::~DBImplReadOnly() {}
+DBImplReadOnly::~DBImplReadOnly() = default;

 // Implementations of the DB interface
 Status DBImplReadOnly::GetImpl(const ReadOptions& read_options,
@@ -169,8 +169,7 @@ Iterator* DBImplReadOnly::NewIterator(const ReadOptions& _read_options,
  SequenceNumber latest_snapshot = versions_->LastSequence();
  SequenceNumber read_seq =
      read_options.snapshot != nullptr
-          ? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
-                ->number_
+          ? static_cast<const SnapshotImpl*>(read_options.snapshot)->number_
          : latest_snapshot;
  ReadCallback* read_callback = nullptr;  // No read callback provided.
  auto db_iter = NewArenaWrappedDbIterator(
@@ -216,8 +215,7 @@ Status DBImplReadOnly::NewIterators(
  SequenceNumber latest_snapshot = versions_->LastSequence();
  SequenceNumber read_seq =
      read_options.snapshot != nullptr
-          ? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
-                ->number_
+          ? static_cast<const SnapshotImpl*>(read_options.snapshot)->number_
          : latest_snapshot;

  autovector<std::tuple<ColumnFamilyData*, SuperVersion*>> cfd_to_sv;
@@ -295,8 +293,7 @@ Status DB::OpenForReadOnly(const Options& options, const std::string& dbname,
  DBOptions db_options(options);
  ColumnFamilyOptions cf_options(options);
  std::vector<ColumnFamilyDescriptor> column_families;
-  column_families.push_back(
-      ColumnFamilyDescriptor(kDefaultColumnFamilyName, cf_options));
+  column_families.emplace_back(kDefaultColumnFamilyName, cf_options);
  std::vector<ColumnFamilyHandle*> handles;

  s = DBImplReadOnly::OpenForReadOnlyWithoutCheck(
@@ -341,7 +338,7 @@ Status DBImplReadOnly::OpenForReadOnlyWithoutCheck(
                           error_if_wal_file_exists);
  if (s.ok()) {
    // set column family handles
-    for (auto cf : column_families) {
+    for (const auto& cf : column_families) {
      auto cfd =
          impl->versions_->GetColumnFamilySet()->GetColumnFamily(cf.name);
      if (cfd == nullptr) {
@@ -31,18 +31,17 @@ class DBImplReadOnly : public DBImpl {
  // TODO: Implement ReadOnly MultiGet?

  using DBImpl::NewIterator;
-  virtual Iterator* NewIterator(const ReadOptions& _read_options,
-                                ColumnFamilyHandle* column_family) override;
+  Iterator* NewIterator(const ReadOptions& _read_options,
+                        ColumnFamilyHandle* column_family) override;

-  virtual Status NewIterators(
-      const ReadOptions& options,
-      const std::vector<ColumnFamilyHandle*>& column_families,
-      std::vector<Iterator*>* iterators) override;
+  Status NewIterators(const ReadOptions& options,
+                      const std::vector<ColumnFamilyHandle*>& column_families,
+                      std::vector<Iterator*>* iterators) override;

  using DBImpl::Put;
-  virtual Status Put(const WriteOptions& /*options*/,
-                     ColumnFamilyHandle* /*column_family*/,
-                     const Slice& /*key*/, const Slice& /*value*/) override {
+  Status Put(const WriteOptions& /*options*/,
+             ColumnFamilyHandle* /*column_family*/, const Slice& /*key*/,
+             const Slice& /*value*/) override {
    return Status::NotSupported("Not supported operation in read only mode.");
  }

@@ -59,37 +58,36 @@ class DBImplReadOnly : public DBImpl {
  }

  using DBImpl::Merge;
-  virtual Status Merge(const WriteOptions& /*options*/,
-                       ColumnFamilyHandle* /*column_family*/,
-                       const Slice& /*key*/, const Slice& /*value*/) override {
+  Status Merge(const WriteOptions& /*options*/,
+               ColumnFamilyHandle* /*column_family*/, const Slice& /*key*/,
+               const Slice& /*value*/) override {
    return Status::NotSupported("Not supported operation in read only mode.");
  }
  using DBImpl::Delete;
-  virtual Status Delete(const WriteOptions& /*options*/,
-                        ColumnFamilyHandle* /*column_family*/,
-                        const Slice& /*key*/) override {
+  Status Delete(const WriteOptions& /*options*/,
+                ColumnFamilyHandle* /*column_family*/,
+                const Slice& /*key*/) override {
    return Status::NotSupported("Not supported operation in read only mode.");
  }
  using DBImpl::SingleDelete;
-  virtual Status SingleDelete(const WriteOptions& /*options*/,
-                              ColumnFamilyHandle* /*column_family*/,
-                              const Slice& /*key*/) override {
+  Status SingleDelete(const WriteOptions& /*options*/,
+                      ColumnFamilyHandle* /*column_family*/,
+                      const Slice& /*key*/) override {
    return Status::NotSupported("Not supported operation in read only mode.");
  }
-  virtual Status Write(const WriteOptions& /*options*/,
-                       WriteBatch* /*updates*/) override {
+  Status Write(const WriteOptions& /*options*/,
+               WriteBatch* /*updates*/) override {
    return Status::NotSupported("Not supported operation in read only mode.");
  }
  using DBImpl::CompactRange;
-  virtual Status CompactRange(const CompactRangeOptions& /*options*/,
-                              ColumnFamilyHandle* /*column_family*/,
-                              const Slice* /*begin*/,
-                              const Slice* /*end*/) override {
+  Status CompactRange(const CompactRangeOptions& /*options*/,
+                      ColumnFamilyHandle* /*column_family*/,
+                      const Slice* /*begin*/, const Slice* /*end*/) override {
    return Status::NotSupported("Not supported operation in read only mode.");
  }

  using DBImpl::CompactFiles;
-  virtual Status CompactFiles(
+  Status CompactFiles(
      const CompactionOptions& /*compact_options*/,
      ColumnFamilyHandle* /*column_family*/,
      const std::vector<std::string>& /*input_file_names*/,
@@ -99,33 +97,33 @@ class DBImplReadOnly : public DBImpl {
    return Status::NotSupported("Not supported operation in read only mode.");
  }

-  virtual Status DisableFileDeletions() override {
+  Status DisableFileDeletions() override {
    return Status::NotSupported("Not supported operation in read only mode.");
  }

-  virtual Status EnableFileDeletions(bool /*force*/) override {
+  Status EnableFileDeletions() override {
    return Status::NotSupported("Not supported operation in read only mode.");
  }
-  virtual Status GetLiveFiles(std::vector<std::string>& ret,
-                              uint64_t* manifest_file_size,
-                              bool /*flush_memtable*/) override {
+  Status GetLiveFiles(std::vector<std::string>& ret,
+                      uint64_t* manifest_file_size,
+                      bool /*flush_memtable*/) override {
    return DBImpl::GetLiveFiles(ret, manifest_file_size,
                                false /* flush_memtable */);
  }

  using DBImpl::Flush;
-  virtual Status Flush(const FlushOptions& /*options*/,
-                       ColumnFamilyHandle* /*column_family*/) override {
+  Status Flush(const FlushOptions& /*options*/,
+               ColumnFamilyHandle* /*column_family*/) override {
    return Status::NotSupported("Not supported operation in read only mode.");
  }

  using DBImpl::SyncWAL;
-  virtual Status SyncWAL() override {
+  Status SyncWAL() override {
    return Status::NotSupported("Not supported operation in read only mode.");
  }

  using DB::IngestExternalFile;
-  virtual Status IngestExternalFile(
+  Status IngestExternalFile(
      ColumnFamilyHandle* /*column_family*/,
      const std::vector<std::string>& /*external_files*/,
      const IngestExternalFileOptions& /*ingestion_options*/) override {
@@ -133,7 +131,7 @@ class DBImplReadOnly : public DBImpl {
  }

  using DB::CreateColumnFamilyWithImport;
-  virtual Status CreateColumnFamilyWithImport(
+  Status CreateColumnFamilyWithImport(
      const ColumnFamilyOptions& /*options*/,
      const std::string& /*column_family_name*/,
      const ImportColumnFamilyOptions& /*import_options*/,
@@ -142,7 +140,7 @@ class DBImplReadOnly : public DBImpl {
    return Status::NotSupported("Not supported operation in read only mode.");
  }

-  virtual Status CreateColumnFamilyWithImport(
+  Status CreateColumnFamilyWithImport(
      const ColumnFamilyOptions& /*options*/,
      const std::string& /*column_family_name*/,
      const ImportColumnFamilyOptions& /*import_options*/,
@@ -152,9 +150,9 @@ class DBImplReadOnly : public DBImpl {
  }

  using DB::ClipColumnFamily;
-  virtual Status ClipColumnFamily(ColumnFamilyHandle* /*column_family*/,
-                                  const Slice& /*begin*/,
-                                  const Slice& /*end*/) override {
+  Status ClipColumnFamily(ColumnFamilyHandle* /*column_family*/,
+                          const Slice& /*begin*/,
+                          const Slice& /*end*/) override {
    return Status::NotSupported("Not supported operation in read only mode.");
  }

@@ -28,7 +28,7 @@ DBImplSecondary::DBImplSecondary(const DBOptions& db_options,
  LogFlush(immutable_db_options_.info_log);
 }

-DBImplSecondary::~DBImplSecondary() {}
+DBImplSecondary::~DBImplSecondary() = default;

 Status DBImplSecondary::Recover(
    const std::vector<ColumnFamilyDescriptor>& column_families,
@@ -327,6 +327,7 @@ Status DBImplSecondary::RecoverLogFiles(
      status = *wal_read_status;
    }
    if (!status.ok()) {
+      wal_read_status->PermitUncheckedError();
      return status;
    }
  }
@@ -497,8 +498,8 @@ Iterator* DBImplSecondary::NewIterator(const ReadOptions& _read_options,

  Iterator* result = nullptr;
  auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(column_family);
+  assert(cfh != nullptr);
  auto cfd = cfh->cfd();
-  ReadCallback* read_callback = nullptr;  // No read callback provided.
  if (read_options.tailing) {
    return NewErrorIterator(Status::NotSupported(
        "tailing iterator not supported in secondary mode"));
@@ -517,27 +518,28 @@ Iterator* DBImplSecondary::NewIterator(const ReadOptions& _read_options,
        return NewErrorIterator(s);
      }
    }
-    result = NewIteratorImpl(read_options, cfd, sv, snapshot, read_callback);
+    result = NewIteratorImpl(read_options, cfh, sv, snapshot,
+                             nullptr /*read_callback*/);
  }
  return result;
 }

 ArenaWrappedDBIter* DBImplSecondary::NewIteratorImpl(
-    const ReadOptions& read_options, ColumnFamilyData* cfd,
+    const ReadOptions& read_options, ColumnFamilyHandleImpl* cfh,
    SuperVersion* super_version, SequenceNumber snapshot,
    ReadCallback* read_callback, bool expose_blob_index, bool allow_refresh) {
-  assert(nullptr != cfd);
+  assert(nullptr != cfh);
  assert(snapshot == kMaxSequenceNumber);
  snapshot = versions_->LastSequence();
  assert(snapshot != kMaxSequenceNumber);
  auto db_iter = NewArenaWrappedDbIterator(
-      env_, read_options, *cfd->ioptions(), super_version->mutable_cf_options,
-      super_version->current, snapshot,
+      env_, read_options, *cfh->cfd()->ioptions(),
+      super_version->mutable_cf_options, super_version->current, snapshot,
      super_version->mutable_cf_options.max_sequential_skip_in_iterations,
-      super_version->version_number, read_callback, this, cfd,
-      expose_blob_index, allow_refresh);
+      super_version->version_number, read_callback, cfh, expose_blob_index,
+      allow_refresh);
  auto internal_iter = NewInternalIterator(
-      db_iter->GetReadOptions(), cfd, super_version, db_iter->GetArena(),
+      db_iter->GetReadOptions(), cfh->cfd(), super_version, db_iter->GetArena(),
      snapshot, /* allow_unprepared_value */ true, db_iter);
  db_iter->SetIterUnderDBIter(internal_iter);
  return db_iter;
@@ -596,28 +598,29 @@ Status DBImplSecondary::NewIterators(
    return Status::NotSupported("snapshot not supported in secondary mode");
  } else {
    SequenceNumber read_seq(kMaxSequenceNumber);
-    autovector<std::tuple<ColumnFamilyData*, SuperVersion*>> cfd_to_sv;
+    autovector<std::tuple<ColumnFamilyHandleImpl*, SuperVersion*>> cfh_to_sv;
    const bool check_read_ts =
        read_options.timestamp && read_options.timestamp->size() > 0;
-    for (auto cfh : column_families) {
-      ColumnFamilyData* cfd = static_cast<ColumnFamilyHandleImpl*>(cfh)->cfd();
+    for (auto cf : column_families) {
+      auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(cf);
+      auto cfd = cfh->cfd();
      SuperVersion* sv = cfd->GetReferencedSuperVersion(this);
-      cfd_to_sv.emplace_back(cfd, sv);
+      cfh_to_sv.emplace_back(cfh, sv);
      if (check_read_ts) {
        const Status s =
            FailIfReadCollapsedHistory(cfd, sv, *(read_options.timestamp));
        if (!s.ok()) {
-          for (auto prev_entry : cfd_to_sv) {
+          for (auto prev_entry : cfh_to_sv) {
            CleanupSuperVersion(std::get<1>(prev_entry));
          }
          return s;
        }
      }
    }
-    assert(cfd_to_sv.size() == column_families.size());
-    for (auto [cfd, sv] : cfd_to_sv) {
+    assert(cfh_to_sv.size() == column_families.size());
+    for (auto [cfh, sv] : cfh_to_sv) {
      iterators->push_back(
-          NewIteratorImpl(read_options, cfd, sv, read_seq, read_callback));
+          NewIteratorImpl(read_options, cfh, sv, read_seq, read_callback));
    }
  }
  return Status::OK();
@@ -802,7 +805,7 @@ Status DB::OpenAsSecondary(
  impl->mutex_.Lock();
  s = impl->Recover(column_families, true, false, false);
  if (s.ok()) {
-    for (auto cf : column_families) {
+    for (const auto& cf : column_families) {
      auto cfd =
          impl->versions_->GetColumnFamilySet()->GetColumnFamily(cf.name);
      if (nullptr == cfd) {
@@ -112,8 +112,8 @@ class DBImplSecondary : public DBImpl {
                        ColumnFamilyHandle* column_family) override;

  ArenaWrappedDBIter* NewIteratorImpl(const ReadOptions& read_options,
-                                      ColumnFamilyData* cfd, SuperVersion* sv,
-                                      SequenceNumber snapshot,
+                                      ColumnFamilyHandleImpl* cfh,
+                                      SuperVersion* sv, SequenceNumber snapshot,
                                      ReadCallback* read_callback,
                                      bool expose_blob_index = false,
                                      bool allow_refresh = true);
@@ -189,7 +189,7 @@ class DBImplSecondary : public DBImpl {
    return Status::NotSupported("Not supported operation in secondary mode.");
  }

-  Status EnableFileDeletions(bool /*force*/) override {
+  Status EnableFileDeletions() override {
    return Status::NotSupported("Not supported operation in secondary mode.");
  }

@@ -220,6 +220,11 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
             write_options.protection_bytes_per_key != 8) {
    return Status::InvalidArgument(
        "`WriteOptions::protection_bytes_per_key` must be zero or eight");
+  } else if (write_options.disableWAL &&
+             immutable_db_options_.recycle_log_file_num > 0) {
+    return Status::InvalidArgument(
+        "WriteOptions::disableWAL option is not supported if "
+        "DBOptions::recycle_log_file_num > 0");
  }
  // TODO: this use of operator bool on `tracer_` can avoid unnecessary lock
  // grabs but does not seem thread-safe.
@@ -325,7 +330,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,

    if (w.ShouldWriteToMemtable()) {
      PERF_TIMER_STOP(write_pre_and_post_process_time);
-      PERF_TIMER_GUARD(write_memtable_time);
+      PERF_TIMER_FOR_WAIT_GUARD(write_memtable_time);

      ColumnFamilyMemTablesImpl column_family_memtables(
          versions_->GetColumnFamilySet());
@@ -557,7 +562,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
    }

    if (status.ok()) {
-      PERF_TIMER_GUARD(write_memtable_time);
+      PERF_TIMER_FOR_WAIT_GUARD(write_memtable_time);

      if (!parallel) {
        // w.sequence will be set inside InsertInto
@@ -797,7 +802,7 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
  WriteThread::WriteGroup memtable_write_group;

  if (w.state == WriteThread::STATE_MEMTABLE_WRITER_LEADER) {
-    PERF_TIMER_GUARD(write_memtable_time);
+    PERF_TIMER_FOR_WAIT_GUARD(write_memtable_time);
    assert(w.ShouldWriteToMemtable());
    write_thread_.EnterAsMemTableWriter(&w, &memtable_write_group);
    if (memtable_write_group.size > 1 &&
@@ -819,6 +824,9 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
  }

  if (w.state == WriteThread::STATE_PARALLEL_MEMTABLE_WRITER) {
+    PERF_TIMER_STOP(write_pre_and_post_process_time);
+    PERF_TIMER_FOR_WAIT_GUARD(write_memtable_time);
+
    assert(w.ShouldWriteToMemtable());
    ColumnFamilyMemTablesImpl column_family_memtables(
        versions_->GetColumnFamilySet());
@@ -828,6 +836,10 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
        0 /*log_number*/, this, true /*concurrent_memtable_writes*/,
        false /*seq_per_batch*/, 0 /*batch_cnt*/, true /*batch_per_txn*/,
        write_options.memtable_insert_hint_per_batch);
+
+    PERF_TIMER_STOP(write_memtable_time);
+    PERF_TIMER_START(write_pre_and_post_process_time);
+
    if (write_thread_.CompleteParallelMemTableWriter(&w)) {
      MemTableInsertStatusCheck(w.status);
      versions_->SetLastSequence(w.write_group->last_sequence);
@@ -860,6 +872,9 @@ Status DBImpl::UnorderedWriteMemtable(const WriteOptions& write_options,
    stats->AddDBStats(InternalStats::kIntStatsNumKeysWritten, total_count);
    RecordTick(stats_, NUMBER_KEYS_WRITTEN, total_count);

+    PERF_TIMER_STOP(write_pre_and_post_process_time);
+    PERF_TIMER_FOR_WAIT_GUARD(write_memtable_time);
+
    ColumnFamilyMemTablesImpl column_family_memtables(
        versions_->GetColumnFamilySet());
    w.status = WriteBatchInternal::InsertInto(
@@ -871,6 +886,8 @@ Status DBImpl::UnorderedWriteMemtable(const WriteOptions& write_options,
    if (write_options.disableWAL) {
      has_unpersisted_data_.store(true, std::memory_order_relaxed);
    }
+
+    PERF_TIMER_START(write_pre_and_post_process_time);
  }

  size_t pending_cnt = pending_memtable_writes_.fetch_sub(1) - 1;
@@ -941,9 +958,13 @@ Status DBImpl::WriteImplWALOnly(
      return status;
    }
  } else {
+    PERF_TIMER_STOP(write_pre_and_post_process_time);
+    PERF_TIMER_FOR_WAIT_GUARD(write_delay_time);
    InstrumentedMutexLock lock(&mutex_);
    Status status =
        DelayWrite(/*num_bytes=*/0ull, *write_thread, write_options);
+    PERF_TIMER_STOP(write_delay_time);
+    PERF_TIMER_START(write_pre_and_post_process_time);
    if (!status.ok()) {
      WriteThread::WriteGroup write_group;
      write_thread->EnterAsBatchGroupLeader(&w, &write_group);
@@ -1212,7 +1233,7 @@ Status DBImpl::PreprocessWrite(const WriteOptions& write_options,
  if (UNLIKELY(status.ok() && (write_controller_.IsStopped() ||
                               write_controller_.NeedsDelay()))) {
    PERF_TIMER_STOP(write_pre_and_post_process_time);
-    PERF_TIMER_GUARD(write_delay_time);
+    PERF_TIMER_FOR_WAIT_GUARD(write_delay_time);
    // We don't know size of curent batch so that we always use the size
    // for previous one. It might create a fairness issue that expiration
    // might happen for smaller writes but larger writes can go through.
@@ -1984,7 +2005,7 @@ Status DBImpl::ThrottleLowPriWritesIfNeeded(const WriteOptions& write_options,
      // is that in case the write is heavy, low pri writes may never have
      // a chance to run. Now we guarantee we are still slowly making
      // progress.
-      PERF_TIMER_GUARD(write_delay_time);
+      PERF_TIMER_FOR_WAIT_GUARD(write_delay_time);
      auto data_size = my_batch->GetDataSize();
      while (data_size > 0) {
        size_t allowed = write_controller_.low_pri_rate_limiter()->RequestToken(
@@ -2119,7 +2140,7 @@ void DBImpl::NotifyOnMemTableSealed(ColumnFamilyData* /*cfd*/,
  }

  mutex_.Unlock();
-  for (auto listener : immutable_db_options_.listeners) {
+  for (const auto& listener : immutable_db_options_.listeners) {
    listener->OnMemTableSealed(mem_table_info);
  }
  mutex_.Lock();
@@ -2157,8 +2178,10 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
    log_write_mutex_.Unlock();
  }
  uint64_t recycle_log_number = 0;
+  // If file deletion is disabled, don't recycle logs since it'll result in
+  // the file getting renamed
  if (creating_new_log && immutable_db_options_.recycle_log_file_num &&
-      !log_recycle_files_.empty()) {
+      !log_recycle_files_.empty() && IsFileDeletionsEnabled()) {
    recycle_log_number = log_recycle_files_.front();
  }
  uint64_t new_log_number =
@@ -2236,7 +2259,7 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
      log_empty_ = true;
      log_dir_synced_ = false;
      logs_.emplace_back(logfile_number_, new_log);
-      alive_log_files_.push_back(LogFileNumberSize(logfile_number_));
+      alive_log_files_.emplace_back(logfile_number_);
    }
  }

@@ -98,7 +98,12 @@ void DumpDBFileSummary(const ImmutableDBOptions& options,
  for (auto& db_path : options.db_paths) {
    if (dbname.compare(db_path.path) != 0) {
      s = env->GetChildren(db_path.path, &files);
-      if (!s.ok()) {
+      if (s.IsNotFound() || s.IsPathNotFound()) {
+        Header(options.info_log,
+               "Directory from db_paths/cf_paths does not yet exist: %s\n",
+               db_path.path.c_str());
+        continue;
+      } else if (!s.ok()) {
        Error(options.info_log, "Error when reading %s dir %s\n",
              db_path.path.c_str(), s.ToString().c_str());
        continue;
@@ -121,12 +126,18 @@ void DumpDBFileSummary(const ImmutableDBOptions& options,

  // Get wal file in wal_dir
  const auto& wal_dir = options.GetWalDir(dbname);
+  bool log_wal_info = true;
  if (!options.IsWalDirSameAsDBPath(dbname)) {
    s = env->GetChildren(wal_dir, &files);
-    if (!s.ok()) {
-      Error(options.info_log, "Error when reading %s dir %s\n", wal_dir.c_str(),
-            s.ToString().c_str());
-      return;
+    if (s.IsNotFound() || s.IsPathNotFound()) {
+      Header(options.info_log,
+             "Write Ahead Log directory does not yet exist: %s\n",
+             wal_dir.c_str());
+      log_wal_info = false;
+    } else if (!s.ok()) {
+      Error(options.info_log, "Error when reading wal dir %s: %s\n",
+            wal_dir.c_str(), s.ToString().c_str());
+      log_wal_info = false;
    }
    wal_info.clear();
    for (const std::string& file : files) {
@@ -146,7 +157,9 @@ void DumpDBFileSummary(const ImmutableDBOptions& options,
      }
    }
  }
-  Header(options.info_log, "Write Ahead Log file in %s: %s\n", wal_dir.c_str(),
-         wal_info.c_str());
+  if (log_wal_info) {
+    Header(options.info_log, "Write Ahead Log file in %s: %s\n",
+           wal_dir.c_str(), wal_info.c_str());
+  }
 }
 }  // namespace ROCKSDB_NAMESPACE
@@ -13,6 +13,120 @@
 #include "util/random.h"

 namespace ROCKSDB_NAMESPACE {
+namespace {
+// A wrapper that allows injection of errors.
+class CorruptionFS : public FileSystemWrapper {
+ public:
+  bool writable_file_error_;
+  int num_writable_file_errors_;
+
+  explicit CorruptionFS(const std::shared_ptr<FileSystem>& _target,
+                        bool fs_buffer)
+      : FileSystemWrapper(_target),
+        writable_file_error_(false),
+        num_writable_file_errors_(0),
+        corruption_trigger_(INT_MAX),
+        read_count_(0),
+        rnd_(300),
+        fs_buffer_(fs_buffer) {}
+  ~CorruptionFS() override {
+    // Assert that the corruption was reset, which means it got triggered
+    assert(corruption_trigger_ == INT_MAX);
+  }
+  const char* Name() const override { return "ErrorEnv"; }
+
+  IOStatus NewWritableFile(const std::string& fname, const FileOptions& opts,
+                           std::unique_ptr<FSWritableFile>* result,
+                           IODebugContext* dbg) override {
+    result->reset();
+    if (writable_file_error_) {
+      ++num_writable_file_errors_;
+      return IOStatus::IOError(fname, "fake error");
+    }
+    return target()->NewWritableFile(fname, opts, result, dbg);
+  }
+
+  void SetCorruptionTrigger(const int trigger) {
+    corruption_trigger_ = trigger;
+    read_count_ = 0;
+  }
+
+  IOStatus NewRandomAccessFile(const std::string& fname,
+                               const FileOptions& opts,
+                               std::unique_ptr<FSRandomAccessFile>* result,
+                               IODebugContext* dbg) override {
+    class CorruptionRandomAccessFile : public FSRandomAccessFileOwnerWrapper {
+     public:
+      CorruptionRandomAccessFile(CorruptionFS& fs,
+                                 std::unique_ptr<FSRandomAccessFile>& file)
+          : FSRandomAccessFileOwnerWrapper(std::move(file)), fs_(fs) {}
+
+      IOStatus Read(uint64_t offset, size_t len, const IOOptions& opts,
+                    Slice* result, char* scratch,
+                    IODebugContext* dbg) const override {
+        IOStatus s = target()->Read(offset, len, opts, result, scratch, dbg);
+        if (opts.verify_and_reconstruct_read) {
+          return s;
+        }
+        if (s.ok() && ++fs_.read_count_ >= fs_.corruption_trigger_) {
+          fs_.read_count_ = 0;
+          fs_.corruption_trigger_ = INT_MAX;
+          char* data = const_cast<char*>(result->data());
+          std::memcpy(
+              data,
+              fs_.rnd_.RandomString(static_cast<int>(result->size())).c_str(),
+              result->size());
+        }
+        return s;
+      }
+
+      IOStatus MultiRead(FSReadRequest* reqs, size_t num_reqs,
+                         const IOOptions& options,
+                         IODebugContext* dbg) override {
+        for (size_t i = 0; i < num_reqs; ++i) {
+          FSReadRequest& req = reqs[i];
+          if (fs_.fs_buffer_) {
+            FSAllocationPtr buffer(new char[req.len], [](void* ptr) {
+              delete[] static_cast<char*>(ptr);
+            });
+            req.fs_scratch = std::move(buffer);
+            req.status = Read(req.offset, req.len, options, &req.result,
+                              static_cast<char*>(req.fs_scratch.get()), dbg);
+          } else {
+            req.status = Read(req.offset, req.len, options, &req.result,
+                              req.scratch, dbg);
+          }
+        }
+        return IOStatus::OK();
+      }
+
+     private:
+      CorruptionFS& fs_;
+    };
+
+    std::unique_ptr<FSRandomAccessFile> file;
+    IOStatus s = target()->NewRandomAccessFile(fname, opts, &file, dbg);
+    EXPECT_OK(s);
+    result->reset(new CorruptionRandomAccessFile(*this, file));
+
+    return s;
+  }
+
+  void SupportedOps(int64_t& supported_ops) override {
+    supported_ops = 1 << FSSupportedOps::kVerifyAndReconstructRead |
+                    1 << FSSupportedOps::kAsyncIO;
+    if (fs_buffer_) {
+      supported_ops |= 1 << FSSupportedOps::kFSBuffer;
+    }
+  }
+
+ private:
+  int corruption_trigger_;
+  int read_count_;
+  Random rnd_;
+  bool fs_buffer_;
+};
+}  // anonymous namespace

 class DBIOFailureTest : public DBTestBase {
 public:
@@ -579,6 +693,132 @@ TEST_F(DBIOFailureTest, CompactionSstSyncError) {
  ASSERT_EQ("bar3", Get(1, "foo"));
 }
 #endif  // !(defined NDEBUG) || !defined(OS_WIN)
+
+class DBIOCorruptionTest
+    : public DBIOFailureTest,
+      public testing::WithParamInterface<std::tuple<bool, bool>> {
+ public:
+  DBIOCorruptionTest() : DBIOFailureTest() {
+    BlockBasedTableOptions bbto;
+    Options options = CurrentOptions();
+
+    base_env_ = env_;
+    EXPECT_NE(base_env_, nullptr);
+    fs_.reset(
+        new CorruptionFS(base_env_->GetFileSystem(), std::get<0>(GetParam())));
+    env_guard_ = NewCompositeEnv(fs_);
+    options.env = env_guard_.get();
+    bbto.num_file_reads_for_auto_readahead = 0;
+    options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+    options.disable_auto_compactions = true;
+
+    Reopen(options);
+  }
+
+  ~DBIOCorruptionTest() {
+    Close();
+    db_ = nullptr;
+  }
+
+ protected:
+  std::unique_ptr<Env> env_guard_;
+  std::shared_ptr<CorruptionFS> fs_;
+  Env* base_env_;
+};
+
+TEST_P(DBIOCorruptionTest, GetReadCorruptionRetry) {
+  CorruptionFS* fs =
+      static_cast<CorruptionFS*>(env_guard_->GetFileSystem().get());
+
+  ASSERT_OK(Put("key1", "val1"));
+  ASSERT_OK(Flush());
+  fs->SetCorruptionTrigger(1);
+
+  std::string val;
+  ReadOptions ro;
+  ro.async_io = std::get<1>(GetParam());
+  ASSERT_OK(dbfull()->Get(ReadOptions(), "key1", &val));
+  ASSERT_EQ(val, "val1");
+}
+
+TEST_P(DBIOCorruptionTest, IterReadCorruptionRetry) {
+  CorruptionFS* fs =
+      static_cast<CorruptionFS*>(env_guard_->GetFileSystem().get());
+
+  ASSERT_OK(Put("key1", "val1"));
+  ASSERT_OK(Flush());
+  fs->SetCorruptionTrigger(1);
+
+  ReadOptions ro;
+  ro.readahead_size = 65536;
+  ro.async_io = std::get<1>(GetParam());
+
+  Iterator* iter = dbfull()->NewIterator(ro);
+  iter->SeekToFirst();
+  while (iter->status().ok() && iter->Valid()) {
+    iter->Next();
+  }
+  ASSERT_OK(iter->status());
+  delete iter;
+}
+
+TEST_P(DBIOCorruptionTest, MultiGetReadCorruptionRetry) {
+  CorruptionFS* fs =
+      static_cast<CorruptionFS*>(env_guard_->GetFileSystem().get());
+
+  ASSERT_OK(Put("key1", "val1"));
+  ASSERT_OK(Put("key2", "val2"));
+  ASSERT_OK(Flush());
+  fs->SetCorruptionTrigger(1);
+
+  std::vector<std::string> keystr{"key1", "key2"};
+  std::vector<Slice> keys{Slice(keystr[0]), Slice(keystr[1])};
+  std::vector<PinnableSlice> values(keys.size());
+  std::vector<Status> statuses(keys.size());
+  ReadOptions ro;
+  ro.async_io = std::get<1>(GetParam());
+  dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(),
+                     keys.data(), values.data(), statuses.data());
+  ASSERT_EQ(values[0].ToString(), "val1");
+  ASSERT_EQ(values[1].ToString(), "val2");
+}
+
+TEST_P(DBIOCorruptionTest, CompactionReadCorruptionRetry) {
+  CorruptionFS* fs =
+      static_cast<CorruptionFS*>(env_guard_->GetFileSystem().get());
+
+  ASSERT_OK(Put("key1", "val1"));
+  ASSERT_OK(Put("key3", "val3"));
+  ASSERT_OK(Flush());
+  ASSERT_OK(Put("key2", "val2"));
+  ASSERT_OK(Flush());
+  fs->SetCorruptionTrigger(1);
+  ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+
+  std::string val;
+  ReadOptions ro;
+  ro.async_io = std::get<1>(GetParam());
+  ASSERT_OK(dbfull()->Get(ro, "key1", &val));
+  ASSERT_EQ(val, "val1");
+}
+
+TEST_P(DBIOCorruptionTest, FlushReadCorruptionRetry) {
+  CorruptionFS* fs =
+      static_cast<CorruptionFS*>(env_guard_->GetFileSystem().get());
+
+  ASSERT_OK(Put("key1", "val1"));
+  fs->SetCorruptionTrigger(1);
+  ASSERT_OK(Flush());
+
+  std::string val;
+  ReadOptions ro;
+  ro.async_io = std::get<1>(GetParam());
+  ASSERT_OK(dbfull()->Get(ro, "key1", &val));
+  ASSERT_EQ(val, "val1");
+}
+
+INSTANTIATE_TEST_CASE_P(DBIOCorruptionTest, DBIOCorruptionTest,
+                        testing::Combine(testing::Bool(), testing::Bool()));
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -43,8 +43,8 @@ DBIter::DBIter(Env* _env, const ReadOptions& read_options,
               const Comparator* cmp, InternalIterator* iter,
               const Version* version, SequenceNumber s, bool arena_mode,
               uint64_t max_sequential_skip_in_iterations,
-               ReadCallback* read_callback, DBImpl* db_impl,
-               ColumnFamilyData* cfd, bool expose_blob_index)
+               ReadCallback* read_callback, ColumnFamilyHandleImpl* cfh,
+               bool expose_blob_index)
    : prefix_extractor_(mutable_cf_options.prefix_extractor.get()),
      env_(_env),
      clock_(ioptions.clock),
@@ -79,8 +79,7 @@ DBIter::DBIter(Env* _env, const ReadOptions& read_options,
      is_blob_(false),
      arena_mode_(arena_mode),
      io_activity_(read_options.io_activity),
-      db_impl_(db_impl),
-      cfd_(cfd),
+      cfh_(cfh),
      timestamp_ub_(read_options.timestamp),
      timestamp_lb_(read_options.iter_start_ts),
      timestamp_size_(timestamp_ub_ ? timestamp_ub_->size() : 0) {
@@ -114,8 +113,8 @@ Status DBIter::GetProperty(std::string prop_name, std::string* prop) {
    *prop = saved_key_.GetUserKey().ToString();
    return Status::OK();
  } else if (prop_name == "rocksdb.iterator.write-time") {
-    // TODO(yuzhangyu): implement return the actual write time.
-    return Status::NotSupported("write time property is under construction");
+    PutFixed64(prop, saved_write_unix_time_);
+    return Status::OK();
  }
  return Status::InvalidArgument("Unidentified property.");
 }
@@ -393,6 +392,7 @@ bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key,
            }
            break;
          case kTypeValue:
+          case kTypeValuePreferredSeqno:
          case kTypeBlobIndex:
          case kTypeWideColumnEntity:
            if (!PrepareValue()) {
@@ -418,8 +418,14 @@ bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key,
                return false;
              }
            } else {
-              assert(ikey_.type == kTypeValue);
-              SetValueAndColumnsFromPlain(iter_.value());
+              assert(ikey_.type == kTypeValue ||
+                     ikey_.type == kTypeValuePreferredSeqno);
+              Slice value = iter_.value();
+              saved_write_unix_time_ = iter_.write_unix_time();
+              if (ikey_.type == kTypeValuePreferredSeqno) {
+                value = ParsePackedValueForValue(value);
+              }
+              SetValueAndColumnsFromPlain(value);
            }

            valid_ = true;
@@ -575,10 +581,15 @@ bool DBIter::MergeValuesNewToOld() {
      return false;
    }

-    if (kTypeValue == ikey.type) {
-      // hit a put, merge the put value with operands and store the
-      // final result in saved_value_. We are done!
-      if (!MergeWithPlainBaseValue(iter_.value(), ikey.user_key)) {
+    if (kTypeValue == ikey.type || kTypeValuePreferredSeqno == ikey.type) {
+      Slice value = iter_.value();
+      saved_write_unix_time_ = iter_.write_unix_time();
+      if (kTypeValuePreferredSeqno == ikey.type) {
+        value = ParsePackedValueForValue(value);
+      }
+      // hit a put or put equivalent, merge the put value with operands and
+      // store the final result in saved_value_. We are done!
+      if (!MergeWithPlainBaseValue(value, ikey.user_key)) {
        return false;
      }
      // iter_ is positioned after put
@@ -701,16 +712,21 @@ bool DBIter::ReverseToForward() {
  // not exist or may have different prefix than the current key().
  // If that's the case, seek iter_ to current key.
  if (!expect_total_order_inner_iter() || !iter_.Valid()) {
-    IterKey last_key;
-    ParsedInternalKey pikey(saved_key_.GetUserKey(), kMaxSequenceNumber,
-                            kValueTypeForSeek);
-    if (timestamp_size_ > 0) {
+    std::string last_key;
+    if (timestamp_size_ == 0) {
+      AppendInternalKey(
+          &last_key, ParsedInternalKey(saved_key_.GetUserKey(),
+                                       kMaxSequenceNumber, kValueTypeForSeek));
+    } else {
      // TODO: pre-create kTsMax.
      const std::string kTsMax(timestamp_size_, '\xff');
-      pikey.SetTimestamp(kTsMax);
+      AppendInternalKeyWithDifferentTimestamp(
+          &last_key,
+          ParsedInternalKey(saved_key_.GetUserKey(), kMaxSequenceNumber,
+                            kValueTypeForSeek),
+          kTsMax);
    }
-    last_key.SetInternalKey(pikey);
-    iter_.Seek(last_key.GetInternalKey());
+    iter_.Seek(last_key);
    RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION);
  }

@@ -840,8 +856,8 @@ bool DBIter::FindValueForCurrentKey() {
  merge_context_.Clear();
  current_entry_is_merged_ = false;
  // last entry before merge (could be kTypeDeletion,
-  // kTypeDeletionWithTimestamp, kTypeSingleDeletion, kTypeValue,
-  // kTypeBlobIndex, or kTypeWideColumnEntity)
+  // kTypeDeletionWithTimestamp, kTypeSingleDeletion, kTypeValue
+  // kTypeBlobIndex, kTypeWideColumnEntity or kTypeValuePreferredSeqno)
  ValueType last_not_merge_type = kTypeDeletion;
  ValueType last_key_entry_type = kTypeDeletion;

@@ -918,10 +934,16 @@ bool DBIter::FindValueForCurrentKey() {
    last_key_entry_type = ikey.type;
    switch (last_key_entry_type) {
      case kTypeValue:
+      case kTypeValuePreferredSeqno:
      case kTypeBlobIndex:
      case kTypeWideColumnEntity:
        if (iter_.iter()->IsValuePinned()) {
-          pinned_value_ = iter_.value();
+          saved_write_unix_time_ = iter_.write_unix_time();
+          if (last_key_entry_type == kTypeValuePreferredSeqno) {
+            pinned_value_ = ParsePackedValueForValue(iter_.value());
+          } else {
+            pinned_value_ = iter_.value();
+          }
        } else {
          valid_ = false;
          status_ = Status::NotSupported(
@@ -1031,7 +1053,8 @@ bool DBIter::FindValueForCurrentKey() {

        return true;
      } else {
-        assert(last_not_merge_type == kTypeValue);
+        assert(last_not_merge_type == kTypeValue ||
+               last_not_merge_type == kTypeValuePreferredSeqno);
        if (!MergeWithPlainBaseValue(pinned_value_, saved_key_.GetUserKey())) {
          return false;
        }
@@ -1039,6 +1062,7 @@ bool DBIter::FindValueForCurrentKey() {
      }
      break;
    case kTypeValue:
+    case kTypeValuePreferredSeqno:
      SetValueAndColumnsFromPlain(pinned_value_);

      break;
@@ -1143,10 +1167,15 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
    Slice ts = ExtractTimestampFromUserKey(ikey.user_key, timestamp_size_);
    saved_timestamp_.assign(ts.data(), ts.size());
  }
-  if (ikey.type == kTypeValue || ikey.type == kTypeBlobIndex ||
-      ikey.type == kTypeWideColumnEntity) {
+  if (ikey.type == kTypeValue || ikey.type == kTypeValuePreferredSeqno ||
+      ikey.type == kTypeBlobIndex || ikey.type == kTypeWideColumnEntity) {
    assert(iter_.iter()->IsValuePinned());
-    pinned_value_ = iter_.value();
+    saved_write_unix_time_ = iter_.write_unix_time();
+    if (ikey.type == kTypeValuePreferredSeqno) {
+      pinned_value_ = ParsePackedValueForValue(iter_.value());
+    } else {
+      pinned_value_ = iter_.value();
+    }
    if (ikey.type == kTypeBlobIndex) {
      if (!SetBlobValueIfNeeded(ikey.user_key, pinned_value_)) {
        return false;
@@ -1159,7 +1188,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
        return false;
      }
    } else {
-      assert(ikey.type == kTypeValue);
+      assert(ikey.type == kTypeValue || ikey.type == kTypeValuePreferredSeqno);
      SetValueAndColumnsFromPlain(pinned_value_);
    }

@@ -1205,8 +1234,12 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
      return false;
    }

-    if (ikey.type == kTypeValue) {
-      if (!MergeWithPlainBaseValue(iter_.value(), saved_key_.GetUserKey())) {
+    if (ikey.type == kTypeValue || ikey.type == kTypeValuePreferredSeqno) {
+      Slice value = iter_.value();
+      if (ikey.type == kTypeValuePreferredSeqno) {
+        value = ParsePackedValueForValue(value);
+      }
+      if (!MergeWithPlainBaseValue(value, saved_key_.GetUserKey())) {
        return false;
      }
      return true;
@@ -1343,18 +1376,23 @@ bool DBIter::FindUserKeyBeforeSavedKey() {

    if (num_skipped >= max_skip_) {
      num_skipped = 0;
-      IterKey last_key;
-      ParsedInternalKey pikey(saved_key_.GetUserKey(), kMaxSequenceNumber,
-                              kValueTypeForSeek);
-      if (timestamp_size_ > 0) {
+      std::string last_key;
+      if (timestamp_size_ == 0) {
+        AppendInternalKey(&last_key, ParsedInternalKey(saved_key_.GetUserKey(),
+                                                       kMaxSequenceNumber,
+                                                       kValueTypeForSeek));
+      } else {
        // TODO: pre-create kTsMax.
        const std::string kTsMax(timestamp_size_, '\xff');
-        pikey.SetTimestamp(kTsMax);
+        AppendInternalKeyWithDifferentTimestamp(
+            &last_key,
+            ParsedInternalKey(saved_key_.GetUserKey(), kMaxSequenceNumber,
+                              kValueTypeForSeek),
+            kTsMax);
      }
-      last_key.SetInternalKey(pikey);
      // It would be more efficient to use SeekForPrev() here, but some
      // iterators may not support it.
-      iter_.Seek(last_key.GetInternalKey());
+      iter_.Seek(last_key);
      RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION);
      if (!iter_.Valid()) {
        break;
@@ -1459,7 +1497,7 @@ void DBIter::Seek(const Slice& target) {
  PERF_CPU_TIMER_GUARD(iter_seek_cpu_nanos, clock_);
  StopWatch sw(clock_, statistics_, DB_SEEK);

-  if (db_impl_ != nullptr && cfd_ != nullptr) {
+  if (cfh_ != nullptr) {
    // TODO: What do we do if this returns an error?
    Slice lower_bound, upper_bound;
    if (iterate_lower_bound_ != nullptr) {
@@ -1472,7 +1510,9 @@ void DBIter::Seek(const Slice& target) {
    } else {
      upper_bound = Slice("");
    }
-    db_impl_->TraceIteratorSeek(cfd_->GetID(), target, lower_bound, upper_bound)
+    cfh_->db()
+        ->TraceIteratorSeek(cfh_->cfd()->GetID(), target, lower_bound,
+                            upper_bound)
        .PermitUncheckedError();
  }

@@ -1533,7 +1573,7 @@ void DBIter::SeekForPrev(const Slice& target) {
  PERF_CPU_TIMER_GUARD(iter_seek_cpu_nanos, clock_);
  StopWatch sw(clock_, statistics_, DB_SEEK);

-  if (db_impl_ != nullptr && cfd_ != nullptr) {
+  if (cfh_ != nullptr) {
    // TODO: What do we do if this returns an error?
    Slice lower_bound, upper_bound;
    if (iterate_lower_bound_ != nullptr) {
@@ -1546,8 +1586,8 @@ void DBIter::SeekForPrev(const Slice& target) {
    } else {
      upper_bound = Slice("");
    }
-    db_impl_
-        ->TraceIteratorSeekForPrev(cfd_->GetID(), target, lower_bound,
+    cfh_->db()
+        ->TraceIteratorSeekForPrev(cfh_->cfd()->GetID(), target, lower_bound,
                                   upper_bound)
        .PermitUncheckedError();
  }
@@ -1711,13 +1751,12 @@ Iterator* NewDBIterator(Env* env, const ReadOptions& read_options,
                        InternalIterator* internal_iter, const Version* version,
                        const SequenceNumber& sequence,
                        uint64_t max_sequential_skip_in_iterations,
-                        ReadCallback* read_callback, DBImpl* db_impl,
-                        ColumnFamilyData* cfd, bool expose_blob_index) {
-  DBIter* db_iter =
-      new DBIter(env, read_options, ioptions, mutable_cf_options,
-                 user_key_comparator, internal_iter, version, sequence, false,
-                 max_sequential_skip_in_iterations, read_callback, db_impl, cfd,
-                 expose_blob_index);
+                        ReadCallback* read_callback,
+                        ColumnFamilyHandleImpl* cfh, bool expose_blob_index) {
+  DBIter* db_iter = new DBIter(
+      env, read_options, ioptions, mutable_cf_options, user_key_comparator,
+      internal_iter, version, sequence, false,
+      max_sequential_skip_in_iterations, read_callback, cfh, expose_blob_index);
  return db_iter;
 }

@@ -118,7 +118,7 @@ class DBIter final : public Iterator {
         const MutableCFOptions& mutable_cf_options, const Comparator* cmp,
         InternalIterator* iter, const Version* version, SequenceNumber s,
         bool arena_mode, uint64_t max_sequential_skip_in_iterations,
-         ReadCallback* read_callback, DBImpl* db_impl, ColumnFamilyData* cfd,
+         ReadCallback* read_callback, ColumnFamilyHandleImpl* cfh,
         bool expose_blob_index);

  // No copying allowed
@@ -367,6 +367,12 @@ class DBIter final : public Iterator {
  // and should not be used across functions. Reusing this object can reduce
  // overhead of calling construction of the function if creating it each time.
  ParsedInternalKey ikey_;
+
+  // The approximate write time for the entry. It is deduced from the entry's
+  // sequence number if the seqno to time mapping is available. For a
+  // kTypeValuePreferredSeqno entry, this is the write time specified by the
+  // user.
+  uint64_t saved_write_unix_time_;
  std::string saved_value_;
  Slice pinned_value_;
  // for prefix seek mode to support prev()
@@ -417,8 +423,7 @@ class DBIter final : public Iterator {
  MergeContext merge_context_;
  LocalStatistics local_stats_;
  PinnedIteratorsManager pinned_iters_mgr_;
-  DBImpl* db_impl_;
-  ColumnFamilyData* cfd_;
+  ColumnFamilyHandleImpl* cfh_;
  const Slice* const timestamp_ub_;
  const Slice* const timestamp_lb_;
  const size_t timestamp_size_;
@@ -428,13 +433,12 @@ class DBIter final : public Iterator {
 // Return a new iterator that converts internal keys (yielded by
 // "*internal_iter") that were live at the specified `sequence` number
 // into appropriate user keys.
-extern Iterator* NewDBIterator(
+Iterator* NewDBIterator(
    Env* env, const ReadOptions& read_options, const ImmutableOptions& ioptions,
    const MutableCFOptions& mutable_cf_options,
    const Comparator* user_key_comparator, InternalIterator* internal_iter,
    const Version* version, const SequenceNumber& sequence,
    uint64_t max_sequential_skip_in_iterations, ReadCallback* read_callback,
-    DBImpl* db_impl = nullptr, ColumnFamilyData* cfd = nullptr,
-    bool expose_blob_index = false);
+    ColumnFamilyHandleImpl* cfh = nullptr, bool expose_blob_index = false);

 }  // namespace ROCKSDB_NAMESPACE
@@ -45,6 +45,14 @@ class TestIterator : public InternalIterator {
    Add(argkey, kTypeValue, argvalue);
  }

+  void AddTimedPut(std::string argkey, std::string argvalue,
+                   uint64_t write_unix_time) {
+    std::string packed_value;
+    [[maybe_unused]] auto packed_value_slice =
+        PackValueAndWriteTime(argvalue, write_unix_time, &packed_value);
+    Add(argkey, kTypeValuePreferredSeqno, packed_value);
+  }
+
  void AddDeletion(std::string argkey) {
    Add(argkey, kTypeDeletion, std::string());
  }
@@ -1388,6 +1396,60 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
  }
 }

+TEST_F(DBIteratorTest, DBIteratorTimedPutBasic) {
+  ReadOptions ro;
+  Options options;
+  options.merge_operator = MergeOperators::CreateFromStringId("stringappend");
+
+  TestIterator* internal_iter = new TestIterator(BytewiseComparator());
+  internal_iter->AddTimedPut("a", "0", /*write_unix_time=*/0);
+  internal_iter->AddMerge("a", "1");
+  internal_iter->AddTimedPut("b", "0", /*write_unix_time=*/0);
+  internal_iter->AddDeletion("b");
+  internal_iter->AddTimedPut("c", "01", /*write_unix_time=*/0);
+  internal_iter->AddTimedPut("c", "02", /*write_unix_time=*/0);
+  internal_iter->AddTimedPut("c", "2", /*write_unix_time=*/0);
+  internal_iter->AddTimedPut("d", "3", /*write_unix_time=*/0);
+  internal_iter->Finish();
+
+  std::unique_ptr<Iterator> db_iter(NewDBIterator(
+      env_, ro, ImmutableOptions(options), MutableCFOptions(options),
+      BytewiseComparator(), internal_iter, nullptr /* version */,
+      7 /* sequence */, /*max_sequential_skip_in_iterations*/ 1,
+      nullptr /* read_callback */));
+  db_iter->SeekToFirst();
+  ASSERT_TRUE(db_iter->Valid());
+  ASSERT_EQ(db_iter->key().ToString(), "a");
+  ASSERT_EQ(db_iter->value().ToString(), "0,1");
+  db_iter->Next();
+  ASSERT_TRUE(db_iter->Valid());
+  ASSERT_EQ(db_iter->key().ToString(), "c");
+  ASSERT_EQ(db_iter->value().ToString(), "2");
+  db_iter->Next();
+  ASSERT_TRUE(db_iter->Valid());
+  ASSERT_EQ(db_iter->key().ToString(), "d");
+  ASSERT_EQ(db_iter->value().ToString(), "3");
+  db_iter->Next();
+  ASSERT_FALSE(db_iter->Valid());
+  ASSERT_OK(db_iter->status());
+
+  db_iter->SeekToLast();
+  ASSERT_TRUE(db_iter->Valid());
+  ASSERT_EQ(db_iter->key().ToString(), "d");
+  ASSERT_EQ(db_iter->value().ToString(), "3");
+  db_iter->Prev();
+  ASSERT_TRUE(db_iter->Valid());
+  ASSERT_EQ(db_iter->key().ToString(), "c");
+  ASSERT_EQ(db_iter->value().ToString(), "2");
+  db_iter->Prev();
+  ASSERT_TRUE(db_iter->Valid());
+  ASSERT_EQ(db_iter->key().ToString(), "a");
+  ASSERT_EQ(db_iter->value().ToString(), "0,1");
+  db_iter->Prev();
+  ASSERT_FALSE(db_iter->Valid());
+  ASSERT_OK(db_iter->status());
+}
+
 TEST_F(DBIteratorTest, DBIterator1) {
  ReadOptions ro;
  Options options;
@@ -2824,7 +2886,7 @@ TEST_F(DBIterWithMergeIterTest, InnerMergeIteratorDataRace4) {
  // Seek() and before calling Prev()
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "MergeIterator::Prev:BeforePrev", [&](void* arg) {
-        IteratorWrapper* it = reinterpret_cast<IteratorWrapper*>(arg);
+        IteratorWrapper* it = static_cast<IteratorWrapper*>(arg);
        if (it->key().starts_with("z")) {
          internal_iter2_->Add("x", kTypeValue, "7", 16u, true);
          internal_iter2_->Add("x", kTypeValue, "7", 15u, true);
@@ -2875,7 +2937,7 @@ TEST_F(DBIterWithMergeIterTest, InnerMergeIteratorDataRace5) {
  // Seek() and before calling Prev()
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "MergeIterator::Prev:BeforePrev", [&](void* arg) {
-        IteratorWrapper* it = reinterpret_cast<IteratorWrapper*>(arg);
+        IteratorWrapper* it = static_cast<IteratorWrapper*>(arg);
        if (it->key().starts_with("z")) {
          internal_iter2_->Add("x", kTypeValue, "7", 16u, true);
          internal_iter2_->Add("x", kTypeValue, "7", 15u, true);
@@ -2922,7 +2984,7 @@ TEST_F(DBIterWithMergeIterTest, InnerMergeIteratorDataRace6) {
  // Seek() and before calling Prev()
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "MergeIterator::Prev:BeforePrev", [&](void* arg) {
-        IteratorWrapper* it = reinterpret_cast<IteratorWrapper*>(arg);
+        IteratorWrapper* it = static_cast<IteratorWrapper*>(arg);
        if (it->key().starts_with("z")) {
          internal_iter2_->Add("x", kTypeValue, "7", 16u, true);
        }
@@ -2971,7 +3033,7 @@ TEST_F(DBIterWithMergeIterTest, InnerMergeIteratorDataRace7) {
  // Seek() and before calling Prev()
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "MergeIterator::Prev:BeforePrev", [&](void* arg) {
-        IteratorWrapper* it = reinterpret_cast<IteratorWrapper*>(arg);
+        IteratorWrapper* it = static_cast<IteratorWrapper*>(arg);
        if (it->key().starts_with("z")) {
          internal_iter2_->Add("x", kTypeValue, "7", 16u, true);
          internal_iter2_->Add("x", kTypeValue, "7", 15u, true);
@@ -3024,7 +3086,7 @@ TEST_F(DBIterWithMergeIterTest, InnerMergeIteratorDataRace8) {
  // before calling Prev()
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "MergeIterator::Prev:BeforePrev", [&](void* arg) {
-        IteratorWrapper* it = reinterpret_cast<IteratorWrapper*>(arg);
+        IteratorWrapper* it = static_cast<IteratorWrapper*>(arg);
        if (it->key().starts_with("z")) {
          internal_iter2_->Add("x", kTypeValue, "7", 16u, true);
          internal_iter2_->Add("y", kTypeValue, "7", 17u, true);
@@ -93,8 +93,8 @@ class DBIteratorTest : public DBIteratorBaseTest,
    if (column_family == nullptr) {
      column_family = db_->DefaultColumnFamily();
    }
-    auto* cfd =
-        static_cast_with_check<ColumnFamilyHandleImpl>(column_family)->cfd();
+    auto* cfh = static_cast_with_check<ColumnFamilyHandleImpl>(column_family);
+    auto* cfd = cfh->cfd();
    SequenceNumber seq = read_options.snapshot != nullptr
                             ? read_options.snapshot->GetSequenceNumber()
                             : db_->GetLatestSequenceNumber();
@@ -109,7 +109,7 @@ class DBIteratorTest : public DBIteratorBaseTest,
    }
    DBImpl* db_impl = dbfull();
    SuperVersion* super_version = cfd->GetReferencedSuperVersion(db_impl);
-    return db_impl->NewIteratorImpl(read_options, cfd, super_version, seq,
+    return db_impl->NewIteratorImpl(read_options, cfh, super_version, seq,
                                    read_callback);
  }

@@ -142,6 +142,13 @@ TEST_P(DBIteratorTest, IteratorProperty) {
    // Get internal key at which the iteration stopped (tombstone in this case).
    ASSERT_OK(iter->GetProperty("rocksdb.iterator.internal-key", &prop_value));
    ASSERT_EQ("2", prop_value);
+
+    prop_value.clear();
+    ASSERT_OK(iter->GetProperty("rocksdb.iterator.write-time", &prop_value));
+    uint64_t write_time;
+    Slice prop_slice = prop_value;
+    ASSERT_TRUE(GetFixed64(&prop_slice, &write_time));
+    ASSERT_EQ(std::numeric_limits<uint64_t>::max(), write_time);
  }
  Close();
 }
@@ -2525,7 +2532,7 @@ TEST_P(DBIteratorTest, RefreshWithSnapshot) {
 TEST_P(DBIteratorTest, CreationFailure) {
  SyncPoint::GetInstance()->SetCallBack(
      "DBImpl::NewInternalIterator:StatusCallback", [](void* arg) {
-        *(reinterpret_cast<Status*>(arg)) = Status::Corruption("test status");
+        *(static_cast<Status*>(arg)) = Status::Corruption("test status");
      });
  SyncPoint::GetInstance()->EnableProcessing();

@@ -3206,13 +3213,13 @@ TEST_F(DBIteratorWithReadCallbackTest, ReadCallback) {
  ASSERT_OK(Put("bar", "v7"));

  SequenceNumber seq2 = db_->GetLatestSequenceNumber();
-  auto* cfd =
-      static_cast_with_check<ColumnFamilyHandleImpl>(db_->DefaultColumnFamily())
-          ->cfd();
+  auto* cfh = static_cast_with_check<ColumnFamilyHandleImpl>(
+      db_->DefaultColumnFamily());
+  auto* cfd = cfh->cfd();
  // The iterator are suppose to see data before seq1.
  DBImpl* db_impl = dbfull();
  SuperVersion* super_version = cfd->GetReferencedSuperVersion(db_impl);
-  Iterator* iter = db_impl->NewIteratorImpl(ReadOptions(), cfd, super_version,
+  Iterator* iter = db_impl->NewIteratorImpl(ReadOptions(), cfh, super_version,
                                            seq2, &callback1);

  // Seek
@@ -3292,7 +3299,7 @@ TEST_F(DBIteratorWithReadCallbackTest, ReadCallback) {

  // The iterator is suppose to see data before seq3.
  super_version = cfd->GetReferencedSuperVersion(db_impl);
-  iter = db_impl->NewIteratorImpl(ReadOptions(), cfd, super_version, seq4,
+  iter = db_impl->NewIteratorImpl(ReadOptions(), cfh, super_version, seq4,
                                  &callback2);
  // Seek to "z", which is visible.
  iter->Seek("z");
@@ -3448,8 +3455,7 @@ TEST_F(DBIteratorTest, ErrorWhenReadFile) {
  SyncPoint::GetInstance()->SetCallBack(
      "RandomAccessFileReader::Read::BeforeReturn",
      [&error_file](void* io_s_ptr) {
-        auto p =
-            reinterpret_cast<std::pair<std::string*, IOStatus*>*>(io_s_ptr);
+        auto p = static_cast<std::pair<std::string*, IOStatus*>*>(io_s_ptr);
        if (p->first->find(error_file) != std::string::npos) {
          *p->second = IOStatus::IOError();
          p->second->SetRetryable(true);
@@ -3529,8 +3535,7 @@ TEST_F(DBIteratorTest, ErrorWhenReadFile) {
  SyncPoint::GetInstance()->ClearAllCallBacks();
  SyncPoint::GetInstance()->SetCallBack(
      "RandomAccessFileReader::Read::AnyOffset", [&f1](void* pair_ptr) {
-        auto p =
-            reinterpret_cast<std::pair<std::string*, IOStatus*>*>(pair_ptr);
+        auto p = static_cast<std::pair<std::string*, IOStatus*>*>(pair_ptr);
        if (p->first->find(f1) != std::string::npos) {
          *p->second = IOStatus::IOError();
          p->second->SetRetryable(true);
@@ -437,14 +437,13 @@ TEST_P(DbKvChecksumTestMergedBatch, WriteToWALCorrupted) {
  // This callback should only be called by the leader thread
  SyncPoint::GetInstance()->SetCallBack(
      "WriteThread::JoinBatchGroup:Wait2", [&](void* arg_leader) {
-        auto* leader = reinterpret_cast<WriteThread::Writer*>(arg_leader);
+        auto* leader = static_cast<WriteThread::Writer*>(arg_leader);
        ASSERT_EQ(leader->state, WriteThread::STATE_GROUP_LEADER);

        // This callback should only be called by the follower thread
        SyncPoint::GetInstance()->SetCallBack(
            "WriteThread::JoinBatchGroup:Wait", [&](void* arg_follower) {
-              auto* follower =
-                  reinterpret_cast<WriteThread::Writer*>(arg_follower);
+              auto* follower = static_cast<WriteThread::Writer*>(arg_follower);
              // The leader thread will wait on this bool and hence wait until
              // this writer joins the write group
              ASSERT_NE(follower->state, WriteThread::STATE_GROUP_LEADER);
@@ -549,14 +548,13 @@ TEST_P(DbKvChecksumTestMergedBatch, WriteToWALWithColumnFamilyCorrupted) {
  // This callback should only be called by the leader thread
  SyncPoint::GetInstance()->SetCallBack(
      "WriteThread::JoinBatchGroup:Wait2", [&](void* arg_leader) {
-        auto* leader = reinterpret_cast<WriteThread::Writer*>(arg_leader);
+        auto* leader = static_cast<WriteThread::Writer*>(arg_leader);
        ASSERT_EQ(leader->state, WriteThread::STATE_GROUP_LEADER);

        // This callback should only be called by the follower thread
        SyncPoint::GetInstance()->SetCallBack(
            "WriteThread::JoinBatchGroup:Wait", [&](void* arg_follower) {
-              auto* follower =
-                  reinterpret_cast<WriteThread::Writer*>(arg_follower);
+              auto* follower = static_cast<WriteThread::Writer*>(arg_follower);
              // The leader thread will wait on this bool and hence wait until
              // this writer joins the write group
              ASSERT_NE(follower->state, WriteThread::STATE_GROUP_LEADER);
@@ -662,7 +660,7 @@ TEST_F(DbKVChecksumWALToWriteBatchTest, WriteBatchChecksumHandoff) {
  SyncPoint::GetInstance()->SetCallBack(
      "DBImpl::RecoverLogFiles:BeforeUpdateProtectionInfo:batch",
      [&](void* batch_ptr) {
-        WriteBatch* batch = reinterpret_cast<WriteBatch*>(batch_ptr);
+        WriteBatch* batch = static_cast<WriteBatch*>(batch_ptr);
        content.assign(batch->Data().data(), batch->GetDataSize());
        Slice batch_content = batch->Data();
        // Corrupt first bit
@@ -672,7 +670,7 @@ TEST_F(DbKVChecksumWALToWriteBatchTest, WriteBatchChecksumHandoff) {
      "DBImpl::RecoverLogFiles:BeforeUpdateProtectionInfo:checksum",
      [&](void* checksum_ptr) {
        // Verify that checksum is produced on the batch content
-        uint64_t checksum = *reinterpret_cast<uint64_t*>(checksum_ptr);
+        uint64_t checksum = *static_cast<uint64_t*>(checksum_ptr);
        ASSERT_EQ(checksum, XXH3_64bits(content.data(), content.size()));
      });
  SyncPoint::GetInstance()->EnableProcessing();
@@ -14,6 +14,7 @@
 #include "db/db_test_util.h"
 #include "env/mock_env.h"
 #include "port/stack_trace.h"
+#include "util/atomic.h"

 namespace ROCKSDB_NAMESPACE {

@@ -147,6 +148,7 @@ TEST_F(DBTestXactLogIterator, TransactionLogIteratorRace) {
 }

 TEST_F(DBTestXactLogIterator, TransactionLogIteratorCheckWhenArchive) {
+  RelaxedAtomic<bool> callback_hit{};
  do {
    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearTrace();
    Options options = OptionsForLogIterTest();
@@ -168,17 +170,23 @@ TEST_F(DBTestXactLogIterator, TransactionLogIteratorCheckWhenArchive) {
    ASSERT_OK(dbfull()->Put(WriteOptions(), "key4", DummyString(1024)));
    ASSERT_OK(dbfull()->Flush(FlushOptions()));

+    callback_hit.StoreRelaxed(false);
    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
        "WalManager::PurgeObsoleteFiles:1", [&](void*) {
          auto iter = OpenTransactionLogIter(0);
          ExpectRecords(4, iter);
+          callback_hit.StoreRelaxed(true);
        });

    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
    ASSERT_OK(dbfull()->Flush(FlushOptions(), cf));
-
    delete cf;
+    // Normally hit several times; WART: perhaps more in parallel after flush
+    // FIXME: this test is flaky
+    // ASSERT_TRUE(callback_hit.LoadRelaxed());
  } while (ChangeCompactOptions());
+  Close();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }
 #endif

@@ -236,7 +244,7 @@ TEST_F(DBTestXactLogIterator, TransactionLogIteratorCorruptedLog) {
    ASSERT_OK(test::TruncateFile(env_, logfile_path,
                                 wal_files.front()->SizeFileBytes() / 2));

-    ASSERT_OK(db_->EnableFileDeletions(/*force=*/false));
+    ASSERT_OK(db_->EnableFileDeletions());

    // Insert a new entry to a new log file
    ASSERT_OK(Put("key1025", DummyString(10)));
@@ -121,7 +121,7 @@ class TestPrefixExtractor : public SliceTransform {

 private:
  const char* separator(const Slice& key) const {
-    return reinterpret_cast<const char*>(memchr(key.data(), '_', key.size()));
+    return static_cast<const char*>(memchr(key.data(), '_', key.size()));
  }
 };

@@ -287,7 +287,7 @@ TEST_F(DBMemTableTest, InsertWithHint) {
  options.env = env_;
  Reopen(options);
  MockMemTableRep* rep =
-      reinterpret_cast<MockMemTableRepFactory*>(options.memtable_factory.get())
+      static_cast<MockMemTableRepFactory*>(options.memtable_factory.get())
          ->rep();
  ASSERT_OK(Put("foo_k1", "foo_v1"));
  ASSERT_EQ(nullptr, rep->last_hint_in());
@@ -430,6 +430,111 @@ TEST_F(DBMergeOperandTest, GetMergeOperandsLargeResultOptimization) {
  }
 }

+TEST_F(DBMergeOperandTest, GetMergeOperandsShortCircuitInMemtable) {
+  const int kNumOperands = 10;
+  const int kNumOperandsToFetch = 5;
+
+  Options options = CurrentOptions();
+  options.merge_operator = MergeOperators::CreateStringAppendOperator();
+  DestroyAndReopen(options);
+
+  Random rnd(301);
+  std::vector<std::string> expected_merge_operands;
+  expected_merge_operands.reserve(kNumOperands);
+  for (int i = 0; i < kNumOperands; ++i) {
+    expected_merge_operands.emplace_back(rnd.RandomString(7 /* len */));
+    ASSERT_OK(Merge("key", expected_merge_operands.back()));
+  }
+
+  std::vector<PinnableSlice> merge_operands(kNumOperands);
+  GetMergeOperandsOptions merge_operands_info;
+  merge_operands_info.expected_max_number_of_operands = kNumOperands;
+  int num_fetched = 0;
+  merge_operands_info.continue_cb = [&](Slice /* value */) {
+    num_fetched++;
+    return num_fetched != kNumOperandsToFetch;
+  };
+  int num_merge_operands = 0;
+  ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(),
+                                  "key", merge_operands.data(),
+                                  &merge_operands_info, &num_merge_operands));
+  ASSERT_EQ(kNumOperandsToFetch, num_merge_operands);
+  ASSERT_EQ(kNumOperandsToFetch, num_fetched);
+
+  for (int i = 0; i < kNumOperandsToFetch; ++i) {
+    ASSERT_EQ(expected_merge_operands[kNumOperands - kNumOperandsToFetch + i],
+              merge_operands[i]);
+  }
+}
+
+TEST_F(DBMergeOperandTest, GetMergeOperandsShortCircuitBaseValue) {
+  // The continuation callback doesn't need to be called on a base value because
+  // there's no remaining work to be saved.
+  Options options = CurrentOptions();
+  options.merge_operator = MergeOperators::CreateStringAppendOperator();
+  DestroyAndReopen(options);
+
+  Random rnd(301);
+  std::string expected_value = rnd.RandomString(7 /* len */);
+  ASSERT_OK(Put("key", expected_value));
+
+  std::vector<PinnableSlice> merge_operands(1);
+  GetMergeOperandsOptions merge_operands_info;
+  merge_operands_info.expected_max_number_of_operands = 1;
+  int num_fetched = 0;
+  merge_operands_info.continue_cb = [&num_fetched](Slice /* value */) {
+    num_fetched++;
+    return true;
+  };
+  int num_merge_operands = 0;
+  ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(),
+                                  "key", merge_operands.data(),
+                                  &merge_operands_info, &num_merge_operands));
+  ASSERT_EQ(1, num_merge_operands);
+  ASSERT_EQ(0, num_fetched);
+
+  ASSERT_EQ(expected_value, merge_operands[0]);
+}
+
+TEST_F(DBMergeOperandTest, GetMergeOperandsShortCircuitInSst) {
+  const int kNumOperands = 10;
+  const int kNumOperandsToFetch = 5;
+
+  Options options = CurrentOptions();
+  options.disable_auto_compactions = true;
+  options.merge_operator = MergeOperators::CreateStringAppendOperator();
+  DestroyAndReopen(options);
+
+  Random rnd(301);
+  std::vector<std::string> expected_merge_operands;
+  expected_merge_operands.reserve(kNumOperands);
+  for (int i = 0; i < kNumOperands; ++i) {
+    expected_merge_operands.emplace_back(rnd.RandomString(7 /* len */));
+    ASSERT_OK(Merge("key", expected_merge_operands.back()));
+    ASSERT_OK(Flush());
+  }
+
+  std::vector<PinnableSlice> merge_operands(kNumOperands);
+  GetMergeOperandsOptions merge_operands_info;
+  merge_operands_info.expected_max_number_of_operands = kNumOperands;
+  int num_fetched = 0;
+  merge_operands_info.continue_cb = [&](Slice /* value */) {
+    num_fetched++;
+    return num_fetched != kNumOperandsToFetch;
+  };
+  int num_merge_operands = 0;
+  ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(),
+                                  "key", merge_operands.data(),
+                                  &merge_operands_info, &num_merge_operands));
+  ASSERT_EQ(kNumOperandsToFetch, num_merge_operands);
+  ASSERT_EQ(kNumOperandsToFetch, num_fetched);
+
+  for (int i = 0; i < kNumOperandsToFetch; ++i) {
+    ASSERT_EQ(expected_merge_operands[kNumOperands - kNumOperandsToFetch + i],
+              merge_operands[i]);
+  }
+}
+
 TEST_F(DBMergeOperandTest, GetMergeOperandsBaseDeletionInImmMem) {
  // In this test, "k1" has a MERGE in a mutable memtable on top of a base
  // DELETE in an immutable memtable.
@@ -1390,7 +1390,7 @@ TEST_F(DBOptionsTest, ChangeCompression) {
  bool compacted = false;
  SyncPoint::GetInstance()->SetCallBack(
      "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) {
-        Compaction* c = reinterpret_cast<Compaction*>(arg);
+        Compaction* c = static_cast<Compaction*>(arg);
        compression_used = c->output_compression();
        compression_opt_used = c->output_compression_opts();
        compacted = true;