Explicitly mark destructors with override (#13212 )

Summary: I saw these compiler warnings while preparing for the 9.10 release: ```cpp '~CompactOnDeletionCollectorFactory' overrides a destructor but is not marked 'override' [-Werror,-Wsuggest-destructor-override] '~CompactForTieringCollectorFactory' overrides a destructor but is not marked 'override' [-Werror,-Wsuggest-destructor-override] ``` This code is from a while ago so I assume that this CI check has been failing for quite some time. We should still clean this up to avoid confusion in the future. Pull Request resolved: https://github.com/facebook/rocksdb/pull/13212 Test Plan: Existing CI checks should pass, and we should not see this CI check failure the next time we try to make a release/patch. Reviewed By: jaykorean Differential Revision: D67287794 Pulled By: archang19 fbshipit-source-id: a11230a919c0b7ef21a7219bf05f567d3d44b2d1
Add files for 9.10 release
2026-11-07 03:22:34 -05:00 · 2024-12-16 12:45:14 -08:00 · 2024-12-13 10:51:47 -08:00 · 2024-12-12 16:33:34 -08:00 · 2024-12-12 11:41:16 -08:00 · 2024-12-09 18:56:27 -08:00
202 changed files with 10660 additions and 2297 deletions
@@ -83,6 +83,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
        "db/log_writer.cc",
        "db/logs_with_prep_tracker.cc",
        "db/malloc_stats.cc",
+        "db/manifest_ops.cc",
        "db/memtable.cc",
        "db/memtable_list.cc",
        "db/merge_helper.cc",
@@ -149,6 +150,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
        "memtable/hash_skiplist_rep.cc",
        "memtable/skiplistrep.cc",
        "memtable/vectorrep.cc",
+        "memtable/wbwi_memtable.cc",
        "memtable/write_buffer_manager.cc",
        "monitoring/histogram.cc",
        "monitoring/histogram_windowing.cc",
@@ -366,6 +368,11 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[

 cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[], deps=[":rocksdb_lib"], headers=[], link_whole=True, extra_test_libs=False)

+cpp_library_wrapper(name="rocksdb_with_faiss_lib", srcs=["utilities/secondary_index/faiss_ivf_index.cc"], deps=[
+        "//faiss:faiss",
+        ":rocksdb_lib",
+    ], headers=[], link_whole=False, extra_test_libs=False)
+
 cpp_library_wrapper(name="rocksdb_test_lib", srcs=[
        "db/db_test_util.cc",
        "db/db_with_timestamp_test_util.cc",
@@ -380,6 +387,20 @@ cpp_library_wrapper(name="rocksdb_test_lib", srcs=[
        "utilities/cassandra/test_utils.cc",
    ], deps=[":rocksdb_lib"], headers=[], link_whole=False, extra_test_libs=True)

+cpp_library_wrapper(name="rocksdb_with_faiss_test_lib", srcs=[
+        "db/db_test_util.cc",
+        "db/db_with_timestamp_test_util.cc",
+        "table/mock_table.cc",
+        "test_util/mock_time_env.cc",
+        "test_util/secondary_cache_test_util.cc",
+        "test_util/testharness.cc",
+        "test_util/testutil.cc",
+        "tools/block_cache_analyzer/block_cache_trace_analyzer.cc",
+        "tools/trace_analyzer_tool.cc",
+        "utilities/agg_merge/test_agg_merge.cc",
+        "utilities/cassandra/test_utils.cc",
+    ], deps=[":rocksdb_with_faiss_lib"], headers=[], link_whole=False, extra_test_libs=True)
+
 cpp_library_wrapper(name="rocksdb_tools_lib", srcs=[
        "test_util/testutil.cc",
        "tools/block_cache_analyzer/block_cache_trace_analyzer.cc",
@@ -5076,6 +5097,12 @@ cpp_unittest_wrapper(name="external_sst_file_test",
            extra_compiler_flags=[])


+cpp_unittest_wrapper(name="faiss_ivf_index_test",
+            srcs=["utilities/secondary_index/faiss_ivf_index_test.cc"],
+            deps=[":rocksdb_with_faiss_test_lib"],
+            extra_compiler_flags=[])
+
+
 cpp_unittest_wrapper(name="fault_injection_test",
            srcs=["db/fault_injection_test.cc"],
            deps=[":rocksdb_test_lib"],
@@ -716,6 +716,7 @@ set(SOURCES
        db/log_reader.cc
        db/log_writer.cc
        db/malloc_stats.cc
+        db/manifest_ops.cc
        db/memtable.cc
        db/memtable_list.cc
        db/merge_helper.cc
@@ -779,6 +780,7 @@ set(SOURCES
        memtable/hash_skiplist_rep.cc
        memtable/skiplistrep.cc
        memtable/vectorrep.cc
+        memtable/wbwi_memtable.cc
        memtable/write_buffer_manager.cc
        monitoring/histogram.cc
        monitoring/histogram_windowing.cc
@@ -1,6 +1,69 @@
 # Rocksdb Change Log
 > NOTE: Entries for next release do not go here. Follow instructions in `unreleased_history/README.txt`

+## 9.10.0 (12/12/2024)
+### New Features
+* Introduce `TransactionOptions::commit_bypass_memtable` to enable transaction commit to bypass memtable insertions. This can be beneficial for transactions with many operations, as it reduces commit time that is mostly spent on memtable insertion.
+
+### Public API Changes
+* Deprecated Remote Compaction APIs (StartV2, WaitForCompleteV2) are completely removed from the codebase
+
+### Behavior Changes
+* DB::KeyMayExist() now follows its function comment, which means `value` parameter can be null, and it will be set only if `value_found` is passed in.
+
+### Bug Fixes
+* Fix the issue where compaction incorrectly drops a key when there is a snapshot with a sequence number of zero.
+* Honor ConfigOptions.ignore_unknown_options in ParseStruct()
+
+### Performance Improvements
+* Enable reuse of file system allocated buffer for synchronous prefetching.
+* In buffered IO mode, try to align writes on power of 2 if checksum handoff is not enabled for the file type being written.
+
+## 9.9.0 (11/18/2024)
+### New Features
+* Multi-Column-Family-Iterator (CoalescingIterator/AttributeGroupIterator) is no longer marked as experimental
+* Adds a new table property "rocksdb.newest.key.time" which records the unix timestamp of the newest key. Uses this table property for FIFO TTL and temperature change compaction.
+
+### Public API Changes
+* Added a new API `Transaction::GetAttributeGroupIterator` that can be used to create a multi-column-family attribute group iterator over the specified column families, including the data from both the transaction and the underlying database. This API is currently supported for optimistic and write-committed pessimistic transactions.
+* Added a new API `Transaction::GetCoalescingIterator` that can be used to create a multi-column-family coalescing iterator over the specified column families, including the data from both the transaction and the underlying database. This API is currently supported for optimistic and write-committed pessimistic transactions.
+
+### Behavior Changes
+* `BaseDeltaIterator` now honors the read option `allow_unprepared_value`.
+
+### Bug Fixes
+* `BaseDeltaIterator` now calls `PrepareValue` on the base iterator in case it has been created with the `allow_unprepared_value` read option set. Earlier, such base iterators could lead to incorrect values being exposed from `BaseDeltaIterator`.
+* Fix a leak of obsolete blob files left open until DB::Close(). This bug was introduced in version 9.4.0.
+* Fix missing cases of corruption retry during DB open and read API processing.
+* Fix a bug for transaction db with 2pc where an old WAL may be retained longer than needed (#13127).
+* Fix leaks of some open SST files (until `DB::Close()`) that are written but never become live due to various failures. (We now have a check for such leaks with no outstanding issues.)
+* Fix a bug for replaying WALs for WriteCommitted transaction DB when its user-defined timestamps setting is toggled on/off between DB sessions.
+
+### Performance Improvements
+* Fix regression in issue #12038 due to `Options::compaction_readahead_size` greater than `max_sectors_kb` (i.e, largest I/O size that the OS issues to a block device defined in linux)
+
+## 9.8.0 (10/25/2024)
+### New Features
+* All non-`block_cache` options in `BlockBasedTableOptions` are now mutable with `DB::SetOptions()`. See also Bug Fixes below.
+* When using iterators with BlobDB, it is now possible to load large values on an on-demand basis, i.e. only if they are actually needed by the application. This can save I/O in use cases where the values associated with certain keys are not needed. For more details, see the new read option `allow_unprepared_value` and the iterator API `PrepareValue`.
+* Add a new file ingestion option `IngestExternalFileOptions::fill_cache` to support not adding blocks from ingested files into block cache during file ingestion.
+* The option `allow_unprepared_value` is now also supported for multi-column-family iterators (i.e. `CoalescingIterator` and `AttributeGroupIterator`).
+* When a file with just one range deletion (standalone range deletion file) is ingested via bulk loading, it will be marked for compaction. During compaction, this type of files can be used to directly filter out some input files that are not protected by any snapshots and completely deleted by the standalone range deletion file.
+
+### Behavior Changes
+* During file ingestion, overlapping files level assignment are done in multiple batches, so that they can potentially be assigned to lower levels other than always land on L0.
+* OPTIONS file to be loaded by remote worker is now preserved so that it does not get purged by the primary host. A similar technique as how we are preserving new SST files from getting purged is used for this. min_options_file_numbers_ is tracked like pending_outputs_ is tracked.
+* Trim readahead_size during scans so data blocks containing keys that are not in the same prefix as the seek key in `Seek()` are not prefetched when `ReadOptions::auto_readahead_size=true` (default value) and `ReadOptions::prefix_same_as_start = true`
+* Assigning levels for external files are done in the same way for universal compaction and leveled compaction. The old behavior tends to assign files to L0 while the new behavior will assign the files to the lowest level possible.
+
+### Bug Fixes
+* Fix a longstanding race condition in SetOptions for `block_based_table_factory` options. The fix has some subtle behavior changes because of copying and replacing the TableFactory on a change with SetOptions, including requiring an Iterator::Refresh() for an existing Iterator to use the latest options.
+* Fix under counting of allocated memory in the compressed secondary cache due to looking at the compressed block size rather than the actual memory allocated, which could be larger due to internal fragmentation.
+* `GetApproximateMemTableStats()` could return disastrously bad estimates 5-25% of the time. The function has been re-engineered to return much better estimates with similar CPU cost.
+* Skip insertion of compressed blocks in the secondary cache if the lowest_used_cache_tier DB option is kVolatileTier.
+* Fix an issue in level compaction where a small CF with small compaction debt can cause the DB to allow parallel compactions. (#13054)
+* Several DB option settings could be lost through `GetOptionsFromString()`, possibly elsewhere as well. Affected options, now fixed:`background_close_inactive_wals`, `write_dbid_to_manifest`, `write_identity_file`, `prefix_seek_opt_in_only`
+
 ## 9.7.0 (09/20/2024)
 ### New Features
 * Make Cache a customizable class that can be instantiated by the object registry.
@@ -23,7 +86,7 @@ by them. Prior to this change they would be orphaned until the DB is re-opened.

 ## 9.6.0 (08/19/2024)
 ### New Features
-* *Best efforts recovery supports recovering to incomplete Version with a clean seqno cut that presents a valid point in time view from the user's perspective, if versioning history doesn't include atomic flush.
+* Best efforts recovery supports recovering to incomplete Version with a clean seqno cut that presents a valid point in time view from the user's perspective, if versioning history doesn't include atomic flush.
 * New option `BlockBasedTableOptions::decouple_partitioned_filters` should improve efficiency in serving read queries because filter and index partitions can consistently target the configured `metadata_block_size`. This option is currently opt-in.
 * Introduce a new mutable CF option `paranoid_memory_checks`. It enables additional validation on data integrity during reads/scanning. Currently, skip list based memtable will validate key ordering during look up and scans.

@@ -35,11 +98,11 @@ by them. Prior to this change they would be orphaned until the DB is re-opened.
 * There may be less intra-L0 compaction triggered by total L0 size being too small. We now use compensated file size (tombstones are assigned some value size) when calculating L0 size and reduce the threshold for L0 size limit. This is to avoid accumulating too much data/tombstones in L0.

 ### Bug Fixes
-* *Make DestroyDB supports slow deletion when it's configured in `SstFileManager`. The slow deletion is subject to the configured `rate_bytes_per_sec`, but not subject to the `max_trash_db_ratio`.
+* Make DestroyDB supports slow deletion when it's configured in `SstFileManager`. The slow deletion is subject to the configured `rate_bytes_per_sec`, but not subject to the `max_trash_db_ratio`.
 * Fixed a bug where we set unprep_seqs_ even when WriteImpl() fails. This was caught by stress test write fault injection in WriteImpl(). This may have incorrectly caused iteration creation failure for unvalidated writes or returned wrong result for WriteUnpreparedTxn::GetUnpreparedSequenceNumbers().
 * Fixed a bug where successful write right after error recovery for last failed write finishes causes duplicate WAL entries
 * Fixed a data race involving the background error status in `unordered_write` mode.
-* *Fix a bug where file snapshot functions like backup, checkpoint may attempt to copy a non-existing manifest file. #12882
+* Fix a bug where file snapshot functions like backup, checkpoint may attempt to copy a non-existing manifest file. #12882
 * Fix a bug where per kv checksum corruption may be ignored in MultiGet().
 * Fix a race condition in pessimistic transactions that could allow multiple transactions with the same name to be registered simultaneously, resulting in a crash or other unpredictable behavior.

@@ -118,7 +181,7 @@ by them. Prior to this change they would be orphaned until the DB is re-opened.
 * Added a new API `GetEntityFromBatchAndDB` to `WriteBatchWithIndex` that can be used for wide-column point lookups with read-your-own-writes consistency. Similarly to `GetFromBatchAndDB`, the API can combine data from the write batch with data from the underlying database if needed. See the API comments for more details.
 * [Experimental] Introduce two new cross-column-family iterators - CoalescingIterator and AttributeGroupIterator. The CoalescingIterator enables users to iterate over multiple column families and access their values and columns. During this iteration, if the same key exists in more than one column family, the keys in the later column family will overshadow the previous ones. The AttributeGroupIterator allows users to gather wide columns per Column Family and create attribute groups while iterating over keys across all CFs.
 * Added a new API `MultiGetEntityFromBatchAndDB` to `WriteBatchWithIndex` that can be used for batched wide-column point lookups with read-your-own-writes consistency. Similarly to `MultiGetFromBatchAndDB`, the API can combine data from the write batch with data from the underlying database if needed. See the API comments for more details.
-* *Adds a `SstFileReader::NewTableIterator` API to support programmatically read a SST file as a raw table file.
+* Adds a `SstFileReader::NewTableIterator` API to support programmatically read a SST file as a raw table file.
 * Add an option to `WaitForCompactOptions` - `wait_for_purge` to make `WaitForCompact()` API wait for background purge to complete

 ### Public API Changes
@@ -147,24 +210,24 @@ the whole DB to be dropped right after migration if the migrated data is larger
 * Fixed a bug where wrong padded bytes are used to generate file checksum and `DataVerificationInfo::checksum` upon file creation
 * Correctly implemented the move semantics of `PinnableWideColumns`.
 * Fixed a bug when the recycle_log_file_num in DBOptions is changed from 0 to non-zero when a DB is reopened. On a subsequent reopen, if a log file created when recycle_log_file_num==0 was reused previously, is alive and is empty, we could end up inserting stale WAL records into the memtable.
-* *Fix a bug where obsolete files' deletion during DB::Open are not rate limited with `SstFilemManager`'s slow deletion feature even if it's configured.
+* Fix a bug where obsolete files' deletion during DB::Open are not rate limited with `SstFilemManager`'s slow deletion feature even if it's configured.

 ## 9.1.0 (03/22/2024)
 ### New Features
 * Added an option, `GetMergeOperandsOptions::continue_cb`, to give users the ability to end `GetMergeOperands()`'s lookup process before all merge operands were found.
-* *Add sanity checks for ingesting external files that currently checks if the user key comparator used to create the file is compatible with the column family's user key comparator.
+* Add sanity checks for ingesting external files that currently checks if the user key comparator used to create the file is compatible with the column family's user key comparator.
 *Support ingesting external files for column family that has user-defined timestamps in memtable only enabled.
 * On file systems that support storage level data checksum and reconstruction, retry SST block reads for point lookups, scans, and flush and compaction if there's a checksum mismatch on the initial read.
 * Some enhancements and fixes to experimental Temperature handling features, including new `default_write_temperature` CF option and opening an `SstFileWriter` with a temperature.
 * `WriteBatchWithIndex` now supports wide-column point lookups via the `GetEntityFromBatch` API. See the API comments for more details.
-* *Implement experimental features: API `Iterator::GetProperty("rocksdb.iterator.write-time")` to allow users to get data's approximate write unix time and write data with a specific write time via `WriteBatch::TimedPut` API.
+* Implement experimental features: API `Iterator::GetProperty("rocksdb.iterator.write-time")` to allow users to get data's approximate write unix time and write data with a specific write time via `WriteBatch::TimedPut` API.

 ### Public API Changes
 * Best-effort recovery (`best_efforts_recovery == true`) may now be used together with atomic flush (`atomic_flush == true`). The all-or-nothing recovery guarantee for atomically flushed data will be upheld.
 * Remove deprecated option `bottommost_temperature`, already replaced by `last_level_temperature`
 * Added new PerfContext counters for block cache bytes read - block_cache_index_read_byte, block_cache_filter_read_byte, block_cache_compression_dict_read_byte, and block_cache_read_byte.
 * Deprecate experimental Remote Compaction APIs - StartV2() and WaitForCompleteV2() and introduce Schedule() and Wait(). The new APIs essentially does the same thing as the old APIs. They allow taking externally generated unique id to wait for remote compaction to complete.
-* *For API `WriteCommittedTransaction::GetForUpdate`, if the column family enables user-defined timestamp, it was mandated that argument `do_validate` cannot be false, and UDT based validation has to be done with a user set read timestamp. It's updated to make the UDT based validation optional if user sets `do_validate` to false and does not set a read timestamp. With this, `GetForUpdate` skips UDT based validation and it's users' responsibility to enforce the UDT invariant. SO DO NOT skip this UDT-based validation if users do not have ways to enforce the UDT invariant. Ways to enforce the invariant on the users side include manage a monotonically increasing timestamp, commit transactions in a single thread etc.
+* For API `WriteCommittedTransaction::GetForUpdate`, if the column family enables user-defined timestamp, it was mandated that argument `do_validate` cannot be false, and UDT based validation has to be done with a user set read timestamp. It's updated to make the UDT based validation optional if user sets `do_validate` to false and does not set a read timestamp. With this, `GetForUpdate` skips UDT based validation and it's users' responsibility to enforce the UDT invariant. SO DO NOT skip this UDT-based validation if users do not have ways to enforce the UDT invariant. Ways to enforce the invariant on the users side include manage a monotonically increasing timestamp, commit transactions in a single thread etc.
 * Defined a new PerfLevel `kEnableWait` to measure time spent by user threads blocked in RocksDB other than mutex, such as a write thread waiting to be added to a write group, a write thread delayed or stalled etc.
 * `RateLimiter`'s API no longer requires the burst size to be the refill size. Users of `NewGenericRateLimiter()` can now provide burst size in `single_burst_bytes`. Implementors of `RateLimiter::SetSingleBurstBytes()` need to adapt their implementations to match the changed API doc.
 * Add `write_memtable_time` to the newly introduced PerfLevel `kEnableWait`.
@@ -197,7 +260,7 @@ MultiGetBenchmarks.multiGetList10 no_column_family 10000 16 100 1024 thrpt 25 76
 ## 9.0.0 (02/16/2024)
 ### New Features
 * Provide support for FSBuffer for point lookups. Also added support for scans and compactions that don't go through prefetching.
-* *Make `SstFileWriter` create SST files without persisting user defined timestamps when the `Option.persist_user_defined_timestamps` flag is set to false.
+* Make `SstFileWriter` create SST files without persisting user defined timestamps when the `Option.persist_user_defined_timestamps` flag is set to false.
 * Add support for user-defined timestamps in APIs `DeleteFilesInRanges` and `GetPropertiesOfTablesInRange`.
 * Mark wal\_compression feature as production-ready. Currently only compatible with ZSTD compression.

@@ -207,9 +270,9 @@ MultiGetBenchmarks.multiGetList10 no_column_family 10000 16 100 1024 thrpt 25 76
 * Exposed mode option to Rate Limiter via c api.
 * Removed deprecated option `access_hint_on_compaction_start`
 * Removed deprecated option `ColumnFamilyOptions::check_flush_compaction_key_order`
-* *Remove the default `WritableFile::GetFileSize` and `FSWritableFile::GetFileSize` implementation that returns 0 and make it pure virtual, so that subclasses are enforced to explicitly provide an implementation.
+* Remove the default `WritableFile::GetFileSize` and `FSWritableFile::GetFileSize` implementation that returns 0 and make it pure virtual, so that subclasses are enforced to explicitly provide an implementation.
 * Removed deprecated option `ColumnFamilyOptions::level_compaction_dynamic_file_size`
-* *Removed tickers with typos "rocksdb.error.handler.bg.errro.count", "rocksdb.error.handler.bg.io.errro.count", "rocksdb.error.handler.bg.retryable.io.errro.count".
+* Removed tickers with typos "rocksdb.error.handler.bg.errro.count", "rocksdb.error.handler.bg.io.errro.count", "rocksdb.error.handler.bg.retryable.io.errro.count".
 * Remove the force mode for `EnableFileDeletions` API because it is unsafe with no known legitimate use.
 * Removed deprecated option `ColumnFamilyOptions::ignore_max_compaction_bytes_for_input`
 * `sst_dump --command=check` now compares the number of records in a table with `num_entries` in table property, and reports corruption if there is a mismatch. API `SstFileDumper::ReadSequential()` is updated to optionally do this verification. (#12322)
@@ -659,7 +659,7 @@ ifneq ($(filter check-headers, $(MAKECMDGOALS)),)
 # TODO: add/support JNI headers
 	DEV_HEADER_DIRS := $(sort include/ $(dir $(ALL_SOURCES)))
 # Some headers like in port/ are platform-specific
-	DEV_HEADERS_TO_CHECK := $(shell $(FIND) $(DEV_HEADER_DIRS) -type f -name '*.h' | grep -E -v 'port/|plugin/|lua/|range_tree/')
+	DEV_HEADERS_TO_CHECK := $(shell $(FIND) $(DEV_HEADER_DIRS) -type f -name '*.h' | grep -E -v 'port/|plugin/|lua/|range_tree/|secondary_index/')
 	PUBLIC_HEADERS_TO_CHECK := $(shell $(FIND) include/ -type f -name '*.h' | grep -E -v 'lua/')
 else
 	DEV_HEADERS_TO_CHECK :=
@@ -2489,7 +2489,7 @@ checkout_folly:
 	fi
 	@# Pin to a particular version for public CI, so that PR authors don't
 	@# need to worry about folly breaking our integration. Update periodically
-	cd third-party/folly && git reset --hard 03041f014b6e6ebb6119ffae8b7a37308f52e913
+	cd third-party/folly && git reset --hard 33f5b67fcaeb8705b04fd1b850873a180dc89aaa
 	@# NOTE: this hack is required for clang in some cases
 	perl -pi -e 's/int rv = syscall/int rv = (int)syscall/' third-party/folly/folly/detail/Futex.cpp
 	@# NOTE: this hack is required for gcc in some cases
@@ -38,12 +38,11 @@ Snowflake [uses](https://www.snowflake.com/blog/how-foundationdb-powers-snowflak
 The Bing search engine from Microsoft uses RocksDB as the storage engine for its web data platform: https://blogs.bing.com/Engineering-Blog/october-2021/RocksDB-in-Microsoft-Bing

 ## LinkedIn
-Two different use cases at Linkedin are using RocksDB as a storage engine:
+1. [Venice](https://venicedb.org/) is a derived data platform using RocksDB as its storage engine. It is LinkedIn's ML feature store, powering thousands of recommender use cases, including the Feed, Video recommendations, and People You May Know.
+2. LinkedIn's follow feed for storing user's activities. Check out the blog post: https://engineering.linkedin.com/blog/2016/03/followfeed--linkedin-s-feed-made-faster-and-smarter
+3. Apache Samza, open source framework for stream processing.

-1. LinkedIn's follow feed for storing user's activities. Check out the blog post: https://engineering.linkedin.com/blog/2016/03/followfeed--linkedin-s-feed-made-faster-and-smarter
-2. Apache Samza, open source framework for stream processing
-
-Learn more about those use cases in a Tech Talk by Ankit Gupta and Naveen Somasundaram: http://www.youtube.com/watch?v=plqVp_OnSzg
+Learn more about LinkedIn's follow feed and Apache Samza in a Tech Talk by Ankit Gupta and Naveen Somasundaram: http://www.youtube.com/watch?v=plqVp_OnSzg

 ## Yahoo
 Yahoo is using RocksDB as a storage engine for their biggest distributed data store Sherpa. Learn more about it here: http://yahooeng.tumblr.com/post/120730204806/sherpa-scales-new-heights
@@ -14,10 +14,10 @@ from targets_builder import TARGETSBuilder, LiteralValue

 from util import ColorString

-# This script generates TARGETS file for Buck.
+# This script generates BUCK file for Buck.
 # Buck is a build tool specifying dependencies among different build targets.
 # User can pass extra dependencies as a JSON object via command line, and this
-# script can include these dependencies in the generate TARGETS file.
+# script can include these dependencies in the generate BUCK file.
 # Usage:
 # $python3 buckifier/buckify_rocksdb.py
 # (This generates a TARGET file without user-specified dependency for unit
@@ -28,7 +28,7 @@ from util import ColorString
 #                      "extra_compiler_flags": ["-DFOO_BAR", "-Os"]
 #                  }
 #         }'
-# (Generated TARGETS file has test_dep and mock1 as dependencies for RocksDB
+# (Generated BUCK file has test_dep and mock1 as dependencies for RocksDB
 # unit tests, and will use the extra_compiler_flags to compile the unit test
 # source.)

@@ -114,9 +114,9 @@ def get_dependencies():
    return deps_map


-# Prepare TARGETS file for buck
-def generate_targets(repo_path, deps_map):
-    print(ColorString.info("Generating TARGETS"))
+# Prepare BUCK file for buck
+def generate_buck(repo_path, deps_map):
+    print(ColorString.info("Generating BUCK"))
    # parsed src.mk file
    src_mk = parse_src_mk(repo_path)
    # get all .cc files
@@ -133,10 +133,10 @@ def generate_targets(repo_path, deps_map):
        # in how the file was generated.
        extra_argv = " '{}'".format(" ".join(sys.argv[1].split()))

-    TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path, extra_argv)
+    BUCK = TARGETSBuilder("%s/BUCK" % repo_path, extra_argv)

    # rocksdb_lib
-    TARGETS.add_library(
+    BUCK.add_library(
        "rocksdb_lib",
        src_mk["LIB_SOURCES"] +
        # always add range_tree, it's only excluded on ppc64, which we don't use internally
@@ -152,7 +152,7 @@ def generate_targets(repo_path, deps_map):
        headers=LiteralValue("glob([\"**/*.h\"])")
    )
    # rocksdb_whole_archive_lib
-    TARGETS.add_library(
+    BUCK.add_library(
        "rocksdb_whole_archive_lib",
        [],
        deps=[
@@ -161,8 +161,17 @@ def generate_targets(repo_path, deps_map):
        extra_external_deps="",
        link_whole=True,
    )
+    # rocksdb_with_faiss_lib
+    BUCK.add_library(
+        "rocksdb_with_faiss_lib",
+        src_mk.get("WITH_FAISS_LIB_SOURCES", []),
+        deps=[
+            "//faiss:faiss",
+            ":rocksdb_lib",
+        ],
+    )
    # rocksdb_test_lib
-    TARGETS.add_library(
+    BUCK.add_library(
        "rocksdb_test_lib",
        src_mk.get("MOCK_LIB_SOURCES", [])
        + src_mk.get("TEST_LIB_SOURCES", [])
@@ -171,8 +180,20 @@ def generate_targets(repo_path, deps_map):
        [":rocksdb_lib"],
        extra_test_libs=True,
    )
+    # rocksdb_with_faiss_test_lib
+    BUCK.add_library(
+        "rocksdb_with_faiss_test_lib",
+        src_mk.get("MOCK_LIB_SOURCES", [])
+        + src_mk.get("TEST_LIB_SOURCES", [])
+        + src_mk.get("EXP_LIB_SOURCES", [])
+        + src_mk.get("ANALYZER_LIB_SOURCES", []),
+        deps=[
+            ":rocksdb_with_faiss_lib",
+        ],
+        extra_test_libs=True,
+    )
    # rocksdb_tools_lib
-    TARGETS.add_library(
+    BUCK.add_library(
        "rocksdb_tools_lib",
        src_mk.get("BENCH_LIB_SOURCES", [])
        + src_mk.get("ANALYZER_LIB_SOURCES", [])
@@ -180,51 +201,51 @@ def generate_targets(repo_path, deps_map):
        [":rocksdb_lib"],
    )
    # rocksdb_cache_bench_tools_lib
-    TARGETS.add_library(
+    BUCK.add_library(
        "rocksdb_cache_bench_tools_lib",
        src_mk.get("CACHE_BENCH_LIB_SOURCES", []),
        [":rocksdb_lib"],
    )
    # rocksdb_stress_lib
-    TARGETS.add_rocksdb_library(
+    BUCK.add_rocksdb_library(
        "rocksdb_stress_lib",
        src_mk.get("ANALYZER_LIB_SOURCES", [])
        + src_mk.get("STRESS_LIB_SOURCES", [])
        + ["test_util/testutil.cc"],
    )
    # ldb binary
-    TARGETS.add_binary(
+    BUCK.add_binary(
        "ldb", ["tools/ldb.cc"], [":rocksdb_tools_lib"]
    )
    # db_stress binary
-    TARGETS.add_binary(
+    BUCK.add_binary(
        "db_stress", ["db_stress_tool/db_stress.cc"], [":rocksdb_stress_lib"]
    )
    # db_bench binary
-    TARGETS.add_binary(
+    BUCK.add_binary(
        "db_bench", ["tools/db_bench.cc"], [":rocksdb_tools_lib"]
    )
    # cache_bench binary
-    TARGETS.add_binary(
+    BUCK.add_binary(
        "cache_bench", ["cache/cache_bench.cc"], [":rocksdb_cache_bench_tools_lib"]
    )
    # bench binaries
    for src in src_mk.get("MICROBENCH_SOURCES", []):
        name = src.rsplit("/", 1)[1].split(".")[0] if "/" in src else src.split(".")[0]
-        TARGETS.add_binary(name, [src], [], extra_bench_libs=True)
+        BUCK.add_binary(name, [src], [], extra_bench_libs=True)
    print(f"Extra dependencies:\n{json.dumps(deps_map)}")

    # Dictionary test executable name -> relative source file path
    test_source_map = {}

-    # c_test.c is added through TARGETS.add_c_test(). If there
+    # c_test.c is added through BUCK.add_c_test(). If there
    # are more than one .c test file, we need to extend
-    # TARGETS.add_c_test() to include other C tests too.
+    # BUCK.add_c_test() to include other C tests too.
    for test_src in src_mk.get("TEST_MAIN_SOURCES_C", []):
        if test_src != "db/c_test.c":
            print("Don't know how to deal with " + test_src)
            return False
-    TARGETS.add_c_test()
+    BUCK.add_c_test()

    try:
        with open(f"{repo_path}/buckifier/bench.json") as json_file:
@@ -239,7 +260,7 @@ def generate_targets(repo_path, deps_map):
                        for metric in overloaded_metric_list:
                            if not isinstance(metric, dict):
                                clean_benchmarks[binary][benchmark].append(metric)
-                TARGETS.add_fancy_bench_config(
+                BUCK.add_fancy_bench_config(
                    config_dict["name"],
                    clean_benchmarks,
                    False,
@@ -261,7 +282,7 @@ def generate_targets(repo_path, deps_map):
                            if not isinstance(metric, dict):
                                clean_benchmarks[binary][benchmark].append(metric)
            for config_dict in slow_fancy_bench_config_list:
-                TARGETS.add_fancy_bench_config(
+                BUCK.add_fancy_bench_config(
                    config_dict["name"] + "_slow",
                    clean_benchmarks,
                    True,
@@ -274,15 +295,20 @@ def generate_targets(repo_path, deps_map):
    except Exception:
        pass

-    TARGETS.add_test_header()
+    BUCK.add_test_header()

    for test_src in src_mk.get("TEST_MAIN_SOURCES", []):
        test = test_src.split(".c")[0].strip().split("/")[-1].strip()
-        test_source_map[test] = test_src
+        test_source_map[test] = (test_src, False)
        print("" + test + " " + test_src)

+    for test_src in src_mk.get("WITH_FAISS_TEST_MAIN_SOURCES", []):
+        test = test_src.split(".c")[0].strip().split("/")[-1].strip()
+        test_source_map[test] = (test_src, True)
+        print("" + test + " " + test_src + " [FAISS]")
+
    for target_alias, deps in deps_map.items():
-        for test, test_src in sorted(test_source_map.items()):
+        for test, (test_src, with_faiss) in sorted(test_source_map.items()):
            if len(test) == 0:
                print(ColorString.warning("Failed to get test name for %s" % test_src))
                continue
@@ -291,31 +317,39 @@ def generate_targets(repo_path, deps_map):

            if test in _EXPORTED_TEST_LIBS:
                test_library = "%s_lib" % test_target_name
-                TARGETS.add_library(
+                BUCK.add_library(
                    test_library,
                    [test_src],
                    deps=[":rocksdb_test_lib"],
                    extra_test_libs=True,
                )
-                TARGETS.register_test(
+                BUCK.register_test(
                    test_target_name,
                    test_src,
                    deps=json.dumps(deps["extra_deps"] + [":" + test_library]),
                    extra_compiler_flags=json.dumps(deps["extra_compiler_flags"]),
                )
            else:
-                TARGETS.register_test(
-                    test_target_name,
-                    test_src,
-                    deps=json.dumps(deps["extra_deps"] + [":rocksdb_test_lib"]),
-                    extra_compiler_flags=json.dumps(deps["extra_compiler_flags"]),
-                )
-    TARGETS.export_file("tools/db_crashtest.py")
+                if with_faiss:
+                    BUCK.register_test(
+                        test_target_name,
+                        test_src,
+                        deps=json.dumps(deps["extra_deps"] + [":rocksdb_with_faiss_test_lib"]),
+                        extra_compiler_flags=json.dumps(deps["extra_compiler_flags"]),
+                    )
+                else:
+                    BUCK.register_test(
+                        test_target_name,
+                        test_src,
+                        deps=json.dumps(deps["extra_deps"] + [":rocksdb_test_lib"]),
+                        extra_compiler_flags=json.dumps(deps["extra_compiler_flags"]),
+                    )
+    BUCK.export_file("tools/db_crashtest.py")

-    print(ColorString.info("Generated TARGETS Summary:"))
-    print(ColorString.info("- %d libs" % TARGETS.total_lib))
-    print(ColorString.info("- %d binarys" % TARGETS.total_bin))
-    print(ColorString.info("- %d tests" % TARGETS.total_test))
+    print(ColorString.info("Generated BUCK Summary:"))
+    print(ColorString.info("- %d libs" % BUCK.total_lib))
+    print(ColorString.info("- %d binarys" % BUCK.total_bin))
+    print(ColorString.info("- %d tests" % BUCK.total_test))
    return True


@@ -335,10 +369,10 @@ def exit_with_error(msg):

 def main():
    deps_map = get_dependencies()
-    # Generate TARGETS file for buck
-    ok = generate_targets(get_rocksdb_path(), deps_map)
+    # Generate BUCK file for buck
+    ok = generate_buck(get_rocksdb_path(), deps_map)
    if not ok:
-        exit_with_error("Failed to generate TARGETS files")
+        exit_with_error("Failed to generate BUCK files")


 if __name__ == "__main__":
@@ -1,32 +1,44 @@
 #!/usr/bin/env bash
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-# If clang_format_diff.py command is not specfied, we assume we are able to
-# access directly without any path.

-TGT_DIFF=`git diff TARGETS | head -n 1`
+if [[ ! -f "BUCK" ]]
+then
+  echo "BUCK file is missing!"
+  echo "Please do not remove / rename BUCK file in your commit(s)."
+  exit 1
+fi
+
+TGT_DIFF=`git diff BUCK | head -n 1`

 if [ ! -z "$TGT_DIFF" ]
 then
-  echo "TARGETS file has uncommitted changes. Skip this check."
+  echo "BUCK file has uncommitted changes. Skip this check."
  exit 0
 fi

-echo Backup original TARGETS file.
+echo Backup original BUCK file.

-cp TARGETS TARGETS.bkp
+cp BUCK BUCK.bkp

 ${PYTHON:-python3} buckifier/buckify_rocksdb.py

-TGT_DIFF=`git diff TARGETS | head -n 1`
+if [[ ! -f "BUCK" ]]
+then
+  echo "BUCK file went missing after running buckifier/buckify_rocksdb.py!"
+  echo "Please do not remove the BUCK file."
+  exit 1
+fi
+
+TGT_DIFF=`git diff BUCK | head -n 1`

 if [ -z "$TGT_DIFF" ]
 then
-  mv TARGETS.bkp TARGETS
+  mv BUCK.bkp BUCK
  exit 0
 else
-  echo "Please run '${PYTHON:-python3} buckifier/buckify_rocksdb.py' to update TARGETS file."
-  echo "Do not manually update TARGETS file."
+  echo "Please run '${PYTHON:-python3} buckifier/buckify_rocksdb.py' to update BUCK file."
+  echo "Do not manually update BUCK file."
  ${PYTHON:-python3} --version
-  mv TARGETS.bkp TARGETS
+  mv BUCK.bkp BUCK
  exit 1
 fi
@@ -13,11 +13,11 @@ namespace ROCKSDB_NAMESPACE {
 class AttributeGroupIteratorImpl : public AttributeGroupIterator {
 public:
  AttributeGroupIteratorImpl(
-      const Comparator* comparator, bool allow_unprepared_value,
-      const std::vector<ColumnFamilyHandle*>& column_families,
-      const std::vector<Iterator*>& child_iterators)
-      : impl_(comparator, allow_unprepared_value, column_families,
-              child_iterators, ResetFunc(this), PopulateFunc(this)) {}
+      const ReadOptions& read_options, const Comparator* comparator,
+      std::vector<std::pair<ColumnFamilyHandle*, std::unique_ptr<Iterator>>>&&
+          cfh_iter_pairs)
+      : impl_(read_options, comparator, std::move(cfh_iter_pairs),
+              ResetFunc(this), PopulateFunc(this)) {}
  ~AttributeGroupIteratorImpl() override {}

  // No copy allowed
@@ -42,6 +42,7 @@ Status BlobFileCache::GetBlobFileReader(
  assert(blob_file_reader);
  assert(blob_file_reader->IsEmpty());

+  // NOTE: sharing same Cache with table_cache
  const Slice key = GetSliceForKey(&blob_file_number);

  assert(cache_);
@@ -98,4 +99,13 @@ Status BlobFileCache::GetBlobFileReader(
  return Status::OK();
 }

+void BlobFileCache::Evict(uint64_t blob_file_number) {
+  // NOTE: sharing same Cache with table_cache
+  const Slice key = GetSliceForKey(&blob_file_number);
+
+  assert(cache_);
+
+  cache_.get()->Erase(key);
+}
+
 }  // namespace ROCKSDB_NAMESPACE
@@ -36,6 +36,15 @@ class BlobFileCache {
                           uint64_t blob_file_number,
                           CacheHandleGuard<BlobFileReader>* blob_file_reader);

+  // Called when a blob file is obsolete to ensure it is removed from the cache
+  // to avoid effectively leaking the open file and assicated memory
+  void Evict(uint64_t blob_file_number);
+
+  // Used to identify cache entries for blob files (not normally useful)
+  static const Cache::CacheItemHelper* GetHelper() {
+    return CacheInterface::GetBasicHelper();
+  }
+
 private:
  using CacheInterface =
      BasicTypedCacheInterface<BlobFileReader, CacheEntryRole::kMisc>;
@@ -460,9 +460,18 @@ Status BuildTable(
      Status prepare =
          WritableFileWriter::PrepareIOOptions(tboptions.write_options, opts);
      if (prepare.ok()) {
+        // FIXME: track file for "slow" deletion, e.g. into the
+        // VersionSet::obsolete_files_ pipeline
        Status ignored = fs->DeleteFile(fname, opts, dbg);
        ignored.PermitUncheckedError();
      }
+      // Ensure we don't leak table cache entries when throwing away output
+      // files. (The usual logic in PurgeObsoleteFiles is not applicable because
+      // this function deletes the obsolete file itself, while they should
+      // probably go into the VersionSet::obsolete_files_ pipeline.)
+      TableCache::ReleaseObsolete(table_cache->get_cache().get(),
+                                  meta->fd.GetNumber(), nullptr /*handle*/,
+                                  mutable_cf_options.uncache_aggressiveness);
    }

    assert(blob_file_additions || blob_file_paths.empty());
@@ -9,14 +9,14 @@

 namespace ROCKSDB_NAMESPACE {

-// EXPERIMENTAL
 class CoalescingIterator : public Iterator {
 public:
-  CoalescingIterator(const Comparator* comparator, bool allow_unprepared_value,
-                     const std::vector<ColumnFamilyHandle*>& column_families,
-                     const std::vector<Iterator*>& child_iterators)
-      : impl_(comparator, allow_unprepared_value, column_families,
-              child_iterators, ResetFunc(this), PopulateFunc(this)) {}
+  CoalescingIterator(
+      const ReadOptions& read_options, const Comparator* comparator,
+      std::vector<std::pair<ColumnFamilyHandle*, std::unique_ptr<Iterator>>>&&
+          cfh_iter_pairs)
+      : impl_(read_options, comparator, std::move(cfh_iter_pairs),
+              ResetFunc(this), PopulateFunc(this)) {}
  ~CoalescingIterator() override {}

  // No copy allowed
@@ -466,7 +466,7 @@ void SuperVersion::Cleanup() {
  // decrement reference to the immutable MemtableList
  // this SV object was pointing to.
  imm->Unref(&to_delete);
-  MemTable* m = mem->Unref();
+  ReadOnlyMemTable* m = mem->Unref();
  if (m != nullptr) {
    auto* memory_usage = current->cfd()->imm()->current_memory_usage();
    assert(*memory_usage >= m->ApproximateMemoryUsage());
@@ -693,9 +693,9 @@ ColumnFamilyData::~ColumnFamilyData() {
  if (mem_ != nullptr) {
    delete mem_->Unref();
  }
-  autovector<MemTable*> to_delete;
+  autovector<ReadOnlyMemTable*> to_delete;
  imm_.current()->Unref(&to_delete);
-  for (MemTable* m : to_delete) {
+  for (auto* m : to_delete) {
    delete m;
  }

@@ -1208,7 +1208,7 @@ Status ColumnFamilyData::RangesOverlapWithMemtables(
  MergeIteratorBuilder merge_iter_builder(&internal_comparator_, &arena);
  merge_iter_builder.AddIterator(super_version->mem->NewIterator(
      read_opts, /*seqno_to_time_mapping=*/nullptr, &arena,
-      /*prefix_extractor=*/nullptr));
+      /*prefix_extractor=*/nullptr, /*for_flush=*/false));
  super_version->imm->AddIterators(read_opts, /*seqno_to_time_mapping=*/nullptr,
                                   /*prefix_extractor=*/nullptr,
                                   &merge_iter_builder,
@@ -1561,6 +1561,11 @@ Status ColumnFamilyData::SetOptions(
      BuildColumnFamilyOptions(initial_cf_options_, mutable_cf_options_);
  ConfigOptions config_opts;
  config_opts.mutable_options_only = true;
+#ifndef NDEBUG
+  if (TEST_allowSetOptionsImmutableInMutable) {
+    config_opts.mutable_options_only = false;
+  }
+#endif
  Status s = GetColumnFamilyOptionsFromMap(config_opts, cf_opts, options_map,
                                           &cf_opts);
  if (s.ok()) {
@@ -207,7 +207,7 @@ struct SuperVersion {
  // Accessing members of this class is not thread-safe and requires external
  // synchronization (ie db mutex held or on write thread).
  ColumnFamilyData* cfd;
-  MemTable* mem;
+  ReadOnlyMemTable* mem;
  MemTableListVersion* imm;
  Version* current;
  MutableCFOptions mutable_cf_options;
@@ -269,7 +269,7 @@ struct SuperVersion {
  // We need to_delete because during Cleanup(), imm->Unref() returns
  // all memtables that we need to free through this vector. We then
  // delete all those memtables outside of mutex, during destruction
-  autovector<MemTable*> to_delete;
+  autovector<ReadOnlyMemTable*> to_delete;
 };

 Status CheckCompressionSupported(const ColumnFamilyOptions& cf_options);
@@ -388,10 +388,14 @@ class ColumnFamilyData {
  uint64_t GetTotalBlobFileSize() const;  // REQUIRE: DB mutex held
  // REQUIRE: DB mutex held
  void SetMemtable(MemTable* new_mem) {
-    new_mem->SetID(++last_memtable_id_);
+    AssignMemtableID(new_mem);
    mem_ = new_mem;
  }

+  void AssignMemtableID(ReadOnlyMemTable* new_imm) {
+    new_imm->SetID(++last_memtable_id_);
+  }
+
  // calculate the oldest log needed for the durability of this column family
  uint64_t OldestLogToKeep();

@@ -402,6 +406,7 @@ class ColumnFamilyData {
                         SequenceNumber earliest_seq);

  TableCache* table_cache() const { return table_cache_.get(); }
+  BlobFileCache* blob_file_cache() const { return blob_file_cache_.get(); }
  BlobSource* blob_source() const { return blob_source_.get(); }

  // See documentation in compaction_picker.h
@@ -345,8 +345,9 @@ Compaction::Compaction(
          _compaction_reason == CompactionReason::kExternalSstIngestion ||
                  _compaction_reason == CompactionReason::kRefitLevel
              ? Compaction::kInvalidLevel
-              : EvaluatePenultimateLevel(vstorage, immutable_options_,
-                                         start_level_, output_level_)) {
+              : EvaluatePenultimateLevel(vstorage, mutable_cf_options_,
+                                         immutable_options_, start_level_,
+                                         output_level_)) {
  MarkFilesBeingCompacted(true);
  if (is_manual_compaction_) {
    compaction_reason_ = CompactionReason::kManualCompaction;
@@ -367,9 +368,10 @@ Compaction::Compaction(
  }
 #endif

-  // setup input_levels_
+  // setup input_levels_ and filtered_input_levels_
  {
    input_levels_.resize(num_input_levels());
+    filtered_input_levels_.resize(num_input_levels());
    if (earliest_snapshot_.has_value()) {
      FilterInputsForCompactionIterator();
    } else {
@@ -935,6 +937,25 @@ bool Compaction::DoesInputReferenceBlobFiles() const {
  return false;
 }

+uint64_t Compaction::MaxInputFileNewestKeyTime(const InternalKey* start,
+                                               const InternalKey* end) const {
+  uint64_t newest_key_time = kUnknownNewestKeyTime;
+  const InternalKeyComparator& icmp =
+      column_family_data()->internal_comparator();
+  for (const auto& level_files : inputs_) {
+    for (const auto& file : level_files.files) {
+      if (start != nullptr && icmp.Compare(file->largest, *start) < 0) {
+        continue;
+      }
+      if (end != nullptr && icmp.Compare(file->smallest, *end) > 0) {
+        continue;
+      }
+      newest_key_time = std::max(newest_key_time, file->TryGetNewestKeyTime());
+    }
+  }
+  return newest_key_time;
+}
+
 uint64_t Compaction::MinInputFileOldestAncesterTime(
    const InternalKey* start, const InternalKey* end) const {
  uint64_t min_oldest_ancester_time = std::numeric_limits<uint64_t>::max();
@@ -970,6 +991,7 @@ uint64_t Compaction::MinInputFileEpochNumber() const {

 int Compaction::EvaluatePenultimateLevel(
    const VersionStorageInfo* vstorage,
+    const MutableCFOptions& mutable_cf_options,
    const ImmutableOptions& immutable_options, const int start_level,
    const int output_level) {
  // TODO: currently per_key_placement feature only support level and universal
@@ -1001,7 +1023,7 @@ int Compaction::EvaluatePenultimateLevel(
  }

  bool supports_per_key_placement =
-      immutable_options.preclude_last_level_data_seconds > 0;
+      mutable_cf_options.preclude_last_level_data_seconds > 0;

  // it could be overridden by unittest
  TEST_SYNC_POINT_CALLBACK("Compaction::SupportsPerKeyPlacement:Enabled",
@@ -1064,6 +1086,7 @@ void Compaction::FilterInputsForCompactionIterator() {
          ucmp->CompareWithoutTimestamp(rangedel_end_ukey,
                                        file->largest.user_key()) > 0) {
        non_start_level_input_files_filtered_.back().back() = true;
+        filtered_input_levels_[level].push_back(file);
      } else {
        non_start_level_input_files.back().push_back(file);
      }
@@ -184,6 +184,16 @@ class Compaction {
    return &input_levels_[compaction_input_level];
  }

+  // Returns the filtered input files of the specified compaction input level.
+  // For now, only non start level is filtered.
+  const std::vector<FileMetaData*>& filtered_input_levels(
+      size_t compaction_input_level) const {
+    const std::vector<FileMetaData*>& filtered_input_level =
+        filtered_input_levels_[compaction_input_level];
+    assert(compaction_input_level != 0 || filtered_input_level.size() == 0);
+    return filtered_input_level;
+  }
+
  // Maximum size of files to build during this compaction.
  uint64_t max_output_file_size() const { return max_output_file_size_; }

@@ -405,6 +415,12 @@ class Compaction {
    return blob_garbage_collection_age_cutoff_;
  }

+  // start and end are sub compact range. Null if no boundary.
+  // This is used to calculate the newest_key_time table property after
+  // compaction.
+  uint64_t MaxInputFileNewestKeyTime(const InternalKey* start,
+                                     const InternalKey* end) const;
+
  // start and end are sub compact range. Null if no boundary.
  // This is used to filter out some input files' ancester's time range.
  uint64_t MinInputFileOldestAncesterTime(const InternalKey* start,
@@ -434,10 +450,11 @@ class Compaction {
  // penultimate level. The safe key range is populated by
  // `PopulatePenultimateLevelOutputRange()`.
  // Which could potentially disable all penultimate level output.
-  static int EvaluatePenultimateLevel(const VersionStorageInfo* vstorage,
-                                      const ImmutableOptions& immutable_options,
-                                      const int start_level,
-                                      const int output_level);
+  static int EvaluatePenultimateLevel(
+      const VersionStorageInfo* vstorage,
+      const MutableCFOptions& mutable_cf_options,
+      const ImmutableOptions& immutable_options, const int start_level,
+      const int output_level);

  // mark (or clear) all files that are being compacted
  void MarkFilesBeingCompacted(bool being_compacted) const;
@@ -538,10 +555,13 @@ class Compaction {

  // Markers for which non start level input files are filtered out if
  // applicable. Only applicable if earliest_snapshot_ is provided and input
-  // start level has a standalone range deletion file.
+  // start level has a standalone range deletion file. Filtered files are
+  // tracked in `filtered_input_levels_`.
  std::vector<std::vector<bool>> non_start_level_input_files_filtered_;

-  //  bool standalone_range_tombstones_used_for_filtering_inputs_;
+  // All files from inputs_ that are filtered.
+  std::vector<std::vector<FileMetaData*>> filtered_input_levels_;
+
  const double score_;  // score that was used to pick this compaction.

  // Is this compaction creating a file in the bottom most level?
@@ -872,8 +872,8 @@ void CompactionIterator::NextFromInput() {
      if (Valid()) {
        at_next_ = true;
      }
-    } else if (last_snapshot == current_user_key_snapshot_ ||
-               (last_snapshot > 0 &&
+    } else if (last_sequence != kMaxSequenceNumber &&
+               (last_snapshot == current_user_key_snapshot_ ||
                last_snapshot < current_user_key_snapshot_)) {
      // If the earliest snapshot is which this key is visible in
      // is the same as the visibility of a previous instance of the
@@ -833,6 +833,14 @@ TEST_P(CompactionIteratorTest, ConvertToPutAtBottom) {
          true /*bottomost_level*/);
 }

+TEST_P(CompactionIteratorTest, ZeroSeqOfKeyAndSnapshot) {
+  AddSnapshot(0);
+  const std::vector<std::string> input_keys = {
+      test::KeyStr("a", 0, kTypeValue), test::KeyStr("b", 0, kTypeValue)};
+  const std::vector<std::string> input_values = {"a1", "b1"};
+  RunTest(input_keys, input_values, input_keys, input_values);
+}
+
 INSTANTIATE_TEST_CASE_P(CompactionIteratorTestInstance, CompactionIteratorTest,
                        testing::Values(true, false));

@@ -1846,6 +1854,22 @@ TEST_P(CompactionIteratorTsGcTest, SingleDeleteAllKeysOlderThanThreshold) {
  }
 }

+TEST_P(CompactionIteratorTsGcTest, ZeroSeqOfKeyAndSnapshot) {
+  AddSnapshot(0);
+  std::string full_history_ts_low;
+  PutFixed64(&full_history_ts_low, std::numeric_limits<uint64_t>::max());
+  const std::vector<std::string> input_keys = {
+      test::KeyStr(101, "a", 0, kTypeValue),
+      test::KeyStr(102, "b", 0, kTypeValue)};
+  const std::vector<std::string> input_values = {"a1", "b1"};
+  RunTest(input_keys, input_values, input_keys, input_values,
+          /*last_committed_seq=*/kMaxSequenceNumber,
+          /*merge_operator=*/nullptr, /*compaction_filter=*/nullptr,
+          /*bottommost_level=*/false,
+          /*earliest_write_conflict_snapshot=*/kMaxSequenceNumber,
+          /*key_not_exists_beyond_output_level=*/false, &full_history_ts_low);
+}
+
 INSTANTIATE_TEST_CASE_P(CompactionIteratorTsGcTestInstance,
                        CompactionIteratorTsGcTest,
                        testing::Values(true, false));
@@ -288,8 +288,8 @@ void CompactionJob::Prepare() {
  // to encode seqno->time to the output files.

  uint64_t preserve_time_duration =
-      std::max(c->immutable_options()->preserve_internal_time_seconds,
-               c->immutable_options()->preclude_last_level_data_seconds);
+      std::max(c->mutable_cf_options()->preserve_internal_time_seconds,
+               c->mutable_cf_options()->preclude_last_level_data_seconds);

  if (preserve_time_duration > 0) {
    const ReadOptions read_options(Env::IOActivity::kCompaction);
@@ -326,8 +326,8 @@ void CompactionJob::Prepare() {
      seqno_to_time_mapping_.Enforce(_current_time);
      seqno_to_time_mapping_.GetCurrentTieringCutoffSeqnos(
          static_cast<uint64_t>(_current_time),
-          c->immutable_options()->preserve_internal_time_seconds,
-          c->immutable_options()->preclude_last_level_data_seconds,
+          c->mutable_cf_options()->preserve_internal_time_seconds,
+          c->mutable_cf_options()->preclude_last_level_data_seconds,
          &preserve_time_min_seqno_, &preclude_last_level_min_seqno_);
    }
    // For accuracy of the GetProximalSeqnoBeforeTime queries above, we only
@@ -911,19 +911,23 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options,
  ROCKS_LOG_BUFFER(
      log_buffer_,
      "[%s] compacted to: %s, MB/sec: %.1f rd, %.1f wr, level %d, "
-      "files in(%d, %d) out(%d +%d blob) "
-      "MB in(%.1f, %.1f +%.1f blob) out(%.1f +%.1f blob), "
+      "files in(%d, %d) filtered(%d, %d) out(%d +%d blob) "
+      "MB in(%.1f, %.1f +%.1f blob) filtered(%.1f, %.1f) out(%.1f +%.1f blob), "
      "read-write-amplify(%.1f) write-amplify(%.1f) %s, records in: %" PRIu64
      ", records dropped: %" PRIu64 " output_compression: %s\n",
      column_family_name.c_str(), vstorage->LevelSummary(&tmp),
      bytes_read_per_sec, bytes_written_per_sec,
      compact_->compaction->output_level(),
      stats.num_input_files_in_non_output_levels,
-      stats.num_input_files_in_output_level, stats.num_output_files,
+      stats.num_input_files_in_output_level,
+      stats.num_filtered_input_files_in_non_output_levels,
+      stats.num_filtered_input_files_in_output_level, stats.num_output_files,
      stats.num_output_files_blob, stats.bytes_read_non_output_levels / kMB,
      stats.bytes_read_output_level / kMB, stats.bytes_read_blob / kMB,
-      stats.bytes_written / kMB, stats.bytes_written_blob / kMB, read_write_amp,
-      write_amp, status.ToString().c_str(), stats.num_input_records,
+      stats.bytes_skipped_non_output_levels / kMB,
+      stats.bytes_skipped_output_level / kMB, stats.bytes_written / kMB,
+      stats.bytes_written_blob / kMB, read_write_amp, write_amp,
+      status.ToString().c_str(), stats.num_input_records,
      stats.num_dropped_records,
      CompressionTypeToString(compact_->compaction->output_compression())
          .c_str());
@@ -1582,6 +1586,8 @@ Status CompactionJob::FinishCompactionOutputFile(
  const uint64_t current_entries = outputs.NumEntries();

  s = outputs.Finish(s, seqno_to_time_mapping_);
+  TEST_SYNC_POINT_CALLBACK(
+      "CompactionJob::FinishCompactionOutputFile()::AfterFinish", &s);

  if (s.ok()) {
    // With accurate smallest and largest key, we can get a slightly more
@@ -1914,6 +1920,10 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
    oldest_ancester_time = current_time;
  }

+  uint64_t newest_key_time = sub_compact->compaction->MaxInputFileNewestKeyTime(
+      sub_compact->start.has_value() ? &tmp_start : nullptr,
+      sub_compact->end.has_value() ? &tmp_end : nullptr);
+
  // Initialize a SubcompactionState::Output and add it to sub_compact->outputs
  uint64_t epoch_number = sub_compact->compaction->MinInputFileEpochNumber();
  {
@@ -1963,7 +1973,7 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
      cfd->internal_tbl_prop_coll_factories(),
      sub_compact->compaction->output_compression(),
      sub_compact->compaction->output_compression_opts(), cfd->GetID(),
-      cfd->GetName(), sub_compact->compaction->output_level(),
+      cfd->GetName(), sub_compact->compaction->output_level(), newest_key_time,
      bottommost_level_, TableFileCreationReason::kCompaction,
      0 /* oldest_key_time */, current_time, db_id_, db_session_id_,
      sub_compact->compaction->max_output_file_size(), file_number,
@@ -2003,7 +2013,6 @@ bool CompactionJob::UpdateCompactionStats(uint64_t* num_input_range_del) {
  bool has_error = false;
  const ReadOptions read_options(Env::IOActivity::kCompaction);
  const auto& input_table_properties = compaction->GetInputTableProperties();
-  // TODO(yuzhangyu): add dedicated stats for filtered files.
  for (int input_level = 0;
       input_level < static_cast<int>(compaction->num_input_levels());
       ++input_level) {
@@ -2043,6 +2052,23 @@ bool CompactionJob::UpdateCompactionStats(uint64_t* num_input_range_del) {
        *num_input_range_del += file_num_range_del;
      }
    }
+
+    const std::vector<FileMetaData*>& filtered_flevel =
+        compaction->filtered_input_levels(input_level);
+    size_t num_filtered_input_files = filtered_flevel.size();
+    uint64_t* bytes_skipped;
+    if (compaction->level(input_level) != compaction->output_level()) {
+      compaction_stats_.stats.num_filtered_input_files_in_non_output_levels +=
+          static_cast<int>(num_filtered_input_files);
+      bytes_skipped = &compaction_stats_.stats.bytes_skipped_non_output_levels;
+    } else {
+      compaction_stats_.stats.num_filtered_input_files_in_output_level +=
+          static_cast<int>(num_filtered_input_files);
+      bytes_skipped = &compaction_stats_.stats.bytes_skipped_output_level;
+    }
+    for (const FileMetaData* filtered_file_meta : filtered_flevel) {
+      *bytes_skipped += filtered_file_meta->fd.GetFileSize();
+    }
  }

  assert(compaction_job_stats_);
@@ -2067,6 +2093,13 @@ void CompactionJob::UpdateCompactionJobStats(
      stats.num_input_files_in_output_level;
  compaction_job_stats_->num_input_files_at_output_level =
      stats.num_input_files_in_output_level;
+  compaction_job_stats_->num_filtered_input_files =
+      stats.num_filtered_input_files_in_non_output_levels +
+      stats.num_filtered_input_files_in_output_level;
+  compaction_job_stats_->num_filtered_input_files_at_output_level =
+      stats.num_filtered_input_files_in_output_level;
+  compaction_job_stats_->total_skipped_input_bytes =
+      stats.bytes_skipped_non_output_levels + stats.bytes_skipped_output_level;

  // output information
  compaction_job_stats_->total_output_bytes = stats.bytes_written;
@@ -215,8 +215,7 @@ class CompactionJob {
  virtual void RecordCompactionIOStats();
  void CleanupCompaction();

-  // Call compaction filter. Then iterate through input and compact the
-  // kv-pairs
+  // Iterate through input and compact the kv-pairs.
  void ProcessKeyValueCompaction(SubcompactionState* sub_compact);

  CompactionState* compact_;
@@ -434,7 +433,7 @@ struct CompactionServiceOutputFile {
      uint64_t _epoch_number, const std::string& _file_checksum,
      const std::string& _file_checksum_func_name, uint64_t _paranoid_hash,
      bool _marked_for_compaction, UniqueId64x2 _unique_id,
-      const std::shared_ptr<const TableProperties>& _table_properties)
+      const TableProperties& _table_properties)
      : file_name(name),
        smallest_seqno(smallest),
        largest_seqno(largest),
@@ -448,7 +447,7 @@ struct CompactionServiceOutputFile {
        paranoid_hash(_paranoid_hash),
        marked_for_compaction(_marked_for_compaction),
        unique_id(std::move(_unique_id)),
-        table_properties(*_table_properties.get()) {}
+        table_properties(_table_properties) {}
 };

 // CompactionServiceResult contains the compaction result from a different db
@@ -301,13 +301,13 @@ class CompactionJobTestBase : public testing::Test {
    const WriteOptions write_options;
    std::unique_ptr<TableBuilder> table_builder(
        cf_options_.table_factory->NewTableBuilder(
-            TableBuilderOptions(*cfd_->ioptions(), mutable_cf_options_,
-                                read_options, write_options,
-                                cfd_->internal_comparator(),
-                                cfd_->internal_tbl_prop_coll_factories(),
-                                CompressionType::kNoCompression,
-                                CompressionOptions(), 0 /* column_family_id */,
-                                kDefaultColumnFamilyName, -1 /* level */),
+            TableBuilderOptions(
+                *cfd_->ioptions(), mutable_cf_options_, read_options,
+                write_options, cfd_->internal_comparator(),
+                cfd_->internal_tbl_prop_coll_factories(),
+                CompressionType::kNoCompression, CompressionOptions(),
+                0 /* column_family_id */, kDefaultColumnFamilyName,
+                -1 /* level */, kUnknownNewestKeyTime),
            file_writer.get()));
    // Build table.
    for (const auto& kv : contents) {
@@ -1672,9 +1672,6 @@ TEST_F(CompactionJobTest, ResultSerialization) {
    tp.readable_properties.emplace("RP_K2y2",
                                   rnd.RandomString(rnd.Uniform(kStrMaxLen)));

-    std::shared_ptr<const TableProperties> table_properties =
-        std::make_shared<const TableProperties>(tp);
-
    UniqueId64x2 id{rnd64.Uniform(UINT64_MAX), rnd64.Uniform(UINT64_MAX)};
    result.output_files.emplace_back(
        rnd.RandomString(rnd.Uniform(kStrMaxLen)) /* file_name */,
@@ -1690,8 +1687,7 @@ TEST_F(CompactionJobTest, ResultSerialization) {
        file_checksum /* file_checksum */,
        file_checksum_func_name /* file_checksum_func_name */,
        rnd64.Uniform(UINT64_MAX) /* paranoid_hash */,
-        rnd.OneIn(2) /* marked_for_compaction */, id /* unique_id */,
-        table_properties);
+        rnd.OneIn(2) /* marked_for_compaction */, id /* unique_id */, tp);
  }
  result.output_level = rnd.Uniform(10);
  result.output_path = rnd.RandomString(rnd.Uniform(kStrMaxLen));
@@ -351,11 +351,11 @@ Compaction* CompactionPicker::CompactFiles(
      break;
    }
  }
-  assert(output_level == 0 ||
-         !FilesRangeOverlapWithCompaction(
-             input_files, output_level,
-             Compaction::EvaluatePenultimateLevel(vstorage, ioptions_,
-                                                  start_level, output_level)));
+  assert(output_level == 0 || !FilesRangeOverlapWithCompaction(
+                                  input_files, output_level,
+                                  Compaction::EvaluatePenultimateLevel(
+                                      vstorage, mutable_cf_options, ioptions_,
+                                      start_level, output_level)));
 #endif /* !NDEBUG */

  CompressionType compression_type;
@@ -659,8 +659,9 @@ Compaction* CompactionPicker::CompactRange(
    // overlaping outputs in the same level.
    if (FilesRangeOverlapWithCompaction(
            inputs, output_level,
-            Compaction::EvaluatePenultimateLevel(vstorage, ioptions_,
-                                                 start_level, output_level))) {
+            Compaction::EvaluatePenultimateLevel(vstorage, mutable_cf_options,
+                                                 ioptions_, start_level,
+                                                 output_level))) {
      // This compaction output could potentially conflict with the output
      // of a currently running compaction, we cannot run it.
      *manual_conflict = true;
@@ -846,7 +847,8 @@ Compaction* CompactionPicker::CompactRange(
  // overlaping outputs in the same level.
  if (FilesRangeOverlapWithCompaction(
          compaction_inputs, output_level,
-          Compaction::EvaluatePenultimateLevel(vstorage, ioptions_, input_level,
+          Compaction::EvaluatePenultimateLevel(vstorage, mutable_cf_options,
+                                               ioptions_, input_level,
                                               output_level))) {
    // This compaction output could potentially conflict with the output
    // of a currently running compaction, we cannot run it.
@@ -1049,10 +1051,12 @@ Status CompactionPicker::SanitizeCompactionInputFilesForAllLevels(
 }

 Status CompactionPicker::SanitizeAndConvertCompactionInputFiles(
-    std::unordered_set<uint64_t>* input_files,
-    const ColumnFamilyMetaData& cf_meta, const int output_level,
-    const VersionStorageInfo* vstorage,
+    std::unordered_set<uint64_t>* input_files, const int output_level,
+    Version* version,
    std::vector<CompactionInputFiles>* converted_input_files) const {
+  ColumnFamilyMetaData cf_meta;
+  version->GetColumnFamilyMetaData(&cf_meta);
+
  assert(static_cast<int>(cf_meta.levels.size()) - 1 ==
         cf_meta.levels[cf_meta.levels.size() - 1].level);
  assert(converted_input_files);
@@ -1123,7 +1127,8 @@ Status CompactionPicker::SanitizeAndConvertCompactionInputFiles(
  }

  s = GetCompactionInputsFromFileNumbers(converted_input_files, input_files,
-                                         vstorage, CompactionOptions());
+                                         version->storage_info(),
+                                         CompactionOptions());
  if (!s.ok()) {
    return s;
  }
@@ -1132,8 +1137,8 @@ Status CompactionPicker::SanitizeAndConvertCompactionInputFiles(
      FilesRangeOverlapWithCompaction(
          *converted_input_files, output_level,
          Compaction::EvaluatePenultimateLevel(
-              vstorage, ioptions_, (*converted_input_files)[0].level,
-              output_level))) {
+              version->storage_info(), version->GetMutableCFOptions(),
+              ioptions_, (*converted_input_files)[0].level, output_level))) {
    return Status::Aborted(
        "A running compaction is writing to the same output level(s) in an "
        "overlapping key range");
@@ -97,9 +97,8 @@ class CompactionPicker {
  // non-ok status with specific reason.
  //
  Status SanitizeAndConvertCompactionInputFiles(
-      std::unordered_set<uint64_t>* input_files,
-      const ColumnFamilyMetaData& cf_meta, const int output_level,
-      const VersionStorageInfo* vstorage,
+      std::unordered_set<uint64_t>* input_files, const int output_level,
+      Version* version,
      std::vector<CompactionInputFiles>* converted_input_files) const;

  // Free up the files that participated in a compaction
@@ -79,10 +79,14 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction(
      FileMetaData* f = *ritr;
      assert(f);
      if (f->fd.table_reader && f->fd.table_reader->GetTableProperties()) {
+        uint64_t newest_key_time = f->TryGetNewestKeyTime();
        uint64_t creation_time =
            f->fd.table_reader->GetTableProperties()->creation_time;
-        if (creation_time == 0 ||
-            creation_time >= (current_time - mutable_cf_options.ttl)) {
+        uint64_t est_newest_key_time = newest_key_time == kUnknownNewestKeyTime
+                                           ? creation_time
+                                           : newest_key_time;
+        if (est_newest_key_time == kUnknownNewestKeyTime ||
+            est_newest_key_time >= (current_time - mutable_cf_options.ttl)) {
          break;
        }
      }
@@ -102,15 +106,19 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction(
  }

  for (const auto& f : inputs[0].files) {
-    uint64_t creation_time = 0;
    assert(f);
+    uint64_t newest_key_time = f->TryGetNewestKeyTime();
+    uint64_t creation_time = 0;
    if (f->fd.table_reader && f->fd.table_reader->GetTableProperties()) {
      creation_time = f->fd.table_reader->GetTableProperties()->creation_time;
    }
+    uint64_t est_newest_key_time = newest_key_time == kUnknownNewestKeyTime
+                                       ? creation_time
+                                       : newest_key_time;
    ROCKS_LOG_BUFFER(log_buffer,
                     "[%s] FIFO compaction: picking file %" PRIu64
-                     " with creation time %" PRIu64 " for deletion",
-                     cf_name.c_str(), f->fd.GetNumber(), creation_time);
+                     " with estimated newest key time %" PRIu64 " for deletion",
+                     cf_name.c_str(), f->fd.GetNumber(), est_newest_key_time);
  }

  Compaction* c = new Compaction(
@@ -350,38 +358,29 @@ Compaction* FIFOCompactionPicker::PickTemperatureChangeCompaction(
  Temperature compaction_target_temp = Temperature::kLastTemperature;
  if (current_time > min_age) {
    uint64_t create_time_threshold = current_time - min_age;
-    // We will ideally identify a file qualifying for temperature change by
-    // knowing the timestamp for the youngest entry in the file. However, right
-    // now we don't have the information. We infer it by looking at timestamp of
-    // the previous file's (which is just younger) oldest entry's timestamp.
-    // avoid index underflow
    assert(level_files.size() >= 1);
-    for (size_t index = level_files.size() - 1; index >= 1; --index) {
+    for (size_t index = level_files.size(); index >= 1; --index) {
      // Try to add cur_file to compaction inputs.
-      FileMetaData* cur_file = level_files[index];
-      // prev_file is just younger than cur_file
-      FileMetaData* prev_file = level_files[index - 1];
+      FileMetaData* cur_file = level_files[index - 1];
+      FileMetaData* prev_file = index < 2 ? nullptr : level_files[index - 2];
      if (cur_file->being_compacted) {
        // Should not happen since we check for
        // `level0_compactions_in_progress_` above. Here we simply just don't
        // schedule anything.
        return nullptr;
      }
-      uint64_t oldest_ancestor_time = prev_file->TryGetOldestAncesterTime();
-      if (oldest_ancestor_time == kUnknownOldestAncesterTime) {
-        // Older files might not have enough information. It is possible to
-        // handle these files by looking at newer files, but maintaining the
-        // logic isn't worth it.
-        break;
+      uint64_t est_newest_key_time = cur_file->TryGetNewestKeyTime(prev_file);
+      // Newer file could have newest_key_time populated
+      if (est_newest_key_time == kUnknownNewestKeyTime) {
+        continue;
      }
-      if (oldest_ancestor_time > create_time_threshold) {
-        // cur_file is too fresh
+      if (est_newest_key_time > create_time_threshold) {
        break;
      }
      Temperature cur_target_temp = ages[0].temperature;
      for (size_t i = 1; i < ages.size(); ++i) {
        if (current_time >= ages[i].age &&
-            oldest_ancestor_time <= current_time - ages[i].age) {
+            est_newest_key_time <= current_time - ages[i].age) {
          cur_target_temp = ages[i].temperature;
        }
      }
@@ -396,8 +395,8 @@ Compaction* FIFOCompactionPicker::PickTemperatureChangeCompaction(
      ROCKS_LOG_BUFFER(
          log_buffer,
          "[%s] FIFO compaction: picking file %" PRIu64
-          " with next file's oldest time %" PRIu64 " for temperature %s.",
-          cf_name.c_str(), cur_file->fd.GetNumber(), oldest_ancestor_time,
+          " with estimated newest key time %" PRIu64 " for temperature %s.",
+          cf_name.c_str(), cur_file->fd.GetNumber(), est_newest_key_time,
          temperature_to_string[cur_target_temp].c_str());
      break;
    }
@@ -414,8 +414,9 @@ void LevelCompactionBuilder::SetupOtherFilesWithRoundRobinExpansion() {
                                                    &tmp_start_level_inputs) ||
        compaction_picker_->FilesRangeOverlapWithCompaction(
            {tmp_start_level_inputs}, output_level_,
-            Compaction::EvaluatePenultimateLevel(
-                vstorage_, ioptions_, start_level_, output_level_))) {
+            Compaction::EvaluatePenultimateLevel(vstorage_, mutable_cf_options_,
+                                                 ioptions_, start_level_,
+                                                 output_level_))) {
      // Constraint 1a
      tmp_start_level_inputs.clear();
      return;
@@ -489,8 +490,9 @@ bool LevelCompactionBuilder::SetupOtherInputsIfNeeded() {
    // We need to disallow this from happening.
    if (compaction_picker_->FilesRangeOverlapWithCompaction(
            compaction_inputs_, output_level_,
-            Compaction::EvaluatePenultimateLevel(
-                vstorage_, ioptions_, start_level_, output_level_))) {
+            Compaction::EvaluatePenultimateLevel(vstorage_, mutable_cf_options_,
+                                                 ioptions_, start_level_,
+                                                 output_level_))) {
      // This compaction output could potentially conflict with the output
      // of a currently running compaction, we cannot run it.
      return false;
@@ -844,8 +846,9 @@ bool LevelCompactionBuilder::PickFileToCompact() {
                                                    &start_level_inputs_) ||
        compaction_picker_->FilesRangeOverlapWithCompaction(
            {start_level_inputs_}, output_level_,
-            Compaction::EvaluatePenultimateLevel(
-                vstorage_, ioptions_, start_level_, output_level_))) {
+            Compaction::EvaluatePenultimateLevel(vstorage_, mutable_cf_options_,
+                                                 ioptions_, start_level_,
+                                                 output_level_))) {
      // A locked (pending compaction) input-level file was pulled in due to
      // user-key overlap.
      start_level_inputs_.clear();
@@ -13,6 +13,7 @@
 #include "db/compaction/compaction_picker_universal.h"
 #include "db/compaction/file_pri.h"
 #include "rocksdb/advanced_options.h"
+#include "table/mock_table.h"
 #include "table/unique_id_impl.h"
 #include "test_util/testharness.h"
 #include "test_util/testutil.h"
@@ -77,7 +78,7 @@ class CompactionPickerTestBase : public testing::Test {
    ioptions_.level_compaction_dynamic_level_bytes = false;
  }

-  ~CompactionPickerTestBase() override = default;
+  ~CompactionPickerTestBase() override { ClearFiles(); }

  void NewVersionStorage(int num_levels, CompactionStyle style) {
    DeleteVersionStorage();
@@ -103,7 +104,7 @@ class CompactionPickerTestBase : public testing::Test {
  void DeleteVersionStorage() {
    vstorage_.reset();
    temp_vstorage_.reset();
-    files_.clear();
+    ClearFiles();
    file_map_.clear();
    input_files_.clear();
  }
@@ -115,6 +116,7 @@ class CompactionPickerTestBase : public testing::Test {
           size_t compensated_file_size = 0, bool marked_for_compact = false,
           Temperature temperature = Temperature::kUnknown,
           uint64_t oldest_ancestor_time = kUnknownOldestAncesterTime,
+           uint64_t newest_key_time = kUnknownNewestKeyTime,
           Slice ts_of_smallest = Slice(), Slice ts_of_largest = Slice(),
           uint64_t epoch_number = kUnknownEpochNumber) {
    assert(ts_of_smallest.size() == ucmp_->timestamp_size());
@@ -161,7 +163,12 @@ class CompactionPickerTestBase : public testing::Test {
        true /* user_defined_timestamps_persisted */);
    f->compensated_file_size =
        (compensated_file_size != 0) ? compensated_file_size : file_size;
+    // oldest_ancester_time is only used if newest_key_time is not available
    f->oldest_ancester_time = oldest_ancestor_time;
+    TableProperties tp;
+    tp.newest_key_time = newest_key_time;
+    f->fd.table_reader = new mock::MockTableReader(mock::KVVector{}, tp);
+
    vstorage->AddFile(level, f);
    files_.emplace_back(f);
    file_map_.insert({file_number, {f, level}});
@@ -206,6 +213,15 @@ class CompactionPickerTestBase : public testing::Test {
    return opts;
  }

+  void ClearFiles() {
+    for (auto& file : files_) {
+      if (file->fd.table_reader != nullptr) {
+        delete file->fd.table_reader;
+      }
+    }
+    files_.clear();
+  }
+
  std::unique_ptr<VersionStorageInfo> temp_vstorage_;
 };

@@ -1108,235 +1124,326 @@ TEST_F(CompactionPickerTest, NeedsCompactionFIFO) {
 }

 TEST_F(CompactionPickerTest, FIFOToCold1) {
-  NewVersionStorage(1, kCompactionStyleFIFO);
-  const uint64_t kFileSize = 100000;
-  const uint64_t kMaxSize = kFileSize * 100000;
-  uint64_t kColdThreshold = 2000;
+  // Test fallback behavior from newest_key_time to oldest_ancestor_time
+  for (bool newestKeyTimeKnown : {false, true}) {
+    NewVersionStorage(1, kCompactionStyleFIFO);
+    const uint64_t kFileSize = 100000;
+    const uint64_t kMaxSize = kFileSize * 100000;
+    uint64_t kColdThreshold = 2000;

-  fifo_options_.max_table_files_size = kMaxSize;
-  fifo_options_.file_temperature_age_thresholds = {
-      {Temperature::kCold, kColdThreshold}};
-  mutable_cf_options_.compaction_options_fifo = fifo_options_;
-  mutable_cf_options_.level0_file_num_compaction_trigger = 100;
-  mutable_cf_options_.max_compaction_bytes = kFileSize * 100;
-  FIFOCompactionPicker fifo_compaction_picker(ioptions_, &icmp_);
+    fifo_options_.max_table_files_size = kMaxSize;
+    fifo_options_.file_temperature_age_thresholds = {
+        {Temperature::kCold, kColdThreshold}};
+    mutable_cf_options_.compaction_options_fifo = fifo_options_;
+    mutable_cf_options_.level0_file_num_compaction_trigger = 100;
+    mutable_cf_options_.max_compaction_bytes = kFileSize * 100;
+    FIFOCompactionPicker fifo_compaction_picker(ioptions_, &icmp_);

-  int64_t current_time = 0;
-  ASSERT_OK(Env::Default()->GetCurrentTime(&current_time));
-  uint64_t threshold_time =
-      static_cast<uint64_t>(current_time) - kColdThreshold;
-  Add(0 /* level */, 4U /* file_number */, "260", "300", 1 * kFileSize, 0, 2500,
-      2600, 0, true, Temperature::kUnknown,
-      threshold_time - 2000 /* oldest_ancestor_time */);
-  // Qualifies for compaction to kCold.
-  Add(0, 3U, "200", "300", 4 * kFileSize, 0, 2300, 2400, 0, true,
-      Temperature::kUnknown, threshold_time - 3000);
-  UpdateVersionStorageInfo();
+    int64_t current_time = 0;
+    ASSERT_OK(Env::Default()->GetCurrentTime(&current_time));
+    uint64_t threshold_time =
+        static_cast<uint64_t>(current_time) - kColdThreshold;
+    Add(0 /* level */, 4U /* file_number */, "260", "300", 1 * kFileSize, 0,
+        2500, 2600, 0, true, Temperature::kUnknown,
+        newestKeyTimeKnown ? kUnknownOldestAncesterTime
+                           : threshold_time - 2000 /* oldest_ancestor_time */,
+        newestKeyTimeKnown ? threshold_time - 2000
+                           : kUnknownNewestKeyTime /* newest_key_time */);
+    // Qualifies for compaction to kCold.
+    Add(0, 3U, "200", "300", 4 * kFileSize, 0, 2300, 2400, 0, true,
+        Temperature::kUnknown,
+        newestKeyTimeKnown ? kUnknownOldestAncesterTime : threshold_time - 3000,
+        newestKeyTimeKnown ? threshold_time - 3000
+                           : kUnknownNewestKeyTime /* newest_key_time */);
+    UpdateVersionStorageInfo();

-  ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()), true);
-  std::unique_ptr<Compaction> compaction(fifo_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, mutable_db_options_,
-      /*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
-      vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(compaction->compaction_reason(),
-            CompactionReason::kChangeTemperature);
-  ASSERT_EQ(compaction->output_temperature(), Temperature::kCold);
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(3U, compaction->input(0, 0)->fd.GetNumber());
+    ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()), true);
+    std::unique_ptr<Compaction> compaction(
+        fifo_compaction_picker.PickCompaction(
+            cf_name_, mutable_cf_options_, mutable_db_options_,
+            /*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
+            vstorage_.get(), &log_buffer_));
+    ASSERT_TRUE(compaction.get() != nullptr);
+    ASSERT_EQ(compaction->compaction_reason(),
+              CompactionReason::kChangeTemperature);
+    ASSERT_EQ(compaction->output_temperature(), Temperature::kCold);
+    ASSERT_EQ(1U, compaction->num_input_files(0));
+    ASSERT_EQ(3U, compaction->input(0, 0)->fd.GetNumber());
+  }
 }

 TEST_F(CompactionPickerTest, FIFOToColdMaxCompactionSize) {
-  NewVersionStorage(1, kCompactionStyleFIFO);
-  const uint64_t kFileSize = 100000;
-  const uint64_t kMaxSize = kFileSize * 100000;
-  uint64_t kColdThreshold = 2000;
+  // Test fallback behavior from newest_key_time to oldest_ancestor_time
+  for (bool newestKeyTimeKnown : {false, true}) {
+    NewVersionStorage(1, kCompactionStyleFIFO);
+    const uint64_t kFileSize = 100000;
+    const uint64_t kMaxSize = kFileSize * 100000;
+    uint64_t kColdThreshold = 2000;

-  fifo_options_.max_table_files_size = kMaxSize;
-  fifo_options_.file_temperature_age_thresholds = {
-      {Temperature::kCold, kColdThreshold}};
-  mutable_cf_options_.compaction_options_fifo = fifo_options_;
-  mutable_cf_options_.level0_file_num_compaction_trigger = 100;
-  mutable_cf_options_.max_compaction_bytes = kFileSize * 9;
-  FIFOCompactionPicker fifo_compaction_picker(ioptions_, &icmp_);
+    fifo_options_.max_table_files_size = kMaxSize;
+    fifo_options_.file_temperature_age_thresholds = {
+        {Temperature::kCold, kColdThreshold}};
+    mutable_cf_options_.compaction_options_fifo = fifo_options_;
+    mutable_cf_options_.level0_file_num_compaction_trigger = 100;
+    mutable_cf_options_.max_compaction_bytes = kFileSize * 9;
+    FIFOCompactionPicker fifo_compaction_picker(ioptions_, &icmp_);

-  int64_t current_time = 0;
-  ASSERT_OK(Env::Default()->GetCurrentTime(&current_time));
-  uint64_t threshold_time =
-      static_cast<uint64_t>(current_time) - kColdThreshold;
-  Add(0, 6U, "240", "290", 2 * kFileSize, 0, 2900, 3000, 0, true,
-      Temperature::kUnknown, static_cast<uint64_t>(current_time) - 100);
-  Add(0, 5U, "240", "290", 2 * kFileSize, 0, 2700, 2800, 0, true,
-      Temperature::kUnknown, threshold_time + 100);
-  Add(0, 4U, "260", "300", 1 * kFileSize, 0, 2500, 2600, 0, true,
-      Temperature::kUnknown, threshold_time - 2000);
-  // The following two files qualify for compaction to kCold.
-  // But only the last two should be included to respect `max_compaction_bytes`.
-  Add(0, 3U, "200", "300", 4 * kFileSize, 0, 2300, 2400, 0, true,
-      Temperature::kUnknown, threshold_time - 3000);
-  Add(0, 2U, "200", "300", 4 * kFileSize, 0, 2100, 2200, 0, true,
-      Temperature::kUnknown, threshold_time - 4000);
-  Add(0, 1U, "200", "300", 4 * kFileSize, 0, 2000, 2100, 0, true,
-      Temperature::kUnknown, threshold_time - 5000);
-  UpdateVersionStorageInfo();
+    int64_t current_time = 0;
+    ASSERT_OK(Env::Default()->GetCurrentTime(&current_time));
+    uint64_t threshold_time =
+        static_cast<uint64_t>(current_time) - kColdThreshold;
+    Add(0, 6U, "240", "290", 2 * kFileSize, 0, 2900, 3000, 0, true,
+        Temperature::kUnknown,
+        newestKeyTimeKnown ? kUnknownOldestAncesterTime
+                           : static_cast<uint64_t>(current_time) - 100,
+        newestKeyTimeKnown ? static_cast<uint64_t>(current_time) - 100
+                           : kUnknownNewestKeyTime);
+    Add(0, 5U, "240", "290", 2 * kFileSize, 0, 2700, 2800, 0, true,
+        Temperature::kUnknown,
+        newestKeyTimeKnown ? kUnknownOldestAncesterTime : threshold_time + 100,
+        newestKeyTimeKnown ? threshold_time + 100 : kUnknownNewestKeyTime);
+    Add(0, 4U, "260", "300", 1 * kFileSize, 0, 2500, 2600, 0, true,
+        Temperature::kUnknown,
+        newestKeyTimeKnown ? kUnknownOldestAncesterTime : threshold_time - 2000,
+        newestKeyTimeKnown ? threshold_time - 2000 : kUnknownNewestKeyTime);
+    // The following two files qualify for compaction to kCold.
+    // But only the last two should be included to respect
+    // `max_compaction_bytes`.
+    Add(0, 3U, "200", "300", 4 * kFileSize, 0, 2300, 2400, 0, true,
+        Temperature::kUnknown,
+        newestKeyTimeKnown ? kUnknownOldestAncesterTime : threshold_time - 3000,
+        newestKeyTimeKnown ? threshold_time - 3000 : kUnknownNewestKeyTime);
+    Add(0, 2U, "200", "300", 4 * kFileSize, 0, 2100, 2200, 0, true,
+        Temperature::kUnknown,
+        newestKeyTimeKnown ? kUnknownOldestAncesterTime : threshold_time - 4000,
+        newestKeyTimeKnown ? threshold_time - 4000 : kUnknownNewestKeyTime);
+    Add(0, 1U, "200", "300", 4 * kFileSize, 0, 2000, 2100, 0, true,
+        Temperature::kUnknown,
+        newestKeyTimeKnown ? kUnknownOldestAncesterTime : threshold_time - 5000,
+        newestKeyTimeKnown ? threshold_time - 5000 : kUnknownNewestKeyTime);
+    UpdateVersionStorageInfo();

-  ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()), true);
-  std::unique_ptr<Compaction> compaction(fifo_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, mutable_db_options_,
-      /*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
-      vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(compaction->compaction_reason(),
-            CompactionReason::kChangeTemperature);
-  // Compaction picker picks older files first and picks one file at a time.
-  ASSERT_EQ(compaction->output_temperature(), Temperature::kCold);
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
+    ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()), true);
+    std::unique_ptr<Compaction> compaction(
+        fifo_compaction_picker.PickCompaction(
+            cf_name_, mutable_cf_options_, mutable_db_options_,
+            /*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
+            vstorage_.get(), &log_buffer_));
+    ASSERT_TRUE(compaction.get() != nullptr);
+    ASSERT_EQ(compaction->compaction_reason(),
+              CompactionReason::kChangeTemperature);
+    // Compaction picker picks older files first and picks one file at a time.
+    ASSERT_EQ(compaction->output_temperature(), Temperature::kCold);
+    ASSERT_EQ(1U, compaction->num_input_files(0));
+    ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
+  }
 }

 TEST_F(CompactionPickerTest, FIFOToColdWithExistingCold) {
-  NewVersionStorage(1, kCompactionStyleFIFO);
-  const uint64_t kFileSize = 100000;
-  const uint64_t kMaxSize = kFileSize * 100000;
-  uint64_t kColdThreshold = 2000;
+  // Test fallback behavior from newest_key_time to oldest_ancestor_time
+  for (bool newestKeyTimeKnown : {false, true}) {
+    NewVersionStorage(1, kCompactionStyleFIFO);
+    const uint64_t kFileSize = 100000;
+    const uint64_t kMaxSize = kFileSize * 100000;
+    uint64_t kColdThreshold = 2000;

-  fifo_options_.max_table_files_size = kMaxSize;
-  fifo_options_.file_temperature_age_thresholds = {
-      {Temperature::kCold, kColdThreshold}};
-  mutable_cf_options_.compaction_options_fifo = fifo_options_;
-  mutable_cf_options_.level0_file_num_compaction_trigger = 100;
-  mutable_cf_options_.max_compaction_bytes = kFileSize * 100;
-  FIFOCompactionPicker fifo_compaction_picker(ioptions_, &icmp_);
+    fifo_options_.max_table_files_size = kMaxSize;
+    fifo_options_.file_temperature_age_thresholds = {
+        {Temperature::kCold, kColdThreshold}};
+    mutable_cf_options_.compaction_options_fifo = fifo_options_;
+    mutable_cf_options_.level0_file_num_compaction_trigger = 100;
+    mutable_cf_options_.max_compaction_bytes = kFileSize * 100;
+    FIFOCompactionPicker fifo_compaction_picker(ioptions_, &icmp_);

-  int64_t current_time = 0;
-  ASSERT_OK(Env::Default()->GetCurrentTime(&current_time));
-  uint64_t threshold_time =
-      static_cast<uint64_t>(current_time) - kColdThreshold;
-  Add(0, 6U, "240", "290", 2 * kFileSize, 0, 2900, 3000, 0, true,
-      Temperature::kUnknown, static_cast<uint64_t>(current_time) - 100);
-  Add(0, 5U, "240", "290", 2 * kFileSize, 0, 2700, 2800, 0, true,
-      Temperature::kUnknown, threshold_time + 100);
-  Add(0, 4U, "260", "300", 1 * kFileSize, 0, 2500, 2600, 0, true,
-      Temperature::kUnknown, threshold_time - 2000);
-  // The following two files qualify for compaction to kCold.
-  Add(0, 3U, "200", "300", 4 * kFileSize, 0, 2300, 2400, 0, true,
-      Temperature::kUnknown, threshold_time - 3000);
-  Add(0, 2U, "200", "300", 4 * kFileSize, 0, 2100, 2200, 0, true,
-      Temperature::kUnknown, threshold_time - 4000);
-  Add(0, 1U, "200", "300", 4 * kFileSize, 0, 2000, 2100, 0, true,
-      Temperature::kCold, threshold_time - 5000);
-  UpdateVersionStorageInfo();
+    int64_t current_time = 0;
+    ASSERT_OK(Env::Default()->GetCurrentTime(&current_time));
+    uint64_t threshold_time =
+        static_cast<uint64_t>(current_time) - kColdThreshold;
+    Add(0, 6U, "240", "290", 2 * kFileSize, 0, 2900, 3000, 0, true,
+        Temperature::kUnknown,
+        newestKeyTimeKnown ? kUnknownOldestAncesterTime
+                           : static_cast<uint64_t>(current_time) - 100,
+        newestKeyTimeKnown ? static_cast<uint64_t>(current_time) - 100
+                           : kUnknownNewestKeyTime);
+    Add(0, 5U, "240", "290", 2 * kFileSize, 0, 2700, 2800, 0, true,
+        Temperature::kUnknown,
+        newestKeyTimeKnown ? kUnknownOldestAncesterTime : threshold_time + 100,
+        newestKeyTimeKnown ? threshold_time + 100 : kUnknownNewestKeyTime);
+    Add(0, 4U, "260", "300", 1 * kFileSize, 0, 2500, 2600, 0, true,
+        Temperature::kUnknown,
+        newestKeyTimeKnown ? kUnknownOldestAncesterTime : threshold_time - 2000,
+        newestKeyTimeKnown ? threshold_time - 2000 : kUnknownNewestKeyTime);
+    // The following two files qualify for compaction to kCold.
+    Add(0, 3U, "200", "300", 4 * kFileSize, 0, 2300, 2400, 0, true,
+        Temperature::kUnknown,
+        newestKeyTimeKnown ? kUnknownOldestAncesterTime : threshold_time - 3000,
+        newestKeyTimeKnown ? threshold_time - 3000 : kUnknownNewestKeyTime);
+    Add(0, 2U, "200", "300", 4 * kFileSize, 0, 2100, 2200, 0, true,
+        Temperature::kUnknown,
+        newestKeyTimeKnown ? kUnknownOldestAncesterTime : threshold_time - 4000,
+        newestKeyTimeKnown ? threshold_time - 4000 : kUnknownNewestKeyTime);
+    Add(0, 1U, "200", "300", 4 * kFileSize, 0, 2000, 2100, 0, true,
+        Temperature::kCold,
+        newestKeyTimeKnown ? kUnknownOldestAncesterTime : threshold_time - 5000,
+        newestKeyTimeKnown ? threshold_time - 5000 : kUnknownNewestKeyTime);
+    UpdateVersionStorageInfo();

-  ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()), true);
-  std::unique_ptr<Compaction> compaction(fifo_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, mutable_db_options_,
-      /*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
-      vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(compaction->compaction_reason(),
-            CompactionReason::kChangeTemperature);
-  // Compaction picker picks older files first and picks one file at a time.
-  ASSERT_EQ(compaction->output_temperature(), Temperature::kCold);
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(2U, compaction->input(0, 0)->fd.GetNumber());
+    ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()), true);
+    std::unique_ptr<Compaction> compaction(
+        fifo_compaction_picker.PickCompaction(
+            cf_name_, mutable_cf_options_, mutable_db_options_,
+            /*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
+            vstorage_.get(), &log_buffer_));
+    ASSERT_TRUE(compaction.get() != nullptr);
+    ASSERT_EQ(compaction->compaction_reason(),
+              CompactionReason::kChangeTemperature);
+    // Compaction picker picks older files first and picks one file at a time.
+    ASSERT_EQ(compaction->output_temperature(), Temperature::kCold);
+    ASSERT_EQ(1U, compaction->num_input_files(0));
+    ASSERT_EQ(2U, compaction->input(0, 0)->fd.GetNumber());
+  }
 }

 TEST_F(CompactionPickerTest, FIFOToColdWithHotBetweenCold) {
-  NewVersionStorage(1, kCompactionStyleFIFO);
-  const uint64_t kFileSize = 100000;
-  const uint64_t kMaxSize = kFileSize * 100000;
-  uint64_t kColdThreshold = 2000;
+  // Test fallback behavior from newest_key_time to oldest_ancestor_time
+  for (bool newestKeyTimeKnown : {false, true}) {
+    NewVersionStorage(1, kCompactionStyleFIFO);
+    const uint64_t kFileSize = 100000;
+    const uint64_t kMaxSize = kFileSize * 100000;
+    uint64_t kColdThreshold = 2000;

-  fifo_options_.max_table_files_size = kMaxSize;
-  fifo_options_.file_temperature_age_thresholds = {
-      {Temperature::kCold, kColdThreshold}};
-  mutable_cf_options_.compaction_options_fifo = fifo_options_;
-  mutable_cf_options_.level0_file_num_compaction_trigger = 100;
-  mutable_cf_options_.max_compaction_bytes = kFileSize * 100;
-  FIFOCompactionPicker fifo_compaction_picker(ioptions_, &icmp_);
+    fifo_options_.max_table_files_size = kMaxSize;
+    fifo_options_.file_temperature_age_thresholds = {
+        {Temperature::kCold, kColdThreshold}};
+    mutable_cf_options_.compaction_options_fifo = fifo_options_;
+    mutable_cf_options_.level0_file_num_compaction_trigger = 100;
+    mutable_cf_options_.max_compaction_bytes = kFileSize * 100;
+    FIFOCompactionPicker fifo_compaction_picker(ioptions_, &icmp_);

-  int64_t current_time = 0;
-  ASSERT_OK(Env::Default()->GetCurrentTime(&current_time));
-  uint64_t threshold_time =
-      static_cast<uint64_t>(current_time) - kColdThreshold;
-  Add(0, 6U, "240", "290", 2 * kFileSize, 0, 2900, 3000, 0, true,
-      Temperature::kUnknown, static_cast<uint64_t>(current_time) - 100);
-  Add(0, 5U, "240", "290", 2 * kFileSize, 0, 2700, 2800, 0, true,
-      Temperature::kUnknown, threshold_time + 100);
-  Add(0, 4U, "260", "300", 1 * kFileSize, 0, 2500, 2600, 0, true,
-      Temperature::kUnknown, threshold_time - 2000);
-  Add(0, 3U, "200", "300", 4 * kFileSize, 0, 2300, 2400, 0, true,
-      Temperature::kCold, threshold_time - 3000);
-  // Qualifies for compaction to kCold.
-  Add(0, 2U, "200", "300", 4 * kFileSize, 0, 2100, 2200, 0, true,
-      Temperature::kUnknown, threshold_time - 4000);
-  Add(0, 1U, "200", "300", 4 * kFileSize, 0, 2000, 2100, 0, true,
-      Temperature::kCold, threshold_time - 5000);
-  UpdateVersionStorageInfo();
+    int64_t current_time = 0;
+    ASSERT_OK(Env::Default()->GetCurrentTime(&current_time));
+    uint64_t threshold_time =
+        static_cast<uint64_t>(current_time) - kColdThreshold;
+    Add(0, 6U, "240", "290", 2 * kFileSize, 0, 2900, 3000, 0, true,
+        Temperature::kUnknown,
+        newestKeyTimeKnown ? kUnknownOldestAncesterTime
+                           : static_cast<uint64_t>(current_time) - 100,
+        newestKeyTimeKnown ? static_cast<uint64_t>(current_time) - 100
+                           : kUnknownNewestKeyTime);
+    Add(0, 5U, "240", "290", 2 * kFileSize, 0, 2700, 2800, 0, true,
+        Temperature::kUnknown,
+        newestKeyTimeKnown ? kUnknownOldestAncesterTime : threshold_time + 100,
+        newestKeyTimeKnown ? threshold_time + 100 : kUnknownNewestKeyTime);
+    Add(0, 4U, "260", "300", 1 * kFileSize, 0, 2500, 2600, 0, true,
+        Temperature::kUnknown,
+        newestKeyTimeKnown ? kUnknownOldestAncesterTime : threshold_time - 2000,
+        newestKeyTimeKnown ? threshold_time - 2000 : kUnknownNewestKeyTime);
+    Add(0, 3U, "200", "300", 4 * kFileSize, 0, 2300, 2400, 0, true,
+        Temperature::kCold,
+        newestKeyTimeKnown ? kUnknownOldestAncesterTime : threshold_time - 3000,
+        newestKeyTimeKnown ? threshold_time - 3000 : kUnknownNewestKeyTime);
+    // Qualifies for compaction to kCold.
+    Add(0, 2U, "200", "300", 4 * kFileSize, 0, 2100, 2200, 0, true,
+        Temperature::kUnknown,
+        newestKeyTimeKnown ? kUnknownOldestAncesterTime : threshold_time - 4000,
+        newestKeyTimeKnown ? threshold_time - 4000 : kUnknownNewestKeyTime);
+    Add(0, 1U, "200", "300", 4 * kFileSize, 0, 2000, 2100, 0, true,
+        Temperature::kCold,
+        newestKeyTimeKnown ? kUnknownOldestAncesterTime : threshold_time - 5000,
+        newestKeyTimeKnown ? threshold_time - 5000 : kUnknownNewestKeyTime);
+    UpdateVersionStorageInfo();

-  ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()), true);
-  std::unique_ptr<Compaction> compaction(fifo_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, mutable_db_options_,
-      /*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
-      vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(compaction->compaction_reason(),
-            CompactionReason::kChangeTemperature);
-  ASSERT_EQ(compaction->output_temperature(), Temperature::kCold);
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(2U, compaction->input(0, 0)->fd.GetNumber());
+    ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()), true);
+    std::unique_ptr<Compaction> compaction(
+        fifo_compaction_picker.PickCompaction(
+            cf_name_, mutable_cf_options_, mutable_db_options_,
+            /*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
+            vstorage_.get(), &log_buffer_));
+    ASSERT_TRUE(compaction.get() != nullptr);
+    ASSERT_EQ(compaction->compaction_reason(),
+              CompactionReason::kChangeTemperature);
+    ASSERT_EQ(compaction->output_temperature(), Temperature::kCold);
+    ASSERT_EQ(1U, compaction->num_input_files(0));
+    ASSERT_EQ(2U, compaction->input(0, 0)->fd.GetNumber());
+  }
 }

 TEST_F(CompactionPickerTest, FIFOToHotAndWarm) {
-  NewVersionStorage(1, kCompactionStyleFIFO);
-  const uint64_t kFileSize = 100000;
-  const uint64_t kMaxSize = kFileSize * 100000;
-  uint64_t kWarmThreshold = 10000;
-  uint64_t kHotThreshold = 2000;
+  // Test fallback behavior from newest_key_time to oldest_ancestor_time
+  for (bool newestKeyTimeKnown : {false, true}) {
+    NewVersionStorage(1, kCompactionStyleFIFO);
+    const uint64_t kFileSize = 100000;
+    const uint64_t kMaxSize = kFileSize * 100000;
+    uint64_t kWarmThreshold = 10000;
+    uint64_t kHotThreshold = 2000;

-  fifo_options_.max_table_files_size = kMaxSize;
-  // Test that multiple threshold works.
-  fifo_options_.file_temperature_age_thresholds = {
-      {Temperature::kHot, kHotThreshold}, {Temperature::kWarm, kWarmThreshold}};
-  mutable_cf_options_.compaction_options_fifo = fifo_options_;
-  mutable_cf_options_.level0_file_num_compaction_trigger = 100;
-  mutable_cf_options_.max_compaction_bytes = kFileSize * 100;
-  FIFOCompactionPicker fifo_compaction_picker(ioptions_, &icmp_);
+    fifo_options_.max_table_files_size = kMaxSize;
+    // Test that multiple threshold works.
+    fifo_options_.file_temperature_age_thresholds = {
+        {Temperature::kHot, kHotThreshold},
+        {Temperature::kWarm, kWarmThreshold}};
+    mutable_cf_options_.compaction_options_fifo = fifo_options_;
+    mutable_cf_options_.level0_file_num_compaction_trigger = 100;
+    mutable_cf_options_.max_compaction_bytes = kFileSize * 100;
+    FIFOCompactionPicker fifo_compaction_picker(ioptions_, &icmp_);

-  int64_t current_time = 0;
-  ASSERT_OK(Env::Default()->GetCurrentTime(&current_time));
-  uint64_t hot_threshold_time =
-      static_cast<uint64_t>(current_time) - kHotThreshold;
-  uint64_t warm_threshold_time =
-      static_cast<uint64_t>(current_time) - kWarmThreshold;
-  Add(0, 6U, "240", "290", 2 * kFileSize, 0, 2900, 3000, 0, true,
-      Temperature::kUnknown, static_cast<uint64_t>(current_time) - 100);
-  Add(0, 5U, "240", "290", 2 * kFileSize, 0, 2700, 2800, 0, true,
-      Temperature::kUnknown, hot_threshold_time + 100);
-  Add(0, 4U, "260", "300", 1 * kFileSize, 0, 2500, 2600, 0, true,
-      Temperature::kUnknown, hot_threshold_time - 200);
-  // Qualifies for Hot
-  Add(0, 3U, "200", "300", 4 * kFileSize, 0, 2300, 2400, 0, true,
-      Temperature::kUnknown, warm_threshold_time - 100);
-  // Qualifies for Warm
-  Add(0, 2U, "200", "300", 4 * kFileSize, 0, 2100, 2200, 0, true,
-      Temperature::kUnknown, warm_threshold_time - 4000);
-  Add(0, 1U, "200", "300", 4 * kFileSize, 0, 2000, 2100, 0, true,
-      Temperature::kUnknown, warm_threshold_time - 5000);
-  UpdateVersionStorageInfo();
+    int64_t current_time = 0;
+    ASSERT_OK(Env::Default()->GetCurrentTime(&current_time));
+    uint64_t hot_threshold_time =
+        static_cast<uint64_t>(current_time) - kHotThreshold;
+    uint64_t warm_threshold_time =
+        static_cast<uint64_t>(current_time) - kWarmThreshold;
+    Add(0, 6U, "240", "290", 2 * kFileSize, 0, 2900, 3000, 0, true,
+        Temperature::kUnknown,
+        newestKeyTimeKnown ? kUnknownOldestAncesterTime
+                           : static_cast<uint64_t>(current_time) - 100,
+        newestKeyTimeKnown ? static_cast<uint64_t>(current_time) - 100
+                           : kUnknownNewestKeyTime);
+    Add(0, 5U, "240", "290", 2 * kFileSize, 0, 2700, 2800, 0, true,
+        Temperature::kUnknown,
+        newestKeyTimeKnown ? kUnknownOldestAncesterTime
+                           : hot_threshold_time + 100,
+        newestKeyTimeKnown ? hot_threshold_time + 100 : kUnknownNewestKeyTime);
+    Add(0, 4U, "260", "300", 1 * kFileSize, 0, 2500, 2600, 0, true,
+        Temperature::kUnknown,
+        newestKeyTimeKnown ? kUnknownOldestAncesterTime
+                           : hot_threshold_time - 200,
+        newestKeyTimeKnown ? hot_threshold_time - 200 : kUnknownNewestKeyTime);
+    // Qualifies for Hot
+    Add(0, 3U, "200", "300", 4 * kFileSize, 0, 2300, 2400, 0, true,
+        Temperature::kUnknown,
+        newestKeyTimeKnown ? kUnknownOldestAncesterTime
+                           : warm_threshold_time - 100,
+        newestKeyTimeKnown ? warm_threshold_time - 100 : kUnknownNewestKeyTime);
+    // Qualifies for Warm
+    Add(0, 2U, "200", "300", 4 * kFileSize, 0, 2100, 2200, 0, true,
+        Temperature::kUnknown,
+        newestKeyTimeKnown ? kUnknownOldestAncesterTime
+                           : warm_threshold_time - 4000,
+        newestKeyTimeKnown ? warm_threshold_time - 4000
+                           : kUnknownNewestKeyTime);
+    Add(0, 1U, "200", "300", 4 * kFileSize, 0, 2000, 2100, 0, true,
+        Temperature::kUnknown,
+        newestKeyTimeKnown ? kUnknownOldestAncesterTime
+                           : warm_threshold_time - 5000,
+        newestKeyTimeKnown ? warm_threshold_time - 5000
+                           : kUnknownNewestKeyTime);
+    UpdateVersionStorageInfo();

-  ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()), true);
-  std::unique_ptr<Compaction> compaction(fifo_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, mutable_db_options_,
-      /*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
-      vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(compaction->compaction_reason(),
-            CompactionReason::kChangeTemperature);
-  // Compaction picker picks older files first and picks one file at a time.
-  ASSERT_EQ(compaction->output_temperature(), Temperature::kWarm);
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
+    ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()), true);
+    std::unique_ptr<Compaction> compaction(
+        fifo_compaction_picker.PickCompaction(
+            cf_name_, mutable_cf_options_, mutable_db_options_,
+            /*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
+            vstorage_.get(), &log_buffer_));
+    ASSERT_TRUE(compaction.get() != nullptr);
+    ASSERT_EQ(compaction->compaction_reason(),
+              CompactionReason::kChangeTemperature);
+    // Compaction picker picks older files first and picks one file at a time.
+    ASSERT_EQ(compaction->output_temperature(), Temperature::kWarm);
+    ASSERT_EQ(1U, compaction->num_input_files(0));
+    ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
+  }
 }

 TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) {
@@ -3097,18 +3204,21 @@ TEST_F(CompactionPickerTest, UniversalMarkedCompactionFullOverlap) {
  Add(0, 1U, "150", "200", kFileSize, 0, 500, 550, /*compensated_file_size*/ 0,
      /*marked_for_compact*/ false, /* temperature*/ Temperature::kUnknown,
      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+      /*newest_key_time*/ kUnknownNewestKeyTime,
      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
      /*epoch_number*/ 3);
  Add(0, 2U, "201", "250", 2 * kFileSize, 0, 401, 450,
      /*compensated_file_size*/ 0, /*marked_for_compact*/ false,
      /* temperature*/ Temperature::kUnknown,
      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+      /*newest_key_time*/ kUnknownNewestKeyTime,
      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
      /*epoch_number*/ 2);
  Add(0, 4U, "260", "300", 4 * kFileSize, 0, 260, 300,
      /*compensated_file_size*/ 0, /*marked_for_compact*/ false,
      /* temperature*/ Temperature::kUnknown,
      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+      /*newest_key_time*/ kUnknownNewestKeyTime,
      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
      /*epoch_number*/ 1);
  Add(3, 5U, "010", "080", 8 * kFileSize, 0, 200, 251);
@@ -3136,6 +3246,7 @@ TEST_F(CompactionPickerTest, UniversalMarkedCompactionFullOverlap) {
  Add(0, 7U, "150", "200", kFileSize, 0, 551, 600, 0, true,
      /* temperature*/ Temperature::kUnknown,
      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+      /*newest_key_time*/ kUnknownNewestKeyTime,
      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
      /*epoch_number*/ 4);
  UpdateVersionStorageInfo();
@@ -3163,6 +3274,7 @@ TEST_F(CompactionPickerTest, UniversalMarkedCompactionFullOverlap2) {
  Add(0, 4U, "260", "300", 4 * kFileSize, 0, 260, 300, 0, true,
      /* temperature*/ Temperature::kUnknown,
      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+      /*newest_key_time*/ kUnknownNewestKeyTime,
      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
      /*epoch_number*/ 1);
  Add(3, 5U, "240", "290", 8 * kFileSize, 0, 201, 250);
@@ -3189,12 +3301,14 @@ TEST_F(CompactionPickerTest, UniversalMarkedCompactionFullOverlap2) {
  Add(0, 1U, "150", "200", kFileSize, 0, 500, 550, /*compensated_file_size*/ 0,
      /*marked_for_compact*/ false, /* temperature*/ Temperature::kUnknown,
      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+      /*newest_key_time*/ kUnknownNewestKeyTime,
      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
      /*epoch_number*/ 3);
  Add(0, 2U, "201", "250", 2 * kFileSize, 0, 401, 450,
      /*compensated_file_size*/ 0, /*marked_for_compact*/ false,
      /* temperature*/ Temperature::kUnknown,
      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+      /*newest_key_time*/ kUnknownNewestKeyTime,
      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
      /*epoch_number*/ 2);
  UpdateVersionStorageInfo();
@@ -3371,23 +3485,27 @@ TEST_F(CompactionPickerTest, UniversalMarkedL0Overlap2) {
      /*compensated_file_size*/ 0, /*marked_for_compact*/ false,
      /* temperature*/ Temperature::kUnknown,
      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+      /*newest_key_time*/ kUnknownNewestKeyTime,
      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
      /*epoch_number*/ 4);
  Add(0, 5U, "240", "290", 2 * kFileSize, 0, 201, 250, 0, true,
      /* temperature*/ Temperature::kUnknown,
      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+      /*newest_key_time*/ kUnknownNewestKeyTime,
      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
      /*epoch_number*/ 3);
  Add(0, 3U, "301", "350", 4 * kFileSize, 0, 101, 150,
      /*compensated_file_size*/ 0, /*marked_for_compact*/ false,
      /* temperature*/ Temperature::kUnknown,
      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+      /*newest_key_time*/ kUnknownNewestKeyTime,
      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
      /*epoch_number*/ 2);
  Add(0, 6U, "501", "750", 8 * kFileSize, 0, 50, 100,
      /*compensated_file_size*/ 0, /*marked_for_compact*/ false,
      /* temperature*/ Temperature::kUnknown,
      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+      /*newest_key_time*/ kUnknownNewestKeyTime,
      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
      /*epoch_number*/ 1);
  UpdateVersionStorageInfo();
@@ -3414,12 +3532,14 @@ TEST_F(CompactionPickerTest, UniversalMarkedL0Overlap2) {
      /*marked_for_compact*/ false,
      /* temperature*/ Temperature::kUnknown,
      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+      /*newest_key_time*/ kUnknownNewestKeyTime,
      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
      /*epoch_number*/ 6);
  Add(0, 2U, "201", "250", kFileSize, 0, 401, 450, /*compensated_file_size*/ 0,
      /*marked_for_compact*/ false,
      /* temperature*/ Temperature::kUnknown,
      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+      /*newest_key_time*/ kUnknownNewestKeyTime,
      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
      /*epoch_number*/ 5);
  UpdateVersionStorageInfo();
@@ -3489,7 +3609,7 @@ TEST_F(CompactionPickerTest, UniversalSizeAmpTierCompactionNonLastLevel) {
  const int kLastLevel = kNumLevels - 1;

  ioptions_.compaction_style = kCompactionStyleUniversal;
-  ioptions_.preclude_last_level_data_seconds = 1000;
+  mutable_cf_options_.preclude_last_level_data_seconds = 1000;
  mutable_cf_options_.compaction_options_universal
      .max_size_amplification_percent = 200;
  // To avoid any L0 file exclusion in size amp compaction intended for reducing
@@ -3529,7 +3649,7 @@ TEST_F(CompactionPickerTest, UniversalSizeRatioTierCompactionLastLevel) {
  const int kPenultimateLevel = kLastLevel - 1;

  ioptions_.compaction_style = kCompactionStyleUniversal;
-  ioptions_.preclude_last_level_data_seconds = 1000;
+  mutable_cf_options_.preclude_last_level_data_seconds = 1000;
  mutable_cf_options_.compaction_options_universal
      .max_size_amplification_percent = 200;
  UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
@@ -3567,7 +3687,7 @@ TEST_F(CompactionPickerTest, UniversalSizeAmpTierCompactionNotSuport) {
  const int kLastLevel = kNumLevels - 1;

  ioptions_.compaction_style = kCompactionStyleUniversal;
-  ioptions_.preclude_last_level_data_seconds = 1000;
+  mutable_cf_options_.preclude_last_level_data_seconds = 1000;
  mutable_cf_options_.compaction_options_universal
      .max_size_amplification_percent = 200;
  // To avoid any L0 file exclusion in size amp compaction intended for reducing
@@ -3605,7 +3725,7 @@ TEST_F(CompactionPickerTest, UniversalSizeAmpTierCompactionLastLevel) {
  const int kPenultimateLevel = kLastLevel - 1;

  ioptions_.compaction_style = kCompactionStyleUniversal;
-  ioptions_.preclude_last_level_data_seconds = 1000;
+  mutable_cf_options_.preclude_last_level_data_seconds = 1000;
  mutable_cf_options_.compaction_options_universal
      .max_size_amplification_percent = 200;
  UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
@@ -3653,7 +3773,8 @@ TEST_F(CompactionPickerU64TsTest, Overlap) {
        /*file_size=*/1U, /*path_id=*/0,
        /*smallest_seq=*/100, /*largest_seq=*/100, /*compensated_file_size=*/0,
        /*marked_for_compact=*/false, /*temperature=*/Temperature::kUnknown,
-        /*oldest_ancestor_time=*/kUnknownOldestAncesterTime, ts1, ts2);
+        /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+        /*newest_key_time*/ kUnknownNewestKeyTime, ts1, ts2);
    UpdateVersionStorageInfo();
  }

@@ -3719,11 +3840,13 @@ TEST_F(CompactionPickerU64TsTest, CannotTrivialMoveUniversal) {
  Add(1, 1U, "150", "150", kFileSize, /*path_id=*/0, /*smallest_seq=*/100,
      /*largest_seq=*/100, /*compensated_file_size=*/kFileSize,
      /*marked_for_compact=*/false, Temperature::kUnknown,
-      kUnknownOldestAncesterTime, ts1, ts2);
+      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+      /*newest_key_time*/ kUnknownNewestKeyTime, ts1, ts2);
  Add(2, 2U, "150", "150", kFileSize, /*path_id=*/0, /*smallest_seq=*/100,
      /*largest_seq=*/100, /*compensated_file_size=*/kFileSize,
      /*marked_for_compact=*/false, Temperature::kUnknown,
-      kUnknownOldestAncesterTime, ts3, ts4);
+      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+      /*newest_key_time*/ kUnknownNewestKeyTime, ts3, ts4);
  UpdateVersionStorageInfo();

  std::unique_ptr<Compaction> compaction(
@@ -3786,8 +3909,8 @@ TEST_P(PerKeyPlacementCompactionPickerTest, OverlapWithNormalCompaction) {
  ASSERT_EQ(enable_per_key_placement_,
            level_compaction_picker.FilesRangeOverlapWithCompaction(
                input_files, 6,
-                Compaction::EvaluatePenultimateLevel(vstorage_.get(), ioptions_,
-                                                     0, 6)));
+                Compaction::EvaluatePenultimateLevel(
+                    vstorage_.get(), mutable_cf_options_, ioptions_, 0, 6)));
 }

 TEST_P(PerKeyPlacementCompactionPickerTest, NormalCompactionOverlap) {
@@ -3874,8 +3997,8 @@ TEST_P(PerKeyPlacementCompactionPickerTest,
  ASSERT_EQ(enable_per_key_placement_,
            universal_compaction_picker.FilesRangeOverlapWithCompaction(
                input_files, 6,
-                Compaction::EvaluatePenultimateLevel(vstorage_.get(), ioptions_,
-                                                     0, 6)));
+                Compaction::EvaluatePenultimateLevel(
+                    vstorage_.get(), mutable_cf_options_, ioptions_, 0, 6)));
 }

 TEST_P(PerKeyPlacementCompactionPickerTest, NormalCompactionOverlapUniversal) {
@@ -3926,7 +4049,7 @@ TEST_P(PerKeyPlacementCompactionPickerTest, PenultimateOverlapUniversal) {
  // This test is make sure the Tiered compaction would lock whole range of
  // both output level and penultimate level
  if (enable_per_key_placement_) {
-    ioptions_.preclude_last_level_data_seconds = 10000;
+    mutable_cf_options_.preclude_last_level_data_seconds = 10000;
  }

  int num_levels = ioptions_.num_levels;
@@ -3981,7 +4104,7 @@ TEST_P(PerKeyPlacementCompactionPickerTest, PenultimateOverlapUniversal) {

 TEST_P(PerKeyPlacementCompactionPickerTest, LastLevelOnlyOverlapUniversal) {
  if (enable_per_key_placement_) {
-    ioptions_.preclude_last_level_data_seconds = 10000;
+    mutable_cf_options_.preclude_last_level_data_seconds = 10000;
  }

  int num_levels = ioptions_.num_levels;
@@ -4040,7 +4163,7 @@ TEST_P(PerKeyPlacementCompactionPickerTest,
  // This should rarely happen in universal compaction, as the non-empty L5
  // should be included in the compaction.
  if (enable_per_key_placement_) {
-    ioptions_.preclude_last_level_data_seconds = 10000;
+    mutable_cf_options_.preclude_last_level_data_seconds = 10000;
  }

  int num_levels = ioptions_.num_levels;
@@ -4094,7 +4217,7 @@ TEST_P(PerKeyPlacementCompactionPickerTest,
  // penultimate level compaction if there's already an ongoing compaction to
  // the penultimate level
  if (enable_per_key_placement_) {
-    ioptions_.preclude_last_level_data_seconds = 10000;
+    mutable_cf_options_.preclude_last_level_data_seconds = 10000;
  }

  int num_levels = ioptions_.num_levels;
@@ -4135,8 +4258,8 @@ TEST_P(PerKeyPlacementCompactionPickerTest,
  ASSERT_EQ(enable_per_key_placement_,
            universal_compaction_picker.FilesRangeOverlapWithCompaction(
                input_files, 6,
-                Compaction::EvaluatePenultimateLevel(vstorage_.get(), ioptions_,
-                                                     6, 6)));
+                Compaction::EvaluatePenultimateLevel(
+                    vstorage_.get(), mutable_cf_options_, ioptions_, 6, 6)));

  if (!enable_per_key_placement_) {
    std::unique_ptr<Compaction> comp2(universal_compaction_picker.CompactFiles(
@@ -4154,7 +4277,7 @@ TEST_P(PerKeyPlacementCompactionPickerTest,
  // compaction, so it's safe to move data from the last level to the
  // penultimate level.
  if (enable_per_key_placement_) {
-    ioptions_.preclude_last_level_data_seconds = 10000;
+    mutable_cf_options_.preclude_last_level_data_seconds = 10000;
  }

  int num_levels = ioptions_.num_levels;
@@ -4195,7 +4318,8 @@ TEST_P(PerKeyPlacementCompactionPickerTest,
  // always safe to move data up
  ASSERT_FALSE(universal_compaction_picker.FilesRangeOverlapWithCompaction(
      input_files, 6,
-      Compaction::EvaluatePenultimateLevel(vstorage_.get(), ioptions_, 6, 6)));
+      Compaction::EvaluatePenultimateLevel(vstorage_.get(), mutable_cf_options_,
+                                           ioptions_, 6, 6)));

  // 2 compactions can be run in parallel
  std::unique_ptr<Compaction> comp2(universal_compaction_picker.CompactFiles(
@@ -142,7 +142,7 @@ class UniversalCompactionBuilder {

  bool ShouldSkipLastSortedRunForSizeAmpCompaction() const {
    assert(!sorted_runs_.empty());
-    return ioptions_.preclude_last_level_data_seconds > 0 &&
+    return mutable_cf_options_.preclude_last_level_data_seconds > 0 &&
           ioptions_.num_levels > 2 &&
           sorted_runs_.back().level == ioptions_.num_levels - 1 &&
           sorted_runs_.size() > 1;
@@ -994,11 +994,11 @@ Compaction* UniversalCompactionBuilder::PickCompactionToReduceSortedRuns(
    grandparents = vstorage_->LevelFiles(sorted_runs_[first_index_after].level);
  }

-  if (output_level != 0 &&
-      picker_->FilesRangeOverlapWithCompaction(
-          inputs, output_level,
-          Compaction::EvaluatePenultimateLevel(vstorage_, ioptions_,
-                                               start_level, output_level))) {
+  if (output_level != 0 && picker_->FilesRangeOverlapWithCompaction(
+                               inputs, output_level,
+                               Compaction::EvaluatePenultimateLevel(
+                                   vstorage_, mutable_cf_options_, ioptions_,
+                                   start_level, output_level))) {
    return nullptr;
  }
  CompactionReason compaction_reason;
@@ -1343,11 +1343,11 @@ Compaction* UniversalCompactionBuilder::PickIncrementalForReduceSizeAmp(
  }

  // intra L0 compactions outputs could have overlap
-  if (output_level != 0 &&
-      picker_->FilesRangeOverlapWithCompaction(
-          inputs, output_level,
-          Compaction::EvaluatePenultimateLevel(vstorage_, ioptions_,
-                                               start_level, output_level))) {
+  if (output_level != 0 && picker_->FilesRangeOverlapWithCompaction(
+                               inputs, output_level,
+                               Compaction::EvaluatePenultimateLevel(
+                                   vstorage_, mutable_cf_options_, ioptions_,
+                                   start_level, output_level))) {
    return nullptr;
  }

@@ -1487,7 +1487,8 @@ Compaction* UniversalCompactionBuilder::PickDeleteTriggeredCompaction() {
      if (picker_->FilesRangeOverlapWithCompaction(
              inputs, output_level,
              Compaction::EvaluatePenultimateLevel(
-                  vstorage_, ioptions_, start_level, output_level))) {
+                  vstorage_, mutable_cf_options_, ioptions_, start_level,
+                  output_level))) {
        return nullptr;
      }

@@ -1587,11 +1588,11 @@ Compaction* UniversalCompactionBuilder::PickCompactionWithSortedRunRange(
  }

  // intra L0 compactions outputs could have overlap
-  if (output_level != 0 &&
-      picker_->FilesRangeOverlapWithCompaction(
-          inputs, output_level,
-          Compaction::EvaluatePenultimateLevel(vstorage_, ioptions_,
-                                               start_level, output_level))) {
+  if (output_level != 0 && picker_->FilesRangeOverlapWithCompaction(
+                               inputs, output_level,
+                               Compaction::EvaluatePenultimateLevel(
+                                   vstorage_, mutable_cf_options_, ioptions_,
+                                   start_level, output_level))) {
    return nullptr;
  }

@@ -376,16 +376,18 @@ Status CompactionServiceCompactionJob::Run() {
  // Build Output
  compaction_result_->output_level = compact_->compaction->output_level();
  compaction_result_->output_path = output_path_;
-  for (const auto& output_file : sub_compact->GetOutputs()) {
-    auto& meta = output_file.meta;
-    compaction_result_->output_files.emplace_back(
-        MakeTableFileName(meta.fd.GetNumber()), meta.fd.smallest_seqno,
-        meta.fd.largest_seqno, meta.smallest.Encode().ToString(),
-        meta.largest.Encode().ToString(), meta.oldest_ancester_time,
-        meta.file_creation_time, meta.epoch_number, meta.file_checksum,
-        meta.file_checksum_func_name, output_file.validator.GetHash(),
-        meta.marked_for_compaction, meta.unique_id,
-        output_file.table_properties);
+  if (status.ok()) {
+    for (const auto& output_file : sub_compact->GetOutputs()) {
+      auto& meta = output_file.meta;
+      compaction_result_->output_files.emplace_back(
+          MakeTableFileName(meta.fd.GetNumber()), meta.fd.smallest_seqno,
+          meta.fd.largest_seqno, meta.smallest.Encode().ToString(),
+          meta.largest.Encode().ToString(), meta.oldest_ancester_time,
+          meta.file_creation_time, meta.epoch_number, meta.file_checksum,
+          meta.file_checksum_func_name, output_file.validator.GetHash(),
+          meta.marked_for_compaction, meta.unique_id,
+          *output_file.table_properties);
+    }
  }

  TEST_SYNC_POINT_CALLBACK("CompactionServiceCompactionJob::Run:0",
@@ -477,159 +479,6 @@ static std::unordered_map<std::string, OptionTypeInfo> cs_input_type_info = {
      OptionType::kUInt64T, OptionVerificationType::kNormal,
      OptionTypeFlags::kNone}},
 };
-static std::unordered_map<std::string, OptionTypeInfo>
-    table_properties_type_info = {
-        {"orig_file_number",
-         {offsetof(struct TableProperties, orig_file_number),
-          OptionType::kUInt64T, OptionVerificationType::kNormal,
-          OptionTypeFlags::kNone}},
-        {"data_size",
-         {offsetof(struct TableProperties, data_size), OptionType::kUInt64T,
-          OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
-        {"index_size",
-         {offsetof(struct TableProperties, index_size), OptionType::kUInt64T,
-          OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
-        {"index_partitions",
-         {offsetof(struct TableProperties, index_partitions),
-          OptionType::kUInt64T, OptionVerificationType::kNormal,
-          OptionTypeFlags::kNone}},
-        {"top_level_index_size",
-         {offsetof(struct TableProperties, top_level_index_size),
-          OptionType::kUInt64T, OptionVerificationType::kNormal,
-          OptionTypeFlags::kNone}},
-        {"index_key_is_user_key",
-         {offsetof(struct TableProperties, index_key_is_user_key),
-          OptionType::kUInt64T, OptionVerificationType::kNormal,
-          OptionTypeFlags::kNone}},
-        {"index_value_is_delta_encoded",
-         {offsetof(struct TableProperties, index_value_is_delta_encoded),
-          OptionType::kUInt64T, OptionVerificationType::kNormal,
-          OptionTypeFlags::kNone}},
-        {"filter_size",
-         {offsetof(struct TableProperties, filter_size), OptionType::kUInt64T,
-          OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
-        {"raw_key_size",
-         {offsetof(struct TableProperties, raw_key_size), OptionType::kUInt64T,
-          OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
-        {"raw_value_size",
-         {offsetof(struct TableProperties, raw_value_size),
-          OptionType::kUInt64T, OptionVerificationType::kNormal,
-          OptionTypeFlags::kNone}},
-        {"num_data_blocks",
-         {offsetof(struct TableProperties, num_data_blocks),
-          OptionType::kUInt64T, OptionVerificationType::kNormal,
-          OptionTypeFlags::kNone}},
-        {"num_entries",
-         {offsetof(struct TableProperties, num_entries), OptionType::kUInt64T,
-          OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
-        {"num_filter_entries",
-         {offsetof(struct TableProperties, num_filter_entries),
-          OptionType::kUInt64T, OptionVerificationType::kNormal,
-          OptionTypeFlags::kNone}},
-        {"num_deletions",
-         {offsetof(struct TableProperties, num_deletions), OptionType::kUInt64T,
-          OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
-        {"num_merge_operands",
-         {offsetof(struct TableProperties, num_merge_operands),
-          OptionType::kUInt64T, OptionVerificationType::kNormal,
-          OptionTypeFlags::kNone}},
-        {"num_range_deletions",
-         {offsetof(struct TableProperties, num_range_deletions),
-          OptionType::kUInt64T, OptionVerificationType::kNormal,
-          OptionTypeFlags::kNone}},
-        {"format_version",
-         {offsetof(struct TableProperties, format_version),
-          OptionType::kUInt64T, OptionVerificationType::kNormal,
-          OptionTypeFlags::kNone}},
-        {"fixed_key_len",
-         {offsetof(struct TableProperties, fixed_key_len), OptionType::kUInt64T,
-          OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
-        {"column_family_id",
-         {offsetof(struct TableProperties, column_family_id),
-          OptionType::kUInt64T, OptionVerificationType::kNormal,
-          OptionTypeFlags::kNone}},
-        {"creation_time",
-         {offsetof(struct TableProperties, creation_time), OptionType::kUInt64T,
-          OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
-        {"oldest_key_time",
-         {offsetof(struct TableProperties, oldest_key_time),
-          OptionType::kUInt64T, OptionVerificationType::kNormal,
-          OptionTypeFlags::kNone}},
-        {"file_creation_time",
-         {offsetof(struct TableProperties, file_creation_time),
-          OptionType::kUInt64T, OptionVerificationType::kNormal,
-          OptionTypeFlags::kNone}},
-        {"slow_compression_estimated_data_size",
-         {offsetof(struct TableProperties,
-                   slow_compression_estimated_data_size),
-          OptionType::kUInt64T, OptionVerificationType::kNormal,
-          OptionTypeFlags::kNone}},
-        {"fast_compression_estimated_data_size",
-         {offsetof(struct TableProperties,
-                   fast_compression_estimated_data_size),
-          OptionType::kUInt64T, OptionVerificationType::kNormal,
-          OptionTypeFlags::kNone}},
-        {"external_sst_file_global_seqno_offset",
-         {offsetof(struct TableProperties,
-                   external_sst_file_global_seqno_offset),
-          OptionType::kUInt64T, OptionVerificationType::kNormal,
-          OptionTypeFlags::kNone}},
-        {"tail_start_offset",
-         {offsetof(struct TableProperties, tail_start_offset),
-          OptionType::kUInt64T, OptionVerificationType::kNormal,
-          OptionTypeFlags::kNone}},
-        {"user_defined_timestamps_persisted",
-         {offsetof(struct TableProperties, user_defined_timestamps_persisted),
-          OptionType::kUInt64T, OptionVerificationType::kNormal,
-          OptionTypeFlags::kNone}},
-        {"key_largest_seqno",
-         {offsetof(struct TableProperties, key_largest_seqno),
-          OptionType::kUInt64T, OptionVerificationType::kNormal,
-          OptionTypeFlags::kNone}},
-        {"db_id",
-         {offsetof(struct TableProperties, db_id), OptionType::kEncodedString}},
-        {"db_session_id",
-         {offsetof(struct TableProperties, db_session_id),
-          OptionType::kEncodedString}},
-        {"db_host_id",
-         {offsetof(struct TableProperties, db_host_id),
-          OptionType::kEncodedString}},
-        {"column_family_name",
-         {offsetof(struct TableProperties, column_family_name),
-          OptionType::kEncodedString}},
-        {"filter_policy_name",
-         {offsetof(struct TableProperties, filter_policy_name),
-          OptionType::kEncodedString}},
-        {"comparator_name",
-         {offsetof(struct TableProperties, comparator_name),
-          OptionType::kEncodedString}},
-        {"merge_operator_name",
-         {offsetof(struct TableProperties, merge_operator_name),
-          OptionType::kEncodedString}},
-        {"prefix_extractor_name",
-         {offsetof(struct TableProperties, prefix_extractor_name),
-          OptionType::kEncodedString}},
-        {"property_collectors_names",
-         {offsetof(struct TableProperties, property_collectors_names),
-          OptionType::kEncodedString}},
-        {"compression_name",
-         {offsetof(struct TableProperties, compression_name),
-          OptionType::kEncodedString}},
-        {"compression_options",
-         {offsetof(struct TableProperties, compression_options),
-          OptionType::kEncodedString}},
-        {"seqno_to_time_mapping",
-         {offsetof(struct TableProperties, seqno_to_time_mapping),
-          OptionType::kEncodedString}},
-        {"user_collected_properties",
-         OptionTypeInfo::StringMap(
-             offsetof(struct TableProperties, user_collected_properties),
-             OptionVerificationType::kNormal, OptionTypeFlags::kNone)},
-        {"readable_properties",
-         OptionTypeInfo::StringMap(
-             offsetof(struct TableProperties, readable_properties),
-             OptionVerificationType::kNormal, OptionTypeFlags::kNone)},
-};

 static std::unordered_map<std::string, OptionTypeInfo>
    cs_output_file_type_info = {
@@ -687,11 +536,29 @@ static std::unordered_map<std::string, OptionTypeInfo>
             OptionVerificationType::kNormal, OptionTypeFlags::kNone,
             {0, OptionType::kUInt64T})},
        {"table_properties",
-         OptionTypeInfo::Struct(
-             "table_properties", &table_properties_type_info,
-             offsetof(struct CompactionServiceOutputFile, table_properties),
-             OptionVerificationType::kNormal, OptionTypeFlags::kNone)},
-};
+         {offsetof(struct CompactionServiceOutputFile, table_properties),
+          OptionType::kStruct, OptionVerificationType::kNormal,
+          OptionTypeFlags::kNone,
+          [](const ConfigOptions& opts, const std::string& /*name*/,
+             const std::string& value, void* addr) {
+            auto table_properties = static_cast<TableProperties*>(addr);
+            return TableProperties::Parse(opts, value, table_properties);
+          },
+          [](const ConfigOptions& opts, const std::string& /*name*/,
+             const void* addr, std::string* value) {
+            const auto table_properties =
+                static_cast<const TableProperties*>(addr);
+            std::string result;
+            auto status = table_properties->Serialize(opts, &result);
+            *value = "{" + result + "}";
+            return status;
+          },
+          [](const ConfigOptions& opts, const std::string& /*name*/,
+             const void* addr1, const void* addr2, std::string* mismatch) {
+            const auto this_one = static_cast<const TableProperties*>(addr1);
+            const auto that_one = static_cast<const TableProperties*>(addr2);
+            return this_one->AreEqual(opts, that_one, mismatch);
+          }}}};

 static std::unordered_map<std::string, OptionTypeInfo>
    compaction_job_stats_type_info = {
@@ -416,6 +416,38 @@ TEST_F(CompactionServiceTest, ManualCompaction) {
  ASSERT_TRUE(result.stats.is_remote_compaction);
 }

+TEST_F(CompactionServiceTest, CompactionOutputFileIOError) {
+  Options options = CurrentOptions();
+  options.disable_auto_compactions = true;
+  ReopenWithCompactionService(&options);
+  GenerateTestData();
+
+  auto my_cs = GetCompactionService();
+
+  SyncPoint::GetInstance()->SetCallBack(
+      "CompactionJob::FinishCompactionOutputFile()::AfterFinish",
+      [&](void* status) {
+        // override status
+        auto s = static_cast<Status*>(status);
+        *s = Status::IOError("Injected IOError!");
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  std::string start_str = Key(15);
+  std::string end_str = Key(45);
+  Slice start(start_str);
+  Slice end(end_str);
+  uint64_t comp_num = my_cs->GetCompactionNum();
+  ASSERT_NOK(db_->CompactRange(CompactRangeOptions(), &start, &end));
+  ASSERT_GE(my_cs->GetCompactionNum(), comp_num + 1);
+
+  CompactionServiceResult result;
+  my_cs->GetResult(&result);
+  ASSERT_NOK(result.status);
+  ASSERT_TRUE(result.stats.is_manual_compaction);
+  ASSERT_TRUE(result.stats.is_remote_compaction);
+}
+
 TEST_F(CompactionServiceTest, PreservedOptionsLocalCompaction) {
  Options options = CurrentOptions();
  options.level0_file_num_compaction_trigger = 2;
@@ -483,21 +515,29 @@ TEST_F(CompactionServiceTest, PreservedOptionsRemoteCompaction) {
    ASSERT_OK(Flush());
  }

-  bool is_primary_called = false;
-  // This will be called twice. One from primary and one from remote.
-  // Try changing the option when called from remote. Otherwise, the new option
-  // will be used
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
+      {{"CompactionServiceTest::OptionsFileChanged",
+        "DBImplSecondary::OpenAndCompact::BeforeLoadingOptions:1"}});
+
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::BackgroundCompaction:NonTrivial:BeforeRun", [&](void* /*arg*/) {
-        if (!is_primary_called) {
-          is_primary_called = true;
-          return;
-        }
-        // Change the option right before the compaction run
+      "DBImplSecondary::OpenAndCompact::BeforeLoadingOptions:0",
+      [&](void* arg) {
+        auto options_file_number = static_cast<uint64_t*>(arg);
+        // Change the option twice before the compaction run
        ASSERT_OK(dbfull()->SetOptions(
            {{"level0_file_num_compaction_trigger", "4"}}));
        ASSERT_EQ(4, dbfull()->GetOptions().level0_file_num_compaction_trigger);
-        dbfull()->TEST_DeleteObsoleteFiles();
+        ASSERT_TRUE(dbfull()->versions_->options_file_number() >
+                    *options_file_number);
+
+        // Change the option twice before the compaction run
+        ASSERT_OK(dbfull()->SetOptions(
+            {{"level0_file_num_compaction_trigger", "5"}}));
+        ASSERT_EQ(5, dbfull()->GetOptions().level0_file_num_compaction_trigger);
+        ASSERT_TRUE(dbfull()->versions_->options_file_number() >
+                    *options_file_number);
+
+        TEST_SYNC_POINT("CompactionServiceTest::OptionsFileChanged");
      });

  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
@@ -34,9 +34,16 @@ void SubcompactionState::Cleanup(Cache* cache) {

  if (!status.ok()) {
    for (const auto& out : GetOutputs()) {
-      // If this file was inserted into the table cache then remove
-      // them here because this compaction was not committed.
-      TableCache::Evict(cache, out.meta.fd.GetNumber());
+      // If this file was inserted into the table cache then remove it here
+      // because this compaction was not committed. This is not strictly
+      // required because of a backstop TableCache::Evict() in
+      // PurgeObsoleteFiles() but is our opportunity to apply
+      // uncache_aggressiveness. TODO: instead, put these files into the
+      // VersionSet::obsolete_files_ pipeline so that they don't have to
+      // be picked up by scanning the DB directory.
+      TableCache::ReleaseObsolete(
+          cache, out.meta.fd.GetNumber(), nullptr /*handle*/,
+          compaction->mutable_cf_options()->uncache_aggressiveness);
    }
  }
  // TODO: sub_compact.io_status is not checked like status. Not sure if thats
@@ -9,15 +9,26 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.

 #include "db/db_test_util.h"
+#include "options/cf_options.h"
 #include "port/stack_trace.h"
 #include "rocksdb/iostats_context.h"
 #include "rocksdb/listener.h"
 #include "rocksdb/utilities/debug.h"
 #include "rocksdb/utilities/table_properties_collectors.h"
 #include "test_util/mock_time_env.h"
+#include "util/defer.h"
 #include "utilities/merge_operators.h"

 namespace ROCKSDB_NAMESPACE {
+namespace {
+ConfigOptions GetStrictConfigOptions() {
+  ConfigOptions config_options;
+  config_options.ignore_unknown_options = false;
+  config_options.ignore_unsupported_options = false;
+  config_options.input_strings_escaped = false;
+  return config_options;
+}
+}  // namespace

 class TieredCompactionTest : public DBTestBase {
 public:
@@ -1234,82 +1245,9 @@ TEST_F(TieredCompactionTest, RangeBasedTieredStorageLevel) {
  db_->ReleaseSnapshot(temp_snap);
 }

-TEST_F(TieredCompactionTest, CheckInternalKeyRange) {
-  // When compacting keys from the last level to penultimate level,
-  // output to penultimate level should be within internal key range
-  // of input files from penultimate level.
-  // Set up:
-  // L5:
-  //  File 1: DeleteRange[1, 3)@4, File 2: [3@5, 100@6]
-  // L6:
-  // File 3: [2@1, 3@2], File 4: [50@3]
-  //
-  // When File 1 and File 3 are being compacted,
-  // Key(3) cannot be compacted up, otherwise it causes
-  // inconsistency where File 3's Key(3) has a lower sequence number
-  // than File 2's Key(3).
-  const int kNumLevels = 7;
-  auto options = CurrentOptions();
-  SetColdTemperature(options);
-  options.level_compaction_dynamic_level_bytes = true;
-  options.num_levels = kNumLevels;
-  options.statistics = CreateDBStatistics();
-  options.max_subcompactions = 10;
-  options.preclude_last_level_data_seconds = 10000;
-  DestroyAndReopen(options);
-  auto cmp = options.comparator;
-
-  std::string hot_start = Key(0);
-  std::string hot_end = Key(0);
-  SyncPoint::GetInstance()->SetCallBack(
-      "CompactionIterator::PrepareOutput.context", [&](void* arg) {
-        auto context = static_cast<PerKeyPlacementContext*>(arg);
-        context->output_to_penultimate_level =
-            cmp->Compare(context->key, hot_start) >= 0 &&
-            cmp->Compare(context->key, hot_end) < 0;
-      });
-  SyncPoint::GetInstance()->EnableProcessing();
-  // File 1
-  ASSERT_OK(Put(Key(2), "val2"));
-  ASSERT_OK(Put(Key(3), "val3"));
-  ASSERT_OK(Flush());
-  MoveFilesToLevel(6);
-  // File 2
-  ASSERT_OK(Put(Key(50), "val50"));
-  ASSERT_OK(Flush());
-  MoveFilesToLevel(6);
-
-  const Snapshot* snapshot = db_->GetSnapshot();
-  hot_end = Key(100);
-  std::string start = Key(1);
-  std::string end = Key(3);
-  ASSERT_OK(
-      db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), start, end));
-  ASSERT_OK(Flush());
-  MoveFilesToLevel(5);
-  // File 3
-  ASSERT_OK(Put(Key(3), "vall"));
-  ASSERT_OK(Put(Key(100), "val100"));
-  ASSERT_OK(Flush());
-  MoveFilesToLevel(5);
-  // Try to compact keys up
-  CompactRangeOptions cro;
-  cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
-  start = Key(1);
-  end = Key(2);
-  Slice begin_slice(start);
-  Slice end_slice(end);
-  ASSERT_OK(db_->CompactRange(cro, &begin_slice, &end_slice));
-  // Without internal key range checking, we get the following error:
-  // Corruption: force_consistency_checks(DEBUG): VersionBuilder: L5 has
-  // overlapping ranges: file #18 largest key: '6B6579303030303033' seq:102,
-  // type:1 vs. file #15 smallest key: '6B6579303030303033' seq:104, type:1
-  db_->ReleaseSnapshot(snapshot);
-}
-
-class PrecludeLastLevelTest : public DBTestBase {
+class PrecludeLastLevelTestBase : public DBTestBase {
 public:
-  PrecludeLastLevelTest(std::string test_name = "preclude_last_level_test")
+  PrecludeLastLevelTestBase(std::string test_name = "preclude_last_level_test")
      : DBTestBase(test_name, /*env_do_fsync=*/false) {
    mock_clock_ = std::make_shared<MockSystemClock>(env_->GetSystemClock());
    mock_clock_->SetCurrentTime(kMockStartTime);
@@ -1334,9 +1272,52 @@ class PrecludeLastLevelTest : public DBTestBase {
        });
    mock_clock_->SetCurrentTime(kMockStartTime);
  }
+
+  void ApplyConfigChangeImpl(
+      bool dynamic, Options* options,
+      const std::unordered_map<std::string, std::string>& config_change,
+      const std::unordered_map<std::string, std::string>& db_config_change) {
+    if (dynamic) {
+      if (config_change.size() > 0) {
+        // FIXME: temporary while preserve/preclude options are not user mutable
+        SaveAndRestore<bool> m(&TEST_allowSetOptionsImmutableInMutable, true);
+        ASSERT_OK(db_->SetOptions(config_change));
+      }
+      if (db_config_change.size() > 0) {
+        ASSERT_OK(db_->SetDBOptions(db_config_change));
+      }
+    } else {
+      if (config_change.size() > 0) {
+        ASSERT_OK(GetColumnFamilyOptionsFromMap(
+            GetStrictConfigOptions(), *options, config_change, options));
+      }
+      if (db_config_change.size() > 0) {
+        ASSERT_OK(GetDBOptionsFromMap(GetStrictConfigOptions(), *options,
+                                      db_config_change, options));
+      }
+      Reopen(*options);
+    }
+  }
 };

-TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeManualCompaction) {
+class PrecludeLastLevelTest : public PrecludeLastLevelTestBase,
+                              public testing::WithParamInterface<bool> {
+ public:
+  using PrecludeLastLevelTestBase::PrecludeLastLevelTestBase;
+
+  bool UseDynamicConfig() const { return GetParam(); }
+
+  void ApplyConfigChange(
+      Options* options,
+      const std::unordered_map<std::string, std::string>& config_change,
+      const std::unordered_map<std::string, std::string>& db_config_change =
+          {}) {
+    ApplyConfigChangeImpl(UseDynamicConfig(), options, config_change,
+                          db_config_change);
+  }
+};
+
+TEST_P(PrecludeLastLevelTest, MigrationFromPreserveTimeManualCompaction) {
  const int kNumTrigger = 4;
  const int kNumLevels = 7;
  const int kNumKeys = 100;
@@ -1367,9 +1348,8 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeManualCompaction) {
  ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel());

  // enable preclude feature
-  options.preclude_last_level_data_seconds = 10000;
-  options.last_level_temperature = Temperature::kCold;
-  Reopen(options);
+  ApplyConfigChange(&options, {{"preclude_last_level_data_seconds", "10000"},
+                               {"last_level_temperature", "kCold"}});

  // all data is hot, even they're in the last level
  ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0);
@@ -1393,7 +1373,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeManualCompaction) {
  Close();
 }

-TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeAutoCompaction) {
+TEST_P(PrecludeLastLevelTest, MigrationFromPreserveTimeAutoCompaction) {
  const int kNumTrigger = 4;
  const int kNumLevels = 7;
  const int kNumKeys = 100;
@@ -1423,16 +1403,17 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeAutoCompaction) {
  // all data is pushed to the last level
  ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel());

-  // enable preclude feature
-  options.preclude_last_level_data_seconds = 10000;
-  options.last_level_temperature = Temperature::kCold;
+  // enable preclude feature, and...
  // make sure it won't trigger Size Amp compaction, unlike normal Size Amp
  // compaction which is typically a last level compaction, when tiered Storage
  // ("preclude_last_level") is enabled, size amp won't include the last level.
  // As the last level would be in cold tier and the size would not be a
  // problem, which also avoid frequent hot to cold storage compaction.
-  options.compaction_options_universal.max_size_amplification_percent = 400;
-  Reopen(options);
+  ApplyConfigChange(
+      &options,
+      {{"preclude_last_level_data_seconds", "10000"},
+       {"last_level_temperature", "kCold"},
+       {"compaction_options_universal.max_size_amplification_percent", "400"}});

  // all data is hot, even they're in the last level
  ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0);
@@ -1464,7 +1445,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeAutoCompaction) {
  Close();
 }

-TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimePartial) {
+TEST_P(PrecludeLastLevelTest, MigrationFromPreserveTimePartial) {
  const int kNumTrigger = 4;
  const int kNumLevels = 7;
  const int kNumKeys = 100;
@@ -1512,9 +1493,8 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimePartial) {
  }

  // enable preclude feature
-  options.preclude_last_level_data_seconds = 2000;
-  options.last_level_temperature = Temperature::kCold;
-  Reopen(options);
+  ApplyConfigChange(&options, {{"preclude_last_level_data_seconds", "2000"},
+                               {"last_level_temperature", "kCold"}});

  // Generate a sstable and trigger manual compaction
  ASSERT_OK(Put(Key(10), "value"));
@@ -1532,7 +1512,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimePartial) {
  Close();
 }

-TEST_F(PrecludeLastLevelTest, SmallPrecludeTime) {
+TEST_P(PrecludeLastLevelTest, SmallPrecludeTime) {
  const int kNumTrigger = 4;
  const int kNumLevels = 7;
  const int kNumKeys = 100;
@@ -1585,13 +1565,203 @@ TEST_F(PrecludeLastLevelTest, SmallPrecludeTime) {
  Close();
 }

-// Test Param: protection_bytes_per_key for WriteBatch
+TEST_P(PrecludeLastLevelTest, CheckInternalKeyRange) {
+  // When compacting keys from the last level to penultimate level,
+  // output to penultimate level should be within internal key range
+  // of input files from penultimate level.
+  // Set up:
+  // L5:
+  //  File 1: DeleteRange[1, 3)@4, File 2: [3@5, 100@6]
+  // L6:
+  // File 3: [2@1, 3@2], File 4: [50@3]
+  //
+  // When File 1 and File 3 are being compacted,
+  // Key(3) cannot be compacted up, otherwise it causes
+  // inconsistency where File 3's Key(3) has a lower sequence number
+  // than File 2's Key(3).
+  const int kNumLevels = 7;
+  auto options = CurrentOptions();
+  options.env = mock_env_.get();
+  options.last_level_temperature = Temperature::kCold;
+  options.level_compaction_dynamic_level_bytes = true;
+  options.num_levels = kNumLevels;
+  options.statistics = CreateDBStatistics();
+  options.max_subcompactions = 10;
+  options.preserve_internal_time_seconds = 10000;
+  DestroyAndReopen(options);
+  // File 3
+  ASSERT_OK(Put(Key(2), "val2"));
+  ASSERT_OK(Put(Key(3), "val3"));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(6);
+  // File 4
+  ASSERT_OK(Put(Key(50), "val50"));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(6);
+
+  ApplyConfigChange(&options, {{"preclude_last_level_data_seconds", "10000"}});
+  const Snapshot* snapshot = db_->GetSnapshot();
+
+  // File 1
+  std::string start = Key(1);
+  std::string end = Key(3);
+  ASSERT_OK(db_->DeleteRange({}, db_->DefaultColumnFamily(), start, end));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(5);
+  // File 2
+  ASSERT_OK(Put(Key(3), "vall"));
+  ASSERT_OK(Put(Key(100), "val100"));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(5);
+
+  ASSERT_EQ("0,0,0,0,0,2,2", FilesPerLevel());
+
+  auto VerifyLogicalState = [&]() {
+    // First with snapshot
+    ASSERT_EQ("val2", Get(Key(2), snapshot));
+    ASSERT_EQ("val3", Get(Key(3), snapshot));
+    ASSERT_EQ("val50", Get(Key(50), snapshot));
+    ASSERT_EQ("NOT_FOUND", Get(Key(100), snapshot));
+
+    // Then without snapshot
+    ASSERT_EQ("NOT_FOUND", Get(Key(2)));
+    ASSERT_EQ("vall", Get(Key(3)));
+    ASSERT_EQ("val50", Get(Key(50)));
+    ASSERT_EQ("val100", Get(Key(100)));
+  };
+
+  VerifyLogicalState();
+
+  // Try to compact keys up
+  CompactRangeOptions cro;
+  cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
+  // Without internal key range checking, we get the following error:
+  // Corruption: force_consistency_checks(DEBUG): VersionBuilder: L5 has
+  // overlapping ranges: file #18 largest key: '6B6579303030303033' seq:102,
+  // type:1 vs. file #15 smallest key: '6B6579303030303033' seq:104, type:1
+  ASSERT_OK(CompactRange(cro, Key(1), Key(2)));
+
+  VerifyLogicalState();
+
+  db_->ReleaseSnapshot(snapshot);
+  Close();
+}
+
+TEST_P(PrecludeLastLevelTest, RangeTombstoneSnapshotMigrateFromLast) {
+  // Reproducer for issue originally described in
+  // https://github.com/facebook/rocksdb/pull/9964/files#r1024449523
+  if (!UseDynamicConfig()) {
+    // Depends on config change while holding a snapshot
+    return;
+  }
+  const int kNumLevels = 7;
+  auto options = CurrentOptions();
+  options.env = mock_env_.get();
+  options.last_level_temperature = Temperature::kCold;
+  options.level_compaction_dynamic_level_bytes = true;
+  options.num_levels = kNumLevels;
+  options.statistics = CreateDBStatistics();
+  options.max_subcompactions = 10;
+  options.preserve_internal_time_seconds = 30000;
+  DestroyAndReopen(options);
+
+  // Entries with much older write time
+  ASSERT_OK(Put(Key(2), "val2"));
+  ASSERT_OK(Put(Key(6), "val6"));
+
+  for (int i = 0; i < 10; i++) {
+    dbfull()->TEST_WaitForPeriodicTaskRun(
+        [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(1000)); });
+  }
+  const Snapshot* snapshot = db_->GetSnapshot();
+
+  ASSERT_OK(db_->DeleteRange({}, db_->DefaultColumnFamily(), Key(1), Key(5)));
+  ASSERT_OK(Put(Key(1), "val1"));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(6);
+  ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel());
+
+  ApplyConfigChange(&options, {{"preclude_last_level_data_seconds", "10000"}});
+
+  // To exercise the WithinPenultimateLevelOutputRange feature, we want files
+  // around the middle file to be compacted on the penultimate level
+  ASSERT_OK(Put(Key(0), "val0"));
+  ASSERT_OK(Flush());
+  ASSERT_OK(Put(Key(3), "val3"));
+  ASSERT_OK(Flush());
+  ASSERT_OK(Put(Key(7), "val7"));
+
+  // FIXME: ideally this wouldn't be necessary to get a seqno to time entry
+  // into a later compaction to get data into the last level
+  dbfull()->TEST_WaitForPeriodicTaskRun(
+      [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(1000)); });
+
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(5);
+  ASSERT_EQ("0,0,0,0,0,3,1", FilesPerLevel());
+
+  auto VerifyLogicalState = [&]() {
+    // First with snapshot
+    if (snapshot) {
+      ASSERT_EQ("NOT_FOUND", Get(Key(0), snapshot));
+      ASSERT_EQ("NOT_FOUND", Get(Key(1), snapshot));
+      ASSERT_EQ("val2", Get(Key(2), snapshot));
+      ASSERT_EQ("NOT_FOUND", Get(Key(3), snapshot));
+      ASSERT_EQ("val6", Get(Key(6), snapshot));
+      ASSERT_EQ("NOT_FOUND", Get(Key(7), snapshot));
+    }
+
+    // Then without snapshot
+    ASSERT_EQ("val0", Get(Key(0)));
+    ASSERT_EQ("val1", Get(Key(1)));
+    ASSERT_EQ("NOT_FOUND", Get(Key(2)));
+    ASSERT_EQ("val3", Get(Key(3)));
+    ASSERT_EQ("val6", Get(Key(6)));
+    ASSERT_EQ("val7", Get(Key(7)));
+  };
+
+  VerifyLogicalState();
+
+  // Try a limited range compaction
+  // FIXME: this currently hits the "Unsafe to store Seq later than snapshot"
+  // error. Needs to work safely for preclude option to be user mutable.
+  // ASSERT_OK(CompactRange({}, Key(3), Key(4)));
+  EXPECT_EQ("0,0,0,0,0,3,1", FilesPerLevel());
+  VerifyLogicalState();
+
+  // Compact everything, but some data still goes to both penultimate and last
+  // levels. A full-range compaction should be safe to "migrate" data from the
+  // last level to penultimate (because of preclude setting change).
+  ASSERT_OK(CompactRange({}, {}, {}));
+  EXPECT_EQ("0,0,0,0,0,1,1", FilesPerLevel());
+  VerifyLogicalState();
+  // Key1 should have been migrated out of the last level
+  auto& meta = *GetLevelFileMetadatas(6)[0];
+  ASSERT_LT(Key(1), meta.smallest.user_key().ToString());
+
+  // Make data eligible for last level
+  db_->ReleaseSnapshot(snapshot);
+  snapshot = nullptr;
+  mock_clock_->MockSleepForSeconds(static_cast<int>(10000));
+
+  ASSERT_OK(CompactRange({}, {}, {}));
+  EXPECT_EQ("0,0,0,0,0,0,1", FilesPerLevel());
+  VerifyLogicalState();
+
+  Close();
+}
+
+INSTANTIATE_TEST_CASE_P(PrecludeLastLevelTest, PrecludeLastLevelTest,
+                        ::testing::Bool());
+
 class TimedPutPrecludeLastLevelTest
-    : public PrecludeLastLevelTest,
+    : public PrecludeLastLevelTestBase,
      public testing::WithParamInterface<size_t> {
 public:
  TimedPutPrecludeLastLevelTest()
-      : PrecludeLastLevelTest("timed_put_preclude_last_level_test") {}
+      : PrecludeLastLevelTestBase("timed_put_preclude_last_level_test") {}
+
+  size_t ProtectionBytesPerKey() const { return GetParam(); }
 };

 TEST_P(TimedPutPrecludeLastLevelTest, FastTrackTimedPutToLastLevel) {
@@ -1609,7 +1779,7 @@ TEST_P(TimedPutPrecludeLastLevelTest, FastTrackTimedPutToLastLevel) {
  options.last_level_temperature = Temperature::kCold;
  DestroyAndReopen(options);
  WriteOptions wo;
-  wo.protection_bytes_per_key = GetParam();
+  wo.protection_bytes_per_key = ProtectionBytesPerKey();

  Random rnd(301);

@@ -1663,7 +1833,7 @@ TEST_P(TimedPutPrecludeLastLevelTest, InterleavedTimedPutAndPut) {
  options.default_write_temperature = Temperature::kHot;
  DestroyAndReopen(options);
  WriteOptions wo;
-  wo.protection_bytes_per_key = GetParam();
+  wo.protection_bytes_per_key = ProtectionBytesPerKey();

  // Start time: kMockStartTime = 10000000;
  ASSERT_OK(TimedPut(0, Key(0), "v0", kMockStartTime - 1 * 24 * 60 * 60, wo));
@@ -1689,7 +1859,7 @@ TEST_P(TimedPutPrecludeLastLevelTest, PreserveTimedPutOnPenultimateLevel) {
  options.default_write_temperature = Temperature::kHot;
  DestroyAndReopen(options);
  WriteOptions wo;
-  wo.protection_bytes_per_key = GetParam();
+  wo.protection_bytes_per_key = ProtectionBytesPerKey();

  // Creating a snapshot to manually control when preferred sequence number is
  // swapped in. An entry's preferred seqno won't get swapped in until it's
@@ -1759,7 +1929,7 @@ TEST_P(TimedPutPrecludeLastLevelTest, AutoTriggerCompaction) {
  options.table_properties_collector_factories.push_back(factory);
  DestroyAndReopen(options);
  WriteOptions wo;
-  wo.protection_bytes_per_key = GetParam();
+  wo.protection_bytes_per_key = ProtectionBytesPerKey();

  Random rnd(301);

@@ -1814,7 +1984,7 @@ TEST_P(TimedPutPrecludeLastLevelTest, AutoTriggerCompaction) {
 INSTANTIATE_TEST_CASE_P(TimedPutPrecludeLastLevelTest,
                        TimedPutPrecludeLastLevelTest, ::testing::Values(0, 8));

-TEST_F(PrecludeLastLevelTest, LastLevelOnlyCompactionPartial) {
+TEST_P(PrecludeLastLevelTest, LastLevelOnlyCompactionPartial) {
  const int kNumTrigger = 4;
  const int kNumLevels = 7;
  const int kNumKeys = 100;
@@ -1845,9 +2015,8 @@ TEST_F(PrecludeLastLevelTest, LastLevelOnlyCompactionPartial) {
  ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel());

  // enable preclude feature
-  options.preclude_last_level_data_seconds = 2000;
-  options.last_level_temperature = Temperature::kCold;
-  Reopen(options);
+  ApplyConfigChange(&options, {{"preclude_last_level_data_seconds", "2000"},
+                               {"last_level_temperature", "kCold"}});

  CompactRangeOptions cro;
  cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
@@ -1878,21 +2047,30 @@ TEST_F(PrecludeLastLevelTest, LastLevelOnlyCompactionPartial) {
  Close();
 }

-class PrecludeLastLevelTestWithParms
-    : public PrecludeLastLevelTest,
-      public testing::WithParamInterface<bool> {
+class PrecludeLastLevelOptionalTest
+    : public PrecludeLastLevelTestBase,
+      public testing::WithParamInterface<std::tuple<bool, bool>> {
 public:
-  PrecludeLastLevelTestWithParms() : PrecludeLastLevelTest() {}
+  bool UseDynamicConfig() const { return std::get<0>(GetParam()); }
+
+  void ApplyConfigChange(
+      Options* options,
+      const std::unordered_map<std::string, std::string>& config_change,
+      const std::unordered_map<std::string, std::string>& db_config_change =
+          {}) {
+    ApplyConfigChangeImpl(UseDynamicConfig(), options, config_change,
+                          db_config_change);
+  }
+
+  bool EnablePrecludeLastLevel() const { return std::get<1>(GetParam()); }
 };

-TEST_P(PrecludeLastLevelTestWithParms, LastLevelOnlyCompactionNoPreclude) {
+TEST_P(PrecludeLastLevelOptionalTest, LastLevelOnlyCompactionNoPreclude) {
  const int kNumTrigger = 4;
  const int kNumLevels = 7;
  const int kNumKeys = 100;
  const int kKeyPerSec = 10;

-  bool enable_preclude_last_level = GetParam();
-
  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleUniversal;
  options.preserve_internal_time_seconds = 2000;
@@ -1950,7 +2128,7 @@ TEST_P(PrecludeLastLevelTestWithParms, LastLevelOnlyCompactionNoPreclude) {
  SyncPoint::GetInstance()->SetCallBack(
      "UniversalCompactionBuilder::PickCompaction:Return", [&](void* arg) {
        auto compaction = static_cast<Compaction*>(arg);
-        if (enable_preclude_last_level && is_manual_compaction_running) {
+        if (EnablePrecludeLastLevel() && is_manual_compaction_running) {
          ASSERT_TRUE(compaction == nullptr);
          verified_compaction_order = true;
        } else {
@@ -1977,12 +2155,11 @@ TEST_P(PrecludeLastLevelTestWithParms, LastLevelOnlyCompactionNoPreclude) {
  SyncPoint::GetInstance()->EnableProcessing();

  // only enable if the Parameter is true
-  if (enable_preclude_last_level) {
-    options.preclude_last_level_data_seconds = 2000;
-  }
-  options.max_background_jobs = 8;
-  options.last_level_temperature = Temperature::kCold;
-  Reopen(options);
+  ApplyConfigChange(&options,
+                    {{"preclude_last_level_data_seconds",
+                      EnablePrecludeLastLevel() ? "2000" : "0"},
+                     {"last_level_temperature", "kCold"}},
+                    {{"max_background_jobs", "8"}});

  auto manual_compaction_thread = port::Thread([this]() {
    CompactRangeOptions cro;
@@ -2011,7 +2188,7 @@ TEST_P(PrecludeLastLevelTestWithParms, LastLevelOnlyCompactionNoPreclude) {

  ASSERT_OK(dbfull()->TEST_WaitForCompact());

-  if (enable_preclude_last_level) {
+  if (EnablePrecludeLastLevel()) {
    ASSERT_NE("0,0,0,0,0,1,1", FilesPerLevel());
  } else {
    ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel());
@@ -2025,7 +2202,7 @@ TEST_P(PrecludeLastLevelTestWithParms, LastLevelOnlyCompactionNoPreclude) {
  Close();
 }

-TEST_P(PrecludeLastLevelTestWithParms, PeriodicCompactionToPenultimateLevel) {
+TEST_P(PrecludeLastLevelOptionalTest, PeriodicCompactionToPenultimateLevel) {
  // Test the last level only periodic compaction should also be blocked by an
  // ongoing compaction in penultimate level if tiered compaction is enabled
  // otherwise, the periodic compaction should just run for the last level.
@@ -2035,8 +2212,6 @@ TEST_P(PrecludeLastLevelTestWithParms, PeriodicCompactionToPenultimateLevel) {
  const int kKeyPerSec = 1;
  const int kNumKeys = 100;

-  bool enable_preclude_last_level = GetParam();
-
  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleUniversal;
  options.preserve_internal_time_seconds = 20000;
@@ -2063,12 +2238,11 @@ TEST_P(PrecludeLastLevelTestWithParms, PeriodicCompactionToPenultimateLevel) {
  ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel());

  // enable preclude feature
-  if (enable_preclude_last_level) {
-    options.preclude_last_level_data_seconds = 20000;
-  }
-  options.max_background_jobs = 8;
-  options.last_level_temperature = Temperature::kCold;
-  Reopen(options);
+  ApplyConfigChange(&options,
+                    {{"preclude_last_level_data_seconds",
+                      EnablePrecludeLastLevel() ? "2000" : "0"},
+                     {"last_level_temperature", "kCold"}},
+                    {{"max_background_jobs", "8"}});

  std::atomic_bool is_size_ratio_compaction_running = false;
  std::atomic_bool verified_last_level_compaction = false;
@@ -2093,7 +2267,7 @@ TEST_P(PrecludeLastLevelTestWithParms, PeriodicCompactionToPenultimateLevel) {
        auto compaction = static_cast<Compaction*>(arg);

        if (is_size_ratio_compaction_running) {
-          if (enable_preclude_last_level) {
+          if (EnablePrecludeLastLevel()) {
            ASSERT_TRUE(compaction == nullptr);
          } else {
            ASSERT_TRUE(compaction != nullptr);
@@ -2151,8 +2325,10 @@ TEST_P(PrecludeLastLevelTestWithParms, PeriodicCompactionToPenultimateLevel) {
  Close();
 }

-INSTANTIATE_TEST_CASE_P(PrecludeLastLevelTestWithParms,
-                        PrecludeLastLevelTestWithParms, testing::Bool());
+INSTANTIATE_TEST_CASE_P(PrecludeLastLevelOptionalTest,
+                        PrecludeLastLevelOptionalTest,
+                        ::testing::Combine(::testing::Bool(),
+                                           ::testing::Bool()));

 // partition the SST into 3 ranges [0, 19] [20, 39] [40, ...]
 class ThreeRangesPartitioner : public SstPartitioner {
@@ -2196,7 +2372,7 @@ class ThreeRangesPartitionerFactory : public SstPartitionerFactory {
  }
 };

-TEST_F(PrecludeLastLevelTest, PartialPenultimateLevelCompaction) {
+TEST_P(PrecludeLastLevelTest, PartialPenultimateLevelCompaction) {
  const int kNumTrigger = 4;
  const int kNumLevels = 7;
  const int kKeyPerSec = 10;
@@ -2207,6 +2383,7 @@ TEST_F(PrecludeLastLevelTest, PartialPenultimateLevelCompaction) {
  options.level0_file_num_compaction_trigger = kNumTrigger;
  options.preserve_internal_time_seconds = 10000;
  options.num_levels = kNumLevels;
+  options.statistics = CreateDBStatistics();
  DestroyAndReopen(options);

  Random rnd(301);
@@ -2244,11 +2421,10 @@ TEST_F(PrecludeLastLevelTest, PartialPenultimateLevelCompaction) {
  // L6: [0,                299]
  ASSERT_EQ("0,0,0,0,0,3,1", FilesPerLevel());

+  ASSERT_OK(options.statistics->Reset());
  // enable tiered storage feature
-  options.preclude_last_level_data_seconds = 10000;
-  options.last_level_temperature = Temperature::kCold;
-  options.statistics = CreateDBStatistics();
-  Reopen(options);
+  ApplyConfigChange(&options, {{"preclude_last_level_data_seconds", "10000"},
+                               {"last_level_temperature", "kCold"}});

  ColumnFamilyMetaData meta;
  db_->GetColumnFamilyMetaData(&meta);
@@ -2294,7 +2470,7 @@ TEST_F(PrecludeLastLevelTest, PartialPenultimateLevelCompaction) {
  Close();
 }

-TEST_F(PrecludeLastLevelTest, RangeDelsCauseFileEndpointsToOverlap) {
+TEST_P(PrecludeLastLevelTest, RangeDelsCauseFileEndpointsToOverlap) {
  const int kNumLevels = 7;
  const int kSecondsPerKey = 10;
  const int kNumFiles = 3;
@@ -2433,12 +2609,25 @@ TEST_F(PrecludeLastLevelTest, RangeDelsCauseFileEndpointsToOverlap) {

 // Tests DBIter::GetProperty("rocksdb.iterator.write-time") return a data's
 // approximate write unix time.
-// Test Param:
-// 1) use tailing iterator or regular iterator (when it applies)
-class IteratorWriteTimeTest : public PrecludeLastLevelTest,
-                              public testing::WithParamInterface<bool> {
+class IteratorWriteTimeTest
+    : public PrecludeLastLevelTestBase,
+      public testing::WithParamInterface<std::tuple<bool, bool>> {
 public:
-  IteratorWriteTimeTest() : PrecludeLastLevelTest("iterator_write_time_test") {}
+  IteratorWriteTimeTest()
+      : PrecludeLastLevelTestBase("iterator_write_time_test") {}
+
+  bool UseTailingIterator() const { return std::get<0>(GetParam()); }
+
+  bool UseDynamicConfig() const { return std::get<1>(GetParam()); }
+
+  void ApplyConfigChange(
+      Options* options,
+      const std::unordered_map<std::string, std::string>& config_change,
+      const std::unordered_map<std::string, std::string>& db_config_change =
+          {}) {
+    ApplyConfigChangeImpl(UseDynamicConfig(), options, config_change,
+                          db_config_change);
+  }

  uint64_t VerifyKeyAndGetWriteTime(Iterator* iter,
                                    const std::string& expected_key) {
@@ -2478,10 +2667,14 @@ TEST_P(IteratorWriteTimeTest, ReadFromMemtables) {
  options.compaction_style = kCompactionStyleUniversal;
  options.env = mock_env_.get();
  options.level0_file_num_compaction_trigger = kNumTrigger;
-  options.preserve_internal_time_seconds = 10000;
  options.num_levels = kNumLevels;
  DestroyAndReopen(options);

+  // While there are issues with tracking seqno 0
+  ASSERT_OK(Delete("something_to_bump_seqno"));
+
+  ApplyConfigChange(&options, {{"preserve_internal_time_seconds", "10000"}});
+
  Random rnd(301);
  for (int i = 0; i < kNumKeys; i++) {
    dbfull()->TEST_WaitForPeriodicTaskRun(
@@ -2495,7 +2688,7 @@ TEST_P(IteratorWriteTimeTest, ReadFromMemtables) {
  }

  ReadOptions ropts;
-  ropts.tailing = GetParam();
+  ropts.tailing = UseTailingIterator();
  int i;

  // Forward iteration
@@ -2535,10 +2728,9 @@ TEST_P(IteratorWriteTimeTest, ReadFromMemtables) {
    ASSERT_EQ(-1, i);
  }

-  // Reopen the DB and disable the seqno to time recording, data with user
-  // specified write time can still get a write time before it's flushed.
-  options.preserve_internal_time_seconds = 0;
-  Reopen(options);
+  // Disable the seqno to time recording. Data with user specified write time
+  // can still get a write time before it's flushed.
+  ApplyConfigChange(&options, {{"preserve_internal_time_seconds", "0"}});
  ASSERT_OK(TimedPut(Key(kKeyWithWriteTime), rnd.RandomString(100),
                     kUserSpecifiedWriteTime));
  {
@@ -2574,10 +2766,14 @@ TEST_P(IteratorWriteTimeTest, ReadFromSstFile) {
  options.compaction_style = kCompactionStyleUniversal;
  options.env = mock_env_.get();
  options.level0_file_num_compaction_trigger = kNumTrigger;
-  options.preserve_internal_time_seconds = 10000;
  options.num_levels = kNumLevels;
  DestroyAndReopen(options);

+  // While there are issues with tracking seqno 0
+  ASSERT_OK(Delete("something_to_bump_seqno"));
+
+  ApplyConfigChange(&options, {{"preserve_internal_time_seconds", "10000"}});
+
  Random rnd(301);
  for (int i = 0; i < kNumKeys; i++) {
    dbfull()->TEST_WaitForPeriodicTaskRun(
@@ -2592,7 +2788,7 @@ TEST_P(IteratorWriteTimeTest, ReadFromSstFile) {

  ASSERT_OK(Flush());
  ReadOptions ropts;
-  ropts.tailing = GetParam();
+  ropts.tailing = UseTailingIterator();
  std::string prop;
  int i;

@@ -2638,10 +2834,9 @@ TEST_P(IteratorWriteTimeTest, ReadFromSstFile) {
    ASSERT_EQ(-1, i);
  }

-  // Reopen the DB and disable the seqno to time recording. Data retrieved from
-  // SST files still have write time available.
-  options.preserve_internal_time_seconds = 0;
-  Reopen(options);
+  // Disable the seqno to time recording. Data retrieved from SST files still
+  // have write time available.
+  ApplyConfigChange(&options, {{"preserve_internal_time_seconds", "0"}});

  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kSecondsPerRecording); });
@@ -2705,11 +2900,12 @@ TEST_P(IteratorWriteTimeTest, MergeReturnsBaseValueWriteTime) {
  options.compaction_style = kCompactionStyleUniversal;
  options.env = mock_env_.get();
  options.level0_file_num_compaction_trigger = kNumTrigger;
-  options.preserve_internal_time_seconds = 10000;
  options.num_levels = kNumLevels;
  options.merge_operator = MergeOperators::CreateStringAppendOperator();
  DestroyAndReopen(options);

+  ApplyConfigChange(&options, {{"preserve_internal_time_seconds", "10000"}});
+
  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(kSecondsPerRecording); });
  ASSERT_OK(Put("foo", "fv1"));
@@ -2720,7 +2916,7 @@ TEST_P(IteratorWriteTimeTest, MergeReturnsBaseValueWriteTime) {
  ASSERT_OK(Merge("foo", "bv1"));

  ReadOptions ropts;
-  ropts.tailing = GetParam();
+  ropts.tailing = UseTailingIterator();
  {
    std::unique_ptr<Iterator> iter(dbfull()->NewIterator(ropts));
    iter->SeekToFirst();
@@ -2738,7 +2934,7 @@ TEST_P(IteratorWriteTimeTest, MergeReturnsBaseValueWriteTime) {
 }

 INSTANTIATE_TEST_CASE_P(IteratorWriteTimeTest, IteratorWriteTimeTest,
-                        testing::Bool());
+                        testing::Combine(testing::Bool(), testing::Bool()));

 }  // namespace ROCKSDB_NAMESPACE

@@ -10623,6 +10623,97 @@ TEST_F(DBCompactionTest, ReleaseCompactionDuringManifestWrite) {
  SyncPoint::GetInstance()->ClearAllCallBacks();
 }

+TEST_F(DBCompactionTest, RecordNewestKeyTimeForTtlCompaction) {
+  Options options;
+  SetTimeElapseOnlySleepOnReopen(&options);
+  options.env = CurrentOptions().env;
+  options.compaction_style = kCompactionStyleFIFO;
+  options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
+  options.write_buffer_size = 10 << 10;  // 10KB
+  options.arena_block_size = 4096;
+  options.compression = kNoCompression;
+  options.create_if_missing = true;
+  options.compaction_options_fifo.allow_compaction = false;
+  options.num_levels = 1;
+  env_->SetMockSleep();
+  options.env = env_;
+  options.ttl = 1 * 60 * 60;  // 1 hour
+  ASSERT_OK(TryReopen(options));
+
+  // Generate and flush 4 files, each about 10KB
+  // Compaction is manually disabled at this point so we can check
+  // each file's newest_key_time
+  Random rnd(301);
+  for (int i = 0; i < 4; i++) {
+    for (int j = 0; j < 10; j++) {
+      ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980)));
+    }
+    ASSERT_OK(Flush());
+    env_->MockSleepForSeconds(5);
+  }
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+  ASSERT_EQ(NumTableFilesAtLevel(0), 4);
+
+  // Check that we are populating newest_key_time on flush
+  std::vector<FileMetaData*> file_metadatas = GetLevelFileMetadatas(0);
+  ASSERT_EQ(file_metadatas.size(), 4);
+  uint64_t first_newest_key_time =
+      file_metadatas[0]->fd.table_reader->GetTableProperties()->newest_key_time;
+  ASSERT_NE(first_newest_key_time, kUnknownNewestKeyTime);
+  // Check that the newest_key_times are in expected ordering
+  uint64_t prev_newest_key_time = first_newest_key_time;
+  for (size_t idx = 1; idx < file_metadatas.size(); idx++) {
+    uint64_t newest_key_time = file_metadatas[idx]
+                                   ->fd.table_reader->GetTableProperties()
+                                   ->newest_key_time;
+
+    ASSERT_LT(newest_key_time, prev_newest_key_time);
+    prev_newest_key_time = newest_key_time;
+    ASSERT_EQ(newest_key_time, file_metadatas[idx]
+                                   ->fd.table_reader->GetTableProperties()
+                                   ->creation_time);
+  }
+  // The delta between the first and last newest_key_times is 15s
+  uint64_t last_newest_key_time = prev_newest_key_time;
+  ASSERT_EQ(15, first_newest_key_time - last_newest_key_time);
+
+  // After compaction, the newest_key_time of the output file should be the max
+  // of the input files
+  options.compaction_options_fifo.allow_compaction = true;
+  ASSERT_OK(TryReopen(options));
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+  ASSERT_EQ(NumTableFilesAtLevel(0), 1);
+  file_metadatas = GetLevelFileMetadatas(0);
+  ASSERT_EQ(file_metadatas.size(), 1);
+  ASSERT_EQ(
+      file_metadatas[0]->fd.table_reader->GetTableProperties()->newest_key_time,
+      first_newest_key_time);
+  // Contrast newest_key_time with creation_time, which records the oldest
+  // ancestor time (15s older than newest_key_time)
+  ASSERT_EQ(
+      file_metadatas[0]->fd.table_reader->GetTableProperties()->creation_time,
+      last_newest_key_time);
+  ASSERT_EQ(file_metadatas[0]->oldest_ancester_time, last_newest_key_time);
+
+  // Make sure TTL of 5s causes compaction
+  env_->MockSleepForSeconds(6);
+
+  // The oldest input file is older than 15s
+  // However the newest of the compaction input files is younger than 15s, so
+  // we don't compact
+  ASSERT_OK(dbfull()->SetOptions({{"ttl", "15"}}));
+  ASSERT_EQ(dbfull()->GetOptions().ttl, 15);
+  ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+  ASSERT_EQ(NumTableFilesAtLevel(0), 1);
+
+  // Now even the youngest input file is too old
+  ASSERT_OK(dbfull()->SetOptions({{"ttl", "5"}}));
+  ASSERT_EQ(dbfull()->GetOptions().ttl, 5);
+  ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+  ASSERT_EQ(NumTableFilesAtLevel(0), 0);
+}
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -659,8 +659,9 @@ Status DBImpl::CloseHelper() {
  // We need to release them before the block cache is destroyed. The block
  // cache may be destroyed inside versions_.reset(), when column family data
  // list is destroyed, so leaving handles in table cache after
-  // versions_.reset() may cause issues.
-  // Here we clean all unreferenced handles in table cache.
+  // versions_.reset() may cause issues. Here we clean all unreferenced handles
+  // in table cache, and (for certain builds/conditions) assert that no obsolete
+  // files are hanging around unreferenced (leak) in the table/blob file cache.
  // Now we assume all user queries have finished, so only version set itself
  // can possibly hold the blocks from block cache. After releasing unreferenced
  // handles here, only handles held by version set left and inside
@@ -668,6 +669,9 @@ Status DBImpl::CloseHelper() {
  // time a handle is released, we erase it from the cache too. By doing that,
  // we can guarantee that after versions_.reset(), table cache is empty
  // so the cache can be safely destroyed.
+#ifndef NDEBUG
+  TEST_VerifyNoObsoleteFilesCached(/*db_mutex_already_held=*/true);
+#endif  // !NDEBUG
  table_cache_->EraseUnRefEntries();

  for (auto& txn_entry : recovered_transactions_) {
@@ -843,20 +847,22 @@ Status DBImpl::RegisterRecordSeqnoTimeWorker(const ReadOptions& read_options,

  uint64_t min_preserve_seconds = std::numeric_limits<uint64_t>::max();
  uint64_t max_preserve_seconds = std::numeric_limits<uint64_t>::min();
-  bool mapping_was_empty = false;
+  std::vector<SuperVersionContext> sv_contexts;
  {
    InstrumentedMutexLock l(&mutex_);

    for (auto cfd : *versions_->GetColumnFamilySet()) {
+      auto& mopts = *cfd->GetLatestMutableCFOptions();
      // preserve time is the max of 2 options.
      uint64_t preserve_seconds =
-          std::max(cfd->ioptions()->preserve_internal_time_seconds,
-                   cfd->ioptions()->preclude_last_level_data_seconds);
+          std::max(mopts.preserve_internal_time_seconds,
+                   mopts.preclude_last_level_data_seconds);
      if (!cfd->IsDropped() && preserve_seconds > 0) {
        min_preserve_seconds = std::min(preserve_seconds, min_preserve_seconds);
        max_preserve_seconds = std::max(preserve_seconds, max_preserve_seconds);
      }
    }
+    size_t old_mapping_size = seqno_to_time_mapping_.Size();
    if (min_preserve_seconds == std::numeric_limits<uint64_t>::max()) {
      // Don't track
      seqno_to_time_mapping_.SetCapacity(0);
@@ -868,9 +874,17 @@ Status DBImpl::RegisterRecordSeqnoTimeWorker(const ReadOptions& read_options,
      seqno_to_time_mapping_.SetCapacity(cap);
      seqno_to_time_mapping_.SetMaxTimeSpan(max_preserve_seconds);
    }
-    mapping_was_empty = seqno_to_time_mapping_.Empty();
+    if (old_mapping_size != seqno_to_time_mapping_.Size()) {
+      InstallSeqnoToTimeMappingInSV(&sv_contexts);
+    }
  }

+  // clean up outside db mutex
+  for (SuperVersionContext& sv_context : sv_contexts) {
+    sv_context.Clean();
+  }
+  sv_contexts.clear();
+
  uint64_t seqno_time_cadence = 0;
  if (min_preserve_seconds != std::numeric_limits<uint64_t>::max()) {
    // round up to 1 when the time_duration is smaller than
@@ -910,8 +924,6 @@ Status DBImpl::RegisterRecordSeqnoTimeWorker(const ReadOptions& read_options,
    assert(!is_new_db || last_seqno_zero);
    if (is_new_db && last_seqno_zero) {
      // Pre-allocate seqnos and pre-populate historical mapping
-      assert(mapping_was_empty);
-
      // We can simply modify these, before writes are allowed
      constexpr uint64_t kMax = kMaxSeqnoTimePairsPerSST;
      versions_->SetLastAllocatedSequence(kMax);
@@ -936,9 +948,10 @@ Status DBImpl::RegisterRecordSeqnoTimeWorker(const ReadOptions& read_options,

      // Pre-populate mappings for reserved sequence numbers.
      RecordSeqnoToTimeMapping(max_preserve_seconds);
-    } else if (mapping_was_empty) {
+    } else {
      if (!last_seqno_zero) {
-        // Ensure at least one mapping (or log a warning)
+        // Ensure at least one mapping (or log a warning), and
+        // an updated entry whenever relevant SetOptions is called
        RecordSeqnoToTimeMapping(/*populate_historical_seconds=*/0);
      } else {
        // FIXME (see limitation described above)
@@ -1299,6 +1312,12 @@ Status DBImpl::SetOptions(
  }
  sv_context.Clean();

+  if (s.ok() && (options_map.count("preserve_internal_time_seconds") > 0 ||
+                 options_map.count("preclude_last_level_data_seconds") > 0)) {
+    s = RegisterRecordSeqnoTimeWorker(read_options, write_options,
+                                      false /* is_new_db*/);
+  }
+
  ROCKS_LOG_INFO(
      immutable_db_options_.info_log,
      "SetOptions() on column family [%s], inputs:", cfd->GetName().c_str());
@@ -1456,8 +1475,6 @@ Status DBImpl::SetDBOptions(
      // TODO(xiez): clarify why apply optimize for read to write options
      file_options_for_compaction_ = fs_->OptimizeForCompactionTableRead(
          file_options_for_compaction_, immutable_db_options_);
-      file_options_for_compaction_.compaction_readahead_size =
-          mutable_db_options_.compaction_readahead_size;
      if (wal_other_option_changed || wal_size_option_changed) {
        WriteThread::Writer w;
        write_thread_.EnterUnbatched(&w, &mutex_);
@@ -2081,7 +2098,8 @@ InternalIterator* DBImpl::NewInternalIterator(
  // Collect iterator for mutable memtable
  auto mem_iter = super_version->mem->NewIterator(
      read_options, super_version->GetSeqnoToTimeMapping(), arena,
-      super_version->mutable_cf_options.prefix_extractor.get());
+      super_version->mutable_cf_options.prefix_extractor.get(),
+      /*for_flush=*/false);
  Status s;
  if (!read_options.ignore_range_deletions) {
    std::unique_ptr<TruncatedRangeDelIterator> mem_tombstone_iter;
@@ -3230,6 +3248,13 @@ Status DBImpl::MultiGetImpl(
      s = Status::Aborted();
      break;
    }
+
+    // This could be a long-running operation
+    bool aborted = ROCKSDB_THREAD_YIELD_CHECK_ABORT();
+    if (aborted) {
+      s = Status::Aborted("Query abort.");
+      break;
+    }
  }

  // Post processing (decrement reference counts and record statistics)
@@ -3474,6 +3499,8 @@ void DBImpl::MultiGetEntityWithCallback(
 Status DBImpl::WrapUpCreateColumnFamilies(
    const ReadOptions& read_options, const WriteOptions& write_options,
    const std::vector<const ColumnFamilyOptions*>& cf_options) {
+  options_mutex_.AssertHeld();
+
  // NOTE: this function is skipped for create_missing_column_families and
  // DB::Open, so new functionality here might need to go into Open also.
  bool register_worker = false;
@@ -3696,6 +3723,8 @@ Status DBImpl::DropColumnFamilies(
 }

 Status DBImpl::DropColumnFamilyImpl(ColumnFamilyHandle* column_family) {
+  options_mutex_.AssertHeld();
+
  // TODO: plumb Env::IOActivity, Env::IOPriority
  const ReadOptions read_options;
  const WriteOptions write_options;
@@ -3713,6 +3742,9 @@ Status DBImpl::DropColumnFamilyImpl(ColumnFamilyHandle* column_family) {
  edit.SetColumnFamily(cfd->GetID());

  Status s;
+  // Save re-aquiring lock for RegisterRecordSeqnoTimeWorker when not
+  // applicable
+  bool used_preserve_preclude = false;
  {
    InstrumentedMutexLock l(&mutex_);
    if (cfd->IsDropped()) {
@@ -3728,9 +3760,11 @@ Status DBImpl::DropColumnFamilyImpl(ColumnFamilyHandle* column_family) {
      write_thread_.ExitUnbatched(&w);
    }
    if (s.ok()) {
-      auto* mutable_cf_options = cfd->GetLatestMutableCFOptions();
-      max_total_in_memory_state_ -= mutable_cf_options->write_buffer_size *
-                                    mutable_cf_options->max_write_buffer_number;
+      auto& moptions = *cfd->GetLatestMutableCFOptions();
+      max_total_in_memory_state_ -=
+          moptions.write_buffer_size * moptions.max_write_buffer_number;
+      used_preserve_preclude = moptions.preserve_internal_time_seconds > 0 ||
+                               moptions.preclude_last_level_data_seconds > 0;
    }

    if (!cf_support_snapshot) {
@@ -3748,8 +3782,7 @@ Status DBImpl::DropColumnFamilyImpl(ColumnFamilyHandle* column_family) {
    bg_cv_.SignalAll();
  }

-  if (cfd->ioptions()->preserve_internal_time_seconds > 0 ||
-      cfd->ioptions()->preclude_last_level_data_seconds > 0) {
+  if (used_preserve_preclude) {
    s = RegisterRecordSeqnoTimeWorker(read_options, write_options,
                                      /* is_new_db */ false);
  }
@@ -3775,7 +3808,6 @@ bool DBImpl::KeyMayExist(const ReadOptions& read_options,
                         ColumnFamilyHandle* column_family, const Slice& key,
                         std::string* value, std::string* timestamp,
                         bool* value_found) {
-  assert(value != nullptr);
  assert(read_options.io_activity == Env::IOActivity::kUnknown);

  if (value_found != nullptr) {
@@ -3792,7 +3824,9 @@ bool DBImpl::KeyMayExist(const ReadOptions& read_options,
  get_impl_options.value_found = value_found;
  get_impl_options.timestamp = timestamp;
  auto s = GetImpl(roptions, key, get_impl_options);
-  value->assign(pinnable_val.data(), pinnable_val.size());
+  if (value_found && *value_found && value) {
+    value->assign(pinnable_val.data(), pinnable_val.size());
+  }

  // If block_cache is enabled and the index block of the table didn't
  // not present in block_cache, the return value will be Status::Incomplete.
@@ -3987,14 +4021,25 @@ std::unique_ptr<IterType> DBImpl::NewMultiCfIterator(
          "Different comparators are being used across CFs"));
    }
  }
+
  std::vector<Iterator*> child_iterators;
  Status s = NewIterators(_read_options, column_families, &child_iterators);
  if (!s.ok()) {
    return error_iterator_func(s);
  }
-  return std::make_unique<ImplType>(
-      column_families[0]->GetComparator(), _read_options.allow_unprepared_value,
-      column_families, std::move(child_iterators));
+
+  assert(column_families.size() == child_iterators.size());
+
+  std::vector<std::pair<ColumnFamilyHandle*, std::unique_ptr<Iterator>>>
+      cfh_iter_pairs;
+  cfh_iter_pairs.reserve(column_families.size());
+  for (size_t i = 0; i < column_families.size(); ++i) {
+    cfh_iter_pairs.emplace_back(column_families[i], child_iterators[i]);
+  }
+
+  return std::make_unique<ImplType>(_read_options,
+                                    column_families[0]->GetComparator(),
+                                    std::move(cfh_iter_pairs));
 }

 Status DBImpl::NewIterators(
@@ -4724,9 +4769,9 @@ void DBImpl::GetApproximateMemTableStats(ColumnFamilyHandle* column_family,
  // Convert user_key into a corresponding internal key.
  InternalKey k1(start.value(), kMaxSequenceNumber, kValueTypeForSeek);
  InternalKey k2(limit.value(), kMaxSequenceNumber, kValueTypeForSeek);
-  MemTable::MemTableStats memStats =
+  ReadOnlyMemTable::MemTableStats memStats =
      sv->mem->ApproximateStats(k1.Encode(), k2.Encode());
-  MemTable::MemTableStats immStats =
+  ReadOnlyMemTable::MemTableStats immStats =
      sv->imm->ApproximateStats(k1.Encode(), k2.Encode());
  *count = memStats.count + immStats.count;
  *size = memStats.size + immStats.size;
@@ -5181,11 +5226,12 @@ Status DBImpl::GetDbIdentity(std::string& identity) const {
  return Status::OK();
 }

-Status DBImpl::GetDbIdentityFromIdentityFile(std::string* identity) const {
+Status DBImpl::GetDbIdentityFromIdentityFile(const IOOptions& opts,
+                                             std::string* identity) const {
  std::string idfilename = IdentityFileName(dbname_);
  const FileOptions soptions;

-  Status s = ReadFileToString(fs_.get(), idfilename, identity);
+  Status s = ReadFileToString(fs_.get(), idfilename, opts, identity);
  if (!s.ok()) {
    return s;
  }
@@ -5496,7 +5542,8 @@ Status DBImpl::WriteOptionsFile(const WriteOptions& write_options,
                                   file_name, fs_.get());

  if (s.ok()) {
-    s = RenameTempFileToOptionsFile(file_name);
+    s = RenameTempFileToOptionsFile(file_name,
+                                    db_options.compaction_service != nullptr);
  }

  if (!s.ok() && GetEnv()->FileExists(file_name).ok()) {
@@ -5573,7 +5620,8 @@ Status DBImpl::DeleteObsoleteOptionsFiles() {
  return Status::OK();
 }

-Status DBImpl::RenameTempFileToOptionsFile(const std::string& file_name) {
+Status DBImpl::RenameTempFileToOptionsFile(const std::string& file_name,
+                                           bool is_remote_compaction_enabled) {
  Status s;

  uint64_t options_file_number = versions_->NewFileNumber();
@@ -5617,7 +5665,7 @@ Status DBImpl::RenameTempFileToOptionsFile(const std::string& file_name) {
      my_disable_delete_obsolete_files = disable_delete_obsolete_files_;
    }

-    if (!my_disable_delete_obsolete_files) {
+    if (!my_disable_delete_obsolete_files && !is_remote_compaction_enabled) {
      // TODO: Should we check for errors here?
      DeleteObsoleteOptionsFiles().PermitUncheckedError();
    }
@@ -5871,8 +5919,8 @@ Status DBImpl::IngestExternalFiles(
  const size_t num_cfs = args.size();
  for (size_t i = 0; i != num_cfs; ++i) {
    if (args[i].external_files.empty()) {
-      char err_msg[128] = {0};
-      snprintf(err_msg, 128, "external_files[%zu] is empty", i);
+      std::string err_msg =
+          "external_files[" + std::to_string(i) + "] is empty";
      return Status::InvalidArgument(err_msg);
    }
    if (i && args[i].options.fill_cache != args[i - 1].options.fill_cache) {
@@ -5938,9 +5986,9 @@ Status DBImpl::IngestExternalFiles(
  uint64_t start_file_number = next_file_number;
  for (size_t i = 1; i != num_cfs; ++i) {
    start_file_number += args[i - 1].external_files.size();
-    auto* cfd =
-        static_cast<ColumnFamilyHandleImpl*>(args[i].column_family)->cfd();
-    SuperVersion* super_version = cfd->GetReferencedSuperVersion(this);
+    SuperVersion* super_version =
+        ingestion_jobs[i].GetColumnFamilyData()->GetReferencedSuperVersion(
+            this);
    Status es = ingestion_jobs[i].Prepare(
        args[i].external_files, args[i].files_checksums,
        args[i].files_checksum_func_names, args[i].file_temperature,
@@ -5954,9 +6002,9 @@ Status DBImpl::IngestExternalFiles(
  TEST_SYNC_POINT("DBImpl::IngestExternalFiles:BeforeLastJobPrepare:0");
  TEST_SYNC_POINT("DBImpl::IngestExternalFiles:BeforeLastJobPrepare:1");
  {
-    auto* cfd =
-        static_cast<ColumnFamilyHandleImpl*>(args[0].column_family)->cfd();
-    SuperVersion* super_version = cfd->GetReferencedSuperVersion(this);
+    SuperVersion* super_version =
+        ingestion_jobs[0].GetColumnFamilyData()->GetReferencedSuperVersion(
+            this);
    Status es = ingestion_jobs[0].Prepare(
        args[0].external_files, args[0].files_checksums,
        args[0].files_checksum_func_names, args[0].file_temperature,
@@ -6007,8 +6055,7 @@ Status DBImpl::IngestExternalFiles(
    bool at_least_one_cf_need_flush = false;
    std::vector<bool> need_flush(num_cfs, false);
    for (size_t i = 0; i != num_cfs; ++i) {
-      auto* cfd =
-          static_cast<ColumnFamilyHandleImpl*>(args[i].column_family)->cfd();
+      auto* cfd = ingestion_jobs[i].GetColumnFamilyData();
      if (cfd->IsDropped()) {
        // TODO (yanqin) investigate whether we should abort ingestion or
        // proceed with other non-dropped column families.
@@ -6040,12 +6087,10 @@ Status DBImpl::IngestExternalFiles(
        for (size_t i = 0; i != num_cfs; ++i) {
          if (need_flush[i]) {
            mutex_.Unlock();
-            auto* cfd =
-                static_cast<ColumnFamilyHandleImpl*>(args[i].column_family)
-                    ->cfd();
-            status = FlushMemTable(cfd, flush_opts,
-                                   FlushReason::kExternalFileIngestion,
-                                   true /* entered_write_thread */);
+            status =
+                FlushMemTable(ingestion_jobs[i].GetColumnFamilyData(),
+                              flush_opts, FlushReason::kExternalFileIngestion,
+                              true /* entered_write_thread */);
            mutex_.Lock();
            if (!status.ok()) {
              break;
@@ -6053,6 +6098,13 @@ Status DBImpl::IngestExternalFiles(
          }
        }
      }
+      if (status.ok()) {
+        for (size_t i = 0; i != num_cfs; ++i) {
+          if (immutable_db_options_.atomic_flush || need_flush[i]) {
+            ingestion_jobs[i].SetFlushedBeforeRun();
+          }
+        }
+      }
    }
    // Run ingestion jobs.
    if (status.ok()) {
@@ -6073,11 +6125,8 @@ Status DBImpl::IngestExternalFiles(
      autovector<autovector<VersionEdit*>> edit_lists;
      uint32_t num_entries = 0;
      for (size_t i = 0; i != num_cfs; ++i) {
-        auto* cfd =
-            static_cast<ColumnFamilyHandleImpl*>(args[i].column_family)->cfd();
-        if (cfd->IsDropped()) {
-          continue;
-        }
+        auto* cfd = ingestion_jobs[i].GetColumnFamilyData();
+        assert(!cfd->IsDropped());
        cfds_to_commit.push_back(cfd);
        mutable_cf_options_list.push_back(cfd->GetLatestMutableCFOptions());
        autovector<VersionEdit*> edit_list;
@@ -6127,20 +6176,16 @@ Status DBImpl::IngestExternalFiles(

    if (status.ok()) {
      for (size_t i = 0; i != num_cfs; ++i) {
-        auto* cfd =
-            static_cast<ColumnFamilyHandleImpl*>(args[i].column_family)->cfd();
-        if (!cfd->IsDropped()) {
-          InstallSuperVersionAndScheduleWork(cfd, &sv_ctxs[i],
-                                             *cfd->GetLatestMutableCFOptions());
+        auto* cfd = ingestion_jobs[i].GetColumnFamilyData();
+        assert(!cfd->IsDropped());
+        InstallSuperVersionAndScheduleWork(cfd, &sv_ctxs[i],
+                                           *cfd->GetLatestMutableCFOptions());
 #ifndef NDEBUG
-          if (0 == i && num_cfs > 1) {
-            TEST_SYNC_POINT(
-                "DBImpl::IngestExternalFiles:InstallSVForFirstCF:0");
-            TEST_SYNC_POINT(
-                "DBImpl::IngestExternalFiles:InstallSVForFirstCF:1");
-          }
-#endif  // !NDEBUG
+        if (0 == i && num_cfs > 1) {
+          TEST_SYNC_POINT("DBImpl::IngestExternalFiles:InstallSVForFirstCF:0");
+          TEST_SYNC_POINT("DBImpl::IngestExternalFiles:InstallSVForFirstCF:1");
        }
+#endif  // !NDEBUG
      }
    } else if (versions_->io_status().IsIOError()) {
      // Error while writing to MANIFEST.
@@ -6182,8 +6227,7 @@ Status DBImpl::IngestExternalFiles(
  }
  if (status.ok()) {
    for (size_t i = 0; i != num_cfs; ++i) {
-      auto* cfd =
-          static_cast<ColumnFamilyHandleImpl*>(args[i].column_family)->cfd();
+      auto* cfd = ingestion_jobs[i].GetColumnFamilyData();
      if (!cfd->IsDropped()) {
        NotifyOnExternalFileIngested(cfd, ingestion_jobs[i]);
      }
@@ -48,6 +48,7 @@
 #include "db/write_controller.h"
 #include "db/write_thread.h"
 #include "logging/event_logger.h"
+#include "memtable/wbwi_memtable.h"
 #include "monitoring/instrumented_mutex.h"
 #include "options/db_options.h"
 #include "port/port.h"
@@ -60,6 +61,7 @@
 #include "rocksdb/transaction_log.h"
 #include "rocksdb/user_write_callback.h"
 #include "rocksdb/utilities/replayer.h"
+#include "rocksdb/utilities/write_batch_with_index.h"
 #include "rocksdb/write_buffer_manager.h"
 #include "table/merging_iterator.h"
 #include "util/autovector.h"
@@ -363,12 +365,10 @@ class DBImpl : public DB {
  const Snapshot* GetSnapshot() override;
  void ReleaseSnapshot(const Snapshot* snapshot) override;

-  // EXPERIMENTAL
  std::unique_ptr<Iterator> NewCoalescingIterator(
      const ReadOptions& options,
      const std::vector<ColumnFamilyHandle*>& column_families) override;

-  // EXPERIMENTAL
  std::unique_ptr<AttributeGroupIterator> NewAttributeGroupIterator(
      const ReadOptions& options,
      const std::vector<ColumnFamilyHandle*>& column_families) override;
@@ -482,7 +482,8 @@ class DBImpl : public DB {

  Status GetDbIdentity(std::string& identity) const override;

-  virtual Status GetDbIdentityFromIdentityFile(std::string* identity) const;
+  virtual Status GetDbIdentityFromIdentityFile(const IOOptions& opts,
+                                               std::string* identity) const;

  Status GetDbSessionId(std::string& session_id) const override;

@@ -1239,9 +1240,14 @@ class DBImpl : public DB {
  static Status TEST_ValidateOptions(const DBOptions& db_options) {
    return ValidateOptions(db_options);
  }
-
 #endif  // NDEBUG

+  // In certain configurations, verify that the table/blob file cache only
+  // contains entries for live files, to check for effective leaks of open
+  // files. This can only be called when purging of obsolete files has
+  // "settled," such as during parts of DB Close().
+  void TEST_VerifyNoObsoleteFilesCached(bool db_mutex_already_held) const;
+
  // persist stats to column family "_persistent_stats"
  void PersistStats();

@@ -1463,7 +1469,8 @@ class DBImpl : public DB {
  // The following two functions can only be called when:
  // 1. WriteThread::Writer::EnterUnbatched() is used.
  // 2. db_mutex is NOT held
-  Status RenameTempFileToOptionsFile(const std::string& file_name);
+  Status RenameTempFileToOptionsFile(const std::string& file_name,
+                                     bool is_remote_compaction_enabled);
  Status DeleteObsoleteOptionsFiles();

  void NotifyOnManualFlushScheduled(autovector<ColumnFamilyData*> cfds,
@@ -1502,6 +1509,23 @@ class DBImpl : public DB {

  void EraseThreadStatusDbInfo() const;

+  // For CFs that has updates in `wbwi`, their memtable will be switched,
+  // and `wbwi` will be added as the latest immutable memtable.
+  //
+  // REQUIRES: this thread is currently at the front of the main writer queue.
+  // @param prep_log refers to the WAL that contains prepare record
+  // for the transaction based on wbwi.
+  // @param assigned_seqno Sequence numbers for the ingested memtable.
+  // @param last_seqno the value of versions_->LastSequence() after the write
+  // ingests `wbwi` is done.
+  // @param memtable_updated Whether the same write that ingests wbwi has
+  // updated memtable. This is useful for determining whether to set bg
+  // error when IngestWBWI fails.
+  Status IngestWBWI(std::shared_ptr<WriteBatchWithIndex> wbwi,
+                    const WBWIMemTable::SeqnoRange& assigned_seqno,
+                    uint64_t min_prep_log, SequenceNumber last_seqno,
+                    bool memtable_updated, bool ignore_missing_cf);
+
  // If disable_memtable is set the application logic must guarantee that the
  // batch will still be skipped from memtable during the recovery. An excption
  // to this is seq_per_batch_ mode, in which since each batch already takes one
@@ -1517,6 +1541,16 @@ class DBImpl : public DB {
  // batch_cnt is expected to be non-zero in seq_per_batch mode and
  // indicates the number of sub-patches. A sub-patch is a subset of the write
  // batch that does not have duplicate keys.
+  // `callback` is called before WAL write.
+  // See more in comment above WriteCallback::Callback().
+  // pre_release_callback is called after WAL write and before memtable write.
+  // See more in comment above PreReleaseCallback::Callback().
+  // post_memtable_callback is called after memtable write but before publishing
+  // the sequence number to readers.
+  //
+  // The main write queue. This is the only write queue that updates
+  // LastSequence. When using one write queue, the same sequence also indicates
+  // the last published sequence.
  Status WriteImpl(const WriteOptions& options, WriteBatch* updates,
                   WriteCallback* callback = nullptr,
                   UserWriteCallback* user_write_cb = nullptr,
@@ -1524,7 +1558,9 @@ class DBImpl : public DB {
                   bool disable_memtable = false, uint64_t* seq_used = nullptr,
                   size_t batch_cnt = 0,
                   PreReleaseCallback* pre_release_callback = nullptr,
-                   PostMemTableCallback* post_memtable_callback = nullptr);
+                   PostMemTableCallback* post_memtable_callback = nullptr,
+                   std::shared_ptr<WriteBatchWithIndex> wbwi = nullptr,
+                   uint64_t min_prep_log = 0);

  Status PipelinedWriteImpl(const WriteOptions& options, WriteBatch* updates,
                            WriteCallback* callback = nullptr,
@@ -1587,7 +1623,7 @@ class DBImpl : public DB {
  // Read/create DB identity file (as appropriate), and write DB ID to
  // version_edit if provided.
  Status SetupDBId(const WriteOptions& write_options, bool read_only,
-                   bool is_new_db, VersionEdit* version_edit);
+                   bool is_new_db, bool is_retry, VersionEdit* version_edit);
  // Assign db_id_ and write DB ID to version_edit if provided.
  void SetDBId(std::string&& id, bool read_only, VersionEdit* version_edit);

@@ -1704,7 +1740,7 @@ class DBImpl : public DB {

  struct WriteContext {
    SuperVersionContext superversion_context;
-    autovector<MemTable*> memtables_to_free_;
+    autovector<ReadOnlyMemTable*> memtables_to_free_;

    explicit WriteContext(bool create_superversion = false)
        : superversion_context(create_superversion) {}
@@ -2046,7 +2082,21 @@ class DBImpl : public DB {

  Status TrimMemtableHistory(WriteContext* context);

-  Status SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context);
+  // Switches the current live memtable to immutable/read-only memtable.
+  // A new WAL is created if the current WAL is not empty.
+  // If `new_imm` is not nullptr, it will be added as the newest immutable
+  // memtable, if and only if OK status is returned.
+  // `last_seqno` needs to be provided if `new_imm` is not nullptr. It is
+  // the value of versions_->LastSequence() after the write that ingests new_imm
+  // is done.
+  //
+  // REQUIRES: mutex_ is held
+  // REQUIRES: this thread is currently at the front of the writer queue
+  // REQUIRES: this thread is currently at the front of the 2nd writer queue if
+  // two_write_queues_ is true (This is to simplify the reasoning.)
+  Status SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
+                        ReadOnlyMemTable* new_imm = nullptr,
+                        SequenceNumber last_seqno = 0);

  // Select and output column families qualified for atomic flush in
  // `selected_cfds`. If `provided_candidate_cfds` is non-empty, it will be used
@@ -2210,8 +2260,6 @@ class DBImpl : public DB {
  void TrackOrUntrackFiles(const std::vector<std::string>& existing_data_files,
                           bool track);

-  ColumnFamilyData* GetColumnFamilyDataByName(const std::string& cf_name);
-
  void MaybeScheduleFlushOrCompaction();

  struct FlushRequest {
@@ -3002,7 +3050,8 @@ CompressionType GetCompressionFlush(const ImmutableCFOptions& ioptions,
 VersionEdit GetDBRecoveryEditForObsoletingMemTables(
    VersionSet* vset, const ColumnFamilyData& cfd,
    const autovector<VersionEdit*>& edit_list,
-    const autovector<MemTable*>& memtables, LogsWithPrepTracker* prep_tracker);
+    const autovector<ReadOnlyMemTable*>& memtables,
+    LogsWithPrepTracker* prep_tracker);

 // Return the earliest log file to keep after the memtable flush is
 // finalized.
@@ -3013,13 +3062,13 @@ VersionEdit GetDBRecoveryEditForObsoletingMemTables(
 uint64_t PrecomputeMinLogNumberToKeep2PC(
    VersionSet* vset, const ColumnFamilyData& cfd_to_flush,
    const autovector<VersionEdit*>& edit_list,
-    const autovector<MemTable*>& memtables_to_flush,
+    const autovector<ReadOnlyMemTable*>& memtables_to_flush,
    LogsWithPrepTracker* prep_tracker);
 // For atomic flush.
 uint64_t PrecomputeMinLogNumberToKeep2PC(
    VersionSet* vset, const autovector<ColumnFamilyData*>& cfds_to_flush,
    const autovector<autovector<VersionEdit*>>& edit_lists,
-    const autovector<const autovector<MemTable*>*>& memtables_to_flush,
+    const autovector<const autovector<ReadOnlyMemTable*>*>& memtables_to_flush,
    LogsWithPrepTracker* prep_tracker);

 // In non-2PC mode, WALs with log number < the returned number can be
@@ -3036,11 +3085,11 @@ uint64_t PrecomputeMinLogNumberToKeepNon2PC(
 // will not depend on any WAL file. nullptr means no memtable is being flushed.
 // The function is only applicable to 2pc mode.
 uint64_t FindMinPrepLogReferencedByMemTable(
-    VersionSet* vset, const autovector<MemTable*>& memtables_to_flush);
+    VersionSet* vset, const autovector<ReadOnlyMemTable*>& memtables_to_flush);
 // For atomic flush.
 uint64_t FindMinPrepLogReferencedByMemTable(
    VersionSet* vset,
-    const autovector<const autovector<MemTable*>*>& memtables_to_flush);
+    const autovector<const autovector<ReadOnlyMemTable*>*>& memtables_to_flush);

 // Fix user-supplied options to be reasonable
 template <class T, class V>
@@ -753,7 +753,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(

  if (s.ok()) {
    autovector<ColumnFamilyData*> tmp_cfds;
-    autovector<const autovector<MemTable*>*> mems_list;
+    autovector<const autovector<ReadOnlyMemTable*>*> mems_list;
    autovector<const MutableCFOptions*> mutable_cf_options_list;
    autovector<FileMetaData*> tmp_file_meta;
    autovector<std::list<std::unique_ptr<FlushJobInfo>>*>
@@ -1457,11 +1457,6 @@ Status DBImpl::CompactFilesImpl(
    input_set.insert(TableFileNameToNumber(file_name));
  }

-  ColumnFamilyMetaData cf_meta;
-  // TODO(yhchiang): can directly use version here if none of the
-  // following functions call is pluggable to external developers.
-  version->GetColumnFamilyMetaData(&cf_meta);
-
  if (output_path_id < 0) {
    if (cfd->ioptions()->cf_paths.size() == 1U) {
      output_path_id = 0;
@@ -1482,7 +1477,7 @@ Status DBImpl::CompactFilesImpl(

  std::vector<CompactionInputFiles> input_files;
  Status s = cfd->compaction_picker()->SanitizeAndConvertCompactionInputFiles(
-      &input_set, cf_meta, output_level, version->storage_info(), &input_files);
+      &input_set, output_level, version, &input_files);
  TEST_SYNC_POINT(
      "DBImpl::CompactFilesImpl::PostSanitizeAndConvertCompactionInputFiles");
  if (!s.ok()) {
@@ -3991,7 +3986,10 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
      // Sanity checking that compaction files are freed.
      for (size_t i = 0; i < c->num_input_levels(); i++) {
        for (size_t j = 0; j < c->inputs(i)->size(); j++) {
-          assert(!c->input(i, j)->being_compacted);
+          // When status is not OK, compaction's result installation failed and
+          // no new Version installed. The files could have been released and
+          // picked up again by other compaction attempts.
+          assert(!c->input(i, j)->being_compacted || !status.ok());
        }
      }
      std::unordered_set<Compaction*>* cip = c->column_family_data()
@@ -6,9 +6,10 @@
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
-
 #ifndef NDEBUG
+#include <iostream>

+#include "db/blob/blob_file_cache.h"
 #include "db/column_family.h"
 #include "db/db_impl/db_impl.h"
 #include "db/error_handler.h"
@@ -257,7 +258,7 @@ size_t DBImpl::TEST_LogsWithPrepSize() {
 }

 uint64_t DBImpl::TEST_FindMinPrepLogReferencedByMemTable() {
-  autovector<MemTable*> empty_list;
+  autovector<ReadOnlyMemTable*> empty_list;
  return FindMinPrepLogReferencedByMemTable(versions_.get(), empty_list);
 }

@@ -321,5 +322,70 @@ size_t DBImpl::TEST_EstimateInMemoryStatsHistorySize() const {
  InstrumentedMutexLock l(&const_cast<DBImpl*>(this)->stats_history_mutex_);
  return EstimateInMemoryStatsHistorySize();
 }
+
+void DBImpl::TEST_VerifyNoObsoleteFilesCached(
+    bool db_mutex_already_held) const {
+  // This check is somewhat expensive and obscure to make a part of every
+  // unit test in every build variety. Thus, we only enable it for ASAN builds.
+  if (!kMustFreeHeapAllocations) {
+    return;
+  }
+
+  std::optional<InstrumentedMutexLock> l;
+  if (db_mutex_already_held) {
+    mutex_.AssertHeld();
+  } else {
+    l.emplace(&mutex_);
+  }
+
+  if (!opened_successfully_) {
+    // We don't need to pro-actively clean up open files during DB::Open()
+    // if we know we are about to fail and clean up in Close().
+    return;
+  }
+  if (disable_delete_obsolete_files_ > 0) {
+    // For better or worse, DB::Close() is allowed with deletions disabled.
+    // Since we generally associate clean-up of open files with deleting them,
+    // we allow "obsolete" open files when deletions are disabled.
+    return;
+  }
+
+  // Live and "quarantined" files are allowed to be open in table cache
+  std::set<uint64_t> live_and_quar_files;
+  for (auto cfd : *versions_->GetColumnFamilySet()) {
+    if (cfd->IsDropped()) {
+      continue;
+    }
+    // Iterate over live versions
+    Version* current = cfd->current();
+    Version* ver = current;
+    do {
+      // Sneakily add both SST and blob files to the same list
+      std::vector<uint64_t> live_files_vec;
+      ver->AddLiveFiles(&live_files_vec, &live_files_vec);
+      live_and_quar_files.insert(live_files_vec.begin(), live_files_vec.end());
+
+      ver = ver->Next();
+    } while (ver != current);
+  }
+  {
+    const auto& quar_files = error_handler_.GetFilesToQuarantine();
+    live_and_quar_files.insert(quar_files.begin(), quar_files.end());
+  }
+  auto fn = [&live_and_quar_files](const Slice& key, Cache::ObjectPtr, size_t,
+                                   const Cache::CacheItemHelper*) {
+    // See TableCache and BlobFileCache
+    assert(key.size() == sizeof(uint64_t));
+    uint64_t file_number;
+    GetUnaligned(reinterpret_cast<const uint64_t*>(key.data()), &file_number);
+    // Assert file is in live/quarantined set
+    if (live_and_quar_files.find(file_number) == live_and_quar_files.end()) {
+      std::cerr << "File " << file_number << " is not live nor quarantined"
+                << std::endl;
+      assert(false);
+    }
+  };
+  table_cache_->ApplyToAllEntries(fn, {});
+}
 }  // namespace ROCKSDB_NAMESPACE
 #endif  // NDEBUG
@@ -449,14 +449,8 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
      // File is being deleted (actually obsolete)
      auto number = file.metadata->fd.GetNumber();
      candidate_files.emplace_back(MakeTableFileName(number), file.path);
-      if (handle == nullptr) {
-        // For files not "pinned" in table cache
-        handle = TableCache::Lookup(table_cache_.get(), number);
-      }
-      if (handle) {
-        TableCache::ReleaseObsolete(table_cache_.get(), handle,
-                                    file.uncache_aggressiveness);
-      }
+      TableCache::ReleaseObsolete(table_cache_.get(), number, handle,
+                                  file.uncache_aggressiveness);
    }
    file.DeleteMetadata();
  }
@@ -572,9 +566,17 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
      case kTableFile:
        // If the second condition is not there, this makes
        // DontDeletePendingOutputs fail
+        // FIXME: but should NOT keep if it came from sst_delete_files?
        keep = (sst_live_set.find(number) != sst_live_set.end()) ||
               number >= state.min_pending_output;
        if (!keep) {
+          // NOTE: sometimes redundant (if came from sst_delete_files)
+          // We don't know which column family is applicable here so we don't
+          // know what uncache_aggressiveness would be used with
+          // ReleaseObsolete(). Anyway, obsolete files ideally go into
+          // sst_delete_files for better/quicker handling, and this is just a
+          // backstop.
+          TableCache::Evict(table_cache_.get(), number);
          files_to_del.insert(number);
        }
        break;
@@ -739,7 +741,8 @@ void DBImpl::DeleteObsoleteFiles() {
 VersionEdit GetDBRecoveryEditForObsoletingMemTables(
    VersionSet* vset, const ColumnFamilyData& cfd,
    const autovector<VersionEdit*>& edit_list,
-    const autovector<MemTable*>& memtables, LogsWithPrepTracker* prep_tracker) {
+    const autovector<ReadOnlyMemTable*>& memtables,
+    LogsWithPrepTracker* prep_tracker) {
  VersionEdit wal_deletion_edit;
  uint64_t min_wal_number_to_keep = 0;
  assert(edit_list.size() > 0);
@@ -769,12 +772,12 @@ VersionEdit GetDBRecoveryEditForObsoletingMemTables(
 }

 uint64_t FindMinPrepLogReferencedByMemTable(
-    VersionSet* vset, const autovector<MemTable*>& memtables_to_flush) {
+    VersionSet* vset, const autovector<ReadOnlyMemTable*>& memtables_to_flush) {
  uint64_t min_log = 0;

  // we must look through the memtables for two phase transactions
  // that have been committed but not yet flushed
-  std::unordered_set<MemTable*> memtables_to_flush_set(
+  std::unordered_set<ReadOnlyMemTable*> memtables_to_flush_set(
      memtables_to_flush.begin(), memtables_to_flush.end());
  for (auto loop_cfd : *vset->GetColumnFamilySet()) {
    if (loop_cfd->IsDropped()) {
@@ -799,12 +802,12 @@ uint64_t FindMinPrepLogReferencedByMemTable(
 }

 uint64_t FindMinPrepLogReferencedByMemTable(
-    VersionSet* vset,
-    const autovector<const autovector<MemTable*>*>& memtables_to_flush) {
+    VersionSet* vset, const autovector<const autovector<ReadOnlyMemTable*>*>&
+                          memtables_to_flush) {
  uint64_t min_log = 0;

-  std::unordered_set<MemTable*> memtables_to_flush_set;
-  for (const autovector<MemTable*>* memtables : memtables_to_flush) {
+  std::unordered_set<ReadOnlyMemTable*> memtables_to_flush_set;
+  for (const autovector<ReadOnlyMemTable*>* memtables : memtables_to_flush) {
    memtables_to_flush_set.insert(memtables->begin(), memtables->end());
  }
  for (auto loop_cfd : *vset->GetColumnFamilySet()) {
@@ -896,7 +899,7 @@ uint64_t PrecomputeMinLogNumberToKeepNon2PC(
 uint64_t PrecomputeMinLogNumberToKeep2PC(
    VersionSet* vset, const ColumnFamilyData& cfd_to_flush,
    const autovector<VersionEdit*>& edit_list,
-    const autovector<MemTable*>& memtables_to_flush,
+    const autovector<ReadOnlyMemTable*>& memtables_to_flush,
    LogsWithPrepTracker* prep_tracker) {
  assert(vset != nullptr);
  assert(prep_tracker != nullptr);
@@ -937,7 +940,7 @@ uint64_t PrecomputeMinLogNumberToKeep2PC(
 uint64_t PrecomputeMinLogNumberToKeep2PC(
    VersionSet* vset, const autovector<ColumnFamilyData*>& cfds_to_flush,
    const autovector<autovector<VersionEdit*>>& edit_lists,
-    const autovector<const autovector<MemTable*>*>& memtables_to_flush,
+    const autovector<const autovector<ReadOnlyMemTable*>*>& memtables_to_flush,
    LogsWithPrepTracker* prep_tracker) {
  assert(vset != nullptr);
  assert(prep_tracker != nullptr);
@@ -980,7 +983,8 @@ void DBImpl::SetDBId(std::string&& id, bool read_only,
 }

 Status DBImpl::SetupDBId(const WriteOptions& write_options, bool read_only,
-                         bool is_new_db, VersionEdit* version_edit) {
+                         bool is_new_db, bool is_retry,
+                         VersionEdit* version_edit) {
  Status s;
  if (!is_new_db) {
    // Check for the IDENTITY file and create it if not there or
@@ -988,7 +992,11 @@ Status DBImpl::SetupDBId(const WriteOptions& write_options, bool read_only,
    std::string db_id_in_file;
    s = fs_->FileExists(IdentityFileName(dbname_), IOOptions(), nullptr);
    if (s.ok()) {
-      s = GetDbIdentityFromIdentityFile(&db_id_in_file);
+      IOOptions opts;
+      if (is_retry) {
+        opts.verify_and_reconstruct_read = true;
+      }
+      s = GetDbIdentityFromIdentityFile(opts, &db_id_in_file);
      if (s.ok() && !db_id_in_file.empty()) {
        if (db_id_.empty()) {
          // Loaded from file and wasn't already known from manifest
@@ -301,7 +301,7 @@ Status DBImpl::NewDB(std::vector<std::string>* new_filenames) {
  VersionEdit new_db_edit;
  const WriteOptions write_options(Env::IOActivity::kDBOpen);
  Status s = SetupDBId(write_options, /*read_only=*/false, /*is_new_db=*/true,
-                       &new_db_edit);
+                       /*is_retry=*/false, &new_db_edit);
  if (!s.ok()) {
    return s;
  }
@@ -575,6 +575,7 @@ Status DBImpl::Recover(
  }
  if (s.ok() && !read_only) {
    for (auto cfd : *versions_->GetColumnFamilySet()) {
+      auto& moptions = *cfd->GetLatestMutableCFOptions();
      // Try to trivially move files down the LSM tree to start from bottommost
      // level when level_compaction_dynamic_level_bytes is enabled. This should
      // only be useful when user is migrating to turning on this option.
@@ -592,14 +593,14 @@ Status DBImpl::Recover(
      if (cfd->ioptions()->compaction_style ==
              CompactionStyle::kCompactionStyleLevel &&
          cfd->ioptions()->level_compaction_dynamic_level_bytes &&
-          !cfd->GetLatestMutableCFOptions()->disable_auto_compactions) {
+          !moptions.disable_auto_compactions) {
        int to_level = cfd->ioptions()->num_levels - 1;
        // last level is reserved
        // allow_ingest_behind does not support Level Compaction,
        // and per_key_placement can have infinite compaction loop for Level
        // Compaction. Adjust to_level here just to be safe.
        if (cfd->ioptions()->allow_ingest_behind ||
-            cfd->ioptions()->preclude_last_level_data_seconds > 0) {
+            moptions.preclude_last_level_data_seconds > 0) {
          to_level -= 1;
        }
        // Whether this column family has a level trivially moved
@@ -675,11 +676,11 @@ Status DBImpl::Recover(
    // Already set up DB ID in NewDB
  } else if (immutable_db_options_.write_dbid_to_manifest && recovery_ctx) {
    VersionEdit edit;
-    s = SetupDBId(write_options, read_only, is_new_db, &edit);
+    s = SetupDBId(write_options, read_only, is_new_db, is_retry, &edit);
    recovery_ctx->UpdateVersionEdits(
        versions_->GetColumnFamilySet()->GetDefault(), edit);
  } else {
-    s = SetupDBId(write_options, read_only, is_new_db, nullptr);
+    s = SetupDBId(write_options, read_only, is_new_db, is_retry, nullptr);
  }
  assert(!s.ok() || !db_id_.empty());
  ROCKS_LOG_INFO(immutable_db_options_.info_log, "DB ID: %s\n", db_id_.c_str());
@@ -1274,7 +1275,8 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& wal_numbers,
          reader.GetRecordedTimestampSize();
      status = HandleWriteBatchTimestampSizeDifference(
          &batch, running_ts_sz, record_ts_sz,
-          TimestampSizeConsistencyMode::kReconcileInconsistency, &new_batch);
+          TimestampSizeConsistencyMode::kReconcileInconsistency, seq_per_batch_,
+          batch_per_txn_, &new_batch);
      if (!status.ok()) {
        return status;
      }
@@ -1371,6 +1373,9 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& wal_numbers,
        }
      }
    }
+    ROCKS_LOG_INFO(immutable_db_options_.info_log,
+                   "Recovered to log #%" PRIu64 " seq #%" PRIu64, wal_number,
+                   *next_sequence);

    if (!status.ok() || old_log_record) {
      if (status.IsNotSupported()) {
@@ -1403,10 +1408,6 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& wal_numbers,
        if (corrupted_wal_found != nullptr) {
          *corrupted_wal_found = true;
        }
-        ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                       "Point in time recovered to log #%" PRIu64
-                       " seq #%" PRIu64,
-                       wal_number, *next_sequence);
      } else {
        assert(immutable_db_options_.wal_recovery_mode ==
                   WALRecoveryMode::kTolerateCorruptedTailRecords ||
@@ -1679,7 +1680,8 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
                  ro, /*seqno_to_time_mapping=*/nullptr, &arena,
                  /*prefix_extractor=*/nullptr, ts_sz)
            : mem->NewIterator(ro, /*seqno_to_time_mapping=*/nullptr, &arena,
-                               /*prefix_extractor=*/nullptr));
+                               /*prefix_extractor=*/nullptr,
+                               /*for_flush=*/true));
    ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
                    "[%s] [WriteLevel0TableForRecovery]"
                    " Level-0 table #%" PRIu64 ": started",
@@ -1735,10 +1737,11 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
          cfd->internal_comparator(), cfd->internal_tbl_prop_coll_factories(),
          GetCompressionFlush(*cfd->ioptions(), mutable_cf_options),
          mutable_cf_options.compression_opts, cfd->GetID(), cfd->GetName(),
-          0 /* level */, false /* is_bottommost */,
-          TableFileCreationReason::kRecovery, 0 /* oldest_key_time */,
-          0 /* file_creation_time */, db_id_, db_session_id_,
-          0 /* target_file_size */, meta.fd.GetNumber(), kMaxSequenceNumber);
+          0 /* level */, current_time /* newest_key_time */,
+          false /* is_bottommost */, TableFileCreationReason::kRecovery,
+          0 /* oldest_key_time */, 0 /* file_creation_time */, db_id_,
+          db_session_id_, 0 /* target_file_size */, meta.fd.GetNumber(),
+          kMaxSequenceNumber);
      Version* version = cfd->current();
      version->Ref();
      uint64_t num_input_entries = 0;
@@ -1768,7 +1771,7 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
        s = io_s;
      }

-      uint64_t total_num_entries = mem->num_entries();
+      uint64_t total_num_entries = mem->NumEntries();
      if (s.ok() && total_num_entries != num_input_entries) {
        std::string msg = "Expected " + std::to_string(total_num_entries) +
                          " entries in memtable, but read " +
@@ -8,7 +8,7 @@
 #include "db/arena_wrapped_db_iter.h"
 #include "db/db_impl/compacted_db_impl.h"
 #include "db/db_impl/db_impl.h"
-#include "db/db_iter.h"
+#include "db/manifest_ops.h"
 #include "db/merge_context.h"
 #include "logging/logging.h"
 #include "monitoring/perf_context_imp.h"
@@ -265,8 +265,8 @@ Status OpenForReadOnlyCheckExistence(const DBOptions& db_options,
    const std::shared_ptr<FileSystem>& fs = db_options.env->GetFileSystem();
    std::string manifest_path;
    uint64_t manifest_file_number;
-    s = VersionSet::GetCurrentManifestPath(dbname, fs.get(), &manifest_path,
-                                           &manifest_file_number);
+    s = GetCurrentManifestPath(dbname, fs.get(), /*is_retry=*/false,
+                               &manifest_path, &manifest_file_number);
  } else {
    // Historic behavior that doesn't necessarily make sense
    s = db_options.env->CreateDirIfMissing(dbname);
@@ -233,7 +233,8 @@ Status DBImplSecondary::RecoverLogFiles(
          reader->GetRecordedTimestampSize();
      status = HandleWriteBatchTimestampSizeDifference(
          &batch, running_ts_sz, record_ts_sz,
-          TimestampSizeConsistencyMode::kVerifyConsistency);
+          TimestampSizeConsistencyMode::kVerifyConsistency, seq_per_batch_,
+          batch_per_txn_);
      if (!status.ok()) {
        break;
      }
@@ -247,9 +248,7 @@ Status DBImplSecondary::RecoverLogFiles(
          if (cfd == nullptr) {
            continue;
          }
-          if (cfds_changed->count(cfd) == 0) {
-            cfds_changed->insert(cfd);
-          }
+          cfds_changed->insert(cfd);
          const std::vector<FileMetaData*>& l0_files =
              cfd->current()->storage_info()->LevelFiles(0);
          SequenceNumber seq =
@@ -957,6 +956,10 @@ Status DB::OpenAndCompact(
  config_options.env = override_options.env;
  std::vector<ColumnFamilyDescriptor> all_column_families;

+  TEST_SYNC_POINT_CALLBACK(
+      "DBImplSecondary::OpenAndCompact::BeforeLoadingOptions:0",
+      &compaction_input.options_file_number);
+  TEST_SYNC_POINT("DBImplSecondary::OpenAndCompact::BeforeLoadingOptions:1");
  std::string options_file_name =
      OptionsFileName(name, compaction_input.options_file_number);

@@ -12,6 +12,7 @@
 #include "db/error_handler.h"
 #include "db/event_helpers.h"
 #include "logging/logging.h"
+#include "memtable/wbwi_memtable.h"
 #include "monitoring/perf_context_imp.h"
 #include "options/options_helper.h"
 #include "test_util/sync_point.h"
@@ -189,16 +190,137 @@ Status DBImpl::WriteWithCallback(const WriteOptions& write_options,
  return s;
 }

-// The main write queue. This is the only write queue that updates LastSequence.
-// When using one write queue, the same sequence also indicates the last
-// published sequence.
+Status DBImpl::IngestWBWI(std::shared_ptr<WriteBatchWithIndex> wbwi,
+                          const WBWIMemTable::SeqnoRange& assigned_seqno,
+                          uint64_t prep_log,
+                          SequenceNumber last_seqno_after_ingest,
+                          bool memtable_updated, bool ignore_missing_cf) {
+  // Keys in new memtable have seqno > last_seqno_after_ingest >= keys in wbwi.
+  assert(assigned_seqno.upper_bound <= last_seqno_after_ingest);
+  // Keys in the current memtable have seqno <= LastSequence() < keys in wbwi.
+  assert(assigned_seqno.lower_bound > versions_->LastSequence());
+  autovector<ReadOnlyMemTable*> memtables;
+  autovector<ColumnFamilyData*> cfds;
+  InstrumentedMutexLock lock(&mutex_);
+  ColumnFamilySet* cf_set = versions_->GetColumnFamilySet();
+
+  // Create WBWIMemTables
+  for (const auto [cf_id, stat] : wbwi->GetCFStats()) {
+    ColumnFamilyData* cfd = cf_set->GetColumnFamily(cf_id);
+    if (!cfd) {
+      if (ignore_missing_cf) {
+        continue;
+      }
+      for (auto mem : memtables) {
+        mem->Unref();
+        delete mem;
+      }
+      for (auto cfd_ptr : cfds) {
+        cfd_ptr->UnrefAndTryDelete();
+      }
+      Status s = Status::InvalidArgument(
+          "Invalid column family id from WriteBatchWithIndex: " +
+          std::to_string(cf_id));
+      if (memtable_updated) {
+        s = Status::Corruption(
+            "Part of the write batch is applied. Memtable is in a inconsistent "
+            "state. " +
+            s.ToString());
+        error_handler_.SetBGError(s, BackgroundErrorReason::kMemTable);
+      }
+
+      return s;
+    }
+    WBWIMemTable* wbwi_memtable =
+        new WBWIMemTable(wbwi, cfd->user_comparator(), cf_id, cfd->ioptions(),
+                         cfd->GetLatestMutableCFOptions(), stat);
+    wbwi_memtable->Ref();
+    wbwi_memtable->AssignSequenceNumbers(assigned_seqno);
+    // This is needed to keep the WAL that contains Prepare alive until
+    // committed data in this memtable is persisted.
+    wbwi_memtable->SetMinPrepLog(prep_log);
+    memtables.push_back(wbwi_memtable);
+    cfd->Ref();
+    cfds.push_back(cfd);
+  }
+
+  // Stop writes to the DB by entering both write threads
+  WriteThread::Writer nonmem_w;
+  if (two_write_queues_) {
+    nonmem_write_thread_.EnterUnbatched(&nonmem_w, &mutex_);
+  }
+  WaitForPendingWrites();
+
+  // Switch memtable and add WBWIMemTables
+  Status s;
+  for (size_t i = 0; i < memtables.size(); ++i) {
+    assert(!immutable_db_options_.atomic_flush);
+    // NOTE: to support atomic flush, need to call
+    // SelectColumnFamiliesForAtomicFlush()
+    WriteContext write_context;
+    // TODO: not switch on empty memtable, may need to update metadata
+    //   like NextLogNumber(), earliest_seqno and memtable id.
+    s = SwitchMemtable(cfds[i], &write_context, memtables[i],
+                       last_seqno_after_ingest);
+    if (!s.ok()) {
+      // SwitchMemtable() can only fail if a new WAL is to be created, this
+      // should only happen for the first call to SwitchMemtable(). log will
+      // be empty and no new WAL is created for the rest of the calls.
+      assert(i == 0);
+      if (i != 0 || memtable_updated) {
+        // escalate error to non-recoverable
+        s = Status::Corruption(
+            "Part of the write batch is applied. Memtable is in a inconsistent "
+            "state. " +
+            s.ToString());
+        error_handler_.SetBGError(s, BackgroundErrorReason::kMemTable);
+      } else {
+        // SwitchMemtable() already sets appropriate bg error
+      }
+      for (size_t j = i; j < memtables.size(); j++) {
+        memtables[j]->Unref();
+        delete memtables[j];
+      }
+      break;
+    }
+  }
+  for (size_t i = 0; i < cfds.size(); ++i) {
+    if (cfds[i]->UnrefAndTryDelete()) {
+      cfds[i] = nullptr;
+    }
+  }
+
+  // exit the second queue before returning
+  if (two_write_queues_) {
+    nonmem_write_thread_.ExitUnbatched(&nonmem_w);
+  }
+  if (s.ok()) {
+    // Trigger flushes for the new immutable memtables.
+    for (const auto cfd : cfds) {
+      if (cfd == nullptr) {
+        continue;
+      }
+      cfd->imm()->FlushRequested();
+      FlushRequest flush_req;
+      // TODO: a new flush reason for ingesting memtable
+      GenerateFlushRequest({cfd}, FlushReason::kExternalFileIngestion,
+                           &flush_req);
+      EnqueuePendingFlush(flush_req);
+    }
+    MaybeScheduleFlushOrCompaction();
+  }
+  return s;
+}
+
 Status DBImpl::WriteImpl(const WriteOptions& write_options,
                         WriteBatch* my_batch, WriteCallback* callback,
                         UserWriteCallback* user_write_cb, uint64_t* log_used,
                         uint64_t log_ref, bool disable_memtable,
                         uint64_t* seq_used, size_t batch_cnt,
                         PreReleaseCallback* pre_release_callback,
-                         PostMemTableCallback* post_memtable_callback) {
+                         PostMemTableCallback* post_memtable_callback,
+                         std::shared_ptr<WriteBatchWithIndex> wbwi,
+                         uint64_t prep_log) {
  assert(!seq_per_batch_ || batch_cnt != 0);
  assert(my_batch == nullptr || my_batch->Count() == 0 ||
         write_options.protection_bytes_per_key == 0 ||
@@ -287,6 +409,23 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
    return Status::NotSupported(
        "DeleteRange is not compatible with row cache.");
  }
+  if (wbwi) {
+    assert(prep_log > 0);
+    // Used only in WriteCommittedTxn::CommitInternal() with no `callback`.
+    assert(!callback);
+    if (immutable_db_options_.unordered_write) {
+      return Status::NotSupported(
+          "Ingesting WriteBatch does not support unordered_write");
+    }
+    if (immutable_db_options_.enable_pipelined_write) {
+      return Status::NotSupported(
+          "Ingesting WriteBatch does not support pipelined_write");
+    }
+    if (immutable_db_options_.atomic_flush) {
+      return Status::NotSupported(
+          "Ingesting WriteBatch does not support atomic_flush");
+    }
+  }
  // Otherwise IsLatestPersistentState optimization does not make sense
  assert(!WriteBatchInternal::IsLatestPersistentState(my_batch) ||
         disable_memtable);
@@ -344,7 +483,8 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
  PERF_TIMER_GUARD(write_pre_and_post_process_time);
  WriteThread::Writer w(write_options, my_batch, callback, user_write_cb,
                        log_ref, disable_memtable, batch_cnt,
-                        pre_release_callback, post_memtable_callback);
+                        pre_release_callback, post_memtable_callback,
+                        /*_ingest_wbwi=*/wbwi != nullptr);
  StopWatch write_sw(immutable_db_options_.clock, stats_, DB_WRITE);

  write_thread_.JoinBatchGroup(&w);
@@ -441,6 +581,9 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
  TEST_SYNC_POINT("DBImpl::WriteImpl:BeforeLeaderEnters");
  last_batch_group_size_ =
      write_thread_.EnterAsBatchGroupLeader(&w, &write_group);
+  if (wbwi) {
+    assert(write_group.size == 1);
+  }

  IOStatus io_s;
  Status pre_release_cb_status;
@@ -494,10 +637,25 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
    // Note about seq_per_batch_: either disableWAL is set for the entire write
    // group or not. In either case we inc seq for each write batch with no
    // failed callback. This means that there could be a batch with
-    // disalbe_memtable in between; although we do not write this batch to
+    // disable_memtable in between; although we do not write this batch to
    // memtable it still consumes a seq. Otherwise, if !seq_per_batch_, we inc
    // the seq per valid written key to mem.
    size_t seq_inc = seq_per_batch_ ? valid_batches : total_count;
+    if (wbwi) {
+      // Reserve sequence numbers for the ingested memtable. We need to reserve
+      // at lease this amount for recovery. During recovery,
+      // transactions do not commit by ingesting WBWI. The sequence number
+      // associated with the commit entry in WAL is used as the starting
+      // sequence number for inserting into memtable. We need to reserve
+      // enough sequence numbers here (at least the number of operations
+      // in write batch) to assign to memtable entries for this transaction.
+      // This prevents updates in different transactions from using out-of-order
+      // sequence numbers or the same key+seqno.
+      //
+      // WBWI ingestion requires not grouping writes, so we don't need to
+      // consider incrementing sequence number for WBWI from other writers.
+      seq_inc += wbwi->GetWriteBatch()->Count();
+    }

    const bool concurrent_update = two_write_queues_;
    // Update stats while we are an exclusive group leader, so we know
@@ -674,6 +832,27 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
    // handle exit, false means somebody else did
    should_exit_batch_group = write_thread_.CompleteParallelMemTableWriter(&w);
  }
+  if (wbwi) {
+    if (status.ok() && w.status.ok()) {
+      // w.batch contains (potentially empty) commit time batch updates,
+      // only ingest wbwi if w.batch is applied to memtable successfully
+      assert(wbwi->GetWriteBatch()->Count() > 0);
+
+      uint32_t memtable_update_count = w.batch->Count();
+      SequenceNumber lb = versions_->LastSequence() + memtable_update_count + 1;
+      SequenceNumber ub = versions_->LastSequence() + memtable_update_count +
+                          wbwi->GetWriteBatch()->Count();
+      assert(ub == last_sequence);
+      if (two_write_queues_) {
+        assert(ub <= versions_->LastAllocatedSequence());
+      }
+      status = IngestWBWI(wbwi, {/*lower_bound=*/lb, /*upper_bound=*/ub},
+                          prep_log, last_sequence,
+                          /*memtable_updated=*/memtable_update_count > 0,
+                          write_options.ignore_missing_column_families);
+    }
+  }
+
  if (should_exit_batch_group) {
    if (status.ok()) {
      for (auto* tmp_w : write_group) {
@@ -687,7 +866,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
        }
      }
      // Note: if we are to resume after non-OK statuses we need to revisit how
-      // we reacts to non-OK statuses here.
+      // we react to non-OK statuses here.
      versions_->SetLastSequence(last_sequence);
    }
    MemTableInsertStatusCheck(w.status);
@@ -1210,7 +1389,6 @@ void DBImpl::MemTableInsertStatusCheck(const Status& status) {
  if (!status.ok()) {
    mutex_.Lock();
    assert(!error_handler_.IsBGWorkStopped());
-    // Maybe change the return status to void?
    error_handler_.SetBGError(status, BackgroundErrorReason::kMemTable);
    mutex_.Unlock();
  }
@@ -1610,6 +1788,8 @@ IOStatus DBImpl::ConcurrentWriteToWAL(
 Status DBImpl::WriteRecoverableState() {
  mutex_.AssertHeld();
  if (!cached_recoverable_state_empty_) {
+    // Only for write-prepared and write-unprepared.
+    assert(seq_per_batch_);
    bool dont_care_bool;
    SequenceNumber next_seq;
    if (two_write_queues_) {
@@ -2202,16 +2382,13 @@ void DBImpl::NotifyOnMemTableSealed(ColumnFamilyData* /*cfd*/,
  mutex_.Lock();
 }

-// REQUIRES: mutex_ is held
-// REQUIRES: this thread is currently at the front of the writer queue
-// REQUIRES: this thread is currently at the front of the 2nd writer queue if
-// two_write_queues_ is true (This is to simplify the reasoning.)
-Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
+Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
+                              ReadOnlyMemTable* new_imm,
+                              SequenceNumber last_seqno) {
  mutex_.AssertHeld();
  assert(lock_wal_count_ == 0);

  // TODO: plumb Env::IOActivity, Env::IOPriority
-  const ReadOptions read_options;
  const WriteOptions write_options;

  log::Writer* new_log = nullptr;
@@ -2247,12 +2424,13 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
  const MutableCFOptions mutable_cf_options = *cfd->GetLatestMutableCFOptions();

  // Set memtable_info for memtable sealed callback
+  // TODO: memtable_info for `new_imm`
  MemTableInfo memtable_info;
  memtable_info.cf_name = cfd->GetName();
  memtable_info.first_seqno = cfd->mem()->GetFirstSequenceNumber();
  memtable_info.earliest_seqno = cfd->mem()->GetEarliestSequenceNumber();
-  memtable_info.num_entries = cfd->mem()->num_entries();
-  memtable_info.num_deletes = cfd->mem()->num_deletes();
+  memtable_info.num_entries = cfd->mem()->NumEntries();
+  memtable_info.num_deletes = cfd->mem()->NumDeletion();
  if (!cfd->ioptions()->persist_user_defined_timestamps &&
      cfd->user_comparator()->timestamp_size() > 0) {
    const Slice& newest_udt = cfd->mem()->GetNewestUDT();
@@ -2274,8 +2452,20 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
    }
  }
  if (s.ok()) {
-    SequenceNumber seq = versions_->LastSequence();
-    new_mem = cfd->ConstructNewMemtable(mutable_cf_options, seq);
+    // FIXME: from the comment for GetEarliestSequenceNumber(), any key with
+    //  seqno >= earliest_seqno should be in this or later memtable. This means
+    //  we should use LastSequence() + 1 or last_seqno + 1 here. And it needs to
+    //  be incremented with file ingestion and other operations that consumes
+    //  sequence number.
+    SequenceNumber seq;
+    if (new_imm) {
+      assert(last_seqno > versions_->LastSequence());
+      seq = last_seqno;
+    } else {
+      seq = versions_->LastSequence();
+    }
+    new_mem =
+        cfd->ConstructNewMemtable(mutable_cf_options, /*earliest_seq=*/seq);
    context->superversion_context.NewSuperVersion();

    ROCKS_LOG_INFO(immutable_db_options_.info_log,
@@ -2357,6 +2547,8 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
        versions_->PreComputeMinLogNumberWithUnflushedData(logfile_number_);
    if (min_wal_number_to_keep >
        versions_->GetWalSet().GetMinWalNumberToKeep()) {
+      // TODO: plumb Env::IOActivity, Env::IOPriority
+      const ReadOptions read_options;
      // Get a snapshot of the empty column families.
      // LogAndApply may release and reacquire db
      // mutex, during that period, column family may become empty (e.g. its
@@ -2414,6 +2606,18 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
  cfd->mem()->SetNextLogNumber(logfile_number_);
  assert(new_mem != nullptr);
  cfd->imm()->Add(cfd->mem(), &context->memtables_to_free_);
+  if (new_imm) {
+    // Need to assign memtable id here before SetMemtable() below assigns id to
+    // the new live memtable
+    cfd->AssignMemtableID(new_imm);
+    // NOTE: new_imm and cfd->mem() references the same WAL and has the same
+    // NextLogNumber(). They should be flushed together. For non-atomic-flush,
+    // we always try to flush all immutable memtable. For atomic flush, these
+    // two memtables will be marked eligible for flush in the same call to
+    // AssignAtomicFlushSeq().
+    new_imm->SetNextLogNumber(logfile_number_);
+    cfd->imm()->Add(new_imm, &context->memtables_to_free_);
+  }
  new_mem->Ref();
  cfd->SetMemtable(new_mem);
  InstallSuperVersionAndScheduleWork(cfd, &context->superversion_context,
@@ -2426,6 +2630,9 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
  // that is okay.  If we did, it most likely means that s was already an error.
  // In any case, ignore any unchecked error for i_os here.
  io_s.PermitUncheckedError();
+  // We guarantee that if a non-ok status is returned, `new_imm` was not added
+  // to the db.
+  assert(s.ok());
  return s;
 }

@@ -27,12 +27,14 @@ class CorruptionFS : public FileSystemWrapper {
        num_writable_file_errors_(0),
        corruption_trigger_(INT_MAX),
        read_count_(0),
+        corrupt_offset_(0),
+        corrupt_len_(0),
        rnd_(300),
        fs_buffer_(fs_buffer),
        verify_read_(verify_read) {}
  ~CorruptionFS() override {
    // Assert that the corruption was reset, which means it got triggered
-    assert(corruption_trigger_ == INT_MAX);
+    assert(corruption_trigger_ == INT_MAX || corrupt_len_ > 0);
  }
  const char* Name() const override { return "ErrorEnv"; }

@@ -48,8 +50,10 @@ class CorruptionFS : public FileSystemWrapper {
  }

  void SetCorruptionTrigger(const int trigger) {
+    MutexLock l(&mutex_);
    corruption_trigger_ = trigger;
    read_count_ = 0;
+    corrupt_fname_.clear();
  }

  IOStatus NewRandomAccessFile(const std::string& fname,
@@ -58,25 +62,31 @@ class CorruptionFS : public FileSystemWrapper {
                               IODebugContext* dbg) override {
    class CorruptionRandomAccessFile : public FSRandomAccessFileOwnerWrapper {
     public:
-      CorruptionRandomAccessFile(CorruptionFS& fs,
+      CorruptionRandomAccessFile(CorruptionFS& fs, const std::string& fname,
                                 std::unique_ptr<FSRandomAccessFile>& file)
-          : FSRandomAccessFileOwnerWrapper(std::move(file)), fs_(fs) {}
+          : FSRandomAccessFileOwnerWrapper(std::move(file)),
+            fs_(fs),
+            fname_(fname) {}

      IOStatus Read(uint64_t offset, size_t len, const IOOptions& opts,
                    Slice* result, char* scratch,
                    IODebugContext* dbg) const override {
        IOStatus s = target()->Read(offset, len, opts, result, scratch, dbg);
        if (opts.verify_and_reconstruct_read) {
+          fs_.MaybeResetOverlapWithCorruptedChunk(fname_, offset,
+                                                  result->size());
          return s;
        }
+
+        MutexLock l(&fs_.mutex_);
        if (s.ok() && ++fs_.read_count_ >= fs_.corruption_trigger_) {
-          fs_.read_count_ = 0;
          fs_.corruption_trigger_ = INT_MAX;
          char* data = const_cast<char*>(result->data());
          std::memcpy(
              data,
              fs_.rnd_.RandomString(static_cast<int>(result->size())).c_str(),
              result->size());
+          fs_.SetCorruptedChunk(fname_, offset, result->size());
        }
        return s;
      }
@@ -101,14 +111,76 @@ class CorruptionFS : public FileSystemWrapper {
        return IOStatus::OK();
      }

+      IOStatus Prefetch(uint64_t /*offset*/, size_t /*n*/,
+                        const IOOptions& /*options*/,
+                        IODebugContext* /*dbg*/) override {
+        return IOStatus::NotSupported("Prefetch");
+      }
+
     private:
      CorruptionFS& fs_;
+      std::string fname_;
    };

    std::unique_ptr<FSRandomAccessFile> file;
    IOStatus s = target()->NewRandomAccessFile(fname, opts, &file, dbg);
    EXPECT_OK(s);
-    result->reset(new CorruptionRandomAccessFile(*this, file));
+    result->reset(new CorruptionRandomAccessFile(*this, fname, file));
+
+    return s;
+  }
+
+  IOStatus NewSequentialFile(const std::string& fname,
+                             const FileOptions& file_opts,
+                             std::unique_ptr<FSSequentialFile>* result,
+                             IODebugContext* dbg) override {
+    class CorruptionSequentialFile : public FSSequentialFileOwnerWrapper {
+     public:
+      CorruptionSequentialFile(CorruptionFS& fs, const std::string& fname,
+                               std::unique_ptr<FSSequentialFile>& file)
+          : FSSequentialFileOwnerWrapper(std::move(file)),
+            fs_(fs),
+            fname_(fname),
+            offset_(0) {}
+
+      IOStatus Read(size_t len, const IOOptions& opts, Slice* result,
+                    char* scratch, IODebugContext* dbg) override {
+        IOStatus s = target()->Read(len, opts, result, scratch, dbg);
+        if (result->size() == 0 ||
+            fname_.find("IDENTITY") != std::string::npos) {
+          return s;
+        }
+
+        if (opts.verify_and_reconstruct_read) {
+          fs_.MaybeResetOverlapWithCorruptedChunk(fname_, offset_,
+                                                  result->size());
+          return s;
+        }
+
+        MutexLock l(&fs_.mutex_);
+        if (s.ok() && ++fs_.read_count_ >= fs_.corruption_trigger_) {
+          fs_.corruption_trigger_ = INT_MAX;
+          char* data = const_cast<char*>(result->data());
+          std::memcpy(
+              data,
+              fs_.rnd_.RandomString(static_cast<int>(result->size())).c_str(),
+              result->size());
+          fs_.SetCorruptedChunk(fname_, offset_, result->size());
+        }
+        offset_ += result->size();
+        return s;
+      }
+
+     private:
+      CorruptionFS& fs_;
+      std::string fname_;
+      size_t offset_;
+    };
+
+    std::unique_ptr<FSSequentialFile> file;
+    IOStatus s = target()->NewSequentialFile(fname, file_opts, &file, dbg);
+    EXPECT_OK(s);
+    result->reset(new CorruptionSequentialFile(*this, fname, file));

    return s;
  }
@@ -123,12 +195,40 @@ class CorruptionFS : public FileSystemWrapper {
    }
  }

+  void SetCorruptedChunk(const std::string& fname, size_t offset, size_t len) {
+    assert(corrupt_fname_.empty());
+
+    corrupt_fname_ = fname;
+    corrupt_offset_ = offset;
+    corrupt_len_ = len;
+  }
+
+  void MaybeResetOverlapWithCorruptedChunk(const std::string& fname,
+                                           size_t offset, size_t len) {
+    if (fname == corrupt_fname_ &&
+        ((offset <= corrupt_offset_ && (offset + len) > corrupt_offset_) ||
+         (offset >= corrupt_offset_ &&
+          offset < (corrupt_offset_ + corrupt_len_)))) {
+      corrupt_fname_.clear();
+    }
+  }
+
+  bool VerifyRetry() { return corrupt_len_ > 0 && corrupt_fname_.empty(); }
+
+  int read_count() { return read_count_; }
+
+  int corruption_trigger() { return corruption_trigger_; }
+
 private:
  int corruption_trigger_;
  int read_count_;
+  std::string corrupt_fname_;
+  size_t corrupt_offset_;
+  size_t corrupt_len_;
  Random rnd_;
  bool fs_buffer_;
  bool verify_read_;
+  port::Mutex mutex_;
 };
 }  // anonymous namespace

@@ -717,6 +817,7 @@ class DBIOCorruptionTest
    bbto.num_file_reads_for_auto_readahead = 0;
    options_.table_factory.reset(NewBlockBasedTableFactory(bbto));
    options_.disable_auto_compactions = true;
+    options_.max_file_opening_threads = 0;

    Reopen(options_);
  }
@@ -857,8 +958,8 @@ TEST_P(DBIOCorruptionTest, FlushReadCorruptionRetry) {
  Status s = Flush();
  if (std::get<2>(GetParam())) {
    ASSERT_OK(s);
-    ASSERT_EQ(stats()->getTickerCount(FILE_READ_CORRUPTION_RETRY_COUNT), 1);
-    ASSERT_EQ(stats()->getTickerCount(FILE_READ_CORRUPTION_RETRY_SUCCESS_COUNT),
+    ASSERT_GT(stats()->getTickerCount(FILE_READ_CORRUPTION_RETRY_COUNT), 1);
+    ASSERT_GT(stats()->getTickerCount(FILE_READ_CORRUPTION_RETRY_SUCCESS_COUNT),
              1);

    std::string val;
@@ -885,8 +986,8 @@ TEST_P(DBIOCorruptionTest, ManifestCorruptionRetry) {

  if (std::get<2>(GetParam())) {
    ASSERT_OK(ReopenDB());
-    ASSERT_EQ(stats()->getTickerCount(FILE_READ_CORRUPTION_RETRY_COUNT), 1);
-    ASSERT_EQ(stats()->getTickerCount(FILE_READ_CORRUPTION_RETRY_SUCCESS_COUNT),
+    ASSERT_GT(stats()->getTickerCount(FILE_READ_CORRUPTION_RETRY_COUNT), 1);
+    ASSERT_GT(stats()->getTickerCount(FILE_READ_CORRUPTION_RETRY_SUCCESS_COUNT),
              1);
  } else {
    ASSERT_EQ(ReopenDB(), Status::Corruption());
@@ -970,6 +1071,57 @@ TEST_P(DBIOCorruptionTest, TablePropertiesCorruptionRetry) {
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
 }

+TEST_P(DBIOCorruptionTest, DBOpenReadCorruptionRetry) {
+  if (!std::get<2>(GetParam())) {
+    return;
+  }
+  CorruptionFS* fs =
+      static_cast<CorruptionFS*>(env_guard_->GetFileSystem().get());
+
+  for (int sst = 0; sst < 3; ++sst) {
+    for (int key = 0; key < 100; ++key) {
+      std::stringstream ss;
+      ss << std::setw(3) << 100 * sst + key;
+      ASSERT_OK(Put("key" + ss.str(), "val" + ss.str()));
+    }
+    ASSERT_OK(Flush());
+  }
+  Close();
+
+  // DB open will create table readers unless we reduce the table cache
+  // capacity.
+  // SanitizeOptions will set max_open_files to minimum of 20. Table cache
+  // is allocated with max_open_files - 10 as capacity. So override
+  // max_open_files to 11 so table cache capacity will become 1. This will
+  // prevent file open during DB open and force the file to be opened
+  // during MultiGet
+  SyncPoint::GetInstance()->SetCallBack(
+      "SanitizeOptions::AfterChangeMaxOpenFiles", [&](void* arg) {
+        int* max_open_files = (int*)arg;
+        *max_open_files = 11;
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  // Progressively increase the IO count trigger for corruption, and verify
+  // that it was retried
+  int corruption_trigger = 1;
+  fs->SetCorruptionTrigger(corruption_trigger);
+  do {
+    fs->SetCorruptionTrigger(corruption_trigger);
+    ASSERT_OK(ReopenDB());
+    for (int sst = 0; sst < 3; ++sst) {
+      for (int key = 0; key < 100; ++key) {
+        std::stringstream ss;
+        ss << std::setw(3) << 100 * sst + key;
+        ASSERT_EQ(Get("key" + ss.str()), "val" + ss.str());
+      }
+    }
+    // Verify that the injected corruption was repaired
+    ASSERT_TRUE(fs->VerifyRetry());
+    corruption_trigger++;
+  } while (fs->corruption_trigger() == INT_MAX);
+}
+
 // The parameters are - 1. Use FS provided buffer, 2. Use async IO ReadOption,
 // 3. Retry with verify_and_reconstruct_read IOOption
 INSTANTIATE_TEST_CASE_P(DBIOCorruptionTest, DBIOCorruptionTest,
@@ -581,6 +581,14 @@ bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key,
    } else {
      iter_.Next();
    }
+
+    // This could be a long-running operation due to tombstones, etc.
+    bool aborted = ROCKSDB_THREAD_YIELD_CHECK_ABORT();
+    if (aborted) {
+      valid_ = false;
+      status_ = Status::Aborted("Query abort.");
+      return false;
+    }
  } while (iter_.Valid());

  valid_ = false;
@@ -56,6 +56,11 @@ class DBOptionsTest : public DBTestBase {
    EXPECT_OK(GetStringFromMutableCFOptions(
        config_options, MutableCFOptions(options), &options_str));
    EXPECT_OK(StringToMap(options_str, &mutable_map));
+    for (auto& opt : TEST_GetImmutableInMutableCFOptions()) {
+      // Not yet mutable but migrated to MutableCFOptions in preparation for
+      // being mutable
+      mutable_map.erase(opt);
+    }
    return mutable_map;
  }

@@ -16,6 +16,7 @@
 #include "db/db_test_util.h"
 #include "db/read_callback.h"
 #include "db/version_edit.h"
+#include "env/fs_readonly.h"
 #include "options/options_helper.h"
 #include "port/port.h"
 #include "port/stack_trace.h"
@@ -36,18 +37,6 @@ namespace ROCKSDB_NAMESPACE {
 class DBTest2 : public DBTestBase {
 public:
  DBTest2() : DBTestBase("db_test2", /*env_do_fsync=*/true) {}
-  std::vector<FileMetaData*> GetLevelFileMetadatas(int level, int cf = 0) {
-    VersionSet* const versions = dbfull()->GetVersionSet();
-    assert(versions);
-    ColumnFamilyData* const cfd =
-        versions->GetColumnFamilySet()->GetColumnFamily(cf);
-    assert(cfd);
-    Version* const current = cfd->current();
-    assert(current);
-    VersionStorageInfo* const storage_info = current->storage_info();
-    assert(storage_info);
-    return storage_info->LevelFiles(level);
-  }
 };

 TEST_F(DBTest2, OpenForReadOnly) {
@@ -2115,16 +2104,15 @@ TEST_P(PinL0IndexAndFilterBlocksTest,
  ASSERT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_ADD));
  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_MISS));

-  std::string value;
  // Miss and hit count should remain the same, they're all pinned.
-  ASSERT_TRUE(db_->KeyMayExist(ReadOptions(), handles_[1], "key", &value));
+  ASSERT_TRUE(db_->KeyMayExist(ReadOptions(), handles_[1], "key", nullptr));
  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));

  // Miss and hit count should remain the same, they're all pinned.
-  value = Get(1, "key");
+  std::string value = Get(1, "key");
  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
@@ -8074,6 +8062,63 @@ TEST_F(DBTest2, TableCacheMissDuringReadFromBlockCacheTier) {
  ASSERT_EQ(orig_num_file_opens, TestGetTickerCount(options, NO_FILE_OPENS));
 }

+TEST_F(DBTest2, GetFileChecksumsFromCurrentManifest_CRC32) {
+  Options opts = CurrentOptions();
+  opts.create_if_missing = true;
+  opts.file_checksum_gen_factory = GetFileChecksumGenCrc32cFactory();
+  opts.level0_file_num_compaction_trigger = 10;
+
+  // Bootstrap the test database.
+  DB* db = nullptr;
+  std::string dbname = test::PerThreadDBPath("file_chksum");
+  ASSERT_OK(DB::Open(opts, dbname, &db));
+
+  WriteOptions wopts;
+  FlushOptions fopts;
+  fopts.wait = true;
+  Random rnd(test::RandomSeed());
+  for (int i = 0; i < 4; i++) {
+    ASSERT_OK(db->Put(wopts, Key(i), rnd.RandomString(100)));
+    ASSERT_OK(db->Flush(fopts));
+  }
+
+  // Obtain rich files metadata for source of truth.
+  std::vector<LiveFileMetaData> live_files;
+  db->GetLiveFilesMetaData(&live_files);
+
+  ASSERT_OK(db->Close());
+  delete db;
+  db = nullptr;
+
+  // Process current MANIFEST file and build internal file checksum mappings.
+  std::unique_ptr<FileChecksumList> checksum_list(NewFileChecksumList());
+  auto read_only_fs =
+      std::make_shared<ReadOnlyFileSystem>(env_->GetFileSystem());
+  ASSERT_OK(experimental::GetFileChecksumsFromCurrentManifest(
+      read_only_fs.get(), dbname, checksum_list.get()));
+
+  ASSERT_TRUE(checksum_list != nullptr);
+
+  // Retrieve files, related checksums and checksum functions.
+  std::vector<uint64_t> file_numbers;
+  std::vector<std::string> checksums;
+  std::vector<std::string> checksum_func_names;
+  ASSERT_OK(checksum_list->GetAllFileChecksums(&file_numbers, &checksums,
+                                               &checksum_func_names));
+
+  // Compare results.
+  ASSERT_EQ(live_files.size(), checksum_list->size());
+  for (size_t i = 0; i < live_files.size(); i++) {
+    std::string stored_checksum;
+    std::string stored_func_name;
+    ASSERT_OK(checksum_list->SearchOneFileChecksum(
+        live_files[i].file_number, &stored_checksum, &stored_func_name));
+
+    ASSERT_EQ(live_files[i].file_checksum, stored_checksum);
+    ASSERT_EQ(live_files[i].file_checksum_func_name, stored_func_name);
+  }
+}
+
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -934,6 +934,13 @@ Status DBTestBase::Get(const std::string& k, PinnableSlice* v) {
  return s;
 }

+Status DBTestBase::CompactRange(const CompactRangeOptions& options,
+                                std::optional<Slice> begin,
+                                std::optional<Slice> end) {
+  return db_->CompactRange(options, begin ? &begin.value() : nullptr,
+                           end ? &end.value() : nullptr);
+}
+
 uint64_t DBTestBase::GetNumSnapshots() {
  uint64_t int_num;
  EXPECT_TRUE(dbfull()->GetIntProperty("rocksdb.num-snapshots", &int_num));
@@ -1263,6 +1270,20 @@ Status DBTestBase::CountFiles(size_t* count) {
  return Status::OK();
 }

+std::vector<FileMetaData*> DBTestBase::GetLevelFileMetadatas(int level,
+                                                             int cf) {
+  VersionSet* const versions = dbfull()->GetVersionSet();
+  assert(versions);
+  ColumnFamilyData* const cfd =
+      versions->GetColumnFamilySet()->GetColumnFamily(cf);
+  assert(cfd);
+  Version* const current = cfd->current();
+  assert(current);
+  VersionStorageInfo* const storage_info = current->storage_info();
+  assert(storage_info);
+  return storage_info->LevelFiles(level);
+}
+
 Status DBTestBase::Size(const Slice& start, const Slice& limit, int cf,
                        uint64_t* size) {
  Range r(start, limit);
@@ -1579,42 +1600,74 @@ std::vector<std::uint64_t> DBTestBase::ListTableFiles(Env* env,
  return file_numbers;
 }

-void DBTestBase::VerifyDBFromMap(std::map<std::string, std::string> true_data,
-                                 size_t* total_reads_res, bool tailing_iter,
-                                 std::map<std::string, Status> status) {
-  size_t total_reads = 0;
+void DBTestBase::VerifyDBFromMap(
+    std::map<std::string, std::string> true_data, size_t* total_reads_res,
+    bool tailing_iter, ReadOptions* ro, ColumnFamilyHandle* cf,
+    std::unordered_set<std::string>* not_found) const {
+  ReadOptions temp_ro;
+  if (!ro) {
+    ro = &temp_ro;
+    ro->verify_checksums = true;
+  }
+  if (!cf) {
+    cf = db_->DefaultColumnFamily();
+  }

-  for (auto& kv : true_data) {
-    Status s = status[kv.first];
-    if (s.ok()) {
-      ASSERT_EQ(Get(kv.first), kv.second);
-    } else {
-      std::string value;
-      ASSERT_EQ(s, db_->Get(ReadOptions(), kv.first, &value));
-    }
+  // Get
+  size_t total_reads = 0;
+  std::string result;
+  for (auto& [k, v] : true_data) {
+    ASSERT_OK(db_->Get(*ro, cf, k, &result)) << "key is " << k;
+    ASSERT_EQ(v, result);
    total_reads++;
  }
+  if (not_found) {
+    for (const auto& k : *not_found) {
+      ASSERT_TRUE(db_->Get(*ro, cf, k, &result).IsNotFound())
+          << "key is " << k << " val is " << result;
+    }
+  }
+
+  // MultiGet
+  std::vector<Slice> key_slice;
+  for (const auto& [k, _] : true_data) {
+    key_slice.emplace_back(k);
+  }
+  std::vector<std::string> values;
+  std::vector<ColumnFamilyHandle*> cfs(key_slice.size(), cf);
+  std::vector<Status> status = db_->MultiGet(*ro, cfs, key_slice, &values);
+  total_reads += key_slice.size();
+  auto data_iter = true_data.begin();
+  for (size_t i = 0; i < key_slice.size(); ++i, ++data_iter) {
+    ASSERT_OK(status[i]);
+    ASSERT_EQ(values[i], data_iter->second);
+  }
+  // MultiGet - not found
+  if (not_found) {
+    key_slice.clear();
+    for (const auto& k : *not_found) {
+      key_slice.emplace_back(k);
+    }
+    cfs = std::vector<ColumnFamilyHandle*>(key_slice.size(), cf);
+    values.clear();
+    status = db_->MultiGet(*ro, cfs, key_slice, &values);
+    for (const auto& s : status) {
+      ASSERT_TRUE(s.IsNotFound());
+    }
+  }

  // Normal Iterator
  {
    int iter_cnt = 0;
-    ReadOptions ro;
-    ro.total_order_seek = true;
-    Iterator* iter = db_->NewIterator(ro);
+    ReadOptions ro_ = *ro;
+    ro_.total_order_seek = true;
+    Iterator* iter = db_->NewIterator(ro_, cf);
    // Verify Iterator::Next()
    iter_cnt = 0;
-    auto data_iter = true_data.begin();
-    Status s;
-    for (iter->SeekToFirst(); iter->Valid(); iter->Next(), data_iter++) {
+    data_iter = true_data.begin();
+    for (iter->SeekToFirst(); iter->Valid(); iter->Next(), ++data_iter) {
      ASSERT_EQ(iter->key().ToString(), data_iter->first);
-      Status current_status = status[data_iter->first];
-      if (!current_status.ok()) {
-        s = current_status;
-      }
-      ASSERT_EQ(iter->status(), s);
-      if (current_status.ok()) {
        ASSERT_EQ(iter->value().ToString(), data_iter->second);
-      }
      iter_cnt++;
      total_reads++;
    }
@@ -1625,20 +1678,12 @@ void DBTestBase::VerifyDBFromMap(std::map<std::string, std::string> true_data,

    // Verify Iterator::Prev()
    // Use a new iterator to make sure its status is clean.
-    iter = db_->NewIterator(ro);
+    iter = db_->NewIterator(ro_, cf);
    iter_cnt = 0;
-    s = Status::OK();
    auto data_rev = true_data.rbegin();
    for (iter->SeekToLast(); iter->Valid(); iter->Prev(), data_rev++) {
      ASSERT_EQ(iter->key().ToString(), data_rev->first);
-      Status current_status = status[data_rev->first];
-      if (!current_status.ok()) {
-        s = current_status;
-      }
-      ASSERT_EQ(iter->status(), s);
-      if (current_status.ok()) {
        ASSERT_EQ(iter->value().ToString(), data_rev->second);
-      }
      iter_cnt++;
      total_reads++;
    }
@@ -1646,12 +1691,20 @@ void DBTestBase::VerifyDBFromMap(std::map<std::string, std::string> true_data,
    ASSERT_EQ(data_rev, true_data.rend())
        << iter_cnt << " / " << true_data.size();

-    // Verify Iterator::Seek()
-    for (const auto& kv : true_data) {
-      iter->Seek(kv.first);
-      ASSERT_EQ(kv.first, iter->key().ToString());
-      ASSERT_EQ(kv.second, iter->value().ToString());
-      total_reads++;
+    // Verify Iterator::Seek() and SeekForPrev()
+    for (const auto& [k, v] : true_data) {
+      for (bool prev : {false, true}) {
+        if (prev) {
+          iter->SeekForPrev(k);
+        } else {
+          iter->Seek(k);
+        }
+        ASSERT_TRUE(iter->Valid());
+        ASSERT_OK(iter->status());
+        ASSERT_EQ(iter->key(), k);
+        ASSERT_EQ(iter->value(), v);
+        ++total_reads;
+      }
    }
    delete iter;
  }
@@ -1659,14 +1712,14 @@ void DBTestBase::VerifyDBFromMap(std::map<std::string, std::string> true_data,
  if (tailing_iter) {
    // Tailing iterator
    int iter_cnt = 0;
-    ReadOptions ro;
-    ro.tailing = true;
-    ro.total_order_seek = true;
-    Iterator* iter = db_->NewIterator(ro);
+    ReadOptions ro_ = *ro;
+    ro_.tailing = true;
+    ro_.total_order_seek = true;
+    Iterator* iter = db_->NewIterator(ro_, cf);

    // Verify ForwardIterator::Next()
    iter_cnt = 0;
-    auto data_iter = true_data.begin();
+    data_iter = true_data.begin();
    for (iter->SeekToFirst(); iter->Valid(); iter->Next(), data_iter++) {
      ASSERT_EQ(iter->key().ToString(), data_iter->first);
      ASSERT_EQ(iter->value().ToString(), data_iter->second);
@@ -1225,6 +1225,9 @@ class DBTestBase : public testing::Test {
                                    const Snapshot* snapshot = nullptr,
                                    const bool async = false);

+  Status CompactRange(const CompactRangeOptions& options,
+                      std::optional<Slice> begin, std::optional<Slice> end);
+
  uint64_t GetNumSnapshots();

  uint64_t GetTimeOldestSnapshots();
@@ -1273,6 +1276,8 @@ class DBTestBase : public testing::Test {

  Status CountFiles(size_t* count);

+  std::vector<FileMetaData*> GetLevelFileMetadatas(int level, int cf = 0);
+
  Status Size(const Slice& start, const Slice& limit, uint64_t* size) {
    return Size(start, limit, 0, size);
  }
@@ -1362,7 +1367,8 @@ class DBTestBase : public testing::Test {
  void VerifyDBFromMap(
      std::map<std::string, std::string> true_data,
      size_t* total_reads_res = nullptr, bool tailing_iter = false,
-      std::map<std::string, Status> status = std::map<std::string, Status>());
+      ReadOptions* ro = nullptr, ColumnFamilyHandle* cf = nullptr,
+      std::unordered_set<std::string>* not_found = nullptr) const;

  void VerifyDBInternal(
      std::vector<std::pair<std::string, std::string>> true_data);
@@ -178,11 +178,9 @@ std::string ParsedInternalKey::DebugString(bool log_err_key, bool hex,
    result += "<redacted>";
  }

-  char buf[50];
-  snprintf(buf, sizeof(buf), "' seq:%" PRIu64 ", type:%d", sequence,
-           static_cast<int>(type));
+  result += "' seq:" + std::to_string(sequence);
+  result += ", type:" + std::to_string(type);

-  result += buf;
  return result;
 }

@@ -132,6 +132,7 @@ void EventHelpers::LogAndNotifyTableFileCreationFinished(
              << table_properties.compression_name << "compression_options"
              << table_properties.compression_options << "creation_time"
              << table_properties.creation_time << "oldest_key_time"
+              << table_properties.newest_key_time << "newest_key_time"
              << table_properties.oldest_key_time << "file_creation_time"
              << table_properties.file_creation_time
              << "slow_compression_estimated_data_size"
@@ -13,6 +13,8 @@
 #include <vector>

 #include "db/db_impl/db_impl.h"
+#include "db/manifest_ops.h"
+#include "db/version_edit_handler.h"
 #include "db/version_util.h"
 #include "logging/logging.h"
 #include "util/atomic.h"
@@ -40,6 +42,58 @@ Status SuggestCompactRange(DB* db, const Slice* begin, const Slice* end) {
  return SuggestCompactRange(db, db->DefaultColumnFamily(), begin, end);
 }

+Status GetFileChecksumsFromCurrentManifest(FileSystem* fs,
+                                           const std::string& dbname,
+                                           FileChecksumList* checksum_list) {
+  std::string manifest_path;
+  uint64_t manifest_file_number;
+  Status s = GetCurrentManifestPath(dbname, fs, true /* is_retry */,
+                                    &manifest_path, &manifest_file_number);
+  if (!s.ok()) {
+    return s;
+  }
+
+  if (checksum_list == nullptr) {
+    return Status::InvalidArgument("checksum_list is nullptr");
+  }
+  assert(checksum_list);
+
+  const ReadOptions read_options(Env::IOActivity::kReadManifest);
+  checksum_list->reset();
+
+  std::unique_ptr<SequentialFileReader> file_reader;
+  {
+    std::unique_ptr<FSSequentialFile> file;
+    s = fs->NewSequentialFile(manifest_path,
+                              fs->OptimizeForManifestRead(FileOptions()), &file,
+                              nullptr /* dbg */);
+    if (!s.ok()) {
+      return s;
+    }
+    file_reader.reset(new SequentialFileReader(std::move(file), manifest_path));
+  }
+
+  struct LogReporter : public log::Reader::Reporter {
+    Status* status_ptr;
+    void Corruption(size_t /*bytes*/, const Status& st) override {
+      if (status_ptr->ok()) {
+        *status_ptr = st;
+      }
+    }
+  } reporter;
+  reporter.status_ptr = &s;
+  log::Reader reader(nullptr, std::move(file_reader), &reporter,
+                     true /* checksum */, 0 /* log_number */);
+
+  // Read all records from the manifest file...
+  uint64_t manifest_file_size = std::numeric_limits<uint64_t>::max();
+  FileChecksumRetriever retriever(read_options, manifest_file_size,
+                                  *checksum_list);
+  retriever.Iterate(reader, &s);
+
+  return retriever.status();
+}
+
 Status UpdateManifestForFilesState(
    const DBOptions& db_opts, const std::string& db_name,
    const std::vector<ColumnFamilyDescriptor>& column_families,
@@ -182,6 +182,36 @@ class ExternalSSTFileBasicTest
        write_global_seqno, verify_checksums_before_ingest, true_data);
  }

+  void VerifyInputFilesInternalStatsForOutputLevel(
+      int output_level, int num_input_files_in_non_output_levels,
+      int num_input_files_in_output_level,
+      int num_filtered_input_files_in_non_output_levels,
+      int num_filtered_input_files_in_output_level,
+      uint64_t bytes_skipped_non_output_levels,
+      uint64_t bytes_skipped_output_level) {
+    ColumnFamilyHandleImpl* cfh =
+        static_cast<ColumnFamilyHandleImpl*>(dbfull()->DefaultColumnFamily());
+    ColumnFamilyData* cfd = cfh->cfd();
+    const InternalStats* internal_stats_ptr = cfd->internal_stats();
+    const std::vector<InternalStats::CompactionStats>& comp_stats =
+        internal_stats_ptr->TEST_GetCompactionStats();
+
+    EXPECT_EQ(num_input_files_in_non_output_levels,
+              comp_stats[output_level].num_input_files_in_non_output_levels);
+    EXPECT_EQ(num_input_files_in_output_level,
+              comp_stats[output_level].num_input_files_in_output_level);
+    EXPECT_EQ(
+        num_filtered_input_files_in_non_output_levels,
+        comp_stats[output_level].num_filtered_input_files_in_non_output_levels);
+    EXPECT_EQ(
+        num_filtered_input_files_in_output_level,
+        comp_stats[output_level].num_filtered_input_files_in_output_level);
+    EXPECT_EQ(bytes_skipped_non_output_levels,
+              comp_stats[output_level].bytes_skipped_non_output_levels);
+    EXPECT_EQ(bytes_skipped_output_level,
+              comp_stats[output_level].bytes_skipped_output_level);
+  }
+
  ~ExternalSSTFileBasicTest() override {
    DestroyDir(env_, sst_files_dir_).PermitUncheckedError();
  }
@@ -241,6 +271,79 @@ TEST_F(ExternalSSTFileBasicTest, Basic) {
  DestroyAndRecreateExternalSSTFilesDir();
 }

+TEST_F(ExternalSSTFileBasicTest, AlignedBufferedWrite) {
+  class AlignedWriteFS : public FileSystemWrapper {
+   public:
+    explicit AlignedWriteFS(const std::shared_ptr<FileSystem>& _target)
+        : FileSystemWrapper(_target) {}
+    ~AlignedWriteFS() override {}
+    const char* Name() const override { return "AlignedWriteFS"; }
+
+    IOStatus NewWritableFile(const std::string& fname, const FileOptions& opts,
+                             std::unique_ptr<FSWritableFile>* result,
+                             IODebugContext* dbg) override {
+      class AlignedWritableFile : public FSWritableFileOwnerWrapper {
+       public:
+        AlignedWritableFile(std::unique_ptr<FSWritableFile>& file)
+            : FSWritableFileOwnerWrapper(std::move(file)), last_write_(false) {}
+
+        using FSWritableFileOwnerWrapper::Append;
+        IOStatus Append(const Slice& data, const IOOptions& options,
+                        IODebugContext* dbg) override {
+          EXPECT_FALSE(last_write_);
+          if ((data.size() & (data.size() - 1)) != 0) {
+            last_write_ = true;
+          }
+          return target()->Append(data, options, dbg);
+        }
+
+       private:
+        bool last_write_;
+      };
+
+      std::unique_ptr<FSWritableFile> file;
+      IOStatus s = target()->NewWritableFile(fname, opts, &file, dbg);
+      if (s.ok()) {
+        result->reset(new AlignedWritableFile(file));
+      }
+      return s;
+    }
+  };
+
+  Options options = CurrentOptions();
+  std::shared_ptr<AlignedWriteFS> aligned_fs =
+      std::make_shared<AlignedWriteFS>(env_->GetFileSystem());
+  std::unique_ptr<Env> wrap_env(
+      new CompositeEnvWrapper(options.env, aligned_fs));
+  options.env = wrap_env.get();
+
+  EnvOptions env_options;
+  env_options.writable_file_max_buffer_size = 64 * 1024 * 1024;
+
+  SstFileWriter sst_file_writer(env_options, options);
+
+  // Current file size should be 0 after sst_file_writer init and before open a
+  // file.
+  ASSERT_EQ(sst_file_writer.FileSize(), 0);
+
+  // file1.sst (0 => 99)
+  std::string file1 = sst_files_dir_ + "file1.sst";
+  ASSERT_OK(sst_file_writer.Open(file1));
+  Random r(301);
+  for (int k = 0; k < 16 * 1024; k++) {
+    uint32_t num = 4096 + r.Uniform(8192);
+    std::string random_string = r.RandomString(num);
+    ASSERT_OK(sst_file_writer.Put(Key(k), random_string));
+  }
+  Status s = sst_file_writer.Finish();
+  ASSERT_OK(s) << s.ToString();
+
+  // Current file size should be non-zero after success write.
+  ASSERT_GT(sst_file_writer.FileSize(), 0);
+
+  DestroyAndRecreateExternalSSTFilesDir();
+}
+
 class ChecksumVerifyHelper {
 private:
  Options options_;
@@ -1818,6 +1921,9 @@ TEST_F(ExternalSSTFileBasicTest, OverlappingFiles) {
  }

  IngestExternalFileOptions ifo;
+  ifo.allow_global_seqno = false;
+  ASSERT_NOK(db_->IngestExternalFile(files, ifo));
+  ifo.allow_global_seqno = true;
  ASSERT_OK(db_->IngestExternalFile(files, ifo));
  ASSERT_EQ(Get("a"), "a1");
  ASSERT_EQ(Get("i"), "i2");
@@ -1838,11 +1944,60 @@ TEST_F(ExternalSSTFileBasicTest, OverlappingFiles) {
  ASSERT_EQ(2, NumTableFilesAtLevel(5));
 }

+class CompactionJobStatsCheckerForFilteredFiles : public EventListener {
+ public:
+  CompactionJobStatsCheckerForFilteredFiles(
+      int num_input_files, int num_input_files_at_output_level,
+      int num_filtered_input_files,
+      int num_filtered_input_files_at_output_level)
+      : num_input_files_(num_input_files),
+        num_input_files_at_output_level_(num_input_files_at_output_level),
+        num_filtered_input_files_(num_filtered_input_files),
+        num_filtered_input_files_at_output_level_(
+            num_filtered_input_files_at_output_level) {}
+
+  void OnCompactionCompleted(DB* /*db*/, const CompactionJobInfo& ci) override {
+    std::lock_guard<std::mutex> lock(mutex_);
+    ASSERT_EQ(num_input_files_, ci.stats.num_input_files);
+    ASSERT_EQ(num_input_files_at_output_level_,
+              ci.stats.num_input_files_at_output_level);
+    ASSERT_EQ(num_filtered_input_files_, ci.stats.num_filtered_input_files);
+    ASSERT_EQ(num_filtered_input_files_at_output_level_,
+              ci.stats.num_filtered_input_files_at_output_level);
+    ASSERT_EQ(ci.stats.total_skipped_input_bytes,
+              expected_compaction_skipped_file_size_);
+  }
+
+  void SetExpectedCompactionSkippedFileSize(uint64_t expected_size) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    expected_compaction_skipped_file_size_ = expected_size;
+  }
+
+ private:
+  int num_input_files_ = 0;
+  int num_input_files_at_output_level_ = 0;
+  int num_filtered_input_files_ = 0;
+  int num_filtered_input_files_at_output_level_ = 0;
+  std::mutex mutex_;
+  uint64_t expected_compaction_skipped_file_size_ = 0;
+};
+
 TEST_F(ExternalSSTFileBasicTest, AtomicReplaceDataWithStandaloneRangeDeletion) {
  Options options = CurrentOptions();
  options.compaction_style = CompactionStyle::kCompactionStyleUniversal;
+  int kCompactionNumInputFiles = 1;
+  int kCompactionNumInputFilesAtOutputLevel = 0;
+  int kCompactionNumFilteredInputFiles = 2;
+  int kCompactionNumFilteredInputFilesAtOutputLevel = 2;
+  auto compaction_listener =
+      std::make_shared<CompactionJobStatsCheckerForFilteredFiles>(
+          kCompactionNumInputFiles, kCompactionNumInputFilesAtOutputLevel,
+          kCompactionNumFilteredInputFiles,
+          kCompactionNumFilteredInputFilesAtOutputLevel);
+  options.listeners.push_back(compaction_listener);
  DestroyAndReopen(options);

+  size_t compaction_skipped_file_size = 0;
  std::vector<std::string> files;
  {
    // Writes first version of data in range partitioned files.
@@ -1853,6 +2008,7 @@ TEST_F(ExternalSSTFileBasicTest, AtomicReplaceDataWithStandaloneRangeDeletion) {
    ASSERT_OK(sst_file_writer.Put("b", "b1"));
    ExternalSstFileInfo file1_info;
    ASSERT_OK(sst_file_writer.Finish(&file1_info));
+    compaction_skipped_file_size += file1_info.file_size;
    files.push_back(std::move(file1));

    std::string file2 = sst_files_dir_ + "file2.sst";
@@ -1861,7 +2017,10 @@ TEST_F(ExternalSSTFileBasicTest, AtomicReplaceDataWithStandaloneRangeDeletion) {
    ASSERT_OK(sst_file_writer.Put("y", "y1"));
    ExternalSstFileInfo file2_info;
    ASSERT_OK(sst_file_writer.Finish(&file2_info));
+    compaction_skipped_file_size += file2_info.file_size;
    files.push_back(std::move(file2));
+    compaction_listener->SetExpectedCompactionSkippedFileSize(
+        compaction_skipped_file_size);
  }

  IngestExternalFileOptions ifo;
@@ -1941,6 +2100,16 @@ TEST_F(ExternalSSTFileBasicTest, AtomicReplaceDataWithStandaloneRangeDeletion) {
  ASSERT_EQ(Get("x"), "x2");
  ASSERT_EQ(Get("y"), "y2");

+  VerifyInputFilesInternalStatsForOutputLevel(
+      /*output_level*/ 6,
+      kCompactionNumInputFiles - kCompactionNumInputFilesAtOutputLevel,
+      kCompactionNumInputFilesAtOutputLevel,
+      kCompactionNumFilteredInputFiles -
+          kCompactionNumFilteredInputFilesAtOutputLevel,
+      kCompactionNumFilteredInputFilesAtOutputLevel,
+      /*bytes_skipped_non_output_levels*/ 0,
+      /*bytes_skipped_output_level*/ compaction_skipped_file_size);
+
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }

@@ -1948,9 +2117,20 @@ TEST_F(ExternalSSTFileBasicTest,
       PartiallyReplaceDataWithOneStandaloneRangeDeletion) {
  Options options = CurrentOptions();
  options.compaction_style = CompactionStyle::kCompactionStyleUniversal;
+  int kCompactionNumInputFiles = 2;
+  int kCompactionNumInputFilesAtOutputLevel = 1;
+  int kCompactionNumFilteredInputFiles = 1;
+  int kCompactionNumFilteredInputFilesAtOutputLevel = 1;
+  auto compaction_listener =
+      std::make_shared<CompactionJobStatsCheckerForFilteredFiles>(
+          kCompactionNumInputFiles, kCompactionNumInputFilesAtOutputLevel,
+          kCompactionNumFilteredInputFiles,
+          kCompactionNumFilteredInputFilesAtOutputLevel);
+  options.listeners.push_back(compaction_listener);
  DestroyAndReopen(options);

  std::vector<std::string> files;
+  size_t compaction_skipped_file_size = 0;
  {
    // Writes first version of data in range partitioned files.
    SstFileWriter sst_file_writer(EnvOptions(), options);
@@ -1960,7 +2140,10 @@ TEST_F(ExternalSSTFileBasicTest,
    ASSERT_OK(sst_file_writer.Put("b", "b1"));
    ExternalSstFileInfo file1_info;
    ASSERT_OK(sst_file_writer.Finish(&file1_info));
+    compaction_skipped_file_size += file1_info.file_size;
    files.push_back(std::move(file1));
+    compaction_listener->SetExpectedCompactionSkippedFileSize(
+        compaction_skipped_file_size);

    std::string file2 = sst_files_dir_ + "file2.sst";
    ASSERT_OK(sst_file_writer.Open(file2));
@@ -2029,6 +2212,17 @@ TEST_F(ExternalSSTFileBasicTest,
  ASSERT_EQ(Get("h"), "h1");
  ASSERT_EQ(Get("x"), "x2");
  ASSERT_EQ(Get("y"), "y");
+
+  VerifyInputFilesInternalStatsForOutputLevel(
+      /*output_level*/ 6,
+      kCompactionNumInputFiles - kCompactionNumInputFilesAtOutputLevel,
+      kCompactionNumInputFilesAtOutputLevel,
+      kCompactionNumFilteredInputFiles -
+          kCompactionNumFilteredInputFilesAtOutputLevel,
+      kCompactionNumFilteredInputFilesAtOutputLevel,
+      /*bytes_skipped_non_output_levels*/ 0,
+      /*bytes_skipped_output_level*/ compaction_skipped_file_size);
+
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }

@@ -2036,15 +2230,29 @@ TEST_F(ExternalSSTFileBasicTest,
       PartiallyReplaceDataWithMultipleStandaloneRangeDeletions) {
  Options options = CurrentOptions();
  options.compaction_style = CompactionStyle::kCompactionStyleUniversal;
+  int kCompactionNumInputFiles = 2;
+  int kCompactionNumInputFilesAtOutputLevel = 0;
+  int kCompactionNumFilteredInputFiles = 2;
+  int kCompactionNumFilteredInputFilesAtOutputLevel = 2;
+  // Two compactions each included on standalone range deletion file that
+  // filters input file on the non start level.
+  auto compaction_listener =
+      std::make_shared<CompactionJobStatsCheckerForFilteredFiles>(
+          kCompactionNumInputFiles / 2,
+          kCompactionNumInputFilesAtOutputLevel / 2,
+          kCompactionNumFilteredInputFiles / 2,
+          kCompactionNumFilteredInputFilesAtOutputLevel / 2);
+  options.listeners.push_back(compaction_listener);
  DestroyAndReopen(options);

  std::vector<std::string> files;
+  ExternalSstFileInfo file1_info;
+  ExternalSstFileInfo file3_info;
  {
    SstFileWriter sst_file_writer(EnvOptions(), options);
    std::string file1 = sst_files_dir_ + "file1.sst";
    ASSERT_OK(sst_file_writer.Open(file1));
    ASSERT_OK(sst_file_writer.Put("a", "a1"));
-    ExternalSstFileInfo file1_info;
    ASSERT_OK(sst_file_writer.Finish(&file1_info));
    files.push_back(std::move(file1));
    std::string file2 = sst_files_dir_ + "file2.sst";
@@ -2056,7 +2264,6 @@ TEST_F(ExternalSSTFileBasicTest,
    std::string file3 = sst_files_dir_ + "file3.sst";
    ASSERT_OK(sst_file_writer.Open(file3));
    ASSERT_OK(sst_file_writer.Put("x", "x1"));
-    ExternalSstFileInfo file3_info;
    ASSERT_OK(sst_file_writer.Finish(&file3_info));
    files.push_back(std::move(file3));
  }
@@ -2104,9 +2311,15 @@ TEST_F(ExternalSSTFileBasicTest,
        size_t* num_input_files = static_cast<size_t*>(arg);
        EXPECT_EQ(1, *num_input_files);
        num_compactions += 1;
+        if (num_compactions == 2) {
+          compaction_listener->SetExpectedCompactionSkippedFileSize(
+              file3_info.file_size);
+        }
      });
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();

+  compaction_listener->SetExpectedCompactionSkippedFileSize(
+      file1_info.file_size);
  ASSERT_OK(db_->IngestExternalFile(files, ifo));

  ASSERT_OK(dbfull()->TEST_WaitForCompact());
@@ -2118,12 +2331,35 @@ TEST_F(ExternalSSTFileBasicTest,
  ASSERT_EQ(Get("a"), "a2");
  ASSERT_EQ(Get("h"), "h");
  ASSERT_EQ(Get("x"), "x2");
+  VerifyInputFilesInternalStatsForOutputLevel(
+      /*output_level*/ 6,
+      kCompactionNumInputFiles - kCompactionNumInputFilesAtOutputLevel,
+      kCompactionNumInputFilesAtOutputLevel,
+      kCompactionNumFilteredInputFiles -
+          kCompactionNumFilteredInputFilesAtOutputLevel,
+      kCompactionNumFilteredInputFilesAtOutputLevel,
+      /*bytes_skipped_non_output_levels*/ 0,
+      /*bytes_skipped_output_level*/ file1_info.file_size +
+          file3_info.file_size);
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }

 TEST_F(ExternalSSTFileBasicTest, StandaloneRangeDeletionEndKeyIsExclusive) {
  Options options = CurrentOptions();
  options.compaction_style = CompactionStyle::kCompactionStyleUniversal;
+  int kCompactionNumInputFiles = 2;
+  int kCompactionNumInputFilesAtOutputLevel = 1;
+  int kCompactionNumFilteredInputFiles = 0;
+  int kCompactionNumFilteredInputFilesAtOutputLevel = 0;
+  auto compaction_listener =
+      std::make_shared<CompactionJobStatsCheckerForFilteredFiles>(
+          kCompactionNumInputFiles, kCompactionNumInputFilesAtOutputLevel,
+          kCompactionNumFilteredInputFiles,
+          kCompactionNumFilteredInputFilesAtOutputLevel);
+  options.listeners.push_back(compaction_listener);
+  // No compaction input files are filtered because the range deletion file's
+  // end is exclusive, so it cannot cover the whole file.
+  compaction_listener->SetExpectedCompactionSkippedFileSize(0);
  DestroyAndReopen(options);

  std::vector<std::string> files;
@@ -2178,6 +2414,16 @@ TEST_F(ExternalSSTFileBasicTest, StandaloneRangeDeletionEndKeyIsExclusive) {

  ASSERT_EQ(Get("a"), "NOT_FOUND");
  ASSERT_EQ(Get("b"), "b");
+
+  VerifyInputFilesInternalStatsForOutputLevel(
+      /*output_level*/ 6,
+      kCompactionNumInputFiles - kCompactionNumInputFilesAtOutputLevel,
+      kCompactionNumInputFilesAtOutputLevel,
+      kCompactionNumFilteredInputFiles -
+          kCompactionNumFilteredInputFilesAtOutputLevel,
+      kCompactionNumFilteredInputFilesAtOutputLevel,
+      /*bytes_skipped_non_output_levels*/ 0,
+      /*bytes_skipped_output_level*/ 0);
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }

@@ -2575,6 +2821,57 @@ TEST_F(ExternalSSTFileBasicTest, StableSnapshotWhileLoggingToManifest) {
  ASSERT_EQ(db_->GetLatestSequenceNumber(), ingested_file_seqno + 1);
 }

+TEST_F(ExternalSSTFileBasicTest, ConcurrentIngestionAndDropColumnFamily) {
+  int kNumCFs = 10;
+  Options options = CurrentOptions();
+  CreateColumnFamilies({"cf_0", "cf_1", "cf_2", "cf_3", "cf_4", "cf_5", "cf_6",
+                        "cf_7", "cf_8", "cf_9"},
+                       options);
+
+  IngestExternalFileArg ingest_arg;
+  IngestExternalFileOptions ifo;
+  std::string external_file = sst_files_dir_ + "/file_to_ingest.sst";
+  SstFileWriter sst_file_writer{EnvOptions(), CurrentOptions()};
+  ASSERT_OK(sst_file_writer.Open(external_file));
+  ASSERT_OK(sst_file_writer.Put("key", "value"));
+  ASSERT_OK(sst_file_writer.Finish());
+  ifo.move_files = false;
+  ingest_arg.external_files = {external_file};
+  ingest_arg.options = ifo;
+
+  std::vector<std::thread> threads;
+  threads.reserve(2 * kNumCFs);
+  std::atomic<int> success_ingestion_count = 0;
+  std::atomic<int> failed_ingestion_count = 0;
+  for (int i = 0; i < kNumCFs; i++) {
+    threads.emplace_back(
+        [this, i]() { ASSERT_OK(db_->DropColumnFamily(handles_[i])); });
+    threads.emplace_back([this, i, ingest_arg, &success_ingestion_count,
+                          &failed_ingestion_count]() {
+      IngestExternalFileArg arg_copy = ingest_arg;
+      arg_copy.column_family = handles_[i];
+      Status s = db_->IngestExternalFiles({arg_copy});
+      ReadOptions ropts;
+      std::string value;
+      if (s.ok()) {
+        ASSERT_OK(db_->Get(ropts, handles_[i], "key", &value));
+        ASSERT_EQ("value", value);
+        success_ingestion_count.fetch_add(1);
+      } else {
+        ASSERT_TRUE(db_->Get(ropts, handles_[i], "key", &value).IsNotFound());
+        failed_ingestion_count.fetch_add(1);
+      }
+    });
+  }
+
+  for (auto& t : threads) {
+    t.join();
+  }
+
+  ASSERT_EQ(kNumCFs, success_ingestion_count + failed_ingestion_count);
+  Close();
+}
+
 INSTANTIATE_TEST_CASE_P(ExternalSSTFileBasicTest, ExternalSSTFileBasicTest,
                        testing::Values(std::make_tuple(true, true),
                                        std::make_tuple(true, false),
@@ -95,6 +95,14 @@ Status ExternalSstFileIngestionJob::Prepare(
        "behind mode.");
  }

+  // Overlapping files need at least two different sequence numbers. If settings
+  // disables global seqno, ingestion will fail anyway, so fail fast in prepare.
+  if (!ingestion_options_.allow_global_seqno && files_overlap_) {
+    return Status::InvalidArgument(
+        "Global seqno is required, but disabled (because external files key "
+        "range overlaps).");
+  }
+
  if (ucmp_->timestamp_size() > 0 && files_overlap_) {
    return Status::NotSupported(
        "Files with overlapping ranges cannot be ingested to column "
@@ -387,8 +395,16 @@ Status ExternalSstFileIngestionJob::NeedsFlush(bool* flush_needed,
 // REQUIRES: we have become the only writer by entering both write_thread_ and
 // nonmem_write_thread_
 Status ExternalSstFileIngestionJob::Run() {
-  Status status;
  SuperVersion* super_version = cfd_->GetSuperVersion();
+  // If column family is flushed after Prepare and before Run, we should have a
+  // specific state of Memtables. The mutable Memtable should be empty, and the
+  // immutable Memtable list should be empty.
+  if (flushed_before_run_ && (super_version->imm->NumNotFlushed() != 0 ||
+                              !super_version->mem->IsEmpty())) {
+    return Status::TryAgain(
+        "Inconsistent memtable state detected when flushed before run.");
+  }
+  Status status;
 #ifndef NDEBUG
  // We should never run the job with a memtable that is overlapping
  // with the files we are ingesting
@@ -212,6 +212,8 @@ class ExternalSstFileIngestionJob {

  ~ExternalSstFileIngestionJob() { UnregisterRange(); }

+  ColumnFamilyData* GetColumnFamilyData() const { return cfd_; }
+
  // Prepare the job by copying external files into the DB.
  Status Prepare(const std::vector<std::string>& external_files_paths,
                 const std::vector<std::string>& files_checksums,
@@ -229,6 +231,8 @@ class ExternalSstFileIngestionJob {
  // Thread-safe
  Status NeedsFlush(bool* flush_needed, SuperVersion* super_version);

+  void SetFlushedBeforeRun() { flushed_before_run_ = true; }
+
  // Will execute the ingestion job and prepare edit() to be applied.
  // REQUIRES: Mutex held
  Status Run();
@@ -371,6 +375,10 @@ class ExternalSstFileIngestionJob {
  bool need_generate_file_checksum_{true};
  std::shared_ptr<IOTracer> io_tracer_;

+  // Flag indicating whether the column family is flushed after `Prepare` and
+  // before `Run`.
+  bool flushed_before_run_{false};
+
  // Below are variables used in (un)registering range for this ingestion job
  //
  // FileMetaData used in inputs of compactions equivalent to this ingestion
@@ -157,7 +157,7 @@ void FlushJob::ReportStartedFlush() {
  IOSTATS_RESET(bytes_written);
 }

-void FlushJob::ReportFlushInputSize(const autovector<MemTable*>& mems) {
+void FlushJob::ReportFlushInputSize(const autovector<ReadOnlyMemTable*>& mems) {
  uint64_t input_size = 0;
  for (auto* mem : mems) {
    input_size += mem->ApproximateMemoryUsage();
@@ -204,7 +204,7 @@ void FlushJob::PickMemTable() {
  // entries mems are (implicitly) sorted in ascending order by their created
  // time. We will use the first memtable's `edit` to keep the meta info for
  // this flush.
-  MemTable* m = mems_[0];
+  ReadOnlyMemTable* m = mems_[0];
  edit_ = m->GetEdits();
  edit_->SetPrevLogNumber(0);
  // SetLogNumber(log_num) indicates logs with number smaller than log_num
@@ -420,9 +420,10 @@ Status FlushJob::MemPurge() {
  std::vector<InternalIterator*> memtables;
  std::vector<std::unique_ptr<FragmentedRangeTombstoneIterator>>
      range_del_iters;
-  for (MemTable* m : mems_) {
+  for (ReadOnlyMemTable* m : mems_) {
    memtables.push_back(m->NewIterator(ro, /*seqno_to_time_mapping=*/nullptr,
-                                       &arena, /*prefix_extractor=*/nullptr));
+                                       &arena, /*prefix_extractor=*/nullptr,
+                                       /*for_flush=*/true));
    auto* range_del_iter = m->NewRangeTombstoneIterator(
        ro, kMaxSequenceNumber, true /* immutable_memtable */);
    if (range_del_iter != nullptr) {
@@ -624,10 +625,13 @@ Status FlushJob::MemPurge() {
        // Construct fragmented memtable range tombstones without mutex
        new_mem->ConstructFragmentedRangeTombstones();
        db_mutex_->Lock();
-        uint64_t new_mem_id = mems_[0]->GetID();
+        // Take the newest id, so that memtables in MemtableList don't have
+        // out-of-order memtable ids.
+        uint64_t new_mem_id = mems_.back()->GetID();

        new_mem->SetID(new_mem_id);
-        new_mem->SetNextLogNumber(mems_[0]->GetNextLogNumber());
+        // Take the latest memtable's next log number.
+        new_mem->SetNextLogNumber(mems_.back()->GetNextLogNumber());

        // This addition will not trigger another flush, because
        // we do not call EnqueuePendingFlush().
@@ -713,11 +717,11 @@ bool FlushJob::MemPurgeDecider(double threshold) {

  // Iterate over each memtable of the set.
  for (auto mem_iter = std::begin(mems_); mem_iter != std::end(mems_);
-       mem_iter++) {
-    MemTable* mt = *mem_iter;
+       ++mem_iter) {
+    ReadOnlyMemTable* mt = *mem_iter;

    // Else sample from the table.
-    uint64_t nentries = mt->num_entries();
+    uint64_t nentries = mt->NumEntries();
    // Corrected Cochran formula for small populations
    // (converges to n0 for large populations).
    uint64_t target_sample_size =
@@ -894,11 +898,12 @@ Status FlushJob::WriteLevel0Table() {
    TEST_SYNC_POINT_CALLBACK("FlushJob::WriteLevel0Table:num_memtables",
                             &mems_size);
    assert(job_context_);
-    for (MemTable* m : mems_) {
-      ROCKS_LOG_INFO(
-          db_options_.info_log,
-          "[%s] [JOB %d] Flushing memtable with next log file: %" PRIu64 "\n",
-          cfd_->GetName().c_str(), job_context_->job_id, m->GetNextLogNumber());
+    for (ReadOnlyMemTable* m : mems_) {
+      ROCKS_LOG_INFO(db_options_.info_log,
+                     "[%s] [JOB %d] Flushing memtable id %" PRIu64
+                     " with next log file: %" PRIu64 "\n",
+                     cfd_->GetName().c_str(), job_context_->job_id, m->GetID(),
+                     m->GetNextLogNumber());
      if (logical_strip_timestamp) {
        memtables.push_back(m->NewTimestampStrippingIterator(
            ro, /*seqno_to_time_mapping=*/nullptr, &arena,
@@ -906,7 +911,7 @@ Status FlushJob::WriteLevel0Table() {
      } else {
        memtables.push_back(
            m->NewIterator(ro, /*seqno_to_time_mapping=*/nullptr, &arena,
-                           /*prefix_extractor=*/nullptr));
+                           /*prefix_extractor=*/nullptr, /*for_flush=*/true));
      }
      auto* range_del_iter =
          logical_strip_timestamp
@@ -917,11 +922,11 @@ Status FlushJob::WriteLevel0Table() {
      if (range_del_iter != nullptr) {
        range_del_iters.emplace_back(range_del_iter);
      }
-      total_num_entries += m->num_entries();
-      total_num_deletes += m->num_deletes();
-      total_data_size += m->get_data_size();
+      total_num_entries += m->NumEntries();
+      total_num_deletes += m->NumDeletion();
+      total_data_size += m->GetDataSize();
      total_memory_usage += m->ApproximateMemoryUsage();
-      total_num_range_deletes += m->num_range_deletes();
+      total_num_range_deletes += m->NumRangeDeletion();
    }

    // TODO(cbi): when memtable is flushed due to number of range deletions
@@ -987,9 +992,10 @@ Status FlushJob::WriteLevel0Table() {
          cfd_->internal_comparator(), cfd_->internal_tbl_prop_coll_factories(),
          output_compression_, mutable_cf_options_.compression_opts,
          cfd_->GetID(), cfd_->GetName(), 0 /* level */,
-          false /* is_bottommost */, TableFileCreationReason::kFlush,
-          oldest_key_time, current_time, db_id_, db_session_id_,
-          0 /* target_file_size */, meta_.fd.GetNumber(),
+          current_time /* newest_key_time */, false /* is_bottommost */,
+          TableFileCreationReason::kFlush, oldest_key_time, current_time,
+          db_id_, db_session_id_, 0 /* target_file_size */,
+          meta_.fd.GetNumber(),
          preclude_last_level_min_seqno_ == kMaxSequenceNumber
              ? preclude_last_level_min_seqno_
              : std::min(earliest_snapshot_, preclude_last_level_min_seqno_));
@@ -1171,7 +1177,7 @@ void FlushJob::GetEffectiveCutoffUDTForPickedMemTables() {
    return;
  }
  // Find the newest user-defined timestamps from all the flushed memtables.
-  for (MemTable* m : mems_) {
+  for (const ReadOnlyMemTable* m : mems_) {
    Slice table_newest_udt = m->GetNewestUDT();
    // Empty memtables can be legitimately created and flushed, for example
    // by error recovery flush attempts.
@@ -1189,7 +1195,7 @@ void FlushJob::GetEffectiveCutoffUDTForPickedMemTables() {
 }

 void FlushJob::GetPrecludeLastLevelMinSeqno() {
-  if (cfd_->ioptions()->preclude_last_level_data_seconds == 0) {
+  if (mutable_cf_options_.preclude_last_level_data_seconds == 0) {
    return;
  }
  int64_t current_time = 0;
@@ -1202,8 +1208,8 @@ void FlushJob::GetPrecludeLastLevelMinSeqno() {
    SequenceNumber preserve_time_min_seqno;
    seqno_to_time_mapping_->GetCurrentTieringCutoffSeqnos(
        static_cast<uint64_t>(current_time),
-        cfd_->ioptions()->preserve_internal_time_seconds,
-        cfd_->ioptions()->preclude_last_level_data_seconds,
+        mutable_cf_options_.preserve_internal_time_seconds,
+        mutable_cf_options_.preclude_last_level_data_seconds,
        &preserve_time_min_seqno, &preclude_last_level_min_seqno_);
  }
 }
@@ -91,7 +91,7 @@ class FlushJob {
             bool* skipped_since_bg_error = nullptr,
             ErrorHandler* error_handler = nullptr);
  void Cancel();
-  const autovector<MemTable*>& GetMemTables() const { return mems_; }
+  const autovector<ReadOnlyMemTable*>& GetMemTables() const { return mems_; }

  std::list<std::unique_ptr<FlushJobInfo>>* GetCommittedFlushJobsInfo() {
    return &committed_flush_jobs_info_;
@@ -101,7 +101,7 @@ class FlushJob {
  friend class FlushJobTest_GetRateLimiterPriorityForWrite_Test;

  void ReportStartedFlush();
-  void ReportFlushInputSize(const autovector<MemTable*>& mems);
+  static void ReportFlushInputSize(const autovector<ReadOnlyMemTable*>& mems);
  void RecordFlushIOStats();
  Status WriteLevel0Table();

@@ -205,7 +205,9 @@ class FlushJob {

  // Variables below are set by PickMemTable():
  FileMetaData meta_;
-  autovector<MemTable*> mems_;
+  // Memtables to be flushed by this job.
+  // Ordered by increasing memtable id, i.e., oldest memtable first.
+  autovector<ReadOnlyMemTable*> mems_;
  VersionEdit* edit_;
  Version* base_;
  bool pick_memtable_called;
@@ -264,7 +264,7 @@ TEST_F(FlushJobTest, NonEmpty) {
  }
  mock::SortKVVector(&inserted_keys);

-  autovector<MemTable*> to_delete;
+  autovector<ReadOnlyMemTable*> to_delete;
  new_mem->ConstructFragmentedRangeTombstones();
  cfd->imm()->Add(new_mem, &to_delete);
  for (auto& m : to_delete) {
@@ -325,7 +325,7 @@ TEST_F(FlushJobTest, FlushMemTablesSingleColumnFamily) {
    }
  }

-  autovector<MemTable*> to_delete;
+  autovector<ReadOnlyMemTable*> to_delete;
  for (auto mem : new_mems) {
    mem->ConstructFragmentedRangeTombstones();
    cfd->imm()->Add(mem, &to_delete);
@@ -380,7 +380,7 @@ TEST_F(FlushJobTest, FlushMemtablesMultipleColumnFamilies) {
  std::vector<uint64_t> memtable_ids;
  std::vector<SequenceNumber> smallest_seqs;
  std::vector<SequenceNumber> largest_seqs;
-  autovector<MemTable*> to_delete;
+  autovector<ReadOnlyMemTable*> to_delete;
  SequenceNumber curr_seqno = 0;
  size_t k = 0;
  for (auto cfd : all_cfds) {
@@ -439,7 +439,7 @@ TEST_F(FlushJobTest, FlushMemtablesMultipleColumnFamilies) {
  for (auto& meta : file_metas) {
    file_meta_ptrs.push_back(&meta);
  }
-  autovector<const autovector<MemTable*>*> mems_list;
+  autovector<const autovector<ReadOnlyMemTable*>*> mems_list;
  for (size_t i = 0; i != all_cfds.size(); ++i) {
    const auto& mems = flush_jobs[i]->GetMemTables();
    mems_list.push_back(&mems);
@@ -528,7 +528,7 @@ TEST_F(FlushJobTest, Snapshots) {
  }
  mock::SortKVVector(&inserted_keys);

-  autovector<MemTable*> to_delete;
+  autovector<ReadOnlyMemTable*> to_delete;
  new_mem->ConstructFragmentedRangeTombstones();
  cfd->imm()->Add(new_mem, &to_delete);
  for (auto& m : to_delete) {
@@ -582,7 +582,7 @@ TEST_F(FlushJobTest, GetRateLimiterPriorityForWrite) {
    }
  }

-  autovector<MemTable*> to_delete;
+  autovector<ReadOnlyMemTable*> to_delete;
  for (auto mem : new_mems) {
    mem->ConstructFragmentedRangeTombstones();
    cfd->imm()->Add(mem, &to_delete);
@@ -654,7 +654,7 @@ TEST_F(FlushJobTest, ReplaceTimedPutWriteTimeWithPreferredSeqno) {
  InternalKey largest_internal_key("foo", SequenceNumber(18), kTypeValue);
  inserted_entries.push_back(
      {largest_internal_key.Encode().ToString(), "fval"});
-  autovector<MemTable*> to_delete;
+  autovector<ReadOnlyMemTable*> to_delete;
  new_mem->ConstructFragmentedRangeTombstones();
  cfd->imm()->Add(new_mem, &to_delete);
  for (auto& m : to_delete) {
@@ -744,7 +744,7 @@ class FlushJobTimestampTest

 TEST_P(FlushJobTimestampTest, AllKeysExpired) {
  ColumnFamilyData* cfd = versions_->GetColumnFamilySet()->GetDefault();
-  autovector<MemTable*> to_delete;
+  autovector<ReadOnlyMemTable*> to_delete;

  {
    MemTable* new_mem = cfd->ConstructNewMemtable(
@@ -810,7 +810,7 @@ TEST_P(FlushJobTimestampTest, AllKeysExpired) {

 TEST_P(FlushJobTimestampTest, NoKeyExpired) {
  ColumnFamilyData* cfd = versions_->GetColumnFamilySet()->GetDefault();
-  autovector<MemTable*> to_delete;
+  autovector<ReadOnlyMemTable*> to_delete;

  {
    MemTable* new_mem = cfd->ConstructNewMemtable(
@@ -714,7 +714,8 @@ void ForwardIterator::RebuildIterators(bool refresh_sv) {
      sv_->GetSeqnoToTimeMapping();
  mutable_iter_ =
      sv_->mem->NewIterator(read_options_, seqno_to_time_mapping, &arena_,
-                            sv_->mutable_cf_options.prefix_extractor.get());
+                            sv_->mutable_cf_options.prefix_extractor.get(),
+                            /*for_flush=*/false);
  sv_->imm->AddIterators(read_options_, seqno_to_time_mapping,
                         sv_->mutable_cf_options.prefix_extractor.get(),
                         &imm_iters_, &arena_);
@@ -784,7 +785,8 @@ void ForwardIterator::RenewIterators() {
      svnew->GetSeqnoToTimeMapping();
  mutable_iter_ =
      svnew->mem->NewIterator(read_options_, seqno_to_time_mapping, &arena_,
-                              svnew->mutable_cf_options.prefix_extractor.get());
+                              svnew->mutable_cf_options.prefix_extractor.get(),
+                              /*for_flush=*/false);
  svnew->imm->AddIterators(read_options_, seqno_to_time_mapping,
                           svnew->mutable_cf_options.prefix_extractor.get(),
                           &imm_iters_, &arena_);
@@ -1301,7 +1301,7 @@ bool InternalStats::HandleNumEntriesActiveMemTable(uint64_t* value,
                                                   DBImpl* /*db*/,
                                                   Version* /*version*/) {
  // Current number of entires in the active memtable
-  *value = cfd_->mem()->num_entries();
+  *value = cfd_->mem()->NumEntries();
  return true;
 }

@@ -1317,7 +1317,7 @@ bool InternalStats::HandleNumDeletesActiveMemTable(uint64_t* value,
                                                   DBImpl* /*db*/,
                                                   Version* /*version*/) {
  // Current number of entires in the active memtable
-  *value = cfd_->mem()->num_deletes();
+  *value = cfd_->mem()->NumDeletion();
  return true;
 }

@@ -1334,11 +1334,11 @@ bool InternalStats::HandleEstimateNumKeys(uint64_t* value, DBImpl* /*db*/,
  // Estimate number of entries in the column family:
  // Use estimated entries in tables + total entries in memtables.
  const auto* vstorage = cfd_->current()->storage_info();
-  uint64_t estimate_keys = cfd_->mem()->num_entries() +
+  uint64_t estimate_keys = cfd_->mem()->NumEntries() +
                           cfd_->imm()->current()->GetTotalNumEntries() +
                           vstorage->GetEstimatedActiveKeys();
  uint64_t estimate_deletes =
-      cfd_->mem()->num_deletes() + cfd_->imm()->current()->GetTotalNumDeletes();
+      cfd_->mem()->NumDeletion() + cfd_->imm()->current()->GetTotalNumDeletes();
  *value = estimate_keys > estimate_deletes * 2
               ? estimate_keys - (estimate_deletes * 2)
               : 0;
@@ -182,6 +182,14 @@ class InternalStats {
    // The number of bytes read from the compaction output level (table files)
    uint64_t bytes_read_output_level;

+    // The number of bytes skipped from all non-output levels because the input
+    // files are filtered by compaction optimizations.
+    uint64_t bytes_skipped_non_output_levels;
+
+    // The number of bytes skipped from the compaction output level because the
+    // input files are filtered by compaction optimizations.
+    uint64_t bytes_skipped_output_level;
+
    // The number of bytes read from blob files
    uint64_t bytes_read_blob;

@@ -201,6 +209,14 @@ class InternalStats {
    // The number of compaction input files in the output level (table files)
    int num_input_files_in_output_level;

+    // The number of non output level compaction input files that are filtered
+    // by compaction optimizations.
+    int num_filtered_input_files_in_non_output_levels;
+
+    // The number of output level compaction input files that are filtered by
+    // compaction optimizations.
+    int num_filtered_input_files_in_output_level;
+
    // The number of compaction output files (table files)
    int num_output_files;

@@ -228,12 +244,16 @@ class InternalStats {
          cpu_micros(0),
          bytes_read_non_output_levels(0),
          bytes_read_output_level(0),
+          bytes_skipped_non_output_levels(0),
+          bytes_skipped_output_level(0),
          bytes_read_blob(0),
          bytes_written(0),
          bytes_written_blob(0),
          bytes_moved(0),
          num_input_files_in_non_output_levels(0),
          num_input_files_in_output_level(0),
+          num_filtered_input_files_in_non_output_levels(0),
+          num_filtered_input_files_in_output_level(0),
          num_output_files(0),
          num_output_files_blob(0),
          num_input_records(0),
@@ -251,12 +271,16 @@ class InternalStats {
          cpu_micros(0),
          bytes_read_non_output_levels(0),
          bytes_read_output_level(0),
+          bytes_skipped_non_output_levels(0),
+          bytes_skipped_output_level(0),
          bytes_read_blob(0),
          bytes_written(0),
          bytes_written_blob(0),
          bytes_moved(0),
          num_input_files_in_non_output_levels(0),
          num_input_files_in_output_level(0),
+          num_filtered_input_files_in_non_output_levels(0),
+          num_filtered_input_files_in_output_level(0),
          num_output_files(0),
          num_output_files_blob(0),
          num_input_records(0),
@@ -280,6 +304,8 @@ class InternalStats {
          cpu_micros(c.cpu_micros),
          bytes_read_non_output_levels(c.bytes_read_non_output_levels),
          bytes_read_output_level(c.bytes_read_output_level),
+          bytes_skipped_non_output_levels(c.bytes_skipped_non_output_levels),
+          bytes_skipped_output_level(c.bytes_skipped_output_level),
          bytes_read_blob(c.bytes_read_blob),
          bytes_written(c.bytes_written),
          bytes_written_blob(c.bytes_written_blob),
@@ -287,6 +313,10 @@ class InternalStats {
          num_input_files_in_non_output_levels(
              c.num_input_files_in_non_output_levels),
          num_input_files_in_output_level(c.num_input_files_in_output_level),
+          num_filtered_input_files_in_non_output_levels(
+              c.num_filtered_input_files_in_non_output_levels),
+          num_filtered_input_files_in_output_level(
+              c.num_filtered_input_files_in_output_level),
          num_output_files(c.num_output_files),
          num_output_files_blob(c.num_output_files_blob),
          num_input_records(c.num_input_records),
@@ -304,6 +334,8 @@ class InternalStats {
      cpu_micros = c.cpu_micros;
      bytes_read_non_output_levels = c.bytes_read_non_output_levels;
      bytes_read_output_level = c.bytes_read_output_level;
+      bytes_skipped_non_output_levels = c.bytes_skipped_non_output_levels;
+      bytes_skipped_output_level = c.bytes_skipped_output_level;
      bytes_read_blob = c.bytes_read_blob;
      bytes_written = c.bytes_written;
      bytes_written_blob = c.bytes_written_blob;
@@ -311,6 +343,10 @@ class InternalStats {
      num_input_files_in_non_output_levels =
          c.num_input_files_in_non_output_levels;
      num_input_files_in_output_level = c.num_input_files_in_output_level;
+      num_filtered_input_files_in_non_output_levels =
+          c.num_filtered_input_files_in_non_output_levels;
+      num_filtered_input_files_in_output_level =
+          c.num_filtered_input_files_in_output_level;
      num_output_files = c.num_output_files;
      num_output_files_blob = c.num_output_files_blob;
      num_input_records = c.num_input_records;
@@ -330,12 +366,16 @@ class InternalStats {
      this->cpu_micros = 0;
      this->bytes_read_non_output_levels = 0;
      this->bytes_read_output_level = 0;
+      this->bytes_skipped_non_output_levels = 0;
+      this->bytes_skipped_output_level = 0;
      this->bytes_read_blob = 0;
      this->bytes_written = 0;
      this->bytes_written_blob = 0;
      this->bytes_moved = 0;
      this->num_input_files_in_non_output_levels = 0;
      this->num_input_files_in_output_level = 0;
+      this->num_filtered_input_files_in_non_output_levels = 0;
+      this->num_filtered_input_files_in_output_level = 0;
      this->num_output_files = 0;
      this->num_output_files_blob = 0;
      this->num_input_records = 0;
@@ -353,6 +393,9 @@ class InternalStats {
      this->cpu_micros += c.cpu_micros;
      this->bytes_read_non_output_levels += c.bytes_read_non_output_levels;
      this->bytes_read_output_level += c.bytes_read_output_level;
+      this->bytes_skipped_non_output_levels +=
+          c.bytes_skipped_non_output_levels;
+      this->bytes_skipped_output_level += c.bytes_skipped_output_level;
      this->bytes_read_blob += c.bytes_read_blob;
      this->bytes_written += c.bytes_written;
      this->bytes_written_blob += c.bytes_written_blob;
@@ -361,6 +404,10 @@ class InternalStats {
          c.num_input_files_in_non_output_levels;
      this->num_input_files_in_output_level +=
          c.num_input_files_in_output_level;
+      this->num_filtered_input_files_in_non_output_levels +=
+          c.num_filtered_input_files_in_non_output_levels;
+      this->num_filtered_input_files_in_output_level +=
+          c.num_filtered_input_files_in_output_level;
      this->num_output_files += c.num_output_files;
      this->num_output_files_blob += c.num_output_files_blob;
      this->num_input_records += c.num_input_records;
@@ -387,6 +434,9 @@ class InternalStats {
      this->cpu_micros -= c.cpu_micros;
      this->bytes_read_non_output_levels -= c.bytes_read_non_output_levels;
      this->bytes_read_output_level -= c.bytes_read_output_level;
+      this->bytes_skipped_non_output_levels -=
+          c.bytes_skipped_non_output_levels;
+      this->bytes_skipped_output_level -= c.bytes_skipped_output_level;
      this->bytes_read_blob -= c.bytes_read_blob;
      this->bytes_written -= c.bytes_written;
      this->bytes_written_blob -= c.bytes_written_blob;
@@ -395,6 +445,10 @@ class InternalStats {
          c.num_input_files_in_non_output_levels;
      this->num_input_files_in_output_level -=
          c.num_input_files_in_output_level;
+      this->num_filtered_input_files_in_non_output_levels -=
+          c.num_filtered_input_files_in_non_output_levels;
+      this->num_filtered_input_files_in_output_level -=
+          c.num_filtered_input_files_in_output_level;
      this->num_output_files -= c.num_output_files;
      this->num_output_files_blob -= c.num_output_files_blob;
      this->num_input_records -= c.num_input_records;
@@ -191,7 +191,7 @@ struct JobContext {
  std::vector<std::string> manifest_delete_files;

  // a list of memtables to be free
-  autovector<MemTable*> memtables_to_free;
+  autovector<ReadOnlyMemTable*> memtables_to_free;

  // contexts for installing superversions for multiple column families
  std::vector<SuperVersionContext> superversion_contexts;
@@ -313,11 +313,11 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
        break;

      default: {
-        char buf[40];
-        snprintf(buf, sizeof(buf), "unknown record type %u", record_type);
+        std::string reason =
+            "unknown record type " + std::to_string(record_type);
        ReportCorruption(
            (fragment.size() + (in_fragmented_record ? scratch->size() : 0)),
-            buf);
+            reason.c_str());
        in_fragmented_record = false;
        scratch->clear();
        break;
@@ -781,12 +781,11 @@ bool FragmentBufferedReader::ReadRecord(Slice* record, std::string* scratch,
        break;

      default: {
-        char buf[40];
-        snprintf(buf, sizeof(buf), "unknown record type %u",
-                 fragment_type_or_err);
+        std::string reason =
+            "unknown record type " + std::to_string(fragment_type_or_err);
        ReportCorruption(
            fragment.size() + (in_fragmented_record_ ? fragments_.size() : 0),
-            buf);
+            reason.c_str());
        in_fragmented_record_ = false;
        fragments_.clear();
        break;
@@ -0,0 +1,47 @@
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
+//
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#include "db/manifest_ops.h"
+
+#include "file/filename.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+Status GetCurrentManifestPath(const std::string& dbname, FileSystem* fs,
+                              bool is_retry, std::string* manifest_path,
+                              uint64_t* manifest_file_number) {
+  assert(fs != nullptr);
+  assert(manifest_path != nullptr);
+  assert(manifest_file_number != nullptr);
+
+  IOOptions opts;
+  std::string fname;
+  if (is_retry) {
+    opts.verify_and_reconstruct_read = true;
+  }
+  Status s = ReadFileToString(fs, CurrentFileName(dbname), opts, &fname);
+  if (!s.ok()) {
+    return s;
+  }
+  if (fname.empty() || fname.back() != '\n') {
+    return Status::Corruption("CURRENT file does not end with newline");
+  }
+  // remove the trailing '\n'
+  fname.resize(fname.size() - 1);
+  FileType type;
+  bool parse_ok = ParseFileName(fname, manifest_file_number, &type);
+  if (!parse_ok || type != kDescriptorFile) {
+    return Status::Corruption("CURRENT file corrupted");
+  }
+  *manifest_path = dbname;
+  if (dbname.back() != '/') {
+    manifest_path->push_back('/');
+  }
+  manifest_path->append(fname);
+  return Status::OK();
+}
+
+}  // namespace ROCKSDB_NAMESPACE
@@ -0,0 +1,20 @@
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
+//
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+#include <cassert>
+
+#include "rocksdb/env.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+// Parameter is_retry=true sets the verify_and_reconstruct_read flag.
+// It comes handy when caller intends to re-read the data with much stronger
+// data integrity checking - e.g. in case of a perceived file corruption.
+Status GetCurrentManifestPath(const std::string& dbname, FileSystem* fs,
+                              bool is_retry, std::string* manifest_path,
+                              uint64_t* manifest_file_number);
+}  // namespace ROCKSDB_NAMESPACE
@@ -79,7 +79,6 @@ MemTable::MemTable(const InternalKeyComparator& cmp,
                   SequenceNumber latest_seq, uint32_t column_family_id)
    : comparator_(cmp),
      moptions_(ioptions, mutable_cf_options),
-      refs_(0),
      kArenaBlockSize(Arena::OptimizeBlockSize(moptions_.arena_block_size)),
      mem_tracker_(write_buffer_manager),
      arena_(moptions_.arena_block_size,
@@ -101,13 +100,9 @@ MemTable::MemTable(const InternalKeyComparator& cmp,
      num_deletes_(0),
      num_range_deletes_(0),
      write_buffer_size_(mutable_cf_options.write_buffer_size),
-      flush_in_progress_(false),
-      flush_completed_(false),
-      file_number_(0),
      first_seqno_(0),
      earliest_seqno_(latest_seq),
      creation_seq_(latest_seq),
-      mem_next_logfile_number_(0),
      min_prep_log_referenced_(0),
      locks_(moptions_.inplace_update_support
                 ? moptions_.inplace_update_num_locks
@@ -118,7 +113,6 @@ MemTable::MemTable(const InternalKeyComparator& cmp,
      insert_with_hint_prefix_extractor_(
          ioptions.memtable_insert_with_hint_prefix_extractor.get()),
      oldest_key_time_(std::numeric_limits<uint64_t>::max()),
-      atomic_flush_seqno_(kMaxSequenceNumber),
      approximate_memory_usage_(0),
      memtable_max_range_deletions_(
          mutable_cf_options.memtable_max_range_deletions) {
@@ -605,7 +599,7 @@ class MemTableIterator : public InternalIterator {
 InternalIterator* MemTable::NewIterator(
    const ReadOptions& read_options,
    UnownedPtr<const SeqnoToTimeMapping> seqno_to_time_mapping, Arena* arena,
-    const SliceTransform* prefix_extractor) {
+    const SliceTransform* prefix_extractor, bool /*for_flush*/) {
  assert(arena != nullptr);
  auto mem = arena->AllocateAligned(sizeof(MemTableIterator));
  return new (mem)
@@ -832,8 +826,8 @@ port::RWMutex* MemTable::GetLock(const Slice& key) {
  return &locks_[GetSliceRangedNPHash(key, locks_.size())];
 }

-MemTable::MemTableStats MemTable::ApproximateStats(const Slice& start_ikey,
-                                                   const Slice& end_ikey) {
+ReadOnlyMemTable::MemTableStats MemTable::ApproximateStats(
+    const Slice& start_ikey, const Slice& end_ikey) {
  uint64_t entry_count = table_->ApproximateNumEntries(start_ikey, end_ikey);
  entry_count += range_del_table_->ApproximateNumEntries(start_ikey, end_ikey);
  if (entry_count == 0) {
@@ -1257,47 +1251,16 @@ static bool SaveValue(void* arg, const char* entry) {
      case kTypeValue:
      case kTypeValuePreferredSeqno: {
        Slice v = GetLengthPrefixedSlice(key_ptr + key_length);
-
        if (type == kTypeValuePreferredSeqno) {
          v = ParsePackedValueForValue(v);
        }

-        *(s->status) = Status::OK();
-
-        if (!s->do_merge) {
-          // Preserve the value with the goal of returning it as part of
-          // raw merge operands to the user
-          // TODO(yanqin) update MergeContext so that timestamps information
-          // can also be retained.
-
-          merge_context->PushOperand(
-              v, s->inplace_update_support == false /* operand_pinned */);
-        } else if (*(s->merge_in_progress)) {
-          assert(s->do_merge);
-
-          if (s->value || s->columns) {
-            // `op_failure_scope` (an output parameter) is not provided (set to
-            // nullptr) since a failure must be propagated regardless of its
-            // value.
-            *(s->status) = MergeHelper::TimedFullMerge(
-                merge_operator, s->key->user_key(),
-                MergeHelper::kPlainBaseValue, v, merge_context->GetOperands(),
-                s->logger, s->statistics, s->clock,
-                /* update_num_ops_stats */ true, /* op_failure_scope */ nullptr,
-                s->value, s->columns);
-          }
-        } else if (s->value) {
-          s->value->assign(v.data(), v.size());
-        } else if (s->columns) {
-          s->columns->SetPlainValue(v);
-        }
-
+        ReadOnlyMemTable::HandleTypeValue(
+            s->key->user_key(), v, s->inplace_update_support == false,
+            s->do_merge, *(s->merge_in_progress), merge_context,
+            s->merge_operator, s->clock, s->statistics, s->logger, s->status,
+            s->value, s->columns, s->is_blob_index);
        *(s->found_final_value) = true;
-
-        if (s->is_blob_index != nullptr) {
-          *(s->is_blob_index) = false;
-        }
-
        return false;
      }
      case kTypeWideColumnEntity: {
@@ -1354,25 +1317,10 @@ static bool SaveValue(void* arg, const char* entry) {
      case kTypeDeletionWithTimestamp:
      case kTypeSingleDeletion:
      case kTypeRangeDeletion: {
-        if (*(s->merge_in_progress)) {
-          if (s->value || s->columns) {
-            // `op_failure_scope` (an output parameter) is not provided (set to
-            // nullptr) since a failure must be propagated regardless of its
-            // value.
-            *(s->status) = MergeHelper::TimedFullMerge(
-                merge_operator, s->key->user_key(), MergeHelper::kNoBaseValue,
-                merge_context->GetOperands(), s->logger, s->statistics,
-                s->clock, /* update_num_ops_stats */ true,
-                /* op_failure_scope */ nullptr, s->value, s->columns);
-          } else {
-            // We have found a final value (a base deletion) and have newer
-            // merge operands that we do not intend to merge. Nothing remains
-            // to be done so assign status to OK.
-            *(s->status) = Status::OK();
-          }
-        } else {
-          *(s->status) = Status::NotFound();
-        }
+        ReadOnlyMemTable::HandleTypeDeletion(
+            s->key->user_key(), *(s->merge_in_progress), s->merge_context,
+            s->merge_operator, s->clock, s->statistics, s->logger, s->status,
+            s->value, s->columns);
        *(s->found_final_value) = true;
        return false;
      }
@@ -18,6 +18,7 @@

 #include "db/dbformat.h"
 #include "db/kv_checksum.h"
+#include "db/merge_helper.h"
 #include "db/range_tombstone_fragmenter.h"
 #include "db/read_callback.h"
 #include "db/seqno_to_time_mapping.h"
@@ -76,88 +77,48 @@ struct MemTablePostProcessInfo {
 };

 using MultiGetRange = MultiGetContext::Range;
-// Note:  Many of the methods in this class have comments indicating that
+
+// For each CF, rocksdb maintains an active memtable that accept writes,
+// and zero or more sealed memtables that we call immutable memtables.
+// This interface contains all methods required for immutable memtables.
+// MemTable class inherit from `ReadOnlyMemTable` and implements additional
+// methods required for active memtables.
+// Immutable memtable list (MemTableList) maintains a list of ReadOnlyMemTable
+// objects. This interface enables feature like direct ingestion of an
+// immutable memtable with custom implementation, bypassing memtable writes.
+//
+// Note: Many of the methods in this class have comments indicating that
 // external synchronization is required as these methods are not thread-safe.
 // It is up to higher layers of code to decide how to prevent concurrent
-// invocation of these methods.  This is usually done by acquiring either
+// invocation of these methods. This is usually done by acquiring either
 // the db mutex or the single writer thread.
 //
 // Some of these methods are documented to only require external
-// synchronization if this memtable is immutable.  Calling MarkImmutable() is
+// synchronization if this memtable is immutable. Calling MarkImmutable() is
 // not sufficient to guarantee immutability.  It is up to higher layers of
 // code to determine if this MemTable can still be modified by other threads.
 // Eg: The Superversion stores a pointer to the current MemTable (that can
 // be modified) and a separate list of the MemTables that can no longer be
 // written to (aka the 'immutable memtables').
-class MemTable {
+//
+// MemTables are reference counted. The initial reference count
+// is zero and the caller must call Ref() at least once.
+class ReadOnlyMemTable {
 public:
-  struct KeyComparator : public MemTableRep::KeyComparator {
-    const InternalKeyComparator comparator;
-    explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) {}
-    int operator()(const char* prefix_len_key1,
-                   const char* prefix_len_key2) const override;
-    int operator()(const char* prefix_len_key,
-                   const DecodedType& key) const override;
-  };
-
-  // MemTables are reference counted.  The initial reference count
-  // is zero and the caller must call Ref() at least once.
-  //
-  // earliest_seq should be the current SequenceNumber in the db such that any
-  // key inserted into this memtable will have an equal or larger seq number.
-  // (When a db is first created, the earliest sequence number will be 0).
-  // If the earliest sequence number is not known, kMaxSequenceNumber may be
-  // used, but this may prevent some transactions from succeeding until the
-  // first key is inserted into the memtable.
-  explicit MemTable(const InternalKeyComparator& comparator,
-                    const ImmutableOptions& ioptions,
-                    const MutableCFOptions& mutable_cf_options,
-                    WriteBufferManager* write_buffer_manager,
-                    SequenceNumber earliest_seq, uint32_t column_family_id);
-  // No copying allowed
-  MemTable(const MemTable&) = delete;
-  MemTable& operator=(const MemTable&) = delete;
-
  // Do not delete this MemTable unless Unref() indicates it not in use.
-  ~MemTable();
+  virtual ~ReadOnlyMemTable() = default;

-  // Increase reference count.
-  // REQUIRES: external synchronization to prevent simultaneous
-  // operations on the same MemTable.
-  void Ref() { ++refs_; }
-
-  // Drop reference count.
-  // If the refcount goes to zero return this memtable, otherwise return null.
-  // REQUIRES: external synchronization to prevent simultaneous
-  // operations on the same MemTable.
-  MemTable* Unref() {
-    --refs_;
-    assert(refs_ >= 0);
-    if (refs_ <= 0) {
-      return this;
-    }
-    return nullptr;
-  }
+  virtual const char* Name() const = 0;

  // Returns an estimate of the number of bytes of data in use by this
  // data structure.
  //
  // REQUIRES: external synchronization to prevent simultaneous
  // operations on the same MemTable (unless this Memtable is immutable).
-  size_t ApproximateMemoryUsage();
-
-  // As a cheap version of `ApproximateMemoryUsage()`, this function doesn't
-  // require external synchronization. The value may be less accurate though
-  size_t ApproximateMemoryUsageFast() const {
-    return approximate_memory_usage_.load(std::memory_order_relaxed);
-  }
+  virtual size_t ApproximateMemoryUsage() = 0;

  // used by MemTableListVersion::MemoryAllocatedBytesExcludingLast
-  size_t MemoryAllocatedBytes() const {
-    return table_->ApproximateMemoryUsage() +
-           range_del_table_->ApproximateMemoryUsage() +
-           arena_.MemoryAllocatedBytes();
-  }
+  virtual size_t MemoryAllocatedBytes() const = 0;

  // Returns a vector of unique random memtable entries of size 'sample_size'.
  //
@@ -172,27 +133,8 @@ class MemTable {
  // REQUIRES: SkipList memtable representation. This function is not
  // implemented for any other type of memtable representation (vectorrep,
  // hashskiplist,...).
-  void UniqueRandomSample(const uint64_t& target_sample_size,
-                          std::unordered_set<const char*>* entries) {
-    // TODO(bjlemaire): at the moment, only supported by skiplistrep.
-    // Extend it to all other memtable representations.
-    table_->UniqueRandomSample(num_entries(), target_sample_size, entries);
-  }
-
-  // This method heuristically determines if the memtable should continue to
-  // host more data.
-  bool ShouldScheduleFlush() const {
-    return flush_state_.load(std::memory_order_relaxed) == FLUSH_REQUESTED;
-  }
-
-  // Returns true if a flush should be scheduled and the caller should
-  // be the one to schedule it
-  bool MarkFlushScheduled() {
-    auto before = FLUSH_REQUESTED;
-    return flush_state_.compare_exchange_strong(before, FLUSH_SCHEDULED,
-                                                std::memory_order_relaxed,
-                                                std::memory_order_relaxed);
-  }
+  virtual void UniqueRandomSample(const uint64_t& target_sample_size,
+                                  std::unordered_set<const char*>* entries) = 0;

  // Return an iterator that yields the contents of the memtable.
  //
@@ -208,18 +150,18 @@ class MemTable {
  //        those allocated in arena.
  // seqno_to_time_mapping: it's used to support return write unix time for the
  // data, currently only needed for iterators serving user reads.
-  InternalIterator* NewIterator(
+  virtual InternalIterator* NewIterator(
      const ReadOptions& read_options,
      UnownedPtr<const SeqnoToTimeMapping> seqno_to_time_mapping, Arena* arena,
-      const SliceTransform* prefix_extractor);
+      const SliceTransform* prefix_extractor, bool for_flush) = 0;

  // Returns an iterator that wraps a MemTableIterator and logically strips the
  // user-defined timestamp of each key. This API is only used by flush when
  // user-defined timestamps in MemTable only feature is enabled.
-  InternalIterator* NewTimestampStrippingIterator(
+  virtual InternalIterator* NewTimestampStrippingIterator(
      const ReadOptions& read_options,
      UnownedPtr<const SeqnoToTimeMapping> seqno_to_time_mapping, Arena* arena,
-      const SliceTransform* prefix_extractor, size_t ts_sz);
+      const SliceTransform* prefix_extractor, size_t ts_sz) = 0;

  // Returns an iterator that yields the range tombstones of the memtable.
  // The caller must ensure that the underlying MemTable remains live
@@ -231,38 +173,23 @@ class MemTable {
  // is constructed when a memtable becomes immutable. Setting the flag to false
  // will always yield correct result, but may incur performance penalty as it
  // always creates a new fragmented range tombstone list.
-  FragmentedRangeTombstoneIterator* NewRangeTombstoneIterator(
+  virtual FragmentedRangeTombstoneIterator* NewRangeTombstoneIterator(
      const ReadOptions& read_options, SequenceNumber read_seq,
-      bool immutable_memtable);
+      bool immutable_memtable) = 0;

  // Returns an iterator that yields the range tombstones of the memtable and
  // logically strips the user-defined timestamp of each key (including start
  // key, and end key). This API is only used by flush when user-defined
  // timestamps in MemTable only feature is enabled.
-  FragmentedRangeTombstoneIterator* NewTimestampStrippingRangeTombstoneIterator(
-      const ReadOptions& read_options, SequenceNumber read_seq, size_t ts_sz);
-
-  Status VerifyEncodedEntry(Slice encoded,
-                            const ProtectionInfoKVOS64& kv_prot_info);
-
-  // Add an entry into memtable that maps key to value at the
-  // specified sequence number and with the specified type.
-  // Typically value will be empty if type==kTypeDeletion.
-  //
-  // REQUIRES: if allow_concurrent = false, external synchronization to prevent
-  // simultaneous operations on the same MemTable.
-  //
-  // Returns `Status::TryAgain` if the `seq`, `key` combination already exists
-  // in the memtable and `MemTableRepFactory::CanHandleDuplicatedKey()` is true.
-  // The next attempt should try a larger value for `seq`.
-  Status Add(SequenceNumber seq, ValueType type, const Slice& key,
-             const Slice& value, const ProtectionInfoKVOS64* kv_prot_info,
-             bool allow_concurrent = false,
-             MemTablePostProcessInfo* post_process_info = nullptr,
-             void** hint = nullptr);
+  virtual FragmentedRangeTombstoneIterator*
+  NewTimestampStrippingRangeTombstoneIterator(const ReadOptions& read_options,
+                                              SequenceNumber read_seq,
+                                              size_t ts_sz) = 0;

  // Used to Get value associated with key or Get Merge Operands associated
  // with key.
+  // Keys are considered if they are no larger than the parameter `key` in
+  // the order defined by comparator and share the save user key with `key`.
  // If do_merge = true the default behavior which is Get value for key is
  // executed. Expected behavior is described right below.
  // If memtable contains a value for key, store it in *value and return true.
@@ -291,14 +218,13 @@ class MemTable {
  // @param immutable_memtable Whether this memtable is immutable. Used
  // internally by NewRangeTombstoneIterator(). See comment above
  // NewRangeTombstoneIterator() for more detail.
-  bool Get(const LookupKey& key, std::string* value,
-           PinnableWideColumns* columns, std::string* timestamp, Status* s,
-           MergeContext* merge_context,
-           SequenceNumber* max_covering_tombstone_seq, SequenceNumber* seq,
-           const ReadOptions& read_opts, bool immutable_memtable,
-           ReadCallback* callback = nullptr, bool* is_blob_index = nullptr,
-           bool do_merge = true);
-
+  virtual bool Get(const LookupKey& key, std::string* value,
+                   PinnableWideColumns* columns, std::string* timestamp,
+                   Status* s, MergeContext* merge_context,
+                   SequenceNumber* max_covering_tombstone_seq,
+                   SequenceNumber* seq, const ReadOptions& read_opts,
+                   bool immutable_memtable, ReadCallback* callback = nullptr,
+                   bool* is_blob_index = nullptr, bool do_merge = true) = 0;
  bool Get(const LookupKey& key, std::string* value,
           PinnableWideColumns* columns, std::string* timestamp, Status* s,
           MergeContext* merge_context,
@@ -315,8 +241,351 @@ class MemTable {
  // @param immutable_memtable Whether this memtable is immutable. Used
  // internally by NewRangeTombstoneIterator(). See comment above
  // NewRangeTombstoneIterator() for more detail.
+  virtual void MultiGet(const ReadOptions& read_options, MultiGetRange* range,
+                        ReadCallback* callback, bool immutable_memtable) = 0;
+
+  // Get total number of entries in the mem table.
+  // REQUIRES: external synchronization to prevent simultaneous
+  // operations on the same MemTable (unless this Memtable is immutable).
+  virtual uint64_t NumEntries() const = 0;
+
+  // Get total number of point deletes in the mem table.
+  // REQUIRES: external synchronization to prevent simultaneous
+  // operations on the same MemTable (unless this Memtable is immutable).
+  virtual uint64_t NumDeletion() const = 0;
+
+  // Get total number of range deletions in the mem table.
+  // REQUIRES: external synchronization to prevent simultaneous
+  // operations on the same MemTable (unless this Memtable is immutable).
+  virtual uint64_t NumRangeDeletion() const = 0;
+
+  virtual uint64_t GetDataSize() const = 0;
+
+  // Returns the sequence number of the first element that was inserted
+  // into the memtable.
+  // REQUIRES: external synchronization to prevent simultaneous
+  // operations on the same MemTable (unless this Memtable is immutable).
+  virtual SequenceNumber GetFirstSequenceNumber() = 0;
+
+  // Returns if there is no entry inserted to the mem table.
+  // REQUIRES: external synchronization to prevent simultaneous
+  // operations on the same MemTable (unless this Memtable is immutable).
+  virtual bool IsEmpty() const = 0;
+
+  // Returns the sequence number that is guaranteed to be smaller than or equal
+  // to the sequence number of any key that could be inserted into this
+  // memtable. It can then be assumed that any write with a larger(or equal)
+  // sequence number will be present in this memtable or a later memtable.
+  //
+  // If the earliest sequence number could not be determined,
+  // kMaxSequenceNumber will be returned.
+  virtual SequenceNumber GetEarliestSequenceNumber() = 0;
+
+  virtual uint64_t GetMinLogContainingPrepSection() = 0;
+
+  // Notify the underlying storage that no more items will be added.
+  // REQUIRES: external synchronization to prevent simultaneous
+  // operations on the same MemTable.
+  // After MarkImmutable() is called, you should not attempt to
+  // write anything to this MemTable().  (Ie. do not call Add() or Update()).
+  virtual void MarkImmutable() = 0;
+
+  // Notify the underlying storage that all data it contained has been
+  // persisted.
+  // REQUIRES: external synchronization to prevent simultaneous
+  // operations on the same MemTable.
+  virtual void MarkFlushed() = 0;
+
+  struct MemTableStats {
+    uint64_t size;
+    uint64_t count;
+  };
+  virtual MemTableStats ApproximateStats(const Slice& start_ikey,
+                                         const Slice& end_ikey) = 0;
+
+  virtual const InternalKeyComparator& GetInternalKeyComparator() const = 0;
+
+  virtual uint64_t ApproximateOldestKeyTime() const = 0;
+
+  // Returns whether a fragmented range tombstone list is already constructed
+  // for this memtable. It should be constructed right before a memtable is
+  // added to an immutable memtable list. Note that if a memtable does not have
+  // any range tombstone, then no range tombstone list will ever be constructed
+  // and true is returned in that case.
+  virtual bool IsFragmentedRangeTombstonesConstructed() const = 0;
+
+  // Get the newest user-defined timestamp contained in this MemTable. Check
+  // `newest_udt_` for what newer means. This method should only be invoked for
+  // an MemTable that has enabled user-defined timestamp feature and set
+  // `persist_user_defined_timestamps` to false. The tracked newest UDT will be
+  // used by flush job in the background to help check the MemTable's
+  // eligibility for Flush.
+  virtual const Slice& GetNewestUDT() const = 0;
+
+  // Increase reference count.
+  // REQUIRES: external synchronization to prevent simultaneous
+  // operations on the same MemTable.
+  void Ref() { ++refs_; }
+
+  // Drop reference count.
+  // If the refcount goes to zero return this memtable, otherwise return null.
+  // REQUIRES: external synchronization to prevent simultaneous
+  // operations on the same MemTable.
+  ReadOnlyMemTable* Unref() {
+    --refs_;
+    assert(refs_ >= 0);
+    if (refs_ <= 0) {
+      return this;
+    }
+    return nullptr;
+  }
+
+  // Returns the edits area that is needed for flushing the memtable
+  VersionEdit* GetEdits() { return &edit_; }
+
+  // Returns the next active logfile number when this memtable is about to
+  // be flushed to storage
+  // REQUIRES: external synchronization to prevent simultaneous
+  // operations on the same MemTable.
+  uint64_t GetNextLogNumber() const { return mem_next_logfile_number_; }
+
+  // Sets the next active logfile number when this memtable is about to
+  // be flushed to storage
+  // REQUIRES: external synchronization to prevent simultaneous
+  // operations on the same MemTable.
+  void SetNextLogNumber(uint64_t num) { mem_next_logfile_number_ = num; }
+
+  // REQUIRES: db_mutex held.
+  void SetID(uint64_t id) { id_ = id; }
+
+  uint64_t GetID() const { return id_; }
+
+  void SetFlushCompleted(bool completed) { flush_completed_ = completed; }
+
+  uint64_t GetFileNumber() const { return file_number_; }
+
+  void SetFileNumber(uint64_t file_num) { file_number_ = file_num; }
+
+  void SetFlushInProgress(bool in_progress) {
+    flush_in_progress_ = in_progress;
+  }
+
+  void SetFlushJobInfo(std::unique_ptr<FlushJobInfo>&& info) {
+    flush_job_info_ = std::move(info);
+  }
+
+  std::unique_ptr<FlushJobInfo> ReleaseFlushJobInfo() {
+    return std::move(flush_job_info_);
+  }
+
+  static void HandleTypeValue(
+      const Slice& lookup_user_key, const Slice& value, bool value_pinned,
+      bool do_merge, bool merge_in_progress, MergeContext* merge_context,
+      const MergeOperator* merge_operator, SystemClock* clock,
+      Statistics* statistics, Logger* info_log, Status* s,
+      std::string* out_value, PinnableWideColumns* out_columns,
+      bool* is_blob_index) {
+    *s = Status::OK();
+
+    if (!do_merge) {
+      // Preserve the value with the goal of returning it as part of
+      // raw merge operands to the user
+      // TODO(yanqin) update MergeContext so that timestamps information
+      // can also be retained.
+      merge_context->PushOperand(value, value_pinned);
+    } else if (merge_in_progress) {
+      assert(do_merge);
+      // `op_failure_scope` (an output parameter) is not provided (set to
+      // nullptr) since a failure must be propagated regardless of its
+      // value.
+      if (out_value || out_columns) {
+        *s = MergeHelper::TimedFullMerge(
+            merge_operator, lookup_user_key, MergeHelper::kPlainBaseValue,
+            value, merge_context->GetOperands(), info_log, statistics, clock,
+            /* update_num_ops_stats */ true,
+            /* op_failure_scope */ nullptr, out_value, out_columns);
+      }
+    } else if (out_value) {
+      out_value->assign(value.data(), value.size());
+    } else if (out_columns) {
+      out_columns->SetPlainValue(value);
+    }
+
+    if (is_blob_index) {
+      *is_blob_index = false;
+    }
+  }
+
+  static void HandleTypeDeletion(
+      const Slice& lookup_user_key, bool merge_in_progress,
+      MergeContext* merge_context, const MergeOperator* merge_operator,
+      SystemClock* clock, Statistics* statistics, Logger* logger, Status* s,
+      std::string* out_value, PinnableWideColumns* out_columns) {
+    if (merge_in_progress) {
+      if (out_value || out_columns) {
+        // `op_failure_scope` (an output parameter) is not provided (set to
+        // nullptr) since a failure must be propagated regardless of its
+        // value.
+        *s = MergeHelper::TimedFullMerge(
+            merge_operator, lookup_user_key, MergeHelper::kNoBaseValue,
+            merge_context->GetOperands(), logger, statistics, clock,
+            /* update_num_ops_stats */ true,
+            /* op_failure_scope */ nullptr, out_value, out_columns);
+      } else {
+        // We have found a final value (a base deletion) and have newer
+        // merge operands that we do not intend to merge. Nothing remains
+        // to be done so assign status to OK.
+        *s = Status::OK();
+      }
+    } else {
+      *s = Status::NotFound();
+    }
+  }
+
+ protected:
+  friend class MemTableList;
+
+  int refs_{0};
+
+  // These are used to manage memtable flushes to storage
+  bool flush_in_progress_{false};  // started the flush
+  bool flush_completed_{false};    // finished the flush
+  uint64_t file_number_{0};
+
+  // The updates to be applied to the transaction log when this
+  // memtable is flushed to storage.
+  VersionEdit edit_;
+
+  // The log files earlier than this number can be deleted.
+  uint64_t mem_next_logfile_number_{0};
+
+  // Memtable id to track flush.
+  uint64_t id_ = 0;
+
+  // Sequence number of the atomic flush that is responsible for this memtable.
+  // The sequence number of atomic flush is a seq, such that no writes with
+  // sequence numbers greater than or equal to seq are flushed, while all
+  // writes with sequence number smaller than seq are flushed.
+  SequenceNumber atomic_flush_seqno_{kMaxSequenceNumber};
+
+  // Flush job info of the current memtable.
+  std::unique_ptr<FlushJobInfo> flush_job_info_;
+};
+
+class MemTable final : public ReadOnlyMemTable {
+ public:
+  struct KeyComparator final : public MemTableRep::KeyComparator {
+    const InternalKeyComparator comparator;
+    explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) {}
+    int operator()(const char* prefix_len_key1,
+                   const char* prefix_len_key2) const override;
+    int operator()(const char* prefix_len_key,
+                   const DecodedType& key) const override;
+  };
+
+  // earliest_seq should be the current SequenceNumber in the db such that any
+  // key inserted into this memtable will have an equal or larger seq number.
+  // (When a db is first created, the earliest sequence number will be 0).
+  // If the earliest sequence number is not known, kMaxSequenceNumber may be
+  // used, but this may prevent some transactions from succeeding until the
+  // first key is inserted into the memtable.
+  explicit MemTable(const InternalKeyComparator& comparator,
+                    const ImmutableOptions& ioptions,
+                    const MutableCFOptions& mutable_cf_options,
+                    WriteBufferManager* write_buffer_manager,
+                    SequenceNumber earliest_seq, uint32_t column_family_id);
+  // No copying allowed
+  MemTable(const MemTable&) = delete;
+  MemTable& operator=(const MemTable&) = delete;
+
+  ~MemTable() override;
+
+  const char* Name() const override { return "MemTable"; }
+
+  size_t ApproximateMemoryUsage() override;
+
+  // As a cheap version of `ApproximateMemoryUsage()`, this function doesn't
+  // require external synchronization. The value may be less accurate though
+  size_t ApproximateMemoryUsageFast() const {
+    return approximate_memory_usage_.load(std::memory_order_relaxed);
+  }
+
+  size_t MemoryAllocatedBytes() const override {
+    return table_->ApproximateMemoryUsage() +
+           range_del_table_->ApproximateMemoryUsage() +
+           arena_.MemoryAllocatedBytes();
+  }
+
+  void UniqueRandomSample(const uint64_t& target_sample_size,
+                          std::unordered_set<const char*>* entries) override {
+    // TODO(bjlemaire): at the moment, only supported by skiplistrep.
+    // Extend it to all other memtable representations.
+    table_->UniqueRandomSample(NumEntries(), target_sample_size, entries);
+  }
+
+  // This method heuristically determines if the memtable should continue to
+  // host more data.
+  bool ShouldScheduleFlush() const {
+    return flush_state_.load(std::memory_order_relaxed) == FLUSH_REQUESTED;
+  }
+
+  // Returns true if a flush should be scheduled and the caller should
+  // be the one to schedule it
+  bool MarkFlushScheduled() {
+    auto before = FLUSH_REQUESTED;
+    return flush_state_.compare_exchange_strong(before, FLUSH_SCHEDULED,
+                                                std::memory_order_relaxed,
+                                                std::memory_order_relaxed);
+  }
+
+  InternalIterator* NewIterator(
+      const ReadOptions& read_options,
+      UnownedPtr<const SeqnoToTimeMapping> seqno_to_time_mapping, Arena* arena,
+      const SliceTransform* prefix_extractor, bool for_flush) override;
+
+  InternalIterator* NewTimestampStrippingIterator(
+      const ReadOptions& read_options,
+      UnownedPtr<const SeqnoToTimeMapping> seqno_to_time_mapping, Arena* arena,
+      const SliceTransform* prefix_extractor, size_t ts_sz) override;
+
+  FragmentedRangeTombstoneIterator* NewRangeTombstoneIterator(
+      const ReadOptions& read_options, SequenceNumber read_seq,
+      bool immutable_memtable) override;
+
+  FragmentedRangeTombstoneIterator* NewTimestampStrippingRangeTombstoneIterator(
+      const ReadOptions& read_options, SequenceNumber read_seq,
+      size_t ts_sz) override;
+
+  Status VerifyEncodedEntry(Slice encoded,
+                            const ProtectionInfoKVOS64& kv_prot_info);
+
+  // Add an entry into memtable that maps key to value at the
+  // specified sequence number and with the specified type.
+  // Typically, value will be empty if type==kTypeDeletion.
+  //
+  // REQUIRES: if allow_concurrent = false, external synchronization to prevent
+  // simultaneous operations on the same MemTable.
+  //
+  // Returns `Status::TryAgain` if the `seq`, `key` combination already exists
+  // in the memtable and `MemTableRepFactory::CanHandleDuplicatedKey()` is true.
+  // The next attempt should try a larger value for `seq`.
+  Status Add(SequenceNumber seq, ValueType type, const Slice& key,
+             const Slice& value, const ProtectionInfoKVOS64* kv_prot_info,
+             bool allow_concurrent = false,
+             MemTablePostProcessInfo* post_process_info = nullptr,
+             void** hint = nullptr);
+
+  using ReadOnlyMemTable::Get;
+  bool Get(const LookupKey& key, std::string* value,
+           PinnableWideColumns* columns, std::string* timestamp, Status* s,
+           MergeContext* merge_context,
+           SequenceNumber* max_covering_tombstone_seq, SequenceNumber* seq,
+           const ReadOptions& read_opts, bool immutable_memtable,
+           ReadCallback* callback = nullptr, bool* is_blob_index = nullptr,
+           bool do_merge = true) override;
+
  void MultiGet(const ReadOptions& read_options, MultiGetRange* range,
-                ReadCallback* callback, bool immutable_memtable);
+                ReadCallback* callback, bool immutable_memtable) override;

  // If `key` exists in current memtable with type value_type and the existing
  // value is at least as large as the new value, updates it in-place. Otherwise
@@ -372,28 +641,19 @@ class MemTable {
    UpdateFlushState();
  }

-  // Get total number of entries in the mem table.
-  // REQUIRES: external synchronization to prevent simultaneous
-  // operations on the same MemTable (unless this Memtable is immutable).
-  uint64_t num_entries() const {
+  uint64_t NumEntries() const override {
    return num_entries_.load(std::memory_order_relaxed);
  }

-  // Get total number of deletes in the mem table.
-  // REQUIRES: external synchronization to prevent simultaneous
-  // operations on the same MemTable (unless this Memtable is immutable).
-  uint64_t num_deletes() const {
+  uint64_t NumDeletion() const override {
    return num_deletes_.load(std::memory_order_relaxed);
  }

-  // Get total number of range deletions in the mem table.
-  // REQUIRES: external synchronization to prevent simultaneous
-  // operations on the same MemTable (unless this Memtable is immutable).
-  uint64_t num_range_deletes() const {
+  uint64_t NumRangeDeletion() const override {
    return num_range_deletes_.load(std::memory_order_relaxed);
  }

-  uint64_t get_data_size() const {
+  uint64_t GetDataSize() const override {
    return data_size_.load(std::memory_order_relaxed);
  }

@@ -413,19 +673,9 @@ class MemTable {
    }
  }

-  // Returns the edits area that is needed for flushing the memtable
-  VersionEdit* GetEdits() { return &edit_; }
+  bool IsEmpty() const override { return first_seqno_ == 0; }

-  // Returns if there is no entry inserted to the mem table.
-  // REQUIRES: external synchronization to prevent simultaneous
-  // operations on the same MemTable (unless this Memtable is immutable).
-  bool IsEmpty() const { return first_seqno_ == 0; }
-
-  // Returns the sequence number of the first element that was inserted
-  // into the memtable.
-  // REQUIRES: external synchronization to prevent simultaneous
-  // operations on the same MemTable (unless this Memtable is immutable).
-  SequenceNumber GetFirstSequenceNumber() {
+  SequenceNumber GetFirstSequenceNumber() override {
    return first_seqno_.load(std::memory_order_relaxed);
  }

@@ -437,14 +687,8 @@ class MemTable {
    return first_seqno_.store(first_seqno, std::memory_order_relaxed);
  }

-  // Returns the sequence number that is guaranteed to be smaller than or equal
-  // to the sequence number of any key that could be inserted into this
-  // memtable. It can then be assumed that any write with a larger(or equal)
-  // sequence number will be present in this memtable or a later memtable.
-  //
-  // If the earliest sequence number could not be determined,
-  // kMaxSequenceNumber will be returned.
-  SequenceNumber GetEarliestSequenceNumber() {
+  SequenceNumber GetEarliestSequenceNumber() override {
+    // With file ingestion and empty memtable, this seqno needs to be fixed.
    return earliest_seqno_.load(std::memory_order_relaxed);
  }

@@ -463,40 +707,18 @@ class MemTable {

  void SetCreationSeq(SequenceNumber sn) { creation_seq_ = sn; }

-  // Returns the next active logfile number when this memtable is about to
-  // be flushed to storage
-  // REQUIRES: external synchronization to prevent simultaneous
-  // operations on the same MemTable.
-  uint64_t GetNextLogNumber() { return mem_next_logfile_number_; }
-
-  // Sets the next active logfile number when this memtable is about to
-  // be flushed to storage
-  // REQUIRES: external synchronization to prevent simultaneous
-  // operations on the same MemTable.
-  void SetNextLogNumber(uint64_t num) { mem_next_logfile_number_ = num; }
-
-  // if this memtable contains data from a committed
-  // two phase transaction we must take note of the
-  // log which contains that data so we can know
-  // when to relese that log
+  // If this memtable contains data from a committed two phase transaction we
+  // must take note of the log which contains that data so we can know when
+  // to release that log.
  void RefLogContainingPrepSection(uint64_t log);
-  uint64_t GetMinLogContainingPrepSection();
+  uint64_t GetMinLogContainingPrepSection() override;

-  // Notify the underlying storage that no more items will be added.
-  // REQUIRES: external synchronization to prevent simultaneous
-  // operations on the same MemTable.
-  // After MarkImmutable() is called, you should not attempt to
-  // write anything to this MemTable().  (Ie. do not call Add() or Update()).
-  void MarkImmutable() {
+  void MarkImmutable() override {
    table_->MarkReadOnly();
    mem_tracker_.DoneAllocating();
  }

-  // Notify the underlying storage that all data it contained has been
-  // persisted.
-  // REQUIRES: external synchronization to prevent simultaneous
-  // operations on the same MemTable.
-  void MarkFlushed() { table_->MarkFlushed(); }
+  void MarkFlushed() override { table_->MarkFlushed(); }

  // return true if the current MemTableRep supports merge operator.
  bool IsMergeOperatorSupported() const {
@@ -509,18 +731,13 @@ class MemTable {
    return table_->IsSnapshotSupported() && !moptions_.inplace_update_support;
  }

-  struct MemTableStats {
-    uint64_t size;
-    uint64_t count;
-  };
-
  MemTableStats ApproximateStats(const Slice& start_ikey,
-                                 const Slice& end_ikey);
+                                 const Slice& end_ikey) override;

  // Get the lock associated for the key
  port::RWMutex* GetLock(const Slice& key);

-  const InternalKeyComparator& GetInternalKeyComparator() const {
+  const InternalKeyComparator& GetInternalKeyComparator() const override {
    return comparator_.comparator;
  }

@@ -528,33 +745,10 @@ class MemTable {
    return &moptions_;
  }

-  uint64_t ApproximateOldestKeyTime() const {
+  uint64_t ApproximateOldestKeyTime() const override {
    return oldest_key_time_.load(std::memory_order_relaxed);
  }

-  // REQUIRES: db_mutex held.
-  void SetID(uint64_t id) { id_ = id; }
-
-  uint64_t GetID() const { return id_; }
-
-  void SetFlushCompleted(bool completed) { flush_completed_ = completed; }
-
-  uint64_t GetFileNumber() const { return file_number_; }
-
-  void SetFileNumber(uint64_t file_num) { file_number_ = file_num; }
-
-  void SetFlushInProgress(bool in_progress) {
-    flush_in_progress_ = in_progress;
-  }
-
-  void SetFlushJobInfo(std::unique_ptr<FlushJobInfo>&& info) {
-    flush_job_info_ = std::move(info);
-  }
-
-  std::unique_ptr<FlushJobInfo> ReleaseFlushJobInfo() {
-    return std::move(flush_job_info_);
-  }
-
  // Returns a heuristic flush decision
  bool ShouldFlushNow();

@@ -565,23 +759,12 @@ class MemTable {
  // SwitchMemtable() may fail.
  void ConstructFragmentedRangeTombstones();

-  // Returns whether a fragmented range tombstone list is already constructed
-  // for this memtable. It should be constructed right before a memtable is
-  // added to an immutable memtable list. Note that if a memtable does not have
-  // any range tombstone, then no range tombstone list will ever be constructed
-  // and true is returned in that case.
-  bool IsFragmentedRangeTombstonesConstructed() const {
+  bool IsFragmentedRangeTombstonesConstructed() const override {
    return fragmented_range_tombstone_list_.get() != nullptr ||
           is_range_del_table_empty_;
  }

-  // Get the newest user-defined timestamp contained in this MemTable. Check
-  // `newest_udt_` for what newer means. This method should only be invoked for
-  // an MemTable that has enabled user-defined timestamp feature and set
-  // `persist_user_defined_timestamps` to false. The tracked newest UDT will be
-  // used by flush job in the background to help check the MemTable's
-  // eligibility for Flush.
-  const Slice& GetNewestUDT() const;
+  const Slice& GetNewestUDT() const override;

  // Returns Corruption status if verification fails.
  static Status VerifyEntryChecksum(const char* entry,
@@ -597,7 +780,6 @@ class MemTable {

  KeyComparator comparator_;
  const ImmutableMemTableOptions moptions_;
-  int refs_;
  const size_t kArenaBlockSize;
  AllocTracker mem_tracker_;
  ConcurrentArena arena_;
@@ -614,15 +796,6 @@ class MemTable {
  // Dynamically changeable memtable option
  std::atomic<size_t> write_buffer_size_;

-  // These are used to manage memtable flushes to storage
-  bool flush_in_progress_;  // started the flush
-  bool flush_completed_;    // finished the flush
-  uint64_t file_number_;    // filled up after flush is complete
-
-  // The updates to be applied to the transaction log when this
-  // memtable is flushed to storage.
-  VersionEdit edit_;
-
  // The sequence number of the kv that was inserted first
  std::atomic<SequenceNumber> first_seqno_;

@@ -632,9 +805,6 @@ class MemTable {

  SequenceNumber creation_seq_;

-  // The log files earlier than this number can be deleted.
-  uint64_t mem_next_logfile_number_;
-
  // the earliest log containing a prepared section
  // which has been inserted into this memtable.
  std::atomic<uint64_t> min_prep_log_referenced_;
@@ -658,15 +828,6 @@ class MemTable {
  // Timestamp of oldest key
  std::atomic<uint64_t> oldest_key_time_;

-  // Memtable id to track flush.
-  uint64_t id_ = 0;
-
-  // Sequence number of the atomic flush that is responsible for this memtable.
-  // The sequence number of atomic flush is a seq, such that no writes with
-  // sequence numbers greater than or equal to seq are flushed, while all
-  // writes with sequence number smaller than seq are flushed.
-  SequenceNumber atomic_flush_seqno_;
-
  // keep track of memory usage in table_, arena_, and range_del_table_.
  // Gets refreshed inside `ApproximateMemoryUsage()` or `ShouldFlushNow`
  std::atomic<uint64_t> approximate_memory_usage_;
@@ -675,9 +836,6 @@ class MemTable {
  // unlimited.
  uint32_t memtable_max_range_deletions_ = 0;

-  // Flush job info of the current memtable.
-  std::unique_ptr<FlushJobInfo> flush_job_info_;
-
  // Size in bytes for the user-defined timestamps.
  size_t ts_sz_;

@@ -31,13 +31,17 @@ class InternalKeyComparator;
 class Mutex;
 class VersionSet;

-void MemTableListVersion::AddMemTable(MemTable* m) {
+void MemTableListVersion::AddMemTable(ReadOnlyMemTable* m) {
+  if (!memlist_.empty()) {
+    // ID can be equal for MemPurge
+    assert(m->GetID() >= memlist_.front()->GetID());
+  }
  memlist_.push_front(m);
  *parent_memtable_list_memory_usage_ += m->ApproximateMemoryUsage();
 }

-void MemTableListVersion::UnrefMemTable(autovector<MemTable*>* to_delete,
-                                        MemTable* m) {
+void MemTableListVersion::UnrefMemTable(
+    autovector<ReadOnlyMemTable*>* to_delete, ReadOnlyMemTable* m) {
  if (m->Unref()) {
    to_delete->push_back(m);
    assert(*parent_memtable_list_memory_usage_ >= m->ApproximateMemoryUsage());
@@ -74,7 +78,7 @@ MemTableListVersion::MemTableListVersion(
 void MemTableListVersion::Ref() { ++refs_; }

 // called by superversion::clean()
-void MemTableListVersion::Unref(autovector<MemTable*>* to_delete) {
+void MemTableListVersion::Unref(autovector<ReadOnlyMemTable*>* to_delete) {
  assert(refs_ >= 1);
  --refs_;
  if (refs_ == 0) {
@@ -92,14 +96,12 @@ void MemTableListVersion::Unref(autovector<MemTable*>* to_delete) {
 }

 int MemTableList::NumNotFlushed() const {
-  int size = static_cast<int>(current_->memlist_.size());
+  int size = current_->NumNotFlushed();
  assert(num_flush_not_started_ <= size);
  return size;
 }

-int MemTableList::NumFlushed() const {
-  return static_cast<int>(current_->memlist_history_.size());
-}
+int MemTableList::NumFlushed() const { return current_->NumFlushed(); }

 // Search all the memtables starting from the most recent one.
 // Return the most recent value found, if any.
@@ -131,7 +133,7 @@ void MemTableListVersion::MultiGet(const ReadOptions& read_options,
 bool MemTableListVersion::GetMergeOperands(
    const LookupKey& key, Status* s, MergeContext* merge_context,
    SequenceNumber* max_covering_tombstone_seq, const ReadOptions& read_opts) {
-  for (MemTable* memtable : memlist_) {
+  for (ReadOnlyMemTable* memtable : memlist_) {
    bool done = memtable->Get(
        key, /*value=*/nullptr, /*columns=*/nullptr, /*timestamp=*/nullptr, s,
        merge_context, max_covering_tombstone_seq, read_opts,
@@ -154,11 +156,11 @@ bool MemTableListVersion::GetFromHistory(
 }

 bool MemTableListVersion::GetFromList(
-    std::list<MemTable*>* list, const LookupKey& key, std::string* value,
-    PinnableWideColumns* columns, std::string* timestamp, Status* s,
-    MergeContext* merge_context, SequenceNumber* max_covering_tombstone_seq,
-    SequenceNumber* seq, const ReadOptions& read_opts, ReadCallback* callback,
-    bool* is_blob_index) {
+    std::list<ReadOnlyMemTable*>* list, const LookupKey& key,
+    std::string* value, PinnableWideColumns* columns, std::string* timestamp,
+    Status* s, MergeContext* merge_context,
+    SequenceNumber* max_covering_tombstone_seq, SequenceNumber* seq,
+    const ReadOptions& read_opts, ReadCallback* callback, bool* is_blob_index) {
  *seq = kMaxSequenceNumber;

  for (auto& memtable : *list) {
@@ -218,7 +220,8 @@ void MemTableListVersion::AddIterators(
    std::vector<InternalIterator*>* iterator_list, Arena* arena) {
  for (auto& m : memlist_) {
    iterator_list->push_back(m->NewIterator(options, seqno_to_time_mapping,
-                                            arena, prefix_extractor));
+                                            arena, prefix_extractor,
+                                            /*for_flush=*/false));
  }
 }

@@ -230,7 +233,8 @@ void MemTableListVersion::AddIterators(
  for (auto& m : memlist_) {
    auto mem_iter =
        m->NewIterator(options, seqno_to_time_mapping,
-                       merge_iter_builder->GetArena(), prefix_extractor);
+                       merge_iter_builder->GetArena(), prefix_extractor,
+                       /*for_flush=*/false);
    if (!add_range_tombstone_iter || options.ignore_range_deletions) {
      merge_iter_builder->AddIterator(mem_iter);
    } else {
@@ -259,14 +263,14 @@ void MemTableListVersion::AddIterators(
 uint64_t MemTableListVersion::GetTotalNumEntries() const {
  uint64_t total_num = 0;
  for (auto& m : memlist_) {
-    total_num += m->num_entries();
+    total_num += m->NumEntries();
  }
  return total_num;
 }

-MemTable::MemTableStats MemTableListVersion::ApproximateStats(
-    const Slice& start_ikey, const Slice& end_ikey) {
-  MemTable::MemTableStats total_stats = {0, 0};
+ReadOnlyMemTable::MemTableStats MemTableListVersion::ApproximateStats(
+    const Slice& start_ikey, const Slice& end_ikey) const {
+  ReadOnlyMemTable::MemTableStats total_stats = {0, 0};
  for (auto& m : memlist_) {
    auto mStats = m->ApproximateStats(start_ikey, end_ikey);
    total_stats.size += mStats.size;
@@ -278,7 +282,7 @@ MemTable::MemTableStats MemTableListVersion::ApproximateStats(
 uint64_t MemTableListVersion::GetTotalNumDeletes() const {
  uint64_t total_num = 0;
  for (auto& m : memlist_) {
-    total_num += m->num_deletes();
+    total_num += m->NumDeletion();
  }
  return total_num;
 }
@@ -304,7 +308,8 @@ SequenceNumber MemTableListVersion::GetFirstSequenceNumber() const {
 }

 // caller is responsible for referencing m
-void MemTableListVersion::Add(MemTable* m, autovector<MemTable*>* to_delete) {
+void MemTableListVersion::Add(ReadOnlyMemTable* m,
+                              autovector<ReadOnlyMemTable*>* to_delete) {
  assert(refs_ == 1);  // only when refs_ == 1 is MemTableListVersion mutable
  AddMemTable(m);
  // m->MemoryAllocatedBytes() is added in MemoryAllocatedBytesExcludingLast
@@ -312,8 +317,8 @@ void MemTableListVersion::Add(MemTable* m, autovector<MemTable*>* to_delete) {
 }

 // Removes m from list of memtables not flushed.  Caller should NOT Unref m.
-void MemTableListVersion::Remove(MemTable* m,
-                                 autovector<MemTable*>* to_delete) {
+void MemTableListVersion::Remove(ReadOnlyMemTable* m,
+                                 autovector<ReadOnlyMemTable*>* to_delete) {
  assert(refs_ == 1);  // only when refs_ == 1 is MemTableListVersion mutable
  memlist_.remove(m);

@@ -364,11 +369,11 @@ bool MemTableListVersion::HistoryShouldBeTrimmed(size_t usage) {
 }

 // Make sure we don't use up too much space in history
-bool MemTableListVersion::TrimHistory(autovector<MemTable*>* to_delete,
+bool MemTableListVersion::TrimHistory(autovector<ReadOnlyMemTable*>* to_delete,
                                      size_t usage) {
  bool ret = false;
  while (HistoryShouldBeTrimmed(usage)) {
-    MemTable* x = memlist_history_.back();
+    ReadOnlyMemTable* x = memlist_history_.back();
    memlist_history_.pop_back();

    UnrefMemTable(to_delete, x);
@@ -398,7 +403,7 @@ bool MemTableList::IsFlushPendingOrRunning() const {

 // Returns the memtables that need to be flushed.
 void MemTableList::PickMemtablesToFlush(uint64_t max_memtable_id,
-                                        autovector<MemTable*>* ret,
+                                        autovector<ReadOnlyMemTable*>* ret,
                                        uint64_t* max_next_log_number) {
  AutoThreadOperationStageUpdater stage_updater(
      ThreadStatus::STAGE_PICK_MEMTABLES_TO_FLUSH);
@@ -411,8 +416,9 @@ void MemTableList::PickMemtablesToFlush(uint64_t max_memtable_id,
  // ret is filled with memtables already sorted in increasing MemTable ID.
  // However, when the mempurge feature is activated, new memtables with older
  // IDs will be added to the memlist.
-  for (auto it = memlist.rbegin(); it != memlist.rend(); ++it) {
-    MemTable* m = *it;
+  auto it = memlist.rbegin();
+  for (; it != memlist.rend(); ++it) {
+    ReadOnlyMemTable* m = *it;
    if (!atomic_flush && m->atomic_flush_seqno_ != kMaxSequenceNumber) {
      atomic_flush = true;
    }
@@ -440,25 +446,32 @@ void MemTableList::PickMemtablesToFlush(uint64_t max_memtable_id,
      break;
    }
  }
+  if (!ret->empty() && it != memlist.rend()) {
+    // checks that the first memtable not picked to flush is not ingested wbwi.
+    // Ingested memtable should be flushed together with the memtable before it
+    // since they map to the same WAL and have the same NextLogNumber().
+    assert(strcmp((*it)->Name(), "WBWIMemTable") != 0);
+  }
  if (!atomic_flush || num_flush_not_started_ == 0) {
    flush_requested_ = false;  // start-flush request is complete
  }
 }

-void MemTableList::RollbackMemtableFlush(const autovector<MemTable*>& mems,
-                                         bool rollback_succeeding_memtables) {
+void MemTableList::RollbackMemtableFlush(
+    const autovector<ReadOnlyMemTable*>& mems,
+    bool rollback_succeeding_memtables) {
  TEST_SYNC_POINT("RollbackMemtableFlush");
  AutoThreadOperationStageUpdater stage_updater(
      ThreadStatus::STAGE_MEMTABLE_ROLLBACK);
 #ifndef NDEBUG
-  for (MemTable* m : mems) {
+  for (ReadOnlyMemTable* m : mems) {
    assert(m->flush_in_progress_);
    assert(m->file_number_ == 0);
  }
 #endif

  if (rollback_succeeding_memtables && !mems.empty()) {
-    std::list<MemTable*>& memlist = current_->memlist_;
+    std::list<ReadOnlyMemTable*>& memlist = current_->memlist_;
    auto it = memlist.rbegin();
    for (; *it != mems[0] && it != memlist.rend(); ++it) {
    }
@@ -468,7 +481,7 @@ void MemTableList::RollbackMemtableFlush(const autovector<MemTable*>& mems,
      ++it;
    }
    while (it != memlist.rend()) {
-      MemTable* m = *it;
+      ReadOnlyMemTable* m = *it;
      // Only rollback complete, not in-progress,
      // in_progress can be flushes that are still writing SSTs
      if (m->flush_completed_) {
@@ -484,7 +497,7 @@ void MemTableList::RollbackMemtableFlush(const autovector<MemTable*>& mems,
    }
  }

-  for (MemTable* m : mems) {
+  for (ReadOnlyMemTable* m : mems) {
    if (m->flush_in_progress_) {
      assert(m->file_number_ == 0);
      m->file_number_ = 0;
@@ -503,10 +516,10 @@ void MemTableList::RollbackMemtableFlush(const autovector<MemTable*>& mems,
 // Status::OK letting a concurrent flush to do actual the recording..
 Status MemTableList::TryInstallMemtableFlushResults(
    ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options,
-    const autovector<MemTable*>& mems, LogsWithPrepTracker* prep_tracker,
-    VersionSet* vset, InstrumentedMutex* mu, uint64_t file_number,
-    autovector<MemTable*>* to_delete, FSDirectory* db_directory,
-    LogBuffer* log_buffer,
+    const autovector<ReadOnlyMemTable*>& mems,
+    LogsWithPrepTracker* prep_tracker, VersionSet* vset, InstrumentedMutex* mu,
+    uint64_t file_number, autovector<ReadOnlyMemTable*>* to_delete,
+    FSDirectory* db_directory, LogBuffer* log_buffer,
    std::list<std::unique_ptr<FlushJobInfo>>* committed_flush_jobs_info,
    bool write_edits) {
  AutoThreadOperationStageUpdater stage_updater(
@@ -553,16 +566,16 @@ Status MemTableList::TryInstallMemtableFlushResults(
    // (in that order) that have finished flushing. Memtables
    // are always committed in the order that they were created.
    uint64_t batch_file_number = 0;
-    size_t batch_count = 0;
    autovector<VersionEdit*> edit_list;
-    autovector<MemTable*> memtables_to_flush;
+    autovector<ReadOnlyMemTable*> memtables_to_flush;
    // enumerate from the last (earliest) element to see how many batch finished
    for (auto it = memlist.rbegin(); it != memlist.rend(); ++it) {
-      MemTable* m = *it;
+      ReadOnlyMemTable* m = *it;
      if (!m->flush_completed_) {
        break;
      }
      if (it == memlist.rbegin() || batch_file_number != m->file_number_) {
+        // Oldest memtable in a new batch.
        batch_file_number = m->file_number_;
        if (m->edit_.GetBlobFileAdditions().empty()) {
          ROCKS_LOG_BUFFER(log_buffer,
@@ -578,17 +591,17 @@ Status MemTableList::TryInstallMemtableFlushResults(
        }

        edit_list.push_back(&m->edit_);
-        memtables_to_flush.push_back(m);
        std::unique_ptr<FlushJobInfo> info = m->ReleaseFlushJobInfo();
        if (info != nullptr) {
          committed_flush_jobs_info->push_back(std::move(info));
        }
      }
-      batch_count++;
+      memtables_to_flush.push_back(m);
    }

+    size_t num_mem_to_flush = memtables_to_flush.size();
    // TODO(myabandeh): Not sure how batch_count could be 0 here.
-    if (batch_count > 0) {
+    if (num_mem_to_flush > 0) {
      VersionEdit edit;
 #ifdef ROCKSDB_ASSERT_STATUS_CHECKED
      if (memtables_to_flush.size() == memlist.size()) {
@@ -612,9 +625,9 @@ Status MemTableList::TryInstallMemtableFlushResults(
          nullptr);
      edit_list.push_back(&edit);

-      const auto manifest_write_cb = [this, cfd, batch_count, log_buffer,
+      const auto manifest_write_cb = [this, cfd, num_mem_to_flush, log_buffer,
                                      to_delete, mu](const Status& status) {
-        RemoveMemTablesOrRestoreFlags(status, cfd, batch_count, log_buffer,
+        RemoveMemTablesOrRestoreFlags(status, cfd, num_mem_to_flush, log_buffer,
                                      to_delete, mu);
      };
      if (write_edits) {
@@ -627,7 +640,7 @@ Status MemTableList::TryInstallMemtableFlushResults(
        // If write_edit is false (e.g: successful mempurge),
        // then remove old memtables, wake up manifest write queue threads,
        // and don't commit anything to the manifest file.
-        RemoveMemTablesOrRestoreFlags(s, cfd, batch_count, log_buffer,
+        RemoveMemTablesOrRestoreFlags(s, cfd, num_mem_to_flush, log_buffer,
                                      to_delete, mu);
        // Note: cfd->SetLogNumber is only called when a VersionEdit
        // is written to MANIFEST. When mempurge is succesful, we skip
@@ -646,7 +659,8 @@ Status MemTableList::TryInstallMemtableFlushResults(
 }

 // New memtables are inserted at the front of the list.
-void MemTableList::Add(MemTable* m, autovector<MemTable*>* to_delete) {
+void MemTableList::Add(ReadOnlyMemTable* m,
+                       autovector<ReadOnlyMemTable*>* to_delete) {
  assert(static_cast<int>(current_->memlist_.size()) >= num_flush_not_started_);
  InstallNewVersion();
  // this method is used to move mutable memtable into an immutable list.
@@ -664,7 +678,8 @@ void MemTableList::Add(MemTable* m, autovector<MemTable*>* to_delete) {
  ResetTrimHistoryNeeded();
 }

-bool MemTableList::TrimHistory(autovector<MemTable*>* to_delete, size_t usage) {
+bool MemTableList::TrimHistory(autovector<ReadOnlyMemTable*>* to_delete,
+                               size_t usage) {
  // Check if history trim is needed first, so that we can avoid installing a
  // new MemTableListVersion without installing a SuperVersion (installed based
  // on return value of this function).
@@ -733,8 +748,8 @@ void MemTableList::InstallNewVersion() {
 }

 void MemTableList::RemoveMemTablesOrRestoreFlags(
-    const Status& s, ColumnFamilyData* cfd, size_t batch_count,
-    LogBuffer* log_buffer, autovector<MemTable*>* to_delete,
+    const Status& s, ColumnFamilyData* cfd, size_t num_mem_to_flush,
+    LogBuffer* log_buffer, autovector<ReadOnlyMemTable*>* to_delete,
    InstrumentedMutex* mu) {
  assert(mu);
  mu->AssertHeld();
@@ -762,8 +777,11 @@ void MemTableList::RemoveMemTablesOrRestoreFlags(
  // read full data as long as column family handle is not deleted, even if
  // the column family is dropped.
  if (s.ok() && !cfd->IsDropped()) {  // commit new state
-    while (batch_count-- > 0) {
-      MemTable* m = current_->memlist_.back();
+    while (num_mem_to_flush-- > 0) {
+      ReadOnlyMemTable* m = current_->memlist_.back();
+      // TODO: The logging can be redundant when we flush multiple memtables
+      // into one SST file. We should only check the edit_ of the oldest
+      // memtable in the group in that case.
      if (m->edit_.GetBlobFileAdditions().empty()) {
        ROCKS_LOG_BUFFER(log_buffer,
                         "[%s] Level-0 commit flush result of table #%" PRIu64
@@ -785,8 +803,8 @@ void MemTableList::RemoveMemTablesOrRestoreFlags(
      ++mem_id;
    }
  } else {
-    for (auto it = current_->memlist_.rbegin(); batch_count-- > 0; ++it) {
-      MemTable* m = *it;
+    for (auto it = current_->memlist_.rbegin(); num_mem_to_flush-- > 0; ++it) {
+      ReadOnlyMemTable* m = *it;
      // commit failed. setup state so that we can flush again.
      if (m->edit_.GetBlobFileAdditions().empty()) {
        ROCKS_LOG_BUFFER(log_buffer,
@@ -814,7 +832,7 @@ void MemTableList::RemoveMemTablesOrRestoreFlags(
 }

 uint64_t MemTableList::PrecomputeMinLogContainingPrepSection(
-    const std::unordered_set<MemTable*>* memtables_to_flush) {
+    const std::unordered_set<ReadOnlyMemTable*>* memtables_to_flush) const {
  uint64_t min_log = 0;

  for (auto& m : current_->memlist_) {
@@ -837,12 +855,12 @@ Status InstallMemtableAtomicFlushResults(
    const autovector<MemTableList*>* imm_lists,
    const autovector<ColumnFamilyData*>& cfds,
    const autovector<const MutableCFOptions*>& mutable_cf_options_list,
-    const autovector<const autovector<MemTable*>*>& mems_list, VersionSet* vset,
-    LogsWithPrepTracker* prep_tracker, InstrumentedMutex* mu,
+    const autovector<const autovector<ReadOnlyMemTable*>*>& mems_list,
+    VersionSet* vset, LogsWithPrepTracker* prep_tracker, InstrumentedMutex* mu,
    const autovector<FileMetaData*>& file_metas,
    const autovector<std::list<std::unique_ptr<FlushJobInfo>>*>&
        committed_flush_jobs_info,
-    autovector<MemTable*>* to_delete, FSDirectory* db_directory,
+    autovector<ReadOnlyMemTable*>* to_delete, FSDirectory* db_directory,
    LogBuffer* log_buffer) {
  AutoThreadOperationStageUpdater stage_updater(
      ThreadStatus::STAGE_MEMTABLE_INSTALL_FLUSH_RESULTS);
@@ -1006,14 +1024,14 @@ Status InstallMemtableAtomicFlushResults(
  return s;
 }

-void MemTableList::RemoveOldMemTables(uint64_t log_number,
-                                      autovector<MemTable*>* to_delete) {
+void MemTableList::RemoveOldMemTables(
+    uint64_t log_number, autovector<ReadOnlyMemTable*>* to_delete) {
  assert(to_delete != nullptr);
  InstallNewVersion();
  auto& memlist = current_->memlist_;
-  autovector<MemTable*> old_memtables;
+  autovector<ReadOnlyMemTable*> old_memtables;
  for (auto it = memlist.rbegin(); it != memlist.rend(); ++it) {
-    MemTable* mem = *it;
+    ReadOnlyMemTable* mem = *it;
    if (mem->GetNextLogNumber() > log_number) {
      break;
    }
@@ -1021,7 +1039,7 @@ void MemTableList::RemoveOldMemTables(uint64_t log_number,
  }

  for (auto it = old_memtables.begin(); it != old_memtables.end(); ++it) {
-    MemTable* mem = *it;
+    ReadOnlyMemTable* mem = *it;
    current_->Remove(mem, to_delete);
    --num_flush_not_started_;
    if (0 == num_flush_not_started_) {
@@ -1044,9 +1062,9 @@ VersionEdit MemTableList::GetEditForDroppingCurrentVersion(

  uint64_t max_next_log_number = 0;
  autovector<VersionEdit*> edit_list;
-  autovector<MemTable*> memtables_to_drop;
+  autovector<ReadOnlyMemTable*> memtables_to_drop;
  for (auto it = memlist.rbegin(); it != memlist.rend(); ++it) {
-    MemTable* m = *it;
+    ReadOnlyMemTable* m = *it;
    memtables_to_drop.push_back(m);
    max_next_log_number = std::max(m->GetNextLogNumber(), max_next_log_number);
  }
@@ -34,11 +34,11 @@ class MemTableList;

 struct FlushJobInfo;

-// keeps a list of immutable memtables in a vector. the list is immutable
-// if refcount is bigger than one. It is used as a state for Get() and
-// Iterator code paths
+// keeps a list of immutable memtables (ReadOnlyMemtable*) in a vector.
+// The list is immutable if refcount is bigger than one. It is used as
+// a state for Get() and iterator code paths.
 //
-// This class is not thread-safe.  External synchronization is required
+// This class is not thread-safe. External synchronization is required
 // (such as holding the db mutex or being on the write thread).
 class MemTableListVersion {
 public:
@@ -49,7 +49,7 @@ class MemTableListVersion {
                               int64_t max_write_buffer_size_to_maintain);

  void Ref();
-  void Unref(autovector<MemTable*>* to_delete = nullptr);
+  void Unref(autovector<ReadOnlyMemTable*>* to_delete = nullptr);

  // Search all the memtables starting from the most recent one.
  // Return the most recent value found, if any.
@@ -127,8 +127,8 @@ class MemTableListVersion {

  uint64_t GetTotalNumDeletes() const;

-  MemTable::MemTableStats ApproximateStats(const Slice& start_ikey,
-                                           const Slice& end_ikey);
+  ReadOnlyMemTable::MemTableStats ApproximateStats(const Slice& start_ikey,
+                                                   const Slice& end_ikey) const;

  // Returns the value of MemTable::GetEarliestSequenceNumber() on the most
  // recent MemTable in this list or kMaxSequenceNumber if the list is empty.
@@ -146,6 +146,10 @@ class MemTableListVersion {

  uint64_t GetID() const { return id_; }

+  int NumNotFlushed() const { return static_cast<int>(memlist_.size()); }
+
+  int NumFlushed() const { return static_cast<int>(memlist_history_.size()); }
+
 private:
  friend class MemTableList;

@@ -153,27 +157,27 @@ class MemTableListVersion {
      const autovector<MemTableList*>* imm_lists,
      const autovector<ColumnFamilyData*>& cfds,
      const autovector<const MutableCFOptions*>& mutable_cf_options_list,
-      const autovector<const autovector<MemTable*>*>& mems_list,
+      const autovector<const autovector<ReadOnlyMemTable*>*>& mems_list,
      VersionSet* vset, LogsWithPrepTracker* prep_tracker,
      InstrumentedMutex* mu, const autovector<FileMetaData*>& file_meta,
      const autovector<std::list<std::unique_ptr<FlushJobInfo>>*>&
          committed_flush_jobs_info,
-      autovector<MemTable*>* to_delete, FSDirectory* db_directory,
+      autovector<ReadOnlyMemTable*>* to_delete, FSDirectory* db_directory,
      LogBuffer* log_buffer);

  // REQUIRE: m is an immutable memtable
-  void Add(MemTable* m, autovector<MemTable*>* to_delete);
+  void Add(ReadOnlyMemTable* m, autovector<ReadOnlyMemTable*>* to_delete);
  // REQUIRE: m is an immutable memtable
-  void Remove(MemTable* m, autovector<MemTable*>* to_delete);
+  void Remove(ReadOnlyMemTable* m, autovector<ReadOnlyMemTable*>* to_delete);

  // Return true if the memtable list should be trimmed to get memory usage
  // under budget.
  bool HistoryShouldBeTrimmed(size_t usage);

  // Trim history, Return true if memtable is trimmed
-  bool TrimHistory(autovector<MemTable*>* to_delete, size_t usage);
+  bool TrimHistory(autovector<ReadOnlyMemTable*>* to_delete, size_t usage);

-  bool GetFromList(std::list<MemTable*>* list, const LookupKey& key,
+  bool GetFromList(std::list<ReadOnlyMemTable*>* list, const LookupKey& key,
                   std::string* value, PinnableWideColumns* columns,
                   std::string* timestamp, Status* s,
                   MergeContext* merge_context,
@@ -182,9 +186,10 @@ class MemTableListVersion {
                   ReadCallback* callback = nullptr,
                   bool* is_blob_index = nullptr);

-  void AddMemTable(MemTable* m);
+  void AddMemTable(ReadOnlyMemTable* m);

-  void UnrefMemTable(autovector<MemTable*>* to_delete, MemTable* m);
+  void UnrefMemTable(autovector<ReadOnlyMemTable*>* to_delete,
+                     ReadOnlyMemTable* m);

  // Calculate the total amount of memory used by memlist_ and memlist_history_
  // excluding the last MemTable in memlist_history_. The reason for excluding
@@ -199,11 +204,11 @@ class MemTableListVersion {
  bool MemtableLimitExceeded(size_t usage);

  // Immutable MemTables that have not yet been flushed.
-  std::list<MemTable*> memlist_;
+  std::list<ReadOnlyMemTable*> memlist_;

  // MemTables that have already been flushed
  // (used during Transaction validation)
-  std::list<MemTable*> memlist_history_;
+  std::list<ReadOnlyMemTable*> memlist_history_;

  // Maximum number of MemTables to keep in memory (including both flushed
  const int max_write_buffer_number_to_maintain_;
@@ -283,7 +288,7 @@ class MemTableList {
  // Returns the earliest memtables that needs to be flushed. The returned
  // memtables are guaranteed to be in the ascending order of created time.
  void PickMemtablesToFlush(uint64_t max_memtable_id,
-                            autovector<MemTable*>* mems,
+                            autovector<ReadOnlyMemTable*>* mems,
                            uint64_t* max_next_log_number = nullptr);

  // Reset status of the given memtable list back to pending state so that
@@ -300,16 +305,16 @@ class MemTableList {
  // Note that we also do rollback in `write_manifest_cb` by calling
  // `RemoveMemTablesOrRestoreFlags()`. There we rollback the entire batch so
  // it is similar to what we do here with rollback_succeeding_memtables=true.
-  void RollbackMemtableFlush(const autovector<MemTable*>& mems,
+  void RollbackMemtableFlush(const autovector<ReadOnlyMemTable*>& mems,
                             bool rollback_succeeding_memtables);

  // Try commit a successful flush in the manifest file. It might just return
  // Status::OK letting a concurrent flush to do the actual the recording.
  Status TryInstallMemtableFlushResults(
      ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options,
-      const autovector<MemTable*>& m, LogsWithPrepTracker* prep_tracker,
+      const autovector<ReadOnlyMemTable*>& m, LogsWithPrepTracker* prep_tracker,
      VersionSet* vset, InstrumentedMutex* mu, uint64_t file_number,
-      autovector<MemTable*>* to_delete, FSDirectory* db_directory,
+      autovector<ReadOnlyMemTable*>* to_delete, FSDirectory* db_directory,
      LogBuffer* log_buffer,
      std::list<std::unique_ptr<FlushJobInfo>>* committed_flush_jobs_info,
      bool write_edits = true);
@@ -319,7 +324,7 @@ class MemTableList {
  // By default, adding memtables will flag that the memtable list needs to be
  // flushed, but in certain situations, like after a mempurge, we may want to
  // avoid flushing the memtable list upon addition of a memtable.
-  void Add(MemTable* m, autovector<MemTable*>* to_delete);
+  void Add(ReadOnlyMemTable* m, autovector<ReadOnlyMemTable*>* to_delete);

  // Returns an estimate of the number of bytes of data in use.
  size_t ApproximateMemoryUsage();
@@ -341,7 +346,7 @@ class MemTableList {
  // memtable list.
  //
  // Return true if memtable is trimmed
-  bool TrimHistory(autovector<MemTable*>* to_delete, size_t usage);
+  bool TrimHistory(autovector<ReadOnlyMemTable*>* to_delete, size_t usage);

  // Returns an estimate of the number of bytes of data used by
  // the unflushed mem-tables.
@@ -390,10 +395,13 @@ class MemTableList {

  size_t* current_memory_usage() { return &current_memory_usage_; }

-  // Returns the min log containing the prep section after memtables listsed in
-  // `memtables_to_flush` are flushed and their status is persisted in manifest.
+  // Returns the WAL number of the oldest WAL that contains a prepared
+  // transaction that corresponds to the content in this MemTableList,
+  // after memtables listed in `memtables_to_flush` are flushed and their
+  // status is persisted in manifest.
  uint64_t PrecomputeMinLogContainingPrepSection(
-      const std::unordered_set<MemTable*>* memtables_to_flush = nullptr);
+      const std::unordered_set<ReadOnlyMemTable*>* memtables_to_flush =
+          nullptr) const;

  uint64_t GetEarliestMemTableID() const {
    auto& memlist = current_->memlist_;
@@ -411,7 +419,7 @@ class MemTableList {
    if (for_atomic_flush) {
      // Scan the memtable list from new to old
      for (auto it = memlist.begin(); it != memlist.end(); ++it) {
-        MemTable* m = *it;
+        ReadOnlyMemTable* m = *it;
        if (m->atomic_flush_seqno_ != kMaxSequenceNumber) {
          return m->GetID();
        }
@@ -431,7 +439,7 @@ class MemTableList {
    // Iterating through the memlist starting at the end, the vector<MemTable*>
    // ret is filled with memtables already sorted in increasing MemTable ID.
    for (auto it = memlist.rbegin(); it != memlist.rend(); ++it) {
-      MemTable* m = *it;
+      ReadOnlyMemTable* m = *it;
      if (m->GetID() > max_memtable_id) {
        break;
      }
@@ -444,7 +452,7 @@ class MemTableList {
    const auto& memlist = current_->memlist_;
    // Scan the memtable list from new to old
    for (auto it = memlist.begin(); it != memlist.end(); ++it) {
-      MemTable* mem = *it;
+      ReadOnlyMemTable* mem = *it;
      if (mem->atomic_flush_seqno_ == kMaxSequenceNumber) {
        mem->atomic_flush_seqno_ = seq;
      } else {
@@ -460,7 +468,7 @@ class MemTableList {
  // was created, i.e. mem->GetNextLogNumber() <= log_number. The memtables are
  // not freed, but put into a vector for future deref and reclamation.
  void RemoveOldMemTables(uint64_t log_number,
-                          autovector<MemTable*>* to_delete);
+                          autovector<ReadOnlyMemTable*>* to_delete);

  // This API is only used by atomic date replacement. To get an edit for
  // dropping the current `MemTableListVersion`.
@@ -473,12 +481,12 @@ class MemTableList {
      const autovector<MemTableList*>* imm_lists,
      const autovector<ColumnFamilyData*>& cfds,
      const autovector<const MutableCFOptions*>& mutable_cf_options_list,
-      const autovector<const autovector<MemTable*>*>& mems_list,
+      const autovector<const autovector<ReadOnlyMemTable*>*>& mems_list,
      VersionSet* vset, LogsWithPrepTracker* prep_tracker,
      InstrumentedMutex* mu, const autovector<FileMetaData*>& file_meta,
      const autovector<std::list<std::unique_ptr<FlushJobInfo>>*>&
          committed_flush_jobs_info,
-      autovector<MemTable*>* to_delete, FSDirectory* db_directory,
+      autovector<ReadOnlyMemTable*>* to_delete, FSDirectory* db_directory,
      LogBuffer* log_buffer);

  // DB mutex held
@@ -488,7 +496,7 @@ class MemTableList {
  // Called after writing to MANIFEST
  void RemoveMemTablesOrRestoreFlags(const Status& s, ColumnFamilyData* cfd,
                                     size_t batch_count, LogBuffer* log_buffer,
-                                     autovector<MemTable*>* to_delete,
+                                     autovector<ReadOnlyMemTable*>* to_delete,
                                     InstrumentedMutex* mu);

  const int min_write_buffer_number_to_merge_;
@@ -529,11 +537,11 @@ Status InstallMemtableAtomicFlushResults(
    const autovector<MemTableList*>* imm_lists,
    const autovector<ColumnFamilyData*>& cfds,
    const autovector<const MutableCFOptions*>& mutable_cf_options_list,
-    const autovector<const autovector<MemTable*>*>& mems_list, VersionSet* vset,
-    LogsWithPrepTracker* prep_tracker, InstrumentedMutex* mu,
+    const autovector<const autovector<ReadOnlyMemTable*>*>& mems_list,
+    VersionSet* vset, LogsWithPrepTracker* prep_tracker, InstrumentedMutex* mu,
    const autovector<FileMetaData*>& file_meta,
    const autovector<std::list<std::unique_ptr<FlushJobInfo>>*>&
        committed_flush_jobs_info,
-    autovector<MemTable*>* to_delete, FSDirectory* db_directory,
+    autovector<ReadOnlyMemTable*>* to_delete, FSDirectory* db_directory,
    LogBuffer* log_buffer);
 }  // namespace ROCKSDB_NAMESPACE
@@ -98,7 +98,8 @@ class MemTableListTest : public testing::Test {
  // structures needed to call this function.
  Status Mock_InstallMemtableFlushResults(
      MemTableList* list, const MutableCFOptions& mutable_cf_options,
-      const autovector<MemTable*>& m, autovector<MemTable*>* to_delete) {
+      const autovector<ReadOnlyMemTable*>& m,
+      autovector<ReadOnlyMemTable*>* to_delete) {
    // Create a mock Logger
    test::NullLogger logger;
    LogBuffer log_buffer(DEBUG_LEVEL, &logger);
@@ -148,8 +149,8 @@ class MemTableListTest : public testing::Test {
  Status Mock_InstallMemtableAtomicFlushResults(
      autovector<MemTableList*>& lists, const autovector<uint32_t>& cf_ids,
      const autovector<const MutableCFOptions*>& mutable_cf_options_list,
-      const autovector<const autovector<MemTable*>*>& mems_list,
-      autovector<MemTable*>* to_delete) {
+      const autovector<const autovector<ReadOnlyMemTable*>*>& mems_list,
+      autovector<ReadOnlyMemTable*>* to_delete) {
    // Create a mock Logger
    test::NullLogger logger;
    LogBuffer log_buffer(DEBUG_LEVEL, &logger);
@@ -227,12 +228,12 @@ TEST_F(MemTableListTest, Empty) {
  ASSERT_FALSE(list.imm_flush_needed.load(std::memory_order_acquire));
  ASSERT_FALSE(list.IsFlushPending());

-  autovector<MemTable*> mems;
+  autovector<ReadOnlyMemTable*> mems;
  list.PickMemtablesToFlush(
      std::numeric_limits<uint64_t>::max() /* memtable_id */, &mems);
  ASSERT_EQ(0, mems.size());

-  autovector<MemTable*> to_delete;
+  autovector<ReadOnlyMemTable*> to_delete;
  list.current()->Unref(&to_delete);
  ASSERT_EQ(0, to_delete.size());
 }
@@ -252,7 +253,7 @@ TEST_F(MemTableListTest, GetTest) {
  MergeContext merge_context;
  InternalKeyComparator ikey_cmp(options.comparator);
  SequenceNumber max_covering_tombstone_seq = 0;
-  autovector<MemTable*> to_delete;
+  autovector<ReadOnlyMemTable*> to_delete;

  LookupKey lkey("key1", seq);
  bool found = list.current()->Get(lkey, &value, /*columns=*/nullptr,
@@ -322,13 +323,14 @@ TEST_F(MemTableListTest, GetTest) {
  ASSERT_TRUE(s.ok() && found);
  ASSERT_EQ(value, "value3.1");

-  ASSERT_EQ(5, mem->num_entries());
-  ASSERT_EQ(1, mem->num_deletes());
+  ASSERT_EQ(5, mem->NumEntries());
+  ASSERT_EQ(1, mem->NumDeletion());

  // Add memtable to list
  // This is to make assert(memtable->IsFragmentedRangeTombstonesConstructed())
  // in MemTableListVersion::GetFromList work.
  mem->ConstructFragmentedRangeTombstones();
+  mem->SetID(1);
  list.Add(mem, &to_delete);

  SequenceNumber saved_seq = seq;
@@ -337,6 +339,7 @@ TEST_F(MemTableListTest, GetTest) {
  WriteBufferManager wb2(options.db_write_buffer_size);
  MemTable* mem2 = new MemTable(cmp, ioptions, MutableCFOptions(options), &wb2,
                                kMaxSequenceNumber, 0 /* column_family_id */);
+  mem2->SetID(2);
  mem2->Ref();

  ASSERT_OK(
@@ -398,7 +401,7 @@ TEST_F(MemTableListTest, GetTest) {
  ASSERT_EQ(2, list.NumNotFlushed());

  list.current()->Unref(&to_delete);
-  for (MemTable* m : to_delete) {
+  for (ReadOnlyMemTable* m : to_delete) {
    delete m;
  }
 }
@@ -418,7 +421,7 @@ TEST_F(MemTableListTest, GetFromHistoryTest) {
  MergeContext merge_context;
  InternalKeyComparator ikey_cmp(options.comparator);
  SequenceNumber max_covering_tombstone_seq = 0;
-  autovector<MemTable*> to_delete;
+  autovector<ReadOnlyMemTable*> to_delete;

  LookupKey lkey("key1", seq);
  bool found = list.current()->Get(lkey, &value, /*columns=*/nullptr,
@@ -491,7 +494,7 @@ TEST_F(MemTableListTest, GetFromHistoryTest) {

  // Flush this memtable from the list.
  // (It will then be a part of the memtable history).
-  autovector<MemTable*> to_flush;
+  autovector<ReadOnlyMemTable*> to_flush;
  list.PickMemtablesToFlush(
      std::numeric_limits<uint64_t>::max() /* memtable_id */, &to_flush);
  ASSERT_EQ(1, to_flush.size());
@@ -636,7 +639,7 @@ TEST_F(MemTableListTest, GetFromHistoryTest) {
  // Cleanup
  list.current()->Unref(&to_delete);
  ASSERT_EQ(3, to_delete.size());
-  for (MemTable* m : to_delete) {
+  for (ReadOnlyMemTable* m : to_delete) {
    delete m;
  }
 }
@@ -651,7 +654,7 @@ TEST_F(MemTableListTest, FlushPendingTest) {
  ImmutableOptions ioptions(options);
  InternalKeyComparator cmp(BytewiseComparator());
  WriteBufferManager wb(options.db_write_buffer_size);
-  autovector<MemTable*> to_delete;
+  autovector<ReadOnlyMemTable*> to_delete;

  // Create MemTableList
  int min_write_buffer_number_to_merge = 3;
@@ -692,7 +695,7 @@ TEST_F(MemTableListTest, FlushPendingTest) {
  // Nothing to flush
  ASSERT_FALSE(list.IsFlushPending());
  ASSERT_FALSE(list.imm_flush_needed.load(std::memory_order_acquire));
-  autovector<MemTable*> to_flush;
+  autovector<ReadOnlyMemTable*> to_flush;
  list.PickMemtablesToFlush(
      std::numeric_limits<uint64_t>::max() /* memtable_id */, &to_flush);
  ASSERT_EQ(0, to_flush.size());
@@ -758,7 +761,7 @@ TEST_F(MemTableListTest, FlushPendingTest) {
  ASSERT_FALSE(list.imm_flush_needed.load(std::memory_order_acquire));

  // Pick tables to flush again
-  autovector<MemTable*> to_flush2;
+  autovector<ReadOnlyMemTable*> to_flush2;
  list.PickMemtablesToFlush(
      std::numeric_limits<uint64_t>::max() /* memtable_id */, &to_flush2);
  ASSERT_EQ(0, to_flush2.size());
@@ -811,7 +814,7 @@ TEST_F(MemTableListTest, FlushPendingTest) {
  ASSERT_TRUE(list.imm_flush_needed.load(std::memory_order_acquire));

  // Pick tables to flush again
-  autovector<MemTable*> to_flush3;
+  autovector<ReadOnlyMemTable*> to_flush3;
  list.PickMemtablesToFlush(
      std::numeric_limits<uint64_t>::max() /* memtable_id */, &to_flush3);
  // Picks newest (fifth oldest)
@@ -821,7 +824,7 @@ TEST_F(MemTableListTest, FlushPendingTest) {
  ASSERT_FALSE(list.imm_flush_needed.load(std::memory_order_acquire));

  // Nothing left to flush
-  autovector<MemTable*> to_flush4;
+  autovector<ReadOnlyMemTable*> to_flush4;
  list.PickMemtablesToFlush(
      std::numeric_limits<uint64_t>::max() /* memtable_id */, &to_flush4);
  ASSERT_EQ(0, to_flush4.size());
@@ -891,7 +894,7 @@ TEST_F(MemTableListTest, FlushPendingTest) {
  memtable_id = 4;
  // Pick tables to flush. The tables to pick must have ID smaller than or
  // equal to 4. Therefore, no table will be selected in this case.
-  autovector<MemTable*> to_flush5;
+  autovector<ReadOnlyMemTable*> to_flush5;
  list.FlushRequested();
  ASSERT_TRUE(list.HasFlushRequested());
  list.PickMemtablesToFlush(memtable_id, &to_flush5);
@@ -932,8 +935,8 @@ TEST_F(MemTableListTest, EmptyAtomicFlushTest) {
  autovector<MemTableList*> lists;
  autovector<uint32_t> cf_ids;
  autovector<const MutableCFOptions*> options_list;
-  autovector<const autovector<MemTable*>*> to_flush;
-  autovector<MemTable*> to_delete;
+  autovector<const autovector<ReadOnlyMemTable*>*> to_flush;
+  autovector<ReadOnlyMemTable*> to_delete;
  Status s = Mock_InstallMemtableAtomicFlushResults(lists, cf_ids, options_list,
                                                    to_flush, &to_delete);
  ASSERT_OK(s);
@@ -995,7 +998,7 @@ TEST_F(MemTableListTest, AtomicFlushTest) {
    cf_ids.push_back(cf_id++);
  }

-  std::vector<autovector<MemTable*>> flush_candidates(num_cfs);
+  std::vector<autovector<ReadOnlyMemTable*>> flush_candidates(num_cfs);

  // Nothing to flush
  for (auto i = 0; i != num_cfs; ++i) {
@@ -1014,7 +1017,7 @@ TEST_F(MemTableListTest, AtomicFlushTest) {
    ASSERT_FALSE(list->IsFlushPending());
    ASSERT_FALSE(list->imm_flush_needed.load(std::memory_order_acquire));
  }
-  autovector<MemTable*> to_delete;
+  autovector<ReadOnlyMemTable*> to_delete;
  // Add tables to the immutable memtalbe lists associated with column families
  for (auto i = 0; i != num_cfs; ++i) {
    for (auto j = 0; j != num_tables_per_cf; ++j) {
@@ -1041,7 +1044,7 @@ TEST_F(MemTableListTest, AtomicFlushTest) {
  autovector<MemTableList*> tmp_lists;
  autovector<uint32_t> tmp_cf_ids;
  autovector<const MutableCFOptions*> tmp_options_list;
-  autovector<const autovector<MemTable*>*> to_flush;
+  autovector<const autovector<ReadOnlyMemTable*>*> to_flush;
  for (auto i = 0; i != num_cfs; ++i) {
    if (!flush_candidates[i].empty()) {
      to_flush.push_back(&flush_candidates[i]);
@@ -1122,7 +1125,7 @@ TEST_F(MemTableListWithTimestampTest, GetTableNewestUDT) {
  std::vector<MemTable*> tables;
  MutableCFOptions mutable_cf_options(options);
  uint64_t current_ts = 0;
-  autovector<MemTable*> to_delete;
+  autovector<ReadOnlyMemTable*> to_delete;
  std::vector<std::string> newest_udts;

  std::string key;
@@ -1162,7 +1165,7 @@ TEST_F(MemTableListWithTimestampTest, GetTableNewestUDT) {
  }

  list.current()->Unref(&to_delete);
-  for (MemTable* m : to_delete) {
+  for (ReadOnlyMemTable* m : to_delete) {
    delete m;
  }
  to_delete.clear();
@@ -24,24 +24,18 @@ struct MultiCfIteratorInfo {
 template <typename ResetFunc, typename PopulateFunc>
 class MultiCfIteratorImpl {
 public:
-  MultiCfIteratorImpl(const Comparator* comparator, bool allow_unprepared_value,
-                      const std::vector<ColumnFamilyHandle*>& column_families,
-                      const std::vector<Iterator*>& child_iterators,
-                      ResetFunc reset_func, PopulateFunc populate_func)
-      : comparator_(comparator),
-        allow_unprepared_value_(allow_unprepared_value),
-        heap_(MultiCfMinHeap(
-            MultiCfHeapItemComparator<std::greater<int>>(comparator_))),
+  MultiCfIteratorImpl(
+      const ReadOptions& read_options, const Comparator* comparator,
+      std::vector<std::pair<ColumnFamilyHandle*, std::unique_ptr<Iterator>>>&&
+          cfh_iter_pairs,
+      ResetFunc reset_func, PopulateFunc populate_func)
+      : allow_unprepared_value_(read_options.allow_unprepared_value),
+        comparator_(comparator),
+        cfh_iter_pairs_(std::move(cfh_iter_pairs)),
        reset_func_(std::move(reset_func)),
-        populate_func_(std::move(populate_func)) {
-    assert(column_families.size() > 0 &&
-           column_families.size() == child_iterators.size());
-    cfh_iter_pairs_.reserve(column_families.size());
-    for (size_t i = 0; i < column_families.size(); ++i) {
-      cfh_iter_pairs_.emplace_back(
-          column_families[i], std::unique_ptr<Iterator>(child_iterators[i]));
-    }
-  }
+        populate_func_(std::move(populate_func)),
+        heap_(MultiCfMinHeap(
+            MultiCfHeapItemComparator<std::greater<int>>(comparator_))) {}
  ~MultiCfIteratorImpl() { status_.PermitUncheckedError(); }

  // No copy allowed
@@ -108,38 +102,21 @@ class MultiCfIteratorImpl {
      return true;
    }

-    auto prepare_value_func = [this](auto& heap, Iterator* iterator) {
-      assert(iterator);
-      assert(iterator->Valid());
-      assert(iterator->status().ok());
-
-      if (!iterator->PrepareValue()) {
-        assert(!iterator->Valid());
-        assert(!iterator->status().ok());
-
-        considerStatus(iterator->status());
-        assert(!status_.ok());
-
-        heap.clear();
-        return false;
-      }
-
-      return true;
-    };
-
    if (std::holds_alternative<MultiCfMaxHeap>(heap_)) {
-      return PopulateIterator(std::get<MultiCfMaxHeap>(heap_),
-                              prepare_value_func);
+      return PopulateIterator(std::get<MultiCfMaxHeap>(heap_));
    }

-    return PopulateIterator(std::get<MultiCfMinHeap>(heap_),
-                            prepare_value_func);
+    return PopulateIterator(std::get<MultiCfMinHeap>(heap_));
  }

 private:
+  Status status_;
+  bool allow_unprepared_value_;
+  const Comparator* comparator_;
  std::vector<std::pair<ColumnFamilyHandle*, std::unique_ptr<Iterator>>>
      cfh_iter_pairs_;
-  Status status_;
+  ResetFunc reset_func_;
+  PopulateFunc populate_func_;

  template <typename CompareOp>
  class MultiCfHeapItemComparator {
@@ -161,9 +138,6 @@ class MultiCfIteratorImpl {
    const Comparator* comparator_;
  };

-  const Comparator* comparator_;
-  bool allow_unprepared_value_;
-
  using MultiCfMinHeap =
      BinaryHeap<MultiCfIteratorInfo,
                 MultiCfHeapItemComparator<std::greater<int>>>;
@@ -174,9 +148,6 @@ class MultiCfIteratorImpl {

  MultiCfIterHeap heap_;

-  ResetFunc reset_func_;
-  PopulateFunc populate_func_;
-
  Iterator* current() const {
    if (std::holds_alternative<MultiCfMaxHeap>(heap_)) {
      auto& max_heap = std::get<MultiCfMaxHeap>(heap_);
@@ -230,10 +201,8 @@ class MultiCfIteratorImpl {
      ++i;
    }
    if (!allow_unprepared_value_ && !heap.empty()) {
-      [[maybe_unused]] const bool result = PopulateIterator(
-          heap,
-          [](auto& /* heap */, Iterator* /* iterator */) { return true; });
-      assert(result);
+      [[maybe_unused]] const bool result = PopulateIterator(heap);
+      assert(result || (!Valid() && !status_.ok()));
    }
  }

@@ -300,15 +269,13 @@ class MultiCfIteratorImpl {
    }

    if (!allow_unprepared_value_ && !heap.empty()) {
-      [[maybe_unused]] const bool result = PopulateIterator(
-          heap,
-          [](auto& /* heap */, Iterator* /* iterator */) { return true; });
-      assert(result);
+      [[maybe_unused]] const bool result = PopulateIterator(heap);
+      assert(result || (!Valid() && !status_.ok()));
    }
  }

-  template <typename BinaryHeap, typename PrepareValueFunc>
-  bool PopulateIterator(BinaryHeap& heap, PrepareValueFunc prepare_value_func) {
+  template <typename BinaryHeap>
+  bool PopulateIterator(BinaryHeap& heap) {
    // 1. Keep the top iterator (by popping it from the heap) and add it to list
    //    to populate
    // 2. For all non-top iterators having the same key as top iter popped
@@ -319,12 +286,33 @@ class MultiCfIteratorImpl {
    //    collected in step 1 and 2 and add all the iters back to the heap
    assert(!heap.empty());

+    auto prepare_value = [this, &heap](Iterator* iterator) {
+      assert(iterator);
+      assert(iterator->Valid());
+      assert(iterator->status().ok());
+
+      if (!iterator->PrepareValue()) {
+        assert(!iterator->Valid());
+        assert(!iterator->status().ok());
+
+        considerStatus(iterator->status());
+        heap.clear();
+
+        assert(!Valid());
+        assert(!status_.ok());
+
+        return false;
+      }
+
+      return true;
+    };
+
    auto top = heap.top();
    assert(top.iterator);
    assert(top.iterator->Valid());
    assert(top.iterator->status().ok());

-    if (!prepare_value_func(heap, top.iterator)) {
+    if (!prepare_value(top.iterator)) {
      return false;
    }

@@ -344,7 +332,7 @@ class MultiCfIteratorImpl {
        break;
      }

-      if (!prepare_value_func(heap, current.iterator)) {
+      if (!prepare_value(current.iterator)) {
        return false;
      }

@@ -413,9 +413,12 @@ class Repairer {
      if (record_status.ok()) {
        const UnorderedMap<uint32_t, size_t>& record_ts_sz =
            reader.GetRecordedTimestampSize();
+        // Use same value for `seq_per_batch` and `batch_per_txn` as
+        // WriteBatchInternal::InsertInto does below.
        record_status = HandleWriteBatchTimestampSizeDifference(
            &batch, running_ts_sz, record_ts_sz,
-            TimestampSizeConsistencyMode::kVerifyConsistency);
+            TimestampSizeConsistencyMode::kVerifyConsistency,
+            /* seq_per_batch */ false, /* batch_per_txn */ true);
        if (record_status.ok()) {
          record_status =
              WriteBatchInternal::InsertInto(&batch, cf_mems, nullptr, nullptr);
@@ -446,7 +449,7 @@ class Repairer {
      Arena arena;
      ScopedArenaPtr<InternalIterator> iter(
          mem->NewIterator(ro, /*seqno_to_time_mapping=*/nullptr, &arena,
-                           /*prefix_extractor=*/nullptr));
+                           /*prefix_extractor=*/nullptr, /*for_flush=*/true));
      int64_t _current_time = 0;
      immutable_db_options_.clock->GetCurrentTime(&_current_time)
          .PermitUncheckedError();  // ignore error
@@ -474,10 +477,10 @@ class Repairer {
          write_option, cfd->internal_comparator(),
          cfd->internal_tbl_prop_coll_factories(), kNoCompression,
          default_compression, cfd->GetID(), cfd->GetName(), -1 /* level */,
-          false /* is_bottommost */, TableFileCreationReason::kRecovery,
-          0 /* oldest_key_time */, 0 /* file_creation_time */,
-          "DB Repairer" /* db_id */, db_session_id_, 0 /*target_file_size*/,
-          meta.fd.GetNumber());
+          current_time /* newest_key_time */, false /* is_bottommost */,
+          TableFileCreationReason::kRecovery, 0 /* oldest_key_time */,
+          0 /* file_creation_time */, "DB Repairer" /* db_id */, db_session_id_,
+          0 /*target_file_size*/, meta.fd.GetNumber());

      SeqnoToTimeMapping empty_seqno_to_time_mapping;
      status = BuildTable(
@@ -60,19 +60,6 @@ class RepairTest : public DBTestBase {
    ASSERT_GT(verify_passed, 0);
    SyncPoint::GetInstance()->DisableProcessing();
  }
-
-  std::vector<FileMetaData*> GetLevelFileMetadatas(int level, int cf = 0) {
-    VersionSet* const versions = dbfull()->GetVersionSet();
-    assert(versions);
-    ColumnFamilyData* const cfd =
-        versions->GetColumnFamilySet()->GetColumnFamily(cf);
-    assert(cfd);
-    Version* const current = cfd->current();
-    assert(current);
-    VersionStorageInfo* const storage_info = current->storage_info();
-    assert(storage_info);
-    return storage_info->LevelFiles(level);
-  }
 };

 TEST_F(RepairTest, SortRepairedDBL0ByEpochNumber) {
@@ -164,6 +164,7 @@ Status TableCache::GetTableReader(
 }

 Cache::Handle* TableCache::Lookup(Cache* cache, uint64_t file_number) {
+  // NOTE: sharing same Cache with BlobFileCache
  Slice key = GetSliceForFileNumber(&file_number);
  return cache->Lookup(key);
 }
@@ -178,6 +179,7 @@ Status TableCache::FindTable(
    size_t max_file_size_for_l0_meta_pin, Temperature file_temperature) {
  PERF_TIMER_GUARD_WITH_CLOCK(find_table_nanos, ioptions_.clock);
  uint64_t number = file_meta.fd.GetNumber();
+  // NOTE: sharing same Cache with BlobFileCache
  Slice key = GetSliceForFileNumber(&number);
  *handle = cache_.Lookup(key);
  TEST_SYNC_POINT_CALLBACK("TableCache::FindTable:0",
@@ -724,13 +726,19 @@ uint64_t TableCache::ApproximateSize(
  return result;
 }

-void TableCache::ReleaseObsolete(Cache* cache, Cache::Handle* h,
+void TableCache::ReleaseObsolete(Cache* cache, uint64_t file_number,
+                                 Cache::Handle* h,
                                 uint32_t uncache_aggressiveness) {
  CacheInterface typed_cache(cache);
  TypedHandle* table_handle = reinterpret_cast<TypedHandle*>(h);
-  TableReader* table_reader = typed_cache.Value(table_handle);
-  table_reader->MarkObsolete(uncache_aggressiveness);
-  typed_cache.ReleaseAndEraseIfLastRef(table_handle);
+  if (table_handle == nullptr) {
+    table_handle = typed_cache.Lookup(GetSliceForFileNumber(&file_number));
+  }
+  if (table_handle != nullptr) {
+    TableReader* table_reader = typed_cache.Value(table_handle);
+    table_reader->MarkObsolete(uncache_aggressiveness);
+    typed_cache.ReleaseAndEraseIfLastRef(table_handle);
+  }
 }

 }  // namespace ROCKSDB_NAMESPACE
@@ -159,12 +159,14 @@ class TableCache {
                         bool skip_range_deletions = false, int level = -1,
                         TypedHandle* table_handle = nullptr);

-  // Evict any entry for the specified file number
+  // Evict any entry for the specified file number. ReleaseObsolete() is
+  // preferred for cleaning up from obsolete files.
  static void Evict(Cache* cache, uint64_t file_number);

  // Handles releasing, erasing, etc. of what should be the last reference
-  // to an obsolete file.
-  static void ReleaseObsolete(Cache* cache, Cache::Handle* handle,
+  // to an obsolete file. `handle` may be nullptr if no prior handle is known.
+  static void ReleaseObsolete(Cache* cache, uint64_t file_number,
+                              Cache::Handle* handle,
                              uint32_t uncache_aggressiveness);

  // Return handle to an existing cache entry if there is one
@@ -58,7 +58,7 @@ void MakeBuilder(
      ioptions, moptions, read_options, write_options, internal_comparator,
      internal_tbl_prop_coll_factories, options.compression,
      options.compression_opts, kTestColumnFamilyId, kTestColumnFamilyName,
-      kTestLevel);
+      kTestLevel, kUnknownNewestKeyTime);
  builder->reset(NewTableBuilder(tboptions, writable->get()));
 }
 }  // namespace
@@ -231,13 +231,14 @@ bool TransactionLogIteratorImpl::IsBatchExpected(
  assert(batch);
  SequenceNumber batchSeq = WriteBatchInternal::Sequence(batch);
  if (batchSeq != expected_seq) {
-    char buf[200];
-    snprintf(buf, sizeof(buf),
-             "Discontinuity in log records. Got seq=%" PRIu64
-             ", Expected seq=%" PRIu64 ", Last flushed seq=%" PRIu64
-             ".Log iterator will reseek the correct batch.",
-             batchSeq, expected_seq, versions_->LastSequence());
-    reporter_.Info(buf);
+    std::ostringstream oss;
+    oss << "Discontinuity in log records. "
+        << "Got seq=" << batchSeq << ", "
+        << "Expected seq=" << expected_seq << ", "
+        << "Last flushed seq=" << versions_->LastSequence() << ". "
+        << "Log iterator will reseek the correct batch.";
+
+    reporter_.Info(oss.str().c_str());
    return false;
  }
  return true;
@@ -24,6 +24,7 @@
 #include <vector>

 #include "cache/cache_reservation_manager.h"
+#include "db/blob/blob_file_cache.h"
 #include "db/blob/blob_file_meta.h"
 #include "db/dbformat.h"
 #include "db/internal_stats.h"
@@ -744,12 +745,9 @@ class VersionBuilder::Rep {
      return Status::Corruption("VersionBuilder", oss.str());
    }

-    // Note: we use C++11 for now but in C++14, this could be done in a more
-    // elegant way using generalized lambda capture.
-    VersionSet* const vs = version_set_;
-    const ImmutableCFOptions* const ioptions = ioptions_;
-
-    auto deleter = [vs, ioptions](SharedBlobFileMetaData* shared_meta) {
+    auto deleter = [vs = version_set_, ioptions = ioptions_,
+                    bc = cfd_ ? cfd_->blob_file_cache()
+                              : nullptr](SharedBlobFileMetaData* shared_meta) {
      if (vs) {
        assert(ioptions);
        assert(!ioptions->cf_paths.empty());
@@ -758,6 +756,9 @@ class VersionBuilder::Rep {
        vs->AddObsoleteBlobFile(shared_meta->GetBlobFileNumber(),
                                ioptions->cf_paths.front().path);
      }
+      if (bc) {
+        bc->Evict(shared_meta->GetBlobFileNumber());
+      }

      delete shared_meta;
    };
@@ -766,7 +767,7 @@ class VersionBuilder::Rep {
        blob_file_number, blob_file_addition.GetTotalBlobCount(),
        blob_file_addition.GetTotalBlobBytes(),
        blob_file_addition.GetChecksumMethod(),
-        blob_file_addition.GetChecksumValue(), deleter);
+        blob_file_addition.GetChecksumValue(), std::move(deleter));

    mutable_blob_file_metas_.emplace(
        blob_file_number, MutableBlobFileMetaData(std::move(shared_meta)));
@@ -107,6 +107,7 @@ class VersionSet;

 constexpr uint64_t kFileNumberMask = 0x3FFFFFFFFFFFFFFF;
 constexpr uint64_t kUnknownOldestAncesterTime = 0;
+constexpr uint64_t kUnknownNewestKeyTime = 0;
 constexpr uint64_t kUnknownFileCreationTime = 0;
 constexpr uint64_t kUnknownEpochNumber = 0;
 // If `Options::allow_ingest_behind` is true, this epoch number
@@ -333,6 +334,27 @@ struct FileMetaData {
    return kUnknownFileCreationTime;
  }

+  // Tries to get the newest key time from the current file
+  // Falls back on oldest ancestor time of previous (newer) file
+  uint64_t TryGetNewestKeyTime(FileMetaData* prev_file = nullptr) {
+    if (fd.table_reader != nullptr &&
+        fd.table_reader->GetTableProperties() != nullptr) {
+      uint64_t newest_key_time =
+          fd.table_reader->GetTableProperties()->newest_key_time;
+      if (newest_key_time != kUnknownNewestKeyTime) {
+        return newest_key_time;
+      }
+    }
+    if (prev_file != nullptr) {
+      uint64_t prev_oldest_ancestor_time =
+          prev_file->TryGetOldestAncesterTime();
+      if (prev_oldest_ancestor_time != kUnknownOldestAncesterTime) {
+        return prev_oldest_ancestor_time;
+      }
+    }
+    return kUnknownNewestKeyTime;
+  }
+
  // WARNING: manual update to this function is needed
  // whenever a new string property is added to FileMetaData
  // to reduce approximation error.
@@ -31,6 +31,7 @@
 #include "db/internal_stats.h"
 #include "db/log_reader.h"
 #include "db/log_writer.h"
+#include "db/manifest_ops.h"
 #include "db/memtable.h"
 #include "db/merge_context.h"
 #include "db/merge_helper.h"
@@ -3386,28 +3387,28 @@ bool ShouldChangeFileTemperature(const ImmutableOptions& ioptions,
  int64_t _current_time;
  auto status = ioptions.clock->GetCurrentTime(&_current_time);
  const uint64_t current_time = static_cast<uint64_t>(_current_time);
-  // We use oldest_ancestor_time of a file to be the estimate age of
-  // the file just older than it. This is the same logic used in
+  // This is the same logic used in
  // FIFOCompactionPicker::PickTemperatureChangeCompaction().
  if (status.ok() && current_time >= ages[0].age) {
    uint64_t create_time_threshold = current_time - ages[0].age;
    Temperature target_temp;
    assert(files.size() >= 1);
-    for (size_t index = files.size() - 1; index >= 1; --index) {
-      FileMetaData* cur_file = files[index];
-      FileMetaData* prev_file = files[index - 1];
+    for (size_t index = files.size(); index >= 1; --index) {
+      FileMetaData* cur_file = files[index - 1];
+      FileMetaData* prev_file = index < 2 ? nullptr : files[index - 2];
      if (!cur_file->being_compacted) {
-        uint64_t oldest_ancestor_time = prev_file->TryGetOldestAncesterTime();
-        if (oldest_ancestor_time == kUnknownOldestAncesterTime) {
-          return false;
+        uint64_t est_newest_key_time = cur_file->TryGetNewestKeyTime(prev_file);
+        // Newer file could have newest_key_time populated
+        if (est_newest_key_time == kUnknownNewestKeyTime) {
+          continue;
        }
-        if (oldest_ancestor_time > create_time_threshold) {
+        if (est_newest_key_time > create_time_threshold) {
          return false;
        }
        target_temp = ages[0].temperature;
        for (size_t i = 1; i < ages.size(); ++i) {
          if (current_time >= ages[i].age &&
-              oldest_ancestor_time <= current_time - ages[i].age) {
+              est_newest_key_time <= current_time - ages[i].age) {
            target_temp = ages[i].temperature;
          }
        }
@@ -4760,7 +4761,7 @@ void VersionStorageInfo::CalculateBaseBytes(const ImmutableOptions& ioptions,
            cur_level_size / options.max_bytes_for_level_multiplier);
        if (lowest_unnecessary_level_ == -1 &&
            cur_level_size <= base_bytes_min &&
-            (ioptions.preclude_last_level_data_seconds == 0 ||
+            (options.preclude_last_level_data_seconds == 0 ||
             i < num_levels_ - 2)) {
          // When per_key_placement is enabled, the penultimate level is
          // necessary.
@@ -4776,7 +4777,7 @@ void VersionStorageInfo::CalculateBaseBytes(const ImmutableOptions& ioptions,
        // which can less than base_bytes_min AND necessary,
        // or there is some unnecessary level.
        assert(first_non_empty_level == num_levels_ - 1 ||
-               ioptions.preclude_last_level_data_seconds > 0 ||
+               options.preclude_last_level_data_seconds > 0 ||
               lowest_unnecessary_level_ != -1);
        // Case 1. If we make target size of last level to be max_level_size,
        // target size of the first non-empty level would be smaller than
@@ -5204,18 +5205,16 @@ VersionSet::~VersionSet() {
  column_family_set_.reset();

  for (auto& file : obsolete_files_) {
-    if (file.metadata->table_reader_handle) {
-      // NOTE: DB is shutting down, so file is probably not obsolete, just
-      // no longer referenced by Versions in memory.
-      // For more context, see comment on "table_cache_->EraseUnRefEntries()"
-      // in DBImpl::CloseHelper().
-      // Using uncache_aggressiveness=0 overrides any previous marking to
-      // attempt to uncache the file's blocks (which after cleaning up
-      // column families could cause use-after-free)
-      TableCache::ReleaseObsolete(table_cache_,
-                                  file.metadata->table_reader_handle,
-                                  /*uncache_aggressiveness=*/0);
-    }
+    // NOTE: DB is shutting down, so file is probably not obsolete, just
+    // no longer referenced by Versions in memory.
+    // For more context, see comment on "table_cache_->EraseUnRefEntries()"
+    // in DBImpl::CloseHelper().
+    // Using uncache_aggressiveness=0 overrides any previous marking to
+    // attempt to uncache the file's blocks (which after cleaning up
+    // column families could cause use-after-free)
+    TableCache::ReleaseObsolete(table_cache_, file.metadata->fd.GetNumber(),
+                                file.metadata->table_reader_handle,
+                                /*uncache_aggressiveness=*/0);
    file.DeleteMetadata();
  }
  obsolete_files_.clear();
@@ -6013,37 +6012,6 @@ Status VersionSet::LogAndApplyHelper(ColumnFamilyData* cfd,
  return builder ? builder->Apply(edit) : Status::OK();
 }

-Status VersionSet::GetCurrentManifestPath(const std::string& dbname,
-                                          FileSystem* fs,
-                                          std::string* manifest_path,
-                                          uint64_t* manifest_file_number) {
-  assert(fs != nullptr);
-  assert(manifest_path != nullptr);
-  assert(manifest_file_number != nullptr);
-
-  std::string fname;
-  Status s = ReadFileToString(fs, CurrentFileName(dbname), &fname);
-  if (!s.ok()) {
-    return s;
-  }
-  if (fname.empty() || fname.back() != '\n') {
-    return Status::Corruption("CURRENT file does not end with newline");
-  }
-  // remove the trailing '\n'
-  fname.resize(fname.size() - 1);
-  FileType type;
-  bool parse_ok = ParseFileName(fname, manifest_file_number, &type);
-  if (!parse_ok || type != kDescriptorFile) {
-    return Status::Corruption("CURRENT file corrupted");
-  }
-  *manifest_path = dbname;
-  if (dbname.back() != '/') {
-    manifest_path->push_back('/');
-  }
-  manifest_path->append(fname);
-  return Status::OK();
-}
-
 Status VersionSet::Recover(
    const std::vector<ColumnFamilyDescriptor>& column_families, bool read_only,
    std::string* db_id, bool no_error_if_files_missing, bool is_retry,
@@ -6052,8 +6020,8 @@ Status VersionSet::Recover(
  // Read "CURRENT" file, which contains a pointer to the current manifest
  // file
  std::string manifest_path;
-  Status s = GetCurrentManifestPath(dbname_, fs_.get(), &manifest_path,
-                                    &manifest_file_number_);
+  Status s = GetCurrentManifestPath(dbname_, fs_.get(), is_retry,
+                                    &manifest_path, &manifest_file_number_);
  if (!s.ok()) {
    return s;
  }
@@ -6298,8 +6266,8 @@ Status VersionSet::ListColumnFamilies(std::vector<std::string>* column_families,
  // Read "CURRENT" file, which contains a pointer to the current manifest file
  std::string manifest_path;
  uint64_t manifest_file_number;
-  Status s =
-      GetCurrentManifestPath(dbname, fs, &manifest_path, &manifest_file_number);
+  Status s = GetCurrentManifestPath(dbname, fs, /*is_retry=*/false,
+                                    &manifest_path, &manifest_file_number);
  if (!s.ok()) {
    return s;
  }
@@ -7497,8 +7465,8 @@ Status ReactiveVersionSet::MaybeSwitchManifest(
  assert(manifest_reader != nullptr);
  Status s;
  std::string manifest_path;
-  s = GetCurrentManifestPath(dbname_, fs_.get(), &manifest_path,
-                             &manifest_file_number_);
+  s = GetCurrentManifestPath(dbname_, fs_.get(), /*is_retry=*/false,
+                             &manifest_path, &manifest_file_number_);
  if (!s.ok()) {
    return s;
  }
@@ -1277,10 +1277,6 @@ class VersionSet {
      const std::vector<std::function<void(const Status&)>>& manifest_wcbs =
          {});

-  static Status GetCurrentManifestPath(const std::string& dbname,
-                                       FileSystem* fs,
-                                       std::string* manifest_filename,
-                                       uint64_t* manifest_file_number);
  void WakeUpWaitingManifestWriters();

  // Recover the last saved descriptor (MANIFEST) from persistent storage.
@@ -1526,7 +1522,6 @@ class VersionSet {
  void GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata);

  void AddObsoleteBlobFile(uint64_t blob_file_number, std::string path) {
-    // TODO: Erase file from BlobFileCache?
    obsolete_blob_files_.emplace_back(blob_file_number, std::move(path));
  }

@@ -15,6 +15,7 @@
 #include "db/db_impl/db_impl.h"
 #include "db/db_test_util.h"
 #include "db/log_writer.h"
+#include "db/manifest_ops.h"
 #include "db/version_edit.h"
 #include "rocksdb/advanced_options.h"
 #include "rocksdb/convenience.h"
@@ -1223,7 +1224,7 @@ class VersionSetTestBase {
      tmp_db_options.env = env_;
      std::unique_ptr<DBImpl> impl(new DBImpl(tmp_db_options, dbname_));
      std::string db_id;
-      ASSERT_OK(impl->GetDbIdentityFromIdentityFile(&db_id));
+      ASSERT_OK(impl->GetDbIdentityFromIdentityFile(IOOptions(), &db_id));
      new_db.SetDBId(db_id);
    }
    new_db.SetLogNumber(0);
@@ -1328,7 +1329,7 @@ class VersionSetTestBase {
              &internal_tbl_prop_coll_factories, kNoCompression,
              CompressionOptions(),
              TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
-              info.column_family, info.level),
+              info.column_family, info.level, kUnknownNewestKeyTime),
          fwriter.get()));
      InternalKey ikey(info.key, 0, ValueType::kTypeValue);
      builder->Add(ikey.Encode(), "value");
@@ -1390,16 +1391,16 @@ class VersionSetTestBase {
  void GetManifestPath(std::string* manifest_path) const {
    assert(manifest_path != nullptr);
    uint64_t manifest_file_number = 0;
-    Status s = versions_->GetCurrentManifestPath(
-        dbname_, fs_.get(), manifest_path, &manifest_file_number);
+    Status s = GetCurrentManifestPath(dbname_, fs_.get(), /*is_retry=*/false,
+                                      manifest_path, &manifest_file_number);
    ASSERT_OK(s);
  }

  void VerifyManifest(std::string* manifest_path) const {
    assert(manifest_path != nullptr);
    uint64_t manifest_file_number = 0;
-    Status s = versions_->GetCurrentManifestPath(
-        dbname_, fs_.get(), manifest_path, &manifest_file_number);
+    Status s = GetCurrentManifestPath(dbname_, fs_.get(), /*is_retry=*/false,
+                                      manifest_path, &manifest_file_number);
    ASSERT_OK(s);
    ASSERT_EQ(1, manifest_file_number);
  }
@@ -3515,7 +3516,7 @@ class VersionSetTestEmptyDb
      tmp_db_options.env = env_;
      std::unique_ptr<DBImpl> impl(new DBImpl(tmp_db_options, dbname_));
      std::string db_id;
-      ASSERT_OK(impl->GetDbIdentityFromIdentityFile(&db_id));
+      ASSERT_OK(impl->GetDbIdentityFromIdentityFile(IOOptions(), &db_id));
      new_db.SetDBId(db_id);
    }
    const std::string manifest_path = DescriptorFileName(dbname_, 1);
@@ -3839,7 +3840,7 @@ class VersionSetTestMissingFiles : public VersionSetTestBase,
      tmp_db_options.env = env_;
      std::unique_ptr<DBImpl> impl(new DBImpl(tmp_db_options, dbname_));
      std::string db_id;
-      ASSERT_OK(impl->GetDbIdentityFromIdentityFile(&db_id));
+      ASSERT_OK(impl->GetDbIdentityFromIdentityFile(IOOptions(), &db_id));
      new_db.SetDBId(db_id);
    }
    {
@@ -129,21 +129,6 @@ Status WideColumnSerialization::Deserialize(Slice& input,
  return Status::OK();
 }

-WideColumns::const_iterator WideColumnSerialization::Find(
-    const WideColumns& columns, const Slice& column_name) {
-  const auto it =
-      std::lower_bound(columns.cbegin(), columns.cend(), column_name,
-                       [](const WideColumn& lhs, const Slice& rhs) {
-                         return lhs.name().compare(rhs) < 0;
-                       });
-
-  if (it == columns.cend() || it->name() != column_name) {
-    return columns.cend();
-  }
-
-  return it;
-}
-
 Status WideColumnSerialization::GetValueOfDefaultColumn(Slice& input,
                                                        Slice& value) {
  WideColumns columns;
@@ -47,8 +47,6 @@ class WideColumnSerialization {

  static Status Deserialize(Slice& input, WideColumns& columns);

-  static WideColumns::const_iterator Find(const WideColumns& columns,
-                                          const Slice& column_name);
  static Status GetValueOfDefaultColumn(Slice& input, Slice& value);

  static constexpr uint32_t kCurrentVersion = 1;
@@ -5,6 +5,7 @@

 #include "db/wide/wide_column_serialization.h"

+#include "db/wide/wide_columns_helper.h"
 #include "test_util/testharness.h"
 #include "util/coding.h"

@@ -99,27 +100,28 @@ TEST(WideColumnSerializationTest, SerializeDeserialize) {
  ASSERT_EQ(columns, deserialized_columns);

  {
-    const auto it = WideColumnSerialization::Find(deserialized_columns, "foo");
+    const auto it = WideColumnsHelper::Find(deserialized_columns.cbegin(),
+                                            deserialized_columns.cend(), "foo");
    ASSERT_NE(it, deserialized_columns.cend());
    ASSERT_EQ(*it, deserialized_columns.front());
  }

  {
-    const auto it =
-        WideColumnSerialization::Find(deserialized_columns, "hello");
+    const auto it = WideColumnsHelper::Find(
+        deserialized_columns.cbegin(), deserialized_columns.cend(), "hello");
    ASSERT_NE(it, deserialized_columns.cend());
    ASSERT_EQ(*it, deserialized_columns.back());
  }

  {
-    const auto it =
-        WideColumnSerialization::Find(deserialized_columns, "fubar");
+    const auto it = WideColumnsHelper::Find(
+        deserialized_columns.cbegin(), deserialized_columns.cend(), "fubar");
    ASSERT_EQ(it, deserialized_columns.cend());
  }

  {
-    const auto it =
-        WideColumnSerialization::Find(deserialized_columns, "snafu");
+    const auto it = WideColumnsHelper::Find(
+        deserialized_columns.cbegin(), deserialized_columns.cend(), "snafu");
    ASSERT_EQ(it, deserialized_columns.cend());
  }
 }
@@ -5,7 +5,6 @@

 #include "db/wide/wide_columns_helper.h"

-#include <algorithm>
 #include <ios>

 #include "db/wide/wide_column_serialization.h"
@@ -42,11 +41,4 @@ Status WideColumnsHelper::DumpSliceAsWideColumns(const Slice& value,
  return s;
 }

-void WideColumnsHelper::SortColumns(WideColumns& columns) {
-  std::sort(columns.begin(), columns.end(),
-            [](const WideColumn& lhs, const WideColumn& rhs) {
-              return lhs.name().compare(rhs.name()) < 0;
-            });
-}
-
 }  // namespace ROCKSDB_NAMESPACE
@@ -4,8 +4,10 @@
 //  (found in the LICENSE.Apache file in the root directory).

 #pragma once
+
+#include <algorithm>
+#include <cassert>
 #include <ostream>
-#include <string>

 #include "rocksdb/rocksdb_namespace.h"
 #include "rocksdb/wide_columns.h"
@@ -34,7 +36,31 @@ class WideColumnsHelper {
    return columns.front().value();
  }

-  static void SortColumns(WideColumns& columns);
+  static void SortColumns(WideColumns& columns) {
+    std::sort(columns.begin(), columns.end(),
+              [](const WideColumn& lhs, const WideColumn& rhs) {
+                return lhs.name().compare(rhs.name()) < 0;
+              });
+  }
+
+  template <typename Iterator>
+  static Iterator Find(Iterator begin, Iterator end, const Slice& column_name) {
+    assert(std::is_sorted(begin, end,
+                          [](const WideColumn& lhs, const WideColumn& rhs) {
+                            return lhs.name().compare(rhs.name()) < 0;
+                          }));
+
+    auto it = std::lower_bound(begin, end, column_name,
+                               [](const WideColumn& lhs, const Slice& rhs) {
+                                 return lhs.name().compare(rhs) < 0;
+                               });
+
+    if (it == end || it->name() != column_name) {
+      return end;
+    }
+
+    return it;
+  }
 };

 }  // namespace ROCKSDB_NAMESPACE
@@ -1160,6 +1160,34 @@ Status WriteBatchInternal::InsertNoop(WriteBatch* b) {
  return Status::OK();
 }

+ValueType WriteBatchInternal::GetBeginPrepareType(bool write_after_commit,
+                                                  bool unprepared_batch) {
+  return write_after_commit
+             ? kTypeBeginPrepareXID
+             : (unprepared_batch ? kTypeBeginUnprepareXID
+                                 : kTypeBeginPersistedPrepareXID);
+}
+
+Status WriteBatchInternal::InsertBeginPrepare(WriteBatch* b,
+                                              bool write_after_commit,
+                                              bool unprepared_batch) {
+  b->rep_.push_back(static_cast<char>(
+      GetBeginPrepareType(write_after_commit, unprepared_batch)));
+  b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) |
+                              ContentFlags::HAS_BEGIN_PREPARE,
+                          std::memory_order_relaxed);
+  return Status::OK();
+}
+
+Status WriteBatchInternal::InsertEndPrepare(WriteBatch* b, const Slice& xid) {
+  b->rep_.push_back(static_cast<char>(kTypeEndPrepareXID));
+  PutLengthPrefixedSlice(&b->rep_, xid);
+  b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) |
+                              ContentFlags::HAS_END_PREPARE,
+                          std::memory_order_relaxed);
+  return Status::OK();
+}
+
 Status WriteBatchInternal::MarkEndPrepare(WriteBatch* b, const Slice& xid,
                                          bool write_after_commit,
                                          bool unprepared_batch) {
@@ -1175,13 +1203,8 @@ Status WriteBatchInternal::MarkEndPrepare(WriteBatch* b, const Slice& xid,

  // rewrite noop as begin marker
  b->rep_[12] = static_cast<char>(
-      write_after_commit ? kTypeBeginPrepareXID
-                         : (unprepared_batch ? kTypeBeginUnprepareXID
-                                             : kTypeBeginPersistedPrepareXID));
-  b->rep_.push_back(static_cast<char>(kTypeEndPrepareXID));
-  PutLengthPrefixedSlice(&b->rep_, xid);
+      GetBeginPrepareType(write_after_commit, unprepared_batch));
  b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) |
-                              ContentFlags::HAS_END_PREPARE |
                              ContentFlags::HAS_BEGIN_PREPARE,
                          std::memory_order_relaxed);
  if (unprepared_batch) {
@@ -1189,7 +1212,7 @@ Status WriteBatchInternal::MarkEndPrepare(WriteBatch* b, const Slice& xid,
                                ContentFlags::HAS_BEGIN_UNPREPARE,
                            std::memory_order_relaxed);
  }
-  return Status::OK();
+  return WriteBatchInternal::InsertEndPrepare(b, xid);
 }

 Status WriteBatchInternal::MarkCommit(WriteBatch* b, const Slice& xid) {
@@ -122,6 +122,15 @@ class WriteBatchInternal {
  static Status PutBlobIndex(WriteBatch* batch, uint32_t column_family_id,
                             const Slice& key, const Slice& value);

+  static ValueType GetBeginPrepareType(bool write_after_commit,
+                                       bool unprepared_batch);
+
+  static Status InsertBeginPrepare(WriteBatch* batch,
+                                   const bool write_after_commit = true,
+                                   bool unprepared_batch = false);
+
+  static Status InsertEndPrepare(WriteBatch* batch, const Slice& xid);
+
  static Status MarkEndPrepare(WriteBatch* batch, const Slice& xid,
                               const bool write_after_commit = true,
                               const bool unprepared_batch = false);
@@ -59,7 +59,8 @@ static std::string PrintContents(WriteBatch* b,
    InternalIterator* iter;
    if (i == 0) {
      iter = mem->NewIterator(ReadOptions(), /*seqno_to_time_mapping=*/nullptr,
-                              &arena, /*prefix_extractor=*/nullptr);
+                              &arena, /*prefix_extractor=*/nullptr,
+                              /*for_flush=*/false);
      arena_iter_guard.reset(iter);
    } else {
      iter = mem->NewRangeTombstoneIterator(ReadOptions(),
@@ -523,8 +523,10 @@ size_t WriteThread::EnterAsBatchGroupLeader(Writer* leader,
        // those are something else. They want to be alone
        (w->callback != nullptr && !w->callback->AllowWriteBatching()) ||
        // dont batch writes that don't want to be batched
-        (size + WriteBatchInternal::ByteSize(w->batch) > max_size)
+        (size + WriteBatchInternal::ByteSize(w->batch) > max_size) ||
        // Do not make batch too big
+        (leader->ingest_wbwi || w->ingest_wbwi)
+        // ingesting WBWI needs to be its own group
    ) {
      // remove from list
      w->link_older->link_newer = w->link_newer;
@@ -148,6 +148,8 @@ class WriteThread {
    Writer* link_older;  // read/write only before linking, or as leader
    Writer* link_newer;  // lazy, read/write only before linking, or as leader

+    bool ingest_wbwi;
+
    Writer()
        : batch(nullptr),
          sync(false),
@@ -174,7 +176,8 @@ class WriteThread {
           WriteCallback* _callback, UserWriteCallback* _user_write_cb,
           uint64_t _log_ref, bool _disable_memtable, size_t _batch_cnt = 0,
           PreReleaseCallback* _pre_release_callback = nullptr,
-           PostMemTableCallback* _post_memtable_callback = nullptr)
+           PostMemTableCallback* _post_memtable_callback = nullptr,
+           bool _ingest_wbwi = false)
        : batch(_batch),
          // TODO: store a copy of WriteOptions instead of its seperated data
          // members
@@ -196,7 +199,8 @@ class WriteThread {
          write_group(nullptr),
          sequence(kMaxSequenceNumber),
          link_older(nullptr),
-          link_newer(nullptr) {}
+          link_newer(nullptr),
+          ingest_wbwi(_ingest_wbwi) {}

    ~Writer() {
      if (made_waitable) {
@@ -613,6 +613,21 @@ class BatchedOpsStressTest : public StressTest {
        // no iterator should finish before the first one
        assert(iters[i]->Valid() &&
               iters[i]->key().starts_with(prefix_slices[i]));
+
+        if (ro_copies[i].allow_unprepared_value) {
+          // Save key in case PrepareValue fails and invalidates the iterator
+          const std::string prepare_value_key =
+              iters[i]->key().ToString(/* hex */ true);
+
+          if (!iters[i]->PrepareValue()) {
+            fprintf(stderr,
+                    "prefix scan error: PrepareValue failed for key %s: %s\n",
+                    prepare_value_key.c_str(),
+                    iters[i]->status().ToString().c_str());
+            continue;
+          }
+        }
+
        values[i] = iters[i]->value().ToString();

        // make sure the last character of the value is the expected digit
@@ -802,6 +802,13 @@ class CfConsistencyStressTest : public StressTest {
         iter->Next()) {
      ++count;

+      if (ro_copy.allow_unprepared_value) {
+        if (!iter->PrepareValue()) {
+          s = iter->status();
+          break;
+        }
+      }
+
      if (!VerifyWideColumns(iter->value(), iter->columns())) {
        s = Status::Corruption("Value and columns inconsistent",
                               DebugString(iter->value(), iter->columns()));
@@ -417,6 +417,10 @@ DECLARE_bool(check_multiget_consistency);
 DECLARE_bool(check_multiget_entity_consistency);
 DECLARE_bool(inplace_update_support);
 DECLARE_uint32(uncache_aggressiveness);
+DECLARE_int32(test_ingest_standalone_range_deletion_one_in);
+DECLARE_bool(allow_unprepared_value);
+DECLARE_string(file_temperature_age_thresholds);
+DECLARE_uint32(commit_bypass_memtable_one_in);

 constexpr long KB = 1024;
 constexpr int kRandomValueMaxFactor = 3;
@@ -546,6 +546,10 @@ DEFINE_uint32(use_timed_put_one_in, 0,
              "If greater than zero, TimedPut is used per every N write ops on "
              "on average.");

+DEFINE_string(file_temperature_age_thresholds, "",
+              "See CompactionOptionsFIFO::file_temperature_age_thresholds. "
+              "empty == unset");
+
 static const bool FLAGS_subcompactions_dummy __attribute__((__unused__)) =
    RegisterFlagValidator(&FLAGS_subcompactions, &ValidateUint32Range);

@@ -835,6 +839,14 @@ DEFINE_bool(use_get_entity, false, "If set, use the GetEntity API for reads");
 DEFINE_bool(use_multi_get_entity, false,
            "If set, use the MultiGetEntity API for reads");

+DEFINE_int32(test_ingest_standalone_range_deletion_one_in, 0,
+             "If non-zero, file ingestion flow will test standalone range "
+             "deletion file once every N file ingestion operations.");
+
+DEFINE_bool(allow_unprepared_value,
+            ROCKSDB_NAMESPACE::ReadOptions().allow_unprepared_value,
+            "Allow lazy loading of values for range scans");
+
 static bool ValidateInt32Percent(const char* flagname, int32_t value) {
  if (value < 0 || value > 100) {
    fprintf(stderr, "Invalid value for --%s: %d, 0<= pct <=100 \n", flagname,
@@ -1457,4 +1469,7 @@ DEFINE_bool(paranoid_memory_checks,
            ROCKSDB_NAMESPACE::Options().paranoid_memory_checks,
            "Sets CF option paranoid_memory_checks.");

+DEFINE_uint32(commit_bypass_memtable_one_in, 0,
+              "If greater than zero, transaction option will set "
+              "commit_bypass_memtable to per every N transactions on average.");
 #endif  // GFLAGS
@@ -413,6 +413,16 @@ bool StressTest::BuildOptionsTable() {
                                                 "2147483646", "2147483647"});
  }

+  if (!FLAGS_file_temperature_age_thresholds.empty()) {
+    // Modify file_temperature_age_thresholds only if it is set initially
+    // (FIFO tiered storage setup)
+    options_tbl.emplace(
+        "file_temperature_age_thresholds",
+        std::vector<std::string>{
+            "{{temperature=kWarm;age=30}:{temperature=kCold;age=300}}",
+            "{{temperature=kCold;age=100}}", "{}"});
+  }
+
  options_table_ = std::move(options_tbl);

  for (const auto& iter : options_table_) {
@@ -795,8 +805,9 @@ void StressTest::ProcessRecoveredPreparedTxnsHelper(Transaction* txn,
  }
 }

-Status StressTest::NewTxn(WriteOptions& write_opts,
-                          std::unique_ptr<Transaction>* out_txn) {
+Status StressTest::NewTxn(WriteOptions& write_opts, ThreadState* thread,
+                          std::unique_ptr<Transaction>* out_txn,
+                          bool* commit_bypass_memtable) {
  if (!FLAGS_use_txn) {
    return Status::InvalidArgument("NewTxn when FLAGS_use_txn is not set");
  }
@@ -811,6 +822,15 @@ Status StressTest::NewTxn(WriteOptions& write_opts,
        FLAGS_use_only_the_last_commit_time_batch_for_recovery;
    txn_options.lock_timeout = 600000;  // 10 min
    txn_options.deadlock_detect = true;
+    if (FLAGS_commit_bypass_memtable_one_in > 0) {
+      assert(FLAGS_txn_write_policy == 0);
+      assert(FLAGS_user_timestamp_size == 0);
+      txn_options.commit_bypass_memtable =
+          thread->rand.OneIn(FLAGS_commit_bypass_memtable_one_in);
+      if (commit_bypass_memtable) {
+        *commit_bypass_memtable = txn_options.commit_bypass_memtable;
+      }
+    }
    out_txn->reset(txn_db_->BeginTransaction(write_opts, txn_options));
    auto istr = std::to_string(txn_id.fetch_add(1));
    Status s = (*out_txn)->SetName("xid" + istr);
@@ -866,11 +886,12 @@ Status StressTest::CommitTxn(Transaction& txn, ThreadState* thread) {
  return s;
 }

-Status StressTest::ExecuteTransaction(
-    WriteOptions& write_opts, ThreadState* thread,
-    std::function<Status(Transaction&)>&& ops) {
+Status StressTest::ExecuteTransaction(WriteOptions& write_opts,
+                                      ThreadState* thread,
+                                      std::function<Status(Transaction&)>&& ops,
+                                      bool* commit_bypass_memtable) {
  std::unique_ptr<Transaction> txn;
-  Status s = NewTxn(write_opts, &txn);
+  Status s = NewTxn(write_opts, thread, &txn, commit_bypass_memtable);
  std::string try_again_messages;
  if (s.ok()) {
    for (int tries = 1;; ++tries) {
@@ -918,6 +939,8 @@ void StressTest::OperateDb(ThreadState* thread) {
  read_opts.auto_readahead_size = FLAGS_auto_readahead_size;
  read_opts.fill_cache = FLAGS_fill_cache;
  read_opts.optimize_multiget_for_io = FLAGS_optimize_multiget_for_io;
+  read_opts.allow_unprepared_value = FLAGS_allow_unprepared_value;
+
  WriteOptions write_opts;
  if (FLAGS_rate_limit_auto_wal_flush) {
    write_opts.rate_limiter_priority = Env::IO_USER;
@@ -1746,6 +1769,15 @@ Status StressTest::TestIterateImpl(ThreadState* thread,
      op_logs += "S " + key.ToString(true) + " ";
    }

+    if (iter->Valid() && ro.allow_unprepared_value) {
+      op_logs += "*";
+
+      if (!iter->PrepareValue()) {
+        assert(!iter->Valid());
+        assert(!iter->status().ok());
+      }
+    }
+
    if (!iter->status().ok() && IsErrorInjectedAndRetryable(iter->status())) {
      return iter->status();
    } else if (!cmp_iter->status().ok() &&
@@ -1777,6 +1809,15 @@ Status StressTest::TestIterateImpl(ThreadState* thread,

      last_op = kLastOpNextOrPrev;

+      if (iter->Valid() && ro.allow_unprepared_value) {
+        op_logs += "*";
+
+        if (!iter->PrepareValue()) {
+          assert(!iter->Valid());
+          assert(!iter->status().ok());
+        }
+      }
+
      if (!iter->status().ok() && IsErrorInjectedAndRetryable(iter->status())) {
        return iter->status();
      } else if (!cmp_iter->status().ok() &&
@@ -1906,7 +1947,9 @@ void StressTest::VerifyIterator(
               << ", iterate_lower_bound: "
               << (ro.iterate_lower_bound
                       ? ro.iterate_lower_bound->ToString(true).c_str()
-                       : "");
+                       : "")
+               << ", allow_unprepared_value: " << ro.allow_unprepared_value;
+
  if (iter->Valid() && !cmp_iter->Valid()) {
    if (pe != nullptr) {
      if (!pe->InDomain(seek_key)) {
@@ -2000,6 +2043,7 @@ void StressTest::VerifyIterator(
      *diverged = true;
    }
  }
+
  if (*diverged) {
    fprintf(stderr, "VerifyIterator failed. Control CF %s\n",
            cmp_cfh->GetName().c_str());
@@ -4133,6 +4177,18 @@ void InitializeOptionsFromFlags(
  options.default_temperature =
      StringToTemperature(FLAGS_default_temperature.c_str());

+  if (!FLAGS_file_temperature_age_thresholds.empty()) {
+    Status s = GetColumnFamilyOptionsFromString(
+        {}, options,
+        "compaction_options_fifo={file_temperature_age_thresholds=" +
+            FLAGS_file_temperature_age_thresholds + "}",
+        &options);
+    if (!s.ok()) {
+      fprintf(stderr, "While setting file_temperature_age_thresholds: %s\n",
+              s.ToString().c_str());
+      exit(1);
+    }
+  }
  options.preclude_last_level_data_seconds =
      FLAGS_preclude_last_level_data_seconds;
  options.preserve_internal_time_seconds = FLAGS_preserve_internal_time_seconds;
@@ -71,14 +71,19 @@ class StressTest {
  void UpdateIfInitialWriteFails(Env* db_stress_env, const Status& write_s,
                                 Status* initial_write_s,
                                 bool* initial_wal_write_may_succeed,
-                                 uint64_t* wait_for_recover_start_time) {
+                                 uint64_t* wait_for_recover_start_time,
+                                 bool commit_bypass_memtable = false) {
    assert(db_stress_env && initial_write_s && initial_wal_write_may_succeed &&
           wait_for_recover_start_time);
    // Only update `initial_write_s`, `initial_wal_write_may_succeed` when the
    // first write fails
    if (!write_s.ok() && (*initial_write_s).ok()) {
      *initial_write_s = write_s;
+      // With commit_bypass_memtable, we create a new WAL after WAL write
+      // succeeds, that wal creation may fail due to injected error. So the
+      // initial wal write may succeed even if status is failed to write to wal
      *initial_wal_write_may_succeed =
+          commit_bypass_memtable ||
          !FaultInjectionTestFS::IsFailedToWriteToWALError(*initial_write_s);
      *wait_for_recover_start_time = db_stress_env->NowMicros();
    }
@@ -138,13 +143,19 @@ class StressTest {
                                                  SharedState* shared);

  // ExecuteTransaction is recommended instead
-  Status NewTxn(WriteOptions& write_opts,
-                std::unique_ptr<Transaction>* out_txn);
+  // @param commit_bypass_memtable Whether commit_bypass_memtable is set to
+  // true in transaction options.
+  Status NewTxn(WriteOptions& write_opts, ThreadState* thread,
+                std::unique_ptr<Transaction>* out_txn,
+                bool* commit_bypass_memtable = nullptr);
  Status CommitTxn(Transaction& txn, ThreadState* thread = nullptr);

  // Creates a transaction, executes `ops`, and tries to commit
+  // @param commit_bypass_memtable Whether commit_bypass_memtable is set to
+  // true in transaction options.
  Status ExecuteTransaction(WriteOptions& write_opts, ThreadState* thread,
-                            std::function<Status(Transaction&)>&& ops);
+                            std::function<Status(Transaction&)>&& ops,
+                            bool* commit_bypass_memtable = nullptr);

  virtual void MaybeClearOneColumnFamily(ThreadState* /* thread */) {}

@@ -544,6 +544,11 @@ Status MultiOpsTxnsStressTest::TestCustomOperations(
    // Should never reach here.
    assert(false);
  }
+  if (!s.ok()) {
+    fprintf(stderr, "Transaction failed %s\n", s.ToString().c_str());
+    fflush(stderr);
+    thread->shared->SafeTerminate();
+  }

  return s;
 }
@@ -579,7 +584,7 @@ Status MultiOpsTxnsStressTest::PrimaryKeyUpdateTxn(ThreadState* thread,
  std::string new_pk = Record::EncodePrimaryKey(new_a);
  std::unique_ptr<Transaction> txn;
  WriteOptions wopts;
-  Status s = NewTxn(wopts, &txn);
+  Status s = NewTxn(wopts, thread, &txn);
  if (!s.ok()) {
    assert(!txn);
    thread->stats.AddErrors(1);
@@ -611,7 +616,12 @@ Status MultiOpsTxnsStressTest::PrimaryKeyUpdateTxn(ThreadState* thread,
    }
    auto& key_gen = key_gen_for_a_[thread->tid];
    key_gen->UndoAllocation(new_a);
-    txn->Rollback().PermitUncheckedError();
+    s = txn->Rollback();
+    if (!s.ok()) {
+      fprintf(stderr, "Transaction rollback failed %s\n", s.ToString().c_str());
+      fflush(stderr);
+      assert(false);
+    }
  });

  ReadOptions ropts;
@@ -699,7 +709,7 @@ Status MultiOpsTxnsStressTest::SecondaryKeyUpdateTxn(ThreadState* thread,
                                                     uint32_t new_c) {
  std::unique_ptr<Transaction> txn;
  WriteOptions wopts;
-  Status s = NewTxn(wopts, &txn);
+  Status s = NewTxn(wopts, thread, &txn);
  if (!s.ok()) {
    assert(!txn);
    thread->stats.AddErrors(1);
@@ -735,7 +745,12 @@ Status MultiOpsTxnsStressTest::SecondaryKeyUpdateTxn(ThreadState* thread,
    }
    auto& key_gen = key_gen_for_c_[thread->tid];
    key_gen->UndoAllocation(new_c);
-    txn->Rollback().PermitUncheckedError();
+    s = txn->Rollback();
+    if (!s.ok()) {
+      fprintf(stderr, "Transaction rollback failed %s\n", s.ToString().c_str());
+      fflush(stderr);
+      assert(false);
+    }
  });

  // TODO (yanqin) try SetSnapshotOnNextOperation(). We currently need to take
@@ -901,7 +916,7 @@ Status MultiOpsTxnsStressTest::UpdatePrimaryIndexValueTxn(ThreadState* thread,
  std::string pk_str = Record::EncodePrimaryKey(a);
  std::unique_ptr<Transaction> txn;
  WriteOptions wopts;
-  Status s = NewTxn(wopts, &txn);
+  Status s = NewTxn(wopts, thread, &txn);
  if (!s.ok()) {
    assert(!txn);
    thread->stats.AddErrors(1);
@@ -927,7 +942,12 @@ Status MultiOpsTxnsStressTest::UpdatePrimaryIndexValueTxn(ThreadState* thread,
    } else {
      thread->stats.AddErrors(1);
    }
-    txn->Rollback().PermitUncheckedError();
+    s = txn->Rollback();
+    if (!s.ok()) {
+      fprintf(stderr, "Transaction rollback failed %s\n", s.ToString().c_str());
+      fflush(stderr);
+      assert(false);
+    }
  });
  ReadOptions ropts;
  ropts.rate_limiter_priority =
@@ -982,7 +1002,7 @@ Status MultiOpsTxnsStressTest::PointLookupTxn(ThreadState* thread,

  std::unique_ptr<Transaction> txn;
  WriteOptions wopts;
-  Status s = NewTxn(wopts, &txn);
+  Status s = NewTxn(wopts, thread, &txn);
  if (!s.ok()) {
    assert(!txn);
    thread->stats.AddErrors(1);
@@ -1026,7 +1046,7 @@ Status MultiOpsTxnsStressTest::RangeScanTxn(ThreadState* thread,

  std::unique_ptr<Transaction> txn;
  WriteOptions wopts;
-  Status s = NewTxn(wopts, &txn);
+  Status s = NewTxn(wopts, thread, &txn);
  if (!s.ok()) {
    assert(!txn);
    thread->stats.AddErrors(1);
@@ -1384,6 +1404,12 @@ Status MultiOpsTxnsStressTest::CommitAndCreateTimestampedSnapshotIfNeeded(
  } else {
    s = txn.Commit();
  }
+  if (!s.ok()) {
+    fprintf(stderr, "Txn %s commit failed with %s\n", txn.GetName().c_str(),
+            s.ToString().c_str());
+    fflush(stderr);
+  }
+
  assert(txn_db_);
  if (FLAGS_create_timestamped_snapshot_one_in > 0 &&
      thread->rand.OneInOpt(50000)) {
@@ -666,7 +666,7 @@ class NonBatchedOpsStressTest : public StressTest {
      if (FLAGS_rate_limit_auto_wal_flush) {
        wo.rate_limiter_priority = Env::IO_USER;
      }
-      Status s = NewTxn(wo, &txn);
+      Status s = NewTxn(wo, thread, &txn);
      if (!s.ok()) {
        fprintf(stderr, "NewTxn error: %s\n", s.ToString().c_str());
        shared->SafeTerminate();
@@ -1127,7 +1127,7 @@ class NonBatchedOpsStressTest : public StressTest {
        write_options.rate_limiter_priority = Env::IO_USER;
      }

-      const Status s = NewTxn(write_options, &txn);
+      const Status s = NewTxn(write_options, thread, &txn);
      if (!s.ok()) {
        fprintf(stderr, "NewTxn error: %s\n", s.ToString().c_str());
        thread->shared->SafeTerminate();
@@ -1515,6 +1515,13 @@ class NonBatchedOpsStressTest : public StressTest {
        }
      }

+      if (ro_copy.allow_unprepared_value) {
+        if (!iter->PrepareValue()) {
+          s = iter->status();
+          break;
+        }
+      }
+
      if (!VerifyWideColumns(iter->value(), iter->columns())) {
        s = Status::Corruption("Value and columns inconsistent",
                               DebugString(iter->value(), iter->columns()));
@@ -1645,6 +1652,7 @@ class NonBatchedOpsStressTest : public StressTest {
    // To track whether WAL write may have succeeded during the initial failed
    // write
    bool initial_wal_write_may_succeed = true;
+    bool commit_bypass_memtable = false;

    PendingExpectedValue pending_expected_value =
        shared->PreparePut(rand_column_family, rand_key);
@@ -1706,9 +1714,10 @@ class NonBatchedOpsStressTest : public StressTest {
            s = db_->Put(write_opts, cfh, k, write_ts, v);
          }
        } else {
-          s = ExecuteTransaction(write_opts, thread, [&](Transaction& txn) {
-            return txn.Put(cfh, k, v);
-          });
+          s = ExecuteTransaction(
+              write_opts, thread,
+              [&](Transaction& txn) { return txn.Put(cfh, k, v); },
+              &commit_bypass_memtable);
        }
      }
      UpdateIfInitialWriteFails(db_stress_env, s, &initial_write_s,
@@ -1772,6 +1781,7 @@ class NonBatchedOpsStressTest : public StressTest {
    // To track whether WAL write may have succeeded during the initial failed
    // write
    bool initial_wal_write_may_succeed = true;
+    bool commit_bypass_memtable = false;

    // Use delete if the key may be overwritten and a single deletion
    // otherwise.
@@ -1796,13 +1806,14 @@ class NonBatchedOpsStressTest : public StressTest {
            s = db_->Delete(write_opts, cfh, key, write_ts);
          }
        } else {
-          s = ExecuteTransaction(write_opts, thread, [&](Transaction& txn) {
-            return txn.Delete(cfh, key);
-          });
+          s = ExecuteTransaction(
+              write_opts, thread,
+              [&](Transaction& txn) { return txn.Delete(cfh, key); },
+              &commit_bypass_memtable);
        }
-        UpdateIfInitialWriteFails(db_stress_env, s, &initial_write_s,
-                                  &initial_wal_write_may_succeed,
-                                  &wait_for_recover_start_time);
+        UpdateIfInitialWriteFails(
+            db_stress_env, s, &initial_write_s, &initial_wal_write_may_succeed,
+            &wait_for_recover_start_time, commit_bypass_memtable);
      } while (!s.ok() && IsErrorInjectedAndRetryable(s) &&
               initial_wal_write_may_succeed);

@@ -1852,13 +1863,14 @@ class NonBatchedOpsStressTest : public StressTest {
            s = db_->SingleDelete(write_opts, cfh, key, write_ts);
          }
        } else {
-          s = ExecuteTransaction(write_opts, thread, [&](Transaction& txn) {
-            return txn.SingleDelete(cfh, key);
-          });
+          s = ExecuteTransaction(
+              write_opts, thread,
+              [&](Transaction& txn) { return txn.SingleDelete(cfh, key); },
+              &commit_bypass_memtable);
        }
-        UpdateIfInitialWriteFails(db_stress_env, s, &initial_write_s,
-                                  &initial_wal_write_may_succeed,
-                                  &wait_for_recover_start_time);
+        UpdateIfInitialWriteFails(
+            db_stress_env, s, &initial_write_s, &initial_wal_write_may_succeed,
+            &wait_for_recover_start_time, commit_bypass_memtable);
      } while (!s.ok() && IsErrorInjectedAndRetryable(s) &&
               initial_wal_write_may_succeed);

@@ -1994,9 +2006,8 @@ class NonBatchedOpsStressTest : public StressTest {
    // a continuous range of keys, the second one with a standalone range
    // deletion for all the keys. This is to exercise the standalone range
    // deletion file's compaction input optimization.
-    // TODO(yuzhangyu): make this an option.
-    bool test_standalone_range_deletion =
-        thread->rand.OneInOpt(10) && FLAGS_delrangepercent > 0;
+    bool test_standalone_range_deletion = thread->rand.OneInOpt(
+        FLAGS_test_ingest_standalone_range_deletion_one_in);
    std::vector<std::string> external_files;
    const std::string sst_filename =
        FLAGS_db + "/." + std::to_string(thread->tid) + ".sst";
@@ -2362,6 +2373,16 @@ class NonBatchedOpsStressTest : public StressTest {
    uint64_t curr = 0;
    while (true) {
      assert(last_key < ub);
+
+      if (iter->Valid() && ro.allow_unprepared_value) {
+        op_logs += "*";
+
+        if (!iter->PrepareValue()) {
+          assert(!iter->Valid());
+          assert(!iter->status().ok());
+        }
+      }
+
      if (!iter->Valid()) {
        if (!iter->status().ok()) {
          if (IsErrorInjectedAndRetryable(iter->status())) {
@@ -2424,6 +2445,16 @@ class NonBatchedOpsStressTest : public StressTest {
    last_key = ub;
    while (true) {
      assert(lb < last_key);
+
+      if (iter->Valid() && ro.allow_unprepared_value) {
+        op_logs += "*";
+
+        if (!iter->PrepareValue()) {
+          assert(!iter->Valid());
+          assert(!iter->status().ok());
+        }
+      }
+
      if (!iter->Valid()) {
        if (!iter->status().ok()) {
          if (IsErrorInjectedAndRetryable(iter->status())) {
@@ -2562,6 +2593,16 @@ class NonBatchedOpsStressTest : public StressTest {
    }

    for (int64_t i = 0; i < num_iter && iter->Valid(); ++i) {
+      if (ro.allow_unprepared_value) {
+        op_logs += "*";
+
+        if (!iter->PrepareValue()) {
+          assert(!iter->Valid());
+          assert(!iter->status().ok());
+          break;
+        }
+      }
+
      if (!check_columns()) {
        return Status::OK();
      }
@@ -2839,7 +2880,7 @@ class NonBatchedOpsStressTest : public StressTest {
          const size_t sz = GenerateValue(value_base, value, sizeof(value));
          const Slice v(value, sz);

-          if (op == Op::PutOrPutEntity) {
+          if (op == Op::PutOrPutEntity || !FLAGS_use_merge) {
            if (FLAGS_use_put_entity_one_in > 0 &&
                (value_base % FLAGS_use_put_entity_one_in) == 0) {
              s = txn->PutEntity(cfh, k, GenerateWideColumns(value_base, v));
@@ -2879,4 +2920,4 @@ StressTest* CreateNonBatchedOpsStressTest() {
 }

 }  // namespace ROCKSDB_NAMESPACE
-#endif  // GFLAGS
+#endif  // GFLAGS
@@ -200,6 +200,11 @@ IOStatus WriteStringToFile(FileSystem* fs, const Slice& data,

 IOStatus ReadFileToString(FileSystem* fs, const std::string& fname,
                          std::string* data) {
+  return ReadFileToString(fs, fname, IOOptions(), data);
+}
+
+IOStatus ReadFileToString(FileSystem* fs, const std::string& fname,
+                          const IOOptions& opts, std::string* data) {
  FileOptions soptions;
  data->clear();
  std::unique_ptr<FSSequentialFile> file;
@@ -212,7 +217,7 @@ IOStatus ReadFileToString(FileSystem* fs, const std::string& fname,
  char* space = new char[kBufferSize];
  while (true) {
    Slice fragment;
-    s = file->Read(kBufferSize, IOOptions(), &fragment, space, nullptr);
+    s = file->Read(kBufferSize, opts, &fragment, space, nullptr);
    if (!s.ok()) {
      break;
    }
@@ -51,6 +51,7 @@
 #include "env/io_posix.h"
 #include "monitoring/iostats_context_imp.h"
 #include "monitoring/thread_status_updater.h"
+#include "options/db_options.h"
 #include "port/lang.h"
 #include "port/port.h"
 #include "rocksdb/options.h"
@@ -930,6 +931,28 @@ class PosixFileSystem : public FileSystem {
    optimized.fallocate_with_keep_size = true;
    return optimized;
  }
+
+  FileOptions OptimizeForCompactionTableRead(
+      const FileOptions& file_options,
+      const ImmutableDBOptions& db_options) const override {
+    FileOptions fo = FileOptions(file_options);
+#ifdef OS_LINUX
+    // To fix https://github.com/facebook/rocksdb/issues/12038
+    if (!file_options.use_direct_reads &&
+        file_options.compaction_readahead_size > 0) {
+      size_t system_limit =
+          GetCompactionReadaheadSizeSystemLimit(db_options.db_paths);
+      if (system_limit > 0 &&
+          file_options.compaction_readahead_size > system_limit) {
+        fo.compaction_readahead_size = system_limit;
+      }
+    }
+#else
+    (void)db_options;
+#endif
+    return fo;
+  }
+
 #ifdef OS_LINUX
  Status RegisterDbPaths(const std::vector<std::string>& paths) override {
    return logical_block_size_cache_.RefAndCacheLogicalBlockSize(paths);
@@ -942,6 +965,36 @@ class PosixFileSystem : public FileSystem {
 private:
  bool forceMmapOff_ = false;  // do we override Env options?

+#ifdef OS_LINUX
+  // Get the minimum "linux system limit" (i.e, the largest I/O size that the OS
+  // can issue to block devices under a directory, also known as
+  // "max_sectors_kb" ) among `db_paths`.
+  // Return 0 if no limit can be found or there is an error in
+  // retrieving such limit.
+  static size_t GetCompactionReadaheadSizeSystemLimit(
+      const std::vector<DbPath>& db_paths) {
+    Status s;
+    size_t limit_kb = 0;
+
+    for (const auto& db_path : db_paths) {
+      size_t dir_max_sectors_kb = 0;
+      s = PosixHelper::GetMaxSectorsKBOfDirectory(db_path.path,
+                                                  &dir_max_sectors_kb);
+      if (!s.ok()) {
+        break;
+      }
+
+      limit_kb = (limit_kb == 0) ? dir_max_sectors_kb
+                                 : std::min(limit_kb, dir_max_sectors_kb);
+    }
+
+    if (s.ok()) {
+      return limit_kb * 1024;
+    } else {
+      return 0;
+    }
+  }
+#endif
  // Returns true iff the named directory exists and is a directory.
  virtual bool DirExists(const std::string& dname) {
    struct stat statbuf;