Fix "no new line at end of file" (#12806 )

Summary: internal CI with -Wnewline-eof complains about this. Pull Request resolved: https://github.com/facebook/rocksdb/pull/12806 Reviewed By: jowlyzhang Differential Revision: D58969551 Pulled By: cbi42 fbshipit-source-id: f97bd23f82129bac8dfd97b0ff4dbf7d2ded95d4
update HISTORY.md for 9.4.fb
2026-11-07 03:22:34 -05:00 · 2024-06-24 16:48:34 -07:00 · 2024-06-23 22:20:59 -07:00 · 2024-06-23 21:54:27 -07:00 · 2024-06-22 16:17:59 -07:00 · 2024-06-22 15:15:02 -07:00
220 changed files with 7938 additions and 2563 deletions
@@ -90,6 +90,9 @@ jobs:
    container:
      image: zjay437/rocksdb:0.6
      options: --shm-size=16gb
+    env:
+      CC: gcc-10
+      CXX: g++-10
    steps:
    - uses: actions/checkout@v4.1.0
    - uses: "./.github/actions/pre-steps"
@@ -104,38 +107,47 @@ jobs:
    container:
      image: zjay437/rocksdb:0.6
      options: --shm-size=16gb
+    env:
+      CC: gcc-10
+      CXX: g++-10
    steps:
    - uses: actions/checkout@v4.1.0
    - uses: "./.github/actions/pre-steps"
    - uses: "./.github/actions/setup-folly"
    - run: "(mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 .. && make V=1 -j20)"
    - uses: "./.github/actions/post-steps"
-  build-linux-gcc-7-with-folly:
+  build-linux-make-with-folly:
    if: ${{ github.repository_owner == 'facebook' }}
    runs-on:
      labels: 16-core-ubuntu
    container:
      image: zjay437/rocksdb:0.6
      options: --shm-size=16gb
+    env:
+      CC: gcc-10
+      CXX: g++-10
    steps:
    - uses: actions/checkout@v4.1.0
    - uses: "./.github/actions/pre-steps"
    - uses: "./.github/actions/setup-folly"
    - uses: "./.github/actions/build-folly"
-    - run: USE_FOLLY=1 LIB_MODE=static CC=gcc-7 CXX=g++-7 V=1 make -j32 check
+    - run: USE_FOLLY=1 LIB_MODE=static V=1 make -j32 check
    - uses: "./.github/actions/post-steps"
-  build-linux-gcc-7-with-folly-lite-no-test:
+  build-linux-make-with-folly-lite-no-test:
    if: ${{ github.repository_owner == 'facebook' }}
    runs-on:
      labels: 16-core-ubuntu
    container:
      image: zjay437/rocksdb:0.6
      options: --shm-size=16gb
+    env:
+      CC: gcc-10
+      CXX: g++-10
    steps:
    - uses: actions/checkout@v4.1.0
    - uses: "./.github/actions/pre-steps"
    - uses: "./.github/actions/setup-folly"
-    - run: USE_FOLLY_LITE=1 CC=gcc-7 CXX=g++-7 V=1 make -j32 all
+    - run: USE_FOLLY_LITE=1 V=1 make -j32 all
    - uses: "./.github/actions/post-steps"
  build-linux-cmake-with-folly-coroutines:
    if: ${{ github.repository_owner == 'facebook' }}
@@ -47,6 +47,9 @@ package/
 unity.a
 tags
 etags
+GPATH
+GRTAGS
+GTAGS
 rocksdb_dump
 rocksdb_undump
 db_test2
@@ -267,8 +267,8 @@ endif(CMAKE_SYSTEM_PROCESSOR MATCHES "s390x")
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "loongarch64")
  CHECK_C_COMPILER_FLAG("-march=loongarch64" HAS_LOONGARCH64)
  if(HAS_LOONGARCH64)
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mcpu=loongarch64 -mtune=loongarch64")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=loongarch64 -mtune=loongarch64")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=loongarch64 -mtune=loongarch64")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=loongarch64 -mtune=loongarch64")
  endif(HAS_LOONGARCH64)
 endif(CMAKE_SYSTEM_PROCESSOR MATCHES "loongarch64")

@@ -488,6 +488,8 @@ if(CMAKE_SYSTEM_NAME MATCHES "Cygwin")
  add_definitions(-fno-builtin-memcmp -DCYGWIN)
 elseif(CMAKE_SYSTEM_NAME MATCHES "Darwin")
  add_definitions(-DOS_MACOSX)
+elseif(CMAKE_SYSTEM_NAME MATCHES "iOS")
+  add_definitions(-DOS_MACOSX -DIOS_CROSS_COMPILE)
 elseif(CMAKE_SYSTEM_NAME MATCHES "Linux")
  add_definitions(-DOS_LINUX)
 elseif(CMAKE_SYSTEM_NAME MATCHES "SunOS")
@@ -615,7 +617,7 @@ if(USE_FOLLY)
    FMT_INST_PATH)
    exec_program(ls ARGS -d ${FOLLY_INST_PATH}/../gflags* OUTPUT_VARIABLE
    GFLAGS_INST_PATH)
-    set(Boost_DIR ${BOOST_INST_PATH}/lib/cmake/Boost-1.78.0)
+    set(Boost_DIR ${BOOST_INST_PATH}/lib/cmake/Boost-1.83.0)
    if(EXISTS ${FMT_INST_PATH}/lib64)
      set(fmt_DIR ${FMT_INST_PATH}/lib64/cmake/fmt)
    else()
@@ -691,6 +693,7 @@ set(SOURCES
        db/db_impl/db_impl_write.cc
        db/db_impl/db_impl_compaction_flush.cc
        db/db_impl/db_impl_files.cc
+        db/db_impl/db_impl_follower.cc
        db/db_impl/db_impl_open.cc
        db/db_impl/db_impl_debug.cc
        db/db_impl/db_impl_experimental.cc
@@ -748,6 +751,7 @@ set(SOURCES
        env/env_encryption.cc
        env/file_system.cc
        env/file_system_tracer.cc
+        env/fs_on_demand.cc
        env/fs_remap.cc
        env/mock_env.cc
        env/unique_id_gen.cc
@@ -935,6 +939,7 @@ set(SOURCES
        utilities/persistent_cache/volatile_tier_impl.cc
        utilities/simulator_cache/cache_simulator.cc
        utilities/simulator_cache/sim_cache.cc
+        utilities/table_properties_collectors/compact_for_tiering_collector.cc
        utilities/table_properties_collectors/compact_on_deletion_collector.cc
        utilities/trace/file_trace_reader_writer.cc
        utilities/trace/replayer_impl.cc
@@ -1037,10 +1042,8 @@ endif()

 else()
  list(APPEND SOURCES
-    db/db_impl/db_impl_follower.cc
    port/port_posix.cc
    env/env_posix.cc
-    env/fs_on_demand.cc
    env/fs_posix.cc
    env/io_posix.cc)
 endif()
@@ -1049,6 +1052,7 @@ if(USE_FOLLY_LITE)
  list(APPEND SOURCES
    third-party/folly/folly/container/detail/F14Table.cpp
    third-party/folly/folly/detail/Futex.cpp
+    third-party/folly/folly/lang/Exception.cpp
    third-party/folly/folly/lang/SafeAssert.cpp
    third-party/folly/folly/lang/ToAscii.cpp
    third-party/folly/folly/ScopeGuard.cpp
@@ -1056,6 +1060,12 @@ if(USE_FOLLY_LITE)
    third-party/folly/folly/synchronization/DistributedMutex.cpp
    third-party/folly/folly/synchronization/ParkingLot.cpp)
  include_directories(${PROJECT_SOURCE_DIR}/third-party/folly)
+  exec_program(python3 ${PROJECT_SOURCE_DIR}/third-party/folly ARGS
+  build/fbcode_builder/getdeps.py show-source-dir boost OUTPUT_VARIABLE
+  BOOST_SOURCE_PATH)
+  exec_program(ls ARGS -d ${BOOST_SOURCE_PATH}/boost* OUTPUT_VARIABLE
+  BOOST_INCLUDE_DIR)
+  include_directories(${BOOST_INCLUDE_DIR})
  add_definitions(-DUSE_FOLLY -DFOLLY_NO_CONFIG)
  list(APPEND THIRDPARTY_LIBS glog)
 endif()
@@ -1479,6 +1489,7 @@ if(WITH_TESTS)
        utilities/persistent_cache/persistent_cache_test.cc
        utilities/simulator_cache/cache_simulator_test.cc
        utilities/simulator_cache/sim_cache_test.cc
+        utilities/table_properties_collectors/compact_for_tiering_collector_test.cc
        utilities/table_properties_collectors/compact_on_deletion_collector_test.cc
        utilities/transactions/optimistic_transaction_test.cc
        utilities/transactions/transaction_test.cc
@@ -1,6 +1,56 @@
 # Rocksdb Change Log
 > NOTE: Entries for next release do not go here. Follow instructions in `unreleased_history/README.txt`

+## 9.4.0 (06/23/2024)
+### New Features
+* Added a `CompactForTieringCollectorFactory` to auto trigger compaction for tiering use case.
+* Optimistic transactions and pessimistic transactions with the WriteCommitted policy now support the `GetEntityForUpdate` API.
+* Added a new "count" command to the ldb repl shell. By default, it prints a count of keys in the database from start to end. The options --from=<key> and/or --to=<key> can be specified to limit the range.
+* Add `rocksdb_writebatch_update_timestamps`, `rocksdb_writebatch_wi_update_timestamps` in C API.
+* Add `rocksdb_iter_refresh` in C API.
+* Add `rocksdb_writebatch_create_with_params`, `rocksdb_writebatch_wi_create_with_params` to create WB and WBWI with all options in C API
+
+### Public API Changes
+* Deprecated names `LogFile` and `VectorLogPtr` in favor of new names `WalFile` and `VectorWalPtr`.
+* Introduce a new universal compaction option CompactionOptionsUniversal::max_read_amp which allows user to define the limit on the number of sorted runs separately from the trigger for compaction (`level0_file_num_compaction_trigger`) #12477.
+
+### Behavior Changes
+* Inactive WALs are immediately closed upon being fully sync-ed rather than in a background thread. This is to ensure LinkFile() is not called on files still open for write, which might not be supported by some FileSystem implementations. This should not be a performance issue, but an opt-out is available with with new DB option `background_close_inactive_wals`.
+
+### Bug Fixes
+* Fix a rare case in which a hard-linked WAL in a Checkpoint is not fully synced (so might lose data on power loss).
+* Fixed the output of the `ldb dump_wal` command for `PutEntity` records so it prints the key and correctly resets the hexadecimal formatting flag after printing the wide-column entity.
+* Fixed an issue where `PutEntity` records were handled incorrectly while rebuilding transactions during recovery.
+* Various read operations could ignore various ReadOptions that might be relevant. Fixed many such cases, which can result in behavior change but a better reflection of specified options.
+
+### Performance Improvements
+* Improved write throughput to memtable when there's a large number of concurrent writers and allow_concurrent_memtable_write=true(#12545)
+
+## 9.3.0 (05/17/2024)
+### New Features
+* Optimistic transactions and pessimistic transactions with the WriteCommitted policy now support the `GetEntity` API.
+* Added new `Iterator` property, "rocksdb.iterator.is-value-pinned", for checking whether the `Slice` returned by `Iterator::value()` can be used until the `Iterator` is destroyed.
+* Optimistic transactions and WriteCommitted pessimistic transactions now support the `MultiGetEntity` API.
+* Optimistic transactions and pessimistic transactions with the WriteCommitted policy now support the `PutEntity` API. Support for read APIs and other write policies (WritePrepared, WriteUnprepared) will be added later.
+
+### Public API Changes
+* Exposed block based metadata cache options via C API
+* Exposed compaction pri via c api.
+* Add a kAdmPolicyAllowAll option to TieredAdmissionPolicy that admits all blocks evicted from the primary block cache into the compressed secondary cache.
+
+### Behavior Changes
+* CompactRange() with change_level=true on a CF with FIFO compaction will return Status::NotSupported().
+* External file ingestion with FIFO compaction will always ingest to L0.
+
+### Bug Fixes
+* Fixed a bug for databases using `DBOptions::allow_2pc == true` (all `TransactionDB`s except `OptimisticTransactionDB`) that have exactly one column family. Due to a missing WAL sync, attempting to open the DB could have returned a `Status::Corruption` with a message like "SST file is ahead of WALs".
+* Fix a bug in CreateColumnFamilyWithImport() where if multiple CFs are imported, we were not resetting files' epoch number and L0 files can have overlapping key range but the same epoch number.
+* Fixed race conditions when `ColumnFamilyOptions::inplace_update_support == true` between user overwrites and reads on the same key.
+* Fix a bug where `CompactFiles()` can compact files of range conflict with other ongoing compactions' when `preclude_last_level_data_seconds > 0` is used
+* Fixed a false positive `Status::Corruption` reported when reopening a DB that used `DBOptions::recycle_log_file_num > 0` and `DBOptions::wal_compression != kNoCompression`.
+* While WAL is locked with LockWAL(), some operations like Flush() and IngestExternalFile() are now blocked as they should have been.
+* Fixed a bug causing stale memory access when using the TieredSecondaryCache with an NVM secondary cache, and a file system that supports return an FS allocated buffer for MultiRead (FSSupportedOps::kFSBuffer is set).
+
 ## 9.2.0 (05/01/2024)
 ### New Features
 * Added two options `deadline` and `max_size_bytes` for CacheDumper to exit early
@@ -103,6 +103,7 @@ dummy := $(shell (export ROCKSDB_ROOT="$(CURDIR)"; \
                  export LIB_MODE="$(LIB_MODE)"; \
 		  export ROCKSDB_CXX_STANDARD="$(ROCKSDB_CXX_STANDARD)"; \
 		  export USE_FOLLY="$(USE_FOLLY)"; \
+		  export USE_FOLLY_LITE="$(USE_FOLLY_LITE)"; \
                  "$(CURDIR)/build_tools/build_detect_platform" "$(CURDIR)/make_config.mk"))
 # this file is generated by the previous line to set build flags and sources
 include make_config.mk
@@ -500,6 +501,17 @@ endif
 ifeq ($(USE_FOLLY_LITE),1)
 	# Path to the Folly source code and include files
 	FOLLY_DIR = ./third-party/folly
+ifneq ($(strip $(BOOST_SOURCE_PATH)),)
+	BOOST_INCLUDE = $(shell (ls -d $(BOOST_SOURCE_PATH)/boost*/))
+	# AIX: pre-defined system headers are surrounded by an extern "C" block
+	ifeq ($(PLATFORM), OS_AIX)
+		PLATFORM_CCFLAGS += -I$(BOOST_INCLUDE)
+		PLATFORM_CXXFLAGS += -I$(BOOST_INCLUDE)
+	else
+		PLATFORM_CCFLAGS += -isystem $(BOOST_INCLUDE)
+		PLATFORM_CXXFLAGS += -isystem $(BOOST_INCLUDE)
+	endif
+endif  # BOOST_SOURCE_PATH
 	# AIX: pre-defined system headers are surrounded by an extern "C" block
 	ifeq ($(PLATFORM), OS_AIX)
 		PLATFORM_CCFLAGS += -I$(FOLLY_DIR)
@@ -636,26 +648,37 @@ TESTS = $(patsubst %.cc, %, $(notdir $(TEST_MAIN_SOURCES)))
 TESTS += $(patsubst %.c, %, $(notdir $(TEST_MAIN_SOURCES_C)))
 TESTS += $(PLUGIN_TESTS)

-# `make check-headers` to very that each header file includes its own
-# dependencies
+# `make check-headers` to verify that each header file includes its own deps
+# and that public headers do not depend on internal headers
 ifneq ($(filter check-headers, $(MAKECMDGOALS)),)
 # TODO: add/support JNI headers
 	DEV_HEADER_DIRS := $(sort include/ $(dir $(ALL_SOURCES)))
 # Some headers like in port/ are platform-specific
-	DEV_HEADERS := $(shell $(FIND) $(DEV_HEADER_DIRS) -type f -name '*.h' | grep -E -v 'port/|plugin/|lua/|range_tree/')
+	DEV_HEADERS_TO_CHECK := $(shell $(FIND) $(DEV_HEADER_DIRS) -type f -name '*.h' | grep -E -v 'port/|plugin/|lua/|range_tree/')
+	PUBLIC_HEADERS_TO_CHECK := $(shell $(FIND) include/ -type f -name '*.h' | grep -E -v 'lua/')
 else
-	DEV_HEADERS :=
+	DEV_HEADERS_TO_CHECK :=
+	PUBLIC_HEADERS_TO_CHECK :=
 endif
-HEADER_OK_FILES = $(patsubst %.h, %.h.ok, $(DEV_HEADERS))
+HEADER_OK_FILES = $(patsubst %.h, %.h.ok, $(DEV_HEADERS_TO_CHECK)) \
+	$(patsubst %.h, %.h.pub, $(PUBLIC_HEADERS_TO_CHECK))

 AM_V_CCH = $(am__v_CCH_$(V))
 am__v_CCH_ = $(am__v_CCH_$(AM_DEFAULT_VERBOSITY))
 am__v_CCH_0 = @echo "  CC.h    " $<;
 am__v_CCH_1 =

+# verify headers include their own dependencies, under dev build settings
 %.h.ok: %.h # .h.ok not actually created, so re-checked on each invocation
 # -DROCKSDB_NAMESPACE=42 ensures the namespace header is included
-	$(AM_V_CCH) echo '#include "$<"' | $(CXX) $(CXXFLAGS) -DROCKSDB_NAMESPACE=42 -x c++ -c - -o /dev/null
+	$(AM_V_CCH) echo '#include "$<"' | $(CXX) $(CXXFLAGS) \
+	  -DROCKSDB_NAMESPACE=42 -x c++ -c - -o /dev/null
+
+# verify public headers do not depend on internal headers, under typical
+# user build settings
+%.h.pub: %.h # .h.pub not actually created, so re-checked on each invocation
+	$(AM_V_CCH) cd include/ && echo '#include "$(patsubst include/%,%,$<)"' | \
+	  $(CXX) -I. -DROCKSDB_NAMESPACE=42 -x c++ -c - -o /dev/null

 check-headers: $(HEADER_OK_FILES)

@@ -1631,6 +1654,9 @@ compaction_job_stats_test: $(OBJ_DIR)/db/compaction/compaction_job_stats_test.o
 compaction_service_test: $(OBJ_DIR)/db/compaction/compaction_service_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)

+compact_for_tiering_collector_test: $(OBJ_DIR)/utilities/table_properties_collectors/compact_for_tiering_collector_test.o $(TEST_LIBRARY) $(LIBRARY)
+	$(AM_LINK)
+
 compact_on_deletion_collector_test: $(OBJ_DIR)/utilities/table_properties_collectors/compact_on_deletion_collector_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)

@@ -2458,14 +2484,15 @@ checkout_folly:
 	fi
 	@# Pin to a particular version for public CI, so that PR authors don't
 	@# need to worry about folly breaking our integration. Update periodically
-	cd third-party/folly && git reset --hard beacd86d63cd71c904632262e6c36f60874d78ba
-	@# A hack to remove boost dependency.
-	@# NOTE: this hack is only needed if building using USE_FOLLY_LITE
-	perl -pi -e 's/^(#include <boost)/\/\/$$1/' third-party/folly/folly/functional/Invoke.h
+	cd third-party/folly && git reset --hard 17054c85c2dee0e84cbee7ef28d7bae7d0b1630e
 	@# NOTE: this hack is required for clang in some cases
 	perl -pi -e 's/int rv = syscall/int rv = (int)syscall/' third-party/folly/folly/detail/Futex.cpp
 	@# NOTE: this hack is required for gcc in some cases
 	perl -pi -e 's/(__has_include.<experimental.memory_resource>.)/__cpp_rtti && $$1/' third-party/folly/folly/memory/MemoryResource.h
+	@# NOTE: this hack is only needed if building target build_folly in the future
+	perl -pi -e 's,https://github.com/jedisct1/libsodium/releases/download/1.0.17/libsodium-1.0.17.tar.gz,https://download.libsodium.org/libsodium/releases/old/unsupported/libsodium-1.0.17.tar.gz,' third-party/folly/build/fbcode_builder/manifests/libsodium
+	@# NOTE: boost source will be needed for any build including `USE_FOLLY_LITE` builds as those depend on boost headers
+	cd third-party/folly && $(PYTHON) build/fbcode_builder/getdeps.py fetch boost

 CXX_M_FLAGS = $(filter -m%, $(CXXFLAGS))

@@ -2477,8 +2504,6 @@ build_folly:
 		echo "Please run checkout_folly first"; \
 		false; \
 	fi
-	# Restore the original version of Invoke.h with boost dependency
-	cd third-party/folly && ${GIT_COMMAND} checkout folly/functional/Invoke.h
 	cd third-party/folly && \
 		CXXFLAGS=" $(CXX_M_FLAGS) -DHAVE_CXX11_ATOMIC " $(PYTHON) build/fbcode_builder/getdeps.py build --no-tests

@@ -4,6 +4,7 @@
 # This file is a Facebook-specific integration for buck builds, so can
 # only be validated by Facebook employees.
 load("//rocks/buckifier:defs.bzl", "cpp_library_wrapper","rocks_cpp_library_wrapper","cpp_binary_wrapper","cpp_unittest_wrapper","fancy_bench_wrapper","add_c_test_wrapper")
+load("@fbcode_macros//build_defs:export_files.bzl", "export_file")


 cpp_library_wrapper(name="rocksdb_lib", srcs=[
@@ -316,6 +317,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
        "utilities/persistent_cache/volatile_tier_impl.cc",
        "utilities/simulator_cache/cache_simulator.cc",
        "utilities/simulator_cache/sim_cache.cc",
+        "utilities/table_properties_collectors/compact_for_tiering_collector.cc",
        "utilities/table_properties_collectors/compact_on_deletion_collector.cc",
        "utilities/trace/file_trace_reader_writer.cc",
        "utilities/trace/replayer_impl.cc",
@@ -393,6 +395,7 @@ rocks_cpp_library_wrapper(name="rocksdb_stress_lib", srcs=[
        "db_stress_tool/cf_consistency_stress.cc",
        "db_stress_tool/db_stress_common.cc",
        "db_stress_tool/db_stress_driver.cc",
+        "db_stress_tool/db_stress_filters.cc",
        "db_stress_tool/db_stress_gflags.cc",
        "db_stress_tool/db_stress_listener.cc",
        "db_stress_tool/db_stress_shared_state.cc",
@@ -410,6 +413,8 @@ rocks_cpp_library_wrapper(name="rocksdb_stress_lib", srcs=[
    ], headers=None)


+cpp_binary_wrapper(name="ldb", srcs=["tools/ldb.cc"], deps=[":rocksdb_tools_lib"], extra_preprocessor_flags=[], extra_bench_libs=False)
+
 cpp_binary_wrapper(name="db_stress", srcs=["db_stress_tool/db_stress.cc"], deps=[":rocksdb_stress_lib"], extra_preprocessor_flags=[], extra_bench_libs=False)

 cpp_binary_wrapper(name="cache_bench", srcs=["cache/cache_bench.cc"], deps=[":rocksdb_cache_bench_tools_lib"], extra_preprocessor_flags=[], extra_bench_libs=False)
@@ -4623,6 +4628,12 @@ cpp_unittest_wrapper(name="compact_files_test",
            extra_compiler_flags=[])


+cpp_unittest_wrapper(name="compact_for_tiering_collector_test",
+            srcs=["utilities/table_properties_collectors/compact_for_tiering_collector_test.cc"],
+            deps=[":rocksdb_test_lib"],
+            extra_compiler_flags=[])
+
+
 cpp_unittest_wrapper(name="compact_on_deletion_collector_test",
            srcs=["utilities/table_properties_collectors/compact_on_deletion_collector_test.cc"],
            deps=[":rocksdb_test_lib"],
@@ -193,6 +193,10 @@ def generate_targets(repo_path, deps_map):
        + src_mk.get("STRESS_LIB_SOURCES", [])
        + ["test_util/testutil.cc"],
    )
+    # ldb binary
+    TARGETS.add_binary(
+        "ldb", ["tools/ldb.cc"], [":rocksdb_tools_lib"]
+    )
    # db_stress binary
    TARGETS.add_binary(
        "db_stress", ["db_stress_tool/db_stress.cc"], [":rocksdb_stress_lib"]
@@ -7,6 +7,7 @@ rocksdb_target_header_template = """# This file \100generated by:
 # This file is a Facebook-specific integration for buck builds, so can
 # only be validated by Facebook employees.
 load("//rocks/buckifier:defs.bzl", "cpp_library_wrapper","rocks_cpp_library_wrapper","cpp_binary_wrapper","cpp_unittest_wrapper","fancy_bench_wrapper","add_c_test_wrapper")
+load("@fbcode_macros//build_defs:export_files.bzl", "export_file")

 """

@@ -585,7 +585,7 @@ EOF
            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lbenchmark"
        fi
    fi
-    if test $USE_FOLLY; then
+    if test $USE_FOLLY || test $USE_FOLLY_LITE; then
        # Test whether libfolly library is installed
        $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
          #include <folly/synchronization/DistributedMutex.h>
@@ -751,6 +751,11 @@ if [ "$USE_FOLLY" ]; then
    FOLLY_PATH=`cd $FOLLY_DIR && $PYTHON build/fbcode_builder/getdeps.py show-inst-dir folly`
  fi
 fi
+if [ "$USE_FOLLY_LITE" ]; then
+  if [ "$FOLLY_DIR" ]; then
+    BOOST_SOURCE_PATH=`cd $FOLLY_DIR && $PYTHON build/fbcode_builder/getdeps.py show-source-dir boost`
+  fi
+fi

 PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS"
 PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS"
@@ -792,6 +797,7 @@ echo "PROFILING_FLAGS=$PROFILING_FLAGS" >> "$OUTPUT"
 echo "FIND=$FIND" >> "$OUTPUT"
 echo "WATCH=$WATCH" >> "$OUTPUT"
 echo "FOLLY_PATH=$FOLLY_PATH" >> "$OUTPUT"
+echo "BOOST_SOURCE_PATH=$BOOST_SOURCE_PATH" >> "$OUTPUT"

 # This will enable some related identifiers for the preprocessor
 if test -n "$JEMALLOC"; then
@@ -18,6 +18,7 @@
 #include "cache/lru_cache.h"
 #include "cache/typed_cache.h"
 #include "port/stack_trace.h"
+#include "table/block_based/block_cache.h"
 #include "test_util/secondary_cache_test_util.h"
 #include "test_util/testharness.h"
 #include "util/coding.h"
@@ -1017,6 +1018,63 @@ INSTANTIATE_TEST_CASE_P(CacheTestInstance, CacheTest,
 INSTANTIATE_TEST_CASE_P(CacheTestInstance, LRUCacheTest,
                        testing::Values(secondary_cache_test_util::kLRU));

+TEST(MiscBlockCacheTest, UncacheAggressivenessAdvisor) {
+  // Aggressiveness to a sequence of Report() calls (as string of 0s and 1s)
+  // exactly until the first ShouldContinue() == false.
+  const std::vector<std::pair<uint32_t, Slice>> expectedTraces{
+      // Aggressiveness 1 aborts on first unsuccessful erasure.
+      {1, "0"},
+      {1, "11111111111111111111110"},
+      // For sufficient evidence, aggressiveness 2 requires a minimum of two
+      // unsuccessful erasures.
+      {2, "00"},
+      {2, "0110"},
+      {2, "1100"},
+      {2, "011111111111111111111111111111111111111111111111111111111111111100"},
+      {2, "0111111111111111111111111111111111110"},
+      // For sufficient evidence, aggressiveness 3 and higher require a minimum
+      // of three unsuccessful erasures.
+      {3, "000"},
+      {3, "01010"},
+      {3, "111000"},
+      {3, "00111111111111111111111111111111111100"},
+      {3, "00111111111111111111110"},
+
+      {4, "000"},
+      {4, "01010"},
+      {4, "111000"},
+      {4, "001111111111111111111100"},
+      {4, "0011111111111110"},
+
+      {6, "000"},
+      {6, "01010"},
+      {6, "111000"},
+      {6, "00111111111111100"},
+      {6, "0011111110"},
+
+      // 69 -> 50% threshold, now up to minimum of 4
+      {69, "0000"},
+      {69, "010000"},
+      {69, "01010000"},
+      {69, "101010100010101000"},
+
+      // 230 -> 10% threshold, appropriately higher minimum
+      {230, "000000000000"},
+      {230, "0000000000010000000000"},
+      {230, "00000000000100000000010000000000"}};
+  for (const auto& [aggressiveness, t] : expectedTraces) {
+    SCOPED_TRACE("aggressiveness=" + std::to_string(aggressiveness) + " with " +
+                 t.ToString());
+    UncacheAggressivenessAdvisor uaa(aggressiveness);
+    for (size_t i = 0; i < t.size(); ++i) {
+      SCOPED_TRACE("i=" + std::to_string(i));
+      ASSERT_TRUE(uaa.ShouldContinue());
+      uaa.Report(t[i] & 1);
+    }
+    ASSERT_FALSE(uaa.ShouldContinue());
+  }
+}
+
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -873,6 +873,111 @@ TEST_P(DBTieredAdmPolicyTest, CompressedCacheAdmission) {
  Destroy(options);
 }

+TEST_F(DBTieredSecondaryCacheTest, FSBufferTest) {
+  class WrapFS : public FileSystemWrapper {
+   public:
+    explicit WrapFS(const std::shared_ptr<FileSystem>& _target)
+        : FileSystemWrapper(_target) {}
+    ~WrapFS() override {}
+    const char* Name() const override { return "WrapFS"; }
+
+    IOStatus NewRandomAccessFile(const std::string& fname,
+                                 const FileOptions& opts,
+                                 std::unique_ptr<FSRandomAccessFile>* result,
+                                 IODebugContext* dbg) override {
+      class WrappedRandomAccessFile : public FSRandomAccessFileOwnerWrapper {
+       public:
+        explicit WrappedRandomAccessFile(
+            std::unique_ptr<FSRandomAccessFile>& file)
+            : FSRandomAccessFileOwnerWrapper(std::move(file)) {}
+
+        IOStatus MultiRead(FSReadRequest* reqs, size_t num_reqs,
+                           const IOOptions& options,
+                           IODebugContext* dbg) override {
+          for (size_t i = 0; i < num_reqs; ++i) {
+            FSReadRequest& req = reqs[i];
+            FSAllocationPtr buffer(new char[req.len], [](void* ptr) {
+              delete[] static_cast<char*>(ptr);
+            });
+            req.fs_scratch = std::move(buffer);
+            req.status = Read(req.offset, req.len, options, &req.result,
+                              static_cast<char*>(req.fs_scratch.get()), dbg);
+          }
+          return IOStatus::OK();
+        }
+      };
+
+      std::unique_ptr<FSRandomAccessFile> file;
+      IOStatus s = target()->NewRandomAccessFile(fname, opts, &file, dbg);
+      EXPECT_OK(s);
+      result->reset(new WrappedRandomAccessFile(file));
+
+      return s;
+    }
+
+    void SupportedOps(int64_t& supported_ops) override {
+      supported_ops = 1 << FSSupportedOps::kAsyncIO;
+      supported_ops |= 1 << FSSupportedOps::kFSBuffer;
+    }
+  };
+
+  if (!LZ4_Supported()) {
+    ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
+    return;
+  }
+
+  std::shared_ptr<WrapFS> wrap_fs =
+      std::make_shared<WrapFS>(env_->GetFileSystem());
+  std::unique_ptr<Env> wrap_env(new CompositeEnvWrapper(env_, wrap_fs));
+  BlockBasedTableOptions table_options;
+  table_options.block_cache = NewCache(250 * 1024, 20 * 1024, 256 * 1024,
+                                       TieredAdmissionPolicy::kAdmPolicyAuto,
+                                       /*ready_before_wait=*/true);
+  table_options.block_size = 4 * 1024;
+  table_options.cache_index_and_filter_blocks = false;
+  Options options = GetDefaultOptions();
+  options.create_if_missing = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  options.statistics = CreateDBStatistics();
+  options.env = wrap_env.get();
+
+  options.paranoid_file_checks = false;
+  DestroyAndReopen(options);
+  Random rnd(301);
+  const int N = 256;
+  for (int i = 0; i < N; i++) {
+    std::string p_v;
+    test::CompressibleString(&rnd, 0.5, 1007, &p_v);
+    ASSERT_OK(Put(Key(i), p_v));
+  }
+
+  ASSERT_OK(Flush());
+
+  std::vector<std::string> keys;
+  std::vector<std::string> values;
+
+  keys.push_back(Key(0));
+  keys.push_back(Key(4));
+  keys.push_back(Key(8));
+  values = MultiGet(keys, /*snapshot=*/nullptr, /*async=*/true);
+  ASSERT_EQ(values.size(), keys.size());
+  for (const auto& value : values) {
+    ASSERT_EQ(1007, value.size());
+  }
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 3u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 3u);
+  ASSERT_EQ(nvm_sec_cache()->num_hits(), 0u);
+
+  std::string v = Get(Key(12));
+  ASSERT_EQ(1007, v.size());
+  ASSERT_EQ(nvm_sec_cache()->num_insert_saved(), 4u);
+  ASSERT_EQ(nvm_sec_cache()->num_misses(), 4u);
+  ASSERT_EQ(options.statistics->getTickerCount(BLOCK_CACHE_MISS), 4u);
+
+  Close();
+  Destroy(options);
+}
+
 INSTANTIATE_TEST_CASE_P(
    DBTieredAdmPolicyTest, DBTieredAdmPolicyTest,
    ::testing::Values(TieredAdmissionPolicy::kAdmPolicyAuto,
@@ -64,7 +64,7 @@ Status BuildTable(
    std::vector<std::unique_ptr<FragmentedRangeTombstoneIterator>>
        range_del_iters,
    FileMetaData* meta, std::vector<BlobFileAddition>* blob_file_additions,
-    std::vector<SequenceNumber> snapshots,
+    std::vector<SequenceNumber> snapshots, SequenceNumber earliest_snapshot,
    SequenceNumber earliest_write_conflict_snapshot,
    SequenceNumber job_snapshot, SnapshotChecker* snapshot_checker,
    bool paranoid_file_checks, InternalStats* internal_stats,
@@ -195,7 +195,7 @@ Status BuildTable(

    const std::atomic<bool> kManualCompactionCanceledFalse{false};
    CompactionIterator c_iter(
-        iter, ucmp, &merge, kMaxSequenceNumber, &snapshots,
+        iter, ucmp, &merge, kMaxSequenceNumber, &snapshots, earliest_snapshot,
        earliest_write_conflict_snapshot, job_snapshot, snapshot_checker, env,
        ShouldReportDetailedTime(env, ioptions.stats),
        true /* internal key corruption is not ok */, range_del_agg.get(),
@@ -57,7 +57,7 @@ Status BuildTable(
    std::vector<std::unique_ptr<FragmentedRangeTombstoneIterator>>
        range_del_iters,
    FileMetaData* meta, std::vector<BlobFileAddition>* blob_file_additions,
-    std::vector<SequenceNumber> snapshots,
+    std::vector<SequenceNumber> snapshots, SequenceNumber earliest_snapshot,
    SequenceNumber earliest_write_conflict_snapshot,
    SequenceNumber job_snapshot, SnapshotChecker* snapshot_checker,
    bool paranoid_file_checks, InternalStats* internal_stats,
@@ -231,6 +231,7 @@ struct rocksdb_livefiles_t {
 };
 struct rocksdb_column_family_handle_t {
  ColumnFamilyHandle* rep;
+  bool immortal;  /* only true for default cf */
 };
 struct rocksdb_column_family_metadata_t {
  ColumnFamilyMetaData rep;
@@ -546,6 +547,7 @@ static bool SaveError(char** errptr, const Status& s) {
  return true;
 }

+// Copies str to a new malloc()-ed buffer. The buffer is not NUL terminated.
 static char* CopyString(const std::string& str) {
  char* result = reinterpret_cast<char*>(malloc(sizeof(char) * str.size()));
  memcpy(result, str.data(), sizeof(char) * str.size());
@@ -906,6 +908,7 @@ rocksdb_t* rocksdb_open_and_trim_history(
    rocksdb_column_family_handle_t* c_handle =
        new rocksdb_column_family_handle_t;
    c_handle->rep = handles[i];
+    c_handle->immortal = false;
    column_family_handles[i] = c_handle;
  }
  rocksdb_t* result = new rocksdb_t;
@@ -936,6 +939,7 @@ rocksdb_t* rocksdb_open_column_families(
    rocksdb_column_family_handle_t* c_handle =
        new rocksdb_column_family_handle_t;
    c_handle->rep = handles[i];
+    c_handle->immortal = false;
    column_family_handles[i] = c_handle;
  }
  rocksdb_t* result = new rocksdb_t;
@@ -971,6 +975,7 @@ rocksdb_t* rocksdb_open_column_families_with_ttl(
    rocksdb_column_family_handle_t* c_handle =
        new rocksdb_column_family_handle_t;
    c_handle->rep = handles[i];
+    c_handle->immortal = false;
    column_family_handles[i] = c_handle;
  }
  rocksdb_t* result = new rocksdb_t;
@@ -1004,6 +1009,7 @@ rocksdb_t* rocksdb_open_for_read_only_column_families(
    rocksdb_column_family_handle_t* c_handle =
        new rocksdb_column_family_handle_t;
    c_handle->rep = handles[i];
+    c_handle->immortal = false;
    column_family_handles[i] = c_handle;
  }
  rocksdb_t* result = new rocksdb_t;
@@ -1035,6 +1041,7 @@ rocksdb_t* rocksdb_open_as_secondary_column_families(
    rocksdb_column_family_handle_t* c_handle =
        new rocksdb_column_family_handle_t;
    c_handle->rep = handles[i];
+    c_handle->immortal = false;
    column_family_handles[i] = c_handle;
  }
  rocksdb_t* result = new rocksdb_t;
@@ -1072,6 +1079,7 @@ rocksdb_column_family_handle_t* rocksdb_create_column_family(
  SaveError(errptr, db->rep->CreateColumnFamily(
                        ColumnFamilyOptions(column_family_options->rep),
                        std::string(column_family_name), &(handle->rep)));
+  handle->immortal = false;
  return handle;
 }

@@ -1095,6 +1103,7 @@ rocksdb_column_family_handle_t** rocksdb_create_column_families(
  for (size_t i = 0; i != handles.size(); ++i) {
    c_handles[i] = new rocksdb_column_family_handle_t;
    c_handles[i]->rep = handles[i];
+    c_handles[i]->immortal = false;
  }

  return c_handles;
@@ -1114,6 +1123,7 @@ rocksdb_column_family_handle_t* rocksdb_create_column_family_with_ttl(
  SaveError(errptr, db_with_ttl->CreateColumnFamilyWithTtl(
                        ColumnFamilyOptions(column_family_options->rep),
                        std::string(column_family_name), &(handle->rep), ttl));
+  handle->immortal = false;
  return handle;
 }

@@ -1135,9 +1145,19 @@ char* rocksdb_column_family_handle_get_name(
  return CopyString(name);
 }

+rocksdb_column_family_handle_t* rocksdb_get_default_column_family_handle(
+    rocksdb_t* db) {
+  rocksdb_column_family_handle_t* handle = new rocksdb_column_family_handle_t;
+  handle->rep = db->rep->DefaultColumnFamily();
+  handle->immortal = true;
+  return handle;
+}
+
 void rocksdb_column_family_handle_destroy(
    rocksdb_column_family_handle_t* handle) {
-  delete handle->rep;
+  if (!handle->immortal) {
+    delete handle->rep;
+  }
  delete handle;
 }

@@ -1373,6 +1393,18 @@ char* rocksdb_get_cf_with_ts(rocksdb_t* db,
  return result;
 }

+char* rocksdb_get_db_identity(rocksdb_t* db, size_t* id_len) {
+  std::string identity_tmp;
+  Status s = db->rep->GetDbIdentity(identity_tmp);
+  if (!s.ok()) {
+    *id_len = 0;
+    return nullptr;
+  }
+
+  *id_len = identity_tmp.size();
+  return CopyString(identity_tmp);
+}
+
 void rocksdb_multi_get(rocksdb_t* db, const rocksdb_readoptions_t* options,
                       size_t num_keys, const char* const* keys_list,
                       const size_t* keys_list_sizes, char** values_list,
@@ -1947,6 +1979,10 @@ void rocksdb_iter_get_error(const rocksdb_iterator_t* iter, char** errptr) {
  SaveError(errptr, iter->rep->status());
 }

+void rocksdb_iter_refresh(const rocksdb_iterator_t* iter, char** errptr) {
+  SaveError(errptr, iter->rep->Refresh());
+}
+
 rocksdb_writebatch_t* rocksdb_writebatch_create() {
  return new rocksdb_writebatch_t;
 }
@@ -1958,6 +1994,15 @@ rocksdb_writebatch_t* rocksdb_writebatch_create_from(const char* rep,
  return b;
 }

+rocksdb_writebatch_t* rocksdb_writebatch_create_with_params(
+    size_t reserved_bytes, size_t max_bytes, size_t protection_bytes_per_key,
+    size_t default_cf_ts_sz) {
+  rocksdb_writebatch_t* b = new rocksdb_writebatch_t;
+  b->rep = WriteBatch(reserved_bytes, max_bytes, protection_bytes_per_key,
+                      default_cf_ts_sz);
+  return b;
+}
+
 void rocksdb_writebatch_destroy(rocksdb_writebatch_t* b) { delete b; }

 void rocksdb_writebatch_clear(rocksdb_writebatch_t* b) { b->rep.Clear(); }
@@ -2223,6 +2268,35 @@ rocksdb_writebatch_wi_t* rocksdb_writebatch_wi_create(
  return b;
 }

+rocksdb_writebatch_wi_t* rocksdb_writebatch_wi_create_with_params(
+    rocksdb_comparator_t* backup_index_comparator, size_t reserved_bytes,
+    unsigned char overwrite_key, size_t max_bytes,
+    size_t protection_bytes_per_key) {
+  rocksdb_writebatch_wi_t* b = new rocksdb_writebatch_wi_t;
+  b->rep = new WriteBatchWithIndex(backup_index_comparator, reserved_bytes,
+                                   overwrite_key, max_bytes,
+                                   protection_bytes_per_key);
+  return b;
+}
+
+void rocksdb_writebatch_update_timestamps(
+    rocksdb_writebatch_t* wb, const char* ts, size_t tslen, void* state,
+    size_t (*get_ts_size)(void*, uint32_t), char** errptr) {
+  SaveError(errptr, wb->rep.UpdateTimestamps(
+                        Slice(ts, tslen), [&get_ts_size, &state](uint32_t cf) {
+                          return (*get_ts_size)(state, cf);
+                        }));
+}
+
+void rocksdb_writebatch_wi_update_timestamps(
+    rocksdb_writebatch_wi_t* wb, const char* ts, size_t tslen, void* state,
+    size_t (*get_ts_size)(void*, uint32_t), char** errptr) {
+  SaveError(errptr, wb->rep->GetWriteBatch()->UpdateTimestamps(
+                        Slice(ts, tslen), [&get_ts_size, &state](uint32_t cf) {
+                          return (*get_ts_size)(state, cf);
+                        }));
+}
+
 void rocksdb_writebatch_wi_destroy(rocksdb_writebatch_wi_t* b) {
  if (b->rep) {
    delete b->rep;
@@ -3885,6 +3959,27 @@ void rocksdb_options_set_plain_table_factory(
  opt->rep.table_factory.reset(factory);
 }

+unsigned char rocksdb_options_get_write_dbid_to_manifest(
+    rocksdb_options_t* opt) {
+  return opt->rep.write_dbid_to_manifest;
+}
+
+void rocksdb_options_set_write_dbid_to_manifest(
+    rocksdb_options_t* opt, unsigned char write_dbid_to_manifest) {
+  opt->rep.write_dbid_to_manifest = write_dbid_to_manifest;
+}
+
+unsigned char rocksdb_options_get_track_and_verify_wals_in_manifest(
+    rocksdb_options_t* opt) {
+  return opt->rep.track_and_verify_wals_in_manifest;
+}
+
+void rocksdb_options_set_track_and_verify_wals_in_manifest(
+    rocksdb_options_t* opt, unsigned char track_and_verify_wals_in_manifest) {
+  opt->rep.track_and_verify_wals_in_manifest =
+      track_and_verify_wals_in_manifest;
+}
+
 void rocksdb_options_set_max_successive_merges(rocksdb_options_t* opt,
                                               size_t v) {
  opt->rep.max_successive_merges = v;
@@ -4264,13 +4359,8 @@ void rocksdb_perfcontext_destroy(rocksdb_perfcontext_t* context) {

 /*
 TODO:
-DB::OpenForReadOnly
-DB::KeyMayExist
 DB::GetOptions
 DB::GetSortedWalFiles
-DB::GetLatestSequenceNumber
-DB::GetUpdatesSince
-DB::GetDbIdentity
 DB::RunManualCompaction
 custom cache
 table_properties_collectors
@@ -5747,6 +5837,7 @@ rocksdb_column_family_handle_t* rocksdb_transactiondb_create_column_family(
  SaveError(errptr, txn_db->rep->CreateColumnFamily(
                        ColumnFamilyOptions(column_family_options->rep),
                        std::string(column_family_name), &(handle->rep)));
+  handle->immortal = false;
  return handle;
 }

@@ -5789,6 +5880,7 @@ rocksdb_transactiondb_t* rocksdb_transactiondb_open_column_families(
    rocksdb_column_family_handle_t* c_handle =
        new rocksdb_column_family_handle_t;
    c_handle->rep = handles[i];
+    c_handle->immortal = false;
    column_family_handles[i] = c_handle;
  }
  rocksdb_transactiondb_t* result = new rocksdb_transactiondb_t;
@@ -6583,6 +6675,7 @@ rocksdb_optimistictransactiondb_open_column_families(
    rocksdb_column_family_handle_t* c_handle =
        new rocksdb_column_family_handle_t;
    c_handle->rep = handles[i];
+    c_handle->immortal = false;
    column_family_handles[i] = c_handle;
  }
  rocksdb_optimistictransactiondb_t* result =
@@ -798,6 +798,11 @@ int main(int argc, char** argv) {
    rocksdb_delete(db, woptions, "foo", 3, &err);
    CheckNoError(err);

+    // get the identity before the backup
+    size_t before_db_id_len = 0;
+    char* before_db_id = rocksdb_get_db_identity(db, &before_db_id_len);
+    CheckCondition(before_db_id_len == 36);
+
    rocksdb_close(db);

    rocksdb_destroy_db(options, dbname, &err);
@@ -818,6 +823,14 @@ int main(int argc, char** argv) {

    CheckGet(db, roptions, "foo", "hello");

+    // the db_identity after the backup is different
+    size_t after_db_id_len = 0;
+    char* after_db_id = rocksdb_get_db_identity(db, &after_db_id_len);
+    CheckCondition(after_db_id_len == 36);
+    CheckCondition(memcmp(after_db_id, before_db_id, after_db_id_len) != 0);
+    Free(&before_db_id);
+    Free(&after_db_id);
+
    rocksdb_backup_engine_close(be);
  }

@@ -833,6 +846,14 @@ int main(int argc, char** argv) {
    rocksdb_checkpoint_create(checkpoint, dbcheckpointname, 0, &err);
    CheckNoError(err);

+    rocksdb_checkpoint_object_destroy(checkpoint);
+    checkpoint = NULL;
+
+    // get the identity before the checkpoint
+    size_t before_db_id_len = 0;
+    char* before_db_id = rocksdb_get_db_identity(db, &before_db_id_len);
+    CheckCondition(before_db_id_len == 36);
+
    // start a new database from the checkpoint
    rocksdb_close(db);
    rocksdb_options_set_error_if_exists(options, 0);
@@ -841,7 +862,13 @@ int main(int argc, char** argv) {

    CheckGet(db, roptions, "foo", "hello");

-    rocksdb_checkpoint_object_destroy(checkpoint);
+    // the db_identity after the checkpoint is different
+    size_t after_db_id_len = 0;
+    char* after_db_id = rocksdb_get_db_identity(db, &after_db_id_len);
+    CheckCondition(after_db_id_len == 36);
+    CheckCondition(memcmp(after_db_id, before_db_id, after_db_id_len) != 0);
+    Free(&before_db_id);
+    Free(&after_db_id);

    rocksdb_close(db);
    rocksdb_destroy_db(options, dbcheckpointname, &err);
@@ -852,6 +879,73 @@ int main(int argc, char** argv) {
    rocksdb_options_set_error_if_exists(options, 1);
  }

+  StartPhase("checkpoint_db_id_in_manifest");
+  {
+    // create new DB with set_write_dbid_to_manifest=true
+    // db_identity is now the same across checkpoints
+    rocksdb_close(db);
+    db = NULL;
+
+    rocksdb_options_t* options_dbid_in_manifest = rocksdb_options_create();
+    rocksdb_options_set_create_if_missing(options_dbid_in_manifest, 1);
+
+    unsigned char write_to_manifest =
+        rocksdb_options_get_write_dbid_to_manifest(options_dbid_in_manifest);
+    CheckCondition(!write_to_manifest);
+    rocksdb_options_set_write_dbid_to_manifest(options_dbid_in_manifest, true);
+    CheckCondition(!write_to_manifest);
+    write_to_manifest =
+        rocksdb_options_get_write_dbid_to_manifest(options_dbid_in_manifest);
+    CheckCondition(write_to_manifest);
+
+    db = rocksdb_open(options_dbid_in_manifest, dbbackupname, &err);
+    CheckNoError(err);
+
+    rocksdb_checkpoint_t* checkpoint =
+        rocksdb_checkpoint_object_create(db, &err);
+    CheckNoError(err);
+
+    rocksdb_checkpoint_create(checkpoint, dbcheckpointname, 0, &err);
+    CheckNoError(err);
+
+    rocksdb_checkpoint_object_destroy(checkpoint);
+    checkpoint = NULL;
+
+    // get the identity before the backup
+    size_t before_db_id_len = 0;
+    char* before_db_id = rocksdb_get_db_identity(db, &before_db_id_len);
+    CheckCondition(before_db_id_len == 36);
+
+    // open the checkpoint
+    rocksdb_close(db);
+    rocksdb_destroy_db(options_dbid_in_manifest, dbbackupname, &err);
+    CheckNoError(err);
+
+    rocksdb_options_set_error_if_exists(options_dbid_in_manifest, 0);
+    db = rocksdb_open(options_dbid_in_manifest, dbcheckpointname, &err);
+    CheckNoError(err);
+
+    // the db_identity after the checkpoint is the same
+    size_t after_db_id_len = 0;
+    char* after_db_id = rocksdb_get_db_identity(db, &after_db_id_len);
+    CheckCondition(after_db_id_len == 36);
+    CheckCondition(memcmp(after_db_id, before_db_id, after_db_id_len) == 0);
+    Free(&before_db_id);
+    Free(&after_db_id);
+
+    rocksdb_close(db);
+    rocksdb_destroy_db(options_dbid_in_manifest, dbcheckpointname, &err);
+    CheckNoError(err);
+    rocksdb_options_destroy(options_dbid_in_manifest);
+    options_dbid_in_manifest = NULL;
+
+    // re-open the default database
+    rocksdb_options_set_error_if_exists(options, 0);
+    db = rocksdb_open(options, dbname, &err);
+    CheckNoError(err);
+    rocksdb_options_set_error_if_exists(options, 1);
+  }
+
  StartPhase("compactall");
  rocksdb_compact_range(db, NULL, 0, NULL, 0);
  CheckGet(db, roptions, "foo", "hello");
@@ -2140,6 +2234,12 @@ int main(int argc, char** argv) {
    CheckCondition(rocksdb_statistics_level_all ==
                   rocksdb_options_get_statistics_level(o));

+    CheckCondition(0 ==
+                   rocksdb_options_get_track_and_verify_wals_in_manifest(o));
+    rocksdb_options_set_track_and_verify_wals_in_manifest(o, 42);
+    CheckCondition(1 ==
+                   rocksdb_options_get_track_and_verify_wals_in_manifest(o));
+
    /* Blob Options */
    rocksdb_options_set_enable_blob_files(o, 1);
    CheckCondition(1 == rocksdb_options_get_enable_blob_files(o));
@@ -538,6 +538,7 @@ ColumnFamilyData::ColumnFamilyData(
      refs_(0),
      initialized_(false),
      dropped_(false),
+      flush_skip_reschedule_(false),
      internal_comparator_(cf_options.comparator),
      initial_cf_options_(SanitizeOptions(db_options, cf_options)),
      ioptions_(db_options, initial_cf_options_),
@@ -1528,6 +1529,20 @@ Status ColumnFamilyData::ValidateOptions(
      }
    }
  }
+
+  if (cf_options.compaction_style == kCompactionStyleUniversal) {
+    int max_read_amp = cf_options.compaction_options_universal.max_read_amp;
+    if (max_read_amp < -1) {
+      return Status::NotSupported(
+          "CompactionOptionsUniversal::max_read_amp should be at least -1.");
+    } else if (0 < max_read_amp &&
+               max_read_amp < cf_options.level0_file_num_compaction_trigger) {
+      return Status::NotSupported(
+          "CompactionOptionsUniversal::max_read_amp limits the number of sorted"
+          " runs but is smaller than the compaction trigger "
+          "level0_file_num_compaction_trigger.");
+    }
+  }
  return s;
 }

@@ -1607,6 +1622,19 @@ FSDirectory* ColumnFamilyData::GetDataDir(size_t path_id) const {
  return data_dirs_[path_id].get();
 }

+void ColumnFamilyData::SetFlushSkipReschedule() {
+  const Comparator* ucmp = user_comparator();
+  const size_t ts_sz = ucmp->timestamp_size();
+  if (ts_sz == 0 || ioptions_.persist_user_defined_timestamps) {
+    return;
+  }
+  flush_skip_reschedule_.store(true);
+}
+
+bool ColumnFamilyData::GetAndClearFlushSkipReschedule() {
+  return flush_skip_reschedule_.exchange(false);
+}
+
 bool ColumnFamilyData::ShouldPostponeFlushToRetainUDT(
    uint64_t max_memtable_id) {
  const Comparator* ucmp = user_comparator();
@@ -329,6 +329,10 @@ class ColumnFamilyData {
  void SetDropped();
  bool IsDropped() const { return dropped_.load(std::memory_order_relaxed); }

+  void SetFlushSkipReschedule();
+
+  bool GetAndClearFlushSkipReschedule();
+
  // thread-safe
  int NumberLevels() const { return ioptions_.num_levels; }

@@ -568,6 +572,10 @@ class ColumnFamilyData {
  // of its files (if missing)
  void RecoverEpochNumbers();

+  int GetUnflushedMemTableCountForWriteStallCheck() const {
+    return (mem_->IsEmpty() ? 0 : 1) + imm_.NumNotFlushed();
+  }
+
 private:
  friend class ColumnFamilySet;
  ColumnFamilyData(uint32_t id, const std::string& name,
@@ -592,6 +600,15 @@ class ColumnFamilyData {
  std::atomic<bool> initialized_;
  std::atomic<bool> dropped_;  // true if client dropped it

+  // When user-defined timestamps in memtable only feature is enabled, this
+  // flag indicates a successfully requested flush that should
+  // skip being rescheduled and haven't undergone the rescheduling check yet.
+  // This flag is cleared when a check skips rescheduling a FlushRequest.
+  // With this flag, automatic flushes in regular cases can continue to
+  // retain UDTs by getting rescheduled as usual while manual flushes and
+  // error recovery flushes will proceed without getting rescheduled.
+  std::atomic<bool> flush_skip_reschedule_;
+
  const InternalKeyComparator internal_comparator_;
  InternalTblPropCollFactories internal_tbl_prop_coll_factories_;

@@ -23,6 +23,7 @@
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "rocksdb/iterator.h"
+#include "rocksdb/listener.h"
 #include "rocksdb/utilities/object_registry.h"
 #include "test_util/sync_point.h"
 #include "test_util/testharness.h"
@@ -34,6 +35,13 @@
 #include "utilities/merge_operators.h"

 namespace ROCKSDB_NAMESPACE {
+namespace {
+std::string EncodeAsUint64(uint64_t number) {
+  std::string result;
+  PutFixed64(&result, number);
+  return result;
+}
+}  // namespace

 static const int kValueSize = 1000;

@@ -3170,6 +3178,8 @@ TEST_P(ColumnFamilyTest, IteratorCloseWALFile1) {
  SpecialEnv env(Env::Default());
  db_options_.env = &env;
  db_options_.max_background_flushes = 1;
+  // When this option is removed, the test will need re-engineering
+  db_options_.background_close_inactive_wals = true;
  column_family_options_.memtable_factory.reset(
      test::NewSpecialSkipListFactory(2));
  Open();
@@ -3222,6 +3232,8 @@ TEST_P(ColumnFamilyTest, IteratorCloseWALFile2) {
  env.SetBackgroundThreads(2, Env::HIGH);
  db_options_.env = &env;
  db_options_.max_background_flushes = 1;
+  // When this option is removed, the test will need re-engineering
+  db_options_.background_close_inactive_wals = true;
  column_family_options_.memtable_factory.reset(
      test::NewSpecialSkipListFactory(2));
  Open();
@@ -3279,6 +3291,8 @@ TEST_P(ColumnFamilyTest, ForwardIteratorCloseWALFile) {
  env.SetBackgroundThreads(2, Env::HIGH);
  db_options_.env = &env;
  db_options_.max_background_flushes = 1;
+  // When this option is removed, the test will need re-engineering
+  db_options_.background_close_inactive_wals = true;
  column_family_options_.memtable_factory.reset(
      test::NewSpecialSkipListFactory(3));
  column_family_options_.level0_file_num_compaction_trigger = 2;
@@ -3603,7 +3617,9 @@ TEST(ColumnFamilyTest, ValidateMemtableKVChecksumOption) {
 }

 // Tests the flushing behavior of a column family to retain user-defined
-// timestamp when `persist_user_defined_timestamp` is false.
+// timestamp when `persist_user_defined_timestamp` is false. The behavior of
+// auto flush is it makes some effort to retain user-defined timestamps while
+// the behavior of manual flush is that it skips retaining UDTs.
 class ColumnFamilyRetainUDTTest : public ColumnFamilyTestBase {
 public:
  ColumnFamilyRetainUDTTest() : ColumnFamilyTestBase(kLatestFormatVersion) {}
@@ -3621,6 +3637,27 @@ class ColumnFamilyRetainUDTTest : public ColumnFamilyTestBase {
    return db_->Put(WriteOptions(), handles_[cf], Slice(key), Slice(ts),
                    Slice(value));
  }
+
+  std::string Get(int cf, const std::string& key, const std::string& read_ts) {
+    ReadOptions ropts;
+    Slice timestamp = read_ts;
+    ropts.timestamp = &timestamp;
+    std::string value;
+    Status s = db_->Get(ropts, handles_[cf], Slice(key), &value);
+    if (s.IsNotFound()) {
+      return "NOT_FOUND";
+    } else if (s.ok()) {
+      return value;
+    }
+    return "";
+  }
+
+  void CheckEffectiveCutoffTime(uint64_t expected_cutoff) {
+    std::string effective_full_history_ts_low;
+    EXPECT_OK(
+        db_->GetFullHistoryTsLow(handles_[0], &effective_full_history_ts_low));
+    EXPECT_EQ(EncodeAsUint64(expected_cutoff), effective_full_history_ts_low);
+  }
 };

 class TestTsComparator : public Comparator {
@@ -3664,7 +3701,9 @@ TEST_F(ColumnFamilyRetainUDTTest, SanityCheck) {
  Close();
 }

-TEST_F(ColumnFamilyRetainUDTTest, FullHistoryTsLowNotSet) {
+class AutoFlushRetainUDTTest : public ColumnFamilyRetainUDTTest {};
+
+TEST_F(AutoFlushRetainUDTTest, FullHistoryTsLowNotSet) {
  SyncPoint::GetInstance()->SetCallBack(
      "DBImpl::BackgroundFlush:CheckFlushRequest:cb", [&](void* arg) {
        ASSERT_NE(nullptr, arg);
@@ -3674,28 +3713,22 @@ TEST_F(ColumnFamilyRetainUDTTest, FullHistoryTsLowNotSet) {

  SyncPoint::GetInstance()->EnableProcessing();
  Open();
-  std::string write_ts;
-  PutFixed64(&write_ts, 1);
-  ASSERT_OK(Put(0, "foo", write_ts, "v1"));
-  // No `full_history_ts_low` explicitly set by user, flush is continued
+  ASSERT_OK(Put(0, "foo", EncodeAsUint64(1), "v1"));
+  // No `full_history_ts_low` explicitly set by user, auto flush is continued
  // without checking if its UDTs expired.
-  ASSERT_OK(Flush(0));
+  ASSERT_OK(dbfull()->TEST_SwitchWAL());
+  ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());

  // After flush, `full_history_ts_low` should be automatically advanced to
  // the effective cutoff timestamp: write_ts + 1
-  std::string cutoff_ts;
-  PutFixed64(&cutoff_ts, 2);
-  std::string effective_full_history_ts_low;
-  ASSERT_OK(
-      db_->GetFullHistoryTsLow(handles_[0], &effective_full_history_ts_low));
-  ASSERT_EQ(cutoff_ts, effective_full_history_ts_low);
+  CheckEffectiveCutoffTime(2);
  Close();

  SyncPoint::GetInstance()->DisableProcessing();
  SyncPoint::GetInstance()->ClearAllCallBacks();
 }

-TEST_F(ColumnFamilyRetainUDTTest, AllKeysExpired) {
+TEST_F(AutoFlushRetainUDTTest, AllKeysExpired) {
  SyncPoint::GetInstance()->SetCallBack(
      "DBImpl::BackgroundFlush:CheckFlushRequest:cb", [&](void* arg) {
        ASSERT_NE(nullptr, arg);
@@ -3705,27 +3738,22 @@ TEST_F(ColumnFamilyRetainUDTTest, AllKeysExpired) {

  SyncPoint::GetInstance()->EnableProcessing();
  Open();
-  std::string write_ts;
-  PutFixed64(&write_ts, 1);
-  ASSERT_OK(Put(0, "foo", write_ts, "v1"));
-  std::string cutoff_ts;
-  PutFixed64(&cutoff_ts, 3);
-  ASSERT_OK(db_->IncreaseFullHistoryTsLow(handles_[0], cutoff_ts));
-  // All keys expired w.r.t the configured `full_history_ts_low`, flush continue
-  // without the need for a re-schedule.
-  ASSERT_OK(Flush(0));
+  ASSERT_OK(Put(0, "foo", EncodeAsUint64(1), "v1"));
+  ASSERT_OK(db_->IncreaseFullHistoryTsLow(handles_[0], EncodeAsUint64(3)));
+  // All keys expired w.r.t the configured `full_history_ts_low`, auto flush
+  // continue without the need for a re-schedule.
+  ASSERT_OK(dbfull()->TEST_SwitchWAL());
+  ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());

  // `full_history_ts_low` stays unchanged after flush.
-  std::string effective_full_history_ts_low;
-  ASSERT_OK(
-      db_->GetFullHistoryTsLow(handles_[0], &effective_full_history_ts_low));
-  ASSERT_EQ(cutoff_ts, effective_full_history_ts_low);
+  CheckEffectiveCutoffTime(3);
  Close();

  SyncPoint::GetInstance()->DisableProcessing();
  SyncPoint::GetInstance()->ClearAllCallBacks();
 }
-TEST_F(ColumnFamilyRetainUDTTest, NotAllKeysExpiredFlushToAvoidWriteStall) {
+
+TEST_F(AutoFlushRetainUDTTest, NotAllKeysExpiredFlushToAvoidWriteStall) {
  SyncPoint::GetInstance()->SetCallBack(
      "DBImpl::BackgroundFlush:CheckFlushRequest:cb", [&](void* arg) {
        ASSERT_NE(nullptr, arg);
@@ -3735,72 +3763,218 @@ TEST_F(ColumnFamilyRetainUDTTest, NotAllKeysExpiredFlushToAvoidWriteStall) {

  SyncPoint::GetInstance()->EnableProcessing();
  Open();
-  std::string cutoff_ts;
-  std::string write_ts;
-  PutFixed64(&write_ts, 1);
-  ASSERT_OK(Put(0, "foo", write_ts, "v1"));
-  PutFixed64(&cutoff_ts, 1);
-  ASSERT_OK(db_->IncreaseFullHistoryTsLow(handles_[0], cutoff_ts));
+  ASSERT_OK(Put(0, "foo", EncodeAsUint64(1), "v1"));
+  ASSERT_OK(db_->IncreaseFullHistoryTsLow(handles_[0], EncodeAsUint64(1)));
  ASSERT_OK(db_->SetOptions(handles_[0], {{"max_write_buffer_number", "1"}}));
-  // Not all keys expired, but flush is continued without a re-schedule because
-  // of risk of write stall.
-  ASSERT_OK(Flush(0));
+  // Not all keys expired, but auto flush is continued without a re-schedule
+  // because of risk of write stall.
+  ASSERT_OK(dbfull()->TEST_SwitchWAL());
+  ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());

  // After flush, `full_history_ts_low` should be automatically advanced to
  // the effective cutoff timestamp: write_ts + 1
-  std::string effective_full_history_ts_low;
-  ASSERT_OK(
-      db_->GetFullHistoryTsLow(handles_[0], &effective_full_history_ts_low));
-
-  cutoff_ts.clear();
-  PutFixed64(&cutoff_ts, 2);
-  ASSERT_EQ(cutoff_ts, effective_full_history_ts_low);
+  CheckEffectiveCutoffTime(2);
  Close();

  SyncPoint::GetInstance()->DisableProcessing();
  SyncPoint::GetInstance()->ClearAllCallBacks();
 }

-TEST_F(ColumnFamilyRetainUDTTest, NotAllKeysExpiredFlushRescheduled) {
-  std::string cutoff_ts;
+TEST_F(AutoFlushRetainUDTTest, NotAllKeysExpiredFlushRescheduled) {
+  std::atomic<int> local_counter{1};
  SyncPoint::GetInstance()->SetCallBack(
      "DBImpl::AfterRetainUDTReschedule:cb", [&](void* /*arg*/) {
        // Increasing full_history_ts_low so all keys expired after the initial
        // FlushRequest is rescheduled
-        cutoff_ts.clear();
-        PutFixed64(&cutoff_ts, 3);
-        ASSERT_OK(db_->IncreaseFullHistoryTsLow(handles_[0], cutoff_ts));
+        ASSERT_OK(
+            db_->IncreaseFullHistoryTsLow(handles_[0], EncodeAsUint64(3)));
      });
  SyncPoint::GetInstance()->SetCallBack(
      "DBImpl::BackgroundFlush:CheckFlushRequest:cb", [&](void* arg) {
        ASSERT_NE(nullptr, arg);
        auto reschedule_count = *static_cast<int*>(arg);
        ASSERT_EQ(2, reschedule_count);
+        local_counter.fetch_add(1);
      });
  SyncPoint::GetInstance()->EnableProcessing();

  Open();
-  std::string write_ts;
-  PutFixed64(&write_ts, 1);
-  ASSERT_OK(Put(0, "foo", write_ts, "v1"));
-  PutFixed64(&cutoff_ts, 1);
-  ASSERT_OK(db_->IncreaseFullHistoryTsLow(handles_[0], cutoff_ts));
+  ASSERT_OK(Put(0, "foo", EncodeAsUint64(1), "v1"));
+  ASSERT_OK(db_->IncreaseFullHistoryTsLow(handles_[0], EncodeAsUint64(1)));
  // Not all keys expired, and there is no risk of write stall. Flush is
  // rescheduled. The actual flush happens after `full_history_ts_low` is
  // increased to mark all keys expired.
-  ASSERT_OK(Flush(0));
+  ASSERT_OK(dbfull()->TEST_SwitchWAL());
+  ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
+  // Make sure callback is not skipped.
+  ASSERT_EQ(2, local_counter);

-  std::string effective_full_history_ts_low;
-  ASSERT_OK(
-      db_->GetFullHistoryTsLow(handles_[0], &effective_full_history_ts_low));
-  // `full_history_ts_low` stays unchanged.
-  ASSERT_EQ(cutoff_ts, effective_full_history_ts_low);
+  CheckEffectiveCutoffTime(3);
  Close();

  SyncPoint::GetInstance()->DisableProcessing();
  SyncPoint::GetInstance()->ClearAllCallBacks();
 }

+class ManualFlushSkipRetainUDTTest : public ColumnFamilyRetainUDTTest {
+ public:
+  // Write an entry with timestamp that is not expired w.r.t cutoff timestamp,
+  // and make sure automatic flush would be rescheduled to retain UDT.
+  void CheckAutomaticFlushRetainUDT(uint64_t write_ts) {
+    std::atomic<int> local_counter{1};
+    SyncPoint::GetInstance()->SetCallBack(
+        "DBImpl::AfterRetainUDTReschedule:cb", [&](void* /*arg*/) {
+          // Increasing full_history_ts_low so all keys expired after the
+          // initial FlushRequest is rescheduled
+          ASSERT_OK(db_->IncreaseFullHistoryTsLow(
+              handles_[0], EncodeAsUint64(write_ts + 1)));
+        });
+    SyncPoint::GetInstance()->SetCallBack(
+        "DBImpl::BackgroundFlush:CheckFlushRequest:cb", [&](void* arg) {
+          ASSERT_NE(nullptr, arg);
+          auto reschedule_count = *static_cast<int*>(arg);
+          ASSERT_EQ(2, reschedule_count);
+          local_counter.fetch_add(1);
+        });
+    SyncPoint::GetInstance()->EnableProcessing();
+    EXPECT_OK(Put(0, "foo", EncodeAsUint64(write_ts),
+                  "foo" + std::to_string(write_ts)));
+    EXPECT_OK(dbfull()->TEST_SwitchWAL());
+    EXPECT_OK(dbfull()->TEST_WaitForFlushMemTable());
+    // Make sure callback is not skipped.
+    EXPECT_EQ(2, local_counter);
+
+    SyncPoint::GetInstance()->DisableProcessing();
+    SyncPoint::GetInstance()->ClearAllCallBacks();
+  }
+};
+
+TEST_F(ManualFlushSkipRetainUDTTest, ManualFlush) {
+  Open();
+  ASSERT_OK(db_->IncreaseFullHistoryTsLow(handles_[0], EncodeAsUint64(0)));
+
+  // Manual flush proceeds without trying to retain UDT.
+  ASSERT_OK(Put(0, "foo", EncodeAsUint64(1), "v1"));
+  ASSERT_OK(Flush(0));
+  CheckEffectiveCutoffTime(2);
+  CheckAutomaticFlushRetainUDT(3);
+
+  Close();
+}
+
+TEST_F(ManualFlushSkipRetainUDTTest, ManualCompaction) {
+  Open();
+  ASSERT_OK(db_->IncreaseFullHistoryTsLow(handles_[0], EncodeAsUint64(0)));
+
+  // Manual compaction proceeds without trying to retain UDT.
+  ASSERT_OK(Put(0, "foo", EncodeAsUint64(1), "v2"));
+  ASSERT_OK(
+      db_->CompactRange(CompactRangeOptions(), handles_[0], nullptr, nullptr));
+  CheckEffectiveCutoffTime(2);
+  CheckAutomaticFlushRetainUDT(3);
+
+  Close();
+}
+
+TEST_F(ManualFlushSkipRetainUDTTest, BulkLoading) {
+  Open();
+  ASSERT_OK(db_->IncreaseFullHistoryTsLow(handles_[0], EncodeAsUint64(0)));
+  ASSERT_OK(Put(0, "foo", EncodeAsUint64(1), "v1"));
+
+  // Test flush behavior in bulk loading scenarios.
+  Options options(db_options_, column_family_options_);
+  std::string sst_files_dir = dbname_ + "/sst_files/";
+  ASSERT_OK(DestroyDir(env_, sst_files_dir));
+  ASSERT_OK(env_->CreateDir(sst_files_dir));
+  SstFileWriter sst_file_writer(EnvOptions(), options);
+  std::string file1 = sst_files_dir + "file1.sst";
+  ASSERT_OK(sst_file_writer.Open(file1));
+  ASSERT_OK(sst_file_writer.Put("foo", EncodeAsUint64(0), "v2"));
+  ExternalSstFileInfo file1_info;
+  ASSERT_OK(sst_file_writer.Finish(&file1_info));
+
+  // Bulk loading in UDT mode doesn't support external file key range overlap
+  // with DB key range.
+  ASSERT_TRUE(db_->IngestExternalFile({file1}, IngestExternalFileOptions())
+                  .IsInvalidArgument());
+
+  std::string file2 = sst_files_dir + "file2.sst";
+  ASSERT_OK(sst_file_writer.Open(file2));
+  ASSERT_OK(sst_file_writer.Put("bar", EncodeAsUint64(0), "val"));
+  ExternalSstFileInfo file2_info;
+  ASSERT_OK(sst_file_writer.Finish(&file2_info));
+  // A successful bulk loading, and it doesn't trigger any flush. As a result
+  // the effective cutoff timestamp is also unchanged.
+  ASSERT_OK(db_->IngestExternalFile({file2}, IngestExternalFileOptions()));
+
+  ASSERT_EQ(Get(0, "foo", EncodeAsUint64(1)), "v1");
+  ASSERT_EQ(Get(0, "bar", EncodeAsUint64(0)), "val");
+  CheckEffectiveCutoffTime(0);
+  CheckAutomaticFlushRetainUDT(1);
+
+  Close();
+}
+
+TEST_F(ManualFlushSkipRetainUDTTest, AutomaticFlushQueued) {
+  Open();
+  ASSERT_OK(db_->IncreaseFullHistoryTsLow(handles_[0], EncodeAsUint64(0)));
+
+  ASSERT_OK(Put(0, "foo", EncodeAsUint64(1), "v1"));
+  ASSERT_OK(dbfull()->TEST_SwitchWAL());
+  CheckEffectiveCutoffTime(0);
+
+  // Default `max_write_buffer_number=2` used, writing another memtable can get
+  // automatic flush to proceed because of memory pressure. Not doing that so
+  // we can test automatic flush gets to proceed because of an ongoing manual
+  // flush attempt.
+  ASSERT_OK(Flush(0));
+  CheckEffectiveCutoffTime(2);
+  CheckAutomaticFlushRetainUDT(3);
+
+  Close();
+}
+
+TEST_F(ManualFlushSkipRetainUDTTest, ConcurrentManualFlushes) {
+  Open();
+  ASSERT_OK(db_->IncreaseFullHistoryTsLow(handles_[0], EncodeAsUint64(0)));
+
+  std::vector<ROCKSDB_NAMESPACE::port::Thread> manual_flush_tds;
+  std::atomic<int> next_ts{0};
+  std::mutex mtx;
+  std::condition_variable cv;
+
+  auto manual_flush = [&](int write_ts) {
+    {
+      std::unique_lock<std::mutex> lock(mtx);
+      cv.wait(lock,
+              [&write_ts, &next_ts] { return write_ts == next_ts.load(); });
+      ASSERT_OK(Put(0, "foo" + std::to_string(write_ts),
+                    EncodeAsUint64(write_ts),
+                    "val_" + std::to_string(write_ts)));
+      next_ts.fetch_add(1);
+      cv.notify_all();
+    }
+    if (write_ts % 2 == 0) {
+      ASSERT_OK(Flush(0));
+    } else {
+      ASSERT_OK(db_->CompactRange(CompactRangeOptions(), handles_[0], nullptr,
+                                  nullptr));
+    }
+  };
+
+  for (int write_ts = 0; write_ts < 10; write_ts++) {
+    manual_flush_tds.emplace_back(manual_flush, write_ts);
+  }
+
+  for (auto& td : manual_flush_tds) {
+    td.join();
+  }
+
+  CheckEffectiveCutoffTime(10);
+  CheckAutomaticFlushRetainUDT(11);
+  Close();
+}
+
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -25,6 +25,7 @@ namespace ROCKSDB_NAMESPACE {
 CompactionIterator::CompactionIterator(
    InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
    SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
+    SequenceNumber earliest_snapshot,
    SequenceNumber earliest_write_conflict_snapshot,
    SequenceNumber job_snapshot, const SnapshotChecker* snapshot_checker,
    Env* env, bool report_detailed_time, bool expect_valid_internal_key,
@@ -40,7 +41,7 @@ CompactionIterator::CompactionIterator(
    const SequenceNumber preserve_time_min_seqno,
    const SequenceNumber preclude_last_level_min_seqno)
    : CompactionIterator(
-          input, cmp, merge_helper, last_sequence, snapshots,
+          input, cmp, merge_helper, last_sequence, snapshots, earliest_snapshot,
          earliest_write_conflict_snapshot, job_snapshot, snapshot_checker, env,
          report_detailed_time, expect_valid_internal_key, range_del_agg,
          blob_file_builder, allow_data_in_errors, enforce_single_del_contracts,
@@ -54,6 +55,7 @@ CompactionIterator::CompactionIterator(
 CompactionIterator::CompactionIterator(
    InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
    SequenceNumber /*last_sequence*/, std::vector<SequenceNumber>* snapshots,
+    SequenceNumber earliest_snapshot,
    SequenceNumber earliest_write_conflict_snapshot,
    SequenceNumber job_snapshot, const SnapshotChecker* snapshot_checker,
    Env* env, bool report_detailed_time, bool expect_valid_internal_key,
@@ -91,9 +93,7 @@ CompactionIterator::CompactionIterator(
      // snapshots_ cannot be nullptr, but we will assert later in the body of
      // the constructor.
      visible_at_tip_(snapshots_ ? snapshots_->empty() : false),
-      earliest_snapshot_(!snapshots_ || snapshots_->empty()
-                             ? kMaxSequenceNumber
-                             : snapshots_->at(0)),
+      earliest_snapshot_(earliest_snapshot),
      info_log_(info_log),
      allow_data_in_errors_(allow_data_in_errors),
      enforce_single_del_contracts_(enforce_single_del_contracts),
@@ -203,6 +203,7 @@ class CompactionIterator {
  CompactionIterator(
      InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
      SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
+      SequenceNumber earliest_snapshot,
      SequenceNumber earliest_write_conflict_snapshot,
      SequenceNumber job_snapshot, const SnapshotChecker* snapshot_checker,
      Env* env, bool report_detailed_time, bool expect_valid_internal_key,
@@ -222,6 +223,7 @@ class CompactionIterator {
  CompactionIterator(
      InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
      SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
+      SequenceNumber earliest_snapshot,
      SequenceNumber earliest_write_conflict_snapshot,
      SequenceNumber job_snapshot, const SnapshotChecker* snapshot_checker,
      Env* env, bool report_detailed_time, bool expect_valid_internal_key,
@@ -296,6 +296,7 @@ class CompactionIteratorTest : public testing::TestWithParam<bool> {
    iter_->SeekToFirst();
    c_iter_.reset(new CompactionIterator(
        iter_.get(), cmp_, merge_helper_.get(), last_sequence, &snapshots_,
+        snapshots_.empty() ? kMaxSequenceNumber : snapshots_.at(0),
        earliest_write_conflict_snapshot, kMaxSequenceNumber,
        snapshot_checker_.get(), Env::Default(),
        false /* report_detailed_time */, false, range_del_agg_.get(),
@@ -174,6 +174,9 @@ CompactionJob::CompactionJob(
      db_mutex_(db_mutex),
      db_error_handler_(db_error_handler),
      existing_snapshots_(std::move(existing_snapshots)),
+      earliest_snapshot_(existing_snapshots_.empty()
+                             ? kMaxSequenceNumber
+                             : existing_snapshots_.at(0)),
      earliest_write_conflict_snapshot_(earliest_write_conflict_snapshot),
      snapshot_checker_(snapshot_checker),
      job_context_(job_context),
@@ -282,6 +285,7 @@ void CompactionJob::Prepare() {

  // collect all seqno->time information from the input files which will be used
  // to encode seqno->time to the output files.
+
  uint64_t preserve_time_duration =
      std::max(c->immutable_options()->preserve_internal_time_seconds,
               c->immutable_options()->preclude_last_level_data_seconds);
@@ -319,28 +323,11 @@ void CompactionJob::Prepare() {
      seqno_to_time_mapping_.Enforce();
    } else {
      seqno_to_time_mapping_.Enforce(_current_time);
-      uint64_t preserve_time =
-          static_cast<uint64_t>(_current_time) > preserve_time_duration
-              ? _current_time - preserve_time_duration
-              : 0;
-      // GetProximalSeqnoBeforeTime tells us the last seqno known to have been
-      // written at or before the given time. + 1 to get the minimum we should
-      // preserve without excluding anything that might have been written on or
-      // after the given time.
-      preserve_time_min_seqno_ =
-          seqno_to_time_mapping_.GetProximalSeqnoBeforeTime(preserve_time) + 1;
-      if (c->immutable_options()->preclude_last_level_data_seconds > 0) {
-        uint64_t preclude_last_level_time =
-            static_cast<uint64_t>(_current_time) >
-                    c->immutable_options()->preclude_last_level_data_seconds
-                ? _current_time -
-                      c->immutable_options()->preclude_last_level_data_seconds
-                : 0;
-        preclude_last_level_min_seqno_ =
-            seqno_to_time_mapping_.GetProximalSeqnoBeforeTime(
-                preclude_last_level_time) +
-            1;
-      }
+      seqno_to_time_mapping_.GetCurrentTieringCutoffSeqnos(
+          static_cast<uint64_t>(_current_time),
+          c->immutable_options()->preserve_internal_time_seconds,
+          c->immutable_options()->preclude_last_level_data_seconds,
+          &preserve_time_min_seqno_, &preclude_last_level_min_seqno_);
    }
    // For accuracy of the GetProximalSeqnoBeforeTime queries above, we only
    // limit the capacity after them.
@@ -1295,8 +1282,9 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {

  auto c_iter = std::make_unique<CompactionIterator>(
      input, cfd->user_comparator(), &merge, versions_->LastSequence(),
-      &existing_snapshots_, earliest_write_conflict_snapshot_, job_snapshot_seq,
-      snapshot_checker_, env_, ShouldReportDetailedTime(env_, stats_),
+      &existing_snapshots_, earliest_snapshot_,
+      earliest_write_conflict_snapshot_, job_snapshot_seq, snapshot_checker_,
+      env_, ShouldReportDetailedTime(env_, stats_),
      /*expect_valid_internal_key=*/true, range_del_agg.get(),
      blob_file_builder.get(), db_options_.allow_data_in_errors,
      db_options_.enforce_single_del_contracts, manual_compaction_canceled_,
@@ -1969,7 +1957,10 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
      cfd->GetName(), sub_compact->compaction->output_level(),
      bottommost_level_, TableFileCreationReason::kCompaction,
      0 /* oldest_key_time */, current_time, db_id_, db_session_id_,
-      sub_compact->compaction->max_output_file_size(), file_number);
+      sub_compact->compaction->max_output_file_size(), file_number,
+      preclude_last_level_min_seqno_ == kMaxSequenceNumber
+          ? preclude_last_level_min_seqno_
+          : std::min(earliest_snapshot_, preclude_last_level_min_seqno_));

  outputs.NewBuilder(tboptions);

@@ -314,6 +314,8 @@ class CompactionJob {
  // deleted because that version is not visible in any snapshot.
  std::vector<SequenceNumber> existing_snapshots_;

+  SequenceNumber earliest_snapshot_;
+
  // This is the earliest snapshot that could be used for write-conflict
  // checking by a transaction.  For any user-key newer than this snapshot, we
  // should make sure not to remove evidence that a write occurred.
@@ -4330,6 +4330,118 @@ TEST_F(CompactionPickerTest, IntraL0WhenL0IsSmall) {
  }
 }

+TEST_F(CompactionPickerTest, UniversalMaxReadAmpLargeDB) {
+  ioptions_.compaction_style = kCompactionStyleUniversal;
+  ioptions_.num_levels = 50;
+  mutable_cf_options_.RefreshDerivedOptions(ioptions_);
+  mutable_cf_options_.compaction_options_universal.size_ratio = 10;
+  mutable_cf_options_.write_buffer_size = 256 << 20;
+  // Avoid space amp compaction
+  mutable_cf_options_.compaction_options_universal
+      .max_size_amplification_percent = 200;
+  const int kMaxRuns = 8;
+  for (int max_read_amp : {kMaxRuns, 0, -1}) {
+    SCOPED_TRACE("max_read_amp = " + std::to_string(max_read_amp));
+    if (max_read_amp == -1) {
+      mutable_cf_options_.level0_file_num_compaction_trigger = kMaxRuns;
+    } else {
+      mutable_cf_options_.level0_file_num_compaction_trigger = 4;
+    }
+    mutable_cf_options_.compaction_options_universal.max_read_amp =
+        max_read_amp;
+    UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
+    uint64_t max_run_size = 20ull << 30;
+    // When max_read_amp = 0, we estimate the number of levels needed based on
+    // size_ratio and write_buffer_size. See more in
+    // UniversalCompactionBuilder::PickCompaction().
+    // With a 20GB last level, we estimate that 8 levels are needed:
+    // L0 256MB
+    // L1 256MB * 1.1 (size_ratio) = 282MB
+    // L2 (256MB + 282MB) * 1.1 = 592MB
+    // L3 1243MB
+    // L4 2610MB
+    // L5 5481MB
+    // L6 11510MB
+    // L7 24171MB > 20GB
+    for (int i = 0; i <= kMaxRuns; ++i) {
+      SCOPED_TRACE("i = " + std::to_string(i));
+      NewVersionStorage(/*num_levels=*/50, kCompactionStyleUniversal);
+      Add(/*level=*/49, /*file_number=*/10, /*smallest=*/"100",
+          /*largest=*/"200", /*file_size=*/max_run_size, /*path_id=*/0,
+          /*smallest_seq=*/0, /*largest_seq=*/0,
+          /*compensated_file_size=*/max_run_size);
+      // Besides the last sorted run, we add additional `i` sorted runs
+      // without triggering space-amp or size-amp compactions.
+      uint64_t file_size = 1 << 20;
+      for (int j = 0; j < i; ++j) {
+        Add(/*level=*/j, /*file_number=*/100 - j, /*smallest=*/"100",
+            /*largest=*/"200", /*file_size=*/file_size, /*path_id=*/0,
+            /*smallest_seq=*/100 - j, /*largest_seq=*/100 - j,
+            /*compensated_file_size=*/file_size);
+        // to avoid space-amp and size-amp compaction
+        file_size *= 2;
+      }
+      UpdateVersionStorageInfo();
+      // level0_file_num_compaction_trigger is still used as trigger to
+      // check potential compactions
+      ASSERT_EQ(
+          universal_compaction_picker.NeedsCompaction(vstorage_.get()),
+          i + 1 >= mutable_cf_options_.level0_file_num_compaction_trigger);
+      std::unique_ptr<Compaction> compaction(
+          universal_compaction_picker.PickCompaction(
+              cf_name_, mutable_cf_options_, mutable_db_options_,
+              vstorage_.get(), &log_buffer_));
+      if (i == kMaxRuns) {
+        // There are in total i + 1 > kMaxRuns sorted runs.
+        // This triggers compaction ignoring size_ratio.
+        ASSERT_NE(nullptr, compaction);
+        ASSERT_EQ(CompactionReason::kUniversalSortedRunNum,
+                  compaction->compaction_reason());
+        // First two runs are compacted
+        ASSERT_EQ(0, compaction->start_level());
+        ASSERT_EQ(1, compaction->output_level());
+        ASSERT_EQ(1U, compaction->num_input_files(0));
+        ASSERT_EQ(1U, compaction->num_input_files(1));
+      } else {
+        ASSERT_EQ(nullptr, compaction);
+      }
+    }
+  }
+}
+
+TEST_F(CompactionPickerTest, UniversalMaxReadAmpSmallDB) {
+  ioptions_.compaction_style = kCompactionStyleUniversal;
+  ioptions_.num_levels = 50;
+  mutable_cf_options_.RefreshDerivedOptions(ioptions_);
+  mutable_cf_options_.level0_file_num_compaction_trigger = 1;
+  mutable_cf_options_.compaction_options_universal.size_ratio = 10;
+  mutable_cf_options_.write_buffer_size = 256 << 20;
+  mutable_cf_options_.compaction_options_universal
+      .max_size_amplification_percent = 200;
+  const int kMaxRuns = 1;
+  for (int max_read_amp : {-1, kMaxRuns, 0}) {
+    SCOPED_TRACE("max_read_amp = " + std::to_string(max_read_amp));
+    mutable_cf_options_.compaction_options_universal.max_read_amp =
+        max_read_amp;
+    UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
+    NewVersionStorage(/*num_levels=*/50, kCompactionStyleUniversal);
+    // max_run_size is much smaller than write_buffer_size,
+    // only 1 level is needed.
+    uint64_t max_run_size = 8 << 10;
+    Add(/*level=*/49, /*file_number=*/10, /*smallest=*/"100",
+        /*largest=*/"200", /*file_size=*/max_run_size, /*path_id=*/0,
+        /*smallest_seq=*/0, /*largest_seq=*/0,
+        /*compensated_file_size=*/max_run_size);
+    UpdateVersionStorageInfo();
+    ASSERT_TRUE(universal_compaction_picker.NeedsCompaction(vstorage_.get()));
+    std::unique_ptr<Compaction> compaction(
+        universal_compaction_picker.PickCompaction(
+            cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(),
+            &log_buffer_));
+    ASSERT_EQ(nullptr, compaction);
+  }
+}
+
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -227,6 +227,7 @@ class UniversalCompactionBuilder {
  const InternalKeyComparator* icmp_;
  double score_;
  std::vector<SortedRun> sorted_runs_;
+  uint64_t max_run_size_;
  const std::string& cf_name_;
  const MutableCFOptions& mutable_cf_options_;
  const MutableDBOptions& mutable_db_options_;
@@ -235,7 +236,8 @@ class UniversalCompactionBuilder {
  LogBuffer* log_buffer_;

  static std::vector<UniversalCompactionBuilder::SortedRun> CalculateSortedRuns(
-      const VersionStorageInfo& vstorage, int last_level);
+      const VersionStorageInfo& vstorage, int last_level,
+      uint64_t* max_run_size);

  // Pick a path ID to place a newly generated file, with its estimated file
  // size.
@@ -440,11 +442,15 @@ void UniversalCompactionBuilder::SortedRun::DumpSizeInfo(

 std::vector<UniversalCompactionBuilder::SortedRun>
 UniversalCompactionBuilder::CalculateSortedRuns(
-    const VersionStorageInfo& vstorage, int last_level) {
+    const VersionStorageInfo& vstorage, int last_level,
+    uint64_t* max_run_size) {
+  assert(max_run_size);
+  *max_run_size = 0;
  std::vector<UniversalCompactionBuilder::SortedRun> ret;
  for (FileMetaData* f : vstorage.LevelFiles(0)) {
    ret.emplace_back(0, f, f->fd.GetFileSize(), f->compensated_file_size,
                     f->being_compacted);
+    *max_run_size = std::max(*max_run_size, f->fd.GetFileSize());
  }
  for (int level = 1; level <= last_level; level++) {
    uint64_t total_compensated_size = 0U;
@@ -466,6 +472,7 @@ UniversalCompactionBuilder::CalculateSortedRuns(
      ret.emplace_back(level, nullptr, total_size, total_compensated_size,
                       being_compacted);
    }
+    *max_run_size = std::max(*max_run_size, total_size);
  }
  return ret;
 }
@@ -477,13 +484,16 @@ Compaction* UniversalCompactionBuilder::PickCompaction() {
  score_ = vstorage_->CompactionScore(kLevel0);
  int max_output_level =
      vstorage_->MaxOutputLevel(ioptions_.allow_ingest_behind);
-  sorted_runs_ = CalculateSortedRuns(*vstorage_, max_output_level);
+  max_run_size_ = 0;
+  sorted_runs_ =
+      CalculateSortedRuns(*vstorage_, max_output_level, &max_run_size_);
+  int file_num_compaction_trigger =
+      mutable_cf_options_.level0_file_num_compaction_trigger;

  if (sorted_runs_.size() == 0 ||
      (vstorage_->FilesMarkedForPeriodicCompaction().empty() &&
       vstorage_->FilesMarkedForCompaction().empty() &&
-       sorted_runs_.size() < (unsigned int)mutable_cf_options_
-                                 .level0_file_num_compaction_trigger)) {
+       sorted_runs_.size() < (unsigned int)file_num_compaction_trigger)) {
    ROCKS_LOG_BUFFER(log_buffer_, "[%s] Universal: nothing to do\n",
                     cf_name_.c_str());
    TEST_SYNC_POINT_CALLBACK(
@@ -505,11 +515,9 @@ Compaction* UniversalCompactionBuilder::PickCompaction() {
    TEST_SYNC_POINT_CALLBACK("PostPickPeriodicCompaction", c);
  }

-  // Check for size amplification.
  if (c == nullptr &&
-      sorted_runs_.size() >=
-          static_cast<size_t>(
-              mutable_cf_options_.level0_file_num_compaction_trigger)) {
+      sorted_runs_.size() >= static_cast<size_t>(file_num_compaction_trigger)) {
+    // Check for size amplification.
    if ((c = PickCompactionToReduceSizeAmp()) != nullptr) {
      TEST_SYNC_POINT("PickCompactionToReduceSizeAmpReturnNonnullptr");
      ROCKS_LOG_BUFFER(log_buffer_, "[%s] Universal: compacting for size amp\n",
@@ -527,13 +535,48 @@ Compaction* UniversalCompactionBuilder::PickCompaction() {
                         cf_name_.c_str());
      } else {
        // Size amplification and file size ratios are within configured limits.
-        // If max read amplification is exceeding configured limits, then force
-        // compaction without looking at filesize ratios and try to reduce
-        // the number of files to fewer than level0_file_num_compaction_trigger.
+        // If max read amplification exceeds configured limits, then force
+        // compaction to reduce the number sorted runs without looking at file
+        // size ratios.
+
        // This is guaranteed by NeedsCompaction()
        assert(sorted_runs_.size() >=
-               static_cast<size_t>(
-                   mutable_cf_options_.level0_file_num_compaction_trigger));
+               static_cast<size_t>(file_num_compaction_trigger));
+        int max_num_runs =
+            mutable_cf_options_.compaction_options_universal.max_read_amp;
+        if (max_num_runs < 0) {
+          // any value < -1 is not valid
+          assert(max_num_runs == -1);
+          // By default, fall back to `level0_file_num_compaction_trigger`
+          max_num_runs = file_num_compaction_trigger;
+        } else if (max_num_runs == 0) {
+          if (mutable_cf_options_.compaction_options_universal.stop_style ==
+              kCompactionStopStyleTotalSize) {
+            // 0 means auto-tuning by RocksDB. We estimate max num run based on
+            // max_run_size, size_ratio and write buffer size:
+            // Assume the size of the lowest level size is equal to
+            // write_buffer_size. Each subsequent level is the max size without
+            // triggering size_ratio compaction. `max_num_runs` is the minimum
+            // number of levels required such that the target size of the
+            // largest level is at least `max_run_size_`.
+            max_num_runs = 1;
+            double cur_level_max_size =
+                static_cast<double>(mutable_cf_options_.write_buffer_size);
+            double total_run_size = 0;
+            while (cur_level_max_size < static_cast<double>(max_run_size_)) {
+              // This loop should not take too many iterations since
+              // cur_level_max_size at least doubles each iteration.
+              total_run_size += cur_level_max_size;
+              cur_level_max_size = (100.0 + ratio) / 100.0 * total_run_size;
+              ++max_num_runs;
+            }
+          } else {
+            // TODO: implement the auto-tune logic for this stop style
+            max_num_runs = file_num_compaction_trigger;
+          }
+        } else {
+          // max_num_runs > 0, it's the limit on the number of sorted run
+        }
        // Get the total number of sorted runs that are not being compacted
        int num_sr_not_compacted = 0;
        for (size_t i = 0; i < sorted_runs_.size(); i++) {
@@ -544,17 +587,25 @@ Compaction* UniversalCompactionBuilder::PickCompaction() {

        // The number of sorted runs that are not being compacted is greater
        // than the maximum allowed number of sorted runs
-        if (num_sr_not_compacted >
-            mutable_cf_options_.level0_file_num_compaction_trigger) {
-          unsigned int num_files =
-              num_sr_not_compacted -
-              mutable_cf_options_.level0_file_num_compaction_trigger + 1;
+        if (num_sr_not_compacted > max_num_runs) {
+          unsigned int num_files = num_sr_not_compacted - max_num_runs + 1;
          if ((c = PickCompactionToReduceSortedRuns(UINT_MAX, num_files)) !=
              nullptr) {
            ROCKS_LOG_BUFFER(log_buffer_,
-                             "[%s] Universal: compacting for file num -- %u\n",
-                             cf_name_.c_str(), num_files);
+                             "[%s] Universal: compacting for file num, to "
+                             "compact file num -- %u, max num runs allowed"
+                             "-- %d, max_run_size -- %" PRIu64 "\n",
+                             cf_name_.c_str(), num_files, max_num_runs,
+                             max_run_size_);
          }
+        } else {
+          ROCKS_LOG_BUFFER(
+              log_buffer_,
+              "[%s] Universal: skipping compaction for file num, num runs not "
+              "being compacted -- %u, max num runs allowed -- %d, max_run_size "
+              "-- %" PRIu64 "\n",
+              cf_name_.c_str(), num_sr_not_compacted, max_num_runs,
+              max_run_size_);
        }
      }
    }
@@ -13,6 +13,7 @@
 #include "rocksdb/iostats_context.h"
 #include "rocksdb/listener.h"
 #include "rocksdb/utilities/debug.h"
+#include "rocksdb/utilities/table_properties_collectors.h"
 #include "test_util/mock_time_env.h"
 #include "utilities/merge_operators.h"

@@ -1734,6 +1735,82 @@ TEST_P(TimedPutPrecludeLastLevelTest, PreserveTimedPutOnPenultimateLevel) {
  Close();
 }

+TEST_P(TimedPutPrecludeLastLevelTest, AutoTriggerCompaction) {
+  const int kNumTrigger = 10;
+  const int kNumLevels = 7;
+  const int kNumKeys = 200;
+
+  Options options = CurrentOptions();
+  options.compaction_style = kCompactionStyleUniversal;
+  options.preclude_last_level_data_seconds = 60;
+  options.preserve_internal_time_seconds = 0;
+  options.env = mock_env_.get();
+  options.level0_file_num_compaction_trigger = kNumTrigger;
+  options.num_levels = kNumLevels;
+  options.last_level_temperature = Temperature::kCold;
+  ConfigOptions config_options;
+  config_options.ignore_unsupported_options = false;
+  std::shared_ptr<TablePropertiesCollectorFactory> factory;
+  std::string id = CompactForTieringCollectorFactory::kClassName();
+  ASSERT_OK(TablePropertiesCollectorFactory::CreateFromString(
+      config_options, "compaction_trigger_ratio=0.4; id=" + id, &factory));
+  auto collector_factory =
+      factory->CheckedCast<CompactForTieringCollectorFactory>();
+  options.table_properties_collector_factories.push_back(factory);
+  DestroyAndReopen(options);
+  WriteOptions wo;
+  wo.protection_bytes_per_key = GetParam();
+
+  Random rnd(301);
+
+  dbfull()->TEST_WaitForPeriodicTaskRun([&] {
+    mock_clock_->MockSleepForSeconds(static_cast<int>(rnd.Uniform(10) + 1));
+  });
+
+  for (int i = 0; i < kNumKeys / 4; i++) {
+    ASSERT_OK(Put(Key(i), rnd.RandomString(100), wo));
+    dbfull()->TEST_WaitForPeriodicTaskRun([&] {
+      mock_clock_->MockSleepForSeconds(static_cast<int>(rnd.Uniform(2)));
+    });
+  }
+  // Create one file with regular Put.
+  ASSERT_OK(Flush());
+
+  // Create one file with TimedPut.
+  // These data are eligible to be put on the last level once written to db
+  // and compaction will fast track them to the last level.
+  for (int i = kNumKeys / 4; i < kNumKeys / 2; i++) {
+    ASSERT_OK(TimedPut(0, Key(i), rnd.RandomString(100), 50, wo));
+  }
+  ASSERT_OK(Flush());
+
+  // TimedPut file moved to the last level via auto triggered compaction.
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+  ASSERT_EQ("1,0,0,0,0,0,1", FilesPerLevel());
+  ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0);
+  ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
+
+  collector_factory->SetCompactionTriggerRatio(1.1);
+  for (int i = kNumKeys / 2; i < kNumKeys * 3 / 4; i++) {
+    ASSERT_OK(TimedPut(0, Key(i), rnd.RandomString(100), 50, wo));
+  }
+  ASSERT_OK(Flush());
+
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+  ASSERT_EQ("2,0,0,0,0,0,1", FilesPerLevel());
+
+  collector_factory->SetCompactionTriggerRatio(0);
+  for (int i = kNumKeys * 3 / 4; i < kNumKeys; i++) {
+    ASSERT_OK(TimedPut(0, Key(i), rnd.RandomString(100), 50, wo));
+  }
+  ASSERT_OK(Flush());
+
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+  ASSERT_EQ("3,0,0,0,0,0,1", FilesPerLevel());
+
+  Close();
+}
+
 INSTANTIATE_TEST_CASE_P(TimedPutPrecludeLastLevelTest,
                        TimedPutPrecludeLastLevelTest, ::testing::Values(0, 8));

@@ -71,6 +71,9 @@ Status VerifySstFileChecksumInternal(const Options& options,
  } else {
    return s;
  }
+  if (!s.ok()) {
+    return s;
+  }
  std::unique_ptr<TableReader> table_reader;
  std::unique_ptr<RandomAccessFileReader> file_reader(
      new RandomAccessFileReader(
@@ -26,6 +26,7 @@
 #include "rocksdb/table_properties.h"
 #include "table/block_based/block_based_table_reader.h"
 #include "table/unique_id_impl.h"
+#include "test_util/secondary_cache_test_util.h"
 #include "util/compression.h"
 #include "util/defer.h"
 #include "util/hash.h"
@@ -740,118 +741,6 @@ class LookupLiarCache : public CacheWrapper {

 }  // anonymous namespace

-TEST_F(DBBlockCacheTest, AddRedundantStats) {
-  const size_t capacity = size_t{1} << 25;
-  const int num_shard_bits = 0;  // 1 shard
-  int iterations_tested = 0;
-  for (const std::shared_ptr<Cache>& base_cache :
-       {NewLRUCache(capacity, num_shard_bits),
-        // FixedHyperClockCache
-        HyperClockCacheOptions(
-            capacity,
-            BlockBasedTableOptions().block_size /*estimated_value_size*/,
-            num_shard_bits)
-            .MakeSharedCache(),
-        // AutoHyperClockCache
-        HyperClockCacheOptions(capacity, 0 /*estimated_value_size*/,
-                               num_shard_bits)
-            .MakeSharedCache()}) {
-    if (!base_cache) {
-      // Skip clock cache when not supported
-      continue;
-    }
-    ++iterations_tested;
-    Options options = CurrentOptions();
-    options.create_if_missing = true;
-    options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
-
-    std::shared_ptr<LookupLiarCache> cache =
-        std::make_shared<LookupLiarCache>(base_cache);
-
-    BlockBasedTableOptions table_options;
-    table_options.cache_index_and_filter_blocks = true;
-    table_options.block_cache = cache;
-    table_options.filter_policy.reset(NewBloomFilterPolicy(50));
-    options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-    DestroyAndReopen(options);
-
-    // Create a new table.
-    ASSERT_OK(Put("foo", "value"));
-    ASSERT_OK(Put("bar", "value"));
-    ASSERT_OK(Flush());
-    ASSERT_EQ(1, NumTableFilesAtLevel(0));
-
-    // Normal access filter+index+data.
-    ASSERT_EQ("value", Get("foo"));
-
-    ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD));
-    ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD));
-    ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD));
-    // --------
-    ASSERT_EQ(3, TestGetTickerCount(options, BLOCK_CACHE_ADD));
-
-    ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD_REDUNDANT));
-    ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD_REDUNDANT));
-    ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD_REDUNDANT));
-    // --------
-    ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_ADD_REDUNDANT));
-
-    // Againt access filter+index+data, but force redundant load+insert on index
-    cache->SetNthLookupNotFound(2);
-    ASSERT_EQ("value", Get("bar"));
-
-    ASSERT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD));
-    ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD));
-    ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD));
-    // --------
-    ASSERT_EQ(4, TestGetTickerCount(options, BLOCK_CACHE_ADD));
-
-    ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD_REDUNDANT));
-    ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD_REDUNDANT));
-    ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD_REDUNDANT));
-    // --------
-    ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_ADD_REDUNDANT));
-
-    // Access just filter (with high probability), and force redundant
-    // load+insert
-    cache->SetNthLookupNotFound(1);
-    ASSERT_EQ("NOT_FOUND", Get("this key was not added"));
-
-    EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD));
-    EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD));
-    EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD));
-    // --------
-    EXPECT_EQ(5, TestGetTickerCount(options, BLOCK_CACHE_ADD));
-
-    EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD_REDUNDANT));
-    EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD_REDUNDANT));
-    EXPECT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD_REDUNDANT));
-    // --------
-    EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_ADD_REDUNDANT));
-
-    // Access just data, forcing redundant load+insert
-    ReadOptions read_options;
-    std::unique_ptr<Iterator> iter{db_->NewIterator(read_options)};
-    cache->SetNthLookupNotFound(1);
-    iter->SeekToFirst();
-    ASSERT_TRUE(iter->Valid());
-    ASSERT_EQ(iter->key(), "bar");
-
-    EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD));
-    EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD));
-    EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD));
-    // --------
-    EXPECT_EQ(6, TestGetTickerCount(options, BLOCK_CACHE_ADD));
-
-    EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD_REDUNDANT));
-    EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD_REDUNDANT));
-    EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD_REDUNDANT));
-    // --------
-    EXPECT_EQ(3, TestGetTickerCount(options, BLOCK_CACHE_ADD_REDUNDANT));
-  }
-  EXPECT_GE(iterations_tested, 1);
-}
-
 TEST_F(DBBlockCacheTest, ParanoidFileChecks) {
  Options options = CurrentOptions();
  options.create_if_missing = true;
@@ -1347,6 +1236,274 @@ TEST_F(DBBlockCacheTest, HyperClockCacheReportProblems) {
  EXPECT_EQ(logger->PopCounts(), (std::array<int, 3>{{0, 1, 0}}));
 }

+class DBBlockCacheTypeTest
+    : public DBBlockCacheTest,
+      public secondary_cache_test_util::WithCacheTypeParam {};
+
+INSTANTIATE_TEST_CASE_P(DBBlockCacheTypeTestInstance, DBBlockCacheTypeTest,
+                        secondary_cache_test_util::GetTestingCacheTypes());
+
+TEST_P(DBBlockCacheTypeTest, AddRedundantStats) {
+  BlockBasedTableOptions table_options;
+
+  const size_t capacity = size_t{1} << 25;
+  const int num_shard_bits = 0;  // 1 shard
+  estimated_value_size_ = table_options.block_size;
+  std::shared_ptr<Cache> base_cache =
+      NewCache(capacity, num_shard_bits, /*strict_capacity_limit=*/false);
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
+
+  std::shared_ptr<LookupLiarCache> cache =
+      std::make_shared<LookupLiarCache>(base_cache);
+
+  table_options.cache_index_and_filter_blocks = true;
+  table_options.block_cache = cache;
+  table_options.filter_policy.reset(NewBloomFilterPolicy(50));
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  DestroyAndReopen(options);
+
+  // Create a new table.
+  ASSERT_OK(Put("foo", "value"));
+  ASSERT_OK(Put("bar", "value"));
+  ASSERT_OK(Flush());
+  ASSERT_EQ(1, NumTableFilesAtLevel(0));
+
+  // Normal access filter+index+data.
+  ASSERT_EQ("value", Get("foo"));
+
+  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD));
+  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD));
+  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD));
+  // --------
+  ASSERT_EQ(3, TestGetTickerCount(options, BLOCK_CACHE_ADD));
+
+  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD_REDUNDANT));
+  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD_REDUNDANT));
+  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD_REDUNDANT));
+  // --------
+  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_ADD_REDUNDANT));
+
+  // Againt access filter+index+data, but force redundant load+insert on index
+  cache->SetNthLookupNotFound(2);
+  ASSERT_EQ("value", Get("bar"));
+
+  ASSERT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD));
+  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD));
+  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD));
+  // --------
+  ASSERT_EQ(4, TestGetTickerCount(options, BLOCK_CACHE_ADD));
+
+  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD_REDUNDANT));
+  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD_REDUNDANT));
+  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD_REDUNDANT));
+  // --------
+  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_ADD_REDUNDANT));
+
+  // Access just filter (with high probability), and force redundant
+  // load+insert
+  cache->SetNthLookupNotFound(1);
+  ASSERT_EQ("NOT_FOUND", Get("this key was not added"));
+
+  EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD));
+  EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD));
+  EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD));
+  // --------
+  EXPECT_EQ(5, TestGetTickerCount(options, BLOCK_CACHE_ADD));
+
+  EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD_REDUNDANT));
+  EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD_REDUNDANT));
+  EXPECT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD_REDUNDANT));
+  // --------
+  EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_ADD_REDUNDANT));
+
+  // Access just data, forcing redundant load+insert
+  ReadOptions read_options;
+  std::unique_ptr<Iterator> iter{db_->NewIterator(read_options)};
+  cache->SetNthLookupNotFound(1);
+  iter->SeekToFirst();
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_EQ(iter->key(), "bar");
+
+  EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD));
+  EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD));
+  EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD));
+  // --------
+  EXPECT_EQ(6, TestGetTickerCount(options, BLOCK_CACHE_ADD));
+
+  EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD_REDUNDANT));
+  EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD_REDUNDANT));
+  EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD_REDUNDANT));
+  // --------
+  EXPECT_EQ(3, TestGetTickerCount(options, BLOCK_CACHE_ADD_REDUNDANT));
+}
+
+namespace {
+std::string AltKey(int i) {
+  char buf[100];
+  snprintf(buf, sizeof(buf), "altkey%06d", i);
+  return std::string(buf);
+}
+}  // namespace
+
+TEST_P(DBBlockCacheTypeTest, Uncache) {
+  for (bool partitioned : {false, true}) {
+    SCOPED_TRACE("partitioned=" + std::to_string(partitioned));
+    for (uint32_t ua : {0, 1, 2, 10000}) {
+      SCOPED_TRACE("ua=" + std::to_string(ua));
+
+      BlockBasedTableOptions table_options;
+      Options options = CurrentOptions();
+      options.uncache_aggressiveness = ua;
+      options.create_if_missing = true;
+      // Don't allow background operations to keep Versions referenced
+      options.stats_dump_period_sec = 0;
+      options.stats_persist_period_sec = 0;
+      auto stats = ROCKSDB_NAMESPACE::CreateDBStatistics();
+      options.statistics = stats;
+
+      const size_t capacity = size_t{1} << 25;
+      const int num_shard_bits = 0;  // 1 shard
+      estimated_value_size_ = table_options.block_size;
+      std::shared_ptr<Cache> cache =
+          NewCache(capacity, num_shard_bits, /*strict_capacity_limit=*/false);
+
+      table_options.cache_index_and_filter_blocks = true;
+      table_options.block_cache = cache;
+      table_options.filter_policy.reset(NewBloomFilterPolicy(10));
+      table_options.partition_filters = partitioned;
+      table_options.index_type =
+          partitioned ? BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch
+                      : BlockBasedTableOptions::IndexType::kBinarySearch;
+      options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+      DestroyAndReopen(options);
+
+      size_t kBaselineCount = 1;  // Because of entry stats collector
+
+      ASSERT_EQ(kBaselineCount, cache->GetOccupancyCount());
+      ASSERT_EQ(0U, cache->GetUsage());
+
+      constexpr uint8_t kNumDataBlocks = 10;
+      constexpr uint8_t kNumFiles = 3;
+      for (int i = 0; i < kNumDataBlocks; i++) {
+        // Force some overlap with ordering
+        ASSERT_OK(Put(Key((i * 7) % kNumDataBlocks),
+                      Random::GetTLSInstance()->RandomBinaryString(
+                          static_cast<int>(table_options.block_size))));
+        if (i >= kNumDataBlocks - kNumFiles) {
+          ASSERT_OK(Flush());
+        }
+      }
+      ASSERT_EQ(int{kNumFiles}, NumTableFilesAtLevel(0));
+
+      for (int i = 0; i < kNumDataBlocks; i++) {
+        ASSERT_NE(Get(Key(i)), "NOT_FOUND");
+      }
+
+      size_t meta_blocks_per_file = /*index & filter*/ 2U * (1U + partitioned);
+      ASSERT_EQ(
+          cache->GetOccupancyCount(),
+          kBaselineCount + kNumDataBlocks + meta_blocks_per_file * kNumFiles);
+      ASSERT_GE(cache->GetUsage(), kNumDataBlocks * table_options.block_size);
+
+      // Combine into one file, making the originals obsolete
+      ASSERT_OK(db_->CompactRange({}, nullptr, nullptr));
+      ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork());
+
+      ASSERT_EQ(1, NumTableFilesAtLevel(1));
+
+      for (int i = 0; i < kNumDataBlocks; i++) {
+        ASSERT_NE(Get(Key(i)), "NOT_FOUND");
+      }
+
+      if (ua == 0) {
+        // Expect to see cache entries for new file and obsolete files
+        EXPECT_EQ(cache->GetOccupancyCount(),
+                  kBaselineCount + kNumDataBlocks * 2U +
+                      meta_blocks_per_file * (kNumFiles + 1));
+        EXPECT_GE(cache->GetUsage(),
+                  kNumDataBlocks * table_options.block_size * 2U);
+      } else {
+        // Expect only to see cache entries for new file
+        EXPECT_EQ(cache->GetOccupancyCount(),
+                  kBaselineCount + kNumDataBlocks + meta_blocks_per_file);
+        EXPECT_GE(cache->GetUsage(), kNumDataBlocks * table_options.block_size);
+        EXPECT_LT(cache->GetUsage(),
+                  kNumDataBlocks * table_options.block_size * 2U);
+      }
+
+      size_t alt_baseline_count = cache->GetOccupancyCount();
+      size_t alt_baseline_usage = cache->GetUsage();
+      ASSERT_OK(stats->Reset());
+      // We aren't generally cleaning up cache entries on DB::Close, especially
+      // because someone might just re-open the same DB.
+      Reopen(options);
+      for (int i = 0; i < kNumDataBlocks; i++) {
+        ASSERT_NE(Get(Key(i)), "NOT_FOUND");
+      }
+
+      EXPECT_EQ(cache->GetOccupancyCount(), alt_baseline_count);
+      EXPECT_EQ(cache->GetUsage(), alt_baseline_usage);
+
+      // Check for unnecessary unncessary cache churn
+      ASSERT_EQ(stats->getTickerCount(BLOCK_CACHE_ADD), 0U);
+      ASSERT_EQ(stats->getTickerCount(BLOCK_CACHE_MISS), 0U);
+      ASSERT_GT(stats->getTickerCount(BLOCK_CACHE_HIT), 0U);
+
+      // And now do a similar test as above except with trivial moves, making
+      // sure that we aren't falsely uncaching in that case, which would cause
+      // unnecessary cache misses. Using AltKey instead of Key to avoid
+      // interference.
+      for (int i = 0; i < kNumDataBlocks; i++) {
+        // No overlap
+        ASSERT_OK(
+            Put(AltKey(i), Random::GetTLSInstance()->RandomBinaryString(
+                               static_cast<int>(table_options.block_size))));
+        if (i >= kNumDataBlocks - kNumFiles) {
+          ASSERT_OK(Flush());
+        }
+      }
+      ASSERT_EQ(int{kNumFiles}, NumTableFilesAtLevel(0));
+
+      for (int i = 0; i < kNumDataBlocks; i++) {
+        ASSERT_NE(Get(AltKey(i)), "NOT_FOUND");
+      }
+
+      ASSERT_EQ(cache->GetOccupancyCount(),
+                alt_baseline_count + kNumDataBlocks +
+                    meta_blocks_per_file * kNumFiles);
+      ASSERT_GE(cache->GetUsage(),
+                alt_baseline_usage + kNumDataBlocks * table_options.block_size);
+
+      ASSERT_OK(stats->Reset());
+
+      // Make trivial move
+      {
+        auto a = AltKey(0);
+        auto b = AltKey(kNumDataBlocks);
+        Slice slice_a{a};
+        Slice slice_b{b};
+        ASSERT_OK(db_->CompactRange({}, &slice_a, &slice_b));
+      }
+      ASSERT_EQ(/*old*/ 1 + /*new*/ int{kNumFiles}, NumTableFilesAtLevel(1));
+
+      for (int i = 0; i < kNumDataBlocks; i++) {
+        ASSERT_NE(Get(AltKey(i)), "NOT_FOUND");
+      }
+
+      // Should be the same if trivial move
+      ASSERT_EQ(cache->GetOccupancyCount(),
+                alt_baseline_count + kNumDataBlocks +
+                    meta_blocks_per_file * kNumFiles);
+
+      // Check for unnecessary unncessary cache churn
+      ASSERT_EQ(stats->getTickerCount(BLOCK_CACHE_ADD), 0U);
+      ASSERT_EQ(stats->getTickerCount(BLOCK_CACHE_MISS), 0U);
+      ASSERT_GT(stats->getTickerCount(BLOCK_CACHE_HIT), 0U);
+    }
+  }
+}

 class DBBlockCacheKeyTest
    : public DBTestBase,
@@ -91,7 +91,7 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
  return Status::OK();
 }

-Status DBImpl::GetSortedWalFiles(VectorLogPtr& files) {
+Status DBImpl::GetSortedWalFiles(VectorWalPtr& files) {
  // Record tracked WALs as a (minimum) cross-check for directory scan
  std::vector<uint64_t> required_by_manifest;

@@ -152,11 +152,11 @@ Status DBImpl::GetSortedWalFiles(VectorLogPtr& files) {
  }

  if (s.ok()) {
-    size_t wal_size = files.size();
+    size_t wal_count = files.size();
    ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                   "Number of log files %" ROCKSDB_PRIszt " (%" ROCKSDB_PRIszt
+                   "Number of WAL files %" ROCKSDB_PRIszt " (%" ROCKSDB_PRIszt
                   " required by manifest)",
-                   wal_size, required_by_manifest.size());
+                   wal_count, required_by_manifest.size());
 #ifndef NDEBUG
    std::ostringstream wal_names;
    for (const auto& wal : files) {
@@ -177,7 +177,7 @@ Status DBImpl::GetSortedWalFiles(VectorLogPtr& files) {
  return s;
 }

-Status DBImpl::GetCurrentWalFile(std::unique_ptr<LogFile>* current_log_file) {
+Status DBImpl::GetCurrentWalFile(std::unique_ptr<WalFile>* current_log_file) {
  uint64_t current_logfile_number;
  {
    InstrumentedMutexLock l(&mutex_);
@@ -198,13 +198,14 @@ Status DBImpl::GetLiveFilesStorageInfo(
  // NOTE: This implementation was largely migrated from Checkpoint.

  Status s;
-  VectorLogPtr live_wal_files;
+  VectorWalPtr live_wal_files;
  bool flush_memtable = true;
  if (!immutable_db_options_.allow_2pc) {
    if (opts.wal_size_for_flush == std::numeric_limits<uint64_t>::max()) {
      flush_memtable = false;
    } else if (opts.wal_size_for_flush > 0) {
-      // If the outstanding log files are small, we skip the flush.
+      // FIXME: avoid querying the filesystem for current WAL state
+      // If the outstanding WAL files are small, we skip the flush.
      s = GetSortedWalFiles(live_wal_files);

      if (!s.ok()) {
@@ -229,12 +230,18 @@ Status DBImpl::GetLiveFilesStorageInfo(
  // metadata.
  mutex_.Lock();
  if (flush_memtable) {
-    Status status = FlushForGetLiveFiles();
-    if (!status.ok()) {
-      mutex_.Unlock();
-      ROCKS_LOG_ERROR(immutable_db_options_.info_log, "Cannot Flush data %s\n",
-                      status.ToString().c_str());
-      return status;
+    bool wal_locked = lock_wal_count_ > 0;
+    if (wal_locked) {
+      ROCKS_LOG_INFO(immutable_db_options_.info_log,
+                     "Can't FlushForGetLiveFiles while WAL is locked");
+    } else {
+      Status status = FlushForGetLiveFiles();
+      if (!status.ok()) {
+        mutex_.Unlock();
+        ROCKS_LOG_ERROR(immutable_db_options_.info_log,
+                        "Cannot Flush data %s\n", status.ToString().c_str());
+        return status;
+      }
    }
  }

@@ -310,6 +317,8 @@ Status DBImpl::GetLiveFilesStorageInfo(
  const uint64_t options_number = versions_->options_file_number();
  const uint64_t options_size = versions_->options_file_size_;
  const uint64_t min_log_num = MinLogNumberToKeep();
+  // Ensure consistency with manifest for track_and_verify_wals_in_manifest
+  const uint64_t max_log_num = logfile_number_;

  mutex_.Unlock();

@@ -373,10 +382,13 @@ Status DBImpl::GetLiveFilesStorageInfo(
  TEST_SYNC_POINT("CheckpointImpl::CreateCheckpoint:SavedLiveFiles2");

  if (s.ok()) {
-    // To maximize the effectiveness of track_and_verify_wals_in_manifest,
-    // sync WAL when it is enabled.
-    s = FlushWAL(
-        immutable_db_options_.track_and_verify_wals_in_manifest /* sync */);
+    // FlushWAL is required to ensure we can physically copy everything
+    // logically written to the WAL. (Sync not strictly required for
+    // active WAL to be copied rather than hard linked, even when
+    // Checkpoint guarantees that the copied-to file is sync-ed. Plus we can't
+    // help track_and_verify_wals_in_manifest after manifest_size is
+    // already determined.)
+    s = FlushWAL(/*sync=*/false);
    if (s.IsNotSupported()) {  // read-only DB or similar
      s = Status::OK();
    }
@@ -385,21 +397,51 @@ Status DBImpl::GetLiveFilesStorageInfo(
  TEST_SYNC_POINT("CheckpointImpl::CreateCustomCheckpoint:AfterGetLive1");
  TEST_SYNC_POINT("CheckpointImpl::CreateCustomCheckpoint:AfterGetLive2");

-  // If we have more than one column family, we also need to get WAL files.
+  // Even after WAL flush, there could be multiple WALs that are not
+  // fully synced. Although the output DB of a Checkpoint or Backup needs
+  // to be fully synced on return, we don't strictly need to sync this
+  // DB (the input DB). If we allow Checkpoint to hard link an inactive
+  // WAL that isn't fully synced, that could result in an insufficiently
+  // sync-ed Checkpoint. Here we get the set of WALs that are potentially
+  // unsynced or still being written to, to prevent them from being hard
+  // linked. Enforcing max_log_num from above ensures any new WALs after
+  // GetOpenWalSizes() and before GetSortedWalFiles() are not included in
+  // the results.
+  // NOTE: we might still hard link a file that is open for writing, even
+  // if we don't do any more writes to it.
+  //
+  // In a step toward reducing unnecessary file metadata queries, we also
+  // get and use our known flushed sizes for those WALs.
+  // FIXME: eventually we should not be using filesystem queries at all for
+  // the required set of WAL files.
+  //
+  // However for recycled log files, we just copy the whole file,
+  // for better or worse.
+  //
+  std::map<uint64_t, uint64_t> open_wal_number_to_size;
+  bool recycling_log_files = immutable_db_options_.recycle_log_file_num > 0;
+  if (s.ok() && !recycling_log_files) {
+    s = GetOpenWalSizes(open_wal_number_to_size);
+  }
+
+  // [old comment] If we have more than one column family, we also need to get
+  // WAL files.
  if (s.ok()) {
+    // FIXME: avoid querying the filesystem for current WAL state
    s = GetSortedWalFiles(live_wal_files);
  }
  if (!s.ok()) {
    return s;
  }

-  size_t wal_size = live_wal_files.size();
+  size_t wal_count = live_wal_files.size();
  // Link WAL files. Copy exact size of last one because it is the only one
  // that has changes after the last flush.
  auto wal_dir = immutable_db_options_.GetWalDir();
-  for (size_t i = 0; s.ok() && i < wal_size; ++i) {
+  for (size_t i = 0; s.ok() && i < wal_count; ++i) {
    if ((live_wal_files[i]->Type() == kAliveLogFile) &&
-        (!flush_memtable || live_wal_files[i]->LogNumber() >= min_log_num)) {
+        (!flush_memtable || live_wal_files[i]->LogNumber() >= min_log_num) &&
+        live_wal_files[i]->LogNumber() <= max_log_num) {
      results.emplace_back();
      LiveFileStorageInfo& info = results.back();
      auto f = live_wal_files[i]->PathName();
@@ -408,12 +450,29 @@ Status DBImpl::GetLiveFilesStorageInfo(
      info.directory = wal_dir;
      info.file_number = live_wal_files[i]->LogNumber();
      info.file_type = kWalFile;
-      info.size = live_wal_files[i]->SizeFileBytes();
-      // Trim the log either if its the last one, or log file recycling is
-      // enabled. In the latter case, a hard link doesn't prevent the file
-      // from being renamed and recycled. So we need to copy it instead.
-      info.trim_to_size = (i + 1 == wal_size) ||
-                          (immutable_db_options_.recycle_log_file_num > 0);
+      if (recycling_log_files) {
+        info.size = live_wal_files[i]->SizeFileBytes();
+        // Recyclable WAL files must be copied instead of hard linked
+        info.trim_to_size = true;
+      } else {
+        auto it = open_wal_number_to_size.find(info.file_number);
+        if (it == open_wal_number_to_size.end()) {
+          // Known fully synced and no future writes (in part from
+          // max_log_num check). Ok to hard link
+          info.size = live_wal_files[i]->SizeFileBytes();
+          assert(!info.trim_to_size);
+        } else {
+          // Marked as (possibly) still open -> use our known flushed size
+          // and force file copy instead of hard link
+          info.size = it->second;
+          info.trim_to_size = true;
+          // FIXME: this is needed as long as db_stress uses
+          // SetReadUnsyncedData(false), because it will only be able to
+          // copy the synced portion of the WAL, which under
+          // SetReadUnsyncedData(false) is given by the reported file size.
+          info.size = std::min(info.size, live_wal_files[i]->SizeFileBytes());
+        }
+      }
      if (opts.include_checksum_info) {
        info.file_checksum_func_name = kUnknownFileChecksumFuncName;
        info.file_checksum = kUnknownFileChecksum;
@@ -86,8 +86,8 @@ TEST_F(DBFlushTest, SyncFail) {
  options.env = fault_injection_env.get();

  SyncPoint::GetInstance()->LoadDependency(
-      {{"DBFlushTest::SyncFail:1", "DBImpl::SyncClosedLogs:Start"},
-       {"DBImpl::SyncClosedLogs:Failed", "DBFlushTest::SyncFail:2"}});
+      {{"DBFlushTest::SyncFail:1", "DBImpl::SyncClosedWals:Start"},
+       {"DBImpl::SyncClosedWals:Failed", "DBFlushTest::SyncFail:2"}});
  SyncPoint::GetInstance()->EnableProcessing();

  CreateAndReopenWithCF({"pikachu"}, options);
@@ -111,8 +111,8 @@ TEST_F(DBFlushTest, SyncSkip) {
  Options options = CurrentOptions();

  SyncPoint::GetInstance()->LoadDependency(
-      {{"DBFlushTest::SyncSkip:1", "DBImpl::SyncClosedLogs:Skip"},
-       {"DBImpl::SyncClosedLogs:Skip", "DBFlushTest::SyncSkip:2"}});
+      {{"DBFlushTest::SyncSkip:1", "DBImpl::SyncClosedWals:Skip"},
+       {"DBImpl::SyncClosedWals:Skip", "DBFlushTest::SyncSkip:2"}});
  SyncPoint::GetInstance()->EnableProcessing();

  Reopen(options);
@@ -2381,7 +2381,7 @@ TEST_F(DBFlushTest, PickRightMemtables) {
  SyncPoint::GetInstance()->DisableProcessing();
  SyncPoint::GetInstance()->ClearAllCallBacks();
  SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::SyncClosedLogs:BeforeReLock", [&](void* /*arg*/) {
+      "DBImpl::SyncClosedWals:BeforeReLock", [&](void* /*arg*/) {
        ASSERT_OK(db_->Put(WriteOptions(), handles_[1], "what", "v"));
        auto* cfhi =
            static_cast_with_check<ColumnFamilyHandleImpl>(handles_[1]);
@@ -637,7 +637,7 @@ Status DBImpl::CloseHelper() {
      if (!s.ok()) {
        ROCKS_LOG_WARN(
            immutable_db_options_.info_log,
-            "Unable to Sync WAL file %s with error -- %s",
+            "Unable to clear writer for WAL %s with error -- %s",
            LogFileName(immutable_db_options_.GetWalDir(), log_number).c_str(),
            s.ToString().c_str());
        // Retain the first error
@@ -1549,64 +1549,132 @@ bool DBImpl::WALBufferIsEmpty() {
  return res;
 }

+Status DBImpl::GetOpenWalSizes(std::map<uint64_t, uint64_t>& number_to_size) {
+  assert(number_to_size.empty());
+  InstrumentedMutexLock l(&log_write_mutex_);
+  for (auto& log : logs_) {
+    auto* open_file = log.writer->file();
+    if (open_file) {
+      number_to_size[log.number] = open_file->GetFlushedSize();
+    }
+  }
+  return Status::OK();
+}
+
 Status DBImpl::SyncWAL() {
  TEST_SYNC_POINT("DBImpl::SyncWAL:Begin");
-  autovector<log::Writer*, 1> logs_to_sync;
-  bool need_log_dir_sync;
-  uint64_t current_log_number;
+  WriteOptions write_options;
+  VersionEdit synced_wals;
+  Status s = SyncWalImpl(/*include_current_wal=*/true, write_options,
+                         /*job_context=*/nullptr, &synced_wals,
+                         /*error_recovery_in_prog=*/false);
+
+  if (s.ok() && synced_wals.IsWalAddition()) {
+    InstrumentedMutexLock l(&mutex_);
+    // TODO: plumb Env::IOActivity, Env::IOPriority
+    const ReadOptions read_options;
+    s = ApplyWALToManifest(read_options, write_options, &synced_wals);
+  }
+
+  TEST_SYNC_POINT("DBImpl::SyncWAL:BeforeMarkLogsSynced:2");
+  return s;
+}
+
+IOStatus DBImpl::SyncWalImpl(bool include_current_wal,
+                             const WriteOptions& write_options,
+                             JobContext* job_context, VersionEdit* synced_wals,
+                             bool error_recovery_in_prog) {
+  autovector<log::Writer*, 1> wals_to_sync;
+  bool need_wal_dir_sync;
+  // Number of a WAL that was active at the start of call and maybe is by
+  // the end of the call.
+  uint64_t maybe_active_number;
+  // Sync WALs up to this number
+  uint64_t up_to_number;

  {
    InstrumentedMutexLock l(&log_write_mutex_);
    assert(!logs_.empty());

-    // This SyncWAL() call only cares about logs up to this number.
-    current_log_number = logfile_number_;
+    maybe_active_number = logfile_number_;
+    up_to_number =
+        include_current_wal ? maybe_active_number : maybe_active_number - 1;

-    while (logs_.front().number <= current_log_number &&
-           logs_.front().IsSyncing()) {
+    while (logs_.front().number <= up_to_number && logs_.front().IsSyncing()) {
      log_sync_cv_.Wait();
    }
    // First check that logs are safe to sync in background.
+    if (include_current_wal &&
+        !logs_.back().writer->file()->writable_file()->IsSyncThreadSafe()) {
+      return IOStatus::NotSupported(
+          "SyncWAL() is not supported for this implementation of WAL file",
+          immutable_db_options_.allow_mmap_writes
+              ? "try setting Options::allow_mmap_writes to false"
+              : Slice());
+    }
    for (auto it = logs_.begin();
-         it != logs_.end() && it->number <= current_log_number; ++it) {
-      if (!it->writer->file()->writable_file()->IsSyncThreadSafe()) {
-        return Status::NotSupported(
-            "SyncWAL() is not supported for this implementation of WAL file",
-            immutable_db_options_.allow_mmap_writes
-                ? "try setting Options::allow_mmap_writes to false"
-                : Slice());
+         it != logs_.end() && it->number <= up_to_number; ++it) {
+      auto& log = *it;
+      // Ensure the head of logs_ is marked as getting_synced if any is.
+      log.PrepareForSync();
+      // If last sync failed on a later WAL, this could be a fully synced
+      // and closed WAL that just needs to be recorded as synced in the
+      // manifest.
+      if (log.writer->file()) {
+        wals_to_sync.push_back(log.writer);
      }
    }
-    for (auto it = logs_.begin();
-         it != logs_.end() && it->number <= current_log_number; ++it) {
-      auto& log = *it;
-      log.PrepareForSync();
-      logs_to_sync.push_back(log.writer);
-    }

-    need_log_dir_sync = !log_dir_synced_;
+    need_wal_dir_sync = !log_dir_synced_;
  }

-  TEST_SYNC_POINT("DBWALTest::SyncWALNotWaitWrite:1");
+  if (include_current_wal) {
+    TEST_SYNC_POINT("DBWALTest::SyncWALNotWaitWrite:1");
+  }
  RecordTick(stats_, WAL_FILE_SYNCED);
-  Status status;
-  IOStatus io_s;
-  // TODO: plumb Env::IOActivity, Env::IOPriority
-  const ReadOptions read_options;
-  const WriteOptions write_options;
  IOOptions opts;
-  io_s = WritableFileWriter::PrepareIOOptions(write_options, opts);
-  if (!io_s.ok()) {
-    status = io_s;
-  }
+  IOStatus io_s = WritableFileWriter::PrepareIOOptions(write_options, opts);
+  std::list<log::Writer*> wals_internally_closed;
  if (io_s.ok()) {
-    for (log::Writer* log : logs_to_sync) {
-      io_s =
-          log->file()->SyncWithoutFlush(opts, immutable_db_options_.use_fsync);
+    for (log::Writer* log : wals_to_sync) {
+      if (job_context) {
+        ROCKS_LOG_INFO(immutable_db_options_.info_log,
+                       "[JOB %d] Syncing log #%" PRIu64, job_context->job_id,
+                       log->get_log_number());
+      }
+      if (error_recovery_in_prog) {
+        log->file()->reset_seen_error();
+      }
+      if (log->get_log_number() >= maybe_active_number) {
+        assert(log->get_log_number() == maybe_active_number);
+        io_s = log->file()->SyncWithoutFlush(opts,
+                                             immutable_db_options_.use_fsync);
+      } else {
+        io_s = log->file()->Sync(opts, immutable_db_options_.use_fsync);
+      }
      if (!io_s.ok()) {
-        status = io_s;
        break;
      }
+      // WALs can be closed when purging obsolete files, but if recycling is
+      // enabled, the log file is closed here so that it can be reused. And
+      // immediate closure here upon final sync makes it easier to guarantee
+      // that Checkpoint doesn't LinkFile on a WAL still open for write, which
+      // might be unsupported for some FileSystem implementations. Close here
+      // should be inexpensive because flush and sync are done, so the kill
+      // switch background_close_inactive_wals is expected to be removed in
+      // the future.
+      if (log->get_log_number() < maybe_active_number &&
+          (immutable_db_options_.recycle_log_file_num > 0 ||
+           !immutable_db_options_.background_close_inactive_wals)) {
+        if (error_recovery_in_prog) {
+          log->file()->reset_seen_error();
+        }
+        io_s = log->file()->Close(opts);
+        wals_internally_closed.push_back(log);
+        if (!io_s.ok()) {
+          break;
+        }
+      }
    }
  }
  if (!io_s.ok()) {
@@ -1616,31 +1684,34 @@ Status DBImpl::SyncWAL() {
    // future writes
    IOStatusCheck(io_s);
  }
-  if (status.ok() && need_log_dir_sync) {
-    status = directories_.GetWalDir()->FsyncWithDirOptions(
+  if (io_s.ok() && need_wal_dir_sync) {
+    io_s = directories_.GetWalDir()->FsyncWithDirOptions(
        IOOptions(), nullptr,
        DirFsyncOptions(DirFsyncOptions::FsyncReason::kNewFileSynced));
  }
-  TEST_SYNC_POINT("DBWALTest::SyncWALNotWaitWrite:2");
+  if (include_current_wal) {
+    TEST_SYNC_POINT("DBWALTest::SyncWALNotWaitWrite:2");

-  TEST_SYNC_POINT("DBImpl::SyncWAL:BeforeMarkLogsSynced:1");
-  VersionEdit synced_wals;
+    TEST_SYNC_POINT("DBImpl::SyncWAL:BeforeMarkLogsSynced:1");
+  } else {
+    TEST_SYNC_POINT_CALLBACK("DBImpl::SyncClosedWals:BeforeReLock",
+                             /*arg=*/nullptr);
+  }
  {
    InstrumentedMutexLock l(&log_write_mutex_);
-    if (status.ok()) {
-      MarkLogsSynced(current_log_number, need_log_dir_sync, &synced_wals);
+    for (auto* wal : wals_internally_closed) {
+      // We can only modify the state of log::Writer under the mutex
+      bool was_closed = wal->PublishIfClosed();
+      assert(was_closed);
+      (void)was_closed;
+    }
+    if (io_s.ok()) {
+      MarkLogsSynced(up_to_number, need_wal_dir_sync, synced_wals);
    } else {
-      MarkLogsNotSynced(current_log_number);
+      MarkLogsNotSynced(up_to_number);
    }
  }
-  if (status.ok() && synced_wals.IsWalAddition()) {
-    InstrumentedMutexLock l(&mutex_);
-    status = ApplyWALToManifest(read_options, write_options, &synced_wals);
-  }
-
-  TEST_SYNC_POINT("DBImpl::SyncWAL:BeforeMarkLogsSynced:2");
-
-  return status;
+  return io_s;
 }

 Status DBImpl::ApplyWALToManifest(const ReadOptions& read_options,
@@ -1758,16 +1829,19 @@ void DBImpl::MarkLogsSynced(uint64_t up_to, bool synced_dir,
          wal.GetPreSyncSize() > 0) {
        synced_wals->AddWal(wal.number, WalMetadata(wal.GetPreSyncSize()));
      }
-      // Check if the file has been closed, i.e wal.writer->file() == nullptr
-      // which can happen if log recycling is enabled, or if all the data in
-      // the log has been synced
+      // Reclaim closed WALs (wal.writer->file() == nullptr), and if we don't
+      // need to close before that (background_close_inactive_wals) we can
+      // opportunistically reclaim WALs that happen to be fully synced.
+      // (Probably not worth extra code and mutex release to opportunistically
+      // close WALs that became eligible since last holding the mutex.
+      // FindObsoleteFiles can take care of it.)
      if (wal.writer->file() == nullptr ||
-          wal.GetPreSyncSize() == wal.writer->file()->GetFlushedSize()) {
+          (immutable_db_options_.background_close_inactive_wals &&
+           wal.GetPreSyncSize() == wal.writer->file()->GetFlushedSize())) {
        // Fully synced
        logs_to_free_.push_back(wal.ReleaseWriter());
        it = logs_.erase(it);
      } else {
-        assert(wal.GetPreSyncSize() < wal.writer->file()->GetFlushedSize());
        wal.FinishSync();
        ++it;
      }
@@ -4469,8 +4543,11 @@ bool DBImpl::GetAggregatedIntProperty(const Slice& property,
  if (property_info == nullptr || property_info->handle_int == nullptr) {
    return false;
  }
+  auto aggregator = CreateIntPropertyAggregator(property);
+  if (aggregator == nullptr) {
+    return false;
+  }

-  uint64_t sum = 0;
  bool ret = true;
  {
    // Needs mutex to protect the list of column families.
@@ -4484,14 +4561,14 @@ bool DBImpl::GetAggregatedIntProperty(const Slice& property,
      // GetIntPropertyInternal may release db mutex and re-acquire it.
      mutex_.AssertHeld();
      if (ret) {
-        sum += value;
+        aggregator->Add(cfd, value);
      } else {
        ret = false;
        break;
      }
    }
  }
-  *aggregated_value = sum;
+  *aggregated_value = aggregator->Aggregate();
  return ret;
 }

@@ -9,6 +9,7 @@
 #pragma once

 #include <atomic>
+#include <cstdint>
 #include <deque>
 #include <functional>
 #include <limits>
@@ -57,6 +58,7 @@
 #include "rocksdb/status.h"
 #include "rocksdb/trace_reader_writer.h"
 #include "rocksdb/transaction_log.h"
+#include "rocksdb/user_write_callback.h"
 #include "rocksdb/utilities/replayer.h"
 #include "rocksdb/write_buffer_manager.h"
 #include "table/merging_iterator.h"
@@ -231,6 +233,10 @@ class DBImpl : public DB {
  using DB::Write;
  Status Write(const WriteOptions& options, WriteBatch* updates) override;

+  using DB::WriteWithCallback;
+  Status WriteWithCallback(const WriteOptions& options, WriteBatch* updates,
+                           UserWriteCallback* user_write_cb) override;
+
  using DB::Get;
  Status Get(const ReadOptions& _read_options,
             ColumnFamilyHandle* column_family, const Slice& key,
@@ -501,8 +507,14 @@ class DBImpl : public DB {
  // All the returned filenames start with "/"
  Status GetLiveFiles(std::vector<std::string>&, uint64_t* manifest_file_size,
                      bool flush_memtable = true) override;
-  Status GetSortedWalFiles(VectorLogPtr& files) override;
-  Status GetCurrentWalFile(std::unique_ptr<LogFile>* current_log_file) override;
+  Status GetSortedWalFiles(VectorWalPtr& files) override;
+  // Get the known flushed sizes of WALs that might still be written to
+  // or have pending sync.
+  // NOTE: unlike alive_log_files_, this function includes WALs that might
+  // be obsolete (but not obsolete to a pending Checkpoint) and not yet fully
+  // synced.
+  Status GetOpenWalSizes(std::map<uint64_t, uint64_t>& number_to_size);
+  Status GetCurrentWalFile(std::unique_ptr<WalFile>* current_log_file) override;
  Status GetCreationTimeOfOldestFile(uint64_t* creation_time) override;

  Status GetUpdatesSince(
@@ -688,7 +700,8 @@ class DBImpl : public DB {
  // thread to determine whether it is safe to perform the write.
  virtual Status WriteWithCallback(const WriteOptions& write_options,
                                   WriteBatch* my_batch,
-                                   WriteCallback* callback);
+                                   WriteCallback* callback,
+                                   UserWriteCallback* user_write_cb = nullptr);

  // Returns the sequence number that is guaranteed to be smaller than or equal
  // to the sequence number of any key that could be inserted into the current
@@ -1447,6 +1460,9 @@ class DBImpl : public DB {
  Status RenameTempFileToOptionsFile(const std::string& file_name);
  Status DeleteObsoleteOptionsFiles();

+  void NotifyOnManualFlushScheduled(autovector<ColumnFamilyData*> cfds,
+                                    FlushReason flush_reason);
+
  void NotifyOnFlushBegin(ColumnFamilyData* cfd, FileMetaData* file_meta,
                          const MutableCFOptions& mutable_cf_options,
                          int job_id, FlushReason flush_reason);
@@ -1497,6 +1513,7 @@ class DBImpl : public DB {
  // batch that does not have duplicate keys.
  Status WriteImpl(const WriteOptions& options, WriteBatch* updates,
                   WriteCallback* callback = nullptr,
+                   UserWriteCallback* user_write_cb = nullptr,
                   uint64_t* log_used = nullptr, uint64_t log_ref = 0,
                   bool disable_memtable = false, uint64_t* seq_used = nullptr,
                   size_t batch_cnt = 0,
@@ -1505,6 +1522,7 @@ class DBImpl : public DB {

  Status PipelinedWriteImpl(const WriteOptions& options, WriteBatch* updates,
                            WriteCallback* callback = nullptr,
+                            UserWriteCallback* user_write_cb = nullptr,
                            uint64_t* log_used = nullptr, uint64_t log_ref = 0,
                            bool disable_memtable = false,
                            uint64_t* seq_used = nullptr);
@@ -1531,7 +1549,8 @@ class DBImpl : public DB {
  // marks start of a new sub-batch.
  Status WriteImplWALOnly(
      WriteThread* write_thread, const WriteOptions& options,
-      WriteBatch* updates, WriteCallback* callback, uint64_t* log_used,
+      WriteBatch* updates, WriteCallback* callback,
+      UserWriteCallback* user_write_cb, uint64_t* log_used,
      const uint64_t log_ref, uint64_t* seq_used, const size_t sub_batch_cnt,
      PreReleaseCallback* pre_release_callback, const AssignOrder assign_order,
      const PublishLastSeq publish_last_seq, const bool disable_memtable);
@@ -1703,8 +1722,11 @@ class DBImpl : public DB {
      return w;
    }
    Status ClearWriter() {
-      // TODO: plumb Env::IOActivity, Env::IOPriority
-      Status s = writer->WriteBuffer(WriteOptions());
+      Status s;
+      if (writer->file()) {
+        // TODO: plumb Env::IOActivity, Env::IOPriority
+        s = writer->WriteBuffer(WriteOptions());
+      }
      delete writer;
      writer = nullptr;
      return s;
@@ -1719,10 +1741,16 @@ class DBImpl : public DB {

    void PrepareForSync() {
      assert(!getting_synced);
-      // Size is expected to be monotonically increasing.
-      assert(writer->file()->GetFlushedSize() >= pre_sync_size);
+      // Ensure the head of logs_ is marked as getting_synced if any is.
      getting_synced = true;
-      pre_sync_size = writer->file()->GetFlushedSize();
+      // If last sync failed on a later WAL, this could be a fully synced
+      // and closed WAL that just needs to be recorded as synced in the
+      // manifest.
+      if (writer->file()) {
+        // Size is expected to be monotonically increasing.
+        assert(writer->file()->GetFlushedSize() >= pre_sync_size);
+        pre_sync_size = writer->file()->GetFlushedSize();
+      }
    }

    void FinishSync() {
@@ -1920,7 +1948,7 @@ class DBImpl : public DB {
  void ReleaseFileNumberFromPendingOutputs(
      std::unique_ptr<std::list<uint64_t>::iterator>& v);

-  IOStatus SyncClosedLogs(const WriteOptions& write_options,
+  IOStatus SyncClosedWals(const WriteOptions& write_options,
                          JobContext* job_context, VersionEdit* synced_wals,
                          bool error_recovery_in_prog);

@@ -2054,17 +2082,22 @@ class DBImpl : public DB {
      mutex_.Lock();
    }

-    if (!immutable_db_options_.unordered_write) {
-      // Then the writes are finished before the next write group starts
-      return;
+    if (immutable_db_options_.unordered_write) {
+      // Wait for the ones who already wrote to the WAL to finish their
+      // memtable write.
+      if (pending_memtable_writes_.load() != 0) {
+        // XXX: suspicious wait while holding DB mutex?
+        std::unique_lock<std::mutex> guard(switch_mutex_);
+        switch_cv_.wait(guard,
+                        [&] { return pending_memtable_writes_.load() == 0; });
+      }
+    } else {
+      // (Writes are finished before the next write group starts.)
    }

-    // Wait for the ones who already wrote to the WAL to finish their
-    // memtable write.
-    if (pending_memtable_writes_.load() != 0) {
-      std::unique_lock<std::mutex> guard(switch_mutex_);
-      switch_cv_.wait(guard,
-                      [&] { return pending_memtable_writes_.load() == 0; });
+    // Wait for any LockWAL to clear
+    while (lock_wal_count_ > 0) {
+      bg_cv_.Wait();
    }
  }

@@ -2174,7 +2207,8 @@ class DBImpl : public DB {
  void GenerateFlushRequest(const autovector<ColumnFamilyData*>& cfds,
                            FlushReason flush_reason, FlushRequest* req);

-  void SchedulePendingFlush(const FlushRequest& req);
+  // Returns true if `req` is successfully enqueued.
+  bool SchedulePendingFlush(const FlushRequest& req);

  void SchedulePendingCompaction(ColumnFamilyData* cfd);
  void SchedulePendingPurge(std::string fname, std::string dir_to_sync,
@@ -2250,6 +2284,11 @@ class DBImpl : public DB {
  ColumnFamilyData* PickCompactionFromQueue(
      std::unique_ptr<TaskLimiterToken>* token, LogBuffer* log_buffer);

+  IOStatus SyncWalImpl(bool include_current_wal,
+                       const WriteOptions& write_options,
+                       JobContext* job_context, VersionEdit* synced_wals,
+                       bool error_recovery_in_prog);
+
  // helper function to call after some of the logs_ were synced
  void MarkLogsSynced(uint64_t up_to, bool synced_dir, VersionEdit* edit);
  Status ApplyWALToManifest(const ReadOptions& read_options,
@@ -2522,8 +2561,10 @@ class DBImpl : public DB {

  bool persistent_stats_cfd_exists_ = true;

-  // alive_log_files_ is protected by mutex_ and log_write_mutex_ with details
-  // as follows:
+  // The current WAL file and those that have not been found obsolete from
+  // memtable flushes. A WAL not on this list might still be pending writer
+  // flush and/or sync and close and might still be in logs_. alive_log_files_
+  // is protected by mutex_ and log_write_mutex_ with details as follows:
  // 1. read by FindObsoleteFiles() which can be called in either application
  //    thread or RocksDB bg threads, both mutex_ and log_write_mutex_ are
  //    held.
@@ -2574,7 +2615,7 @@ class DBImpl : public DB {
  // 8. read by MarkLogsNotSynced() and MarkLogsSynced() are protected by
  //    log_write_mutex_.
  // 9. erase() by MarkLogsSynced() protected by log_write_mutex_.
-  // 10. read by SyncClosedLogs() protected by only log_write_mutex_. This can
+  // 10. read by SyncClosedWals() protected by only log_write_mutex_. This can
  //     happen in bg flush threads after DB::Open() returns success to
  //     applications.
  // 11. reads, e.g. front(), iteration, and back() called by PreprocessWrite()
@@ -2587,7 +2628,7 @@ class DBImpl : public DB {
  // 13. emplace_back() by SwitchMemtable() hold both mutex_ and
  //     log_write_mutex_. This happens in the write group leader. Can conflict
  //     with bg threads calling FindObsoleteFiles(), MarkLogsSynced(),
-  //     SyncClosedLogs(), etc. as well as application threads calling
+  //     SyncClosedWals(), etc. as well as application threads calling
  //     FlushWAL(), SyncWAL(), LockWAL(). This is fine because all parties
  //     require at least log_write_mutex_.
  // 14. iteration called in WriteToWAL(write_group) protected by
@@ -87,6 +87,9 @@ bool DBImpl::ShouldRescheduleFlushRequestToRetainUDT(
  mutex_.AssertHeld();
  assert(flush_req.cfd_to_max_mem_id_to_persist.size() == 1);
  ColumnFamilyData* cfd = flush_req.cfd_to_max_mem_id_to_persist.begin()->first;
+  if (cfd->GetAndClearFlushSkipReschedule()) {
+    return false;
+  }
  uint64_t max_memtable_id =
      flush_req.cfd_to_max_mem_id_to_persist.begin()->second;
  if (cfd->IsDropped() ||
@@ -98,15 +101,20 @@ bool DBImpl::ShouldRescheduleFlushRequestToRetainUDT(
  // alleviated if we continue with the flush instead of postponing it.
  const auto& mutable_cf_options = *cfd->GetLatestMutableCFOptions();

-  // Taking the status of the active Memtable into consideration so that we are
-  // not just checking if DB is currently already in write stall mode.
-  int mem_to_flush = cfd->mem()->ApproximateMemoryUsageFast() >=
-                             cfd->mem()->write_buffer_size() / 2
-                         ? 1
-                         : 0;
+  // Use the same criteria as WaitUntilFlushWouldNotStallWrites does w.r.t
+  // defining what a write stall is about to happen means. If this uses a
+  // stricter criteria, for example, a write stall is about to happen if the
+  // last memtable is 10% full, there is a possibility that manual flush could
+  // be waiting in `WaitUntilFlushWouldNotStallWrites` with the incorrect
+  // expectation that others will clear up the excessive memtables and
+  // eventually let it proceed. The others in this case won't start clearing
+  // until the last memtable is 10% full. To avoid that scenario, the criteria
+  // this uses should be the same or less strict than
+  // `WaitUntilFlushWouldNotStallWrites` does.
  WriteStallCondition write_stall =
      ColumnFamilyData::GetWriteStallConditionAndCause(
-          cfd->imm()->NumNotFlushed() + mem_to_flush, /*num_l0_files=*/0,
+          cfd->GetUnflushedMemTableCountForWriteStallCheck(),
+          /*num_l0_files=*/0,
          /*num_compaction_needed_bytes=*/0, mutable_cf_options,
          *cfd->ioptions())
          .first;
@@ -116,89 +124,19 @@ bool DBImpl::ShouldRescheduleFlushRequestToRetainUDT(
  return true;
 }

-IOStatus DBImpl::SyncClosedLogs(const WriteOptions& write_options,
+IOStatus DBImpl::SyncClosedWals(const WriteOptions& write_options,
                                JobContext* job_context,
                                VersionEdit* synced_wals,
                                bool error_recovery_in_prog) {
-  TEST_SYNC_POINT("DBImpl::SyncClosedLogs:Start");
-  InstrumentedMutexLock l(&log_write_mutex_);
-  autovector<log::Writer*, 1> logs_to_sync;
-  uint64_t current_log_number = logfile_number_;
-  while (logs_.front().number < current_log_number &&
-         logs_.front().IsSyncing()) {
-    log_sync_cv_.Wait();
+  TEST_SYNC_POINT("DBImpl::SyncClosedWals:Start");
+
+  IOStatus io_s = SyncWalImpl(/*include_current_wal*/ false, write_options,
+                              job_context, synced_wals, error_recovery_in_prog);
+  if (!io_s.ok()) {
+    TEST_SYNC_POINT("DBImpl::SyncClosedWals:Failed");
+  } else {
+    TEST_SYNC_POINT("DBImpl::SyncClosedWals:end");
  }
-  for (auto it = logs_.begin();
-       it != logs_.end() && it->number < current_log_number; ++it) {
-    auto& log = *it;
-    log.PrepareForSync();
-    logs_to_sync.push_back(log.writer);
-  }
-
-  IOStatus io_s;
-  if (!logs_to_sync.empty()) {
-    log_write_mutex_.Unlock();
-
-    assert(job_context);
-
-    for (log::Writer* log : logs_to_sync) {
-      ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                     "[JOB %d] Syncing log #%" PRIu64, job_context->job_id,
-                     log->get_log_number());
-      if (error_recovery_in_prog) {
-        log->file()->reset_seen_error();
-      }
-
-      IOOptions io_options;
-      io_s = WritableFileWriter::PrepareIOOptions(write_options, io_options);
-      if (!io_s.ok()) {
-        break;
-      }
-      io_s = log->file()->Sync(io_options, immutable_db_options_.use_fsync);
-      if (!io_s.ok()) {
-        break;
-      }
-
-      if (immutable_db_options_.recycle_log_file_num > 0) {
-        if (error_recovery_in_prog) {
-          log->file()->reset_seen_error();
-        }
-        // Normally the log file is closed when purging obsolete file, but if
-        // log recycling is enabled, the log file is closed here so that it
-        // can be reused.
-        io_s = log->Close(write_options);
-        if (!io_s.ok()) {
-          break;
-        }
-      }
-    }
-    if (io_s.ok()) {
-      IOOptions io_options;
-      io_s = WritableFileWriter::PrepareIOOptions(write_options, io_options);
-      if (io_s.ok()) {
-        io_s = directories_.GetWalDir()->FsyncWithDirOptions(
-            io_options, nullptr,
-            DirFsyncOptions(DirFsyncOptions::FsyncReason::kNewFileSynced));
-      }
-    }
-
-    TEST_SYNC_POINT_CALLBACK("DBImpl::SyncClosedLogs:BeforeReLock",
-                             /*arg=*/nullptr);
-    log_write_mutex_.Lock();
-
-    // "number <= current_log_number - 1" is equivalent to
-    // "number < current_log_number".
-    if (io_s.ok()) {
-      MarkLogsSynced(current_log_number - 1, true, synced_wals);
-    } else {
-      MarkLogsNotSynced(current_log_number - 1);
-    }
-    if (!io_s.ok()) {
-      TEST_SYNC_POINT("DBImpl::SyncClosedLogs:Failed");
-      return io_s;
-    }
-  }
-  TEST_SYNC_POINT("DBImpl::SyncClosedLogs:end");
  return io_s;
 }

@@ -237,12 +175,12 @@ Status DBImpl::FlushMemTableToOutputFile(
  // If needs_to_sync_closed_wals is true, we need to record the current
  // maximum memtable ID of this column family so that a later PickMemtables()
  // call will not pick memtables whose IDs are higher. This is due to the fact
-  // that SyncClosedLogs() may release the db mutex, and memtable switch can
+  // that SyncClosedWals() may release the db mutex, and memtable switch can
  // happen for this column family in the meantime. The newly created memtables
  // have their data backed by unsynced WALs, thus they cannot be included in
  // this flush job.
  // Another reason why we must record the current maximum memtable ID of this
-  // column family: SyncClosedLogs() may release db mutex, thus it's possible
+  // column family: SyncClosedWals() may release db mutex, thus it's possible
  // for application to continue to insert into memtables increasing db's
  // sequence number. The application may take a snapshot, but this snapshot is
  // not included in `snapshot_seqs` which will be passed to flush job because
@@ -256,7 +194,7 @@ Status DBImpl::FlushMemTableToOutputFile(

  // If needs_to_sync_closed_wals is false, then the flush job will pick ALL
  // existing memtables of the column family when PickMemTable() is called
-  // later. Although we won't call SyncClosedLogs() in this case, we may still
+  // later. Although we won't call SyncClosedWals() in this case, we may still
  // call the callbacks of the listeners, i.e. NotifyOnFlushBegin() which also
  // releases and re-acquires the db mutex. In the meantime, the application
  // can still insert into the memtables and increase the db's sequence number.
@@ -286,12 +224,12 @@ Status DBImpl::FlushMemTableToOutputFile(
  bool need_cancel = false;
  IOStatus log_io_s = IOStatus::OK();
  if (needs_to_sync_closed_wals) {
-    // SyncClosedLogs() may unlock and re-lock the log_write_mutex multiple
+    // SyncClosedWals() may unlock and re-lock the log_write_mutex multiple
    // times.
    VersionEdit synced_wals;
    bool error_recovery_in_prog = error_handler_.IsRecoveryInProgress();
    mutex_.Unlock();
-    log_io_s = SyncClosedLogs(write_options, job_context, &synced_wals,
+    log_io_s = SyncClosedWals(write_options, job_context, &synced_wals,
                              error_recovery_in_prog);
    mutex_.Lock();
    if (log_io_s.ok() && synced_wals.IsWalAddition()) {
@@ -306,7 +244,7 @@ Status DBImpl::FlushMemTableToOutputFile(
      error_handler_.SetBGError(log_io_s, BackgroundErrorReason::kFlush);
    }
  } else {
-    TEST_SYNC_POINT("DBImpl::SyncClosedLogs:Skip");
+    TEST_SYNC_POINT("DBImpl::SyncClosedWals:Skip");
  }
  s = log_io_s;

@@ -580,7 +518,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
    VersionEdit synced_wals;
    bool error_recovery_in_prog = error_handler_.IsRecoveryInProgress();
    mutex_.Unlock();
-    log_io_s = SyncClosedLogs(write_options, job_context, &synced_wals,
+    log_io_s = SyncClosedWals(write_options, job_context, &synced_wals,
                              error_recovery_in_prog);
    mutex_.Lock();
    if (log_io_s.ok() && synced_wals.IsWalAddition()) {
@@ -2310,6 +2248,23 @@ void DBImpl::GenerateFlushRequest(const autovector<ColumnFamilyData*>& cfds,
  }
 }

+void DBImpl::NotifyOnManualFlushScheduled(autovector<ColumnFamilyData*> cfds,
+                                          FlushReason flush_reason) {
+  if (immutable_db_options_.listeners.size() == 0U) {
+    return;
+  }
+  if (shutting_down_.load(std::memory_order_acquire)) {
+    return;
+  }
+  std::vector<ManualFlushInfo> info;
+  for (ColumnFamilyData* cfd : cfds) {
+    info.push_back({cfd->GetID(), cfd->GetName(), flush_reason});
+  }
+  for (const auto& listener : immutable_db_options_.listeners) {
+    listener->OnManualFlushScheduled(this, info);
+  }
+}
+
 Status DBImpl::FlushMemTable(ColumnFamilyData* cfd,
                             const FlushOptions& flush_options,
                             FlushReason flush_reason,
@@ -2414,7 +2369,14 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd,
        }
      }
      for (const auto& req : flush_reqs) {
-        SchedulePendingFlush(req);
+        assert(req.cfd_to_max_mem_id_to_persist.size() == 1);
+        ColumnFamilyData* loop_cfd =
+            req.cfd_to_max_mem_id_to_persist.begin()->first;
+        bool already_queued_for_flush = loop_cfd->queued_for_flush();
+        bool flush_req_enqueued = SchedulePendingFlush(req);
+        if (already_queued_for_flush || flush_req_enqueued) {
+          loop_cfd->SetFlushSkipReschedule();
+        }
      }
      MaybeScheduleFlushOrCompaction();
    }
@@ -2426,6 +2388,8 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd,
      }
    }
  }
+
+  NotifyOnManualFlushScheduled({cfd}, flush_reason);
  TEST_SYNC_POINT("DBImpl::FlushMemTable:AfterScheduleFlush");
  TEST_SYNC_POINT("DBImpl::FlushMemTable:BeforeWaitForBgFlush");
  if (s.ok() && flush_options.wait) {
@@ -2570,6 +2534,7 @@ Status DBImpl::AtomicFlushMemTables(
      }
    }
  }
+  NotifyOnManualFlushScheduled(cfds, flush_reason);
  TEST_SYNC_POINT("DBImpl::AtomicFlushMemTables:AfterScheduleFlush");
  TEST_SYNC_POINT("DBImpl::AtomicFlushMemTables:BeforeWaitForBgFlush");
  if (s.ok() && flush_options.wait) {
@@ -2627,7 +2592,9 @@ Status DBImpl::RetryFlushesForErrorRecovery(FlushReason flush_reason,
          flush_reason,
          {{cfd,
            std::numeric_limits<uint64_t>::max() /* max_mem_id_to_persist */}}};
-      SchedulePendingFlush(flush_req);
+      if (SchedulePendingFlush(flush_req)) {
+        cfd->SetFlushSkipReschedule();
+      };
    }
  }
  MaybeScheduleFlushOrCompaction();
@@ -2715,13 +2682,13 @@ Status DBImpl::WaitUntilFlushWouldNotStallWrites(ColumnFamilyData* cfd,
      // mode due to pending compaction bytes, but that's less common
      // No extra immutable Memtable will be created if the current Memtable is
      // empty.
-      int mem_to_flush = cfd->mem()->IsEmpty() ? 0 : 1;
-      write_stall_condition = ColumnFamilyData::GetWriteStallConditionAndCause(
-                                  cfd->imm()->NumNotFlushed() + mem_to_flush,
-                                  vstorage->l0_delay_trigger_count() + 1,
-                                  vstorage->estimated_compaction_needed_bytes(),
-                                  mutable_cf_options, *cfd->ioptions())
-                                  .first;
+      write_stall_condition =
+          ColumnFamilyData::GetWriteStallConditionAndCause(
+              cfd->GetUnflushedMemTableCountForWriteStallCheck(),
+              vstorage->l0_delay_trigger_count() + 1,
+              vstorage->estimated_compaction_needed_bytes(), mutable_cf_options,
+              *cfd->ioptions())
+              .first;
    } while (write_stall_condition != WriteStallCondition::kNormal);
  }
  return Status::OK();
@@ -3033,13 +3000,14 @@ ColumnFamilyData* DBImpl::PickCompactionFromQueue(
  return cfd;
 }

-void DBImpl::SchedulePendingFlush(const FlushRequest& flush_req) {
+bool DBImpl::SchedulePendingFlush(const FlushRequest& flush_req) {
  mutex_.AssertHeld();
+  bool enqueued = false;
  if (reject_new_background_jobs_) {
-    return;
+    return enqueued;
  }
  if (flush_req.cfd_to_max_mem_id_to_persist.empty()) {
-    return;
+    return enqueued;
  }
  if (!immutable_db_options_.atomic_flush) {
    // For the non-atomic flush case, we never schedule multiple column
@@ -3054,6 +3022,7 @@ void DBImpl::SchedulePendingFlush(const FlushRequest& flush_req) {
      cfd->set_queued_for_flush(true);
      ++unscheduled_flushes_;
      flush_queue_.push_back(flush_req);
+      enqueued = true;
    }
  } else {
    for (auto& iter : flush_req.cfd_to_max_mem_id_to_persist) {
@@ -3062,7 +3031,9 @@ void DBImpl::SchedulePendingFlush(const FlushRequest& flush_req) {
    }
    ++unscheduled_flushes_;
    flush_queue_.push_back(flush_req);
+    enqueued = true;
  }
+  return enqueued;
 }

 void DBImpl::SchedulePendingCompaction(ColumnFamilyData* cfd) {
@@ -312,6 +312,26 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
        // logs_ could have changed while we were waiting.
        continue;
      }
+      // This WAL file is not live, so it's OK if we never sync the rest of it.
+      // If it's already closed, then it's been fully synced. If
+      // !background_close_inactive_wals then we need to Close it before
+      // removing from logs_ but not blocking while holding log_write_mutex_.
+      if (!immutable_db_options_.background_close_inactive_wals &&
+          log.writer->file()) {
+        // We are taking ownership of and pinning the front entry, so we can
+        // expect it to be the same after releasing and re-acquiring the lock
+        log.PrepareForSync();
+        log_write_mutex_.Unlock();
+        // TODO: maybe check the return value of Close.
+        // TODO: plumb Env::IOActivity, Env::IOPriority
+        auto s = log.writer->file()->Close({});
+        s.PermitUncheckedError();
+        log_write_mutex_.Lock();
+        log.writer->PublishIfClosed();
+        assert(&log == &logs_.front());
+        log.FinishSync();
+        log_sync_cv_.SignalAll();
+      }
      logs_to_free_.push_back(log.ReleaseWriter());
      logs_.pop_front();
    }
@@ -410,12 +430,24 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
      state.manifest_delete_files.size());
  // We may ignore the dbname when generating the file names.
  for (auto& file : state.sst_delete_files) {
-    if (!file.only_delete_metadata) {
-      candidate_files.emplace_back(
-          MakeTableFileName(file.metadata->fd.GetNumber()), file.path);
-    }
-    if (file.metadata->table_reader_handle) {
-      table_cache_->Release(file.metadata->table_reader_handle);
+    auto* handle = file.metadata->table_reader_handle;
+    if (file.only_delete_metadata) {
+      if (handle) {
+        // Simply release handle of file that is not being deleted
+        table_cache_->Release(handle);
+      }
+    } else {
+      // File is being deleted (actually obsolete)
+      auto number = file.metadata->fd.GetNumber();
+      candidate_files.emplace_back(MakeTableFileName(number), file.path);
+      if (handle == nullptr) {
+        // For files not "pinned" in table cache
+        handle = TableCache::Lookup(table_cache_.get(), number);
+      }
+      if (handle) {
+        TableCache::ReleaseObsolete(table_cache_.get(), handle,
+                                    file.uncache_aggressiveness);
+      }
    }
    file.DeleteMetadata();
  }
@@ -491,7 +523,7 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
  for (const auto w : state.logs_to_free) {
    // TODO: maybe check the return value of Close.
    // TODO: plumb Env::IOActivity, Env::IOPriority
-    auto s = w->Close(WriteOptions());
+    auto s = w->Close({});
    s.PermitUncheckedError();
  }

@@ -577,8 +609,6 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
    std::string fname;
    std::string dir_to_sync;
    if (type == kTableFile) {
-      // evict from cache
-      TableCache::Evict(table_cache_.get(), number);
      fname = MakeTableFileName(candidate_file.file_path, number);
      dir_to_sync = candidate_file.file_path;
    } else if (type == kBlobFile) {
@@ -5,6 +5,7 @@

 #include "db/db_impl/db_impl_follower.h"

+#include <algorithm>
 #include <cinttypes>

 #include "db/arena_wrapped_db_iter.h"
@@ -301,7 +301,10 @@ Status DBImpl::NewDB(std::vector<std::string>* new_filenames) {
  }
  if (immutable_db_options_.write_dbid_to_manifest) {
    std::string temp_db_id;
-    GetDbIdentityFromIdentityFile(&temp_db_id);
+    s = GetDbIdentityFromIdentityFile(&temp_db_id);
+    if (!s.ok()) {
+      return s;
+    }
    new_db.SetDBId(temp_db_id);
  }
  new_db.SetLogNumber(0);
@@ -1670,6 +1673,8 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
      SequenceNumber earliest_write_conflict_snapshot;
      std::vector<SequenceNumber> snapshot_seqs =
          snapshots_.GetAll(&earliest_write_conflict_snapshot);
+      SequenceNumber earliest_snapshot =
+          (snapshot_seqs.empty() ? kMaxSequenceNumber : snapshot_seqs.at(0));
      auto snapshot_checker = snapshot_checker_.get();
      if (use_custom_gc_ && snapshot_checker == nullptr) {
        snapshot_checker = DisableGCSnapshotChecker::Instance();
@@ -1689,6 +1694,7 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
      IOStatus io_s;
      const ReadOptions read_option(Env::IOActivity::kDBOpen);
      const WriteOptions write_option(Env::IO_HIGH, Env::IOActivity::kDBOpen);
+
      TableBuilderOptions tboptions(
          *cfd->ioptions(), mutable_cf_options, read_option, write_option,
          cfd->internal_comparator(), cfd->internal_tbl_prop_coll_factories(),
@@ -1697,21 +1703,22 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
          0 /* level */, false /* is_bottommost */,
          TableFileCreationReason::kRecovery, 0 /* oldest_key_time */,
          0 /* file_creation_time */, db_id_, db_session_id_,
-          0 /* target_file_size */, meta.fd.GetNumber());
+          0 /* target_file_size */, meta.fd.GetNumber(), kMaxSequenceNumber);
      Version* version = cfd->current();
      version->Ref();
      uint64_t num_input_entries = 0;
-      s = BuildTable(
-          dbname_, versions_.get(), immutable_db_options_, tboptions,
-          file_options_for_compaction_, cfd->table_cache(), iter.get(),
-          std::move(range_del_iters), &meta, &blob_file_additions,
-          snapshot_seqs, earliest_write_conflict_snapshot, kMaxSequenceNumber,
-          snapshot_checker, paranoid_file_checks, cfd->internal_stats(), &io_s,
-          io_tracer_, BlobFileCreationReason::kRecovery,
-          nullptr /* seqno_to_time_mapping */, &event_logger_, job_id,
-          nullptr /* table_properties */, write_hint,
-          nullptr /*full_history_ts_low*/, &blob_callback_, version,
-          &num_input_entries);
+      s = BuildTable(dbname_, versions_.get(), immutable_db_options_, tboptions,
+                     file_options_for_compaction_, cfd->table_cache(),
+                     iter.get(), std::move(range_del_iters), &meta,
+                     &blob_file_additions, snapshot_seqs, earliest_snapshot,
+                     earliest_write_conflict_snapshot, kMaxSequenceNumber,
+                     snapshot_checker, paranoid_file_checks,
+                     cfd->internal_stats(), &io_s, io_tracer_,
+                     BlobFileCreationReason::kRecovery,
+                     nullptr /* seqno_to_time_mapping */, &event_logger_,
+                     job_id, nullptr /* table_properties */, write_hint,
+                     nullptr /*full_history_ts_low*/, &blob_callback_, version,
+                     &num_input_entries);
      version->Unref();
      LogFlush(immutable_db_options_.info_log);
      ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
@@ -155,21 +155,36 @@ Status DBImpl::Write(const WriteOptions& write_options, WriteBatch* my_batch) {
  }
  if (s.ok()) {
    s = WriteImpl(write_options, my_batch, /*callback=*/nullptr,
+                  /*user_write_cb=*/nullptr,
                  /*log_used=*/nullptr);
  }
  return s;
 }

 Status DBImpl::WriteWithCallback(const WriteOptions& write_options,
-                                 WriteBatch* my_batch,
-                                 WriteCallback* callback) {
+                                 WriteBatch* my_batch, WriteCallback* callback,
+                                 UserWriteCallback* user_write_cb) {
  Status s;
  if (write_options.protection_bytes_per_key > 0) {
    s = WriteBatchInternal::UpdateProtectionInfo(
        my_batch, write_options.protection_bytes_per_key);
  }
  if (s.ok()) {
-    s = WriteImpl(write_options, my_batch, callback, nullptr);
+    s = WriteImpl(write_options, my_batch, callback, user_write_cb);
+  }
+  return s;
+}
+
+Status DBImpl::WriteWithCallback(const WriteOptions& write_options,
+                                 WriteBatch* my_batch,
+                                 UserWriteCallback* user_write_cb) {
+  Status s;
+  if (write_options.protection_bytes_per_key > 0) {
+    s = WriteBatchInternal::UpdateProtectionInfo(
+        my_batch, write_options.protection_bytes_per_key);
+  }
+  if (s.ok()) {
+    s = WriteImpl(write_options, my_batch, /*callback=*/nullptr, user_write_cb);
  }
  return s;
 }
@@ -179,9 +194,9 @@ Status DBImpl::WriteWithCallback(const WriteOptions& write_options,
 // published sequence.
 Status DBImpl::WriteImpl(const WriteOptions& write_options,
                         WriteBatch* my_batch, WriteCallback* callback,
-                         uint64_t* log_used, uint64_t log_ref,
-                         bool disable_memtable, uint64_t* seq_used,
-                         size_t batch_cnt,
+                         UserWriteCallback* user_write_cb, uint64_t* log_used,
+                         uint64_t log_ref, bool disable_memtable,
+                         uint64_t* seq_used, size_t batch_cnt,
                         PreReleaseCallback* pre_release_callback,
                         PostMemTableCallback* post_memtable_callback) {
  assert(!seq_per_batch_ || batch_cnt != 0);
@@ -268,6 +283,10 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
    return Status::NotSupported(
        "seq_per_batch currently does not honor post_memtable_callback");
  }
+  if (my_batch->HasDeleteRange() && immutable_db_options_.row_cache) {
+    return Status::NotSupported(
+        "DeleteRange is not compatible with row cache.");
+  }
  // Otherwise IsLatestPersistentState optimization does not make sense
  assert(!WriteBatchInternal::IsLatestPersistentState(my_batch) ||
         disable_memtable);
@@ -284,10 +303,10 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
        seq_per_batch_ ? kDoAssignOrder : kDontAssignOrder;
    // Otherwise it is WAL-only Prepare batches in WriteCommitted policy and
    // they don't consume sequence.
-    return WriteImplWALOnly(&nonmem_write_thread_, write_options, my_batch,
-                            callback, log_used, log_ref, seq_used, batch_cnt,
-                            pre_release_callback, assign_order,
-                            kDontPublishLastSeq, disable_memtable);
+    return WriteImplWALOnly(
+        &nonmem_write_thread_, write_options, my_batch, callback, user_write_cb,
+        log_used, log_ref, seq_used, batch_cnt, pre_release_callback,
+        assign_order, kDontPublishLastSeq, disable_memtable);
  }

  if (immutable_db_options_.unordered_write) {
@@ -299,9 +318,9 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
    // Use a write thread to i) optimize for WAL write, ii) publish last
    // sequence in in increasing order, iii) call pre_release_callback serially
    Status status = WriteImplWALOnly(
-        &write_thread_, write_options, my_batch, callback, log_used, log_ref,
-        &seq, sub_batch_cnt, pre_release_callback, kDoAssignOrder,
-        kDoPublishLastSeq, disable_memtable);
+        &write_thread_, write_options, my_batch, callback, user_write_cb,
+        log_used, log_ref, &seq, sub_batch_cnt, pre_release_callback,
+        kDoAssignOrder, kDoPublishLastSeq, disable_memtable);
    TEST_SYNC_POINT("DBImpl::WriteImpl:UnorderedWriteAfterWriteWAL");
    if (!status.ok()) {
      return status;
@@ -318,17 +337,20 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
  }

  if (immutable_db_options_.enable_pipelined_write) {
-    return PipelinedWriteImpl(write_options, my_batch, callback, log_used,
-                              log_ref, disable_memtable, seq_used);
+    return PipelinedWriteImpl(write_options, my_batch, callback, user_write_cb,
+                              log_used, log_ref, disable_memtable, seq_used);
  }

  PERF_TIMER_GUARD(write_pre_and_post_process_time);
-  WriteThread::Writer w(write_options, my_batch, callback, log_ref,
-                        disable_memtable, batch_cnt, pre_release_callback,
-                        post_memtable_callback);
+  WriteThread::Writer w(write_options, my_batch, callback, user_write_cb,
+                        log_ref, disable_memtable, batch_cnt,
+                        pre_release_callback, post_memtable_callback);
  StopWatch write_sw(immutable_db_options_.clock, stats_, DB_WRITE);

  write_thread_.JoinBatchGroup(&w);
+  if (w.state == WriteThread::STATE_PARALLEL_MEMTABLE_CALLER) {
+    write_thread_.SetMemWritersEachStride(&w);
+  }
  if (w.state == WriteThread::STATE_PARALLEL_MEMTABLE_WRITER) {
    // we are a non-leader in a parallel group

@@ -679,6 +701,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,

 Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
                                  WriteBatch* my_batch, WriteCallback* callback,
+                                  UserWriteCallback* user_write_cb,
                                  uint64_t* log_used, uint64_t log_ref,
                                  bool disable_memtable, uint64_t* seq_used) {
  PERF_TIMER_GUARD(write_pre_and_post_process_time);
@@ -686,8 +709,8 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,

  WriteContext write_context;

-  WriteThread::Writer w(write_options, my_batch, callback, log_ref,
-                        disable_memtable, /*_batch_cnt=*/0,
+  WriteThread::Writer w(write_options, my_batch, callback, user_write_cb,
+                        log_ref, disable_memtable, /*_batch_cnt=*/0,
                        /*_pre_release_callback=*/nullptr);
  write_thread_.JoinBatchGroup(&w);
  TEST_SYNC_POINT("DBImplWrite::PipelinedWriteImpl:AfterJoinBatchGroup");
@@ -826,7 +849,9 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
    // so we need to set its status to pass ASSERT_STATUS_CHECKED
    memtable_write_group.status.PermitUncheckedError();
  }
-
+  if (w.state == WriteThread::STATE_PARALLEL_MEMTABLE_CALLER) {
+    write_thread_.SetMemWritersEachStride(&w);
+  }
  if (w.state == WriteThread::STATE_PARALLEL_MEMTABLE_WRITER) {
    PERF_TIMER_STOP(write_pre_and_post_process_time);
    PERF_TIMER_FOR_WAIT_GUARD(write_memtable_time);
@@ -866,7 +891,8 @@ Status DBImpl::UnorderedWriteMemtable(const WriteOptions& write_options,
  PERF_TIMER_GUARD(write_pre_and_post_process_time);
  StopWatch write_sw(immutable_db_options_.clock, stats_, DB_WRITE);

-  WriteThread::Writer w(write_options, my_batch, callback, log_ref,
+  WriteThread::Writer w(write_options, my_batch, callback,
+                        /*user_write_cb=*/nullptr, log_ref,
                        false /*disable_memtable*/);

  if (w.CheckCallback(this) && w.ShouldWriteToMemtable()) {
@@ -916,13 +942,15 @@ Status DBImpl::UnorderedWriteMemtable(const WriteOptions& write_options,
 // applicable in a two-queue setting.
 Status DBImpl::WriteImplWALOnly(
    WriteThread* write_thread, const WriteOptions& write_options,
-    WriteBatch* my_batch, WriteCallback* callback, uint64_t* log_used,
+    WriteBatch* my_batch, WriteCallback* callback,
+    UserWriteCallback* user_write_cb, uint64_t* log_used,
    const uint64_t log_ref, uint64_t* seq_used, const size_t sub_batch_cnt,
    PreReleaseCallback* pre_release_callback, const AssignOrder assign_order,
    const PublishLastSeq publish_last_seq, const bool disable_memtable) {
  PERF_TIMER_GUARD(write_pre_and_post_process_time);
-  WriteThread::Writer w(write_options, my_batch, callback, log_ref,
-                        disable_memtable, sub_batch_cnt, pre_release_callback);
+  WriteThread::Writer w(write_options, my_batch, callback, user_write_cb,
+                        log_ref, disable_memtable, sub_batch_cnt,
+                        pre_release_callback);
  StopWatch write_sw(immutable_db_options_.clock, stats_, DB_WRITE);

  write_thread->JoinBatchGroup(&w);
@@ -1489,6 +1517,11 @@ IOStatus DBImpl::WriteToWAL(const WriteThread::WriteGroup& write_group,
    RecordTick(stats_, WAL_FILE_BYTES, log_size);
    stats->AddDBStats(InternalStats::kIntStatsWriteWithWal, write_with_wal);
    RecordTick(stats_, WRITE_WITH_WAL, write_with_wal);
+    for (auto* writer : write_group) {
+      if (!writer->CallbackFailed()) {
+        writer->CheckPostWalWriteCallback();
+      }
+    }
  }
  return io_s;
 }
@@ -1553,6 +1586,11 @@ IOStatus DBImpl::ConcurrentWriteToWAL(
    stats->AddDBStats(InternalStats::kIntStatsWriteWithWal, write_with_wal,
                      concurrent);
    RecordTick(stats_, WRITE_WITH_WAL, write_with_wal);
+    for (auto* writer : write_group) {
+      if (!writer->CallbackFailed()) {
+        writer->CheckPostWalWriteCallback();
+      }
+    }
  }
  return io_s;
 }
@@ -2156,6 +2194,8 @@ void DBImpl::NotifyOnMemTableSealed(ColumnFamilyData* /*cfd*/,
 // two_write_queues_ is true (This is to simplify the reasoning.)
 Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
  mutex_.AssertHeld();
+  assert(lock_wal_count_ == 0);
+
  // TODO: plumb Env::IOActivity, Env::IOPriority
  const ReadOptions read_options;
  const WriteOptions write_options;
@@ -2199,6 +2239,11 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
  memtable_info.earliest_seqno = cfd->mem()->GetEarliestSequenceNumber();
  memtable_info.num_entries = cfd->mem()->num_entries();
  memtable_info.num_deletes = cfd->mem()->num_deletes();
+  if (!cfd->ioptions()->persist_user_defined_timestamps &&
+      cfd->user_comparator()->timestamp_size() > 0) {
+    const Slice& newest_udt = cfd->mem()->GetNewestUDT();
+    memtable_info.newest_udt.assign(newest_udt.data(), newest_udt.size());
+  }
  // Log this later after lock release. It may be outdated, e.g., if background
  // flush happens before logging, but that should be ok.
  int num_imm_unflushed = cfd->imm()->NumNotFlushed();
@@ -2218,14 +2263,15 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
    SequenceNumber seq = versions_->LastSequence();
    new_mem = cfd->ConstructNewMemtable(mutable_cf_options, seq);
    context->superversion_context.NewSuperVersion();
+
+    ROCKS_LOG_INFO(immutable_db_options_.info_log,
+                   "[%s] New memtable created with log file: #%" PRIu64
+                   ". Immutable memtables: %d.\n",
+                   cfd->GetName().c_str(), new_log_number, num_imm_unflushed);
+    // There should be no concurrent write as the thread is at the front of
+    // writer queue
+    cfd->mem()->ConstructFragmentedRangeTombstones();
  }
-  ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                 "[%s] New memtable created with log file: #%" PRIu64
-                 ". Immutable memtables: %d.\n",
-                 cfd->GetName().c_str(), new_log_number, num_imm_unflushed);
-  // There should be no concurrent write as the thread is at the front of
-  // writer queue
-  cfd->mem()->ConstructFragmentedRangeTombstones();

  mutex_.Lock();
  if (recycle_log_number != 0) {
@@ -192,6 +192,49 @@ TEST_F(DBPropertiesTest, GetAggregatedIntPropertyTest) {
  }
 }

+TEST_F(DBPropertiesTest, AggregateBlockCacheProperty) {
+  constexpr size_t kCapacity = 1000;
+  LRUCacheOptions co;
+  co.capacity = kCapacity;
+  co.num_shard_bits = 0;
+  co.metadata_charge_policy = kDontChargeCacheMetadata;
+  auto block_cache = NewLRUCache(co);
+
+  // All columns families share the same block cache.
+  Options options = CurrentOptions();
+  BlockBasedTableOptions table_opt;
+  table_opt.no_block_cache = false;
+  table_opt.block_cache = block_cache;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_opt));
+
+  CreateAndReopenWithCF({"one", "two", "three", "four"}, options);
+
+  // Insert unpinned block to the cache
+  constexpr size_t kSize1 = 100;
+  ASSERT_OK(block_cache->Insert("block1", nullptr /*value*/,
+                                &kNoopCacheItemHelper, kSize1));
+  // Insert pinned block to the cache
+  constexpr size_t kSize2 = 200;
+  Cache::Handle* block2 = nullptr;
+  ASSERT_OK(block_cache->Insert("block2", nullptr /*value*/,
+                                &kNoopCacheItemHelper, kSize2, &block2));
+
+  uint64_t value;
+  ASSERT_TRUE(db_->GetAggregatedIntProperty(DB::Properties::kBlockCacheCapacity,
+                                            &value));
+  ASSERT_EQ(value, kCapacity);
+
+  ASSERT_TRUE(
+      db_->GetAggregatedIntProperty(DB::Properties::kBlockCacheUsage, &value));
+  ASSERT_EQ(value, kSize1 + kSize2);
+
+  ASSERT_TRUE(db_->GetAggregatedIntProperty(
+      DB::Properties::kBlockCachePinnedUsage, &value));
+  ASSERT_EQ(value, kSize2);
+
+  block_cache->Release(block2);
+}
+
 namespace {
 void VerifySimilar(uint64_t a, uint64_t b, double bias) {
  ASSERT_EQ(a == 0U, b == 0U);
@@ -1083,7 +1126,6 @@ TEST_F(DBPropertiesTest, EstimateCompressionRatio) {
  ASSERT_GT(CompressionRatioAtLevel(1), 10.0);
 }

-
 class CountingUserTblPropCollector : public TablePropertiesCollector {
 public:
  const char* Name() const override { return "CountingUserTblPropCollector"; }
@@ -2366,7 +2408,6 @@ TEST_F(DBPropertiesTest, TableMetaIndexKeys) {
  } while (ChangeOptions());
 }

-
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -3820,6 +3820,10 @@ TEST_F(DBRangeDelTest, RowCache) {
  ASSERT_OK(wb.DeleteRange(Key(1), Key(5)));
  ASSERT_TRUE(db_->Write(WriteOptions(), &wb).IsNotSupported());
  ASSERT_EQ(Get(Key(3)), "val");
+  // By default, memtable insertion failure will turn the DB to read-only mode.
+  // The check for delete range should happen before that to fail early
+  // and should not turn db into read-only mdoe.
+  ASSERT_OK(Put(Key(5), "foo"));
 }
 }  // namespace ROCKSDB_NAMESPACE

@@ -5645,6 +5645,8 @@ TEST_F(DBTest, DynamicUniversalCompactionOptions) {
  ASSERT_EQ(
      dbfull()->GetOptions().compaction_options_universal.allow_trivial_move,
      false);
+  ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.max_read_amp,
+            -1);

  ASSERT_OK(dbfull()->SetOptions(
      {{"compaction_options_universal", "{size_ratio=7;}"}}));
@@ -5666,9 +5668,11 @@ TEST_F(DBTest, DynamicUniversalCompactionOptions) {
  ASSERT_EQ(
      dbfull()->GetOptions().compaction_options_universal.allow_trivial_move,
      false);
+  ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.max_read_amp,
+            -1);

-  ASSERT_OK(dbfull()->SetOptions(
-      {{"compaction_options_universal", "{min_merge_width=11;}"}}));
+  ASSERT_OK(dbfull()->SetOptions({{"compaction_options_universal",
+                                   "{min_merge_width=11;max_read_amp=0;}"}}));
  ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.size_ratio, 7u);
  ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.min_merge_width,
            11u);
@@ -5687,6 +5691,8 @@ TEST_F(DBTest, DynamicUniversalCompactionOptions) {
  ASSERT_EQ(
      dbfull()->GetOptions().compaction_options_universal.allow_trivial_move,
      false);
+  ASSERT_EQ(dbfull()->GetOptions().compaction_options_universal.max_read_amp,
+            0);
 }

 TEST_F(DBTest, FileCreationRandomFailure) {
@@ -6347,6 +6353,8 @@ TEST_F(DBTest, PromoteL0) {
  Options options = CurrentOptions();
  options.disable_auto_compactions = true;
  options.write_buffer_size = 10 * 1024 * 1024;
+  // Exercise what was a use-after-free (ASAN failure) under ~VersionSet()
+  options.uncache_aggressiveness = 300;
  DestroyAndReopen(options);

  // non overlapping ranges
@@ -4154,7 +4154,7 @@ TEST_F(DBTest2, LiveFilesOmitObsoleteFiles) {
  TEST_SYNC_POINT("DBTest2::LiveFilesOmitObsoleteFiles:FlushTriggered");

  ASSERT_OK(db_->DisableFileDeletions());
-  VectorLogPtr log_files;
+  VectorWalPtr log_files;
  ASSERT_OK(db_->GetSortedWalFiles(log_files));
  TEST_SYNC_POINT("DBTest2::LiveFilesOmitObsoleteFiles:LiveFilesCaptured");
  for (const auto& log_file : log_files) {
@@ -276,16 +276,16 @@ class SpecialEnv : public EnvWrapper {
      SpecialEnv* env_;
      std::unique_ptr<WritableFile> base_;
    };
-    class WalFile : public WritableFile {
+    class SpecialWalFile : public WritableFile {
     public:
-      WalFile(SpecialEnv* env, std::unique_ptr<WritableFile>&& b)
+      SpecialWalFile(SpecialEnv* env, std::unique_ptr<WritableFile>&& b)
          : env_(env), base_(std::move(b)) {
        env_->num_open_wal_file_.fetch_add(1);
      }
-      virtual ~WalFile() { env_->num_open_wal_file_.fetch_add(-1); }
+      virtual ~SpecialWalFile() { env_->num_open_wal_file_.fetch_add(-1); }
      Status Append(const Slice& data) override {
 #if !(defined NDEBUG) || !defined(OS_WIN)
-        TEST_SYNC_POINT("SpecialEnv::WalFile::Append:1");
+        TEST_SYNC_POINT("SpecialEnv::SpecialWalFile::Append:1");
 #endif
        Status s;
        if (env_->log_write_error_.load(std::memory_order_acquire)) {
@@ -299,7 +299,7 @@ class SpecialEnv : public EnvWrapper {
          s = base_->Append(data);
        }
 #if !(defined NDEBUG) || !defined(OS_WIN)
-        TEST_SYNC_POINT("SpecialEnv::WalFile::Append:2");
+        TEST_SYNC_POINT("SpecialEnv::SpecialWalFile::Append:2");
 #endif
        return s;
      }
@@ -419,7 +419,7 @@ class SpecialEnv : public EnvWrapper {
      } else if (strstr(f.c_str(), "MANIFEST") != nullptr) {
        r->reset(new ManifestFile(this, std::move(*r)));
      } else if (strstr(f.c_str(), "log") != nullptr) {
-        r->reset(new WalFile(this, std::move(*r)));
+        r->reset(new SpecialWalFile(this, std::move(*r)));
      } else {
        r->reset(new OtherFile(this, std::move(*r)));
      }
@@ -14,6 +14,7 @@
 #include "port/stack_trace.h"
 #include "rocksdb/file_system.h"
 #include "test_util/sync_point.h"
+#include "util/defer.h"
 #include "util/udt_util.h"
 #include "utilities/fault_injection_env.h"
 #include "utilities/fault_injection_fs.h"
@@ -270,8 +271,10 @@ TEST_F(DBWALTest, SyncWALNotWaitWrite) {
  ASSERT_OK(Put("foo3", "bar3"));

  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({
-      {"SpecialEnv::WalFile::Append:1", "DBWALTest::SyncWALNotWaitWrite:1"},
-      {"DBWALTest::SyncWALNotWaitWrite:2", "SpecialEnv::WalFile::Append:2"},
+      {"SpecialEnv::SpecialWalFile::Append:1",
+       "DBWALTest::SyncWALNotWaitWrite:1"},
+      {"DBWALTest::SyncWALNotWaitWrite:2",
+       "SpecialEnv::SpecialWalFile::Append:2"},
  });
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();

@@ -1469,6 +1472,93 @@ TEST_F(DBWALTest, SyncMultipleLogs) {
  ASSERT_OK(dbfull()->SyncWAL());
 }

+TEST_F(DBWALTest, SyncWalPartialFailure) {
+  class MyTestFileSystem : public FileSystemWrapper {
+   public:
+    explicit MyTestFileSystem(std::shared_ptr<FileSystem> base)
+        : FileSystemWrapper(std::move(base)) {}
+
+    static const char* kClassName() { return "MyTestFileSystem"; }
+    const char* Name() const override { return kClassName(); }
+    IOStatus NewWritableFile(const std::string& fname,
+                             const FileOptions& file_opts,
+                             std::unique_ptr<FSWritableFile>* result,
+                             IODebugContext* dbg) override {
+      IOStatus s = target()->NewWritableFile(fname, file_opts, result, dbg);
+      if (s.ok()) {
+        *result =
+            std::make_unique<MyTestWritableFile>(std::move(*result), *this);
+      }
+      return s;
+    }
+
+    AcqRelAtomic<uint32_t> syncs_before_failure_{UINT32_MAX};
+
+   protected:
+    class MyTestWritableFile : public FSWritableFileOwnerWrapper {
+     public:
+      MyTestWritableFile(std::unique_ptr<FSWritableFile>&& file,
+                         MyTestFileSystem& fs)
+          : FSWritableFileOwnerWrapper(std::move(file)), fs_(fs) {}
+
+      IOStatus Sync(const IOOptions& options, IODebugContext* dbg) override {
+        int prev_val = fs_.syncs_before_failure_.FetchSub(1);
+        if (prev_val == 0) {
+          return IOStatus::IOError("fault");
+        } else {
+          return target()->Sync(options, dbg);
+        }
+      }
+
+     protected:
+      MyTestFileSystem& fs_;
+    };
+  };
+
+  Options options = CurrentOptions();
+  options.max_write_buffer_number = 4;
+  options.track_and_verify_wals_in_manifest = true;
+  options.max_bgerror_resume_count = 0;  // manual resume
+
+  auto custom_fs =
+      std::make_shared<MyTestFileSystem>(options.env->GetFileSystem());
+  std::unique_ptr<Env> fault_fs_env(NewCompositeEnv(custom_fs));
+  options.env = fault_fs_env.get();
+  Reopen(options);
+  Defer closer([this]() { Close(); });
+
+  // This is the simplest way to get
+  // * one inactive WAL, synced
+  // * one inactive WAL, not synced, and
+  // * one active WAL, not synced
+  // with a single thread, to exercise as much logic as we reasonably can.
+  ASSERT_OK(static_cast_with_check<DBImpl>(db_)->PauseBackgroundWork());
+  ASSERT_OK(Put("key1", "val1"));
+  ASSERT_OK(static_cast_with_check<DBImpl>(db_)->TEST_SwitchMemtable());
+  ASSERT_OK(db_->SyncWAL());
+  ASSERT_OK(Put("key2", "val2"));
+  ASSERT_OK(static_cast_with_check<DBImpl>(db_)->TEST_SwitchMemtable());
+  ASSERT_OK(Put("key3", "val3"));
+
+  // Allow 1 of the WALs to sync, but another won't
+  custom_fs->syncs_before_failure_.Store(1);
+  ASSERT_NOK(db_->SyncWAL());
+
+  // Stuck in this state. (This could previously cause a segfault.)
+  ASSERT_NOK(db_->SyncWAL());
+
+  // Can't Resume because WAL write failure is considered non-recoverable,
+  // regardless of the IOStatus itself. (Can/should be fixed?)
+  ASSERT_NOK(db_->Resume());
+
+  // Verify no data loss after reopen.
+  // Also Close() could previously crash in this state.
+  Reopen(options);
+  ASSERT_EQ("val1", Get("key1"));
+  ASSERT_EQ("val2", Get("key2"));
+  ASSERT_EQ("val3", Get("key3"));
+}
+
 // Github issue 1339. Prior the fix we read sequence id from the first log to
 // a local variable, then keep increase the variable as we replay logs,
 // ignoring actual sequence id of the records. This is incorrect if some writes
@@ -666,11 +666,25 @@ TEST_P(DBWriteTest, LockWALInEffect) {
  // try the 1st WAL created during open
  ASSERT_OK(Put("key0", "value"));
  ASSERT_NE(options.manual_wal_flush, dbfull()->WALBufferIsEmpty());
+
  ASSERT_OK(db_->LockWAL());
+
  ASSERT_TRUE(dbfull()->WALBufferIsEmpty());
+  uint64_t wal_num = dbfull()->TEST_GetCurrentLogNumber();
+  // Manual flush with wait=false should abruptly fail with TryAgain
+  FlushOptions flush_opts;
+  flush_opts.wait = false;
+  for (bool allow_write_stall : {true, false}) {
+    flush_opts.allow_write_stall = allow_write_stall;
+    ASSERT_TRUE(db_->Flush(flush_opts).IsTryAgain());
+  }
+  ASSERT_EQ(wal_num, dbfull()->TEST_GetCurrentLogNumber());
+
  ASSERT_OK(db_->UnlockWAL());
-  // try the 2nd wal created during SwitchWAL
+
+  // try the 2nd wal created during SwitchWAL (not locked this time)
  ASSERT_OK(dbfull()->TEST_SwitchWAL());
+  ASSERT_NE(wal_num, dbfull()->TEST_GetCurrentLogNumber());
  ASSERT_OK(Put("key1", "value"));
  ASSERT_NE(options.manual_wal_flush, dbfull()->WALBufferIsEmpty());
  ASSERT_OK(db_->LockWAL());
@@ -709,21 +723,57 @@ TEST_P(DBWriteTest, LockWALInEffect) {
 }

 TEST_P(DBWriteTest, LockWALConcurrentRecursive) {
+  // This is a micro-stress test of LockWAL and concurrency handling.
+  // It is considered the most convenient way to balance functional
+  // coverage and reproducibility (vs. the two extremes of (a) unit tests
+  // tailored to specific interleavings and (b) db_stress)
  Options options = GetOptions();
  Reopen(options);
-  ASSERT_OK(Put("k1", "val"));
+  ASSERT_OK(Put("k1", "k1_orig"));
  ASSERT_OK(db_->LockWAL());  // 0 -> 1
  auto frozen_seqno = db_->GetLatestSequenceNumber();
-  std::atomic<bool> t1_completed{false};
-  port::Thread t1{[&]() {
-    // Won't finish until WAL unlocked
-    ASSERT_OK(Put("k1", "val2"));
-    t1_completed = true;
+
+  std::string ingest_file = dbname_ + "/external.sst";
+  {
+    SstFileWriter sst_file_writer(EnvOptions(), options);
+    ASSERT_OK(sst_file_writer.Open(ingest_file));
+    ASSERT_OK(sst_file_writer.Put("k2", "k2_val"));
+    ExternalSstFileInfo external_info;
+    ASSERT_OK(sst_file_writer.Finish(&external_info));
+  }
+  AcqRelAtomic<bool> parallel_ingest_completed{false};
+  port::Thread parallel_ingest{[&]() {
+    IngestExternalFileOptions ingest_opts;
+    ingest_opts.move_files = true;  // faster than copy
+    // Shouldn't finish until WAL unlocked
+    ASSERT_OK(db_->IngestExternalFile({ingest_file}, ingest_opts));
+    parallel_ingest_completed.Store(true);
+  }};
+
+  AcqRelAtomic<bool> flush_completed{false};
+  port::Thread parallel_flush{[&]() {
+    FlushOptions flush_opts;
+    // NB: Flush with wait=false case is tested above in LockWALInEffect
+    flush_opts.wait = true;
+    // allow_write_stall = true blocks in fewer cases
+    flush_opts.allow_write_stall = true;
+    // Shouldn't finish until WAL unlocked
+    ASSERT_OK(db_->Flush(flush_opts));
+    flush_completed.Store(true);
+  }};
+
+  AcqRelAtomic<bool> parallel_put_completed{false};
+  port::Thread parallel_put{[&]() {
+    // This can make certain failure scenarios more likely:
+    //   sleep(1);
+    // Shouldn't finish until WAL unlocked
+    ASSERT_OK(Put("k1", "k1_mod"));
+    parallel_put_completed.Store(true);
  }};

  ASSERT_OK(db_->LockWAL());  // 1 -> 2
  // Read-only ops are OK
-  ASSERT_EQ(Get("k1"), "val");
+  ASSERT_EQ(Get("k1"), "k1_orig");
  {
    std::vector<LiveFileStorageInfo> files;
    LiveFilesStorageInfoOptions lf_opts;
@@ -732,29 +782,35 @@ TEST_P(DBWriteTest, LockWALConcurrentRecursive) {
    ASSERT_OK(db_->GetLiveFilesStorageInfo({lf_opts}, &files));
  }

-  port::Thread t2{[&]() {
+  port::Thread parallel_lock_wal{[&]() {
    ASSERT_OK(db_->LockWAL());  // 2 -> 3 or 1 -> 2
  }};

  ASSERT_OK(db_->UnlockWAL());  // 2 -> 1 or 3 -> 2
-  // Give t1 an extra chance to jump in case of bug
+  // Give parallel_put an extra chance to jump in case of bug
  std::this_thread::yield();
-  t2.join();
-  ASSERT_FALSE(t1_completed.load());
+  parallel_lock_wal.join();
+  ASSERT_FALSE(parallel_put_completed.Load());
+  ASSERT_FALSE(parallel_ingest_completed.Load());
+  ASSERT_FALSE(flush_completed.Load());

  // Should now have 2 outstanding LockWAL
-  ASSERT_EQ(Get("k1"), "val");
+  ASSERT_EQ(Get("k1"), "k1_orig");

  ASSERT_OK(db_->UnlockWAL());  // 2 -> 1

-  ASSERT_FALSE(t1_completed.load());
-  ASSERT_EQ(Get("k1"), "val");
+  ASSERT_FALSE(parallel_put_completed.Load());
+  ASSERT_FALSE(parallel_ingest_completed.Load());
+  ASSERT_FALSE(flush_completed.Load());
+
+  ASSERT_EQ(Get("k1"), "k1_orig");
+  ASSERT_EQ(Get("k2"), "NOT_FOUND");
  ASSERT_EQ(frozen_seqno, db_->GetLatestSequenceNumber());

  // Ensure final Unlock is concurrency safe and extra Unlock is safe but
  // non-OK
  std::atomic<int> unlock_ok{0};
-  port::Thread t3{[&]() {
+  port::Thread parallel_stuff{[&]() {
    if (db_->UnlockWAL().ok()) {
      unlock_ok++;
    }
@@ -767,18 +823,23 @@ TEST_P(DBWriteTest, LockWALConcurrentRecursive) {
  if (db_->UnlockWAL().ok()) {
    unlock_ok++;
  }
-  t3.join();
+  parallel_stuff.join();

  // There was one extra unlock, so just one non-ok
  ASSERT_EQ(unlock_ok.load(), 2);

  // Write can proceed
-  t1.join();
-  ASSERT_TRUE(t1_completed.load());
-  ASSERT_EQ(Get("k1"), "val2");
+  parallel_put.join();
+  ASSERT_TRUE(parallel_put_completed.Load());
+  ASSERT_EQ(Get("k1"), "k1_mod");
+  parallel_ingest.join();
+  ASSERT_TRUE(parallel_ingest_completed.Load());
+  ASSERT_EQ(Get("k2"), "k2_val");
+  parallel_flush.join();
+  ASSERT_TRUE(flush_completed.Load());
  // And new writes
-  ASSERT_OK(Put("k2", "val"));
-  ASSERT_EQ(Get("k2"), "val");
+  ASSERT_OK(Put("k3", "val"));
+  ASSERT_EQ(Get("k3"), "val");
 }

 TEST_P(DBWriteTest, ConcurrentlyDisabledWAL) {
@@ -425,7 +425,7 @@ TEST_F(DBErrorHandlingFSTest, FlushWALWriteRetryableError) {

  listener->EnableAutoRecovery(false);
  SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::SyncClosedLogs:Start",
+      "DBImpl::SyncClosedWals:Start",
      [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
  SyncPoint::GetInstance()->EnableProcessing();

@@ -470,7 +470,7 @@ TEST_F(DBErrorHandlingFSTest, FlushWALAtomicWriteRetryableError) {

  listener->EnableAutoRecovery(false);
  SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::SyncClosedLogs:Start",
+      "DBImpl::SyncClosedWals:Start",
      [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
  SyncPoint::GetInstance()->EnableProcessing();

@@ -674,10 +674,6 @@ TEST_F(ExternalSSTFileBasicTest, NoCopy) {
  ASSERT_EQ(file3_info.smallest_key, Key(110));
  ASSERT_EQ(file3_info.largest_key, Key(124));

-  ASSERT_OK(dbfull()->LockWAL());
-  // TODO(FIXME): should not allow file ingestion.
-  // With below line, ingestion will block, without it, ingestion can go
-  // through. ASSERT_OK(dbfull()->Put(WriteOptions(), "vo", "vo"));
  s = DeprecatedAddFile({file1}, true /* move file */);
  ASSERT_OK(s) << s.ToString();
  ASSERT_EQ(Status::NotFound(), env_->FileExists(file1));
@@ -342,8 +342,7 @@ Status ExternalSstFileIngestionJob::NeedsFlush(bool* flush_needed,
  autovector<UserKeyRange> ranges;
  ranges.reserve(n);
  for (const IngestedFileInfo& file_to_ingest : files_to_ingest_) {
-    ranges.emplace_back(file_to_ingest.smallest_internal_key.user_key(),
-                        file_to_ingest.largest_internal_key.user_key());
+    ranges.emplace_back(file_to_ingest.start_ukey, file_to_ingest.limit_ukey);
  }
  Status status = cfd_->RangesOverlapWithMemtables(
      ranges, super_version, db_options_.allow_data_in_errors, flush_needed);
@@ -930,6 +929,17 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo(
    }
  }

+  const size_t ts_sz = ucmp->timestamp_size();
+  Slice smallest = file_to_ingest->smallest_internal_key.user_key();
+  Slice largest = file_to_ingest->largest_internal_key.user_key();
+  if (ts_sz > 0) {
+    AppendUserKeyWithMaxTimestamp(&file_to_ingest->start_ukey, smallest, ts_sz);
+    AppendUserKeyWithMinTimestamp(&file_to_ingest->limit_ukey, largest, ts_sz);
+  } else {
+    file_to_ingest->start_ukey.assign(smallest.data(), smallest.size());
+    file_to_ingest->limit_ukey.assign(largest.data(), largest.size());
+  }
+
  auto s =
      GetSstInternalUniqueId(file_to_ingest->table_properties.db_id,
                             file_to_ingest->table_properties.db_session_id,
@@ -982,9 +992,8 @@ Status ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile(
    if (lvl > 0 && lvl < vstorage->base_level()) {
      continue;
    }
-    if (cfd_->RangeOverlapWithCompaction(
-            file_to_ingest->smallest_internal_key.user_key(),
-            file_to_ingest->largest_internal_key.user_key(), lvl)) {
+    if (cfd_->RangeOverlapWithCompaction(file_to_ingest->start_ukey,
+                                         file_to_ingest->limit_ukey, lvl)) {
      // We must use L0 or any level higher than `lvl` to be able to overwrite
      // the compaction output keys that we overlap with in this level, We also
      // need to assign this file a seqno to overwrite the compaction output
@@ -994,9 +1003,8 @@ Status ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile(
    } else if (vstorage->NumLevelFiles(lvl) > 0) {
      bool overlap_with_level = false;
      status = sv->current->OverlapWithLevelIterator(
-          ro, env_options_, file_to_ingest->smallest_internal_key.user_key(),
-          file_to_ingest->largest_internal_key.user_key(), lvl,
-          &overlap_with_level);
+          ro, env_options_, file_to_ingest->start_ukey,
+          file_to_ingest->limit_ukey, lvl, &overlap_with_level);
      if (!status.ok()) {
        return status;
      }
@@ -1165,9 +1173,8 @@ bool ExternalSstFileIngestionJob::IngestedFileFitInLevel(
  }

  auto* vstorage = cfd_->current()->storage_info();
-  Slice file_smallest_user_key(
-      file_to_ingest->smallest_internal_key.user_key());
-  Slice file_largest_user_key(file_to_ingest->largest_internal_key.user_key());
+  Slice file_smallest_user_key(file_to_ingest->start_ukey);
+  Slice file_largest_user_key(file_to_ingest->limit_ukey);

  if (vstorage->OverlapInLevel(level, &file_smallest_user_key,
                               &file_largest_user_key)) {
@@ -32,6 +32,17 @@ struct IngestedFileInfo {
  InternalKey smallest_internal_key;
  // Largest internal key in external file
  InternalKey largest_internal_key;
+  // NOTE: use below two fields for all `*Overlap*` types of checks instead of
+  // smallest_internal_key.user_key() and largest_internal_key.user_key().
+  // The smallest / largest user key contained in the file for key range checks.
+  // These could be different from smallest_internal_key.user_key(), and
+  // largest_internal_key.user_key() when user-defined timestamps are enabled,
+  // because the check is about making sure the user key without timestamps part
+  // does not overlap. To achieve that, the smallest user key will be updated
+  // with the maximum timestamp while the largest user key will be updated with
+  // the min timestamp. It's otherwise the same.
+  std::string start_ukey;
+  std::string limit_ukey;
  // Sequence number for keys in external file
  SequenceNumber original_seqno;
  // Offset of the global sequence number field in the file, will
@@ -3358,11 +3358,60 @@ TEST_F(ExternalSSTFileWithTimestampTest, Basic) {

    ASSERT_OK(IngestExternalUDTFile({file4}));

+    for (int k = 200; k < 250; k++) {
+      VerifyValueAndTs(Key(k), EncodeAsUint64(k), Key(k) + "_val",
+                       EncodeAsUint64(k));
+    }
+
    // In UDT mode, any external file that can be successfully ingested also
    // should not overlap with the db. As a result, they can always get the
    // seq 0 assigned.
    ASSERT_EQ(db_->GetLatestSequenceNumber(), seq_num_before_ingestion);

+    // file5.sst (Key(200), ts = 199)
+    // While DB has (Key(200), ts = 200) => user key without timestamp overlaps
+    std::string file5 = sst_files_dir_ + "file5.sst";
+    ASSERT_OK(sst_file_writer.Open(file5));
+    ASSERT_OK(
+        sst_file_writer.Put(Key(200), EncodeAsUint64(199), Key(200) + "_val"));
+
+    ExternalSstFileInfo file5_info;
+    ASSERT_OK(sst_file_writer.Finish(&file5_info));
+    ASSERT_TRUE(IngestExternalUDTFile({file5}).IsInvalidArgument());
+
+    // file6.sst (Key(200), ts = 201)
+    // While DB has (Key(200), ts = 200) => user key without timestamp overlaps
+    std::string file6 = sst_files_dir_ + "file6.sst";
+    ASSERT_OK(sst_file_writer.Open(file6));
+    ASSERT_OK(
+        sst_file_writer.Put(Key(200), EncodeAsUint64(201), Key(0) + "_val"));
+
+    ExternalSstFileInfo file6_info;
+    ASSERT_OK(sst_file_writer.Finish(&file6_info));
+    ASSERT_TRUE(IngestExternalUDTFile({file6}).IsInvalidArgument());
+
+    // Check memtable overlap.
+    ASSERT_OK(dbfull()->Put(WriteOptions(), Key(250), EncodeAsUint64(250),
+                            Key(250) + "_val"));
+
+    std::string file7 = sst_files_dir_ + "file7.sst";
+    ASSERT_OK(sst_file_writer.Open(file7));
+    ASSERT_OK(
+        sst_file_writer.Put(Key(250), EncodeAsUint64(249), Key(250) + "_val2"));
+
+    ExternalSstFileInfo file7_info;
+    ASSERT_OK(sst_file_writer.Finish(&file7_info));
+    ASSERT_TRUE(IngestExternalUDTFile({file7}).IsInvalidArgument());
+
+    std::string file8 = sst_files_dir_ + "file8.sst";
+    ASSERT_OK(sst_file_writer.Open(file8));
+    ASSERT_OK(
+        sst_file_writer.Put(Key(250), EncodeAsUint64(251), Key(250) + "_val3"));
+
+    ExternalSstFileInfo file8_info;
+    ASSERT_OK(sst_file_writer.Finish(&file8_info));
+    ASSERT_TRUE(IngestExternalUDTFile({file8}).IsInvalidArgument());
+
    DestroyAndRecreateExternalSSTFilesDir();
  } while (ChangeOptions(kSkipPlainTable | kSkipFIFOCompaction |
                         kRangeDelSkipConfigs));
@@ -12,6 +12,7 @@
 // file data (or entire files) not protected by a "sync".

 #include "db/db_impl/db_impl.h"
+#include "db/db_test_util.h"
 #include "db/log_format.h"
 #include "db/version_set.h"
 #include "env/mock_env.h"
@@ -545,78 +546,6 @@ TEST_P(FaultInjectionTest, WriteBatchWalTerminationTest) {
  ASSERT_EQ(db_->Get(ro, "boys", &val), Status::NotFound());
 }

-TEST_P(FaultInjectionTest, NoDuplicateTrailingEntries) {
-  auto fault_fs = std::make_shared<FaultInjectionTestFS>(FileSystem::Default());
-  fault_fs->EnableWriteErrorInjection();
-  fault_fs->SetFilesystemDirectWritable(false);
-  const std::string file_name = NormalizePath(dbname_ + "/test_file");
-  std::unique_ptr<log::Writer> log_writer = nullptr;
-  constexpr uint64_t log_number = 0;
-  {
-    std::unique_ptr<FSWritableFile> file;
-    const Status s =
-        fault_fs->NewWritableFile(file_name, FileOptions(), &file, nullptr);
-    ASSERT_OK(s);
-    std::unique_ptr<WritableFileWriter> fwriter(
-        new WritableFileWriter(std::move(file), file_name, FileOptions()));
-    log_writer.reset(new log::Writer(std::move(fwriter), log_number,
-                                     /*recycle_log_files=*/false));
-  }
-
-  fault_fs->SetRandomWriteError(
-      0xdeadbeef, /*one_in=*/1, IOStatus::IOError("Injected IOError"),
-      /*inject_for_all_file_types=*/true, /*types=*/{});
-
-  {
-    VersionEdit edit;
-    edit.SetColumnFamily(0);
-    std::string buf;
-    assert(edit.EncodeTo(&buf));
-    const Status s = log_writer->AddRecord(WriteOptions(), buf);
-    ASSERT_NOK(s);
-  }
-
-  fault_fs->DisableWriteErrorInjection();
-
-  // Flush remaining data from its buffer to underlying file.
-  ASSERT_OK(log_writer->file()->writable_file()->Sync(IOOptions(),
-                                                      nullptr /* dbg */));
-  // Closing the log writer will cause WritableFileWriter::Close()
-  log_writer.reset();
-
-  {
-    std::unique_ptr<FSSequentialFile> file;
-    Status s =
-        fault_fs->NewSequentialFile(file_name, FileOptions(), &file, nullptr);
-    ASSERT_OK(s);
-    std::unique_ptr<SequentialFileReader> freader(
-        new SequentialFileReader(std::move(file), file_name));
-    Status log_read_s;
-    class LogReporter : public log::Reader::Reporter {
-     public:
-      Status* status_;
-      explicit LogReporter(Status* _s) : status_(_s) {}
-      void Corruption(size_t /*bytes*/, const Status& _s) override {
-        if (status_->ok()) {
-          *status_ = _s;
-        }
-      }
-    } reporter(&log_read_s);
-    std::unique_ptr<log::Reader> log_reader(new log::Reader(
-        nullptr, std::move(freader), &reporter, /*checksum=*/true, log_number));
-    Slice record;
-    std::string data;
-    size_t count = 0;
-    while (log_reader->ReadRecord(&record, &data) && log_read_s.ok()) {
-      VersionEdit edit;
-      ASSERT_OK(edit.DecodeFrom(data));
-      ++count;
-    }
-    // Verify that only one version edit exists in the file.
-    ASSERT_EQ(1, count);
-  }
-}
-
 INSTANTIATE_TEST_CASE_P(
    FaultTest, FaultInjectionTest,
    ::testing::Values(std::make_tuple(false, kDefault, kEnd),
@@ -629,6 +558,59 @@ INSTANTIATE_TEST_CASE_P(
                      std::make_tuple(false, kSyncWal, kEnd),
                      std::make_tuple(true, kSyncWal, kEnd)));

+class FaultInjectionDBTest : public DBTestBase {
+ public:
+  FaultInjectionDBTest()
+      : DBTestBase("fault_injection_fs_test", /*env_do_fsync=*/false) {}
+};
+
+TEST(FaultInjectionFSTest, ReadUnsyncedData) {
+  std::shared_ptr<FaultInjectionTestFS> fault_fs =
+      std::make_shared<FaultInjectionTestFS>(FileSystem::Default());
+  ASSERT_TRUE(fault_fs->ReadUnsyncedData());
+
+  uint32_t len = Random::GetTLSInstance()->Uniform(10000) + 1;
+  Random rnd(len);
+
+  // Create partially synced file
+  std::string f = test::PerThreadDBPath("read_unsynced.data");
+  std::string data = rnd.RandomString(len);
+  {
+    std::unique_ptr<FSWritableFile> w;
+    ASSERT_OK(fault_fs->NewWritableFile(f, {}, &w, nullptr));
+    uint32_t synced_len = rnd.Uniform(len + 1);
+    ASSERT_OK(w->Append(Slice(data.data(), synced_len), {}, nullptr));
+    ASSERT_OK(w->Sync({}, nullptr));
+    ASSERT_OK(w->Append(Slice(data.data() + synced_len, len - synced_len), {},
+                        nullptr));
+    // no sync here
+    ASSERT_OK(w->Close({}, nullptr));
+  }
+  // Test file size includes unsynced data
+  {
+    uint64_t file_size;
+    ASSERT_OK(fault_fs->GetFileSize(f, {}, &file_size, nullptr));
+    ASSERT_EQ(len, file_size);
+  }
+  // Test read file contents, with two reads that probably don't
+  // align with the unsynced split.
+  {
+    std::unique_ptr<FSSequentialFile> r;
+    ASSERT_OK(fault_fs->NewSequentialFile(f, {}, &r, nullptr));
+    uint32_t first_read_len = rnd.Uniform(len + 1);
+    Slice sl;
+    std::unique_ptr<char[]> scratch(new char[first_read_len]);
+    ASSERT_OK(r->Read(first_read_len, {}, &sl, scratch.get(), nullptr));
+    ASSERT_EQ(first_read_len, sl.size());
+    ASSERT_EQ(0, sl.compare(Slice(data.data(), first_read_len)));
+    uint32_t second_read_len = len - first_read_len;
+    scratch.reset(new char[second_read_len]);
+    ASSERT_OK(r->Read(second_read_len, {}, &sl, scratch.get(), nullptr));
+    ASSERT_EQ(second_read_len, sl.size());
+    ASSERT_EQ(0,
+              sl.compare(Slice(data.data() + first_read_len, second_read_len)));
+  }
+}
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -115,6 +115,9 @@ FlushJob::FlushJob(
      db_mutex_(db_mutex),
      shutting_down_(shutting_down),
      existing_snapshots_(std::move(existing_snapshots)),
+      earliest_snapshot_(existing_snapshots_.empty()
+                             ? kMaxSequenceNumber
+                             : existing_snapshots_.at(0)),
      earliest_write_conflict_snapshot_(earliest_write_conflict_snapshot),
      snapshot_checker_(snapshot_checker),
      job_context_(job_context),
@@ -194,6 +197,7 @@ void FlushJob::PickMemTable() {
  // Track effective cutoff user-defined timestamp during flush if
  // user-defined timestamps can be stripped.
  GetEffectiveCutoffUDTForPickedMemTables();
+  GetPrecludeLastLevelMinSeqno();

  ReportFlushInputSize(mems_);

@@ -502,7 +506,7 @@ Status FlushJob::MemPurge() {
    const std::atomic<bool> kManualCompactionCanceledFalse{false};
    CompactionIterator c_iter(
        iter.get(), (cfd_->internal_comparator()).user_comparator(), &merge,
-        kMaxSequenceNumber, &existing_snapshots_,
+        kMaxSequenceNumber, &existing_snapshots_, earliest_snapshot_,
        earliest_write_conflict_snapshot_, job_snapshot_seq, snapshot_checker_,
        env, ShouldReportDetailedTime(env, ioptions->stats),
        true /* internal key corruption is not ok */, range_del_agg.get(),
@@ -968,14 +972,17 @@ Status FlushJob::WriteLevel0Table() {
          cfd_->GetID(), cfd_->GetName(), 0 /* level */,
          false /* is_bottommost */, TableFileCreationReason::kFlush,
          oldest_key_time, current_time, db_id_, db_session_id_,
-          0 /* target_file_size */, meta_.fd.GetNumber());
+          0 /* target_file_size */, meta_.fd.GetNumber(),
+          preclude_last_level_min_seqno_ == kMaxSequenceNumber
+              ? preclude_last_level_min_seqno_
+              : std::min(earliest_snapshot_, preclude_last_level_min_seqno_));
      const SequenceNumber job_snapshot_seq =
          job_context_->GetJobSnapshotSequence();

      s = BuildTable(
          dbname_, versions_, db_options_, tboptions, file_options_,
          cfd_->table_cache(), iter.get(), std::move(range_del_iters), &meta_,
-          &blob_file_additions, existing_snapshots_,
+          &blob_file_additions, existing_snapshots_, earliest_snapshot_,
          earliest_write_conflict_snapshot_, job_snapshot_seq,
          snapshot_checker_, mutable_cf_options_.paranoid_file_checks,
          cfd_->internal_stats(), &io_s, io_tracer_,
@@ -1154,6 +1161,26 @@ void FlushJob::GetEffectiveCutoffUDTForPickedMemTables() {
  }
 }

+void FlushJob::GetPrecludeLastLevelMinSeqno() {
+  if (cfd_->ioptions()->preclude_last_level_data_seconds == 0) {
+    return;
+  }
+  int64_t current_time = 0;
+  Status s = db_options_.clock->GetCurrentTime(&current_time);
+  if (!s.ok()) {
+    ROCKS_LOG_WARN(db_options_.info_log,
+                   "Failed to get current time in Flush: Status: %s",
+                   s.ToString().c_str());
+  } else {
+    SequenceNumber preserve_time_min_seqno;
+    seqno_to_time_mapping_->GetCurrentTieringCutoffSeqnos(
+        static_cast<uint64_t>(current_time),
+        cfd_->ioptions()->preserve_internal_time_seconds,
+        cfd_->ioptions()->preclude_last_level_data_seconds,
+        &preserve_time_min_seqno, &preclude_last_level_min_seqno_);
+  }
+}
+
 Status FlushJob::MaybeIncreaseFullHistoryTsLowToAboveCutoffUDT() {
  db_mutex_->AssertHeld();
  const auto* ucmp = cfd_->user_comparator();
@@ -143,6 +143,13 @@ class FlushJob {
  // `MaybeIncreaseFullHistoryTsLowToAboveCutoffUDT` for details.
  void GetEffectiveCutoffUDTForPickedMemTables();

+  // If this column family enables tiering feature, it will find the current
+  // `preclude_last_level_min_seqno_`, and the smaller one between this and
+  // the `earliset_snapshot_` will later be announced to user property
+  // collectors. It indicates to tiering use cases which data are old enough to
+  // be placed on the last level.
+  void GetPrecludeLastLevelMinSeqno();
+
  Status MaybeIncreaseFullHistoryTsLowToAboveCutoffUDT();

  const std::string& dbname_;
@@ -161,6 +168,7 @@ class FlushJob {
  InstrumentedMutex* db_mutex_;
  std::atomic<bool>* shutting_down_;
  std::vector<SequenceNumber> existing_snapshots_;
+  SequenceNumber earliest_snapshot_;
  SequenceNumber earliest_write_conflict_snapshot_;
  SnapshotChecker* snapshot_checker_;
  JobContext* job_context_;
@@ -221,6 +229,12 @@ class FlushJob {
  // Keeps track of the newest user-defined timestamp for this flush job if
  // `persist_user_defined_timestamps` flag is false.
  std::string cutoff_udt_;
+
+  // The current minimum seqno that compaction jobs will preclude the data from
+  // the last level. Data with seqnos larger than this or larger than
+  // `earliest_snapshot_` will be output to the penultimate level had it gone
+  // through a compaction to the last level.
+  SequenceNumber preclude_last_level_min_seqno_ = kMaxSequenceNumber;
 };

 }  // namespace ROCKSDB_NAMESPACE
@@ -16,6 +16,7 @@
 #include <limits>
 #include <sstream>
 #include <string>
+#include <unordered_map>
 #include <utility>
 #include <vector>

@@ -33,7 +34,6 @@

 namespace ROCKSDB_NAMESPACE {

-
 const std::map<LevelStatType, LevelStat> InternalStats::compaction_level_stats =
    {
        {LevelStatType::NUM_FILES, LevelStat{"NumFiles", "Files"}},
@@ -2135,5 +2135,64 @@ void InternalStats::DumpCFFileHistogram(std::string* value) {
  value->append(oss.str());
 }

+namespace {
+
+class SumPropertyAggregator : public IntPropertyAggregator {
+ public:
+  SumPropertyAggregator() : aggregated_value_(0) {}
+  virtual ~SumPropertyAggregator() override = default;
+
+  void Add(ColumnFamilyData* cfd, uint64_t value) override {
+    (void)cfd;
+    aggregated_value_ += value;
+  }
+
+  uint64_t Aggregate() const override { return aggregated_value_; }
+
+ private:
+  uint64_t aggregated_value_;
+};
+
+// A block cache may be shared by multiple column families.
+// BlockCachePropertyAggregator ensures that the same cache is only added once.
+class BlockCachePropertyAggregator : public IntPropertyAggregator {
+ public:
+  BlockCachePropertyAggregator() = default;
+  virtual ~BlockCachePropertyAggregator() override = default;
+
+  void Add(ColumnFamilyData* cfd, uint64_t value) override {
+    auto* table_factory = cfd->ioptions()->table_factory.get();
+    assert(table_factory != nullptr);
+    Cache* cache =
+        table_factory->GetOptions<Cache>(TableFactory::kBlockCacheOpts());
+    if (cache != nullptr) {
+      block_cache_properties_.emplace(cache, value);
+    }
+  }
+
+  uint64_t Aggregate() const override {
+    uint64_t sum = 0;
+    for (const auto& p : block_cache_properties_) {
+      sum += p.second;
+    }
+    return sum;
+  }
+
+ private:
+  std::unordered_map<Cache*, uint64_t> block_cache_properties_;
+};
+
+}  // anonymous namespace
+
+std::unique_ptr<IntPropertyAggregator> CreateIntPropertyAggregator(
+    const Slice& property) {
+  if (property == DB::Properties::kBlockCacheCapacity ||
+      property == DB::Properties::kBlockCacheUsage ||
+      property == DB::Properties::kBlockCachePinnedUsage) {
+    return std::make_unique<BlockCachePropertyAggregator>();
+  } else {
+    return std::make_unique<SumPropertyAggregator>();
+  }
+}

 }  // namespace ROCKSDB_NAMESPACE
@@ -432,7 +432,7 @@ class InternalStats {
    explicit CompactionStatsFull() : stats(), penultimate_level_stats() {}

    explicit CompactionStatsFull(CompactionReason reason, int c)
-        : stats(reason, c), penultimate_level_stats(reason, c){}
+        : stats(reason, c), penultimate_level_stats(reason, c) {}

    uint64_t TotalBytesWritten() const {
      uint64_t bytes_written = stats.bytes_written + stats.bytes_written_blob;
@@ -873,5 +873,24 @@ class InternalStats {
  uint64_t started_at_;
 };

+// IntPropertyAggregator aggregates an integer property across all column
+// families.
+class IntPropertyAggregator {
+ public:
+  IntPropertyAggregator() {}
+  virtual ~IntPropertyAggregator() {}
+
+  IntPropertyAggregator(const IntPropertyAggregator&) = delete;
+  void operator=(const IntPropertyAggregator&) = delete;
+
+  // Add a column family's property value to the aggregator.
+  virtual void Add(ColumnFamilyData* cfd, uint64_t value) = 0;
+
+  // Get the aggregated value.
+  virtual uint64_t Aggregate() const = 0;
+};
+
+std::unique_ptr<IntPropertyAggregator> CreateIntPropertyAggregator(
+    const Slice& property);

 }  // namespace ROCKSDB_NAMESPACE
@@ -487,9 +487,11 @@ unsigned int Reader::ReadPhysicalRecord(Slice* result, size_t* drop_size,
         type == kRecyclableUserDefinedTimestampSizeType);
    if (is_recyclable_type) {
      header_size = kRecyclableHeaderSize;
-      if (end_of_buffer_offset_ - buffer_.size() == 0) {
-        recycled_ = true;
+      if (first_record_read_ && !recycled_) {
+        // A recycled log should have started with a recycled record
+        return kBadRecord;
      }
+      recycled_ = true;
      // We need enough for the larger header
      if (buffer_.size() < static_cast<size_t>(kRecyclableHeaderSize)) {
        int r = kEof;
@@ -867,9 +869,12 @@ bool FragmentBufferedReader::TryReadFragment(
  int header_size = kHeaderSize;
  if ((type >= kRecyclableFullType && type <= kRecyclableLastType) ||
      type == kRecyclableUserDefinedTimestampSizeType) {
-    if (end_of_buffer_offset_ - buffer_.size() == 0) {
-      recycled_ = true;
+    if (first_record_read_ && !recycled_) {
+      // A recycled log should have started with a recycled record
+      *fragment_type_or_err = kBadRecord;
+      return true;
    }
+    recycled_ = true;
    header_size = kRecyclableHeaderSize;
    while (buffer_.size() < static_cast<size_t>(kRecyclableHeaderSize)) {
      size_t old_size = buffer_.size();
@@ -1157,6 +1157,42 @@ TEST_P(CompressionLogTest, AlignedFragmentation) {
  ASSERT_EQ("EOF", Read());
 }

+TEST_P(CompressionLogTest, ChecksumMismatch) {
+  const CompressionType kCompressionType = std::get<2>(GetParam());
+  const bool kCompressionEnabled = kCompressionType != kNoCompression;
+  const bool kRecyclableLog = (std::get<0>(GetParam()) != 0);
+  if (!StreamingCompressionTypeSupported(kCompressionType)) {
+    ROCKSDB_GTEST_SKIP("Test requires support for compression type");
+    return;
+  }
+  ASSERT_OK(SetupTestEnv());
+
+  Write("foooooo");
+  int header_len;
+  if (kRecyclableLog) {
+    header_len = kRecyclableHeaderSize;
+  } else {
+    header_len = kHeaderSize;
+  }
+  int compression_record_len;
+  if (kCompressionEnabled) {
+    compression_record_len = header_len + 4;
+  } else {
+    compression_record_len = 0;
+  }
+  IncrementByte(compression_record_len + header_len /* offset */,
+                14 /* delta */);
+
+  ASSERT_EQ("EOF", Read());
+  if (!kRecyclableLog) {
+    ASSERT_GT(DroppedBytes(), 0U);
+    ASSERT_EQ("OK", MatchError("checksum mismatch"));
+  } else {
+    ASSERT_EQ(0U, DroppedBytes());
+    ASSERT_EQ("", ReportMessage());
+  }
+}
+
 INSTANTIATE_TEST_CASE_P(
    Compression, CompressionLogTest,
    ::testing::Combine(::testing::Values(0, 1), ::testing::Bool(),
@@ -53,6 +53,11 @@ Writer::~Writer() {

 IOStatus Writer::WriteBuffer(const WriteOptions& write_options) {
  if (dest_->seen_error()) {
+#ifndef NDEBUG
+    if (dest_->seen_injected_error()) {
+      return IOStatus::IOError("Seen injected error. Skip writing buffer.");
+    }
+#endif  // NDEBUG
    return IOStatus::IOError("Seen error. Skip writing buffer.");
  }
  IOOptions opts;
@@ -74,9 +79,23 @@ IOStatus Writer::Close(const WriteOptions& write_options) {
  return s;
 }

+bool Writer::PublishIfClosed() {
+  if (dest_->IsClosed()) {
+    dest_.reset();
+    return true;
+  } else {
+    return false;
+  }
+}
+
 IOStatus Writer::AddRecord(const WriteOptions& write_options,
                           const Slice& slice) {
  if (dest_->seen_error()) {
+#ifndef NDEBUG
+    if (dest_->seen_injected_error()) {
+      return IOStatus::IOError("Seen injected error. Skip writing buffer.");
+    }
+#endif  // NDEBUG
    return IOStatus::IOError("Seen error. Skip writing buffer.");
  }
  const char* ptr = slice.data();
@@ -184,6 +203,11 @@ IOStatus Writer::AddCompressionTypeRecord(const WriteOptions& write_options) {
  }

  if (dest_->seen_error()) {
+#ifndef NDEBUG
+    if (dest_->seen_injected_error()) {
+      return IOStatus::IOError("Seen injected error. Skip writing buffer.");
+    }
+#endif  // NDEBUG
    return IOStatus::IOError("Seen error. Skip writing buffer.");
  }

@@ -107,6 +107,11 @@ class Writer {

  IOStatus Close(const WriteOptions& write_options);

+  // If closing the writer through file(), call this afterwards to modify
+  // this object's state to reflect that. Returns true if the destination file
+  // has been closed. If it hasn't been closed, returns false with no change.
+  bool PublishIfClosed();
+
  bool BufferIsEmpty();

  size_t TEST_block_offset() const { return block_offset_; }
@@ -13,6 +13,7 @@
 #include <array>
 #include <limits>
 #include <memory>
+#include <optional>

 #include "db/dbformat.h"
 #include "db/kv_checksum.h"
@@ -617,8 +618,9 @@ FragmentedRangeTombstoneIterator* MemTable::NewRangeTombstoneIteratorInternal(
 }

 void MemTable::ConstructFragmentedRangeTombstones() {
-  assert(!IsFragmentedRangeTombstonesConstructed(false));
-  // There should be no concurrent Construction
+  // There should be no concurrent Construction.
+  // We could also check fragmented_range_tombstone_list_ to avoid repeate
+  // constructions. We just construct them here again to be safe.
  if (!is_range_del_table_empty_.load(std::memory_order_relaxed)) {
    // TODO: plumb Env::IOActivity, Env::IOPriority
    auto* unfragmented_iter = new MemTableIterator(
@@ -955,9 +957,9 @@ static bool SaveValue(void* arg, const char* entry) {
                                             s->key->user_key())) {
    // Correct user key
    TEST_SYNC_POINT_CALLBACK("Memtable::SaveValue:Found:entry", &entry);
-    std::unique_ptr<ReadLock> read_lock;
+    std::optional<ReadLock> read_lock;
    if (s->inplace_update_support) {
-      read_lock.reset(new ReadLock(s->mem->GetLock(s->key->user_key())));
+      read_lock.emplace(s->mem->GetLock(s->key->user_key()));
    }

    if (s->protection_bytes_per_key > 0) {
@@ -534,21 +534,21 @@ class MemTable {
  // Returns a heuristic flush decision
  bool ShouldFlushNow();

+  // Updates `fragmented_range_tombstone_list_` that will be used to serve reads
+  // when this memtable becomes an immutable memtable (in some
+  // MemtableListVersion::memlist_). Should be called when this memtable is
+  // about to become immutable. May be called multiple times since
+  // SwitchMemtable() may fail.
  void ConstructFragmentedRangeTombstones();

  // Returns whether a fragmented range tombstone list is already constructed
  // for this memtable. It should be constructed right before a memtable is
  // added to an immutable memtable list. Note that if a memtable does not have
-  // any range tombstone, then no range tombstone list will ever be constructed.
-  // @param allow_empty Specifies whether a memtable with no range tombstone is
-  // considered to have its fragmented range tombstone list constructed.
-  bool IsFragmentedRangeTombstonesConstructed(bool allow_empty = true) const {
-    if (allow_empty) {
-      return fragmented_range_tombstone_list_.get() != nullptr ||
-             is_range_del_table_empty_;
-    } else {
-      return fragmented_range_tombstone_list_.get() != nullptr;
-    }
+  // any range tombstone, then no range tombstone list will ever be constructed
+  // and true is returned in that case.
+  bool IsFragmentedRangeTombstonesConstructed() const {
+    return fragmented_range_tombstone_list_.get() != nullptr ||
+           is_range_del_table_empty_;
  }

  // Get the newest user-defined timestamp contained in this MemTable. Check
@@ -85,7 +85,7 @@ class MultiCfIteratorImpl {
  void Next() {
    assert(Valid());
    auto& min_heap = GetHeap<MultiCfMinHeap>([this]() {
-      Slice target = key();
+      std::string target(key().data(), key().size());
      InitMinHeap();
      Seek(target);
    });
@@ -94,7 +94,7 @@ class MultiCfIteratorImpl {
  void Prev() {
    assert(Valid());
    auto& max_heap = GetHeap<MultiCfMaxHeap>([this]() {
-      Slice target = key();
+      std::string target(key().data(), key().size());
      InitMaxHeap();
      SeekForPrev(target);
    });
@@ -198,8 +198,15 @@ class MultiCfIteratorImpl {

  template <typename BinaryHeap, typename AdvanceFuncType>
  void AdvanceIterator(BinaryHeap& heap, AdvanceFuncType advance_func) {
-    assert(!heap.empty());
    reset_func_();
+    // It is possible for one or more child iters are at invalid keys due to
+    // manual prefix iteration. For such cases, we consider the result of the
+    // multi-cf-iter is also undefined.
+    // https://github.com/facebook/rocksdb/wiki/Prefix-Seek#manual-prefix-iterating
+    // for details about manual prefix iteration
+    if (heap.empty()) {
+      return;
+    }

    // 1. Keep the top iterator (by popping it from the heap)
    // 2. Make sure all others have iterated past the top iterator key slice
@@ -17,16 +17,6 @@
 #include "table/internal_iterator.h"

 namespace ROCKSDB_NAMESPACE {
-struct FragmentedRangeTombstoneList;
-
-struct FragmentedRangeTombstoneListCache {
-  // ensure only the first reader needs to initialize l
-  std::mutex reader_mutex;
-  std::unique_ptr<FragmentedRangeTombstoneList> tombstones = nullptr;
-  // readers will first check this bool to avoid
-  std::atomic<bool> initialized = false;
-};
-
 struct FragmentedRangeTombstoneList {
 public:
  // A compact representation of a "stack" of range tombstone fragments, which
@@ -124,6 +114,14 @@ struct FragmentedRangeTombstoneList {
  uint64_t total_tombstone_payload_bytes_;
 };

+struct FragmentedRangeTombstoneListCache {
+  // ensure only the first reader needs to initialize l
+  std::mutex reader_mutex;
+  std::unique_ptr<FragmentedRangeTombstoneList> tombstones = nullptr;
+  // readers will first check this bool to avoid
+  std::atomic<bool> initialized = false;
+};
+
 // FragmentedRangeTombstoneIterator converts an InternalIterator of a range-del
 // meta block into an iterator over non-overlapping tombstone fragments. The
 // tombstone fragmentation process should be more efficient than the range
@@ -480,9 +480,10 @@ class Repairer {
          dbname_, /* versions */ nullptr, immutable_db_options_, tboptions,
          file_options_, table_cache_.get(), iter.get(),
          std::move(range_del_iters), &meta, nullptr /* blob_file_additions */,
-          {}, kMaxSequenceNumber, kMaxSequenceNumber, snapshot_checker,
-          false /* paranoid_file_checks*/, nullptr /* internal_stats */, &io_s,
-          nullptr /*IOTracer*/, BlobFileCreationReason::kRecovery,
+          {}, kMaxSequenceNumber, kMaxSequenceNumber, kMaxSequenceNumber,
+          snapshot_checker, false /* paranoid_file_checks*/,
+          nullptr /* internal_stats */, &io_s, nullptr /*IOTracer*/,
+          BlobFileCreationReason::kRecovery,
          nullptr /* seqno_to_time_mapping */, nullptr /* event_logger */,
          0 /* job_id */, nullptr /* table_properties */, write_hint);
      ROCKS_LOG_INFO(db_options_.info_log,
@@ -69,6 +69,36 @@ SequenceNumber SeqnoToTimeMapping::GetProximalSeqnoBeforeTime(
  return it->seqno;
 }

+void SeqnoToTimeMapping::GetCurrentTieringCutoffSeqnos(
+    uint64_t current_time, uint64_t preserve_internal_time_seconds,
+    uint64_t preclude_last_level_data_seconds,
+    SequenceNumber* preserve_time_min_seqno,
+    SequenceNumber* preclude_last_level_min_seqno) const {
+  uint64_t preserve_time_duration = std::max(preserve_internal_time_seconds,
+                                             preclude_last_level_data_seconds);
+  if (preserve_time_duration <= 0) {
+    return;
+  }
+  uint64_t preserve_time = current_time > preserve_time_duration
+                               ? current_time - preserve_time_duration
+                               : 0;
+  // GetProximalSeqnoBeforeTime tells us the last seqno known to have been
+  // written at or before the given time. + 1 to get the minimum we should
+  // preserve without excluding anything that might have been written on or
+  // after the given time.
+  if (preserve_time_min_seqno) {
+    *preserve_time_min_seqno = GetProximalSeqnoBeforeTime(preserve_time) + 1;
+  }
+  if (preclude_last_level_data_seconds > 0 && preclude_last_level_min_seqno) {
+    uint64_t preclude_last_level_time =
+        current_time > preclude_last_level_data_seconds
+            ? current_time - preclude_last_level_data_seconds
+            : 0;
+    *preclude_last_level_min_seqno =
+        GetProximalSeqnoBeforeTime(preclude_last_level_time) + 1;
+  }
+}
+
 void SeqnoToTimeMapping::EnforceMaxTimeSpan(uint64_t now) {
  assert(enforced_);  // at least sorted
  uint64_t cutoff_time;
@@ -213,6 +213,15 @@ class SeqnoToTimeMapping {
  // must be in enforced state as a precondition.
  SequenceNumber GetProximalSeqnoBeforeTime(uint64_t time) const;

+  // Given current time, the configured `preserve_internal_time_seconds`, and
+  // `preclude_last_level_data_seconds`, find the relevant cutoff sequence
+  // numbers for tiering.
+  void GetCurrentTieringCutoffSeqnos(
+      uint64_t current_time, uint64_t preserve_internal_time_seconds,
+      uint64_t preclude_last_level_data_seconds,
+      SequenceNumber* preserve_time_min_seqno,
+      SequenceNumber* preclude_last_level_min_seqno) const;
+
  // Encode to a binary string by appending to `dest`.
  // Because this is a const operation depending on sortedness, the structure
  // must be in enforced state as a precondition.
@@ -163,6 +163,11 @@ Status TableCache::GetTableReader(
  return s;
 }

+Cache::Handle* TableCache::Lookup(Cache* cache, uint64_t file_number) {
+  Slice key = GetSliceForFileNumber(&file_number);
+  return cache->Lookup(key);
+}
+
 Status TableCache::FindTable(
    const ReadOptions& ro, const FileOptions& file_options,
    const InternalKeyComparator& internal_comparator,
@@ -727,4 +732,14 @@ uint64_t TableCache::ApproximateSize(

  return result;
 }
+
+void TableCache::ReleaseObsolete(Cache* cache, Cache::Handle* h,
+                                 uint32_t uncache_aggressiveness) {
+  CacheInterface typed_cache(cache);
+  TypedHandle* table_handle = reinterpret_cast<TypedHandle*>(h);
+  TableReader* table_reader = typed_cache.Value(table_handle);
+  table_reader->MarkObsolete(uncache_aggressiveness);
+  typed_cache.ReleaseAndEraseIfLastRef(table_handle);
+}
+
 }  // namespace ROCKSDB_NAMESPACE
@@ -165,6 +165,14 @@ class TableCache {
  // Evict any entry for the specified file number
  static void Evict(Cache* cache, uint64_t file_number);

+  // Handles releasing, erasing, etc. of what should be the last reference
+  // to an obsolete file.
+  static void ReleaseObsolete(Cache* cache, Cache::Handle* handle,
+                              uint32_t uncache_aggressiveness);
+
+  // Return handle to an existing cache entry if there is one
+  static Cache::Handle* Lookup(Cache* cache, uint64_t file_number);
+
  // Find table reader
  // @param skip_filters Disables loading/accessing the filter block
  // @param level == -1 means not specified
@@ -44,7 +44,9 @@ class InternalTblPropCollFactory {
  virtual ~InternalTblPropCollFactory() {}
  // has to be thread-safe
  virtual InternalTblPropColl* CreateInternalTblPropColl(
-      uint32_t column_family_id, int level_at_creation) = 0;
+      uint32_t column_family_id, int level_at_creation, int num_levels,
+      SequenceNumber last_level_inclusive_max_seqno_threshold =
+          kMaxSequenceNumber) = 0;

  // The name of the properties collector can be used for debugging purpose.
  virtual const char* Name() const = 0;
@@ -92,10 +94,15 @@ class UserKeyTablePropertiesCollectorFactory
      std::shared_ptr<TablePropertiesCollectorFactory> user_collector_factory)
      : user_collector_factory_(user_collector_factory) {}
  InternalTblPropColl* CreateInternalTblPropColl(
-      uint32_t column_family_id, int level_at_creation) override {
+      uint32_t column_family_id, int level_at_creation, int num_levels,
+      SequenceNumber last_level_inclusive_max_seqno_threshold =
+          kMaxSequenceNumber) override {
    TablePropertiesCollectorFactory::Context context;
    context.column_family_id = column_family_id;
    context.level_at_creation = level_at_creation;
+    context.num_levels = num_levels;
+    context.last_level_inclusive_max_seqno_threshold =
+        last_level_inclusive_max_seqno_threshold;
    TablePropertiesCollector* collector =
        user_collector_factory_->CreateTablePropertiesCollector(context);
    if (collector) {
@@ -209,7 +209,9 @@ class RegularKeysStartWithAFactory : public InternalTblPropCollFactory,
    }
  }
  InternalTblPropColl* CreateInternalTblPropColl(
-      uint32_t /*column_family_id*/, int /* level_at_creation */) override {
+      uint32_t /*column_family_id*/, int /* level_at_creation */,
+      int /* num_levels */,
+      SequenceNumber /* last_level_inclusive_max_seqno_threshold */) override {
    return new RegularKeysStartWithAInternal();
  }
  const char* Name() const override { return "RegularKeysStartWithA"; }
@@ -18,7 +18,7 @@ TransactionLogIteratorImpl::TransactionLogIteratorImpl(
    const std::string& dir, const ImmutableDBOptions* options,
    const TransactionLogIterator::ReadOptions& read_options,
    const EnvOptions& soptions, const SequenceNumber seq,
-    std::unique_ptr<VectorLogPtr> files, VersionSet const* const versions,
+    std::unique_ptr<VectorWalPtr> files, VersionSet const* const versions,
    const bool seq_per_batch, const std::shared_ptr<IOTracer>& io_tracer)
    : dir_(dir),
      options_(options),
@@ -44,7 +44,7 @@ TransactionLogIteratorImpl::TransactionLogIteratorImpl(
 }

 Status TransactionLogIteratorImpl::OpenLogFile(
-    const LogFile* log_file,
+    const WalFile* log_file,
    std::unique_ptr<SequentialFileReader>* file_reader) {
  FileSystemPtr fs(options_->fs, io_tracer_);
  std::unique_ptr<FSSequentialFile> file;
@@ -281,7 +281,7 @@ void TransactionLogIteratorImpl::UpdateCurrentWriteBatch(const Slice& record) {
  current_status_ = Status::OK();
 }

-Status TransactionLogIteratorImpl::OpenLogReader(const LogFile* log_file) {
+Status TransactionLogIteratorImpl::OpenLogReader(const WalFile* log_file) {
  std::unique_ptr<SequentialFileReader> file;
  Status s = OpenLogFile(log_file, &file);
  if (!s.ok()) {
@@ -19,9 +19,9 @@

 namespace ROCKSDB_NAMESPACE {

-class LogFileImpl : public LogFile {
+class WalFileImpl : public WalFile {
 public:
-  LogFileImpl(uint64_t logNum, WalFileType logType, SequenceNumber startSeq,
+  WalFileImpl(uint64_t logNum, WalFileType logType, SequenceNumber startSeq,
              uint64_t sizeBytes)
      : logNumber_(logNum),
        type_(logType),
@@ -43,7 +43,7 @@ class LogFileImpl : public LogFile {

  uint64_t SizeFileBytes() const override { return sizeFileBytes_; }

-  bool operator<(const LogFile& that) const {
+  bool operator<(const WalFile& that) const {
    return LogNumber() < that.LogNumber();
  }

@@ -60,7 +60,7 @@ class TransactionLogIteratorImpl : public TransactionLogIterator {
      const std::string& dir, const ImmutableDBOptions* options,
      const TransactionLogIterator::ReadOptions& read_options,
      const EnvOptions& soptions, const SequenceNumber seqNum,
-      std::unique_ptr<VectorLogPtr> files, VersionSet const* const versions,
+      std::unique_ptr<VectorWalPtr> files, VersionSet const* const versions,
      const bool seq_per_batch, const std::shared_ptr<IOTracer>& io_tracer);

  bool Valid() override;
@@ -77,7 +77,7 @@ class TransactionLogIteratorImpl : public TransactionLogIterator {
  const TransactionLogIterator::ReadOptions read_options_;
  const EnvOptions& soptions_;
  SequenceNumber starting_sequence_number_;
-  std::unique_ptr<VectorLogPtr> files_;
+  std::unique_ptr<VectorWalPtr> files_;
  // Used only to get latest seq. num
  // TODO(icanadi) can this be just a callback?
  VersionSet const* const versions_;
@@ -92,7 +92,7 @@ class TransactionLogIteratorImpl : public TransactionLogIterator {
  std::unique_ptr<WriteBatch> current_batch_;
  std::unique_ptr<log::Reader> current_log_reader_;
  std::string scratch_;
-  Status OpenLogFile(const LogFile* log_file,
+  Status OpenLogFile(const WalFile* log_file,
                     std::unique_ptr<SequentialFileReader>* file);

  struct LogReporter : public log::Reader::Reporter {
@@ -123,6 +123,6 @@ class TransactionLogIteratorImpl : public TransactionLogIterator {
  bool IsBatchExpected(const WriteBatch* batch, SequenceNumber expected_seq);
  // Update current batch if a continuous batch is found.
  void UpdateCurrentWriteBatch(const Slice& record);
-  Status OpenLogReader(const LogFile* file);
+  Status OpenLogReader(const WalFile* file);
 };
 }  // namespace ROCKSDB_NAMESPACE
@@ -364,14 +364,15 @@ class DumpManifestHandler : public VersionEditHandler {

  Status ApplyVersionEdit(VersionEdit& edit, ColumnFamilyData** cfd) override {
    // Write out each individual edit
-    if (verbose_ && !json_) {
+    if (json_) {
      // Print out DebugStrings. Can include non-terminating null characters.
-      fwrite(edit.DebugString(hex_).data(), sizeof(char),
-             edit.DebugString(hex_).size(), stdout);
-    } else if (json_) {
+      std::string edit_dump_str = edit.DebugJSON(count_, hex_);
+      fwrite(edit_dump_str.data(), sizeof(char), edit_dump_str.size(), stdout);
+      fwrite("\n", sizeof(char), 1, stdout);
+    } else if (verbose_) {
      // Print out DebugStrings. Can include non-terminating null characters.
-      fwrite(edit.DebugString(hex_).data(), sizeof(char),
-             edit.DebugString(hex_).size(), stdout);
+      std::string edit_dump_str = edit.DebugString(hex_);
+      fwrite(edit_dump_str.data(), sizeof(char), edit_dump_str.size(), stdout);
    }
    ++count_;
    return VersionEditHandler::ApplyVersionEdit(edit, cfd);
@@ -857,10 +857,14 @@ Version::~Version() {
      f->refs--;
      if (f->refs <= 0) {
        assert(cfd_ != nullptr);
+        // When not in the process of closing the DB, we'll have a superversion
+        // to get current mutable options from
+        auto* sv = cfd_->GetSuperVersion();
        uint32_t path_id = f->fd.GetPathId();
        assert(path_id < cfd_->ioptions()->cf_paths.size());
        vset_->obsolete_files_.emplace_back(
            f, cfd_->ioptions()->cf_paths[path_id].path,
+            sv ? sv->mutable_cf_options.uncache_aggressiveness : 0,
            cfd_->GetFileMetadataCacheReservationManager());
      }
    }
@@ -3501,6 +3505,10 @@ void VersionStorageInfo::ComputeCompactionScore(
          score = kScoreForNeedCompaction;
        }
      } else {
+        // For universal compaction, if a user configures `max_read_amp`, then
+        // the score may be a false positive signal.
+        // `level0_file_num_compaction_trigger` is used as a trigger to check
+        // if there is any compaction work to do.
        score = static_cast<double>(num_sorted_runs) /
                mutable_cf_options.level0_file_num_compaction_trigger;
        if (compaction_style_ == kCompactionStyleLevel && num_levels() > 1) {
@@ -5188,13 +5196,21 @@ Status VersionSet::Close(FSDirectory* db_dir, InstrumentedMutex* mu) {
 }

 VersionSet::~VersionSet() {
-  // we need to delete column_family_set_ because its destructor depends on
-  // VersionSet
+  // Must clean up column families to make all files "obsolete"
  column_family_set_.reset();
+
  for (auto& file : obsolete_files_) {
    if (file.metadata->table_reader_handle) {
-      table_cache_->Release(file.metadata->table_reader_handle);
-      TableCache::Evict(table_cache_, file.metadata->fd.GetNumber());
+      // NOTE: DB is shutting down, so file is probably not obsolete, just
+      // no longer referenced by Versions in memory.
+      // For more context, see comment on "table_cache_->EraseUnRefEntries()"
+      // in DBImpl::CloseHelper().
+      // Using uncache_aggressiveness=0 overrides any previous marking to
+      // attempt to uncache the file's blocks (which after cleaning up
+      // column families could cause use-after-free)
+      TableCache::ReleaseObsolete(table_cache_,
+                                  file.metadata->table_reader_handle,
+                                  /*uncache_aggressiveness=*/0);
    }
    file.DeleteMetadata();
  }
@@ -797,16 +797,20 @@ struct ObsoleteFileInfo {
  // the file, usually because the file is trivial moved so two FileMetadata
  // is managing the file.
  bool only_delete_metadata = false;
+  // To apply to this file
+  uint32_t uncache_aggressiveness = 0;

  ObsoleteFileInfo() noexcept
      : metadata(nullptr), only_delete_metadata(false) {}
  ObsoleteFileInfo(FileMetaData* f, const std::string& file_path,
+                   uint32_t _uncache_aggressiveness,
                   std::shared_ptr<CacheReservationManager>
                       file_metadata_cache_res_mgr_arg = nullptr)
      : metadata(f),
        path(file_path),
-        only_delete_metadata(false),
-        file_metadata_cache_res_mgr(file_metadata_cache_res_mgr_arg) {}
+        uncache_aggressiveness(_uncache_aggressiveness),
+        file_metadata_cache_res_mgr(
+            std::move(file_metadata_cache_res_mgr_arg)) {}

  ObsoleteFileInfo(const ObsoleteFileInfo&) = delete;
  ObsoleteFileInfo& operator=(const ObsoleteFileInfo&) = delete;
@@ -816,9 +820,13 @@ struct ObsoleteFileInfo {
  }

  ObsoleteFileInfo& operator=(ObsoleteFileInfo&& rhs) noexcept {
-    path = std::move(rhs.path);
    metadata = rhs.metadata;
    rhs.metadata = nullptr;
+    path = std::move(rhs.path);
+    only_delete_metadata = rhs.only_delete_metadata;
+    rhs.only_delete_metadata = false;
+    uncache_aggressiveness = rhs.uncache_aggressiveness;
+    rhs.uncache_aggressiveness = 0;
    file_metadata_cache_res_mgr = rhs.file_metadata_cache_res_mgr;
    rhs.file_metadata_cache_res_mgr = nullptr;

@@ -1495,10 +1503,7 @@ class VersionSet {
  void GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata);

  void AddObsoleteBlobFile(uint64_t blob_file_number, std::string path) {
-    assert(table_cache_);
-
-    table_cache_->Erase(GetSliceForKey(&blob_file_number));
-
+    // TODO: Erase file from BlobFileCache?
    obsolete_blob_files_.emplace_back(blob_file_number, std::move(path));
  }

@@ -1676,6 +1681,8 @@ class VersionSet {
  // Current size of manifest file
  uint64_t manifest_file_size_;

+  // Obsolete files, or during DB shutdown any files not referenced by what's
+  // left of the in-memory LSM state.
  std::vector<ObsoleteFileInfo> obsolete_files_;
  std::vector<ObsoleteBlobFileInfo> obsolete_blob_files_;
  std::vector<std::string> obsolete_manifests_;
@@ -44,13 +44,13 @@ Status WalManager::DeleteFile(const std::string& fname, uint64_t number) {
  return s;
 }

-Status WalManager::GetSortedWalFiles(VectorLogPtr& files) {
+Status WalManager::GetSortedWalFiles(VectorWalPtr& files) {
  // First get sorted files in db dir, then get sorted files from archived
  // dir, to avoid a race condition where a log file is moved to archived
  // dir in between.
  Status s;
  // list wal files in main db dir.
-  VectorLogPtr logs;
+  VectorWalPtr logs;
  s = GetSortedWalsOfType(wal_dir_, logs, kAliveLogFile);
  if (!s.ok()) {
    return s;
@@ -72,8 +72,8 @@ Status WalManager::GetSortedWalFiles(VectorLogPtr& files) {
      return s;
    }
  } else if (!exists.IsNotFound()) {
-    assert(s.IsIOError());
-    return s;
+    assert(s.ok());
+    return exists;
  }

  uint64_t latest_archived_log_number = 0;
@@ -113,7 +113,7 @@ Status WalManager::GetUpdatesSince(
  //  Get all sorted Wal Files.
  //  Do binary search and open files and find the seq number.

-  std::unique_ptr<VectorLogPtr> wal_files(new VectorLogPtr);
+  std::unique_ptr<VectorWalPtr> wal_files(new VectorWalPtr);
  Status s = GetSortedWalFiles(*wal_files);
  if (!s.ok()) {
    return s;
@@ -173,7 +173,6 @@ void WalManager::PurgeObsoleteWALFiles() {
  if (!s.ok()) {
    ROCKS_LOG_ERROR(db_options_.info_log, "Can't get archive files: %s",
                    s.ToString().c_str());
-    assert(false);
    return;
  }

@@ -249,7 +248,7 @@ void WalManager::PurgeObsoleteWALFiles() {
  }

  size_t files_del_num = log_files_num - files_keep_num;
-  VectorLogPtr archived_logs;
+  VectorWalPtr archived_logs;
  s = GetSortedWalsOfType(archival_dir, archived_logs, kArchivedLogFile);
  if (!s.ok()) {
    ROCKS_LOG_WARN(db_options_.info_log,
@@ -291,7 +290,7 @@ void WalManager::ArchiveWALFile(const std::string& fname, uint64_t number) {
 }

 Status WalManager::GetSortedWalsOfType(const std::string& path,
-                                       VectorLogPtr& log_files,
+                                       VectorWalPtr& log_files,
                                       WalFileType log_type) {
  std::vector<std::string> all_files;
  const Status status = env_->GetChildren(path, &all_files);
@@ -338,20 +337,20 @@ Status WalManager::GetSortedWalsOfType(const std::string& path,
      }

      log_files.emplace_back(
-          new LogFileImpl(number, log_type, sequence, size_bytes));
+          new WalFileImpl(number, log_type, sequence, size_bytes));
    }
  }
  std::sort(
      log_files.begin(), log_files.end(),
-      [](const std::unique_ptr<LogFile>& a, const std::unique_ptr<LogFile>& b) {
-        LogFileImpl* a_impl = static_cast_with_check<LogFileImpl>(a.get());
-        LogFileImpl* b_impl = static_cast_with_check<LogFileImpl>(b.get());
+      [](const std::unique_ptr<WalFile>& a, const std::unique_ptr<WalFile>& b) {
+        WalFileImpl* a_impl = static_cast_with_check<WalFileImpl>(a.get());
+        WalFileImpl* b_impl = static_cast_with_check<WalFileImpl>(b.get());
        return *a_impl < *b_impl;
      });
  return status;
 }

-Status WalManager::RetainProbableWalFiles(VectorLogPtr& all_logs,
+Status WalManager::RetainProbableWalFiles(VectorWalPtr& all_logs,
                                          const SequenceNumber target) {
  int64_t start = 0;  // signed to avoid overflow when target is < first file.
  int64_t end = static_cast<int64_t>(all_logs.size()) - 1;
@@ -424,7 +423,7 @@ Status WalManager::ReadFirstRecord(const WalFileType type,
 }

 Status WalManager::GetLiveWalFile(uint64_t number,
-                                  std::unique_ptr<LogFile>* log_file) {
+                                  std::unique_ptr<WalFile>* log_file) {
  if (!log_file) {
    return Status::InvalidArgument("log_file not preallocated.");
  }
@@ -442,7 +441,7 @@ Status WalManager::GetLiveWalFile(uint64_t number,
    return s;
  }

-  log_file->reset(new LogFileImpl(number, kAliveLogFile,
+  log_file->reset(new WalFileImpl(number, kAliveLogFile,
                                  0,  // SequenceNumber
                                  size_bytes));

@@ -49,7 +49,7 @@ class WalManager {
        wal_in_db_path_(db_options_.IsWalDirSameAsDBPath()),
        io_tracer_(io_tracer) {}

-  Status GetSortedWalFiles(VectorLogPtr& files);
+  Status GetSortedWalFiles(VectorWalPtr& files);

  // Allow user to tail transaction log to find all recent changes to the
  // database that are newer than `seq_number`.
@@ -64,7 +64,7 @@ class WalManager {

  Status DeleteFile(const std::string& fname, uint64_t number);

-  Status GetLiveWalFile(uint64_t number, std::unique_ptr<LogFile>* log_file);
+  Status GetLiveWalFile(uint64_t number, std::unique_ptr<WalFile>* log_file);

  Status TEST_ReadFirstRecord(const WalFileType type, const uint64_t number,
                              SequenceNumber* sequence) {
@@ -77,12 +77,12 @@ class WalManager {
  }

 private:
-  Status GetSortedWalsOfType(const std::string& path, VectorLogPtr& log_files,
+  Status GetSortedWalsOfType(const std::string& path, VectorWalPtr& log_files,
                             WalFileType type);
  // Requires: all_logs should be sorted with earliest log file first
  // Retains all log files in all_logs which contain updates with seq no.
  // Greater Than or Equal to the requested SequenceNumber.
-  Status RetainProbableWalFiles(VectorLogPtr& all_logs,
+  Status RetainProbableWalFiles(VectorWalPtr& all_logs,
                                const SequenceNumber target);

  // ReadFirstRecord checks the read_first_record_cache_ to see if the entry
@@ -6,6 +6,7 @@
 #include "db/wide/wide_columns_helper.h"

 #include <algorithm>
+#include <ios>

 #include "db/wide/wide_column_serialization.h"

@@ -15,6 +16,9 @@ void WideColumnsHelper::DumpWideColumns(const WideColumns& columns,
  if (columns.empty()) {
    return;
  }
+
+  const std::ios_base::fmtflags orig_flags = os.flags();
+
  if (hex) {
    os << std::hex;
  }
@@ -23,6 +27,8 @@ void WideColumnsHelper::DumpWideColumns(const WideColumns& columns,
  for (++it; it != columns.end(); ++it) {
    os << ' ' << *it;
  }
+
+  os.flags(orig_flags);
 }

 Status WideColumnsHelper::DumpSliceAsWideColumns(const Slice& value,
@@ -65,6 +65,7 @@
 #include "port/lang.h"
 #include "rocksdb/merge_operator.h"
 #include "rocksdb/system_clock.h"
+#include "util/aligned_storage.h"
 #include "util/autovector.h"
 #include "util/cast_util.h"
 #include "util/coding.h"
@@ -1900,7 +1901,7 @@ class MemTableInserter : public WriteBatch::Handler {
  // Make creation optional but do not incur
  // std::unique_ptr additional allocation
  using MemPostInfoMap = std::map<MemTable*, MemTablePostProcessInfo>;
-  using PostMapType = std::aligned_storage<sizeof(MemPostInfoMap)>::type;
+  using PostMapType = aligned_storage<MemPostInfoMap>::type;
  PostMapType mem_post_info_map_;
  // current recovered transaction we are rebuilding (recovery)
  WriteBatch* rebuilding_trx_;
@@ -1914,7 +1915,7 @@ class MemTableInserter : public WriteBatch::Handler {
  bool write_before_prepare_;
  // Whether this batch was unprepared or not
  bool unprepared_batch_;
-  using DupDetector = std::aligned_storage<sizeof(DuplicateDetector)>::type;
+  using DupDetector = aligned_storage<DuplicateDetector>::type;
  DupDetector duplicate_detector_;
  bool dup_dectector_on_;

@@ -1922,7 +1923,7 @@ class MemTableInserter : public WriteBatch::Handler {
  bool hint_created_;
  // Hints for this batch
  using HintMap = std::unordered_map<MemTable*, void*>;
-  using HintMapType = std::aligned_storage<sizeof(HintMap)>::type;
+  using HintMapType = aligned_storage<HintMap>::type;
  HintMapType hint_;

  HintMap& GetHintMap() {
@@ -2121,14 +2122,15 @@ class MemTableInserter : public WriteBatch::Handler {
    return true;
  }

+  template <typename RebuildTxnOp>
  Status PutCFImpl(uint32_t column_family_id, const Slice& key,
                   const Slice& value, ValueType value_type,
+                   RebuildTxnOp rebuild_txn_op,
                   const ProtectionInfoKVOS64* kv_prot_info) {
    // optimize for non-recovery mode
    if (UNLIKELY(write_after_commit_ && rebuilding_trx_ != nullptr)) {
      // TODO(ajkr): propagate `ProtectionInfoKVOS64`.
-      return WriteBatchInternal::Put(rebuilding_trx_, column_family_id, key,
-                                     value);
+      return rebuild_txn_op(rebuilding_trx_, column_family_id, key, value);
      // else insert the values to the memtable right away
    }

@@ -2139,8 +2141,8 @@ class MemTableInserter : public WriteBatch::Handler {
        // The CF is probably flushed and hence no need for insert but we still
        // need to keep track of the keys for upcoming rollback/commit.
        // TODO(ajkr): propagate `ProtectionInfoKVOS64`.
-        ret_status = WriteBatchInternal::Put(rebuilding_trx_, column_family_id,
-                                             key, value);
+        ret_status =
+            rebuild_txn_op(rebuilding_trx_, column_family_id, key, value);
        if (ret_status.ok()) {
          MaybeAdvanceSeq(IsDuplicateKeySeq(column_family_id, key));
        }
@@ -2264,8 +2266,8 @@ class MemTableInserter : public WriteBatch::Handler {
    if (UNLIKELY(ret_status.ok() && rebuilding_trx_ != nullptr)) {
      assert(!write_after_commit_);
      // TODO(ajkr): propagate `ProtectionInfoKVOS64`.
-      ret_status = WriteBatchInternal::Put(rebuilding_trx_, column_family_id,
-                                           key, value);
+      ret_status =
+          rebuild_txn_op(rebuilding_trx_, column_family_id, key, value);
    }
    return ret_status;
  }
@@ -2274,15 +2276,21 @@ class MemTableInserter : public WriteBatch::Handler {
               const Slice& value) override {
    const auto* kv_prot_info = NextProtectionInfo();
    Status ret_status;
+
+    auto rebuild_txn_op = [](WriteBatch* rebuilding_trx, uint32_t cf_id,
+                             const Slice& k, const Slice& v) -> Status {
+      return WriteBatchInternal::Put(rebuilding_trx, cf_id, k, v);
+    };
+
    if (kv_prot_info != nullptr) {
      // Memtable needs seqno, doesn't need CF ID
      auto mem_kv_prot_info =
          kv_prot_info->StripC(column_family_id).ProtectS(sequence_);
      ret_status = PutCFImpl(column_family_id, key, value, kTypeValue,
-                             &mem_kv_prot_info);
+                             rebuild_txn_op, &mem_kv_prot_info);
    } else {
      ret_status = PutCFImpl(column_family_id, key, value, kTypeValue,
-                             nullptr /* kv_prot_info */);
+                             rebuild_txn_op, nullptr /* kv_prot_info */);
    }
    // TODO: this assumes that if TryAgain status is returned to the caller,
    // the operation is actually tried again. The proper way to do this is to
@@ -2301,15 +2309,23 @@ class MemTableInserter : public WriteBatch::Handler {
    std::string value_buf;
    Slice packed_value =
        PackValueAndWriteTime(value, unix_write_time, &value_buf);
+
+    auto rebuild_txn_op = [](WriteBatch* /* rebuilding_trx */,
+                             uint32_t /* cf_id */, const Slice& /* k */,
+                             const Slice& /* v */) -> Status {
+      return Status::NotSupported();
+    };
+
    if (kv_prot_info != nullptr) {
      auto mem_kv_prot_info =
          kv_prot_info->StripC(column_family_id).ProtectS(sequence_);
      ret_status = PutCFImpl(column_family_id, key, packed_value,
-                             kTypeValuePreferredSeqno, &mem_kv_prot_info);
+                             kTypeValuePreferredSeqno, rebuild_txn_op,
+                             &mem_kv_prot_info);
    } else {
-      ret_status =
-          PutCFImpl(column_family_id, key, packed_value,
-                    kTypeValuePreferredSeqno, nullptr /* kv_prot_info */);
+      ret_status = PutCFImpl(column_family_id, key, packed_value,
+                             kTypeValuePreferredSeqno, rebuild_txn_op,
+                             nullptr /* kv_prot_info */);
    }

    // TODO: this assumes that if TryAgain status is returned to the caller,
@@ -2327,14 +2343,27 @@ class MemTableInserter : public WriteBatch::Handler {
    const auto* kv_prot_info = NextProtectionInfo();

    Status s;
+
+    auto rebuild_txn_op = [](WriteBatch* rebuilding_trx, uint32_t cf_id,
+                             const Slice& k, Slice entity) -> Status {
+      WideColumns columns;
+      const Status st = WideColumnSerialization::Deserialize(entity, columns);
+      if (!st.ok()) {
+        return st;
+      }
+
+      return WriteBatchInternal::PutEntity(rebuilding_trx, cf_id, k, columns);
+    };
+
    if (kv_prot_info) {
      // Memtable needs seqno, doesn't need CF ID
      auto mem_kv_prot_info =
          kv_prot_info->StripC(column_family_id).ProtectS(sequence_);
      s = PutCFImpl(column_family_id, key, value, kTypeWideColumnEntity,
-                    &mem_kv_prot_info);
+                    rebuild_txn_op, &mem_kv_prot_info);
    } else {
      s = PutCFImpl(column_family_id, key, value, kTypeWideColumnEntity,
+                    rebuild_txn_op,
                    /* kv_prot_info */ nullptr);
    }

@@ -2521,11 +2550,6 @@ class MemTableInserter : public WriteBatch::Handler {
    assert(ret_status.ok());

    if (db_ != nullptr) {
-      if (db_->immutable_db_options().row_cache) {
-        ret_status.PermitUncheckedError();
-        return Status::NotSupported(
-            "DeleteRange is not compatible with row cache.");
-      }
      auto cf_handle = cf_mems_->GetColumnFamilyHandle();
      if (cf_handle == nullptr) {
        cf_handle = db_->DefaultColumnFamily();
@@ -2778,16 +2802,23 @@ class MemTableInserter : public WriteBatch::Handler {
                        const Slice& value) override {
    const auto* kv_prot_info = NextProtectionInfo();
    Status ret_status;
+
+    auto rebuild_txn_op = [](WriteBatch* /* rebuilding_trx */,
+                             uint32_t /* cf_id */, const Slice& /* k */,
+                             const Slice& /* v */) -> Status {
+      return Status::NotSupported();
+    };
+
    if (kv_prot_info != nullptr) {
      // Memtable needs seqno, doesn't need CF ID
      auto mem_kv_prot_info =
          kv_prot_info->StripC(column_family_id).ProtectS(sequence_);
      // Same as PutCF except for value type.
      ret_status = PutCFImpl(column_family_id, key, value, kTypeBlobIndex,
-                             &mem_kv_prot_info);
+                             rebuild_txn_op, &mem_kv_prot_info);
    } else {
      ret_status = PutCFImpl(column_family_id, key, value, kTypeBlobIndex,
-                             nullptr /* kv_prot_info */);
+                             rebuild_txn_op, nullptr /* kv_prot_info */);
    }
    if (UNLIKELY(ret_status.IsTryAgain())) {
      DecrementProtectionInfoIdxForTryAgain();
@@ -2,8 +2,6 @@
 //  This source code is licensed under both the GPLv2 (found in the
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
-
-
 #include "db/write_callback.h"

 #include <atomic>
@@ -15,6 +13,7 @@
 #include "db/db_impl/db_impl.h"
 #include "port/port.h"
 #include "rocksdb/db.h"
+#include "rocksdb/user_write_callback.h"
 #include "rocksdb/write_batch.h"
 #include "test_util/sync_point.h"
 #include "test_util/testharness.h"
@@ -84,6 +83,28 @@ class MockWriteCallback : public WriteCallback {
  bool AllowWriteBatching() override { return allow_batching_; }
 };

+class MockUserWriteCallback : public UserWriteCallback {
+ public:
+  std::atomic<bool> write_enqueued_{false};
+  std::atomic<bool> wal_write_done_{false};
+
+  MockUserWriteCallback() = default;
+
+  MockUserWriteCallback(const MockUserWriteCallback& other) {
+    write_enqueued_.store(other.write_enqueued_.load());
+    wal_write_done_.store(other.wal_write_done_.load());
+  }
+
+  void OnWriteEnqueued() override { write_enqueued_.store(true); }
+
+  void OnWalWriteFinish() override { wal_write_done_.store(true); }
+
+  void Reset() {
+    write_enqueued_.store(false);
+    wal_write_done_.store(false);
+  }
+};
+
 #if !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN)
 class WriteCallbackPTest
    : public WriteCallbackTest,
@@ -119,9 +140,11 @@ TEST_P(WriteCallbackPTest, WriteWithCallbackTest) {
      kvs_.clear();
      write_batch_.Clear();
      callback_.was_called_.store(false);
+      user_write_cb_.Reset();
    }

    MockWriteCallback callback_;
+    MockUserWriteCallback user_write_cb_;
    WriteBatch write_batch_;
    std::vector<std::pair<string, string>> kvs_;
  };
@@ -327,18 +350,26 @@ TEST_P(WriteCallbackPTest, WriteWithCallbackTest) {
        ASSERT_OK(WriteBatchInternal::InsertNoop(&write_op.write_batch_));
        const size_t ONE_BATCH = 1;
        s = db_impl->WriteImpl(woptions, &write_op.write_batch_,
-                               &write_op.callback_, nullptr, 0, false, nullptr,
-                               ONE_BATCH,
+                               &write_op.callback_, &write_op.user_write_cb_,
+                               nullptr, 0, false, nullptr, ONE_BATCH,
                               two_queues_ ? &publish_seq_callback : nullptr);
      } else {
        s = db_impl->WriteWithCallback(woptions, &write_op.write_batch_,
-                                       &write_op.callback_);
+                                       &write_op.callback_,
+                                       &write_op.user_write_cb_);
      }

+      ASSERT_TRUE(write_op.user_write_cb_.write_enqueued_.load());
      if (write_op.callback_.should_fail_) {
        ASSERT_TRUE(s.IsBusy());
+        ASSERT_FALSE(write_op.user_write_cb_.wal_write_done_.load());
      } else {
        ASSERT_OK(s);
+        if (enable_WAL_) {
+          ASSERT_TRUE(write_op.user_write_cb_.wal_write_done_.load());
+        } else {
+          ASSERT_FALSE(write_op.user_write_cb_.wal_write_done_.load());
+        }
      }
    };

@@ -440,6 +471,16 @@ TEST_F(WriteCallbackTest, WriteCallBackTest) {
  ASSERT_OK(s);
  ASSERT_EQ("value.a2", value);

+  MockUserWriteCallback user_write_cb;
+  WriteBatch wb4;
+  ASSERT_OK(wb4.Put("a", "value.a4"));
+
+  ASSERT_OK(db->WriteWithCallback(write_options, &wb4, &user_write_cb));
+  ASSERT_OK(db->Get(read_options, "a", &value));
+  ASSERT_EQ(value, "value.a4");
+  ASSERT_TRUE(user_write_cb.write_enqueued_.load());
+  ASSERT_TRUE(user_write_cb.wal_write_done_.load());
+
  delete db;
  ASSERT_OK(DestroyDB(dbname, options));
 }
@@ -404,6 +404,8 @@ void WriteThread::JoinBatchGroup(Writer* w) {

  bool linked_as_leader = LinkOne(w, &newest_writer_);

+  w->CheckWriteEnqueuedCallback();
+
  if (linked_as_leader) {
    SetState(w, STATE_GROUP_LEADER);
  }
@@ -428,6 +430,7 @@ void WriteThread::JoinBatchGroup(Writer* w) {
    TEST_SYNC_POINT_CALLBACK("WriteThread::JoinBatchGroup:BeganWaiting", w);
    AwaitState(w,
               STATE_GROUP_LEADER | STATE_MEMTABLE_WRITER_LEADER |
+                   STATE_PARALLEL_MEMTABLE_CALLER |
                   STATE_PARALLEL_MEMTABLE_WRITER | STATE_COMPLETED,
               &jbg_ctx);
    TEST_SYNC_POINT_CALLBACK("WriteThread::JoinBatchGroup:DoneWaiting", w);
@@ -656,12 +659,57 @@ void WriteThread::ExitAsMemTableWriter(Writer* /*self*/,
  SetState(leader, STATE_COMPLETED);
 }

+void WriteThread::SetMemWritersEachStride(Writer* w) {
+  WriteGroup* write_group = w->write_group;
+  Writer* last_writer = write_group->last_writer;
+
+  // The stride is the same for each writer in write_group, so w will
+  // call the writers with the same number in write_group mod total size
+  size_t stride = static_cast<size_t>(std::sqrt(write_group->size));
+  size_t count = 0;
+  while (w) {
+    if (count++ % stride == 0) {
+      SetState(w, STATE_PARALLEL_MEMTABLE_WRITER);
+    }
+    w = (w == last_writer) ? nullptr : w->link_newer;
+  }
+}
+
 void WriteThread::LaunchParallelMemTableWriters(WriteGroup* write_group) {
  assert(write_group != nullptr);
-  write_group->running.store(write_group->size);
-  for (auto w : *write_group) {
-    SetState(w, STATE_PARALLEL_MEMTABLE_WRITER);
+  size_t group_size = write_group->size;
+  write_group->running.store(group_size);
+
+  // The minimum number to allow the group use parallel caller mode.
+  // The number must no lower than 3;
+  const size_t MinParallelSize = 20;
+
+  // The group_size is too small, and there is no need to have
+  // the parallel partial callers.
+  if (group_size < MinParallelSize) {
+    for (auto w : *write_group) {
+      SetState(w, STATE_PARALLEL_MEMTABLE_WRITER);
+    }
+    return;
  }
+
+  // The stride is equal to std::sqrt(group_size) which can minimize
+  // the total number of leader SetSate.
+  // Set the leader itself STATE_PARALLEL_MEMTABLE_WRITER, and set
+  // (stride-1) writers to be STATE_PARALLEL_MEMTABLE_CALLER.
+  size_t stride = static_cast<size_t>(std::sqrt(group_size));
+  auto w = write_group->leader;
+  SetState(w, STATE_PARALLEL_MEMTABLE_WRITER);
+
+  for (size_t i = 1; i < stride; i++) {
+    w = w->link_newer;
+    SetState(w, STATE_PARALLEL_MEMTABLE_CALLER);
+  }
+
+  // After setting all STATE_PARALLEL_MEMTABLE_CALLER, the leader also
+  // does the job as STATE_PARALLEL_MEMTABLE_CALLER.
+  w = w->link_newer;
+  SetMemWritersEachStride(w);
 }

 static WriteThread::AdaptationContext cpmtw_ctx(
@@ -788,8 +836,8 @@ void WriteThread::ExitAsBatchGroupLeader(WriteGroup& write_group,
    }

    AwaitState(leader,
-               STATE_MEMTABLE_WRITER_LEADER | STATE_PARALLEL_MEMTABLE_WRITER |
-                   STATE_COMPLETED,
+               STATE_MEMTABLE_WRITER_LEADER | STATE_PARALLEL_MEMTABLE_CALLER |
+                   STATE_PARALLEL_MEMTABLE_WRITER | STATE_COMPLETED,
               &eabgl_ctx);
  } else {
    Writer* head = newest_writer_.load(std::memory_order_acquire);
@@ -22,7 +22,9 @@
 #include "rocksdb/options.h"
 #include "rocksdb/status.h"
 #include "rocksdb/types.h"
+#include "rocksdb/user_write_callback.h"
 #include "rocksdb/write_batch.h"
+#include "util/aligned_storage.h"
 #include "util/autovector.h"

 namespace ROCKSDB_NAMESPACE {
@@ -71,6 +73,12 @@ class WriteThread {
    // A state indicating that the thread may be waiting using StateMutex()
    // and StateCondVar()
    STATE_LOCKED_WAITING = 32,
+
+    // The state used to inform a waiting writer that it has become a
+    // caller to call some other waiting writers to write to memtable
+    // by calling SetMemWritersEachStride. After doing
+    // this, it will also write to memtable.
+    STATE_PARALLEL_MEMTABLE_CALLER = 64,
  };

  struct Writer;
@@ -127,6 +135,7 @@ class WriteThread {
    uint64_t log_used;  // log number that this batch was inserted into
    uint64_t log_ref;   // log number that memtable insert should reference
    WriteCallback* callback;
+    UserWriteCallback* user_write_cb;
    bool made_waitable;          // records lazy construction of mutex and cv
    std::atomic<uint8_t> state;  // write under StateMutex() or pre-link
    WriteGroup* write_group;
@@ -134,8 +143,8 @@ class WriteThread {
    Status status;
    Status callback_status;  // status returned by callback->Callback()

-    std::aligned_storage<sizeof(std::mutex)>::type state_mutex_bytes;
-    std::aligned_storage<sizeof(std::condition_variable)>::type state_cv_bytes;
+    aligned_storage<std::mutex>::type state_mutex_bytes;
+    aligned_storage<std::condition_variable>::type state_cv_bytes;
    Writer* link_older;  // read/write only before linking, or as leader
    Writer* link_newer;  // lazy, read/write only before linking, or as leader

@@ -153,6 +162,7 @@ class WriteThread {
          log_used(0),
          log_ref(0),
          callback(nullptr),
+          user_write_cb(nullptr),
          made_waitable(false),
          state(STATE_INIT),
          write_group(nullptr),
@@ -161,8 +171,8 @@ class WriteThread {
          link_newer(nullptr) {}

    Writer(const WriteOptions& write_options, WriteBatch* _batch,
-           WriteCallback* _callback, uint64_t _log_ref, bool _disable_memtable,
-           size_t _batch_cnt = 0,
+           WriteCallback* _callback, UserWriteCallback* _user_write_cb,
+           uint64_t _log_ref, bool _disable_memtable, size_t _batch_cnt = 0,
           PreReleaseCallback* _pre_release_callback = nullptr,
           PostMemTableCallback* _post_memtable_callback = nullptr)
        : batch(_batch),
@@ -180,6 +190,7 @@ class WriteThread {
          log_used(0),
          log_ref(_log_ref),
          callback(_callback),
+          user_write_cb(_user_write_cb),
          made_waitable(false),
          state(STATE_INIT),
          write_group(nullptr),
@@ -203,6 +214,18 @@ class WriteThread {
      return callback_status.ok();
    }

+    void CheckWriteEnqueuedCallback() {
+      if (user_write_cb != nullptr) {
+        user_write_cb->OnWriteEnqueued();
+      }
+    }
+
+    void CheckPostWalWriteCallback() {
+      if (user_write_cb != nullptr) {
+        user_write_cb->OnWalWriteFinish();
+      }
+    }
+
    void CreateMutex() {
      if (!made_waitable) {
        // Note that made_waitable is tracked separately from state
@@ -323,10 +346,19 @@ class WriteThread {
  // Causes JoinBatchGroup to return STATE_PARALLEL_MEMTABLE_WRITER for all of
  // the non-leader members of this write batch group.  Sets Writer::sequence
  // before waking them up.
+  // If the size of write_group n is not small, the leader will call n^0.5
+  // members to be PARALLEL_MEMTABLE_CALLER in the write_group to help to set
+  // other's status parallel. This ensures that the cost to call SetState
+  // sequentially does not exceed 2(n^0.5).
  //
  // WriteGroup* write_group: Extra state used to coordinate the parallel add
  void LaunchParallelMemTableWriters(WriteGroup* write_group);

+  // One of the every stride=N number writer in the WriteGroup are set to the
+  // MemTableWriters, where N is equal to square of the total number of this
+  // write_group, and all of these MemTableWriters will write to memtable.
+  void SetMemWritersEachStride(Writer* w);
+
  // Reports the completion of w's batch to the parallel group leader, and
  // waits for the rest of the parallel batch to complete.  Returns true
  // if this thread is the last to complete, and hence should advance
@@ -4,6 +4,7 @@ add_executable(db_stress${ARTIFACT_SUFFIX}
  db_stress.cc
  db_stress_common.cc
  db_stress_driver.cc
+  db_stress_filters.cc
  db_stress_gflags.cc
  db_stress_listener.cc
  db_stress_shared_state.cc
@@ -590,6 +590,11 @@ class BatchedOpsStressTest : public StressTest {
        // For half of the time, set the upper bound to the next prefix
        ub_slices[i] = upper_bounds[i];
        ro_copies[i].iterate_upper_bound = &(ub_slices[i]);
+        if (FLAGS_use_sqfc_for_range_queries) {
+          ro_copies[i].table_filter =
+              sqfc_factory_->GetTableFilterForRangeQuery(prefix_slices[i],
+                                                         ub_slices[i]);
+        }
      }

      iters[i].reset(db_->NewIterator(ro_copies[i], cfh));
@@ -73,7 +73,10 @@ class CfConsistencyStressTest : public StressTest {
      status = db_->Write(write_opts, &batch);
    }

-    if (!status.ok()) {
+    if (IsRetryableInjectedError(status)) {
+      fprintf(stdout, "multi put or merge error: %s\n",
+              status.ToString().c_str());
+    } else if (!status.ok()) {
      fprintf(stderr, "multi put or merge error: %s\n",
              status.ToString().c_str());
      thread->stats.AddErrors(1);
@@ -96,7 +99,9 @@ class CfConsistencyStressTest : public StressTest {
      batch.Delete(cfh, key);
    }
    Status s = db_->Write(write_opts, &batch);
-    if (!s.ok()) {
+    if (IsRetryableInjectedError(s)) {
+      fprintf(stdout, "multidel error: %s\n", s.ToString().c_str());
+    } else if (!s.ok()) {
      fprintf(stderr, "multidel error: %s\n", s.ToString().c_str());
      thread->stats.AddErrors(1);
    } else {
@@ -125,7 +130,9 @@ class CfConsistencyStressTest : public StressTest {
      batch.DeleteRange(cfh, key, end_key);
    }
    Status s = db_->Write(write_opts, &batch);
-    if (!s.ok()) {
+    if (IsRetryableInjectedError(s)) {
+      fprintf(stdout, "multi del range error: %s\n", s.ToString().c_str());
+    } else if (!s.ok()) {
      fprintf(stderr, "multi del range error: %s\n", s.ToString().c_str());
      thread->stats.AddErrors(1);
    } else {
@@ -170,6 +177,15 @@ class CfConsistencyStressTest : public StressTest {
      std::string value0;
      s = db_->Get(readoptionscopy, column_families_[rand_column_families[0]],
                   key, &value0);
+
+      // Temporarily disable error injection for verification
+      if (fault_fs_guard) {
+        fault_fs_guard->DisableThreadLocalErrorInjection(
+            FaultInjectionIOType::kRead);
+        fault_fs_guard->DisableThreadLocalErrorInjection(
+            FaultInjectionIOType::kMetadataRead);
+      }
+
      if (s.ok() || s.IsNotFound()) {
        bool found = s.ok();
        for (size_t i = 1; i < rand_column_families.size(); i++) {
@@ -214,6 +230,13 @@ class CfConsistencyStressTest : public StressTest {
        }
      }

+      //  Enable back error injection disabled for verification
+      if (fault_fs_guard) {
+        fault_fs_guard->EnableThreadLocalErrorInjection(
+            FaultInjectionIOType::kRead);
+        fault_fs_guard->EnableThreadLocalErrorInjection(
+            FaultInjectionIOType::kMetadataRead);
+      }
      db_->ReleaseSnapshot(snapshot);
    }
    if (!is_consistent) {
@@ -225,6 +248,8 @@ class CfConsistencyStressTest : public StressTest {
      thread->stats.AddGets(1, 1);
    } else if (s.IsNotFound()) {
      thread->stats.AddGets(1, 0);
+    } else if (IsRetryableInjectedError(s)) {
+      fprintf(stdout, "TestGet error: %s\n", s.ToString().c_str());
    } else {
      fprintf(stderr, "TestGet error: %s\n", s.ToString().c_str());
      thread->stats.AddErrors(1);
@@ -261,6 +286,8 @@ class CfConsistencyStressTest : public StressTest {
      } else if (s.IsNotFound()) {
        // not found case
        thread->stats.AddGets(1, 0);
+      } else if (IsRetryableInjectedError(s)) {
+        fprintf(stdout, "MultiGet error: %s\n", s.ToString().c_str());
      } else {
        // errors case
        fprintf(stderr, "MultiGet error: %s\n", s.ToString().c_str());
@@ -297,7 +324,12 @@ class CfConsistencyStressTest : public StressTest {
      PinnableWideColumns result;
      s = db_->GetEntity(read_opts, cfh, key, &result);

-      if (s.ok()) {
+      if (IsRetryableInjectedError(s)) {
+        fprintf(stdout,
+                "GetEntity error: inconsistent columns for key %s, entity %s\n",
+                StringToHex(key).c_str(),
+                WideColumnsToHex(result.columns()).c_str());
+      } else if (s.ok()) {
        if (!VerifyWideColumns(result.columns())) {
          fprintf(
              stderr,
@@ -323,6 +355,14 @@ class CfConsistencyStressTest : public StressTest {
                         column_families_[rand_column_families[0]], key,
                         &cmp_result);

+      //  Temporarily disable error injection for verification
+      if (fault_fs_guard) {
+        fault_fs_guard->DisableThreadLocalErrorInjection(
+            FaultInjectionIOType::kRead);
+
+        fault_fs_guard->DisableThreadLocalErrorInjection(
+            FaultInjectionIOType::kMetadataRead);
+      }
      if (s.ok() || s.IsNotFound()) {
        const bool cmp_found = s.ok();

@@ -458,6 +498,14 @@ class CfConsistencyStressTest : public StressTest {
          }
        }
      }
+
+      //  Enable back error injection disabled for verification
+      if (fault_fs_guard) {
+        fault_fs_guard->EnableThreadLocalErrorInjection(
+            FaultInjectionIOType::kRead);
+        fault_fs_guard->EnableThreadLocalErrorInjection(
+            FaultInjectionIOType::kMetadataRead);
+      }
    }

    if (!is_consistent) {
@@ -469,6 +517,8 @@ class CfConsistencyStressTest : public StressTest {
      thread->stats.AddGets(1, 1);
    } else if (s.IsNotFound()) {
      thread->stats.AddGets(1, 0);
+    } else if (IsRetryableInjectedError(s)) {
+      fprintf(stdout, "TestGetEntity error: %s\n", s.ToString().c_str());
    } else {
      fprintf(stderr, "TestGetEntity error: %s\n", s.ToString().c_str());
      thread->stats.AddErrors(1);
@@ -540,7 +590,11 @@ class CfConsistencyStressTest : public StressTest {
        for (size_t j = 0; j < num_cfs; ++j) {
          const Status& s = result[j].status();
          const WideColumns& columns = result[j].columns();
-          if (!s.ok() && !s.IsNotFound()) {
+          if (IsRetryableInjectedError(s)) {
+            fprintf(stdout, "TestMultiGetEntity (AttributeGroup) error: %s\n",
+                    s.ToString().c_str());
+            break;
+          } else if (!s.ok() && !s.IsNotFound()) {
            fprintf(stderr, "TestMultiGetEntity (AttributeGroup) error: %s\n",
                    s.ToString().c_str());
            thread->stats.AddErrors(1);
@@ -645,7 +699,11 @@ class CfConsistencyStressTest : public StressTest {
          const Status& s = statuses[j];
          const WideColumns& columns = results[j].columns();

-          if (!s.ok() && !s.IsNotFound()) {
+          if (IsRetryableInjectedError(s)) {
+            fprintf(stdout, "TestMultiGetEntity error: %s\n",
+                    s.ToString().c_str());
+            break;
+          } else if (!s.ok() && !s.IsNotFound()) {
            fprintf(stderr, "TestMultiGetEntity error: %s\n",
                    s.ToString().c_str());
            thread->stats.AddErrors(1);
@@ -746,6 +804,10 @@ class CfConsistencyStressTest : public StressTest {
    if (GetNextPrefix(prefix, &upper_bound) && thread->rand.OneIn(2)) {
      ub_slice = Slice(upper_bound);
      ro_copy.iterate_upper_bound = &ub_slice;
+      if (FLAGS_use_sqfc_for_range_queries) {
+        ro_copy.table_filter =
+            sqfc_factory_->GetTableFilterForRangeQuery(prefix, ub_slice);
+      }
    }

    ColumnFamilyHandle* const cfh =
@@ -776,7 +838,9 @@ class CfConsistencyStressTest : public StressTest {
      s = iter->status();
    }

-    if (!s.ok()) {
+    if (IsRetryableInjectedError(s)) {
+      fprintf(stdout, "TestPrefixScan error: %s\n", s.ToString().c_str());
+    } else if (!s.ok()) {
      fprintf(stderr, "TestPrefixScan error: %s\n", s.ToString().c_str());
      thread->stats.AddErrors(1);

@@ -200,7 +200,7 @@ void CompressedCacheSetCapacityThread(void* v) {
        // Lower by upto 50% of usable block cache capacity
        adjustment = (adjustment * thread->rand.Uniform(50)) / 100;
        block_cache->SetCapacity(capacity - adjustment);
-        fprintf(stdout, "New cache capacity = %lu\n",
+        fprintf(stdout, "New cache capacity = %zu\n",
                block_cache->GetCapacity());
        db_stress_env->SleepForMicroseconds(10 * 1000 * 1000);
        block_cache->SetCapacity(capacity);
@@ -395,6 +395,16 @@ bool VerifyWideColumns(const WideColumns& columns) {
  return VerifyWideColumns(value_of_default, columns);
 }

+bool VerifyIteratorAttributeGroups(
+    const IteratorAttributeGroups& attribute_groups) {
+  for (const auto& attribute_group : attribute_groups) {
+    if (!VerifyWideColumns(attribute_group.columns())) {
+      return false;
+    }
+  }
+  return true;
+}
+
 std::string GetNowNanos() {
  uint64_t t = db_stress_env->NowNanos();
  std::string ret;
@@ -89,7 +89,6 @@ DECLARE_string(options_file);
 DECLARE_int64(active_width);
 DECLARE_bool(test_batches_snapshots);
 DECLARE_bool(atomic_flush);
-DECLARE_int32(manual_wal_flush_one_in);
 DECLARE_int32(lock_wal_one_in);
 DECLARE_bool(test_cf_consistency);
 DECLARE_bool(test_multi_ops_txns);
@@ -136,6 +135,7 @@ DECLARE_int32(universal_size_ratio);
 DECLARE_int32(universal_min_merge_width);
 DECLARE_int32(universal_max_merge_width);
 DECLARE_int32(universal_max_size_amplification_percent);
+DECLARE_int32(universal_max_read_amp);
 DECLARE_int32(clear_column_family_one_in);
 DECLARE_int32(get_live_files_apis_one_in);
 DECLARE_int32(get_all_column_family_metadata_one_in);
@@ -169,6 +169,9 @@ DECLARE_int32(bloom_before_level);
 DECLARE_bool(partition_filters);
 DECLARE_bool(optimize_filters_for_memory);
 DECLARE_bool(detect_filter_construct_corruption);
+DECLARE_string(sqfc_name);
+DECLARE_uint32(sqfc_version);
+DECLARE_bool(use_sqfc_for_range_queries);
 DECLARE_int32(index_type);
 DECLARE_int32(data_block_index_type);
 DECLARE_string(db);
@@ -189,7 +192,6 @@ DECLARE_uint64(bytes_per_sync);
 DECLARE_uint64(wal_bytes_per_sync);
 DECLARE_int32(kill_random_test);
 DECLARE_string(kill_exclude_prefixes);
-DECLARE_bool(disable_wal);
 DECLARE_uint64(recycle_log_file_num);
 DECLARE_int64(target_file_size_base);
 DECLARE_int32(target_file_size_multiplier);
@@ -254,6 +256,7 @@ DECLARE_int32(prefix_size);
 DECLARE_bool(use_merge);
 DECLARE_uint32(use_put_entity_one_in);
 DECLARE_bool(use_attribute_group);
+DECLARE_bool(use_multi_cf_iterator);
 DECLARE_bool(use_full_merge_v1);
 DECLARE_int32(sync_wal_one_in);
 DECLARE_bool(avoid_unnecessary_blocking_io);
@@ -314,8 +317,6 @@ DECLARE_int32(blob_cache_numshardbits);
 DECLARE_int32(prepopulate_blob_cache);

 DECLARE_int32(approximate_size_one_in);
-DECLARE_bool(sync_fault_injection);
-
 DECLARE_bool(best_efforts_recovery);
 DECLARE_bool(skip_verifydb);
 DECLARE_bool(enable_compaction_filter);
@@ -415,6 +416,7 @@ DECLARE_bool(enable_memtable_insert_with_hint_prefix_extractor);
 DECLARE_bool(check_multiget_consistency);
 DECLARE_bool(check_multiget_entity_consistency);
 DECLARE_bool(inplace_update_support);
+DECLARE_uint32(uncache_aggressiveness);

 constexpr long KB = 1024;
 constexpr int kRandomValueMaxFactor = 3;
@@ -761,6 +763,8 @@ WideColumns GenerateExpectedWideColumns(uint32_t value_base,
                                        const Slice& slice);
 bool VerifyWideColumns(const Slice& value, const WideColumns& columns);
 bool VerifyWideColumns(const WideColumns& columns);
+bool VerifyIteratorAttributeGroups(
+    const IteratorAttributeGroups& attribute_groups);

 AttributeGroups GenerateAttributeGroups(
    const std::vector<ColumnFamilyHandle*>& cfhs, uint32_t value_base,
@@ -8,6 +8,7 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 //

+#include "db_stress_tool/db_stress_shared_state.h"
 #ifdef GFLAGS
 #include "db_stress_tool/db_stress_common.h"
 #include "utilities/fault_injection_fs.h"
@@ -81,31 +82,6 @@ bool RunStressTestImpl(SharedState* shared) {
  stress->InitDb(shared);
  stress->FinishInitDb(shared);

-  if (FLAGS_write_fault_one_in) {
-    if (!FLAGS_sync_fault_injection) {
-      // unsynced WAL loss is not supported without sync_fault_injection
-      fault_fs_guard->SetDirectWritableTypes({kWalFile});
-    }
-    IOStatus error_msg;
-    if (FLAGS_inject_error_severity <= 1 || FLAGS_inject_error_severity > 2) {
-      error_msg = IOStatus::IOError("Retryable injected write error");
-      error_msg.SetRetryable(true);
-    } else if (FLAGS_inject_error_severity == 2) {
-      error_msg = IOStatus::IOError("Fatal injected write error");
-      error_msg.SetDataLoss(true);
-    }
-    // TODO: inject write error for other file types including
-    //  MANIFEST, CURRENT, and WAL files.
-    fault_fs_guard->SetRandomWriteError(
-        shared->GetSeed(), FLAGS_write_fault_one_in, error_msg,
-        /*inject_for_all_file_types=*/false, {FileType::kTableFile});
-    fault_fs_guard->SetFilesystemDirectWritable(false);
-    fault_fs_guard->EnableWriteErrorInjection();
-  }
-  if (FLAGS_sync_fault_injection) {
-    fault_fs_guard->SetFilesystemDirectWritable(false);
-  }
-
  uint32_t n = FLAGS_threads;
  uint64_t now = clock->NowMicros();
  fprintf(stdout, "%s Initializing worker threads\n",
@@ -182,6 +158,15 @@ bool RunStressTestImpl(SharedState* shared) {
      if (!FLAGS_expected_values_dir.empty()) {
        stress->TrackExpectedState(shared);
      }
+
+      // Since wrie fault and sync fault implementations are coupled with each
+      // other in `TestFSWritableFile()`, we can not enable or disable only one
+      // of the two.
+      // TODO(hx235): decouple implementations of write fault injection and sync
+      // fault injection.
+      if (FLAGS_sync_fault_injection || FLAGS_write_fault_one_in > 0) {
+        fault_fs_guard->SetFilesystemDirectWritable(false);
+      }
      now = clock->NowMicros();
      fprintf(stdout, "%s Starting database operations\n",
              clock->TimeToString(now / 1000000).c_str());
@@ -0,0 +1,93 @@
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#include "db_stress_tool/db_stress_filters.h"
+
+#include <memory>
+#include <mutex>
+
+namespace ROCKSDB_NAMESPACE {
+
+#ifdef GFLAGS
+
+using experimental::KeySegmentsExtractor;
+using experimental::MakeSharedBytewiseMinMaxSQFC;
+using experimental::SelectKeySegment;
+using experimental::SstQueryFilterConfigs;
+using experimental::SstQueryFilterConfigsManager;
+
+namespace {
+class VariableWidthExtractor : public KeySegmentsExtractor {
+ public:
+  const char* Name() const override { return "VariableWidthExtractor"; }
+
+  void Extract(const Slice& key_or_bound, KeyKind /*kind*/,
+               Result* result) const override {
+    uint32_t len = static_cast<uint32_t>(key_or_bound.size());
+    // This uses as delimiter any zero byte that follows a non-zero byte.
+    // And in accordance with best practice, the delimiter is part of the
+    // segment leading up to it.
+    bool prev_non_zero = false;
+    for (uint32_t i = 0; i < len; ++i) {
+      if ((prev_non_zero && key_or_bound[i] == 0) || i + 1 == len) {
+        result->segment_ends.push_back(i + 1);
+      }
+      prev_non_zero = key_or_bound[i] != 0;
+    }
+  }
+};
+const auto kVariableWidthExtractor = std::make_shared<VariableWidthExtractor>();
+class FixedWidthExtractor : public KeySegmentsExtractor {
+ public:
+  const char* Name() const override { return "FixedWidthExtractor"; }
+
+  void Extract(const Slice& key_or_bound, KeyKind /*kind*/,
+               Result* result) const override {
+    uint32_t len = static_cast<uint32_t>(key_or_bound.size());
+    // Fixed 8-byte segments, with any leftovers going into another
+    // segment.
+    uint32_t i = 0;
+    while (i + 8 <= len) {
+      i += 8;
+      result->segment_ends.push_back(i);
+    }
+    if (i < len) {
+      result->segment_ends.push_back(len);
+    }
+  }
+};
+const auto kFixedWidthExtractor = std::make_shared<FixedWidthExtractor>();
+// NOTE: MinMax filter on segment 0 is not normally useful because of metadata
+// on smallest and largest key for each SST file. But it doesn't hurt to test.
+const auto kFilter0 = MakeSharedBytewiseMinMaxSQFC(SelectKeySegment(0));
+const auto kFilter1 = MakeSharedBytewiseMinMaxSQFC(SelectKeySegment(1));
+const auto kFilter2 = MakeSharedBytewiseMinMaxSQFC(SelectKeySegment(2));
+const auto kFilter3 = MakeSharedBytewiseMinMaxSQFC(SelectKeySegment(3));
+const SstQueryFilterConfigs fooConfigs1{{kFilter0, kFilter2},
+                                        kVariableWidthExtractor};
+const SstQueryFilterConfigs fooConfigs2{{kFilter1, kFilter3},
+                                        kVariableWidthExtractor};
+const SstQueryFilterConfigs barConfigs2{{kFilter1, kFilter2},
+                                        kFixedWidthExtractor};
+const SstQueryFilterConfigsManager::Data data = {
+    {1, {{"foo", fooConfigs1}}},
+    {2, {{"foo", fooConfigs2}, {"bar", barConfigs2}}},
+};
+}  // namespace
+
+SstQueryFilterConfigsManager& DbStressSqfcManager() {
+  std::once_flag flag;
+  static std::shared_ptr<SstQueryFilterConfigsManager> mgr;
+  std::call_once(flag, []() {
+    Status s = SstQueryFilterConfigsManager::MakeShared(data, &mgr);
+    assert(s.ok());
+    assert(mgr);
+  });
+  return *mgr;
+}
+
+#endif  // GFLAGS
+
+}  // namespace ROCKSDB_NAMESPACE
@@ -0,0 +1,16 @@
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#ifdef GFLAGS
+#pragma once
+
+#include "rocksdb/experimental.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+experimental::SstQueryFilterConfigsManager& DbStressSqfcManager();
+
+}  // namespace ROCKSDB_NAMESPACE
+#endif  // GFLAGS
@@ -306,6 +306,9 @@ DEFINE_int32(universal_max_merge_width, 0,
 DEFINE_int32(universal_max_size_amplification_percent, 0,
             "The max size amplification for universal style compaction");

+DEFINE_int32(universal_max_read_amp, -1,
+             "The limit on the number of sorted runs");
+
 DEFINE_int32(clear_column_family_one_in, 1000000,
             "With a chance of 1/N, delete a column family and then recreate "
             "it again. If N == 0, never drop/create column families. "
@@ -579,6 +582,16 @@ DEFINE_bool(
        .detect_filter_construct_corruption,
    "Detect corruption during new Bloom Filter and Ribbon Filter construction");

+DEFINE_string(sqfc_name, "foo",
+              "Config name to select from SstQueryFilterConfigsManager.");
+
+DEFINE_uint32(sqfc_version, 0,
+              "User-defined filtering version to select from "
+              "SstQueryFilterConfigsManager. 0 = disable writing filters");
+
+DEFINE_bool(use_sqfc_for_range_queries, true,
+            "Apply SstQueryFilters to range queries");
+
 DEFINE_int32(
    index_type,
    static_cast<int32_t>(
@@ -955,6 +968,9 @@ DEFINE_uint32(use_put_entity_one_in, 0,
 DEFINE_bool(use_attribute_group, false,
            "If set, use the attribute_group API to put/get entities");

+DEFINE_bool(use_multi_cf_iterator, false,
+            "If set, use the multi_cf_iterator for TestIterate");
+
 DEFINE_bool(use_full_merge_v1, false,
            "On true, use a merge operator that implement the deprecated "
            "version of FullMerge");
@@ -1012,6 +1028,14 @@ DEFINE_int32(approximate_size_one_in, 64,
 DEFINE_int32(read_fault_one_in, 1000,
             "On non-zero, enables fault injection on read");

+DEFINE_bool(error_recovery_with_no_fault_injection, false,
+            "If true, error recovery will be done without fault injection if "
+            "fault injection is enabled");
+
+DEFINE_int32(metadata_read_fault_one_in, 1000,
+             "On non-zero, enables fault injection on metadata read (i.e, "
+             "directory and file metadata read)");
+
 DEFINE_int32(get_property_one_in, 1000,
             "If non-zero, then DB::GetProperty() will be called to get various"
             " properties for every N ops on average. 0 indicates that"
@@ -1071,6 +1095,10 @@ DEFINE_int32(write_fault_one_in, 0,
             "On non-zero, enables fault injection on write. Currently only"
             "injects write error when writing to SST files.");

+DEFINE_int32(metadata_write_fault_one_in, 1000,
+             "On non-zero, enables fault injection on metadata write (i.e, "
+             "directory and file metadata write)");
+
 DEFINE_uint64(user_timestamp_size, 0,
              "Number of bytes for a user-defined timestamp. Currently, only "
              "8-byte is supported");
@@ -1079,6 +1107,11 @@ DEFINE_bool(persist_user_defined_timestamps, true,
            "Flag to indicate whether user-defined timestamps will be persisted"
            " during Flush");

+DEFINE_int32(open_metadata_read_fault_one_in, 0,
+             "On non-zero, enables fault injection on file metadata read (i.e, "
+             "directory and file metadata read)"
+             "during DB reopen.");
+
 DEFINE_int32(open_metadata_write_fault_one_in, 0,
             "On non-zero, enables fault injection on file metadata write "
             "during DB reopen.");
@@ -1400,4 +1433,11 @@ DEFINE_bool(check_multiget_entity_consistency, true,
 DEFINE_bool(inplace_update_support,
            ROCKSDB_NAMESPACE::Options().inplace_update_support,
            "Options.inplace_update_support");
+
+DEFINE_uint32(uncache_aggressiveness,
+              ROCKSDB_NAMESPACE::ColumnFamilyOptions().uncache_aggressiveness,
+              "Aggressiveness of erasing cache entries that are likely "
+              "obsolete. 0 = disabled, 1 = minimum, 100 = moderate, 10000 = "
+              "normal max");
+
 #endif  // GFLAGS
@@ -71,22 +71,53 @@ class DbStressListener : public EventListener {
    VerifyFilePath(info.file_path);
    // pretending doing some work here
    RandomSleep();
-    if (FLAGS_read_fault_one_in) {
-      (void)fault_fs_guard->GetAndResetErrorCount();
-      fault_fs_guard->DisableErrorInjection();
+    if (fault_fs_guard) {
+      fault_fs_guard->DisableThreadLocalErrorInjection(
+          FaultInjectionIOType::kRead);
+      fault_fs_guard->DisableThreadLocalErrorInjection(
+          FaultInjectionIOType::kWrite);
+      fault_fs_guard->DisableThreadLocalErrorInjection(
+          FaultInjectionIOType::kMetadataRead);
+      fault_fs_guard->DisableThreadLocalErrorInjection(
+          FaultInjectionIOType::kMetadataWrite);
    }
  }

  void OnFlushBegin(DB* /*db*/,
                    const FlushJobInfo& /*flush_job_info*/) override {
    RandomSleep();
-    if (FLAGS_read_fault_one_in) {
-      // Hardcoded to inject retryable error as a non-retryable error would put
-      // the DB in read-only mode and then it would crash on the next write.
-      fault_fs_guard->SetThreadLocalReadErrorContext(
-          static_cast<uint32_t>(FLAGS_seed), FLAGS_read_fault_one_in,
-          true /* retryable */);
-      fault_fs_guard->EnableErrorInjection();
+    if (fault_fs_guard) {
+      fault_fs_guard->SetThreadLocalErrorContext(
+          FaultInjectionIOType::kRead, static_cast<uint32_t>(FLAGS_seed),
+          FLAGS_read_fault_one_in,
+          FLAGS_inject_error_severity == 1 /* retryable */,
+          FLAGS_inject_error_severity == 2 /* has_data_loss*/);
+      fault_fs_guard->EnableThreadLocalErrorInjection(
+          FaultInjectionIOType::kRead);
+
+      fault_fs_guard->SetThreadLocalErrorContext(
+          FaultInjectionIOType::kWrite, static_cast<uint32_t>(FLAGS_seed),
+          FLAGS_write_fault_one_in,
+          FLAGS_inject_error_severity == 1 /* retryable */,
+          FLAGS_inject_error_severity == 2 /* has_data_loss*/);
+      fault_fs_guard->EnableThreadLocalErrorInjection(
+          FaultInjectionIOType::kWrite);
+
+      fault_fs_guard->SetThreadLocalErrorContext(
+          FaultInjectionIOType::kMetadataRead,
+          static_cast<uint32_t>(FLAGS_seed), FLAGS_metadata_read_fault_one_in,
+          FLAGS_inject_error_severity == 1 /* retryable */,
+          FLAGS_inject_error_severity == 2 /* has_data_loss*/);
+      fault_fs_guard->EnableThreadLocalErrorInjection(
+          FaultInjectionIOType::kMetadataRead);
+
+      fault_fs_guard->SetThreadLocalErrorContext(
+          FaultInjectionIOType::kMetadataWrite,
+          static_cast<uint32_t>(FLAGS_seed), FLAGS_metadata_write_fault_one_in,
+          FLAGS_inject_error_severity == 1 /* retryable */,
+          FLAGS_inject_error_severity == 2 /* has_data_loss*/);
+      fault_fs_guard->EnableThreadLocalErrorInjection(
+          FaultInjectionIOType::kMetadataWrite);
    }
  }

@@ -112,20 +143,51 @@ class DbStressListener : public EventListener {
  }

  void OnSubcompactionBegin(const SubcompactionJobInfo& /* si */) override {
-    if (FLAGS_read_fault_one_in) {
-      // Hardcoded to inject retryable error as a non-retryable error would put
-      // the DB in read-only mode and then it would crash on the next write.
-      fault_fs_guard->SetThreadLocalReadErrorContext(
-          static_cast<uint32_t>(FLAGS_seed), FLAGS_read_fault_one_in,
-          true /* retryable */);
-      fault_fs_guard->EnableErrorInjection();
+    if (fault_fs_guard) {
+      fault_fs_guard->SetThreadLocalErrorContext(
+          FaultInjectionIOType::kRead, static_cast<uint32_t>(FLAGS_seed),
+          FLAGS_read_fault_one_in,
+          FLAGS_inject_error_severity == 1 /* retryable */,
+          FLAGS_inject_error_severity == 2 /* has_data_loss*/);
+      fault_fs_guard->EnableThreadLocalErrorInjection(
+          FaultInjectionIOType::kRead);
+
+      fault_fs_guard->SetThreadLocalErrorContext(
+          FaultInjectionIOType::kWrite, static_cast<uint32_t>(FLAGS_seed),
+          FLAGS_write_fault_one_in,
+          FLAGS_inject_error_severity == 1 /* retryable */,
+          FLAGS_inject_error_severity == 2 /* has_data_loss*/);
+      fault_fs_guard->EnableThreadLocalErrorInjection(
+          FaultInjectionIOType::kWrite);
+
+      fault_fs_guard->SetThreadLocalErrorContext(
+          FaultInjectionIOType::kMetadataRead,
+          static_cast<uint32_t>(FLAGS_seed), FLAGS_metadata_read_fault_one_in,
+          FLAGS_inject_error_severity == 1 /* retryable */,
+          FLAGS_inject_error_severity == 2 /* has_data_loss*/);
+      fault_fs_guard->EnableThreadLocalErrorInjection(
+          FaultInjectionIOType::kMetadataRead);
+
+      fault_fs_guard->SetThreadLocalErrorContext(
+          FaultInjectionIOType::kMetadataWrite,
+          static_cast<uint32_t>(FLAGS_seed), FLAGS_metadata_write_fault_one_in,
+          FLAGS_inject_error_severity == 1 /* retryable */,
+          FLAGS_inject_error_severity == 2 /* has_data_loss*/);
+      fault_fs_guard->EnableThreadLocalErrorInjection(
+          FaultInjectionIOType::kMetadataWrite);
    }
  }

  void OnSubcompactionCompleted(const SubcompactionJobInfo& /* si */) override {
-    if (FLAGS_read_fault_one_in) {
-      (void)fault_fs_guard->GetAndResetErrorCount();
-      fault_fs_guard->DisableErrorInjection();
+    if (fault_fs_guard) {
+      fault_fs_guard->DisableThreadLocalErrorInjection(
+          FaultInjectionIOType::kRead);
+      fault_fs_guard->DisableThreadLocalErrorInjection(
+          FaultInjectionIOType::kWrite);
+      fault_fs_guard->DisableThreadLocalErrorInjection(
+          FaultInjectionIOType::kMetadataRead);
+      fault_fs_guard->DisableThreadLocalErrorInjection(
+          FaultInjectionIOType::kMetadataWrite);
    }
  }

@@ -211,10 +273,35 @@ class DbStressListener : public EventListener {
                            Status /* bg_error */,
                            bool* /* auto_recovery */) override {
    RandomSleep();
+    if (FLAGS_error_recovery_with_no_fault_injection && fault_fs_guard) {
+      fault_fs_guard->DisableThreadLocalErrorInjection(
+          FaultInjectionIOType::kRead);
+      fault_fs_guard->DisableThreadLocalErrorInjection(
+          FaultInjectionIOType::kWrite);
+      fault_fs_guard->DisableThreadLocalErrorInjection(
+          FaultInjectionIOType::kMetadataRead);
+      fault_fs_guard->DisableThreadLocalErrorInjection(
+          FaultInjectionIOType::kMetadataWrite);
+      // TODO(hx235): only exempt the flush thread during error recovery instead
+      // of all the flush threads from error injection
+      fault_fs_guard->SetIOActivtiesExemptedFromFaultInjection(
+          {Env::IOActivity::kFlush});
+    }
  }

  void OnErrorRecoveryCompleted(Status /* old_bg_error */) override {
    RandomSleep();
+    if (FLAGS_error_recovery_with_no_fault_injection && fault_fs_guard) {
+      fault_fs_guard->EnableThreadLocalErrorInjection(
+          FaultInjectionIOType::kRead);
+      fault_fs_guard->EnableThreadLocalErrorInjection(
+          FaultInjectionIOType::kWrite);
+      fault_fs_guard->EnableThreadLocalErrorInjection(
+          FaultInjectionIOType::kMetadataRead);
+      fault_fs_guard->EnableThreadLocalErrorInjection(
+          FaultInjectionIOType::kMetadataWrite);
+      fault_fs_guard->SetIOActivtiesExemptedFromFaultInjection({});
+    }
  }

 protected:
@@ -29,8 +29,15 @@ DECLARE_int32(clear_column_family_one_in);
 DECLARE_bool(test_batches_snapshots);
 DECLARE_int32(compaction_thread_pool_adjust_interval);
 DECLARE_int32(continuous_verification_interval);
+DECLARE_bool(error_recovery_with_no_fault_injection);
+DECLARE_bool(sync_fault_injection);
+DECLARE_bool(disable_wal);
+DECLARE_int32(manual_wal_flush_one_in);
+DECLARE_int32(metadata_read_fault_one_in);
+DECLARE_int32(metadata_write_fault_one_in);
 DECLARE_int32(read_fault_one_in);
 DECLARE_int32(write_fault_one_in);
+DECLARE_int32(open_metadata_read_fault_one_in);
 DECLARE_int32(open_metadata_write_fault_one_in);
 DECLARE_int32(open_write_fault_one_in);
 DECLARE_int32(open_read_fault_one_in);
@@ -7,11 +7,13 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.

+#include "rocksdb/io_status.h"
 #ifdef GFLAGS
 #pragma once

 #include "db_stress_tool/db_stress_common.h"
 #include "db_stress_tool/db_stress_shared_state.h"
+#include "rocksdb/experimental.h"

 namespace ROCKSDB_NAMESPACE {
 class SystemClock;
@@ -19,6 +21,7 @@ class Transaction;
 class TransactionDB;
 class OptimisticTransactionDB;
 struct TransactionDBOptions;
+using experimental::SstQueryFilterConfigsManager;

 class StressTest {
 public:
@@ -41,8 +44,24 @@ class StressTest {
  virtual void VerifyDb(ThreadState* thread) const = 0;
  virtual void ContinuouslyVerifyDb(ThreadState* /*thread*/) const = 0;
  void PrintStatistics();
+  bool MightHaveDataLoss() {
+    return FLAGS_sync_fault_injection || FLAGS_write_fault_one_in > 0 ||
+           FLAGS_metadata_write_fault_one_in > 0 || FLAGS_disable_wal ||
+           FLAGS_manual_wal_flush_one_in > 0;
+  }

 protected:
+  static int GetMinInjectedErrorCount(int error_count_1, int error_count_2) {
+    if (error_count_1 > 0 && error_count_2 > 0) {
+      return std::min(error_count_1, error_count_2);
+    } else if (error_count_1 > 0) {
+      return error_count_1;
+    } else if (error_count_2 > 0) {
+      return error_count_2;
+    } else {
+      return 0;
+    }
+  }
  Status AssertSame(DB* db, ColumnFamilyHandle* cf,
                    ThreadState::SnapshotState& snap_state);

@@ -169,6 +188,20 @@ class StressTest {
                             const std::vector<int>& rand_column_families,
                             const std::vector<int64_t>& rand_keys);

+  // Given a key K, this creates an attribute group iterator which scans to K
+  // and then does a random sequence of Next/Prev operations. Called only when
+  // use_attribute_group=1
+  virtual Status TestIterateAttributeGroups(
+      ThreadState* thread, const ReadOptions& read_opts,
+      const std::vector<int>& rand_column_families,
+      const std::vector<int64_t>& rand_keys);
+
+  template <typename IterType, typename NewIterFunc, typename VerifyFunc>
+  Status TestIterateImpl(ThreadState* thread, const ReadOptions& read_opts,
+                         const std::vector<int>& rand_column_families,
+                         const std::vector<int64_t>& rand_keys,
+                         NewIterFunc new_iter_func, VerifyFunc verify_func);
+
  virtual Status TestIterateAgainstExpected(
      ThreadState* /* thread */, const ReadOptions& /* read_opts */,
      const std::vector<int>& /* rand_column_families */,
@@ -192,10 +225,12 @@ class StressTest {
  // diverged = true if the two iterator is already diverged.
  // True if verification passed, false if not.
  // op_logs is the information to print when validation fails.
+  template <typename IterType, typename VerifyFuncType>
  void VerifyIterator(ThreadState* thread, ColumnFamilyHandle* cmp_cfh,
-                      const ReadOptions& ro, Iterator* iter, Iterator* cmp_iter,
+                      const ReadOptions& ro, IterType* iter, Iterator* cmp_iter,
                      LastIterateOp op, const Slice& seek_key,
-                      const std::string& op_logs, bool* diverged);
+                      const std::string& op_logs, VerifyFuncType verifyFunc,
+                      bool* diverged);

  virtual Status TestBackupRestore(ThreadState* thread,
                                   const std::vector<int>& rand_column_families,
@@ -224,13 +259,13 @@ class StressTest {

  Status MaybeReleaseSnapshots(ThreadState* thread, uint64_t i);

-  Status VerifyGetLiveFiles() const;
-  Status VerifyGetLiveFilesMetaData() const;
-  Status VerifyGetLiveFilesStorageInfo() const;
-  Status VerifyGetAllColumnFamilyMetaData() const;
+  Status TestGetLiveFiles() const;
+  Status TestGetLiveFilesMetaData() const;
+  Status TestGetLiveFilesStorageInfo() const;
+  Status TestGetAllColumnFamilyMetaData() const;

-  Status VerifyGetSortedWalFiles() const;
-  Status VerifyGetCurrentWalFile() const;
+  Status TestGetSortedWalFiles() const;
+  Status TestGetCurrentWalFile() const;
  void TestGetProperty(ThreadState* thread) const;
  Status TestGetPropertiesOfAllTables() const;

@@ -245,6 +280,11 @@ class StressTest {
    return Status::NotSupported("TestCustomOperations() must be overridden");
  }

+  bool IsRetryableInjectedError(const Status& s) const {
+    return s.getState() && std::strstr(s.getState(), "inject") &&
+           !status_to_io_status(Status(s)).GetDataLoss();
+  }
+
  void ProcessStatus(SharedState* shared, std::string msg, const Status& s,
                     bool ignore_injected_error = true) const;

@@ -303,6 +343,7 @@ class StressTest {
  std::unordered_map<std::string, std::vector<std::string>> options_table_;
  std::vector<std::string> options_index_;
  std::atomic<bool> db_preload_finished_;
+  std::shared_ptr<SstQueryFilterConfigsManager::Factory> sqfc_factory_;

  // Fields used for continuous verification from another thread
  DB* cmp_db_;
@@ -344,7 +385,9 @@ void InitializeOptionsFromFlags(
 // from OPTIONS file.
 void InitializeOptionsGeneral(
    const std::shared_ptr<Cache>& cache,
-    const std::shared_ptr<const FilterPolicy>& filter_policy, Options& options);
+    const std::shared_ptr<const FilterPolicy>& filter_policy,
+    const std::shared_ptr<SstQueryFilterConfigsManager::Factory>& sqfc_factory,
+    Options& options);

 // If no OPTIONS file is specified, set up `options` so that we can test
 // user-defined timestamp which requires `-user_timestamp_size=8`.
@@ -20,10 +20,10 @@
 // NOTE that if FLAGS_test_batches_snapshots is set, the test will have
 // different behavior. See comment of the flag for details.

-#include "db_stress_tool/db_stress_shared_state.h"
 #ifdef GFLAGS
 #include "db_stress_tool/db_stress_common.h"
 #include "db_stress_tool/db_stress_driver.h"
+#include "db_stress_tool/db_stress_shared_state.h"
 #include "rocksdb/convenience.h"
 #include "utilities/fault_injection_fs.h"

@@ -82,9 +82,11 @@ int db_stress_tool(int argc, char** argv) {
  dbsl_env_wrapper_guard = std::make_shared<CompositeEnvWrapper>(raw_env);
  db_stress_listener_env = dbsl_env_wrapper_guard.get();

-  if (FLAGS_read_fault_one_in || FLAGS_sync_fault_injection ||
-      FLAGS_write_fault_one_in || FLAGS_open_metadata_write_fault_one_in ||
-      FLAGS_open_write_fault_one_in || FLAGS_open_read_fault_one_in) {
+  if (FLAGS_open_metadata_read_fault_one_in ||
+      FLAGS_open_metadata_write_fault_one_in || FLAGS_open_read_fault_one_in ||
+      FLAGS_open_write_fault_one_in || FLAGS_metadata_read_fault_one_in ||
+      FLAGS_metadata_write_fault_one_in || FLAGS_read_fault_one_in ||
+      FLAGS_write_fault_one_in || FLAGS_sync_fault_injection) {
    FaultInjectionTestFS* fs =
        new FaultInjectionTestFS(raw_env->GetFileSystem());
    fault_fs_guard.reset(fs);
@@ -301,11 +303,11 @@ int db_stress_tool(int argc, char** argv) {
  }

  if (FLAGS_use_put_entity_one_in > 0 &&
-      (FLAGS_use_full_merge_v1 || FLAGS_use_txn || FLAGS_test_multi_ops_txns ||
+      (FLAGS_use_full_merge_v1 || FLAGS_test_multi_ops_txns ||
       FLAGS_user_timestamp_size > 0)) {
    fprintf(stderr,
-            "Wide columns are incompatible with V1 Merge, transactions, and "
-            "user-defined timestamps\n");
+            "Wide columns are incompatible with V1 Merge, the multi-op "
+            "transaction test, and user-defined timestamps\n");
    exit(1);
  }

@@ -432,6 +432,13 @@ Status MultiOpsTxnsStressTest::TestIterate(
  return s;
 }

+Status MultiOpsTxnsStressTest::TestIterateAttributeGroups(
+    ThreadState* /*thread*/, const ReadOptions& /*read_opts*/,
+    const std::vector<int>& /*rand_column_families*/,
+    const std::vector<int64_t>& /*rand_keys*/) {
+  return Status::NotSupported();
+}
+
 // Not intended for use.
 Status MultiOpsTxnsStressTest::TestPut(ThreadState* /*thread*/,
                                       WriteOptions& /*write_opts*/,
@@ -749,6 +756,10 @@ Status MultiOpsTxnsStressTest::SecondaryKeyUpdateTxn(ThreadState* thread,
  ropts.iterate_upper_bound = &iter_ub;
  ropts.rate_limiter_priority =
      FLAGS_rate_limit_user_ops ? Env::IO_USER : Env::IO_TOTAL;
+  if (FLAGS_use_sqfc_for_range_queries) {
+    ropts.table_filter =
+        sqfc_factory_->GetTableFilterForRangeQuery(old_sk_prefix, iter_ub);
+  }
  it = txn->GetIterator(ropts);

  assert(it);
@@ -1107,6 +1118,10 @@ void MultiOpsTxnsStressTest::VerifyDb(ThreadState* thread) const {
    ropts.snapshot = snapshot;
    ropts.total_order_seek = true;
    ropts.iterate_upper_bound = &iter_ub;
+    if (FLAGS_use_sqfc_for_range_queries) {
+      ropts.table_filter =
+          sqfc_factory_->GetTableFilterForRangeQuery(start_key, iter_ub);
+    }

    std::unique_ptr<Iterator> it(db_->NewIterator(ropts));
    for (it->Seek(start_key); it->Valid(); it->Next()) {
@@ -1615,6 +1630,10 @@ void MultiOpsTxnsStressTest::ScanExistingDb(SharedState* shared, int threads) {
    ropts.iterate_lower_bound = &pk_lb;
    ropts.iterate_upper_bound = &pk_ub;
    ropts.total_order_seek = true;
+    if (FLAGS_use_sqfc_for_range_queries) {
+      ropts.table_filter =
+          sqfc_factory_->GetTableFilterForRangeQuery(pk_lb, pk_ub);
+    }
    std::unique_ptr<Iterator> it(db_->NewIterator(ropts));

    for (it->SeekToFirst(); it->Valid(); it->Next()) {
@@ -228,6 +228,11 @@ class MultiOpsTxnsStressTest : public StressTest {
                     const std::vector<int>& rand_column_families,
                     const std::vector<int64_t>& rand_keys) override;

+  Status TestIterateAttributeGroups(
+      ThreadState* thread, const ReadOptions& read_opts,
+      const std::vector<int>& rand_column_families,
+      const std::vector<int64_t>& rand_keys) override;
+
  Status TestPut(ThreadState* thread, WriteOptions& write_opts,
                 const ReadOptions& read_opts, const std::vector<int>& cf_ids,
                 const std::vector<int64_t>& keys, char (&value)[100]) override;
@@ -457,7 +457,6 @@ Status PosixHelper::GetLogicalBlockSizeOfDirectory(const std::string& directory,
                                                   size_t* size) {
  int fd = open(directory.c_str(), O_DIRECTORY | O_RDONLY);
  if (fd == -1) {
-    close(fd);
    return Status::IOError("Cannot open directory " + directory);
  }
  *size = PosixHelper::GetLogicalBlockSizeOfFd(fd);
@@ -59,7 +59,10 @@ IOStatus CopyFile(FileSystem* fs, const std::string& source,
      return io_s;
    }
    if (slice.size() == 0) {
-      return IOStatus::Corruption("file too small");
+      return IOStatus::Corruption(
+          "File smaller than expected for copy: " + source + " expecting " +
+          std::to_string(size) + " more bytes after " +
+          std::to_string(dest_writer->GetFileSize()));
    }

    io_s = dest_writer->Append(opts, slice);
@@ -226,7 +229,10 @@ IOStatus GenerateOneFileChecksum(
                                  io_s.ToString());
    }
    if (slice.size() == 0) {
-      return IOStatus::Corruption("file too small");
+      return IOStatus::Corruption(
+          "File smaller than expected for checksum: " + file_path +
+          " expecting " + std::to_string(size) + " more bytes after " +
+          std::to_string(offset));
    }
    checksum_generator->Update(slice.data(), slice.size());
    size -= slice.size();