Update version and HISTORY.md for 11.4.3 release

Summary:
Pull Request resolved: https://github.com/facebook/rocksdb/pull/14881

Read-only DB instances can share a directory with a live writer, so their view of live files is only a snapshot from the time they opened. After read-only DB open began setting `opened_successfully_` for its intended open-success semantics, that state had an unintended side effect: the common close path treated a successful read-only open like a read-write open and ran obsolete-file cleanup. If the writer created or made files live after the read-only handle opened, that cleanup could use the stale read-only live set and delete files still needed by the writer.

This change records whether a `DBImpl` was opened read-only and skips close-time obsolete-file cleanup for read-only DBs instead of simply keeping `opened_successfully_=false`, which could be confusing and easily mistaken in the future again.

For completeness I also updated other callsites of `opened_successfully_`, but those should not be real bugs as ReadOnly DBs do not run flushes/compactions or write to WALs.

Reviewed By: xingbowang

Differential Revision: D109622445

fbshipit-source-id: d7be4b20fce86ccb218a63ac6f5b707b316aac79

.github/actions/pre-steps/action.yml

@@ -2,6 +2,13 @@ name: pre-steps
 runs:
   using: composite
   steps:
+  - name: Dump ulimits for diagnostics
+    run: |
+      echo "=== ulimits (soft) ==="
+      ulimit -a -S || true
+      echo "=== ulimits (hard) ==="
+      ulimit -a -H || true
+    shell: bash
   - name: Install lld linker for faster builds
     run: apt-get update -y && apt-get install -y lld 2>/dev/null || true
     shell: bash

.github/actions/windows-build-steps/action.yml

@@ -3,7 +3,7 @@ inputs:
   suite-run:
     description: Comma-separated list of test suites to run (empty to skip C++ tests)
     required: false
-    default: arena_test,db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test
+    default: arena_test,db_basic_test,db_test,db_etc2_test,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test
   run-java:
     description: Whether to run Java tests
     required: false

.github/workflows/pr-jobs.yml

@@ -373,14 +373,17 @@ jobs:
           crash_duration: 240
     container:
       image: ghcr.io/facebook/rocksdb_ubuntu:22.1
-      options: --shm-size=16gb
+      options: --shm-size=16gb --ulimit nofile=1048576:1048576
     steps:
     - uses: actions/checkout@v4.1.0
     - uses: "./.github/actions/pre-steps"
     - uses: "./.github/actions/setup-ccache"
       with:
         cache-key-prefix: mini-crashtest
-    - run: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=${{ matrix.crash_duration }} --max_key=2500000' ${{ matrix.crash_test_target }}
+    - run: |
+        ulimit -S -n $(ulimit -Hn) # Raise open files soft limit to hard limit
+        echo "Updated ulimit -Hn: $(ulimit -Hn) -Sn: $(ulimit -Sn)"
+        make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=${{ matrix.crash_duration }} --max_key=2500000' ${{ matrix.crash_test_target }}
     - uses: "./.github/actions/teardown-ccache"
       if: always()
     - uses: "./.github/actions/post-steps"
@@ -534,7 +537,7 @@ jobs:
           suite_run: db_test
           run_java: "false"
         - test_shard: other
-          suite_run: arena_test,db_basic_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test
+          suite_run: arena_test,db_basic_test,db_etc2_test,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test
           run_java: "false"
         - test_shard: java
           suite_run: ""

.gitignore

@@ -52,7 +52,7 @@ GRTAGS
 GTAGS
 rocksdb_dump
 rocksdb_undump
-db_test2
+db_etc2_test
 trace_analyzer
 block_cache_trace_analyzer
 io_tracer_parser

BUCK

@@ -4868,6 +4868,12 @@ cpp_unittest_wrapper(name="db_encryption_test",
             extra_compiler_flags=[])
 
 
+cpp_unittest_wrapper(name="db_etc2_test",
+            srcs=["db/db_etc2_test.cc"],
+            deps=[":rocksdb_test_lib"],
+            extra_compiler_flags=[])
+
+
 cpp_unittest_wrapper(name="db_etc3_test",
             srcs=["db/db_etc3_test.cc"],
             deps=[":rocksdb_test_lib"],
@@ -5024,12 +5030,6 @@ cpp_unittest_wrapper(name="db_test",
             extra_compiler_flags=[])
 
 
-cpp_unittest_wrapper(name="db_test2",
-            srcs=["db/db_test2.cc"],
-            deps=[":rocksdb_test_lib"],
-            extra_compiler_flags=[])
-
-
 cpp_unittest_wrapper(name="db_universal_compaction_test",
             srcs=["db/db_universal_compaction_test.cc"],
             deps=[":rocksdb_test_lib"],

CLAUDE.md

@@ -209,12 +209,11 @@ The following patterns emerged as frequent sources of review feedback:
 ## Important tips
 
 ### Build system
-* There are 3 build system. Make, CMake, BUCK(meta internal).
+* There are 3 build system. Make for git clones, BUCK (meta internal) for hg
+  clones, and CMake for some special cases.
 * When a new .cc file is added, update Makefile, CMakeLists.txt, src.mk, BUCK.
 * Don't manually edit BUCK file, after updating src.mk, run
     /usr/local/bin/python3 buckifier/buckify_rocksdb.py to update it
-* Use make to build and run the test. CMake and BUCK are not used locally.
-* Use `make dbg` command to build all of the unit test in debug mode.
 * For -j in make command, use the number of CPU cores to decide it.
 
 ### When to run `make clean` (avoid mixing build modes)

CMakeLists.txt

@@ -654,6 +654,8 @@ if(USE_FOLLY)
     FMT_INST_PATH)
     exec_program(ls ARGS -d ${FOLLY_INST_PATH}/../gflags* OUTPUT_VARIABLE
     GFLAGS_INST_PATH)
+    exec_program(ls ARGS -d ${FOLLY_INST_PATH}/../glog* OUTPUT_VARIABLE
+    GLOG_INST_PATH)
     set(Boost_DIR ${BOOST_INST_PATH}/lib/cmake/Boost-1.83.0)
     if(EXISTS ${FMT_INST_PATH}/lib64)
       set(fmt_DIR ${FMT_INST_PATH}/lib64/cmake/fmt)
@@ -661,6 +663,42 @@ if(USE_FOLLY)
       set(fmt_DIR ${FMT_INST_PATH}/lib/cmake/fmt)
     endif()
     set(gflags_DIR ${GFLAGS_INST_PATH}/lib/cmake/gflags)
+    if(EXISTS ${GLOG_INST_PATH}/lib64/cmake/glog)
+      set(GLOG_CMAKE_DIR ${GLOG_INST_PATH}/lib64/cmake/glog)
+    elseif(EXISTS ${GLOG_INST_PATH}/lib/cmake/glog)
+      set(GLOG_CMAKE_DIR ${GLOG_INST_PATH}/lib/cmake/glog)
+    endif()
+    if(NOT GLOG_CMAKE_DIR)
+      find_library(GETDEPS_GLOG_LIBRARY NAMES glogd glog
+        PATHS ${GLOG_INST_PATH}/lib64 ${GLOG_INST_PATH}/lib
+        NO_DEFAULT_PATH)
+      if(NOT GETDEPS_GLOG_LIBRARY OR
+         NOT EXISTS ${GLOG_INST_PATH}/include/glog/logging.h)
+        message(FATAL_ERROR "Could not find getdeps glog under "
+          "${GLOG_INST_PATH}")
+      endif()
+      set(GLOG_CMAKE_DIR ${CMAKE_CURRENT_BINARY_DIR}/getdeps-glog-cmake)
+      file(MAKE_DIRECTORY ${GLOG_CMAKE_DIR})
+      file(WRITE ${GLOG_CMAKE_DIR}/glog-config.cmake
+        "if(NOT TARGET glog::glog)\n"
+        "  add_library(glog::glog UNKNOWN IMPORTED)\n"
+        "  set_target_properties(glog::glog PROPERTIES\n"
+        "    IMPORTED_LOCATION \"${GETDEPS_GLOG_LIBRARY}\"\n"
+        "    INTERFACE_INCLUDE_DIRECTORIES \"${GLOG_INST_PATH}/include\")\n"
+        "endif()\n"
+        "set(glog_FOUND TRUE)\n"
+        "set(Glog_FOUND TRUE)\n"
+        "set(GLOG_FOUND TRUE)\n"
+        "set(GLOG_LIBRARY \"${GETDEPS_GLOG_LIBRARY}\")\n"
+        "set(GLOG_LIBRARIES \"${GETDEPS_GLOG_LIBRARY}\")\n"
+        "set(GLOG_INCLUDE_DIR \"${GLOG_INST_PATH}/include\")\n")
+    endif()
+    set(glog_DIR ${GLOG_CMAKE_DIR})
+    set(Glog_DIR ${GLOG_CMAKE_DIR})
+    # Unlike gflags, glog may also be installed system-wide. Pre-resolve the
+    # getdeps copy so folly-config.cmake cannot fall back to another version.
+    find_package(glog CONFIG REQUIRED PATHS ${GLOG_CMAKE_DIR}
+                 NO_DEFAULT_PATH)
 
     exec_program(sed ARGS -i 's/gflags_shared//g'
     ${FOLLY_INST_PATH}/lib/cmake/folly/folly-targets.cmake)
@@ -1453,6 +1491,7 @@ if(WITH_TESTS)
         db/db_clip_test.cc
         db/db_dynamic_level_test.cc
         db/db_encryption_test.cc
+        db/db_etc2_test.cc
         db/db_etc3_test.cc
         db/db_flush_test.cc
         db/db_inplace_update_test.cc
@@ -1476,7 +1515,6 @@ if(WITH_TESTS)
         db/db_table_properties_test.cc
         db/db_tailing_iter_test.cc
         db/db_test.cc
-        db/db_test2.cc
         db/db_logical_block_size_cache_test.cc
         db/db_universal_compaction_test.cc
         db/db_wal_test.cc

HISTORY.md

@@ -1,12 +1,40 @@
 # Rocksdb Change Log
 > NOTE: Entries for next release do not go here. Follow instructions in `unreleased_history/README.txt`
 
-## 11.3.2 (06/24/2026)
+## 11.4.3 (06/24/2026)
 ### Bug Fixes
 * Fixed a bug where closing a read-only DB instance could delete live SST files created by a concurrent read-write DB sharing the same directory.
 
-## 11.3.1 (06/04/2026)
-* Don't call `EventListener::OnCompactionPreCommit` during DB shutdown, like other compaction callbacks
+## 11.4.2 (06/08/2026)
+### Public API Changes
+* Added experimental read-scoped block buffer provider API, configured through `ReadOptions::read_scoped_block_buffer_provider`, for supported block-based table iterator scans and MultiScan reads to use caller-provided read-scoped storage for final data-block contents. When configured, supported provider-backed scan data-block reads bypass the data-block cache. The provider is ignored when mmap reads are enabled. RocksDB may still use ordinary temporary scratch for serialized block bytes, such as when a block may be compressed. Get and MultiGet do not currently provide API guarantees for this provider.
+
+### Bug Fixes
+* Fixed a use after free bug in Multiscan when async IO is used
+
+## 11.4.1 (06/05/2026)
+* No-op patch used to sync with internal fb version.
+
+## 11.4.0 (06/02/2026)
+### Public API Changes
+* Added `rocksdb_options_set_memtable_batch_lookup_optimization()` and `rocksdb_options_get_memtable_batch_lookup_optimization()` to the C API, exposing the existing `AdvancedColumnFamilyOptions::memtable_batch_lookup_optimization` field. This allows C API users (and downstream language bindings) to enable the skip-list memtable's batch-lookup optimization for `MultiGet`, which caches the search path between consecutive keys and reduces per-key cost from O(log N) to O(log d) where d is the distance between consecutive keys.
+* Added `rocksdb_readoptions_set_optimize_multiget_for_io()` and `rocksdb_readoptions_get_optimize_multiget_for_io()` to the C API, exposing the existing `ReadOptions::optimize_multiget_for_io` field. This allows C API users (and downstream language bindings) to opt out of the multi-level parallel MultiGet path, which only takes effect when the library is built with `USE_COROUTINES`.
+* Added `rocksdb_block_based_table_index_block_search_type_auto` enum constant and the `rocksdb_block_based_options_set_uniform_cv_threshold()` setter to the C API. The constant exposes `BlockSearchType::kAuto`, and the setter exposes `BlockBasedTableOptions::uniform_cv_threshold`. Both are required for `kAuto` index-block search to take effect from the C API: without setting `uniform_cv_threshold >= 0`, the per-block `is_uniform` footer bit is never written, and `kAuto` falls back to binary search at read time.
+* Added `EventListener::OnDBShutdownBegin`, a callback that fires once when a DB begins shutdown, including when RocksDB cleans up a failed `DB::Open()` attempt.
+* Added a new PerfContext counter `blob_cache_read_byte` for blob cache bytes read.
+
+### Behavior Changes
+* Remote compaction now falls back to local compaction when the primary cannot deserialize a successful `CompactionServiceResult` before installing any remote output files.
+* StringToMap (rocksdb/convenience.h) now preserves the outer braces of nested values so each map entry is in self-contained form and can be embedded directly into another `key=value;` string (e.g. SetOptions) without losing inner ';' delimiters. A new symmetric MapToString utility is provided.
+
+### Bug Fixes
+* Fixed a bug where `AbortAllCompactions()` could leave automatic compaction work unscheduled when aborting before the background worker picked it, causing compaction and write stalls until DB restart.
+* Fix a bug where db had a false positive compaction corruption error due to remote compaction service result with `has_accurate_num_input_records=false` not being serialized.
+* Fixed WritePrepared TransactionDB cleanup after retryable commit or rollback write failures so prepared transactions remain rollbackable instead of leaving unresolved prepared state.
+* Fix a rare corruption bug for tiered compaction that incorrectly moved last level range tombstones into proximal level. This corruption error is surfaced when force_consistency_checks is enabled.
+
+### Performance Improvements
+* Reduce the likelihood of user-facing metadata operations blocking on MANIFEST rotation when file creation is slow, such as on remote storage. MANIFEST write batches containing foreground edits from `CreateColumnFamily()`, `DropColumnFamily()`, `CreateColumnFamilyWithImport()`, `IngestExternalFile()`, and `DeleteFilesInRanges()` now get a relaxed file size threshold for triggering MANIFEST rotation; background-only MANIFEST write batches, such as compaction and flush, continue to use the normal threshold. This change should make auto-tuning manifest file size more attractive (see `max_manifest_file_size` and `max_manifest_space_amp_pct` options).
 
 ## 11.3.0 (05/15/2026)
 ### New Features

Makefile

@@ -887,23 +887,43 @@ coverage: clean
 parallel_tests = $(patsubst %,parallel_%,$(PARALLEL_TEST))
 .PHONY: gen_parallel_tests $(parallel_tests)
 # Shard size controls how many test cases run per process. The actual number
-# of shards per binary is: min(ceil(test_count / GTEST_SHARD_SIZE), NCORES * 8)
+# of shards per binary is: min(ceil(test_count / shard_size), NCORES * 8)
 # This adapts to machine size: many small shards on beefy machines, fewer
 # larger shards on CI (typically 2-4 cores).
 GTEST_SHARD_SIZE ?= 10
 NCORES ?= $(shell nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)
 
+# Per-binary overrides for GTEST_SHARD_SIZE, used to chop up binaries whose
+# individual test cases are slow enough that the default of 10 leaves one
+# shard on the critical path of "make check". Format: whitespace-separated
+# tokens "BINARY:SIZE". Sizes were tuned from `make suggest-slow-tests`
+# output to keep max shard time under ~20s on a beefy box. Reducing too far
+# adds per-shard fixed overhead, so don't use 1 unless individual tests are
+# >5s.
+SHARD_SIZE_OVERRIDES ?= \
+  point_lock_manager_stress_test:1 \
+  external_sst_file_test:1 \
+  external_sst_file_basic_test:2 \
+  db_test:3 \
+  compaction_service_test:1
+
 $(parallel_tests):
 	$(AM_V_at)TEST_BINARY=$(patsubst parallel_%,%,$@); \
   TEST_COUNT=` \
     (./$$TEST_BINARY --gtest_list_tests 2>/dev/null || echo "  list_failure") \
     | grep -c '^ '`; \
 	if [ "$$TEST_COUNT" -le 0 ]; then TEST_COUNT=1; fi; \
+	SHARD_SIZE=$(GTEST_SHARD_SIZE); \
+	for o in $(SHARD_SIZE_OVERRIDES); do \
+		case "$$o" in \
+			"$$TEST_BINARY":*) SHARD_SIZE=$${o#*:} ;; \
+		esac; \
+	done; \
 	MAX_SHARDS=$$(( $(NCORES) * 8 )); \
-	NUM_SHARDS=$$(( (TEST_COUNT + $(GTEST_SHARD_SIZE) - 1) / $(GTEST_SHARD_SIZE) )); \
+	NUM_SHARDS=$$(( (TEST_COUNT + SHARD_SIZE - 1) / SHARD_SIZE )); \
 	if [ "$$NUM_SHARDS" -gt "$$MAX_SHARDS" ]; then NUM_SHARDS=$$MAX_SHARDS; fi; \
 	if [ "$$NUM_SHARDS" -le 0 ]; then NUM_SHARDS=1; fi; \
-	echo "  Generating $$NUM_SHARDS shards for $$TEST_BINARY ($$TEST_COUNT tests)"; \
+	echo "  Generating $$NUM_SHARDS shards for $$TEST_BINARY ($$TEST_COUNT tests, shard_size=$$SHARD_SIZE)"; \
 	SHARD_IDX=0; \
 	while [ "$$SHARD_IDX" -lt "$$NUM_SHARDS" ]; do \
 		if [ -n "$(CI_TOTAL_SHARDS)" ] && [ $$(( $$SHARD_IDX % $(CI_TOTAL_SHARDS) )) -ne $(CI_SHARD_INDEX) ]; then \
@@ -929,31 +949,64 @@ gen_parallel_tests:
 	$(AM_V_at)$(FIND) t -type f -name 'run-*' -exec rm -f {} \;
 	$(MAKE) $(parallel_tests)
 
-# Reorder input lines (which are one per test) so that the
-# longest-running tests appear first in the output.
-# Do this by prefixing each selected name with its duration,
-# sort the resulting names, and remove the leading numbers.
-# FIXME: the "100" we prepend is a fake time, for now.
-# FIXME: squirrel away timings from each run and use them
-# (when present) on subsequent runs to order these tests.
+# Reorder input lines (one per test or per shard) so the longest-running
+# tests/shards start first under "make check" parallel scheduling. The trick
+# is to prefix matched names with a fake duration (100), sort numerically
+# descending, then strip the prefix. Slow-matched tests therefore come ahead
+# of all unmatched tests; ordering within each group is unspecified.
 #
-# Without this reordering, these two tests would happen to start only
-# after almost all other tests had completed, thus adding 100 seconds
-# to the duration of parallel "make check".  That's the difference
-# between 4 minutes (old) and 2m20s (new).
+# Why this matters: gnu_parallel hands out jobs in input order. If a slow
+# test starts only after most others have finished, the run is bottlenecked
+# on its tail. Front-loading slow tests overlaps them with the bulk of
+# faster ones for much better wall-clock time.
 #
-# 152.120 PASS t/DBTest.FileCreationRandomFailure
-# 107.816 PASS t/DBTest.EncodeDecompressedBlockSizeTest
+# Maintenance:
+#   1) Run `make check` so LOG is up-to-date.
+#   2) Run `make suggest-slow-tests` to see candidate slow binaries.
+#   3) Add binaries with high per-shard time (slow critical path) or high
+#      total time across many shards (need early queueing for fan-out).
+# Each alternative is wrapped in `^.*NAME.*$$` so the *whole input line* is
+# captured into $$1; then `s,(...),100 $$1,` prepends "100 " to the line.
 #
-# With sharded test execution, prioritize binaries known to be slow.
-# These generate many shards and should start early for good load balancing.
+# Tiers below are based on observed timings (see suggest-slow-tests).
+# Tier 1: max single-shard time >= 30s (critical-path bottlenecks).
+# Tier 2: max single-shard time 15-30s.
+# Tier 3: huge total time across many tiny shards; front-loading them keeps
+#         the tail of the run busy while big shards finish.
 slow_test_regexp = \
-	^.*block_based_table_reader_test.*$$|^.*table_test.*$$|^.*block_test.*$$|^.*write_prepared_transaction_test.*$$|^.*transaction_test.*$$|^.*external_sst_file_test.*$$|^.*db_wal_test.*$$|^.*db_with_timestamp_basic_test.*$$|^.*db_test-.*$$
+	^.*point_lock_manager_stress_test.*$$|^.*db_test.*$$|^.*external_sst_file_test.*$$|^.*compaction_service_test.*$$|^.*corruption_test.*$$|^.*comparator_db_test.*$$|^.*external_sst_file_basic_test.*$$|^.*rate_limiter_test.*$$|^.*db_compaction_test.*$$|^.*write_prepared_transaction_test.*$$|^.*db_merge_operator_test.*$$|\
+	^.*db_dynamic_level_test.*$$|^.*db_bloom_filter_test.*$$|^.*error_handler_fs_test.*$$|^.*merge_helper_test.*$$|^.*transaction_test.*$$|^.*db_kv_checksum_test.*$$|^.*inlineskiplist_test.*$$|\
+	^.*db_with_timestamp_basic_test.*$$|^.*table_test.*$$|^.*db_wal_test.*$$|^.*block_based_table_reader_test.*$$|^.*block_test.*$$
 prioritize_long_running_tests =						\
   perl -pe 's,($(slow_test_regexp)),100 $$1,'				\
     | sort -k1,1gr							\
     | sed 's/^[.0-9]* //'
 
+# Helper: print binaries observed to be slow in the last `make check` run.
+# Use this to decide whether `slow_test_regexp` above needs updating.
+# Aggregates per-binary across shards; flags any binary whose max single-shard
+# time is >= 20s OR whose total time is >= 200s.
+.PHONY: suggest-slow-tests
+suggest-slow-tests:
+	@if [ ! -s LOG ]; then \
+	  echo "No (or empty) LOG file. Run 'make check' first." >&2; exit 1; \
+	fi
+	@bash -c '$(quoted_perl_command)' < LOG | awk '\
+	  { \
+	    t=$$1; name=$$3; \
+	    sub(/^t\/run-/, "", name); \
+	    sub(/-shard-[0-9]+$$/, "", name); \
+	    total[name] += t; \
+	    if (t > max[name]) max[name] = t; \
+	    count[name]++; \
+	  } \
+	  END { \
+	    printf "%8s %8s %5s  %s\n", "TOTAL_s", "MAX_s", "SHRDS", "BINARY"; \
+	    for (n in total) \
+	      if (max[n] >= 20 || total[n] >= 200) \
+	        printf "%8.1f %8.1f %5d  %s\n", total[n], max[n], count[n], n; \
+	  }' | (read header; echo "$$header"; sort -k2,2gr)
+
 # "make check" uses
 # Run with "make J=1 check" to disable parallelism in "make check".
 # Run with "make J=200% check" to run two parallel jobs per core.
@@ -1531,7 +1584,7 @@ db_open_with_config_test: $(OBJ_DIR)/db/db_open_with_config_test.o $(TEST_LIBRAR
 db_test: $(OBJ_DIR)/db/db_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
-db_test2: $(OBJ_DIR)/db/db_test2.o $(TEST_LIBRARY) $(LIBRARY)
+db_etc2_test: $(OBJ_DIR)/db/db_etc2_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
 db_etc3_test: $(OBJ_DIR)/db/db_etc3_test.o $(TEST_LIBRARY) $(LIBRARY)

build_tools/getdeps_fallback_mirror.py

@@ -1,66 +1,188 @@
+#!/usr/bin/env python3
 #  Copyright (c) Meta Platforms, Inc. and affiliates.
 #  This source code is licensed under both the GPLv2 (found in the COPYING file in the root directory)
 #  and the Apache 2.0 License (found in the LICENSE.Apache file in the root directory).
-
-#!/usr/bin/env python3
 """
 Pre-download packages with unreliable mirrors using fallback mirrors.
 Reads package info from folly's getdeps manifest files.
 """
-import sys
-import os
-import hashlib
-import subprocess
 import configparser
+import hashlib
+import os
+import shutil
+import sys
+import urllib.request
+
+DOWNLOAD_TIMEOUT_SECONDS = 120
+DOWNLOAD_CHUNK_BYTES = 64 * 1024
+MAX_DOWNLOAD_BYTES = 50 * 1024 * 1024
+
+MIRROR_FALLBACKS = {
+    "ftpmirror.gnu.org/gnu/": [
+        "https://mirrors.kernel.org/gnu/",
+        "https://ftpmirror.gnu.org/gnu/",
+        "https://ftp.gnu.org/gnu/",
+    ],
+    "ftp.gnu.org/gnu/": [
+        "https://mirrors.kernel.org/gnu/",
+        "https://ftpmirror.gnu.org/gnu/",
+        "https://ftp.gnu.org/gnu/",
+    ],
+}
+
+# These packages must have URLs matching MIRROR_FALLBACKS; other packages are
+# left for getdeps.py's normal download path.
+PACKAGES_TO_CHECK = ("autoconf", "automake", "libtool", "libiberty")
+
 
 def sha256_file(path):
     """Calculate SHA256 hash of a file."""
     h = hashlib.sha256()
     try:
-        with open(path, 'rb') as f:
-            for chunk in iter(lambda: f.read(65536), b''):
+        with open(path, "rb") as f:
+            for chunk in iter(lambda: f.read(65536), b""):
                 h.update(chunk)
         return h.hexdigest()
     except Exception:
         return None
 
+
 def parse_manifest(manifest_path):
     """Parse a getdeps manifest file to extract download info."""
-    config = configparser.ConfigParser()
+    # folly manifests can contain bare keys in sections unrelated to downloads.
+    config = configparser.ConfigParser(allow_no_value=True, interpolation=None)
     try:
-        config.read(manifest_path)
-        if 'download' in config:
-            return {
-                'url': config['download'].get('url', ''),
-                'sha256': config['download'].get('sha256', ''),
-            }
-    except Exception:
-        pass
+        with open(manifest_path, encoding="utf-8") as manifest_file:
+            config.read_file(manifest_file)
+    except Exception as ex:
+        print(f"  {os.path.basename(manifest_path)}: WARNING - parse failed: {ex}")
+        return None
+
+    if "download" in config:
+        return {
+            "url": config["download"].get("url", ""),
+            "sha256": config["download"].get("sha256", ""),
+        }
     return None
 
+
+def file_size(path):
+    try:
+        return os.path.getsize(path)
+    except Exception:
+        return None
+
+
 def get_fallback_mirrors(url):
     """Get fallback mirror URLs for a given URL."""
-    # Fallback mirror patterns for known unreliable hosts
-    mirror_fallbacks = {
-        "ftp.gnu.org/gnu/": [
-            "https://mirrors.kernel.org/gnu/",
-            "https://ftpmirror.gnu.org/gnu/",
-            "https://ftp.gnu.org/gnu/",
-        ],
-        "ftpmirror.gnu.org/gnu/": [
-            "https://mirrors.kernel.org/gnu/",
-            "https://ftpmirror.gnu.org/gnu/",
-            "https://ftp.gnu.org/gnu/",
-        ],
-    }
-
-    for pattern, mirrors in mirror_fallbacks.items():
+    for pattern, mirrors in MIRROR_FALLBACKS.items():
         if pattern in url:
             # Extract the path after the pattern
             path_start = url.find(pattern) + len(pattern)
             path = url[path_start:]
             return [mirror + path for mirror in mirrors]
-    return [url]  # No fallback, use original
+    return []
+
+
+def download_url(url, filepath):
+    """Download URL to filepath without leaving partial files behind."""
+    tmp_filepath = filepath + ".tmp"
+    if os.path.exists(tmp_filepath):
+        os.remove(tmp_filepath)
+
+    request = urllib.request.Request(
+        url, headers={"User-Agent": "rocksdb-getdeps-fallback/1.0"}
+    )
+    try:
+        with urllib.request.urlopen(
+            request, timeout=DOWNLOAD_TIMEOUT_SECONDS
+        ) as response, open(tmp_filepath, "wb") as output:
+            copied = 0
+            while True:
+                chunk = response.read(DOWNLOAD_CHUNK_BYTES)
+                if not chunk:
+                    break
+
+                copied += len(chunk)
+                if copied > MAX_DOWNLOAD_BYTES:
+                    raise Exception(
+                        f"download exceeds {MAX_DOWNLOAD_BYTES} bytes"
+                    )
+                output.write(chunk)
+        os.replace(tmp_filepath, filepath)
+    finally:
+        if os.path.exists(tmp_filepath):
+            os.remove(tmp_filepath)
+
+
+def prepare_download(package, info, download_dir, cache_dir):
+    url = info["url"]
+    expected_sha256 = info["sha256"]
+    mirrors = get_fallback_mirrors(url)
+    if not mirrors:
+        return False
+
+    if not expected_sha256:
+        print(f"  {package}: WARNING - skipped fallback without sha256")
+        return False
+
+    # getdeps uses format: {package}-{filename}
+    filename = f"{package}-{os.path.basename(url)}"
+    filepath = os.path.join(download_dir, filename)
+    cache_path = os.path.join(cache_dir, filename)
+
+    # Check if already valid.
+    actual_sha256 = sha256_file(filepath) if os.path.exists(filepath) else None
+    if actual_sha256 == expected_sha256:
+        print(f"  {filename}: OK (already downloaded)")
+        return True
+    if actual_sha256 is not None:
+        print(
+            f"  {filename}: WARNING - removing invalid download "
+            f"sha256={actual_sha256}"
+        )
+        os.remove(filepath)
+
+    # The cache is only an opportunistic single-build accelerator; callers
+    # should not share it across concurrent builds without external locking.
+    actual_sha256 = sha256_file(cache_path) if os.path.exists(cache_path) else None
+    if actual_sha256 == expected_sha256:
+        print(f"  {filename}: OK (from cache)")
+        shutil.copy2(cache_path, filepath)
+        return True
+    if actual_sha256 is not None:
+        print(
+            f"  {filename}: WARNING - removing invalid cache "
+            f"sha256={actual_sha256}"
+        )
+        os.remove(cache_path)
+
+    # Try fallback mirrors.
+    for mirror_url in mirrors:
+        print(f"  {filename}: trying {mirror_url}...")
+        try:
+            download_url(mirror_url, filepath)
+        except Exception as ex:
+            print(f"  {filename}: WARNING - download failed: {ex}")
+            continue
+
+        actual_sha256 = sha256_file(filepath)
+        if actual_sha256 == expected_sha256:
+            size = file_size(filepath)
+            print(f"  {filename}: OK (downloaded, {size} bytes)")
+            shutil.copy2(filepath, cache_path)
+            return True
+
+        size = file_size(filepath)
+        print(
+            f"  {filename}: WARNING - sha256 mismatch from {mirror_url}: "
+            f"expected={expected_sha256} actual={actual_sha256} size={size}"
+        )
+        os.remove(filepath)
+
+    print(f"  {filename}: WARNING - all mirrors failed")
+    return False
+
 
 def main():
     if len(sys.argv) != 4:
@@ -68,60 +190,39 @@ def main():
         sys.exit(1)
 
     download_dir, cache_dir, manifests_dir = sys.argv[1], sys.argv[2], sys.argv[3]
+    os.makedirs(download_dir, exist_ok=True)
+    os.makedirs(cache_dir, exist_ok=True)
 
-    # Packages known to have unreliable mirrors
-    packages_to_check = ["autoconf", "automake", "libtool"]
-
-    for package in packages_to_check:
+    checked = 0
+    ready = 0
+    for package in PACKAGES_TO_CHECK:
         manifest_path = os.path.join(manifests_dir, package)
-        if not os.path.exists(manifest_path):
+        if not os.path.isfile(manifest_path):
             continue
 
         info = parse_manifest(manifest_path)
-        if not info or not info['url'] or not info['sha256']:
+        if not info or not info["url"]:
             continue
 
-        # Determine filename from URL
-        url = info['url']
-        expected_sha256 = info['sha256']
-        url_filename = os.path.basename(url)
-
-        # getdeps uses format: {package}-{filename}
-        filename = f"{package}-{url_filename}"
-        filepath = os.path.join(download_dir, filename)
-        cache_path = os.path.join(cache_dir, filename)
-
-        # Check if already valid
-        if os.path.exists(filepath) and sha256_file(filepath) == expected_sha256:
-            print(f"  {filename}: OK (already downloaded)")
+        if not info["sha256"]:
+            print(f"  {package}: WARNING - skipped fallback without sha256")
             continue
 
-        # Check cache
-        if os.path.exists(cache_path) and sha256_file(cache_path) == expected_sha256:
-            print(f"  {filename}: OK (from cache)")
-            subprocess.run(['cp', cache_path, filepath], check=True)
+        if not get_fallback_mirrors(info["url"]):
+            print(
+                f"  {package}: WARNING - skipped fallback without known mirror "
+                f"for {info['url']}"
+            )
             continue
 
-        # Try fallback mirrors
-        mirrors = get_fallback_mirrors(url)
-        downloaded = False
-        for mirror_url in mirrors:
-            print(f"  {filename}: trying {mirror_url}...")
-            try:
-                subprocess.run(['wget', '-q', '-O', filepath, mirror_url], check=True, timeout=120)
-                if sha256_file(filepath) == expected_sha256:
-                    print(f"  {filename}: OK (downloaded)")
-                    subprocess.run(['cp', filepath, cache_path], check=False)
-                    downloaded = True
-                    break
-                else:
-                    os.remove(filepath)
-            except Exception:
-                if os.path.exists(filepath):
-                    os.remove(filepath)
+        checked += 1
+        try:
+            if prepare_download(package, info, download_dir, cache_dir):
+                ready += 1
+        except Exception as ex:
+            print(f"  {package}: WARNING - fallback preparation failed: {ex}")
 
-        if not downloaded:
-            print(f"  {filename}: WARNING - all mirrors failed")
+    print(f"  fallback mirror downloads ready: {ready}/{checked}")
 
 if __name__ == "__main__":
     main()

db/blob/blob_file_reader.cc

@@ -20,6 +20,7 @@
 #include "table/format.h"
 #include "table/multiget_context.h"
 #include "test_util/sync_point.h"
+#include "util/aligned_buffer.h"
 #include "util/compression.h"
 #include "util/stop_watch.h"
 
@@ -165,7 +166,7 @@ Status BlobFileReader::ReadHeader(const RandomAccessFileReader* file_reader,
 
   Slice header_slice;
   Buffer buf;
-  AlignedBuf aligned_buf;
+  AlignedBuffer direct_io_buffer;
 
   {
     TEST_SYNC_POINT("BlobFileReader::ReadHeader:ReadFromFile");
@@ -175,7 +176,7 @@ Status BlobFileReader::ReadHeader(const RandomAccessFileReader* file_reader,
 
     const Status s =
         ReadFromFile(file_reader, read_options, read_offset, read_size,
-                     statistics, &header_slice, &buf, &aligned_buf);
+                     statistics, &header_slice, &buf, &direct_io_buffer);
     if (!s.ok()) {
       return s;
     }
@@ -216,7 +217,7 @@ Status BlobFileReader::ReadFooter(const RandomAccessFileReader* file_reader,
 
   Slice footer_slice;
   Buffer buf;
-  AlignedBuf aligned_buf;
+  AlignedBuffer direct_io_buffer;
 
   {
     TEST_SYNC_POINT("BlobFileReader::ReadFooter:ReadFromFile");
@@ -226,7 +227,7 @@ Status BlobFileReader::ReadFooter(const RandomAccessFileReader* file_reader,
 
     const Status s =
         ReadFromFile(file_reader, read_options, read_offset, read_size,
-                     statistics, &footer_slice, &buf, &aligned_buf);
+                     statistics, &footer_slice, &buf, &direct_io_buffer);
     if (!s.ok()) {
       return s;
     }
@@ -257,10 +258,11 @@ Status BlobFileReader::ReadFromFile(const RandomAccessFileReader* file_reader,
                                     const ReadOptions& read_options,
                                     uint64_t read_offset, size_t read_size,
                                     Statistics* statistics, Slice* slice,
-                                    Buffer* buf, AlignedBuf* aligned_buf) {
+                                    Buffer* buf,
+                                    AlignedBuffer* direct_io_buffer) {
   assert(slice);
   assert(buf);
-  assert(aligned_buf);
+  assert(direct_io_buffer);
 
   assert(file_reader);
 
@@ -277,15 +279,15 @@ Status BlobFileReader::ReadFromFile(const RandomAccessFileReader* file_reader,
 
   if (file_reader->use_direct_io()) {
     constexpr char* scratch = nullptr;
+    AlignedBufferAllocationContext direct_io_context{direct_io_buffer};
 
     s = file_reader->Read(io_options, read_offset, read_size, slice, scratch,
-                          aligned_buf, &dbg);
+                          &direct_io_context, &dbg);
   } else {
     buf->reset(new char[read_size]);
-    constexpr AlignedBuf* aligned_scratch = nullptr;
 
     s = file_reader->Read(io_options, read_offset, read_size, slice, buf->get(),
-                          aligned_scratch, &dbg);
+                          nullptr, &dbg);
   }
 
   if (!s.ok()) {
@@ -349,7 +351,7 @@ Status BlobFileReader::GetBlob(
 
   Slice record_slice;
   Buffer buf;
-  AlignedBuf aligned_buf;
+  AlignedBuffer direct_io_buffer;
 
   bool prefetched = false;
 
@@ -379,7 +381,7 @@ Status BlobFileReader::GetBlob(
     const Status s =
         ReadFromFile(file_reader_.get(), read_options, record_offset,
                      static_cast<size_t>(record_size), statistics_,
-                     &record_slice, &buf, &aligned_buf);
+                     &record_slice, &buf, &direct_io_buffer);
     if (!s.ok()) {
       return s;
     }
@@ -477,7 +479,7 @@ void BlobFileReader::MultiGetBlob(
   }
 
   Buffer buf;
-  AlignedBuf aligned_buf;
+  AlignedBuffer direct_io_buffer;
 
   Status s;
   bool direct_io = file_reader_->use_direct_io();
@@ -500,8 +502,9 @@ void BlobFileReader::MultiGetBlob(
   IODebugContext dbg;
   s = file_reader_->PrepareIOOptions(read_options, opts, &dbg);
   if (s.ok()) {
+    AlignedBufferAllocationContext direct_io_context{&direct_io_buffer};
     s = file_reader_->MultiRead(opts, read_reqs.data(), read_reqs.size(),
-                                direct_io ? &aligned_buf : nullptr, &dbg);
+                                &direct_io_context, &dbg);
   }
   if (!s.ok()) {
     for (auto& req : read_reqs) {

db/blob/blob_file_reader.h

@@ -108,7 +108,7 @@ class BlobFileReader {
                              const ReadOptions& read_options,
                              uint64_t read_offset, size_t read_size,
                              Statistics* statistics, Slice* slice, Buffer* buf,
-                             AlignedBuf* aligned_buf);
+                             AlignedBuffer* direct_io_buffer);
 
   static Status VerifyBlob(const Slice& record_slice, const Slice& user_key,
                            uint64_t value_size);

db/blob/blob_source.cc

@@ -74,6 +74,7 @@ Status BlobSource::GetBlobFromCache(
     assert(cached_blob->GetValue());
 
     PERF_COUNTER_ADD(blob_cache_hit_count, 1);
+    PERF_COUNTER_ADD(blob_cache_read_byte, cached_blob->GetValue()->size());
     RecordTick(statistics_, BLOB_DB_CACHE_HIT);
     RecordTick(statistics_, BLOB_DB_CACHE_BYTES_READ,
                cached_blob->GetValue()->size());

db/blob/blob_source_test.cc

@@ -229,6 +229,7 @@ TEST_F(BlobSourceTest, GetBlobsFromCache) {
     // Retrieved the blob cache num_blobs * 3 times via TEST_BlobInCache,
     // GetBlob, and TEST_BlobInCache.
     ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, 0);
+    ASSERT_EQ((int)get_perf_context()->blob_cache_read_byte, 0);
     ASSERT_EQ((int)get_perf_context()->blob_read_count, num_blobs);
     ASSERT_EQ((int)get_perf_context()->blob_read_byte, total_bytes);
     ASSERT_GE((int)get_perf_context()->blob_checksum_time, 0);
@@ -262,6 +263,8 @@ TEST_F(BlobSourceTest, GetBlobsFromCache) {
       blob_bytes += blob_sizes[i];
       total_bytes += bytes_read;
       ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, i);
+      ASSERT_EQ((int)get_perf_context()->blob_cache_read_byte,
+                blob_bytes - blob_sizes[i]);
       ASSERT_EQ((int)get_perf_context()->blob_read_count, i + 1);
       ASSERT_EQ((int)get_perf_context()->blob_read_byte, total_bytes);
 
@@ -269,11 +272,13 @@ TEST_F(BlobSourceTest, GetBlobsFromCache) {
                                                blob_offsets[i]));
 
       ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, i + 1);
+      ASSERT_EQ((int)get_perf_context()->blob_cache_read_byte, blob_bytes);
       ASSERT_EQ((int)get_perf_context()->blob_read_count, i + 1);
       ASSERT_EQ((int)get_perf_context()->blob_read_byte, total_bytes);
     }
 
     ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, num_blobs);
+    ASSERT_EQ((int)get_perf_context()->blob_cache_read_byte, blob_bytes);
     ASSERT_EQ((int)get_perf_context()->blob_read_count, num_blobs);
     ASSERT_EQ((int)get_perf_context()->blob_read_byte, total_bytes);
 
@@ -312,6 +317,7 @@ TEST_F(BlobSourceTest, GetBlobsFromCache) {
     // Retrieved the blob cache num_blobs * 3 times via TEST_BlobInCache,
     // GetBlob, and TEST_BlobInCache.
     ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, num_blobs * 3);
+    ASSERT_EQ((int)get_perf_context()->blob_cache_read_byte, blob_bytes * 3);
     ASSERT_EQ((int)get_perf_context()->blob_read_count, 0);  // without i/o
     ASSERT_EQ((int)get_perf_context()->blob_read_byte, 0);   // without i/o
 
@@ -690,6 +696,8 @@ TEST_F(BlobSourceTest, MultiGetBlobsFromMultiFiles) {
     // TEST_BlobInCache.
     ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count,
               num_blobs * blob_files);
+    ASSERT_EQ((int)get_perf_context()->blob_cache_read_byte,
+              blob_value_bytes * blob_files);
     ASSERT_EQ((int)get_perf_context()->blob_read_count,
               num_blobs * blob_files);  // blocking i/o
     ASSERT_EQ((int)get_perf_context()->blob_read_byte,
@@ -750,6 +758,8 @@ TEST_F(BlobSourceTest, MultiGetBlobsFromMultiFiles) {
     // via MultiGetBlob and TEST_BlobInCache.
     ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count,
               num_blobs * blob_files * 2);
+    ASSERT_EQ((int)get_perf_context()->blob_cache_read_byte,
+              blob_value_bytes * blob_files * 2);
     ASSERT_EQ((int)get_perf_context()->blob_read_count,
               0);  // blocking i/o
     ASSERT_EQ((int)get_perf_context()->blob_read_byte,

db/c.cc

@@ -3771,6 +3771,11 @@ void rocksdb_block_based_options_set_index_block_search_type(
       static_cast<BlockBasedTableOptions::BlockSearchType>(v);
 }
 
+void rocksdb_block_based_options_set_uniform_cv_threshold(
+    rocksdb_block_based_table_options_t* options, double v) {
+  options->rep.uniform_cv_threshold = v;
+}
+
 void rocksdb_block_based_options_set_data_block_hash_ratio(
     rocksdb_block_based_table_options_t* options, double v) {
   options->rep.data_block_hash_table_util_ratio = v;
@@ -5398,6 +5403,16 @@ size_t rocksdb_options_get_memtable_huge_page_size(rocksdb_options_t* opt) {
   return opt->rep.memtable_huge_page_size;
 }
 
+void rocksdb_options_set_memtable_batch_lookup_optimization(
+    rocksdb_options_t* opt, unsigned char v) {
+  opt->rep.memtable_batch_lookup_optimization = v;
+}
+
+unsigned char rocksdb_options_get_memtable_batch_lookup_optimization(
+    rocksdb_options_t* opt) {
+  return opt->rep.memtable_batch_lookup_optimization;
+}
+
 void rocksdb_options_set_hash_skip_list_rep(rocksdb_options_t* opt,
                                             size_t bucket_count,
                                             int32_t skiplist_height,
@@ -5829,6 +5844,8 @@ uint64_t rocksdb_perfcontext_metric(rocksdb_perfcontext_t* context,
       return rep->number_async_seek;
     case rocksdb_blob_cache_hit_count:
       return rep->blob_cache_hit_count;
+    case rocksdb_blob_cache_read_byte:
+      return rep->blob_cache_read_byte;
     case rocksdb_blob_read_count:
       return rep->blob_read_count;
     case rocksdb_blob_read_byte:
@@ -6247,6 +6264,16 @@ unsigned char rocksdb_readoptions_get_async_io(rocksdb_readoptions_t* opt) {
   return opt->rep.async_io;
 }
 
+void rocksdb_readoptions_set_optimize_multiget_for_io(
+    rocksdb_readoptions_t* opt, unsigned char v) {
+  opt->rep.optimize_multiget_for_io = v;
+}
+
+unsigned char rocksdb_readoptions_get_optimize_multiget_for_io(
+    rocksdb_readoptions_t* opt) {
+  return opt->rep.optimize_multiget_for_io;
+}
+
 void rocksdb_readoptions_set_timestamp(rocksdb_readoptions_t* opt,
                                        const char* ts, size_t tslen) {
   if (ts == nullptr) {

db/c_test.c

@@ -888,6 +888,9 @@ int main(int argc, char** argv) {
   rocksdb_block_based_options_set_block_cache(table_options, cache);
   rocksdb_block_based_options_set_data_block_index_type(table_options, 1);
   rocksdb_block_based_options_set_data_block_hash_ratio(table_options, 0.75);
+  rocksdb_block_based_options_set_index_block_search_type(
+      table_options, rocksdb_block_based_table_index_block_search_type_auto);
+  rocksdb_block_based_options_set_uniform_cv_threshold(table_options, 0.2);
   rocksdb_block_based_options_set_top_level_index_pinning_tier(table_options,
                                                                1);
   rocksdb_block_based_options_set_partition_pinning_tier(table_options, 2);
@@ -2812,6 +2815,15 @@ int main(int argc, char** argv) {
     rocksdb_options_set_memtable_huge_page_size(o, 25);
     CheckCondition(25 == rocksdb_options_get_memtable_huge_page_size(o));
 
+    // memtable_batch_lookup_optimization defaults to 0; flip to 1 to verify
+    // the setter is wired through to the underlying C++ ColumnFamilyOptions
+    // field.
+    CheckCondition(0 ==
+                   rocksdb_options_get_memtable_batch_lookup_optimization(o));
+    rocksdb_options_set_memtable_batch_lookup_optimization(o, 1);
+    CheckCondition(1 ==
+                   rocksdb_options_get_memtable_batch_lookup_optimization(o));
+
     rocksdb_options_set_max_successive_merges(o, 26);
     CheckCondition(26 == rocksdb_options_get_max_successive_merges(o));
 
@@ -2998,6 +3010,8 @@ int main(int argc, char** argv) {
                    rocksdb_options_get_memtable_prefix_bloom_size_ratio(copy));
     CheckCondition(24 == rocksdb_options_get_max_compaction_bytes(copy));
     CheckCondition(25 == rocksdb_options_get_memtable_huge_page_size(copy));
+    CheckCondition(
+        1 == rocksdb_options_get_memtable_batch_lookup_optimization(copy));
     CheckCondition(26 == rocksdb_options_get_max_successive_merges(copy));
     CheckCondition(27 == rocksdb_options_get_bloom_locality(copy));
     CheckCondition(1 == rocksdb_options_get_inplace_update_support(copy));
@@ -3432,6 +3446,12 @@ int main(int argc, char** argv) {
     rocksdb_readoptions_set_async_io(ro, 1);
     CheckCondition(1 == rocksdb_readoptions_get_async_io(ro));
 
+    // optimize_multiget_for_io defaults to true (1); flip to 0 to verify the
+    // setter is wired through to the underlying C++ ReadOptions field.
+    CheckCondition(1 == rocksdb_readoptions_get_optimize_multiget_for_io(ro));
+    rocksdb_readoptions_set_optimize_multiget_for_io(ro, 0);
+    CheckCondition(0 == rocksdb_readoptions_get_optimize_multiget_for_io(ro));
+
     rocksdb_readoptions_destroy(ro);
   }
 

db/compaction/compaction_job.cc

@@ -2065,10 +2065,18 @@ Status CompactionJob::FinishCompactionOutputFile(
     std::pair<SequenceNumber, SequenceNumber> keep_seqno_range{
         0, kMaxSequenceNumber};
     if (sub_compact->compaction->SupportsPerKeyPlacement()) {
+      // Point entries are routed to proximal output only when their seqno is
+      // strictly greater than `proximal_after_seqno_`. Range tombstones use a
+      // [lower, upper) filter, so split them at the next seqno to preserve the
+      // same boundary.
+      SequenceNumber range_del_split_seqno = proximal_after_seqno_;
+      if (range_del_split_seqno < kMaxSequenceNumber) {
+        range_del_split_seqno++;
+      }
       if (outputs.IsProximalLevel()) {
-        keep_seqno_range.first = proximal_after_seqno_;
+        keep_seqno_range.first = range_del_split_seqno;
       } else {
-        keep_seqno_range.second = proximal_after_seqno_;
+        keep_seqno_range.second = range_del_split_seqno;
       }
     }
     CompactionIterationStats range_del_out_stats;

db/compaction/compaction_service_job.cc

@@ -189,14 +189,24 @@ CompactionJob::ProcessKeyValueCompactionWithCompactionService(
     return compaction_status;
   }
 
-  // CompactionServiceJobStatus::kSuccess was returned, but somehow we failed to
-  // read the result. Consider this as an installation failure
   if (!s.ok()) {
-    sub_compact->status = s;
+    // Wait() returned kSuccess, but the primary host could not deserialize the
+    // remote result before importing any output files into this DB. The remote
+    // output is still isolated in the service-managed staging directory, so it
+    // is safe to fall back to local compaction for the same job and notify the
+    // service via OnInstallation(kUseLocal).
+    assert(sub_compact->status.ok());
+    assert(sub_compact->GetMutableCompactionOutputs().empty());
+    assert(sub_compact->GetMutableProximalOutputs().empty());
+    ROCKS_LOG_WARN(db_options_.info_log,
+                   "[%s] [JOB %d] Failed to parse remote compaction result "
+                   "(%s), falling back to local compaction",
+                   compaction->column_family_data()->GetName().c_str(), job_id_,
+                   s.ToString().c_str());
     compaction_result.status.PermitUncheckedError();
     db_options_.compaction_service->OnInstallation(
-        response.scheduled_job_id, CompactionServiceJobStatus::kFailure);
-    return CompactionServiceJobStatus::kFailure;
+        response.scheduled_job_id, CompactionServiceJobStatus::kUseLocal);
+    return CompactionServiceJobStatus::kUseLocal;
   }
   sub_compact->status = compaction_result.status;
 
@@ -641,6 +651,10 @@ static std::unordered_map<std::string, OptionTypeInfo>
         {"cpu_micros",
          {offsetof(struct CompactionJobStats, cpu_micros), OptionType::kUInt64T,
           OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
+        {"has_accurate_num_input_records",
+         {offsetof(struct CompactionJobStats, has_accurate_num_input_records),
+          OptionType::kBoolean, OptionVerificationType::kNormal,
+          OptionTypeFlags::kNone}},
         {"num_input_records",
          {offsetof(struct CompactionJobStats, num_input_records),
           OptionType::kUInt64T, OptionVerificationType::kNormal,

db/compaction/compaction_service_test.cc

@@ -127,6 +127,7 @@ class MyTestCompactionService : public CompactionService {
 
   void OnInstallation(const std::string& /*scheduled_job_id*/,
                       CompactionServiceJobStatus status) override {
+    installation_callback_count_.fetch_add(1);
     final_updated_status_ = status;
   }
 
@@ -167,6 +168,8 @@ class MyTestCompactionService : public CompactionService {
     return final_updated_status_.load();
   }
 
+  int GetOnInstallationCount() { return installation_callback_count_.load(); }
+
  protected:
   InstrumentedMutex mutex_;
   const std::string db_path_;
@@ -195,6 +198,7 @@ class MyTestCompactionService : public CompactionService {
   std::vector<std::shared_ptr<EventListener>> listeners_;
   std::vector<std::shared_ptr<TablePropertiesCollectorFactory>>
       table_properties_collector_factories_;
+  std::atomic_int installation_callback_count_{0};
   std::atomic<CompactionServiceJobStatus> final_updated_status_{
       CompactionServiceJobStatus::kUseLocal};
 
@@ -939,6 +943,37 @@ TEST_F(CompactionServiceTest, VerifyInputRecordCount) {
   SyncPoint::GetInstance()->ClearAllCallBacks();
 }
 
+TEST_F(CompactionServiceTest, InaccurateInputRecordCount) {
+  Options options = CurrentOptions();
+  options.disable_auto_compactions = true;
+  ReopenWithCompactionService(&options);
+  GenerateTestData();
+
+  auto my_cs = GetCompactionService();
+
+  std::string start_str = Key(15);
+  std::string end_str = Key(45);
+  Slice start(start_str);
+  Slice end(end_str);
+  uint64_t comp_num = my_cs->GetCompactionNum();
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "CompactionServiceCompactionJob::Run:0", [&](void* arg) {
+        CompactionServiceResult* compaction_result =
+            *(static_cast<CompactionServiceResult**>(arg));
+        ASSERT_TRUE(compaction_result != nullptr);
+        compaction_result->stats.has_accurate_num_input_records = false;
+        compaction_result->stats.num_input_records = 0;
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &start, &end));
+  ASSERT_GE(my_cs->GetCompactionNum(), comp_num + 1);
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+}
+
 TEST_F(CompactionServiceTest, EmptyResult) {
   Options options = CurrentOptions();
   options.disable_auto_compactions = true;
@@ -1448,7 +1483,7 @@ TEST_F(CompactionServiceTest, FailedToStart) {
   ASSERT_TRUE(s.IsIncomplete());
 }
 
-TEST_F(CompactionServiceTest, InvalidResult) {
+TEST_F(CompactionServiceTest, InvalidResultFallsBackToLocal) {
   Options options = CurrentOptions();
   options.disable_auto_compactions = true;
   ReopenWithCompactionService(&options);
@@ -1456,15 +1491,22 @@ TEST_F(CompactionServiceTest, InvalidResult) {
   GenerateTestData();
 
   auto my_cs = GetCompactionService();
+  ASSERT_EQ(0, my_cs->GetOnInstallationCount());
   my_cs->OverrideWaitResult("Invalid Str");
+  int compaction_num_before = my_cs->GetCompactionNum();
 
   std::string start_str = Key(15);
   std::string end_str = Key(45);
   Slice start(start_str);
   Slice end(end_str);
+  // The remote result is malformed before any installation starts, so the
+  // primary should recover by rerunning the compaction locally for this job.
   Status s = db_->CompactRange(CompactRangeOptions(), &start, &end);
-  ASSERT_FALSE(s.ok());
-  ASSERT_EQ(CompactionServiceJobStatus::kFailure,
+  ASSERT_OK(s);
+  ASSERT_GE(my_cs->GetCompactionNum(), compaction_num_before + 1);
+  VerifyTestData();
+  ASSERT_EQ(1, my_cs->GetOnInstallationCount());
+  ASSERT_EQ(CompactionServiceJobStatus::kUseLocal,
             my_cs->GetFinalCompactionServiceJobStatus());
 }
 

db/compaction/tiered_compaction_test.cc

@@ -2897,6 +2897,61 @@ TEST_P(PrecludeLastLevelTest, RangeDelsCauseFileEndpointsToOverlap) {
   Close();
 }
 
+TEST_F(PrecludeLastLevelTestBase,
+       RangeDelAtProximalSeqnoBoundaryStaysInLastLevel) {
+  constexpr int kNumLevels = 7;
+
+  Options options = CurrentOptions();
+  options.env = mock_env_.get();
+  options.num_levels = kNumLevels;
+  options.disable_auto_compactions = true;
+  DestroyAndReopen(options);
+  Defer close_db([&] { Close(); });
+
+  ASSERT_OK(Put(Key(2), "old"));
+  ASSERT_OK(Put(Key(12), "old"));
+  ManagedSnapshot snapshot(db_.get());
+  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(2),
+                             Key(12)));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(6);
+
+  const int l5_file_keys[][2] = {{0, 4}, {5, 9}};
+  for (const auto& l5_file : l5_file_keys) {
+    ASSERT_OK(Put(Key(l5_file[0]), "hot"));
+    ASSERT_OK(Put(Key(l5_file[1]), "hot"));
+    ASSERT_OK(Flush());
+    MoveFilesToLevel(5);
+  }
+  ASSERT_EQ("0,0,0,0,0,2,1", FilesPerLevel());
+
+  SyncPoint::GetInstance()->SetCallBack(
+      "CompactionJob::PrepareTimes():preclude_last_level_min_seqno",
+      [](void* arg) { *static_cast<SequenceNumber*>(arg) = 0; });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  ASSERT_OK(db_->SetOptions({{"preclude_last_level_data_seconds", "10000"}}));
+
+  auto l5_files = GetLevelFileMetadatas(5);
+  auto l6_files = GetLevelFileMetadatas(6);
+  ASSERT_EQ(2, l5_files.size());
+  ASSERT_EQ(1, l6_files.size());
+
+  ASSERT_OK(db_->CompactFiles(CompactionOptions(),
+                              {MakeTableFileName(l5_files[1]->fd.GetNumber()),
+                               MakeTableFileName(l6_files[0]->fd.GetNumber())},
+                              6));
+
+  uint64_t l6_range_deletions = 0;
+  for (auto* f : GetLevelFileMetadatas(5)) {
+    ASSERT_EQ(0, f->num_range_deletions);
+  }
+  for (auto* f : GetLevelFileMetadatas(6)) {
+    l6_range_deletions += f->num_range_deletions;
+  }
+  ASSERT_GT(l6_range_deletions, 0);
+}
+
 // Tests DBIter::GetProperty("rocksdb.iterator.write-time") return a data's
 // approximate write unix time.
 class IteratorWriteTimeTest

db/db_basic_test.cc

@@ -761,6 +761,111 @@ TEST_F(DBBasicTest, ReuseManifestOnOpenFullStackComposition) {
   EXPECT_EQ("v", Get("k"));
 }
 
+// Regression test for WAL recovery while publishing a fresh MANIFEST. The test
+// stores SSTs in a separate DB path, injects failure after CURRENT points at
+// the new MANIFEST, and simulates crash cleanup; the recovered SST must survive
+// because the synced MANIFEST references it.
+TEST_F(DBBasicTest, RecoverySstDirSyncedBeforeFreshManifestPublish) {
+  auto fault_fs = std::make_shared<FaultInjectionTestFS>(env_->GetFileSystem());
+  std::unique_ptr<Env> fault_env(NewCompositeEnv(fault_fs));
+
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.disable_auto_compactions = true;
+  options.env = fault_env.get();
+  options.reuse_manifest_on_open = false;
+  options.db_paths.emplace_back(dbname_ + "_2", 1ULL << 30);
+
+  DestroyAndReopen(options);
+  ASSERT_OK(Put("base", "value"));
+  ASSERT_OK(Flush());
+
+  ASSERT_OK(Put("recovered", "value"));
+  ASSERT_OK(db_->FlushWAL(true));
+  Close();
+
+  fault_fs->ResetState();
+
+  std::atomic<bool> fail_after_current_publish{true};
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "VersionSet::ProcessManifestWrites:AfterSetCurrentFile", [&](void* arg) {
+        if (fail_after_current_publish.exchange(false)) {
+          ASSERT_NE(nullptr, arg);
+          IOStatus* io_s = static_cast<IOStatus*>(arg);
+          ASSERT_OK(*io_s);
+          *io_s = IOStatus::IOError("injected current publish aftermath");
+        }
+      });
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  Status s = TryReopen(options);
+  ASSERT_TRUE(s.IsIOError()) << s.ToString();
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
+
+  ASSERT_OK(fault_fs->DeleteFilesCreatedAfterLastDirSync(IOOptions(), nullptr));
+
+  s = TryReopen(options);
+  ASSERT_OK(s);
+  ASSERT_EQ("value", Get("base"));
+  ASSERT_EQ("value", Get("recovered"));
+  Close();
+}
+
+// Regression test for WAL recovery while appending to a reused MANIFEST. The
+// first reopen forces recovery to create an SST and then fail after MANIFEST
+// sync. The simulated crash cleanup deletes files without a prior directory
+// sync; the recovered SST must survive because the synced MANIFEST references
+// it.
+TEST_F(DBBasicTest,
+       ReuseManifestOnOpenSyncsRecoverySstDirBeforeManifestAppend) {
+  auto fault_fs = std::make_shared<FaultInjectionTestFS>(env_->GetFileSystem());
+  std::unique_ptr<Env> fault_env(NewCompositeEnv(fault_fs));
+
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.disable_auto_compactions = true;
+  options.env = fault_env.get();
+  options.reuse_manifest_on_open = true;
+
+  DestroyAndReopen(options);
+  ASSERT_OK(Put("base", "value"));
+  ASSERT_OK(Flush());
+
+  ASSERT_OK(Put("recovered", "value"));
+  ASSERT_OK(db_->FlushWAL(true));
+  Close();
+
+  fault_fs->ResetState();
+
+  std::atomic<bool> fail_after_manifest_sync{true};
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "VersionSet::ProcessManifestWrites:AfterSyncManifest", [&](void* arg) {
+        if (fail_after_manifest_sync.exchange(false)) {
+          ASSERT_NE(nullptr, arg);
+          IOStatus* io_s = static_cast<IOStatus*>(arg);
+          ASSERT_OK(*io_s);
+          *io_s = IOStatus::IOError("injected manifest sync aftermath");
+        }
+      });
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  Status s = TryReopen(options);
+  ASSERT_TRUE(s.IsIOError()) << s.ToString();
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
+
+  ASSERT_OK(fault_fs->DeleteFilesCreatedAfterLastDirSync(IOOptions(), nullptr));
+
+  s = TryReopen(options);
+  ASSERT_OK(s);
+  ASSERT_EQ("value", Get("base"));
+  ASSERT_EQ("value", Get("recovered"));
+  Close();
+}
+
 // Direct unit test for WritableFileWriter's initial_file_size parameter:
 // verifies the visible size accessors report the existing on-disk size
 // immediately, rather than the constructor's zero default.

db/db_bloom_filter_test.cc

@@ -2022,7 +2022,11 @@ TEST_F(DBBloomFilterTest, MutableFilterPolicy) {
   double expected_bpk = 10.0;
   // Other configs to try
   std::vector<std::pair<std::string, double>> configs = {
-      {"ribbonfilter:10:-1", 7.0}, {"bloomfilter:5", 5.0}, {"nullptr", 0.0}};
+      {"ribbonfilter:10:-1", 7.0},
+      {"bloomfilter:5", 5.0},
+      {"nullptr", 0.0},
+      // As serialized in OPTIONS file
+      {"{id=ribbonfilter:12:-1;bloom_before_level=-1;}", 8.4}};
 
   table_options.cache_index_and_filter_blocks = true;
   options.table_factory.reset(NewBlockBasedTableFactory(table_options));

db/db_compaction_abort_test.cc

@@ -427,6 +427,38 @@ TEST_F(DBCompactionAbortTest, AbortAutomaticCompaction) {
   }
 }
 
+TEST_F(DBCompactionAbortTest, AbortScheduledAutomaticCompactionBeforePick) {
+  // The goal is to cover an automatic compaction that has been scheduled, but
+  // aborts before the worker picks and removes a column family from the
+  // compaction queue. This pins the scheduler bookkeeping invariant that
+  // ResumeAllCompactions() must be able to schedule that still-queued work.
+  constexpr int kCompactionTrigger = 4;
+  constexpr int kNumL0Files = kCompactionTrigger + 1;
+  Options options = GetOptionsWithStats();
+  options.level0_file_num_compaction_trigger = kCompactionTrigger;
+  options.max_background_compactions = 1;
+  options.max_subcompactions = 1;
+  options.disable_auto_compactions = false;
+  Reopen(options);
+
+  SyncPointAbortHelper helper("BackgroundCallCompaction:0");
+  helper.Setup(dbfull());
+
+  PopulateData(/*num_files=*/kNumL0Files, /*keys_per_file=*/100,
+               /*value_size=*/1000);
+  helper.CleanupAndWait();
+
+  const uint64_t compact_write_bytes_before_resume =
+      stats_->getTickerCount(COMPACT_WRITE_BYTES);
+  dbfull()->ResumeAllCompactions();
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+
+  ASSERT_GT(stats_->getTickerCount(COMPACT_WRITE_BYTES),
+            compact_write_bytes_before_resume);
+  ASSERT_LT(NumTableFilesAtLevel(0), kNumL0Files);
+  VerifyDataIntegrity(/*num_keys=*/100);
+}
+
 TEST_F(DBCompactionAbortTest, AbortAndVerifyNoOutputFiles) {
   Options options = CurrentOptions();
   options.level0_file_num_compaction_trigger = 4;

db/db_etc2_test.cc

@@ -35,7 +35,7 @@ namespace ROCKSDB_NAMESPACE {
 
 class DBTest2 : public DBTestBase {
  public:
-  DBTest2() : DBTestBase("db_test2", /*env_do_fsync=*/true) {}
+  DBTest2() : DBTestBase("db_etc2_test", /*env_do_fsync=*/true) {}
 };
 
 TEST_F(DBTest2, OpenForReadOnly) {

db/db_etc3_test.cc

@@ -3,12 +3,10 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
 
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
 #include "db/db_test_util.h"
+#include "rocksdb/convenience.h"
+#include "rocksdb/metadata.h"
+#include "rocksdb/sst_file_writer.h"
 
 namespace ROCKSDB_NAMESPACE {
 
@@ -48,50 +46,98 @@ TEST_F(DBEtc3Test, AutoTuneManifestSize) {
   ASSERT_EQ(DBOptions{}.max_manifest_space_amp_pct, 500);
 
   Options options = CurrentOptions();
-  ASSERT_OK(db_->SetOptions({{"level0_file_num_compaction_trigger", "20"}}));
+  // Test setup: create many flushed files. Keep level compaction semantics so
+  // DeleteFilesInRanges can remove non-L0 files, but prevent automatic
+  // compactions and write stalls from adding unrelated behavior.
+  options.disable_auto_compactions = true;
+  options.level0_slowdown_writes_trigger = 100;
+  options.level0_stop_writes_trigger = 200;
 
-  // Use large column family names to essentially control the amount of payload
-  // data needed for the manifest file. Drop manifest entries don't include the
-  // CF name so are small.
+  // Test strategy: use large column family names to control the rough amount
+  // of payload added to the MANIFEST. Drop manifest entries do not include the
+  // CF name, so they are small.
+  //
+  // Most CF helper calls piggy-back a background manifest write so the main
+  // auto-tuning phases continue to test the unrelaxed background threshold
+  // even though CF manipulation itself is foreground. Phase-specific foreground
+  // checks disable that piggy-backed background write.
   uint64_t prev_manifest_num = 0, cur_manifest_num = 0;
   std::deque<ColumnFamilyHandle*> handles;
   int counter = 5;
-  auto AddCfFn = [&]() {
+  auto UpdateManifestNumsFrom = [&](uint64_t before_manifest_num) {
+    prev_manifest_num = before_manifest_num;
+    cur_manifest_num = dbfull()->TEST_Current_Manifest_FileNo();
+  };
+  auto BackgroundManifestWriteFn = [&]() {
+    uint64_t before_manifest_num = cur_manifest_num;
+    ASSERT_OK(Put("x", std::to_string(counter++)));
+    ASSERT_OK(Flush());
+    UpdateManifestNumsFrom(before_manifest_num);
+  };
+  auto AddCfFn = [&](bool include_background_manifest_write = true) {
+    uint64_t before_manifest_num = cur_manifest_num;
     std::string name = "cf" + std::to_string(counter++);
     name.resize(1000, 'a');
     ASSERT_OK(db_->CreateColumnFamily(options, name, &handles.emplace_back()));
-    prev_manifest_num = cur_manifest_num;
     cur_manifest_num = dbfull()->TEST_Current_Manifest_FileNo();
+    if (include_background_manifest_write) {
+      BackgroundManifestWriteFn();
+    }
+    UpdateManifestNumsFrom(before_manifest_num);
   };
-  auto DropCfFn = [&]() {
+  auto DropCfFn = [&](bool include_background_manifest_write = true) {
+    uint64_t before_manifest_num = cur_manifest_num;
     ASSERT_OK(db_->DropColumnFamily(handles.front()));
     ASSERT_OK(db_->DestroyColumnFamilyHandle(handles.front()));
     handles.pop_front();
-    prev_manifest_num = cur_manifest_num;
-    cur_manifest_num = dbfull()->TEST_Current_Manifest_FileNo();
-  };
-  auto TrivialManifestWriteFn = [&]() {
-    ASSERT_OK(Put("x", std::to_string(counter++)));
-    ASSERT_OK(Flush());
-    prev_manifest_num = cur_manifest_num;
     cur_manifest_num = dbfull()->TEST_Current_Manifest_FileNo();
+    if (include_background_manifest_write) {
+      BackgroundManifestWriteFn();
+    }
+    UpdateManifestNumsFrom(before_manifest_num);
   };
 
+  // ---- Phase 1: foreground threshold relaxation is bounded ----
+  //
+  // Foreground operations should only get about 25% extra headroom, not an
+  // unbounded threshold. With 1000-byte CF names and a 3000-byte normal limit,
+  // the relaxed limit should allow the first four foreground-only CF additions
+  // but require rotation on the fifth.
+  options.max_manifest_file_size = 3000;
+  options.max_manifest_space_amp_pct = 0;
+  DestroyAndReopen(options);
+  cur_manifest_num = dbfull()->TEST_Current_Manifest_FileNo();
+  prev_manifest_num = cur_manifest_num;
+  for (int i = 1; i <= 4; ++i) {
+    AddCfFn(/*include_background_manifest_write=*/false);
+    ASSERT_EQ(prev_manifest_num, cur_manifest_num);
+  }
+  AddCfFn(/*include_background_manifest_write=*/false);
+  ASSERT_LT(prev_manifest_num, cur_manifest_num);
+  while (!handles.empty()) {
+    ASSERT_OK(db_->DestroyColumnFamilyHandle(handles.front()));
+    handles.pop_front();
+  }
+
+  // ---- Phase 2: no auto-tuning means frequent rotation ----
+  //
   options.max_manifest_file_size = 1000000;
   options.max_manifest_space_amp_pct = 0;  // no auto-tuning yet
   DestroyAndReopen(options);
+  cur_manifest_num = dbfull()->TEST_Current_Manifest_FileNo();
+  prev_manifest_num = cur_manifest_num;
 
-  // With the generous (minimum) maximum manifest size, should not be rotated
+  // With the generous minimum manifest size, should not be rotated.
   AddCfFn();
   AddCfFn();
   AddCfFn();
   ASSERT_EQ(prev_manifest_num, cur_manifest_num);
 
-  // Change options for small max and (still) no auto-tuning
+  // Lower the minimum while still disabling auto-tuning.
   ASSERT_OK(db_->SetDBOptions({{"max_manifest_file_size", "3000"}}));
 
   // Takes effect on the next manifest write
-  TrivialManifestWriteFn();
+  BackgroundManifestWriteFn();
   ASSERT_LT(prev_manifest_num, cur_manifest_num);
 
   // Now we have to rewrite the whole manifest on each write because the
@@ -102,28 +148,31 @@ TEST_F(DBEtc3Test, AutoTuneManifestSize) {
   ASSERT_LT(prev_manifest_num, cur_manifest_num);
   AddCfFn();
   ASSERT_LT(prev_manifest_num, cur_manifest_num);
-  TrivialManifestWriteFn();
+  BackgroundManifestWriteFn();
   ASSERT_LT(prev_manifest_num, cur_manifest_num);
 
+  // ---- Phase 3: auto-tuning raises the background threshold ----
+  //
   // Enabling auto-tuning should fix this, immediately for next manifest writes.
-  // This will allow up to double-ish the size of the compacted manifest,
-  // which last should have been 4000 + some bytes.
+  // This will allow up to roughly double the size of the compacted manifest,
+  // which now includes CF entries plus the piggy-backed background writes.
   ASSERT_EQ(handles.size(), 4U);
-  ASSERT_OK(db_->SetDBOptions({{"max_manifest_space_amp_pct", "105"}}));
+  ASSERT_OK(db_->SetDBOptions({{"max_manifest_space_amp_pct", "95"}}));
 
-  // After 9 CF names should be enough to rotate the manifest
-  for (int i = 1; i <= 5; ++i) {
+  // Auto-tuning lets several more CF names accumulate in the MANIFEST before
+  // the piggy-backed background write crosses the threshold.
+  for (int i = 1; i <= 3; ++i) {
     if ((i % 2) == 1) {
       DropCfFn();
     }
     AddCfFn();
     ASSERT_EQ(prev_manifest_num, cur_manifest_num);
   }
-  TrivialManifestWriteFn();
+  AddCfFn();
   ASSERT_LT(prev_manifest_num, cur_manifest_num);
 
-  // We now have a different last compacted manifest size, should be
-  // able to go beyond 9 CFs named in manifest this time.
+  // We now have a different last compacted manifest size, so the next
+  // threshold should be based on the newly compacted MANIFEST.
   ASSERT_EQ(handles.size(), 6U);
 
   DropCfFn();
@@ -133,11 +182,10 @@ TEST_F(DBEtc3Test, AutoTuneManifestSize) {
     AddCfFn();
     ASSERT_EQ(prev_manifest_num, cur_manifest_num);
   }
-  // We've written 10 named CFs to the manifest. We should be able to
-  // dynamically change the auto-tuning still based on the last "compacted"
-  // manifest size of 7000 + some bytes.
+  // We should be able to dynamically change the auto-tuning still based on
+  // the last "compacted" manifest size.
   ASSERT_OK(db_->SetDBOptions({{"max_manifest_space_amp_pct", "51"}}));
-  TrivialManifestWriteFn();
+  BackgroundManifestWriteFn();
   ASSERT_LT(prev_manifest_num, cur_manifest_num);
   // And the "compacted" manifest size has reset again, so should be changed
   // again sooner.
@@ -146,16 +194,129 @@ TEST_F(DBEtc3Test, AutoTuneManifestSize) {
     AddCfFn();
     ASSERT_EQ(prev_manifest_num, cur_manifest_num);
   }
-  // Enough for manifest change
-  AddCfFn();
+
+  // ---- Phase 4: foreground operations use relaxed threshold ----
+  //
+  // The current MANIFEST is now large enough for the next background manifest
+  // write to rotate it, but still small enough for foreground operations to use
+  // their 25% extra headroom. Assert that each foreground operation stays on
+  // the same MANIFEST, then verify the next background write rotates.
+  const std::string external_files_dir = dbname_ + "/external_files";
+  ASSERT_OK(env_->CreateDirIfMissing(external_files_dir));
+  auto WriteExternalSstFile = [&](const std::string& file_name,
+                                  const std::string& key,
+                                  const std::string& value) {
+    const std::string file_path = external_files_dir + "/" + file_name;
+    Status ignored = env_->DeleteFile(file_path);
+    ignored.PermitUncheckedError();
+
+    SstFileWriter sst_file_writer(EnvOptions(), options);
+    ASSERT_OK(sst_file_writer.Open(file_path));
+    ASSERT_OK(sst_file_writer.Put(key, value));
+    ASSERT_OK(sst_file_writer.Finish());
+  };
+  auto IngestExternalFileForegroundManifestWriteFn =
+      [&](std::string* ingested_key) {
+        uint64_t before_manifest_num = cur_manifest_num;
+        const std::string key = "z" + std::to_string(counter++);
+        const std::string value = "v" + std::to_string(counter++);
+        const std::string file_name =
+            "ingest_" + std::to_string(counter++) + ".sst";
+        const std::string file_path = external_files_dir + "/" + file_name;
+        WriteExternalSstFile(file_name, key, value);
+
+        ASSERT_OK(
+            db_->IngestExternalFile({file_path}, IngestExternalFileOptions()));
+        ASSERT_EQ(value, Get(key));
+        *ingested_key = key;
+        Status ignored = env_->DeleteFile(file_path);
+        ignored.PermitUncheckedError();
+        UpdateManifestNumsFrom(before_manifest_num);
+      };
+  auto CreateColumnFamilyWithImportForegroundManifestWriteFn = [&]() {
+    uint64_t before_manifest_num = cur_manifest_num;
+    const std::string cf_name = "import_cf" + std::to_string(counter++);
+    const std::string key = "import" + std::to_string(counter++);
+    const std::string value = "v" + std::to_string(counter++);
+    const std::string file_name =
+        "import_" + std::to_string(counter++) + ".sst";
+    const std::string file_path = external_files_dir + "/" + file_name;
+    WriteExternalSstFile(file_name, key, value);
+
+    LiveFileMetaData file_metadata;
+    file_metadata.name = file_name;
+    file_metadata.db_path = external_files_dir;
+    file_metadata.smallest_seqno = 0;
+    file_metadata.largest_seqno = 0;
+    file_metadata.level = 0;
+    ExportImportFilesMetaData metadata;
+    metadata.files.push_back(file_metadata);
+    metadata.db_comparator_name = options.comparator->Name();
+
+    ColumnFamilyHandle* import_handle = nullptr;
+    ASSERT_OK(db_->CreateColumnFamilyWithImport(options, cf_name,
+                                                ImportColumnFamilyOptions(),
+                                                metadata, &import_handle));
+    ASSERT_NE(import_handle, nullptr);
+    handles.push_back(import_handle);
+
+    std::string result;
+    ASSERT_OK(db_->Get(ReadOptions(), import_handle, key, &result));
+    ASSERT_EQ(value, result);
+    Status ignored = env_->DeleteFile(file_path);
+    ignored.PermitUncheckedError();
+    UpdateManifestNumsFrom(before_manifest_num);
+  };
+  auto DeleteFilesInRangesForegroundManifestWriteFn =
+      [&](const std::string& key) {
+        uint64_t before_manifest_num = cur_manifest_num;
+        const std::string limit = key + "\xff";
+        std::vector<RangeOpt> ranges;
+        ranges.emplace_back(key, limit);
+        ASSERT_OK(DeleteFilesInRanges(db_.get(), db_->DefaultColumnFamily(),
+                                      ranges.data(), ranges.size(),
+                                      /*include_end=*/false));
+
+        std::string result;
+        ASSERT_TRUE(db_->Get(ReadOptions(), key, &result).IsNotFound());
+        UpdateManifestNumsFrom(before_manifest_num);
+      };
+
+  // Column family manipulation.
+  AddCfFn(/*include_background_manifest_write=*/false);
+  ASSERT_EQ(prev_manifest_num, cur_manifest_num);
+
+  // SetOptions should not write to the MANIFEST. If that regresses, the write
+  // should be treated as a background write and rotate here.
+  {
+    ASSERT_OK(db_->SetOptions({{"level0_slowdown_writes_trigger", "101"}}));
+    UpdateManifestNumsFrom(cur_manifest_num);
+  }
+  ASSERT_EQ(prev_manifest_num, cur_manifest_num);
+
+  // External file ingestion.
+  std::string ingested_key;
+  IngestExternalFileForegroundManifestWriteFn(&ingested_key);
+  ASSERT_EQ(prev_manifest_num, cur_manifest_num);
+
+  // Imported column family creation.
+  CreateColumnFamilyWithImportForegroundManifestWriteFn();
+  ASSERT_EQ(prev_manifest_num, cur_manifest_num);
+
+  // Physical file deletion by range.
+  DeleteFilesInRangesForegroundManifestWriteFn(ingested_key);
+  ASSERT_EQ(prev_manifest_num, cur_manifest_num);
+
+  // Background flush.
+  BackgroundManifestWriteFn();
   ASSERT_LT(prev_manifest_num, cur_manifest_num);
 
-  // ---- Verify persisted compacted manifest size survives close/reopen ----
+  // ---- Phase 5: persisted compacted manifest size survives close/reopen ----
   // Close with CFs still live. Reopen with reuse_manifest_on_open so the
   // manifest is NOT rewritten from scratch. The persisted compacted size
   // should be loaded and used for auto-tuning.
-  // At this point we have 7 CF handles plus default.
-  ASSERT_EQ(handles.size(), 7U);
+  // At this point we have 8 CF handles plus default.
+  ASSERT_EQ(handles.size(), 8U);
 
   // Collect CF names for reopen, then release handles (Close needs this)
   std::vector<std::string> cf_names = {"default"};
@@ -168,18 +329,16 @@ TEST_F(DBEtc3Test, AutoTuneManifestSize) {
   handles.clear();
 
   Close();
-  // Use a large max_manifest_file_size so the reused manifest (which is
-  // already ~10KB) does NOT trigger rotation on the first few writes.
-  // Auto-tuning with the persisted compacted size (~5KB) at 200% amp
-  // gives a tuned threshold of ~15KB. Without persistence, the threshold
-  // would be max(max_manifest_file_size, 0 * anything) =
-  // max_manifest_file_size.
+  // Use a max_manifest_file_size below the reused manifest size. Auto-tuning
+  // with the persisted compacted size at 200% amp keeps the tuned threshold
+  // high enough. Without persistence, the threshold would be
+  // max(max_manifest_file_size, 0 * anything) = max_manifest_file_size.
   //
-  // We set max_manifest_file_size to two values to distinguish:
-  // - 3000: if persisted compacted size is NOT loaded, tuned = 3000,
-  //   and the first AddCf will rotate (manifest is already ~10KB > 3000)
-  // - With persisted compacted size loaded, tuned = max(3000, 5000*3) = 15000,
-  //   so no rotation until we exceed 15KB
+  // With max_manifest_file_size set to 3000, missing persisted compacted size
+  // would keep tuned = 3000 and the first AddCf would rotate because the
+  // reused manifest is already larger. With persisted compacted size loaded,
+  // the tuned threshold is based on the compacted size, so no rotation until
+  // the manifest grows well beyond the minimum.
   options.max_manifest_file_size = 3000;
   options.max_manifest_space_amp_pct = 200;
   options.reuse_manifest_on_open = true;
@@ -191,11 +350,11 @@ TEST_F(DBEtc3Test, AutoTuneManifestSize) {
   // during reopen, because the persisted compacted size keeps the tuned
   // threshold high enough. Without persistence, last_compacted = 0, so
   // tuned = max_manifest_file_size = 3000, and the first LogAndApply
-  // during Open rotates the manifest because it's already ~10KB > 3000.
+  // during Open rotates the manifest because it is already larger than 3000.
   ASSERT_EQ(manifest_num_before_reopen, cur_manifest_num);
 
-  // Adding CFs should still not trigger rotation because the tuned
-  // threshold (~15KB) exceeds the current manifest size (~10KB + adds).
+  // Adding CFs should still not trigger rotation because the tuned threshold
+  // from the persisted compacted size exceeds the current manifest size.
   for (int i = 1; i <= 4; ++i) {
     AddCfFn();
     ASSERT_EQ(prev_manifest_num, cur_manifest_num);
@@ -207,7 +366,7 @@ TEST_F(DBEtc3Test, AutoTuneManifestSize) {
 
   // Wrap up
   while (!handles.empty()) {
-    DropCfFn();
+    DropCfFn(/*include_background_manifest_write=*/false);
   }
 }
 

db/db_impl/db_impl.cc

@@ -496,6 +496,21 @@ void DBImpl::WaitForAsyncFileOpen() {
   }
 }
 
+void DBImpl::NotifyOnDBShutdownBegin() {
+  mutex_.AssertHeld();
+  if (shutdown_notification_sent_ || immutable_db_options_.listeners.empty()) {
+    return;
+  }
+  shutdown_notification_sent_ = true;
+
+  // release lock while notifying events
+  mutex_.Unlock();
+  for (const auto& listener : immutable_db_options_.listeners) {
+    listener->OnDBShutdownBegin(this);
+  }
+  mutex_.Lock();
+}
+
 // Will lock the mutex_,  will wait for completion if wait is true
 void DBImpl::CancelAllBackgroundWork(bool wait) {
   ROCKS_LOG_INFO(immutable_db_options_.info_log,
@@ -542,6 +557,9 @@ void DBImpl::CancelAllBackgroundWork(bool wait) {
     immutable_db_options_.compaction_service->CancelAwaitingJobs();
   }
 
+  if (!already_shutting_down) {
+    NotifyOnDBShutdownBegin();
+  }
   shutting_down_.store(true, std::memory_order_release);
   bg_cv_.SignalAll();
   if (!wait) {
@@ -5837,6 +5855,7 @@ Status DBImpl::DeleteFilesInRanges(ColumnFamilyHandle* column_family,
   }
 
   VersionEdit edit;
+  edit.MarkForegroundOperation();
   std::set<FileMetaData*> deleted_files;
   JobContext job_context(next_job_id_.fetch_add(1), true);
   {
@@ -6947,7 +6966,9 @@ Status DBImpl::IngestExternalFiles(
         assert(!cfd->IsDropped());
         cfds_to_commit.push_back(cfd);
         autovector<VersionEdit*> edit_list;
-        edit_list.push_back(ingestion_jobs[i].edit());
+        auto* edit = ingestion_jobs[i].edit();
+        edit->MarkForegroundOperation();
+        edit_list.push_back(edit);
         edit_lists.push_back(edit_list);
         ++num_entries;
       }
@@ -6962,7 +6983,6 @@ Status DBImpl::IngestExternalFiles(
       }
       status =
           versions_->LogAndApply(cfds_to_commit, read_options, write_options,
-
                                  edit_lists, &mutex_, directories_.GetDbDir());
       // It is safe to update VersionSet last seqno here after LogAndApply since
       // LogAndApply persists last sequence number from VersionEdits,
@@ -7095,6 +7115,7 @@ Status DBImpl::CreateColumnFamilyWithImport(
 
   SuperVersionContext dummy_sv_ctx(/* create_superversion */ true);
   VersionEdit dummy_edit;
+  dummy_edit.MarkForegroundOperation();
   uint64_t next_file_number = 0;
   std::unique_ptr<std::list<uint64_t>::iterator> pending_output_elem;
   {
@@ -7152,9 +7173,10 @@ Status DBImpl::CreateColumnFamilyWithImport(
 
       // Install job edit [Mutex will be unlocked here]
       if (status.ok()) {
-        status = versions_->LogAndApply(cfd, read_options, write_options,
-                                        import_job.edit(), &mutex_,
-                                        directories_.GetDbDir());
+        auto* edit = import_job.edit();
+        edit->MarkForegroundOperation();
+        status = versions_->LogAndApply(cfd, read_options, write_options, edit,
+                                        &mutex_, directories_.GetDbDir());
         if (status.ok()) {
           InstallSuperVersionForConfigChange(cfd, &sv_context);
         }
@@ -7585,6 +7607,7 @@ Status DBImpl::ReserveFileNumbersBeforeIngestion(
   // reuse the file number that has already assigned to the internal file,
   // and this will overwrite the external file. To protect the external
   // file, we have to make sure the file number will never being reused.
+  dummy_edit.MarkForegroundOperation();
   s = versions_->LogAndApply(cfd, read_options, write_options, &dummy_edit,
                              &mutex_, directories_.GetDbDir());
   if (s.ok()) {

db/db_impl/db_impl.h

@@ -1456,6 +1456,13 @@ class DBImpl : public DB {
   std::atomic<int> next_job_id_ = 1;
 
   std::atomic<bool> shutting_down_ = false;
+  // Protected by mutex_. This is separate from shutting_down_ because
+  // OnDBShutdownBegin must fire before shutting_down_ is published, and the
+  // notification releases mutex_ while invoking listeners. Marking that the
+  // notification has started prevents a concurrent or reentrant
+  // CancelAllBackgroundWork() call from firing it again during that unlocked
+  // callback window.
+  bool shutdown_notification_sent_ = false;
 
   // No new background jobs can be queued if true. This is used to prevent new
   // background jobs from being queued after WaitForCompact() completes waiting
@@ -1539,6 +1546,7 @@ class DBImpl : public DB {
                                    const Status& st,
                                    const CompactionJobStats& job_stats,
                                    int job_id);
+  void NotifyOnDBShutdownBegin();
   void NotifyOnMemTableSealed(ColumnFamilyData* cfd,
                               const MemTableInfo& mem_table_info);
 

db/db_impl/db_impl_compaction_flush.cc

@@ -4067,6 +4067,13 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
       status = Status::ShutdownInProgress();
     } else if (compaction_aborted_.load(std::memory_order_acquire) > 0) {
       status = Status::Incomplete(Status::SubCode::kCompactionAborted);
+      if (!is_prepicked) {
+        // This automatic compaction was scheduled from compaction_queue_, but
+        // abort happened before PickCompactionFromQueue() could remove the
+        // queued CF. Restore the unscheduled count so ResumeAllCompactions()
+        // can schedule the still-queued work.
+        unscheduled_compactions_++;
+      }
     } else if (is_manual &&
                manual_compaction->canceled.load(std::memory_order_acquire)) {
       status = Status::Incomplete(Status::SubCode::kManualCompactionPaused);
@@ -4074,10 +4081,14 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
   } else {
     status = error_handler_.GetBGError();
     // If we get here, it means a hard error happened after this compaction
-    // was scheduled by MaybeScheduleFlushOrCompaction(), but before it got
-    // a chance to execute. Since we didn't pop a cfd from the compaction
-    // queue, increment unscheduled_compactions_
-    unscheduled_compactions_++;
+    // was scheduled by MaybeScheduleFlushOrCompaction(), but before it got a
+    // chance to execute. Since non-prepicked work consumed an unscheduled
+    // compaction credit without popping a cfd from the compaction queue,
+    // restore the credit here. Prepicked work was scheduled directly and did
+    // not consume unscheduled_compactions_.
+    if (!is_prepicked) {
+      unscheduled_compactions_++;
+    }
   }
 
   if (!status.ok()) {

db/db_impl/db_impl_open.cc

@@ -2234,6 +2234,16 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
 
   constexpr int level = 0;
 
+  if (s.ok() && has_output) {
+    // Before publishing any recovery edit, make the new SST's directory entry
+    // durable. Reused MANIFESTs have no CURRENT update, and fresh MANIFEST
+    // publication should not be relied on to order the recovered SST entry.
+    s = GetDataDir(cfd, meta.fd.GetPathId())
+            ->FsyncWithDirOptions(
+                IOOptions(), nullptr,
+                DirFsyncOptions(DirFsyncOptions::FsyncReason::kNewFileSynced));
+  }
+
   if (s.ok() && has_output) {
     edit->AddFile(level, meta.fd.GetNumber(), meta.fd.GetPathId(),
                   meta.fd.GetFileSize(), meta.smallest, meta.largest,

db/db_impl/db_impl_secondary.cc

@@ -105,10 +105,22 @@ Status DBImplSecondary::FindNewLogNumbers(std::vector<uint64_t>* logs) {
     return s;
   }
 
-  // if log_readers_ is non-empty, it means we have applied all logs with log
-  // numbers smaller than the smallest log in log_readers_, so there is no
-  // need to pass these logs to RecoverLogFiles
-  uint64_t log_number_min = 0;
+  // Drop readers only after MANIFEST replay says the WAL is obsolete. The
+  // primary can create a higher-number empty WAL before it stops appending to a
+  // lower-number current WAL, so seeing a higher-number WAL is not sufficient.
+  const uint64_t min_log_number_to_keep = versions_->min_log_number_to_keep();
+  for (auto iter = log_readers_.begin(); iter != log_readers_.end();) {
+    if (iter->first < min_log_number_to_keep) {
+      iter = log_readers_.erase(iter);
+    } else {
+      break;
+    }
+  }
+
+  // If log_readers_ is non-empty, it means we have applied all logs with log
+  // numbers smaller than the smallest retained log reader. Otherwise, manifest
+  // recovery tells us which older logs can be skipped.
+  uint64_t log_number_min = min_log_number_to_keep;
   if (!log_readers_.empty()) {
     log_number_min = log_readers_.begin()->first;
   }
@@ -321,12 +333,6 @@ Status DBImplSecondary::RecoverLogFiles(
       return status;
     }
   }
-  // remove logreaders from map after successfully recovering the WAL
-  if (log_readers_.size() > 1) {
-    auto erase_iter = log_readers_.begin();
-    std::advance(erase_iter, log_readers_.size() - 1);
-    log_readers_.erase(log_readers_.begin(), erase_iter);
-  }
   return status;
 }
 

db/db_iter.cc

@@ -1480,7 +1480,9 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
     if (timestamp_lb_ != nullptr) {
       saved_key_.SetInternalKey(ikey);
     } else {
-      saved_key_.SetUserKey(ikey.user_key);
+      saved_key_.SetUserKey(
+          ikey.user_key,
+          !pin_thru_lifetime_ || !iter_.iter()->IsKeyPinned() /* copy */);
     }
 
     valid_ = true;

db/db_iter_test.cc

@@ -216,12 +216,19 @@ class TestIterator : public InternalIterator {
   bool IsKeyPinned() const override { return true; }
   bool IsValuePinned() const override { return true; }
 
+  void Prepare(const MultiScanArgs* /*scan_opts*/) override {
+    ++prepare_call_count_;
+  }
+
+  size_t prepare_call_count() const { return prepare_call_count_; }
+
  private:
   bool initialized_;
   bool valid_;
   size_t sequence_number_;
   size_t iter_;
   size_t steps_ = 0;
+  size_t prepare_call_count_ = 0;
 
   InternalKeyComparator cmp;
   std::vector<std::pair<std::string, std::string>> data_;
@@ -243,6 +250,53 @@ class DBIteratorTest : public testing::Test {
   DBIteratorTest() : env_(Env::Default()) {}
 };
 
+TEST_F(DBIteratorTest, PrepareForwardsValidatedScanRanges) {
+  Options options;
+  ImmutableOptions ioptions = ImmutableOptions(options);
+  MutableCFOptions mutable_cf_options = MutableCFOptions(options);
+  TestIterator* internal_iter = new TestIterator(BytewiseComparator());
+  internal_iter->AddPut("a", "val_a");
+  internal_iter->AddPut("b", "val_b");
+  internal_iter->Finish();
+
+  ReadOptions ro;
+  std::unique_ptr<Iterator> db_iter(DBIter::NewIter(
+      env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
+      internal_iter, nullptr /* version */, 10 /* sequence */,
+      nullptr /* read_callback */, /*active_mem=*/nullptr));
+
+  MultiScanArgs scan_opts(BytewiseComparator());
+  scan_opts.insert("a", "c");
+
+  db_iter->Prepare(scan_opts);
+  ASSERT_OK(db_iter->status());
+  EXPECT_EQ(internal_iter->prepare_call_count(), 1);
+}
+
+TEST_F(DBIteratorTest, PrepareDoesNotForwardInvalidScanRanges) {
+  Options options;
+  ImmutableOptions ioptions = ImmutableOptions(options);
+  MutableCFOptions mutable_cf_options = MutableCFOptions(options);
+  TestIterator* internal_iter = new TestIterator(BytewiseComparator());
+  internal_iter->AddPut("a", "val_a");
+  internal_iter->AddPut("b", "val_b");
+  internal_iter->Finish();
+
+  ReadOptions ro;
+  std::unique_ptr<Iterator> db_iter(DBIter::NewIter(
+      env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
+      internal_iter, nullptr /* version */, 10 /* sequence */,
+      nullptr /* read_callback */, /*active_mem=*/nullptr));
+
+  MultiScanArgs scan_opts(BytewiseComparator());
+  scan_opts.insert("b", "d");
+  scan_opts.insert("a", "c");
+
+  db_iter->Prepare(scan_opts);
+  EXPECT_TRUE(db_iter->status().IsInvalidArgument());
+  EXPECT_EQ(internal_iter->prepare_call_count(), 0);
+}
+
 TEST_F(DBIteratorTest, DBIteratorPrevNext) {
   Options options;
   ImmutableOptions ioptions = ImmutableOptions(options);

db/db_iterator_test.cc

@@ -4395,6 +4395,71 @@ TEST_P(DBMultiScanIteratorTest, RangeAcrossFiles) {
   iter.reset();
 }
 
+TEST_P(DBMultiScanIteratorTest, SortedRangesSkipIODispatcherSort) {
+  auto options = CurrentOptions();
+  options.target_file_size_base = 100 << 10;
+  options.compaction_style = kCompactionStyleUniversal;
+  options.num_levels = 50;
+  options.compression = kNoCompression;
+  DestroyAndReopen(options);
+
+  auto key = [](int i) {
+    std::stringstream ss;
+    ss << "k" << std::setw(5) << std::setfill('0') << i;
+    return ss.str();
+  };
+
+  auto rnd = Random::GetTLSInstance();
+  for (int i = 0; i < 100; ++i) {
+    ASSERT_OK(Put(key(i), rnd->RandomString(2 << 10)));
+  }
+  ASSERT_OK(Flush());
+
+  ASSERT_OK(db_->CompactRange({}, nullptr, nullptr));
+  ASSERT_EQ(2, NumTableFilesAtLevel(49));
+
+  int sort_count = 0;
+  SyncPoint::GetInstance()->SetCallBack(
+      "IODispatcherImpl::SubmitJob:SortBlockHandles",
+      [&](void* /*arg*/) { ++sort_count; });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  auto tracking_dispatcher = std::make_shared<TrackingIODispatcher>();
+  std::vector<std::string> key_ranges({key(10), key(90)});
+  MultiScanArgs scan_options(BytewiseComparator());
+  scan_options.io_dispatcher = tracking_dispatcher;
+  scan_options.use_async_io = false;
+  scan_options.insert(key_ranges[0], key_ranges[1]);
+
+  ReadOptions ro;
+  ro.fill_cache = GetParam();
+  ColumnFamilyHandle* cfh = dbfull()->DefaultColumnFamily();
+  std::unique_ptr<MultiScan> iter =
+      dbfull()->NewMultiScan(ro, cfh, scan_options);
+
+  try {
+    int i = 10;
+    for (auto range : *iter) {
+      for (auto it : range) {
+        ASSERT_EQ(it.first.ToString(), key(i));
+        ++i;
+      }
+    }
+    ASSERT_EQ(i, 90);
+  } catch (MultiScanException& ex) {
+    ASSERT_NOK(ex.status());
+    std::cerr << "Iterator returned status " << ex.what();
+    abort();
+  } catch (std::logic_error& ex) {
+    std::cerr << "Iterator returned logic error " << ex.what();
+    abort();
+  }
+
+  ASSERT_GT(tracking_dispatcher->GetReadSets().size(), 0);
+  EXPECT_EQ(sort_count, 0);
+  iter.reset();
+}
+
 TEST_P(DBMultiScanIteratorTest, FailureTest) {
   auto options = CurrentOptions();
   options.compression = kNoCompression;
@@ -7181,6 +7246,52 @@ TEST_P(ReadPathRangeTombstoneTest,
   db_->ReleaseSnapshot(snap);
 }
 
+// SeekForPrev uses the reverse path: PrevInternal -> FindValueForCurrentKey.
+// When a key has more versions than max_sequential_skip_in_iterations,
+// FindValueForCurrentKey delegates to FindValueForCurrentKeyUsingSeek which
+// re-seeks forward to the newest visible version. That function must preserve
+// key pinning by passing the correct `copy` parameter to
+// saved_key_.SetUserKey().
+TEST_P(DBIteratorTest, SeekForPrevKeyPinnedWithManyVersions) {
+  Options options = CurrentOptions();
+  options.max_sequential_skip_in_iterations = 8;
+  options.statistics = CreateDBStatistics();
+  DestroyAndReopen(options);
+
+  // 20 versions > max_sequential_skip (8), forcing the reseek path.
+  // No flush: keeps all versions alive in the memtable.
+  for (int i = 0; i < 20; i++) {
+    ASSERT_OK(Put("key1", "value" + std::to_string(i)));
+  }
+
+  ReadOptions ro;
+  ro.pin_data = true;
+
+  std::unique_ptr<Iterator> iter(NewIterator(ro));
+  std::string prop_value;
+
+  // Seek (forward) correctly reports key as pinned.
+  iter->Seek("key1");
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_OK(iter->GetProperty("rocksdb.iterator.is-key-pinned", &prop_value));
+  ASSERT_EQ("1", prop_value);
+
+  uint64_t reseeks_before =
+      options.statistics->getTickerCount(NUMBER_OF_RESEEKS_IN_ITERATION);
+
+  iter->SeekForPrev("key1");
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_EQ("key1", iter->key().ToString());
+
+  // Confirm FindValueForCurrentKeyUsingSeek was actually taken.
+  uint64_t reseeks_after =
+      options.statistics->getTickerCount(NUMBER_OF_RESEEKS_IN_ITERATION);
+  ASSERT_GT(reseeks_after, reseeks_before);
+
+  ASSERT_OK(iter->GetProperty("rocksdb.iterator.is-key-pinned", &prop_value));
+  ASSERT_EQ("1", prop_value);
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

db/db_secondary_test.cc

@@ -13,6 +13,7 @@
 #include "db/db_with_timestamp_test_util.h"
 #include "db/wide/wide_column_test_util.h"
 #include "db/write_batch_internal.h"
+#include "file/filename.h"
 #include "port/stack_trace.h"
 #include "rocksdb/utilities/transaction_db.h"
 #include "test_util/sync_point.h"
@@ -922,6 +923,48 @@ TEST_F(DBSecondaryTest, OpenAsSecondaryWALTailing) {
   verify_db_func("new_foo_value_1", "new_bar_value");
 }
 
+TEST_F(DBSecondaryTest, CatchUpTailsCurrentWalWhenFutureWalExists) {
+  // Goal: secondary catch-up must keep tailing the current WAL even when a
+  // higher-number empty WAL is present. This reproduces the async WAL
+  // precreation shape by creating a future WAL before opening the secondary,
+  // then appending another write to the current WAL and catching up again.
+  Options options;
+  options.env = env_;
+  Reopen(options);
+
+  ASSERT_OK(Put("foo", "value0"));
+  ASSERT_OK(Flush());
+  ASSERT_OK(Put("foo", "value1"));
+  ASSERT_OK(db_->FlushWAL(/*sync=*/true));
+
+  std::unique_ptr<WalFile> current_wal;
+  ASSERT_OK(db_->GetCurrentWalFile(&current_wal));
+  ASSERT_NE(nullptr, current_wal);
+
+  const uint64_t future_wal_number = current_wal->LogNumber() + 1000;
+  std::unique_ptr<WritableFile> future_wal_file;
+  ASSERT_OK(env_->NewWritableFile(LogFileName(dbname_, future_wal_number),
+                                  &future_wal_file, EnvOptions()));
+  ASSERT_OK(future_wal_file->Close());
+
+  Options secondary_options;
+  secondary_options.env = env_;
+  secondary_options.max_open_files = -1;
+  OpenSecondary(secondary_options);
+
+  ReadOptions read_options;
+  std::string value;
+  ASSERT_OK(db_secondary_->Get(read_options, "foo", &value));
+  ASSERT_EQ("value1", value);
+
+  ASSERT_OK(Put("foo", "value2"));
+  ASSERT_OK(db_->FlushWAL(/*sync=*/true));
+
+  ASSERT_OK(db_secondary_->TryCatchUpWithPrimary());
+  ASSERT_OK(db_secondary_->Get(read_options, "foo", &value));
+  ASSERT_EQ("value2", value);
+}
+
 TEST_F(DBSecondaryTest, SecondaryTailingBug_ISSUE_8467) {
   Options options;
   options.env = env_;

db/db_test_util.cc

@@ -13,6 +13,7 @@
 #include "db/forward_iterator.h"
 #include "env/fs_readonly.h"
 #include "env/mock_env.h"
+#include "file/file_util.h"
 #include "port/lang.h"
 #include "rocksdb/cache.h"
 #include "rocksdb/convenience.h"
@@ -90,6 +91,8 @@ DBTestBase::DBTestBase(const std::string path, bool env_do_fsync)
   dbname_ = test::PerThreadDBPath(env_, path);
   alternative_wal_dir_ = dbname_ + "/wal";
   alternative_db_log_dir_ = dbname_ + "/db_log_dir";
+  EXPECT_OK(DestroyDir(env_, alternative_wal_dir_));
+  EXPECT_OK(DestroyDir(env_, alternative_db_log_dir_));
   auto options = CurrentOptions();
   options.env = env_;
   auto delete_options = options;
@@ -117,6 +120,8 @@ DBTestBase::~DBTestBase() {
   if (getenv("KEEP_DB")) {
     printf("DB is still at %s\n", dbname_.c_str());
   } else {
+    EXPECT_OK(DestroyDir(env_, alternative_wal_dir_));
+    EXPECT_OK(DestroyDir(env_, alternative_db_log_dir_));
     EXPECT_OK(DestroyDB(dbname_, options));
   }
   // Ensure SstFileManager (and its DeleteScheduler background thread) is

db/listener_test.cc

@@ -31,6 +31,7 @@
 #include "util/mutexlock.h"
 #include "util/rate_limiter_impl.h"
 #include "util/string_util.h"
+#include "utilities/fault_injection_fs.h"
 #include "utilities/merge_operators.h"
 
 namespace ROCKSDB_NAMESPACE {
@@ -316,6 +317,79 @@ TEST_F(EventListenerTest, OnCompactionPreCommitOrdering) {
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
 }
 
+class TestDBShutdownBeginListener : public EventListener {
+ public:
+  void OnDBShutdownBegin(DB* db) override {
+    std::lock_guard<std::mutex> lock(mutex_);
+    ++shutdown_count_;
+    last_db_ = db;
+  }
+
+  int ShutdownCount() const {
+    std::lock_guard<std::mutex> lock(mutex_);
+    return shutdown_count_;
+  }
+
+  DB* LastDB() const {
+    std::lock_guard<std::mutex> lock(mutex_);
+    return last_db_;
+  }
+
+ private:
+  mutable std::mutex mutex_;
+  int shutdown_count_ = 0;
+  DB* last_db_ = nullptr;
+};
+
+TEST_F(EventListenerTest, OnDBShutdownBeginOnceForCancelAndClose) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  auto listener = std::make_shared<TestDBShutdownBeginListener>();
+  options.listeners.emplace_back(listener);
+  DestroyAndReopen(options);
+
+  DB* db = db_.get();
+  dbfull()->CancelAllBackgroundWork(false);
+  EXPECT_EQ(1, listener->ShutdownCount());
+  EXPECT_EQ(db, listener->LastDB());
+
+  Close();
+  EXPECT_EQ(1, listener->ShutdownCount());
+}
+
+TEST_F(EventListenerTest, OnDBShutdownBeginOnFailedOpen) {
+  Close();
+
+  std::shared_ptr<FaultInjectionTestFS> fs(
+      new FaultInjectionTestFS(env_->GetFileSystem()));
+  std::unique_ptr<Env> env(NewCompositeEnv(fs));
+
+  Options options = CurrentOptions();
+  options.env = env.get();
+  options.create_if_missing = true;
+  auto listener = std::make_shared<TestDBShutdownBeginListener>();
+  options.listeners.emplace_back(listener);
+
+  SyncPoint::GetInstance()->SetCallBack(
+      "PersistRocksDBOptions:create",
+      [&](void* /*arg*/) { fs->SetFilesystemActive(false); });
+  SyncPoint::GetInstance()->SetCallBack(
+      "PersistRocksDBOptions:written",
+      [&](void* /*arg*/) { fs->SetFilesystemActive(true); });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  std::unique_ptr<DB> db;
+  Status s = DB::Open(options, dbname_, &db);
+  ASSERT_TRUE(s.IsIOError()) << s.ToString();
+  EXPECT_EQ(1, listener->ShutdownCount());
+  EXPECT_NE(nullptr, listener->LastDB());
+  EXPECT_EQ(nullptr, db.get());
+
+  fs->SetFilesystemActive(true);
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+}
+
 // This simple Listener can only handle one flush at a time.
 class TestFlushListener : public EventListener {
  public:
@@ -1869,7 +1943,7 @@ TEST_F(EventListenerTest, BackgroundJobPressure) {
   listener->Reset();
   sleeping_task.WakeUp();
   sleeping_task.WaitUntilDone();
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+  ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork());
 
   snapshots = listener->GetSnapshots();
   CheckInvariants(snapshots);

db/version_edit.h

@@ -978,6 +978,11 @@ class VersionEdit {
 
   bool IsColumnFamilyDrop() const { return is_column_family_drop_; }
 
+  void MarkForegroundOperation() { is_foreground_operation_ = true; }
+  bool IsForegroundOperation() const {
+    return is_foreground_operation_ || IsColumnFamilyManipulation();
+  }
+
   void MarkNoManifestWriteDummy() { is_no_manifest_write_dummy_ = true; }
   bool IsNoManifestWriteDummy() const { return is_no_manifest_write_dummy_; }
 
@@ -1111,6 +1116,7 @@ class VersionEdit {
   // it also includes column family name.
   bool is_column_family_drop_ = false;
   bool is_column_family_add_ = false;
+  bool is_foreground_operation_ = false;
   std::string column_family_name_;
 
   uint32_t remaining_entries_ = 0;

db/version_set.cc

@@ -6111,9 +6111,32 @@ Status VersionSet::ProcessManifestWrites(
 
   uint64_t prev_manifest_file_size = manifest_file_size_;
   assert(pending_manifest_file_number_ == 0);
+  bool has_foreground_operation = false;
+  for (const VersionEdit* e : batch_edits) {
+    if (e->IsForegroundOperation()) {
+      has_foreground_operation = true;
+      break;
+    }
+  }
+  // For MANIFEST write batches with any foreground operation (external file
+  // ingestion/import, DeleteFilesInRange(s), and column family manipulations
+  // like CreateColumnFamily and DropColumnFamily), relax the size limit by 25%
+  // to reduce the likelihood of a user operation blocking on MANIFEST rotation.
+  // Background-only batches (flush/compaction) still rotate at the normal
+  // threshold.
+  // TODO/future: for workloads like atomic-replace ingestion-only, with zero
+  // or few flushes and compactions, it might be nice to trigger background
+  // manifest rotation if we are beyond the soft limit. But the vast majority
+  // of workloads should have plenty of background manifest ops to avoid
+  // foreground rotation.
+  uint64_t enforced_limit = tuned_max_manifest_file_size_;
+  if (has_foreground_operation) {
+    uint64_t new_limit = enforced_limit + enforced_limit / 4;
+    // don't keep in case of overflow
+    enforced_limit = std::max(enforced_limit, new_limit);
+  }
   if (!skip_manifest_write &&
-      (!descriptor_log_ ||
-       prev_manifest_file_size >= tuned_max_manifest_file_size_)) {
+      (!descriptor_log_ || prev_manifest_file_size >= enforced_limit)) {
     TEST_SYNC_POINT("VersionSet::ProcessManifestWrites:BeforeNewManifest");
     new_descriptor_log = true;
   } else {
@@ -6284,6 +6307,8 @@ Status VersionSet::ProcessManifestWrites(
       io_s = SetCurrentFile(
           write_options, fs_.get(), dbname_, pending_manifest_file_number_,
           file_options_.temperature, dir_contains_current_file);
+      TEST_SYNC_POINT_CALLBACK(
+          "VersionSet::ProcessManifestWrites:AfterSetCurrentFile", &io_s);
       if (!io_s.ok()) {
         s = io_s;
         // Quarantine old manifest file in case new manifest file's CURRENT

db_stress_tool/batched_ops_stress.cc

@@ -13,7 +13,9 @@
 namespace ROCKSDB_NAMESPACE {
 class BatchedOpsStressTest : public StressTest {
  public:
-  BatchedOpsStressTest() = default;
+  BatchedOpsStressTest(int db_index, const std::string& db_path,
+                       const std::string& ev_path, const std::string& sec_path)
+      : StressTest(db_index, db_path, ev_path, sec_path) {}
   virtual ~BatchedOpsStressTest() = default;
 
   bool IsStateTracked() const override { return false; }
@@ -722,7 +724,11 @@ class BatchedOpsStressTest : public StressTest {
   }
 };
 
-StressTest* CreateBatchedOpsStressTest() { return new BatchedOpsStressTest(); }
+StressTest* CreateBatchedOpsStressTest(int db_index, const std::string& db_path,
+                                       const std::string& ev_path,
+                                       const std::string& sec_path) {
+  return new BatchedOpsStressTest(db_index, db_path, ev_path, sec_path);
+}
 
 }  // namespace ROCKSDB_NAMESPACE
 #endif  // GFLAGS

db_stress_tool/cf_consistency_stress.cc

@@ -21,7 +21,10 @@
 namespace ROCKSDB_NAMESPACE {
 class CfConsistencyStressTest : public StressTest {
  public:
-  CfConsistencyStressTest() : batch_id_(0) {}
+  CfConsistencyStressTest(int db_index, const std::string& db_path,
+                          const std::string& ev_path,
+                          const std::string& sec_path)
+      : StressTest(db_index, db_path, ev_path, sec_path), batch_id_(0) {}
 
   ~CfConsistencyStressTest() override = default;
 
@@ -227,10 +230,10 @@ class CfConsistencyStressTest : public StressTest {
                    key, &value0);
 
       // Temporarily disable error injection for verification
-      if (fault_fs_guard) {
-        fault_fs_guard->DisableThreadLocalErrorInjection(
+      if (db_fault_injection_fs_) {
+        db_fault_injection_fs_->DisableThreadLocalErrorInjection(
             FaultInjectionIOType::kRead);
-        fault_fs_guard->DisableThreadLocalErrorInjection(
+        db_fault_injection_fs_->DisableThreadLocalErrorInjection(
             FaultInjectionIOType::kMetadataRead);
       }
 
@@ -279,10 +282,10 @@ class CfConsistencyStressTest : public StressTest {
       }
 
       //  Enable back error injection disabled for verification
-      if (fault_fs_guard) {
-        fault_fs_guard->EnableThreadLocalErrorInjection(
+      if (db_fault_injection_fs_) {
+        db_fault_injection_fs_->EnableThreadLocalErrorInjection(
             FaultInjectionIOType::kRead);
-        fault_fs_guard->EnableThreadLocalErrorInjection(
+        db_fault_injection_fs_->EnableThreadLocalErrorInjection(
             FaultInjectionIOType::kMetadataRead);
       }
       db_->ReleaseSnapshot(snapshot);
@@ -397,10 +400,10 @@ class CfConsistencyStressTest : public StressTest {
                          &cmp_result);
 
       //  Temporarily disable error injection for verification
-      if (fault_fs_guard) {
-        fault_fs_guard->DisableThreadLocalErrorInjection(
+      if (db_fault_injection_fs_) {
+        db_fault_injection_fs_->DisableThreadLocalErrorInjection(
             FaultInjectionIOType::kRead);
-        fault_fs_guard->DisableThreadLocalErrorInjection(
+        db_fault_injection_fs_->DisableThreadLocalErrorInjection(
             FaultInjectionIOType::kMetadataRead);
       }
 
@@ -559,10 +562,10 @@ class CfConsistencyStressTest : public StressTest {
       }
 
       //  Enable back error injection disabled for verification
-      if (fault_fs_guard) {
-        fault_fs_guard->EnableThreadLocalErrorInjection(
+      if (db_fault_injection_fs_) {
+        db_fault_injection_fs_->EnableThreadLocalErrorInjection(
             FaultInjectionIOType::kRead);
-        fault_fs_guard->EnableThreadLocalErrorInjection(
+        db_fault_injection_fs_->EnableThreadLocalErrorInjection(
             FaultInjectionIOType::kMetadataRead);
       }
     }
@@ -1543,8 +1546,11 @@ class CfConsistencyStressTest : public StressTest {
   std::deque<DebugEvent> recent_debug_events_;
 };
 
-StressTest* CreateCfConsistencyStressTest() {
-  return new CfConsistencyStressTest();
+StressTest* CreateCfConsistencyStressTest(int db_index,
+                                          const std::string& db_path,
+                                          const std::string& ev_path,
+                                          const std::string& sec_path) {
+  return new CfConsistencyStressTest(db_index, db_path, ev_path, sec_path);
 }
 
 }  // namespace ROCKSDB_NAMESPACE

db_stress_tool/db_stress_common.cc

@@ -13,16 +13,17 @@
 
 #include <cmath>
 
+#include "db_stress_tool/db_stress_test_base.h"
 #include "file/file_util.h"
 #include "rocksdb/secondary_cache.h"
 #include "util/file_checksum_helper.h"
 #include "util/xxhash.h"
 
-ROCKSDB_NAMESPACE::Env* db_stress_listener_env = nullptr;
-ROCKSDB_NAMESPACE::Env* db_stress_env = nullptr;
-std::shared_ptr<ROCKSDB_NAMESPACE::FaultInjectionTestFS> fault_fs_guard;
+ROCKSDB_NAMESPACE::Env* raw_env = nullptr;
 std::shared_ptr<ROCKSDB_NAMESPACE::SecondaryCache> compressed_secondary_cache;
 std::shared_ptr<ROCKSDB_NAMESPACE::Cache> block_cache;
+std::shared_ptr<ROCKSDB_NAMESPACE::WriteBufferManager> wbm;
+std::shared_ptr<ROCKSDB_NAMESPACE::RateLimiter> rate_limiter;
 enum ROCKSDB_NAMESPACE::CompressionType compression_type_e =
     ROCKSDB_NAMESPACE::kSnappyCompression;
 enum ROCKSDB_NAMESPACE::CompressionType bottommost_compression_type_e =
@@ -114,10 +115,10 @@ void PoolSizeChangeThread(void* v) {
     if (new_thread_pool_size < 1) {
       new_thread_pool_size = 1;
     }
-    db_stress_env->SetBackgroundThreads(new_thread_pool_size,
-                                        ROCKSDB_NAMESPACE::Env::Priority::LOW);
+    raw_env->SetBackgroundThreads(new_thread_pool_size,
+                                  ROCKSDB_NAMESPACE::Env::Priority::LOW);
     // Sleep up to 3 seconds
-    db_stress_env->SleepForMicroseconds(
+    raw_env->SleepForMicroseconds(
         thread->rand.Next() % FLAGS_compaction_thread_pool_adjust_interval *
             1000 +
         1);
@@ -144,7 +145,7 @@ void DbVerificationThread(void* v) {
     if (!shared->HasVerificationFailedYet()) {
       stress_test->ContinuouslyVerifyDb(thread);
     }
-    db_stress_env->SleepForMicroseconds(
+    raw_env->SleepForMicroseconds(
         thread->rand.Next() % FLAGS_continuous_verification_interval * 1000 +
         1);
   }
@@ -166,7 +167,7 @@ void CompressedCacheSetCapacityThread(void* v) {
         return;
       }
     }
-    db_stress_env->SleepForMicroseconds(FLAGS_secondary_cache_update_interval);
+    raw_env->SleepForMicroseconds(FLAGS_secondary_cache_update_interval);
     if (FLAGS_compressed_secondary_cache_size > 0) {
       Status s = compressed_secondary_cache->SetCapacity(0);
       size_t capacity;
@@ -174,7 +175,7 @@ void CompressedCacheSetCapacityThread(void* v) {
         s = compressed_secondary_cache->GetCapacity(capacity);
         assert(capacity == 0);
       }
-      db_stress_env->SleepForMicroseconds(10 * 1000 * 1000);
+      raw_env->SleepForMicroseconds(10 * 1000 * 1000);
       if (s.ok()) {
         s = compressed_secondary_cache->SetCapacity(
             FLAGS_compressed_secondary_cache_size);
@@ -201,7 +202,7 @@ void CompressedCacheSetCapacityThread(void* v) {
         block_cache->SetCapacity(capacity - adjustment);
         fprintf(stdout, "New cache capacity = %zu\n",
                 block_cache->GetCapacity());
-        db_stress_env->SleepForMicroseconds(10 * 1000 * 1000);
+        raw_env->SleepForMicroseconds(10 * 1000 * 1000);
         block_cache->SetCapacity(capacity);
       } else {
         Status s;
@@ -214,7 +215,7 @@ void CompressedCacheSetCapacityThread(void* v) {
         s = UpdateTieredCache(block_cache, /*capacity*/ -1,
                               new_comp_cache_ratio);
         if (s.ok()) {
-          db_stress_env->SleepForMicroseconds(10 * 1000 * 1000);
+          raw_env->SleepForMicroseconds(10 * 1000 * 1000);
         }
         if (s.ok()) {
           s = UpdateTieredCache(block_cache, /*capacity*/ -1,
@@ -231,36 +232,37 @@ void CompressedCacheSetCapacityThread(void* v) {
 
 #ifndef NDEBUG
 static void SetupFaultInjectionForRemoteCompaction(SharedState* shared) {
-  if (!fault_fs_guard) {
+  auto fault_fs = shared->GetStressTest()->GetDbFaultInjectionFs();
+  if (!fault_fs) {
     return;
   }
 
-  fault_fs_guard->SetThreadLocalErrorContext(
+  fault_fs->SetThreadLocalErrorContext(
       FaultInjectionIOType::kRead, shared->GetSeed(), FLAGS_read_fault_one_in,
       FLAGS_inject_error_severity == 1 /* retryable */,
       FLAGS_inject_error_severity == 2 /* has_data_loss*/);
-  fault_fs_guard->EnableThreadLocalErrorInjection(FaultInjectionIOType::kRead);
+  fault_fs->EnableThreadLocalErrorInjection(FaultInjectionIOType::kRead);
 
-  fault_fs_guard->SetThreadLocalErrorContext(
+  fault_fs->SetThreadLocalErrorContext(
       FaultInjectionIOType::kWrite, shared->GetSeed(), FLAGS_write_fault_one_in,
       FLAGS_inject_error_severity == 1 /* retryable */,
       FLAGS_inject_error_severity == 2 /* has_data_loss*/);
-  fault_fs_guard->EnableThreadLocalErrorInjection(FaultInjectionIOType::kWrite);
+  fault_fs->EnableThreadLocalErrorInjection(FaultInjectionIOType::kWrite);
 
-  fault_fs_guard->SetThreadLocalErrorContext(
+  fault_fs->SetThreadLocalErrorContext(
       FaultInjectionIOType::kMetadataRead, shared->GetSeed(),
       FLAGS_metadata_read_fault_one_in,
       FLAGS_inject_error_severity == 1 /* retryable */,
       FLAGS_inject_error_severity == 2 /* has_data_loss*/);
-  fault_fs_guard->EnableThreadLocalErrorInjection(
+  fault_fs->EnableThreadLocalErrorInjection(
       FaultInjectionIOType::kMetadataRead);
 
-  fault_fs_guard->SetThreadLocalErrorContext(
+  fault_fs->SetThreadLocalErrorContext(
       FaultInjectionIOType::kMetadataWrite, shared->GetSeed(),
       FLAGS_metadata_write_fault_one_in,
       FLAGS_inject_error_severity == 1 /* retryable */,
       FLAGS_inject_error_severity == 2 /* has_data_loss*/);
-  fault_fs_guard->EnableThreadLocalErrorInjection(
+  fault_fs->EnableThreadLocalErrorInjection(
       FaultInjectionIOType::kMetadataWrite);
 }
 #endif  // NDEBUG
@@ -268,7 +270,7 @@ static void SetupFaultInjectionForRemoteCompaction(SharedState* shared) {
 static CompactionServiceOptionsOverride CreateOverrideOptions(
     const Options& options, const CompactionServiceJobInfo& job_info) {
   CompactionServiceOptionsOverride override_options{
-      .env = db_stress_env,
+      .env = options.env,
       .file_checksum_gen_factory = options.file_checksum_gen_factory,
       .merge_operator = options.merge_operator,
       .compaction_filter = options.compaction_filter,
@@ -311,14 +313,8 @@ static CompactionServiceOptionsOverride CreateOverrideOptions(
 }
 
 static Status CleanupOutputDirectory(const std::string& output_directory) {
-#ifndef NDEBUG
-  // Temporarily disable fault injection to ensure deletion always succeeds
-  if (fault_fs_guard) {
-    fault_fs_guard->DisableAllThreadLocalErrorInjection();
-  }
-#endif  // NDEBUG
-
-  Status s = DestroyDir(db_stress_env, output_directory);
+  // Uses raw_env (no fault injection); cleanup must always succeed.
+  Status s = DestroyDir(raw_env, output_directory);
   if (!s.ok()) {
     fprintf(stderr,
             "Failed to destroy output directory %s when allow_resumption is "
@@ -327,7 +323,7 @@ static Status CleanupOutputDirectory(const std::string& output_directory) {
   }
 
   if (s.ok()) {
-    s = db_stress_env->CreateDir(output_directory);
+    s = raw_env->CreateDir(output_directory);
     if (!s.ok()) {
       fprintf(stderr,
               "Failed to recreate output directory %s when allow_resumption is "
@@ -336,13 +332,6 @@ static Status CleanupOutputDirectory(const std::string& output_directory) {
     }
   }
 
-#ifndef NDEBUG
-  // Re-enable fault injection after deletion
-  if (fault_fs_guard) {
-    fault_fs_guard->EnableAllThreadLocalErrorInjection();
-  }
-#endif  // NDEBUG
-
   return s;
 }
 
@@ -497,7 +486,7 @@ void RemoteCompactionWorkerThread(void* v) {
           shared, stress_test, rand, successful_compaction_end_to_end_micros);
     }
 
-    db_stress_env->SleepForMicroseconds(
+    raw_env->SleepForMicroseconds(
         thread->rand.Next() % FLAGS_remote_compaction_worker_interval * 1000 +
         1);
   }
@@ -679,7 +668,7 @@ bool VerifyIteratorAttributeGroups(
 }
 
 std::string GetNowNanos() {
-  uint64_t t = db_stress_env->NowNanos();
+  uint64_t t = raw_env->NowNanos();
   std::string ret;
   PutFixed64(&ret, t);
   return ret;
@@ -691,7 +680,7 @@ uint64_t GetWriteUnixTime(ThreadState* thread) {
                FLAGS_preclude_last_level_data_seconds);
   static uint64_t kFallbackTime = std::numeric_limits<uint64_t>::max();
   int64_t write_time = 0;
-  Status s = db_stress_env->GetCurrentTime(&write_time);
+  Status s = raw_env->GetCurrentTime(&write_time);
   uint32_t write_time_mode = thread->rand.Uniform(3);
   if (write_time_mode == 0 || !s.ok()) {
     return kFallbackTime;
@@ -806,6 +795,8 @@ std::shared_ptr<FileChecksumGenFactory> GetFileChecksumImpl(
   return std::make_shared<DbStressChecksumGenFactory>(internal_name);
 }
 
+// Expected values state files are always on local filesystem (Python owns
+// this dir), so use Env::Default() (PosixEnv) even when raw_env is remote.
 Status DeleteFilesInDirectory(const std::string& dirname) {
   std::vector<std::string> filenames;
   Status s = Env::Default()->GetChildren(dirname, &filenames);
@@ -880,8 +871,12 @@ Status DestroyUnverifiedSubdir(const std::string& dirname) {
 Status DbStressDestroyDb(const std::string& db_path) {
   Status s;
   Options options;
-  // NOTE: using db_stress_listener_env in order to see obsolete MANIFEST files
-  options.env = db_stress_listener_env;
+  // Use raw_env (no DbStressFSWrapper) because DbStressFSWrapper renames
+  // MANIFEST files to MANIFEST-xxx_renamed_ instead of deleting them during
+  // normal DB operation (to preserve history for debugging). Using raw_env
+  // here ensures DestroyDB actually deletes these files. DestroyDir below
+  // catches any remaining files not known to DestroyDB.
+  options.env = raw_env;
   // Remove DB files in a principled way to avoid issues
   if (FLAGS_use_blob_db) {
     s = blob_db::DestroyBlobDB(db_path, options, blob_db::BlobDBOptions());
@@ -891,9 +886,8 @@ Status DbStressDestroyDb(const std::string& db_path) {
   if (!s.ok()) {
     return s;
   }
-  // Remove everything else recursively, only reporting success if able to
-  // delete everything
-  return DestroyDir(db_stress_listener_env, db_path);
+  // Remove everything else recursively (catches MANIFEST_renamed_ files)
+  return DestroyDir(raw_env, db_path);
 }
 
 }  // namespace ROCKSDB_NAMESPACE

db_stress_tool/db_stress_common.h

@@ -49,6 +49,7 @@
 #include "rocksdb/advanced_options.h"
 #include "rocksdb/cache.h"
 #include "rocksdb/env.h"
+#include "rocksdb/rate_limiter.h"
 #include "rocksdb/slice.h"
 #include "rocksdb/slice_transform.h"
 #include "rocksdb/statistics.h"
@@ -61,6 +62,7 @@
 #include "rocksdb/utilities/transaction.h"
 #include "rocksdb/utilities/transaction_db.h"
 #include "rocksdb/write_batch.h"
+#include "rocksdb/write_buffer_manager.h"
 #include "test_util/testutil.h"
 #include "util/coding.h"
 #include "util/compression.h"
@@ -191,6 +193,7 @@ DECLARE_string(db);
 DECLARE_string(secondaries_base);
 DECLARE_bool(test_secondary);
 DECLARE_string(expected_values_dir);
+DECLARE_int32(num_dbs);
 DECLARE_bool(expected_state_trace_debug);
 DECLARE_int64(expected_state_trace_debug_key);
 DECLARE_int32(expected_state_trace_debug_max_logs);
@@ -463,6 +466,7 @@ DECLARE_uint32(ingest_wbwi_one_in);
 DECLARE_bool(universal_reduce_file_locking);
 DECLARE_bool(use_multiscan);
 DECLARE_bool(multiscan_use_async_io);
+DECLARE_bool(read_scoped_block_buffer_provider);
 DECLARE_uint64(multiscan_max_prefetch_memory_bytes);
 
 // Compaction deletion trigger declarations for stress testing
@@ -480,13 +484,16 @@ constexpr int kRandomValueMaxFactor = 3;
 constexpr int kValueMaxLen = 100;
 constexpr uint32_t kLargePrimeForCommonFactorSkew = 1872439133;
 
-// wrapped posix environment
-extern ROCKSDB_NAMESPACE::Env* db_stress_env;
-extern ROCKSDB_NAMESPACE::Env* db_stress_listener_env;
-extern std::shared_ptr<ROCKSDB_NAMESPACE::FaultInjectionTestFS> fault_fs_guard;
+// Base env from --env_uri/--fs_uri. No wrappers (no DbStressFSWrapper, no
+// fault injection). Could be PosixEnv or remote. Used for infrastructure ops
+// (threads, time, dirs, cleanup). Per-StressTest env adds fault injection +
+// DbStressFSWrapper on top for DB I/O.
+extern ROCKSDB_NAMESPACE::Env* raw_env;
 extern std::shared_ptr<ROCKSDB_NAMESPACE::SecondaryCache>
     compressed_secondary_cache;
 extern std::shared_ptr<ROCKSDB_NAMESPACE::Cache> block_cache;
+extern std::shared_ptr<ROCKSDB_NAMESPACE::WriteBufferManager> wbm;
+extern std::shared_ptr<ROCKSDB_NAMESPACE::RateLimiter> rate_limiter;
 
 extern enum ROCKSDB_NAMESPACE::CompressionType compression_type_e;
 extern enum ROCKSDB_NAMESPACE::CompressionType bottommost_compression_type_e;
@@ -831,10 +838,21 @@ AttributeGroups GenerateAttributeGroups(
     const std::vector<ColumnFamilyHandle*>& cfhs, uint32_t value_base,
     const Slice& slice);
 
-StressTest* CreateCfConsistencyStressTest();
-StressTest* CreateBatchedOpsStressTest();
-StressTest* CreateNonBatchedOpsStressTest();
-StressTest* CreateMultiOpsTxnsStressTest();
+StressTest* CreateCfConsistencyStressTest(int db_index,
+                                          const std::string& db_path,
+                                          const std::string& ev_path,
+                                          const std::string& sec_path);
+StressTest* CreateBatchedOpsStressTest(int db_index, const std::string& db_path,
+                                       const std::string& ev_path,
+                                       const std::string& sec_path);
+StressTest* CreateNonBatchedOpsStressTest(int db_index,
+                                          const std::string& db_path,
+                                          const std::string& ev_path,
+                                          const std::string& sec_path);
+StressTest* CreateMultiOpsTxnsStressTest(int db_index,
+                                         const std::string& db_path,
+                                         const std::string& ev_path,
+                                         const std::string& sec_path);
 void CheckAndSetOptionsForMultiOpsTxnStressTest();
 void InitializeHotKeyGenerator(double alpha);
 int64_t GetOneHotKeyID(double rand_seed, int64_t max_key);

db_stress_tool/db_stress_compaction_service.cc

@@ -3,11 +3,6 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
 
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
 #ifdef GFLAGS
 
 #include "db_stress_tool/db_stress_compaction_service.h"

db_stress_tool/db_stress_driver.cc

@@ -10,12 +10,14 @@
 
 #include "db_stress_tool/db_stress_shared_state.h"
 #ifdef GFLAGS
+#include <mutex>
+
 #include "db_stress_tool/db_stress_common.h"
 #include "utilities/fault_injection_fs.h"
 
 namespace ROCKSDB_NAMESPACE {
 void ThreadBody(void* v) {
-  ThreadStatusUtil::RegisterThread(db_stress_env, ThreadStatus::USER);
+  ThreadStatusUtil::RegisterThread(raw_env, ThreadStatus::USER);
   ThreadState* thread = static_cast<ThreadState*>(v);
   SharedState* shared = thread->shared;
 
@@ -64,17 +66,21 @@ void ThreadBody(void* v) {
   ThreadStatusUtil::UnregisterThread();
 }
 bool RunStressTestImpl(SharedState* shared) {
-  SystemClock* clock = db_stress_env->GetSystemClock().get();
+  SystemClock* clock = raw_env->GetSystemClock().get();
   StressTest* stress = shared->GetStressTest();
+  const std::string db_label =
+      (FLAGS_num_dbs > 1) ? "[" + stress->GetDbLabel() + "] " : "";
 
   if (shared->ShouldVerifyAtBeginning() && FLAGS_preserve_unverified_changes) {
-    Status s = InitUnverifiedSubdir(FLAGS_db);
-    if (s.ok() && !FLAGS_expected_values_dir.empty()) {
-      s = InitUnverifiedSubdir(FLAGS_expected_values_dir);
+    const auto& db_path = stress->GetDbPath();
+    const auto& ev_dir = stress->GetExpectedValuesDir();
+    Status s = InitUnverifiedSubdir(db_path);
+    if (s.ok() && !ev_dir.empty()) {
+      s = InitUnverifiedSubdir(ev_dir);
     }
     if (!s.ok()) {
-      fprintf(stderr, "Failed to setup unverified state dir: %s\n",
-              s.ToString().c_str());
+      fprintf(stderr, "%sFailed to setup unverified state dir: %s\n",
+              db_label.c_str(), s.ToString().c_str());
       exit(1);
     }
   }
@@ -84,24 +90,15 @@ bool RunStressTestImpl(SharedState* shared) {
 
   uint32_t n = FLAGS_threads;
   uint64_t now = clock->NowMicros();
-  fprintf(stdout, "%s Initializing worker threads\n",
-          clock->TimeToString(now / 1000000).c_str());
+  fprintf(stdout, "%s %sInitializing worker threads\n",
+          clock->TimeToString(now / 1000000).c_str(), db_label.c_str());
 
   shared->SetThreads(n);
 
-  if (FLAGS_compaction_thread_pool_adjust_interval > 0) {
-    shared->IncBgThreads();
-  }
-
   if (FLAGS_continuous_verification_interval > 0) {
     shared->IncBgThreads();
   }
 
-  if (FLAGS_compressed_secondary_cache_size > 0 ||
-      FLAGS_compressed_secondary_cache_ratio > 0.0) {
-    shared->IncBgThreads();
-  }
-
   uint32_t remote_compaction_worker_thread_count =
       FLAGS_remote_compaction_worker_threads;
   if (remote_compaction_worker_thread_count > 0) {
@@ -113,25 +110,39 @@ bool RunStressTestImpl(SharedState* shared) {
   std::vector<ThreadState*> threads(n);
   for (uint32_t i = 0; i < n; i++) {
     threads[i] = new ThreadState(i, shared);
-    db_stress_env->StartThread(ThreadBody, threads[i]);
+    raw_env->StartThread(ThreadBody, threads[i]);
   }
 
+  // Spawn at most one PoolSizeChangeThread globally. All DBs share the
+  // same raw_env, so one thread resizing the pool is sufficient.
+  // IncBgThreads inside call_once so only the launching DB's SharedState
+  // tracks this thread (avoids deadlock for non-first DBs).
   ThreadState bg_thread(0, shared);
+  static std::once_flag pool_size_thread_flag;
   if (FLAGS_compaction_thread_pool_adjust_interval > 0) {
-    db_stress_env->StartThread(PoolSizeChangeThread, &bg_thread);
+    std::call_once(pool_size_thread_flag, [&]() {
+      shared->IncBgThreads();
+      raw_env->StartThread(PoolSizeChangeThread, &bg_thread);
+    });
   }
 
   ThreadState continuous_verification_thread(0, shared);
   if (FLAGS_continuous_verification_interval > 0) {
-    db_stress_env->StartThread(DbVerificationThread,
-                               &continuous_verification_thread);
+    raw_env->StartThread(DbVerificationThread, &continuous_verification_thread);
   }
 
+  // Spawn at most one CompressedCacheSetCapacityThread globally. The cache
+  // is shared across all DBs, and the thread's SetCapacity(0)/assert(==0)
+  // sequence would race if multiple DBs each spawned their own copy.
   ThreadState compressed_cache_set_capacity_thread(0, shared);
+  static std::once_flag compressed_cache_thread_flag;
   if (FLAGS_compressed_secondary_cache_size > 0 ||
       FLAGS_compressed_secondary_cache_ratio > 0.0) {
-    db_stress_env->StartThread(CompressedCacheSetCapacityThread,
-                               &compressed_cache_set_capacity_thread);
+    std::call_once(compressed_cache_thread_flag, [&]() {
+      shared->IncBgThreads();
+      raw_env->StartThread(CompressedCacheSetCapacityThread,
+                           &compressed_cache_set_capacity_thread);
+    });
   }
 
   std::vector<ThreadState*> remote_compaction_worker_threads;
@@ -141,7 +152,7 @@ bool RunStressTestImpl(SharedState* shared) {
     for (uint32_t i = 0; i < remote_compaction_worker_thread_count; i++) {
       ThreadState* ts = new ThreadState(i, shared);
       remote_compaction_worker_threads.push_back(ts);
-      db_stress_env->StartThread(RemoteCompactionWorkerThread, ts);
+      raw_env->StartThread(RemoteCompactionWorkerThread, ts);
     }
   }
 
@@ -156,16 +167,19 @@ bool RunStressTestImpl(SharedState* shared) {
     }
     if (shared->ShouldVerifyAtBeginning()) {
       if (shared->HasVerificationFailedYet()) {
-        fprintf(stderr, "Crash-recovery verification failed :(\n");
+        fprintf(stderr, "%sCrash-recovery verification failed :(\n",
+                db_label.c_str());
       } else {
-        fprintf(stdout, "Crash-recovery verification passed :)\n");
-        Status s = DestroyUnverifiedSubdir(FLAGS_db);
-        if (s.ok() && !FLAGS_expected_values_dir.empty()) {
-          s = DestroyUnverifiedSubdir(FLAGS_expected_values_dir);
+        fprintf(stdout, "%sCrash-recovery verification passed :)\n",
+                db_label.c_str());
+        const auto& ev_dir = stress->GetExpectedValuesDir();
+        Status s = DestroyUnverifiedSubdir(stress->GetDbPath());
+        if (s.ok() && !ev_dir.empty()) {
+          s = DestroyUnverifiedSubdir(ev_dir);
         }
         if (!s.ok()) {
-          fprintf(stderr, "Failed to cleanup unverified state dir: %s\n",
-                  s.ToString().c_str());
+          fprintf(stderr, "%sFailed to cleanup unverified state dir: %s\n",
+                  db_label.c_str(), s.ToString().c_str());
           exit(1);
         }
       }
@@ -174,15 +188,16 @@ bool RunStressTestImpl(SharedState* shared) {
     if (!FLAGS_verification_only) {
       // This is after the verification step to avoid making all those `Get()`s
       // and `MultiGet()`s contend on the DB-wide trace mutex.
-      if (!FLAGS_expected_values_dir.empty()) {
+      if (!stress->GetExpectedValuesDir().empty()) {
         stress->TrackExpectedState(shared);
       }
 
       if (FLAGS_sync_fault_injection || FLAGS_write_fault_one_in > 0) {
-        fault_fs_guard->SetFilesystemDirectWritable(false);
-        fault_fs_guard->SetInjectUnsyncedDataLoss(FLAGS_sync_fault_injection);
+        auto fault_fs = stress->GetDbFaultInjectionFs();
+        fault_fs->SetFilesystemDirectWritable(false);
+        fault_fs->SetInjectUnsyncedDataLoss(FLAGS_sync_fault_injection);
         if (FLAGS_exclude_wal_from_write_fault_injection) {
-          fault_fs_guard->SetFileTypesExcludedFromWriteFaultInjection(
+          fault_fs->SetFileTypesExcludedFromWriteFaultInjection(
               {FileType::kWalFile});
         }
       }
@@ -190,8 +205,8 @@ bool RunStressTestImpl(SharedState* shared) {
         Status s = stress->EnableAutoCompaction();
         assert(s.ok());
       }
-      fprintf(stdout, "%s Starting database operations\n",
-              clock->TimeToString(now / 1000000).c_str());
+      fprintf(stdout, "%s %sStarting database operations\n",
+              clock->TimeToString(now / 1000000).c_str(), db_label.c_str());
 
       shared->SetStart();
       shared->GetCondVar()->SignalAll();
@@ -201,14 +216,17 @@ bool RunStressTestImpl(SharedState* shared) {
 
       now = clock->NowMicros();
       if (FLAGS_test_batches_snapshots) {
-        fprintf(stdout, "%s Limited verification already done during gets\n",
-                clock->TimeToString((uint64_t)now / 1000000).c_str());
+        fprintf(stdout, "%s %sLimited verification already done during gets\n",
+                clock->TimeToString((uint64_t)now / 1000000).c_str(),
+                db_label.c_str());
       } else if (FLAGS_skip_verifydb) {
-        fprintf(stdout, "%s Verification skipped\n",
-                clock->TimeToString((uint64_t)now / 1000000).c_str());
+        fprintf(stdout, "%s %sVerification skipped\n",
+                clock->TimeToString((uint64_t)now / 1000000).c_str(),
+                db_label.c_str());
       } else {
-        fprintf(stdout, "%s Starting verification\n",
-                clock->TimeToString((uint64_t)now / 1000000).c_str());
+        fprintf(stdout, "%s %sStarting verification\n",
+                clock->TimeToString((uint64_t)now / 1000000).c_str(),
+                db_label.c_str());
       }
 
       shared->SetStartVerify();
@@ -230,7 +248,7 @@ bool RunStressTestImpl(SharedState* shared) {
     for (unsigned int i = 1; i < n; i++) {
       threads[0]->stats.Merge(threads[i]->stats);
     }
-    threads[0]->stats.Report("Stress Test");
+    threads[0]->stats.Report((db_label + "Stress Test").c_str());
   }
 
   for (unsigned int i = 0; i < n; i++) {
@@ -241,8 +259,8 @@ bool RunStressTestImpl(SharedState* shared) {
   now = clock->NowMicros();
   if (!FLAGS_skip_verifydb && !FLAGS_test_batches_snapshots &&
       !shared->HasVerificationFailedYet()) {
-    fprintf(stdout, "%s Verification successful\n",
-            clock->TimeToString(now / 1000000).c_str());
+    fprintf(stdout, "%s %sVerification successful\n",
+            clock->TimeToString(now / 1000000).c_str(), db_label.c_str());
   }
 
   if (!FLAGS_verification_only) {
@@ -260,6 +278,9 @@ bool RunStressTestImpl(SharedState* shared) {
       shared->GetCondVar()->Wait();
     }
   }
+  now = clock->NowMicros();
+  fprintf(stdout, "%s %sStress test bg threads finished\n",
+          clock->TimeToString(now / 1000000).c_str(), db_label.c_str());
 
   assert(remote_compaction_worker_threads.size() ==
          remote_compaction_worker_thread_count);
@@ -271,13 +292,13 @@ bool RunStressTestImpl(SharedState* shared) {
   }
 
   if (shared->HasVerificationFailedYet()) {
-    fprintf(stderr, "Verification failed :(\n");
+    fprintf(stderr, "%sVerification failed :(\n", db_label.c_str());
     return false;
   }
   return true;
 }
 bool RunStressTest(SharedState* shared) {
-  ThreadStatusUtil::RegisterThread(db_stress_env, ThreadStatus::USER);
+  ThreadStatusUtil::RegisterThread(raw_env, ThreadStatus::USER);
   bool result = RunStressTestImpl(shared);
   ThreadStatusUtil::UnregisterThread();
   return result;

db_stress_tool/db_stress_gflags.cc

@@ -1705,10 +1705,27 @@ DEFINE_bool(use_multiscan, false,
 DEFINE_bool(multiscan_use_async_io, false,
             "If set, enable async_io for MultiScan operations.");
 
+DEFINE_bool(read_scoped_block_buffer_provider, false,
+            "If set, configure ReadOptions::read_scoped_block_buffer_provider "
+            "with a stress-test provider for supported scan reads.");
+
 DEFINE_uint64(multiscan_max_prefetch_memory_bytes, 0,
               "If non-zero, sets the max_prefetch_memory_bytes on the "
               "IODispatcher used for MultiScan. This limits the total memory "
               "used for prefetching data blocks across all concurrent "
               "MultiScan ReadSets.");
 
+DEFINE_int32(num_dbs, 1,
+             "Number of DB instances to run in parallel. "
+             "For num_dbs=1: --db is the DB path directly (default). "
+             "For num_dbs>1: --db is a parent directory; db_stress "
+             "auto-creates db_0/, db_1/, ... subdirs underneath "
+             "(same for --expected_values_dir and secondaries). "
+             "Per-DB flags: threads, max_key, ops_per_thread, reopen, "
+             "column_families, and all DB options. "
+             "Shared across all instances: background env threads "
+             "(compaction, flush pool), cache, write_buffer_manager, "
+             "compressed_secondary_cache, rate_limiter, and "
+             "compaction_thread_pool_adjust_interval.");
+
 #endif  // GFLAGS

db_stress_tool/db_stress_listener.cc

@@ -7,6 +7,7 @@
 
 #include <cstdint>
 
+#include "db_stress_tool/db_stress_test_base.h"
 #include "file/file_util.h"
 #include "rocksdb/file_system.h"
 #include "util/coding_lean.h"
@@ -15,6 +16,21 @@ namespace ROCKSDB_NAMESPACE {
 
 #ifdef GFLAGS
 
+DbStressListener::DbStressListener(
+    const std::string& db_name, const std::vector<DbPath>& db_paths,
+    const std::vector<ColumnFamilyDescriptor>& column_families,
+    SharedState* shared)
+    : db_name_(db_name),
+      db_paths_(db_paths),
+      column_families_(column_families),
+      num_pending_file_creations_(0),
+      unique_ids_(shared->GetStressTest()->GetExpectedValuesDir().empty()
+                      ? db_name
+                      : shared->GetStressTest()->GetExpectedValuesDir()),
+      shared_(shared),
+      db_fault_injection_fs_(shared->GetStressTest()->GetDbFaultInjectionFs()) {
+}
+
 UniqueIdVerifier::UniqueIdVerifier(const std::string& dir)
     : path_(dir + "/.unique_ids") {
   // We expect such a small number of files generated during this test

db_stress_tool/db_stress_listener.h

@@ -27,8 +27,6 @@
 #include "utilities/fault_injection_fs.h"
 DECLARE_int32(compact_files_one_in);
 
-extern std::shared_ptr<ROCKSDB_NAMESPACE::FaultInjectionTestFS> fault_fs_guard;
-
 namespace ROCKSDB_NAMESPACE {
 
 // Verify across process executions that all seen IDs are unique
@@ -58,34 +56,22 @@ class DbStressListener : public EventListener {
   DbStressListener(const std::string& db_name,
                    const std::vector<DbPath>& db_paths,
                    const std::vector<ColumnFamilyDescriptor>& column_families,
-                   SharedState* shared)
-      : db_name_(db_name),
-        db_paths_(db_paths),
-        column_families_(column_families),
-        num_pending_file_creations_(0),
-        unique_ids_(FLAGS_expected_values_dir.empty()
-                        ? db_name
-                        : FLAGS_expected_values_dir),
-        shared_(shared) {}
+                   SharedState* shared);
 
   const char* Name() const override { return kClassName(); }
   static const char* kClassName() { return "DBStressListener"; }
 
   ~DbStressListener() override { assert(num_pending_file_creations_ == 0); }
 
-  // Signal that the DB is about to shut down. Must be called before DB::Close()
-  // or CancelAllBackgroundWork(). Why? Listener notifications, especially for
-  // compaction, have different (mostly relaxed) contracts during shutdown, and
-  // not all callbacks take a DB object. Ideally, public API improvements would
-  // make this unnecessary in the future.
-  void NotifyShuttingDown() { shutting_down_.Store(true); }
+  void OnDBShutdownBegin(DB* /*db*/) override { shutting_down_.Store(true); }
+
   void OnFlushCompleted(DB* /*db*/, const FlushJobInfo& info) override {
     assert(IsValidColumnFamilyName(info.cf_name));
     VerifyFilePath(info.file_path);
     // pretending doing some work here
     RandomSleep();
-    if (fault_fs_guard) {
-      fault_fs_guard->DisableAllThreadLocalErrorInjection();
+    if (db_fault_injection_fs_) {
+      db_fault_injection_fs_->DisableAllThreadLocalErrorInjection();
     }
     shared_->SetPersistedSeqno(info.largest_seqno);
   }
@@ -93,37 +79,37 @@ class DbStressListener : public EventListener {
   void OnFlushBegin(DB* /*db*/,
                     const FlushJobInfo& /*flush_job_info*/) override {
     RandomSleep();
-    if (fault_fs_guard) {
-      fault_fs_guard->SetThreadLocalErrorContext(
+    if (db_fault_injection_fs_) {
+      db_fault_injection_fs_->SetThreadLocalErrorContext(
           FaultInjectionIOType::kRead, static_cast<uint32_t>(FLAGS_seed),
           FLAGS_read_fault_one_in,
           FLAGS_inject_error_severity == 1 /* retryable */,
           FLAGS_inject_error_severity == 2 /* has_data_loss*/);
-      fault_fs_guard->EnableThreadLocalErrorInjection(
+      db_fault_injection_fs_->EnableThreadLocalErrorInjection(
           FaultInjectionIOType::kRead);
 
-      fault_fs_guard->SetThreadLocalErrorContext(
+      db_fault_injection_fs_->SetThreadLocalErrorContext(
           FaultInjectionIOType::kWrite, static_cast<uint32_t>(FLAGS_seed),
           FLAGS_write_fault_one_in,
           FLAGS_inject_error_severity == 1 /* retryable */,
           FLAGS_inject_error_severity == 2 /* has_data_loss*/);
-      fault_fs_guard->EnableThreadLocalErrorInjection(
+      db_fault_injection_fs_->EnableThreadLocalErrorInjection(
           FaultInjectionIOType::kWrite);
 
-      fault_fs_guard->SetThreadLocalErrorContext(
+      db_fault_injection_fs_->SetThreadLocalErrorContext(
           FaultInjectionIOType::kMetadataRead,
           static_cast<uint32_t>(FLAGS_seed), FLAGS_metadata_read_fault_one_in,
           FLAGS_inject_error_severity == 1 /* retryable */,
           FLAGS_inject_error_severity == 2 /* has_data_loss*/);
-      fault_fs_guard->EnableThreadLocalErrorInjection(
+      db_fault_injection_fs_->EnableThreadLocalErrorInjection(
           FaultInjectionIOType::kMetadataRead);
 
-      fault_fs_guard->SetThreadLocalErrorContext(
+      db_fault_injection_fs_->SetThreadLocalErrorContext(
           FaultInjectionIOType::kMetadataWrite,
           static_cast<uint32_t>(FLAGS_seed), FLAGS_metadata_write_fault_one_in,
           FLAGS_inject_error_severity == 1 /* retryable */,
           FLAGS_inject_error_severity == 2 /* has_data_loss*/);
-      fault_fs_guard->EnableThreadLocalErrorInjection(
+      db_fault_injection_fs_->EnableThreadLocalErrorInjection(
           FaultInjectionIOType::kMetadataWrite);
     }
   }
@@ -250,44 +236,44 @@ class DbStressListener : public EventListener {
   }
 
   void OnSubcompactionBegin(const SubcompactionJobInfo& /* si */) override {
-    if (fault_fs_guard) {
-      fault_fs_guard->SetThreadLocalErrorContext(
+    if (db_fault_injection_fs_) {
+      db_fault_injection_fs_->SetThreadLocalErrorContext(
           FaultInjectionIOType::kRead, static_cast<uint32_t>(FLAGS_seed),
           FLAGS_read_fault_one_in,
           FLAGS_inject_error_severity == 1 /* retryable */,
           FLAGS_inject_error_severity == 2 /* has_data_loss*/);
-      fault_fs_guard->EnableThreadLocalErrorInjection(
+      db_fault_injection_fs_->EnableThreadLocalErrorInjection(
           FaultInjectionIOType::kRead);
 
-      fault_fs_guard->SetThreadLocalErrorContext(
+      db_fault_injection_fs_->SetThreadLocalErrorContext(
           FaultInjectionIOType::kWrite, static_cast<uint32_t>(FLAGS_seed),
           FLAGS_write_fault_one_in,
           FLAGS_inject_error_severity == 1 /* retryable */,
           FLAGS_inject_error_severity == 2 /* has_data_loss*/);
-      fault_fs_guard->EnableThreadLocalErrorInjection(
+      db_fault_injection_fs_->EnableThreadLocalErrorInjection(
           FaultInjectionIOType::kWrite);
 
-      fault_fs_guard->SetThreadLocalErrorContext(
+      db_fault_injection_fs_->SetThreadLocalErrorContext(
           FaultInjectionIOType::kMetadataRead,
           static_cast<uint32_t>(FLAGS_seed), FLAGS_metadata_read_fault_one_in,
           FLAGS_inject_error_severity == 1 /* retryable */,
           FLAGS_inject_error_severity == 2 /* has_data_loss*/);
-      fault_fs_guard->EnableThreadLocalErrorInjection(
+      db_fault_injection_fs_->EnableThreadLocalErrorInjection(
           FaultInjectionIOType::kMetadataRead);
 
-      fault_fs_guard->SetThreadLocalErrorContext(
+      db_fault_injection_fs_->SetThreadLocalErrorContext(
           FaultInjectionIOType::kMetadataWrite,
           static_cast<uint32_t>(FLAGS_seed), FLAGS_metadata_write_fault_one_in,
           FLAGS_inject_error_severity == 1 /* retryable */,
           FLAGS_inject_error_severity == 2 /* has_data_loss*/);
-      fault_fs_guard->EnableThreadLocalErrorInjection(
+      db_fault_injection_fs_->EnableThreadLocalErrorInjection(
           FaultInjectionIOType::kMetadataWrite);
     }
   }
 
   void OnSubcompactionCompleted(const SubcompactionJobInfo& /* si */) override {
-    if (fault_fs_guard) {
-      fault_fs_guard->DisableAllThreadLocalErrorInjection();
+    if (db_fault_injection_fs_) {
+      db_fault_injection_fs_->DisableAllThreadLocalErrorInjection();
     }
   }
 
@@ -380,11 +366,12 @@ class DbStressListener : public EventListener {
                             Status /* bg_error */,
                             bool* /* auto_recovery */) override {
     RandomSleep();
-    if (FLAGS_error_recovery_with_no_fault_injection && fault_fs_guard) {
-      fault_fs_guard->DisableAllThreadLocalErrorInjection();
+    if (FLAGS_error_recovery_with_no_fault_injection &&
+        db_fault_injection_fs_) {
+      db_fault_injection_fs_->DisableAllThreadLocalErrorInjection();
       // TODO(hx235): only exempt the flush thread during error recovery instead
       // of all the flush threads from error injection
-      fault_fs_guard->SetIOActivitiesExcludedFromFaultInjection(
+      db_fault_injection_fs_->SetIOActivitiesExcludedFromFaultInjection(
           {Env::IOActivity::kFlush});
     }
   }
@@ -392,9 +379,10 @@ class DbStressListener : public EventListener {
   void OnErrorRecoveryEnd(
       const BackgroundErrorRecoveryInfo& /*info*/) override {
     RandomSleep();
-    if (FLAGS_error_recovery_with_no_fault_injection && fault_fs_guard) {
-      fault_fs_guard->EnableAllThreadLocalErrorInjection();
-      fault_fs_guard->SetIOActivitiesExcludedFromFaultInjection({});
+    if (FLAGS_error_recovery_with_no_fault_injection &&
+        db_fault_injection_fs_) {
+      db_fault_injection_fs_->EnableAllThreadLocalErrorInjection();
+      db_fault_injection_fs_->SetIOActivitiesExcludedFromFaultInjection({});
     }
   }
 
@@ -485,6 +473,7 @@ class DbStressListener : public EventListener {
   std::atomic<int> num_pending_file_creations_;
   UniqueIdVerifier unique_ids_;
   SharedState* shared_;
+  std::shared_ptr<FaultInjectionTestFS> db_fault_injection_fs_;
   mutable std::mutex bg_pressure_mu_;
   BackgroundJobPressure last_bg_pressure_;
   // Files (by file_number) currently in flight from OnCompactionBegin to
@@ -493,7 +482,8 @@ class DbStressListener : public EventListener {
   // callback. Protected by compacting_files_mu_.
   std::mutex compacting_files_mu_;
   std::unordered_set<uint64_t> compacting_files_;
-  // Set before DB close to suppress false positives from stale tracking.
+  // Set when DBImpl begins shutdown to suppress false positives from stale
+  // tracking when compaction callbacks are skipped during shutdown.
   Atomic<bool> shutting_down_{false};
   // Jobs that have passed OnCompactionPreCommit but not yet
   // OnCompactionCompleted. Maps job_id -> input file numbers.

db_stress_tool/db_stress_shared_state.cc

@@ -11,7 +11,109 @@
 #ifdef GFLAGS
 #include "db_stress_tool/db_stress_shared_state.h"
 
+#include "db_stress_tool/db_stress_test_base.h"
+
 namespace ROCKSDB_NAMESPACE {
 thread_local bool SharedState::ignore_read_error;
+
+SharedState::SharedState(Env* /*env*/, StressTest* stress_test)
+    : cv_(&mu_),
+      seed_(static_cast<uint32_t>(FLAGS_seed)),
+      max_key_(FLAGS_max_key),
+      log2_keys_per_lock_(static_cast<uint32_t>(FLAGS_log2_keys_per_lock)),
+      num_threads_(0),
+      num_initialized_(0),
+      num_populated_(0),
+      vote_reopen_(0),
+      num_done_(0),
+      start_(false),
+      start_verify_(false),
+      num_bg_threads_(0),
+      should_stop_bg_thread_(false),
+      bg_thread_finished_(0),
+      stress_test_(stress_test),
+      verification_failure_(false),
+      should_stop_test_(false),
+      no_overwrite_ids_(GenerateNoOverwriteIds()),
+      expected_state_manager_(nullptr),
+      printing_verification_results_(false),
+      start_timestamp_(Env::Default()->NowNanos()) {
+  Status status;
+  // TODO: We should introduce a way to explicitly disable verification
+  // during shutdown. When that is disabled and FLAGS_expected_values_dir
+  // is empty (disabling verification at startup), we can skip tracking
+  // expected state. Only then should we permit bypassing the below feature
+  // compatibility checks.
+  const auto& expected_values_dir = stress_test_->GetExpectedValuesDir();
+  if (!expected_values_dir.empty()) {
+    if (!std::atomic<uint32_t>{}.is_lock_free() ||
+        !std::atomic<uint64_t>{}.is_lock_free()) {
+      std::ostringstream status_s;
+      status_s << "Cannot use --expected_values_dir on platforms without "
+                  "lock-free "
+               << (!std::atomic<uint32_t>{}.is_lock_free()
+                       ? "std::atomic<uint32_t>"
+                       : "std::atomic<uint64_t>");
+      status = Status::InvalidArgument(status_s.str());
+    }
+
+    if (status.ok() && FLAGS_clear_column_family_one_in > 0) {
+      status = Status::InvalidArgument(
+          "Cannot use --expected_values_dir on when "
+          "--clear_column_family_one_in is greater than zero.");
+    }
+  }
+  if (status.ok()) {
+    if (expected_values_dir.empty()) {
+      expected_state_manager_.reset(
+          new AnonExpectedStateManager(FLAGS_max_key, FLAGS_column_families));
+    } else {
+      expected_state_manager_.reset(new FileExpectedStateManager(
+          FLAGS_max_key, FLAGS_column_families, expected_values_dir));
+    }
+    status = expected_state_manager_->Open();
+  }
+  if (!status.ok()) {
+    fprintf(stderr, "Failed setting up expected state with error: %s\n",
+            status.ToString().c_str());
+    exit(1);  // NOLINT(concurrency-mt-unsafe)
+  }
+
+  if (FLAGS_test_batches_snapshots) {
+    fprintf(stdout, "No lock creation because test_batches_snapshots set\n");
+    return;
+  }
+
+  long num_locks = static_cast<long>(max_key_ >> log2_keys_per_lock_);
+  if (max_key_ & ((1 << log2_keys_per_lock_) - 1)) {
+    num_locks++;
+  }
+  fprintf(stdout, "Creating %ld locks\n", num_locks * FLAGS_column_families);
+  key_locks_.resize(FLAGS_column_families);
+
+  for (int i = 0; i < FLAGS_column_families; ++i) {
+    key_locks_[i].reset(new port::Mutex[num_locks]);
+  }
+  if (FLAGS_read_fault_one_in || FLAGS_metadata_read_fault_one_in) {
+#ifdef NDEBUG
+    // Unsupported in release mode because it relies on
+    // `IGNORE_STATUS_IF_ERROR` to distinguish faults not expected to lead to
+    // failure.
+    fprintf(stderr,
+            "Cannot set nonzero value for --read_fault_one_in in "
+            "release mode.");
+    exit(1);  // NOLINT(concurrency-mt-unsafe)
+#else         // NDEBUG
+    SyncPoint::GetInstance()->SetCallBack("FaultInjectionIgnoreError",
+                                          IgnoreReadErrorCallback);
+    SyncPoint::GetInstance()->EnableProcessing();
+#endif        // NDEBUG
+  }
+}
+
+bool SharedState::ShouldVerifyAtBeginning() const {
+  return !stress_test_->GetExpectedValuesDir().empty();
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 #endif  // GFLAGS

db_stress_tool/db_stress_shared_state.h

@@ -24,7 +24,6 @@ DECLARE_uint64(log2_keys_per_lock);
 DECLARE_int32(threads);
 DECLARE_int32(column_families);
 DECLARE_int32(nooverwritepercent);
-DECLARE_string(expected_values_dir);
 DECLARE_int32(clear_column_family_one_in);
 DECLARE_bool(test_batches_snapshots);
 DECLARE_int32(compaction_thread_pool_adjust_interval);
@@ -78,99 +77,7 @@ class SharedState {
   // for those calls
   static thread_local bool ignore_read_error;
 
-  SharedState(Env* /*env*/, StressTest* stress_test)
-      : cv_(&mu_),
-        seed_(static_cast<uint32_t>(FLAGS_seed)),
-        max_key_(FLAGS_max_key),
-        log2_keys_per_lock_(static_cast<uint32_t>(FLAGS_log2_keys_per_lock)),
-        num_threads_(0),
-        num_initialized_(0),
-        num_populated_(0),
-        vote_reopen_(0),
-        num_done_(0),
-        start_(false),
-        start_verify_(false),
-        num_bg_threads_(0),
-        should_stop_bg_thread_(false),
-        bg_thread_finished_(0),
-        stress_test_(stress_test),
-        verification_failure_(false),
-        should_stop_test_(false),
-        no_overwrite_ids_(GenerateNoOverwriteIds()),
-        expected_state_manager_(nullptr),
-        printing_verification_results_(false),
-        start_timestamp_(Env::Default()->NowNanos()) {
-    Status status;
-    // TODO: We should introduce a way to explicitly disable verification
-    // during shutdown. When that is disabled and FLAGS_expected_values_dir
-    // is empty (disabling verification at startup), we can skip tracking
-    // expected state. Only then should we permit bypassing the below feature
-    // compatibility checks.
-    if (!FLAGS_expected_values_dir.empty()) {
-      if (!std::atomic<uint32_t>{}.is_lock_free() ||
-          !std::atomic<uint64_t>{}.is_lock_free()) {
-        std::ostringstream status_s;
-        status_s << "Cannot use --expected_values_dir on platforms without "
-                    "lock-free "
-                 << (!std::atomic<uint32_t>{}.is_lock_free()
-                         ? "std::atomic<uint32_t>"
-                         : "std::atomic<uint64_t>");
-        status = Status::InvalidArgument(status_s.str());
-      }
-
-      if (status.ok() && FLAGS_clear_column_family_one_in > 0) {
-        status = Status::InvalidArgument(
-            "Cannot use --expected_values_dir on when "
-            "--clear_column_family_one_in is greater than zero.");
-      }
-    }
-    if (status.ok()) {
-      if (FLAGS_expected_values_dir.empty()) {
-        expected_state_manager_.reset(
-            new AnonExpectedStateManager(FLAGS_max_key, FLAGS_column_families));
-      } else {
-        expected_state_manager_.reset(new FileExpectedStateManager(
-            FLAGS_max_key, FLAGS_column_families, FLAGS_expected_values_dir));
-      }
-      status = expected_state_manager_->Open();
-    }
-    if (!status.ok()) {
-      fprintf(stderr, "Failed setting up expected state with error: %s\n",
-              status.ToString().c_str());
-      exit(1);
-    }
-
-    if (FLAGS_test_batches_snapshots) {
-      fprintf(stdout, "No lock creation because test_batches_snapshots set\n");
-      return;
-    }
-
-    long num_locks = static_cast<long>(max_key_ >> log2_keys_per_lock_);
-    if (max_key_ & ((1 << log2_keys_per_lock_) - 1)) {
-      num_locks++;
-    }
-    fprintf(stdout, "Creating %ld locks\n", num_locks * FLAGS_column_families);
-    key_locks_.resize(FLAGS_column_families);
-
-    for (int i = 0; i < FLAGS_column_families; ++i) {
-      key_locks_[i].reset(new port::Mutex[num_locks]);
-    }
-    if (FLAGS_read_fault_one_in || FLAGS_metadata_read_fault_one_in) {
-#ifdef NDEBUG
-      // Unsupported in release mode because it relies on
-      // `IGNORE_STATUS_IF_ERROR` to distinguish faults not expected to lead to
-      // failure.
-      fprintf(stderr,
-              "Cannot set nonzero value for --read_fault_one_in in "
-              "release mode.");
-      exit(1);
-#else   // NDEBUG
-      SyncPoint::GetInstance()->SetCallBack("FaultInjectionIgnoreError",
-                                            IgnoreReadErrorCallback);
-      SyncPoint::GetInstance()->EnableProcessing();
-#endif  // NDEBUG
-    }
-  }
+  SharedState(Env* env, StressTest* stress_test);
 
   ~SharedState() {
 #ifndef NDEBUG
@@ -430,9 +337,7 @@ class SharedState {
     return bg_thread_finished_ == num_bg_threads_;
   }
 
-  bool ShouldVerifyAtBeginning() const {
-    return !FLAGS_expected_values_dir.empty();
-  }
+  bool ShouldVerifyAtBeginning() const;
 
   bool PrintingVerificationResults() {
     bool tmp = false;

db_stress_tool/db_stress_stat.h

@@ -15,6 +15,7 @@
 #include "rocksdb/statistics.h"
 #include "rocksdb/system_clock.h"
 #include "util/gflags_compat.h"
+#include "util/mutexlock.h"
 #include "util/random.h"
 
 DECLARE_bool(histogram);
@@ -181,6 +182,9 @@ class Stats {
     double rate = bytes_mb / elapsed;
     double throughput = (double)done_ / elapsed;
 
+    // Lock stdout so multi-DB reports don't interleave.
+    static port::Mutex report_mutex;
+    MutexLock l(&report_mutex);
     fprintf(stdout, "%-12s: ", name);
     fprintf(stdout, "%.3f micros/op %ld ops/sec\n", seconds_ * 1e6 / done_,
             (long)throughput);

db_stress_tool/db_stress_test_base.h

@@ -13,6 +13,7 @@
 
 #include "db_stress_tool/db_stress_common.h"
 #include "db_stress_tool/db_stress_shared_state.h"
+#include "env/composite_env_wrapper.h"
 #include "rocksdb/experimental.h"
 #include "rocksdb/user_defined_index.h"
 #include "utilities/fault_injection_fs.h"
@@ -22,6 +23,7 @@ class SystemClock;
 class Transaction;
 class TransactionDB;
 class OptimisticTransactionDB;
+class DbStressFSWrapper;
 struct TransactionDBOptions;
 using experimental::SstQueryFilterConfigsManager;
 
@@ -40,11 +42,26 @@ class StressTest {
   // from optimistic transactions when conflict detection retries are exhausted.
   static bool IsExpectedTxnError(const Status& s);
 
-  StressTest();
+  StressTest(int db_index, const std::string& db_path,
+             const std::string& ev_path, const std::string& sec_path);
 
   virtual ~StressTest() {}
 
-  std::shared_ptr<Cache> NewCache(size_t capacity, int32_t num_shard_bits);
+  const std::string& GetDbLabel() const;
+  const std::string& GetDbPath() const;
+  const std::string& GetExpectedValuesDir() const;
+  const std::string& GetSecondariesBase() const;
+
+  // See db_fault_injection_fs_ member.
+  std::shared_ptr<FaultInjectionTestFS> GetDbFaultInjectionFs() const {
+    return db_fault_injection_fs_;
+  }
+
+  // See db_env_ member.
+  Env* GetDbEnv() const;
+
+  static std::shared_ptr<Cache> NewCache(size_t capacity,
+                                         int32_t num_shard_bits);
 
   static std::vector<std::string> GetBlobCompressionTags();
 
@@ -72,7 +89,9 @@ class StressTest {
   void CleanUp();
 
  private:
-  void NotifyListenerShuttingDown();
+  void InitializeListenersForOpen(
+      SharedState* shared,
+      const std::vector<ColumnFamilyDescriptor>& cf_descriptors);
 
  protected:
   static int GetMinInjectedErrorCount(int error_count_1, int error_count_2) {
@@ -87,39 +106,34 @@ class StressTest {
     }
   }
 
-  void UpdateIfInitialWriteFails(Env* db_stress_env, const Status& write_s,
+  void UpdateIfInitialWriteFails(Env* env, const Status& write_s,
                                  Status* initial_write_s,
                                  bool* initial_wal_write_may_succeed,
                                  uint64_t* wait_for_recover_start_time,
                                  bool commit_bypass_memtable = false) {
-    assert(db_stress_env && initial_write_s && initial_wal_write_may_succeed &&
+    assert(env && initial_write_s && initial_wal_write_may_succeed &&
            wait_for_recover_start_time);
-    // Only update `initial_write_s`, `initial_wal_write_may_succeed` when the
-    // first write fails
     if (!write_s.ok() && (*initial_write_s).ok()) {
       *initial_write_s = write_s;
-      // With commit_bypass_memtable, we create a new WAL after WAL write
-      // succeeds, that wal creation may fail due to injected error. So the
-      // initial wal write may succeed even if status is failed to write to wal
       *initial_wal_write_may_succeed =
           commit_bypass_memtable ||
           !FaultInjectionTestFS::IsFailedToWriteToWALError(*initial_write_s);
-      *wait_for_recover_start_time = db_stress_env->NowMicros();
+      *wait_for_recover_start_time = env->NowMicros();
     }
   }
 
-  void PrintWriteRecoveryWaitTimeIfNeeded(Env* db_stress_env,
+  void PrintWriteRecoveryWaitTimeIfNeeded(Env* env,
                                           const Status& initial_write_s,
                                           bool initial_wal_write_may_succeed,
                                           uint64_t wait_for_recover_start_time,
                                           const std::string& thread_name) {
-    assert(db_stress_env);
+    assert(env);
     bool waited_for_recovery = !initial_write_s.ok() &&
                                IsErrorInjectedAndRetryable(initial_write_s) &&
                                initial_wal_write_may_succeed;
     if (waited_for_recovery) {
       uint64_t elapsed_sec =
-          (db_stress_env->NowMicros() - wait_for_recover_start_time) / 1000000;
+          (env->NowMicros() - wait_for_recover_start_time) / 1000000;
       if (elapsed_sec > 10) {
         fprintf(stdout,
                 "%s thread slept to wait for write recovery for "
@@ -306,6 +320,35 @@ class StressTest {
     kLastOpSeekToLast
   };
 
+  // Enum used to track MANIFEST verification mode during DB reopen
+  enum ManifestVerifyMode {
+    MANIFEST_VERIFY_NONE,
+    // MANIFEST file should be reused (same file number), CURRENT should not
+    // change. Used when reuse_manifest_on_open=1. Warnings are logged on
+    // failure but test continues.
+    MANIFEST_VERIFY_REUSE,
+    // MANIFEST file should be reused AND no recovery writes should occur.
+    // Used when both reuse_manifest_on_open=1 and
+    // optimize_manifest_for_recovery=1. Warnings are logged on failure but test
+    // continues.
+    MANIFEST_VERIFY_NO_WRITE,
+    // Strict verification: MANIFEST file must be reused with ZERO writes
+    // (complete avoidance). Used when ALL conditions for complete avoidance
+    // are met. Failures are FATAL and will terminate the test.
+    // Conditions for STRICT mode:
+    // - Both reuse_manifest_on_open=1 and optimize_manifest_for_recovery=1
+    // - Not in best_efforts_recovery mode
+    // - avoid_flush_during_recovery=true (no flush during recovery)
+    // - write_dbid_to_manifest=0 (no DB_ID write on open)
+    // - metadata_write_fault_one_in=0 (no fault injection)
+    // - open_metadata_write_fault_one_in=0 (no fault injection)
+    // - MANIFEST not corrupted, not at size limit
+    // Note: avoid_flush_during_shutdown is NOT required. If it leaves data
+    // in WAL but avoid_flush_during_recovery=true prevents flushing it,
+    // MANIFEST still won't be written.
+    MANIFEST_VERIFY_STRICT
+  };
+
   // Compare the two iterator, iter and cmp_iter are in the same position,
   // unless iter might be made invalidate or undefined because of
   // upper or lower bounds, or prefix extractor.
@@ -416,7 +459,20 @@ class StressTest {
 
   void CleanUpColumnFamilies();
 
-  std::shared_ptr<Cache> cache_;
+  void RecordManifestStateBeforeReopen();
+  void VerifyManifestNotRewritten();
+
+  int db_index_;
+  std::string db_label_;
+  std::string db_path_;
+  std::string expected_values_path_;
+  std::string secondaries_path_;
+  // Wraps raw_env->GetFileSystem(). See DbStressFSWrapper.
+  std::shared_ptr<DbStressFSWrapper> db_stress_fs_;
+  // Wraps db_stress_fs_ when NeedsFaultInjection(). Null otherwise.
+  std::shared_ptr<FaultInjectionTestFS> db_fault_injection_fs_;
+  // Wraps the outermost FS above. Set as options_.env for all DB I/O.
+  std::unique_ptr<CompositeEnvWrapper> db_env_;
   std::shared_ptr<Cache> compressed_cache_;
   std::shared_ptr<const FilterPolicy> filter_policy_;
   std::unique_ptr<DB> db_owner_;
@@ -442,6 +498,12 @@ class StressTest {
   std::unique_ptr<DB> secondary_db_;
   std::vector<ColumnFamilyHandle*> secondary_cfhs_;
   bool is_db_stopped_;
+
+  // MANIFEST verification state for reopen
+  ManifestVerifyMode manifest_verify_mode_;
+  uint64_t manifest_file_number_before_reopen_;
+  uint64_t manifest_file_size_before_reopen_;
+  std::string current_file_content_before_reopen_;
 };
 
 // Load options from OPTIONS file and populate `options`.

db_stress_tool/db_stress_tool.cc

@@ -22,6 +22,7 @@
 
 #ifdef GFLAGS
 #include <iostream>
+#include <thread>
 
 #include "db_stress_tool/db_stress_common.h"
 #include "db_stress_tool/db_stress_driver.h"
@@ -33,11 +34,79 @@
 namespace ROCKSDB_NAMESPACE {
 namespace {
 static std::shared_ptr<ROCKSDB_NAMESPACE::Env> env_guard;
-static std::shared_ptr<ROCKSDB_NAMESPACE::Env> env_wrapper_guard;
 static std::shared_ptr<ROCKSDB_NAMESPACE::Env> legacy_env_wrapper_guard;
-static std::shared_ptr<ROCKSDB_NAMESPACE::CompositeEnvWrapper>
-    dbsl_env_wrapper_guard;
-static std::shared_ptr<CompositeEnvWrapper> fault_env_guard;
+// Raw pointers for signal-safe crash callback. Signal handlers can only
+// access file-static/global variables; can't capture StressTest instances.
+static std::vector<ROCKSDB_NAMESPACE::FaultInjectionTestFS*>
+    fault_fs_for_crash_report;
+
+int ValidateNumDbsFlags() {
+  if (FLAGS_num_dbs < 1) {
+    fprintf(stderr, "Error: --num_dbs must be >= 1\n");
+    return 1;
+  }
+  if (FLAGS_num_dbs > 1) {
+    if (FLAGS_clear_column_family_one_in > 0) {
+      fprintf(stderr,
+              "Error: --num_dbs > 1 incompatible with "
+              "--clear_column_family_one_in\n");
+      return 1;
+    }
+    if (FLAGS_test_multi_ops_txns) {
+      fprintf(stderr,
+              "Error: --num_dbs > 1 incompatible with "
+              "--test_multi_ops_txns\n");
+      return 1;
+    }
+  }
+  return 0;
+}
+
+int DestroyAllDbs() {
+  bool all_ok = true;
+  const int num_dbs = FLAGS_num_dbs;
+  auto destroy_one = [&](const std::string& db_path) {
+    Status s = DbStressDestroyDb(db_path);
+    if (s.ok()) {
+      fprintf(stdout, "Successfully destroyed db at %s\n", db_path.c_str());
+    } else {
+      fprintf(stderr, "Failed to destroy db at %s: %s\n", db_path.c_str(),
+              s.ToString().c_str());
+      all_ok = false;
+    }
+  };
+  if (num_dbs == 1) {
+    destroy_one(FLAGS_db);
+  } else {
+    for (int i = 0; i < num_dbs; i++) {
+      destroy_one(FLAGS_db + "/db_" + std::to_string(i));
+    }
+    DestroyDir(raw_env, FLAGS_db);
+  }
+  return all_ok ? 0 : 1;
+}
+
+void RegisterCrashCallbacks(
+    const std::vector<std::unique_ptr<StressTest>>& stress_tests, int num_dbs) {
+  fault_fs_for_crash_report.resize(num_dbs, nullptr);
+  bool any_fault_fs = false;
+  for (int i = 0; i < num_dbs; i++) {
+    fault_fs_for_crash_report[i] =
+        stress_tests[i]->GetDbFaultInjectionFs().get();
+    if (fault_fs_for_crash_report[i]) {
+      any_fault_fs = true;
+    }
+  }
+  if (any_fault_fs) {
+    port::RegisterCrashCallback([]() {
+      for (auto* fs : fault_fs_for_crash_report) {
+        if (fs) {
+          fs->PrintRecentInjectedErrors();
+        }
+      }
+    });
+  }
+}
 
 int ReturnFlagValidationError(const char* message) {
   std::cerr << "Error: " << message << '\n';
@@ -67,8 +136,6 @@ int db_stress_tool(int argc, char** argv) {
       StringToCompressionType(FLAGS_bottommost_compression_type.c_str());
   checksum_type_e = StringToChecksumType(FLAGS_checksum_type.c_str());
 
-  Env* raw_env;
-
   int env_opts = !FLAGS_env_uri.empty() + !FLAGS_fs_uri.empty();
   if (env_opts > 1) {
     fprintf(stderr, "Error: --env_uri and --fs_uri are mutually exclusive\n");
@@ -82,57 +149,10 @@ int db_stress_tool(int argc, char** argv) {
             s.ToString().c_str());
     exit(1);
   }
-  dbsl_env_wrapper_guard = std::make_shared<CompositeEnvWrapper>(raw_env);
-  db_stress_listener_env = dbsl_env_wrapper_guard.get();
 
-  if (FLAGS_open_metadata_read_fault_one_in ||
-      FLAGS_open_metadata_write_fault_one_in || FLAGS_open_read_fault_one_in ||
-      FLAGS_open_write_fault_one_in || FLAGS_metadata_read_fault_one_in ||
-      FLAGS_metadata_write_fault_one_in || FLAGS_read_fault_one_in ||
-      FLAGS_write_fault_one_in || FLAGS_sync_fault_injection) {
-    FaultInjectionTestFS* fs =
-        new FaultInjectionTestFS(raw_env->GetFileSystem());
-    fault_fs_guard.reset(fs);
-    // Info logs are debugging artifacts, so exclude them from fault injection
-    // and keep error accounting focused on DB data and metadata.
-    fault_fs_guard->SetFileTypesExcludedFromFaultInjection(
-        {FileType::kInfoLogFile});
-    // Set it to direct writable here to initially bypass any fault injection
-    // during DB open This will correspondingly be overwritten in
-    // StressTest::Open() for open fault injection and in RunStressTestImpl()
-    // for proper fault injection setup.
-    fault_fs_guard->SetFilesystemDirectWritable(true);
-    fault_env_guard =
-        std::make_shared<CompositeEnvWrapper>(raw_env, fault_fs_guard);
-    raw_env = fault_env_guard.get();
-
-    // Register a crash callback so that recently injected errors are
-    // printed to stderr when the process crashes (SIGABRT, SIGSEGV, etc.).
-    // This helps diagnose stress test failures caused by fault injection.
-    port::RegisterCrashCallback([]() {
-      if (fault_fs_guard) {
-        fault_fs_guard->PrintRecentInjectedErrors();
-      }
-    });
-  }
-
-  auto db_stress_fs =
-      std::make_shared<DbStressFSWrapper>(raw_env->GetFileSystem());
-  env_wrapper_guard =
-      std::make_shared<CompositeEnvWrapper>(raw_env, db_stress_fs);
-  db_stress_env = env_wrapper_guard.get();
-
-  // Handle --destroy_db_and_exit early, before other option validation
+  // Handle --destroy_db_and_exit early
   if (FLAGS_destroy_db_and_exit) {
-    s = DbStressDestroyDb(FLAGS_db);
-    if (s.ok()) {
-      fprintf(stdout, "Successfully destroyed db at %s\n", FLAGS_db.c_str());
-      return 0;
-    } else {
-      fprintf(stderr, "Failed to destroy db at %s: %s\n", FLAGS_db.c_str(),
-              s.ToString().c_str());
-      return 1;
-    }
+    return DestroyAllDbs();
   }
 
   // Handle --delete_dir_and_exit early, before other option validation
@@ -149,14 +169,21 @@ int db_stress_tool(int argc, char** argv) {
     }
   }
 
+  {
+    int rc = ValidateNumDbsFlags();
+    if (rc != 0) {
+      return rc;
+    }
+  }
+
   FLAGS_rep_factory = StringToRepFactory(FLAGS_memtablerep.c_str());
 
   // The number of background threads should be at least as much the
   // max number of concurrent compactions.
-  db_stress_env->SetBackgroundThreads(FLAGS_max_background_compactions,
-                                      ROCKSDB_NAMESPACE::Env::Priority::LOW);
-  db_stress_env->SetBackgroundThreads(FLAGS_num_bottom_pri_threads,
-                                      ROCKSDB_NAMESPACE::Env::Priority::BOTTOM);
+  raw_env->SetBackgroundThreads(FLAGS_max_background_compactions,
+                                ROCKSDB_NAMESPACE::Env::Priority::LOW);
+  raw_env->SetBackgroundThreads(FLAGS_num_bottom_pri_threads,
+                                ROCKSDB_NAMESPACE::Env::Priority::BOTTOM);
   if (FLAGS_prefixpercent > 0 && FLAGS_prefix_size < 0) {
     fprintf(stderr,
             "Error: prefixpercent is non-zero while prefix_size is "
@@ -356,42 +383,75 @@ int db_stress_tool(int argc, char** argv) {
   // Choose a location for the test database if none given with --db=<path>
   if (FLAGS_db.empty()) {
     std::string default_db_path;
-    db_stress_env->GetTestDirectory(&default_db_path);
+    raw_env->GetTestDirectory(&default_db_path);
     default_db_path += "/dbstress";
     FLAGS_db = default_db_path;
   }
 
-  // Now that FLAGS_db is resolved, set the fault injection log file path
-  // so that PrintAll() writes to a file instead of stderr (signal-safe).
-  // Store the log in TEST_TMPDIR (outside the DB directory) so it survives
-  // DB reopen (which cleans untracked files) and gets included in the
-  // sandcastle db.tar.gz artifact for post-failure analysis.
-  if (fault_fs_guard) {
-    std::string log_dir;
-    const char* test_tmpdir = getenv("TEST_TMPDIR");
-    if (test_tmpdir && test_tmpdir[0] != '\0') {
-      log_dir = test_tmpdir;
-    } else {
-      log_dir = "/tmp";
-    }
-    std::string log_path = log_dir + "/fault_injection_" +
-                           std::to_string(getpid()) + "_" +
-                           std::to_string(time(nullptr)) + ".log";
-    fault_fs_guard->SetInjectedErrorLogPath(log_path);
-  }
-
-  if ((FLAGS_test_secondary || FLAGS_continuous_verification_interval > 0) &&
-      FLAGS_secondaries_base.empty()) {
-    std::string default_secondaries_path;
-    db_stress_env->GetTestDirectory(&default_secondaries_path);
-    default_secondaries_path += "/dbstress_secondaries";
-    s = db_stress_env->CreateDirIfMissing(default_secondaries_path);
+  // For num_dbs=1: --db and --expected_values_dir are paths used as-is.
+  // For num_dbs>1: they are parent directories; C++ creates db_0/, db_1/, ...
+  // subdirs underneath. DB::Open(create_if_missing) creates each DB dir.
+  // Python (db_crashtest.py) also creates EV dirs; C++ creates them here as a
+  // fallback for direct CLI usage.
+  const int num_dbs = FLAGS_num_dbs;
+  if (num_dbs > 1) {
+    s = raw_env->CreateDirIfMissing(FLAGS_db);
     if (!s.ok()) {
-      fprintf(stderr, "Failed to create directory %s: %s\n",
-              default_secondaries_path.c_str(), s.ToString().c_str());
+      fprintf(stderr, "Failed to create directory %s: %s\n", FLAGS_db.c_str(),
+              s.ToString().c_str());
       exit(1);
     }
-    FLAGS_secondaries_base = default_secondaries_path;
+  }
+  std::vector<std::string> db_paths;
+  std::vector<std::string> ev_paths;
+  for (int i = 0; i < num_dbs; i++) {
+    std::string suffix = (num_dbs == 1) ? "" : "/db_" + std::to_string(i);
+    db_paths.push_back(FLAGS_db + suffix);
+    if (!FLAGS_expected_values_dir.empty()) {
+      std::string ep = FLAGS_expected_values_dir + suffix;
+      s = Env::Default()->CreateDirIfMissing(ep);
+      if (!s.ok()) {
+        fprintf(stderr, "Failed to create directory %s: %s\n", ep.c_str(),
+                s.ToString().c_str());
+        exit(1);
+      }
+      ev_paths.push_back(std::move(ep));
+    }
+  }
+  if (ev_paths.empty()) {
+    ev_paths.resize(num_dbs);
+  }
+
+  // Secondary paths: C++ owns creation entirely.
+  std::vector<std::string> sec_paths;
+  if (FLAGS_test_secondary || FLAGS_continuous_verification_interval > 0) {
+    std::string sec_parent;
+    if (!FLAGS_secondaries_base.empty()) {
+      sec_parent = FLAGS_secondaries_base;
+    } else {
+      raw_env->GetTestDirectory(&sec_parent);
+      sec_parent += "/dbstress_secondaries";
+    }
+    s = raw_env->CreateDirIfMissing(sec_parent);
+    if (!s.ok()) {
+      fprintf(stderr, "Failed to create directory %s: %s\n", sec_parent.c_str(),
+              s.ToString().c_str());
+      exit(1);
+    }
+    for (int i = 0; i < num_dbs; i++) {
+      std::string suffix = (num_dbs == 1) ? "" : "/db_" + std::to_string(i);
+      std::string sec_path = sec_parent + suffix;
+      s = raw_env->CreateDirIfMissing(sec_path);
+      if (!s.ok()) {
+        fprintf(stderr, "Failed to create directory %s: %s\n", sec_path.c_str(),
+                s.ToString().c_str());
+        exit(1);
+      }
+      sec_paths.push_back(std::move(sec_path));
+    }
+  }
+  if (sec_paths.empty()) {
+    sec_paths.resize(num_dbs);
   }
 
   if (FLAGS_best_efforts_recovery &&
@@ -511,25 +571,72 @@ int db_stress_tool(int argc, char** argv) {
     key_gen_ctx.weights.emplace_back(key_gen_ctx.window -
                                      keys_per_level * (levels - 1));
   }
-  std::unique_ptr<ROCKSDB_NAMESPACE::SharedState> shared;
-  std::unique_ptr<ROCKSDB_NAMESPACE::StressTest> stress;
-  if (FLAGS_test_cf_consistency) {
-    stress.reset(CreateCfConsistencyStressTest());
-  } else if (FLAGS_test_batches_snapshots) {
-    stress.reset(CreateBatchedOpsStressTest());
-  } else if (FLAGS_test_multi_ops_txns) {
-    stress.reset(CreateMultiOpsTxnsStressTest());
-  } else {
-    stress.reset(CreateNonBatchedOpsStressTest());
-  }
-  // Initialize the Zipfian pre-calculated array
+  // Initialize shared resources (hot-key generator, block cache, WBM,
+  // rate limiter) once so that all DB instances share them.
   InitializeHotKeyGenerator(FLAGS_hot_key_alpha);
-  shared.reset(new SharedState(db_stress_env, stress.get()));
-  bool run_stress_test = RunStressTest(shared.get());
-  // Close DB in CleanUp() before destructor to prevent race between destructor
-  // and operations in listener callbacks (e.g. MultiOpsTxnsStressListener).
-  stress->CleanUp();
-  return run_stress_test ? 0 : 1;
+  block_cache =
+      StressTest::NewCache(FLAGS_cache_size, FLAGS_cache_numshardbits);
+  if (FLAGS_use_write_buffer_manager) {
+    wbm = std::make_shared<WriteBufferManager>(FLAGS_db_write_buffer_size,
+                                               block_cache);
+  }
+  if (FLAGS_rate_limiter_bytes_per_sec > 0) {
+    rate_limiter.reset(NewGenericRateLimiter(
+        FLAGS_rate_limiter_bytes_per_sec, 1000 /* refill_period_us */,
+        10 /* fairness */,
+        FLAGS_rate_limit_bg_reads ? RateLimiter::Mode::kReadsOnly
+                                  : RateLimiter::Mode::kWritesOnly));
+  }
+
+  // Phase 1: Create StressTest instances (one per DB).
+  std::vector<std::unique_ptr<StressTest>> stress_tests(num_dbs);
+  for (int i = 0; i < num_dbs; i++) {
+    if (FLAGS_test_cf_consistency) {
+      stress_tests[i].reset(CreateCfConsistencyStressTest(
+          i, db_paths[i], ev_paths[i], sec_paths[i]));
+    } else if (FLAGS_test_batches_snapshots) {
+      stress_tests[i].reset(CreateBatchedOpsStressTest(
+          i, db_paths[i], ev_paths[i], sec_paths[i]));
+    } else if (FLAGS_test_multi_ops_txns) {
+      stress_tests[i].reset(CreateMultiOpsTxnsStressTest(
+          i, db_paths[i], ev_paths[i], sec_paths[i]));
+    } else {
+      stress_tests[i].reset(CreateNonBatchedOpsStressTest(
+          i, db_paths[i], ev_paths[i], sec_paths[i]));
+    }
+  }
+
+  RegisterCrashCallbacks(stress_tests, num_dbs);
+
+  // Phase 2: Create SharedState for every DB before any worker thread starts.
+  std::vector<std::unique_ptr<SharedState>> shared_states(num_dbs);
+  for (int i = 0; i < num_dbs; i++) {
+    shared_states[i].reset(new SharedState(raw_env, stress_tests[i].get()));
+  }
+
+  // Phase 3: Launch each DB's stress test on its own thread.
+  std::vector<int> results(num_dbs, 0);
+  std::vector<std::thread> stress_test_runners;
+  stress_test_runners.reserve(num_dbs);
+  for (int i = 0; i < num_dbs; i++) {
+    stress_test_runners.emplace_back([i, &results, &shared_states]() {
+      results[i] = RunStressTest(shared_states[i].get()) ? 1 : 0;
+    });
+  }
+
+  // Phase 4: Wait for all DB threads, collect results, clean up.
+  bool all_passed = true;
+  for (int i = 0; i < num_dbs; i++) {
+    stress_test_runners[i].join();
+    if (!results[i]) {
+      all_passed = false;
+    }
+    stress_tests[i]->CleanUp();
+  }
+  for (auto& fs : fault_fs_for_crash_report) {
+    fs = nullptr;
+  }
+  return all_passed ? 0 : 1;
 }
 
 }  // namespace ROCKSDB_NAMESPACE

db_stress_tool/multi_ops_txns_stress.cc

@@ -40,7 +40,7 @@ DEFINE_int32(clear_wp_commit_cache_one_in, 0,
              "write-unprepared transactions.");
 
 extern "C" bool rocksdb_write_prepared_TEST_ShouldClearCommitCache(void) {
-  static Random rand(static_cast<uint32_t>(db_stress_env->NowMicros()));
+  static Random rand(static_cast<uint32_t>(raw_env->NowMicros()));
   return FLAGS_clear_wp_commit_cache_one_in > 0 &&
          rand.OneIn(FLAGS_clear_wp_commit_cache_one_in);
 }
@@ -1413,7 +1413,7 @@ Status MultiOpsTxnsStressTest::CommitAndCreateTimestampedSnapshotIfNeeded(
   Status s;
   if (FLAGS_create_timestamped_snapshot_one_in > 0 &&
       thread->rand.OneInOpt(FLAGS_create_timestamped_snapshot_one_in)) {
-    uint64_t ts = db_stress_env->NowNanos();
+    uint64_t ts = raw_env->NowNanos();
     std::shared_ptr<const Snapshot> snapshot;
     s = txn.CommitAndTryCreateSnapshot(/*notifier=*/nullptr, ts, &snapshot);
   } else {
@@ -1428,7 +1428,7 @@ Status MultiOpsTxnsStressTest::CommitAndCreateTimestampedSnapshotIfNeeded(
   assert(txn_db_);
   if (FLAGS_create_timestamped_snapshot_one_in > 0 &&
       thread->rand.OneInOpt(50000)) {
-    uint64_t now = db_stress_env->NowNanos();
+    uint64_t now = raw_env->NowNanos();
     constexpr uint64_t time_diff = static_cast<uint64_t>(1000) * 1000 * 1000;
     txn_db_->ReleaseTimestampedSnapshotsOlderThan(now - time_diff);
   }
@@ -1767,8 +1767,11 @@ void MultiOpsTxnsStressTest::ScanExistingDb(SharedState* shared, int threads) {
   }
 }
 
-StressTest* CreateMultiOpsTxnsStressTest() {
-  return new MultiOpsTxnsStressTest();
+StressTest* CreateMultiOpsTxnsStressTest(int db_index,
+                                         const std::string& db_path,
+                                         const std::string& ev_path,
+                                         const std::string& sec_path) {
+  return new MultiOpsTxnsStressTest(db_index, db_path, ev_path, sec_path);
 }
 
 void CheckAndSetOptionsForMultiOpsTxnStressTest() {

db_stress_tool/multi_ops_txns_stress.h

@@ -9,6 +9,7 @@
 
 #ifdef GFLAGS
 #include "db_stress_tool/db_stress_common.h"
+#include "util/atomic.h"
 
 namespace ROCKSDB_NAMESPACE {
 
@@ -191,7 +192,10 @@ class MultiOpsTxnsStressTest : public StressTest {
     uint32_t c_{0};
   };
 
-  MultiOpsTxnsStressTest() {}
+  MultiOpsTxnsStressTest(int db_index, const std::string& db_path,
+                         const std::string& ev_path,
+                         const std::string& sec_path)
+      : StressTest(db_index, db_path, ev_path, sec_path) {}
 
   ~MultiOpsTxnsStressTest() override {}
 
@@ -423,11 +427,16 @@ class MultiOpsTxnsStressListener : public EventListener {
 
   ~MultiOpsTxnsStressListener() override {}
 
+  void OnDBShutdownBegin(DB* /*db*/) override { shutting_down_.Store(true); }
+
   void OnFlushCompleted(DB* db, const FlushJobInfo& info) override {
     assert(db);
 #ifdef NDEBUG
     (void)db;
 #endif
+    if (shutting_down_.Load()) {
+      return;
+    }
     assert(info.cf_id == 0);
     const ReadOptions read_options(Env::IOActivity::kFlush);
     stress_test_->VerifyPkSkFast(read_options, info.job_id);
@@ -438,6 +447,9 @@ class MultiOpsTxnsStressListener : public EventListener {
 #ifdef NDEBUG
     (void)db;
 #endif
+    if (shutting_down_.Load()) {
+      return;
+    }
     assert(info.cf_id == 0);
     const ReadOptions read_options(Env::IOActivity::kCompaction);
     stress_test_->VerifyPkSkFast(read_options, info.job_id);
@@ -445,6 +457,7 @@ class MultiOpsTxnsStressListener : public EventListener {
 
  private:
   MultiOpsTxnsStressTest* const stress_test_ = nullptr;
+  Atomic<bool> shutting_down_{false};
 };
 
 }  // namespace ROCKSDB_NAMESPACE

db_stress_tool/no_batched_ops_stress.cc

@@ -24,7 +24,10 @@
 namespace ROCKSDB_NAMESPACE {
 class NonBatchedOpsStressTest : public StressTest {
  public:
-  NonBatchedOpsStressTest() = default;
+  NonBatchedOpsStressTest(int db_index, const std::string& db_path,
+                          const std::string& ev_path,
+                          const std::string& sec_path)
+      : StressTest(db_index, db_path, ev_path, sec_path) {}
 
   virtual ~NonBatchedOpsStressTest() = default;
 
@@ -265,20 +268,20 @@ class NonBatchedOpsStressTest : public StressTest {
             std::string from_db;
 
             // Temporarily disable error injection to verify the secondary
-            if (fault_fs_guard) {
-              fault_fs_guard->DisableThreadLocalErrorInjection(
+            if (db_fault_injection_fs_) {
+              db_fault_injection_fs_->DisableThreadLocalErrorInjection(
                   FaultInjectionIOType::kRead);
-              fault_fs_guard->DisableThreadLocalErrorInjection(
+              db_fault_injection_fs_->DisableThreadLocalErrorInjection(
                   FaultInjectionIOType::kMetadataRead);
             }
 
             s = secondary_db_->Get(options, secondary_cfhs_[cf], key, &from_db);
 
             // Re-enable error injection after verifying the secondary
-            if (fault_fs_guard) {
-              fault_fs_guard->EnableThreadLocalErrorInjection(
+            if (db_fault_injection_fs_) {
+              db_fault_injection_fs_->EnableThreadLocalErrorInjection(
                   FaultInjectionIOType::kRead);
-              fault_fs_guard->EnableThreadLocalErrorInjection(
+              db_fault_injection_fs_->EnableThreadLocalErrorInjection(
                   FaultInjectionIOType::kMetadataRead);
             }
 
@@ -688,10 +691,10 @@ class NonBatchedOpsStressTest : public StressTest {
     bool read_older_ts = MaybeUseOlderTimestampForPointLookup(
         thread, read_ts_str, read_ts_slice, read_opts_copy);
 
-    if (fault_fs_guard) {
-      fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
+    if (db_fault_injection_fs_) {
+      db_fault_injection_fs_->GetAndResetInjectedThreadLocalErrorCount(
           FaultInjectionIOType::kRead);
-      fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
+      db_fault_injection_fs_->GetAndResetInjectedThreadLocalErrorCount(
           FaultInjectionIOType::kMetadataRead);
       SharedState::ignore_read_error = false;
     }
@@ -703,11 +706,11 @@ class NonBatchedOpsStressTest : public StressTest {
         thread->shared->Get(rand_column_families[0], rand_keys[0]);
 
     int injected_error_count = 0;
-    if (fault_fs_guard) {
+    if (db_fault_injection_fs_) {
       injected_error_count = GetMinInjectedErrorCount(
-          fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
+          db_fault_injection_fs_->GetAndResetInjectedThreadLocalErrorCount(
               FaultInjectionIOType::kRead),
-          fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
+          db_fault_injection_fs_->GetAndResetInjectedThreadLocalErrorCount(
               FaultInjectionIOType::kMetadataRead));
       if (!SharedState::ignore_read_error && injected_error_count > 0 &&
           (s.ok() || s.IsNotFound())) {
@@ -716,9 +719,9 @@ class NonBatchedOpsStressTest : public StressTest {
         MutexLock l(thread->shared->GetMutex());
         fprintf(stderr, "Didn't get expected error from Get\n");
         fprintf(stderr, "Callstack that injected the fault\n");
-        fault_fs_guard->PrintInjectedThreadLocalErrorBacktrace(
+        db_fault_injection_fs_->PrintInjectedThreadLocalErrorBacktrace(
             FaultInjectionIOType::kRead);
-        fault_fs_guard->PrintInjectedThreadLocalErrorBacktrace(
+        db_fault_injection_fs_->PrintInjectedThreadLocalErrorBacktrace(
             FaultInjectionIOType::kMetadataRead);
         std::terminate();
       }
@@ -821,10 +824,10 @@ class NonBatchedOpsStressTest : public StressTest {
     std::unique_ptr<Transaction> txn;
     if (use_txn) {
       // TODO(hx235): test fault injection with MultiGet() with transactions
-      if (fault_fs_guard) {
-        fault_fs_guard->DisableThreadLocalErrorInjection(
+      if (db_fault_injection_fs_) {
+        db_fault_injection_fs_->DisableThreadLocalErrorInjection(
             FaultInjectionIOType::kRead);
-        fault_fs_guard->DisableThreadLocalErrorInjection(
+        db_fault_injection_fs_->DisableThreadLocalErrorInjection(
             FaultInjectionIOType::kMetadataRead);
       }
       WriteOptions wo;
@@ -850,20 +853,20 @@ class NonBatchedOpsStressTest : public StressTest {
     int injected_error_count = 0;
 
     if (!use_txn) {
-      if (fault_fs_guard) {
-        fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
+      if (db_fault_injection_fs_) {
+        db_fault_injection_fs_->GetAndResetInjectedThreadLocalErrorCount(
             FaultInjectionIOType::kRead);
-        fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
+        db_fault_injection_fs_->GetAndResetInjectedThreadLocalErrorCount(
             FaultInjectionIOType::kMetadataRead);
         SharedState::ignore_read_error = false;
       }
       db_->MultiGet(readoptionscopy, cfh, num_keys, keys.data(), values.data(),
                     statuses.data());
-      if (fault_fs_guard) {
+      if (db_fault_injection_fs_) {
         injected_error_count = GetMinInjectedErrorCount(
-            fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
+            db_fault_injection_fs_->GetAndResetInjectedThreadLocalErrorCount(
                 FaultInjectionIOType::kRead),
-            fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
+            db_fault_injection_fs_->GetAndResetInjectedThreadLocalErrorCount(
                 FaultInjectionIOType::kMetadataRead));
 
         if (injected_error_count > 0) {
@@ -883,9 +886,9 @@ class NonBatchedOpsStressTest : public StressTest {
                     "num_keys %zu Expected %d errors, seen at least %d\n",
                     num_keys, injected_error_count, stat_nok_nfound);
             fprintf(stderr, "Callstack that injected the fault\n");
-            fault_fs_guard->PrintInjectedThreadLocalErrorBacktrace(
+            db_fault_injection_fs_->PrintInjectedThreadLocalErrorBacktrace(
                 FaultInjectionIOType::kRead);
-            fault_fs_guard->PrintInjectedThreadLocalErrorBacktrace(
+            db_fault_injection_fs_->PrintInjectedThreadLocalErrorBacktrace(
                 FaultInjectionIOType::kMetadataRead);
             std::terminate();
           }
@@ -952,10 +955,10 @@ class NonBatchedOpsStressTest : public StressTest {
             const Status& s,
             const std::optional<ExpectedValue>& ryw_expected_value) -> bool {
       //  Temporarily disable error injection for verification
-      if (fault_fs_guard) {
-        fault_fs_guard->DisableThreadLocalErrorInjection(
+      if (db_fault_injection_fs_) {
+        db_fault_injection_fs_->DisableThreadLocalErrorInjection(
             FaultInjectionIOType::kRead);
-        fault_fs_guard->DisableThreadLocalErrorInjection(
+        db_fault_injection_fs_->DisableThreadLocalErrorInjection(
             FaultInjectionIOType::kMetadataRead);
       }
 
@@ -1049,10 +1052,10 @@ class NonBatchedOpsStressTest : public StressTest {
       }
 
       // Enable back error injection disbled for checking results
-      if (fault_fs_guard) {
-        fault_fs_guard->DisableThreadLocalErrorInjection(
+      if (db_fault_injection_fs_) {
+        db_fault_injection_fs_->DisableThreadLocalErrorInjection(
             FaultInjectionIOType::kRead);
-        fault_fs_guard->DisableThreadLocalErrorInjection(
+        db_fault_injection_fs_->DisableThreadLocalErrorInjection(
             FaultInjectionIOType::kMetadataRead);
       }
       return check_multiget_res;
@@ -1091,10 +1094,10 @@ class NonBatchedOpsStressTest : public StressTest {
     if (use_txn) {
       txn->Rollback().PermitUncheckedError();
       // Enable back error injection disbled for transactions
-      if (fault_fs_guard) {
-        fault_fs_guard->EnableThreadLocalErrorInjection(
+      if (db_fault_injection_fs_) {
+        db_fault_injection_fs_->EnableThreadLocalErrorInjection(
             FaultInjectionIOType::kRead);
-        fault_fs_guard->EnableThreadLocalErrorInjection(
+        db_fault_injection_fs_->EnableThreadLocalErrorInjection(
             FaultInjectionIOType::kMetadataRead);
       }
     }
@@ -1141,10 +1144,10 @@ class NonBatchedOpsStressTest : public StressTest {
     const ExpectedValue pre_read_expected_value =
         thread->shared->Get(column_family, key);
 
-    if (fault_fs_guard) {
-      fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
+    if (db_fault_injection_fs_) {
+      db_fault_injection_fs_->GetAndResetInjectedThreadLocalErrorCount(
           FaultInjectionIOType::kRead);
-      fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
+      db_fault_injection_fs_->GetAndResetInjectedThreadLocalErrorCount(
           FaultInjectionIOType::kMetadataRead);
       SharedState::ignore_read_error = false;
     }
@@ -1164,11 +1167,11 @@ class NonBatchedOpsStressTest : public StressTest {
         thread->shared->Get(column_family, key);
 
     int injected_error_count = 0;
-    if (fault_fs_guard) {
+    if (db_fault_injection_fs_) {
       injected_error_count = GetMinInjectedErrorCount(
-          fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
+          db_fault_injection_fs_->GetAndResetInjectedThreadLocalErrorCount(
               FaultInjectionIOType::kRead),
-          fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
+          db_fault_injection_fs_->GetAndResetInjectedThreadLocalErrorCount(
               FaultInjectionIOType::kMetadataRead));
       if (!SharedState::ignore_read_error && injected_error_count > 0 &&
           (s.ok() || s.IsNotFound())) {
@@ -1177,9 +1180,9 @@ class NonBatchedOpsStressTest : public StressTest {
         MutexLock l(thread->shared->GetMutex());
         fprintf(stderr, "Didn't get expected error from GetEntity\n");
         fprintf(stderr, "Callstack that injected the fault\n");
-        fault_fs_guard->PrintInjectedThreadLocalErrorBacktrace(
+        db_fault_injection_fs_->PrintInjectedThreadLocalErrorBacktrace(
             FaultInjectionIOType::kRead);
-        fault_fs_guard->PrintInjectedThreadLocalErrorBacktrace(
+        db_fault_injection_fs_->PrintInjectedThreadLocalErrorBacktrace(
             FaultInjectionIOType::kMetadataRead);
         std::terminate();
       }
@@ -1281,10 +1284,10 @@ class NonBatchedOpsStressTest : public StressTest {
     if (FLAGS_use_txn) {
       // TODO(hx235): test fault injection with MultiGetEntity() with
       // transactions
-      if (fault_fs_guard) {
-        fault_fs_guard->DisableThreadLocalErrorInjection(
+      if (db_fault_injection_fs_) {
+        db_fault_injection_fs_->DisableThreadLocalErrorInjection(
             FaultInjectionIOType::kRead);
-        fault_fs_guard->DisableThreadLocalErrorInjection(
+        db_fault_injection_fs_->DisableThreadLocalErrorInjection(
             FaultInjectionIOType::kMetadataRead);
       }
       WriteOptions write_options;
@@ -1318,11 +1321,11 @@ class NonBatchedOpsStressTest : public StressTest {
     int injected_error_count = 0;
 
     auto verify_expected_errors = [&](auto get_status) {
-      assert(fault_fs_guard);
+      assert(db_fault_injection_fs_);
       injected_error_count = GetMinInjectedErrorCount(
-          fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
+          db_fault_injection_fs_->GetAndResetInjectedThreadLocalErrorCount(
               FaultInjectionIOType::kRead),
-          fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
+          db_fault_injection_fs_->GetAndResetInjectedThreadLocalErrorCount(
               FaultInjectionIOType::kMetadataRead));
       if (injected_error_count) {
         int stat_nok_nfound = 0;
@@ -1344,9 +1347,9 @@ class NonBatchedOpsStressTest : public StressTest {
           fprintf(stderr, "num_keys %zu Expected %d errors, seen %d\n",
                   num_keys, injected_error_count, stat_nok_nfound);
           fprintf(stderr, "Call stack that injected the fault\n");
-          fault_fs_guard->PrintInjectedThreadLocalErrorBacktrace(
+          db_fault_injection_fs_->PrintInjectedThreadLocalErrorBacktrace(
               FaultInjectionIOType::kRead);
-          fault_fs_guard->PrintInjectedThreadLocalErrorBacktrace(
+          db_fault_injection_fs_->PrintInjectedThreadLocalErrorBacktrace(
               FaultInjectionIOType::kMetadataRead);
           std::terminate();
         }
@@ -1356,10 +1359,10 @@ class NonBatchedOpsStressTest : public StressTest {
     auto check_results = [&](auto get_columns, auto get_status,
                              auto do_extra_check, auto call_get_entity) {
       // Temporarily disable error injection for checking results
-      if (fault_fs_guard) {
-        fault_fs_guard->DisableThreadLocalErrorInjection(
+      if (db_fault_injection_fs_) {
+        db_fault_injection_fs_->DisableThreadLocalErrorInjection(
             FaultInjectionIOType::kRead);
-        fault_fs_guard->DisableThreadLocalErrorInjection(
+        db_fault_injection_fs_->DisableThreadLocalErrorInjection(
             FaultInjectionIOType::kMetadataRead);
       }
       const bool check_get_entity =
@@ -1538,10 +1541,10 @@ class NonBatchedOpsStressTest : public StressTest {
         }
       }
       // Enable back error injection disbled for checking results
-      if (fault_fs_guard) {
-        fault_fs_guard->EnableThreadLocalErrorInjection(
+      if (db_fault_injection_fs_) {
+        db_fault_injection_fs_->EnableThreadLocalErrorInjection(
             FaultInjectionIOType::kRead);
-        fault_fs_guard->EnableThreadLocalErrorInjection(
+        db_fault_injection_fs_->EnableThreadLocalErrorInjection(
             FaultInjectionIOType::kMetadataRead);
       }
     };
@@ -1624,19 +1627,19 @@ class NonBatchedOpsStressTest : public StressTest {
 
       txn->Rollback().PermitUncheckedError();
       // Enable back error injection disbled for transactions
-      if (fault_fs_guard) {
-        fault_fs_guard->EnableThreadLocalErrorInjection(
+      if (db_fault_injection_fs_) {
+        db_fault_injection_fs_->EnableThreadLocalErrorInjection(
             FaultInjectionIOType::kRead);
-        fault_fs_guard->EnableThreadLocalErrorInjection(
+        db_fault_injection_fs_->EnableThreadLocalErrorInjection(
             FaultInjectionIOType::kMetadataRead);
       }
     } else if (FLAGS_use_attribute_group) {
       // AttributeGroup MultiGetEntity verification
 
-      if (fault_fs_guard) {
-        fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
+      if (db_fault_injection_fs_) {
+        db_fault_injection_fs_->GetAndResetInjectedThreadLocalErrorCount(
             FaultInjectionIOType::kRead);
-        fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
+        db_fault_injection_fs_->GetAndResetInjectedThreadLocalErrorCount(
             FaultInjectionIOType::kMetadataRead);
         SharedState::ignore_read_error = false;
       }
@@ -1652,7 +1655,7 @@ class NonBatchedOpsStressTest : public StressTest {
       db_->MultiGetEntity(read_opts_copy, num_keys, key_slices.data(),
                           results.data());
 
-      if (fault_fs_guard) {
+      if (db_fault_injection_fs_) {
         verify_expected_errors(
             [&](size_t i) { return results[i][0].status(); });
       }
@@ -1668,10 +1671,10 @@ class NonBatchedOpsStressTest : public StressTest {
     } else {
       // Non-AttributeGroup MultiGetEntity verification
 
-      if (fault_fs_guard) {
-        fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
+      if (db_fault_injection_fs_) {
+        db_fault_injection_fs_->GetAndResetInjectedThreadLocalErrorCount(
             FaultInjectionIOType::kRead);
-        fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
+        db_fault_injection_fs_->GetAndResetInjectedThreadLocalErrorCount(
             FaultInjectionIOType::kMetadataRead);
         SharedState::ignore_read_error = false;
       }
@@ -1682,7 +1685,7 @@ class NonBatchedOpsStressTest : public StressTest {
       db_->MultiGetEntity(read_opts_copy, cfh, num_keys, key_slices.data(),
                           results.data(), statuses.data());
 
-      if (fault_fs_guard) {
+      if (db_fault_injection_fs_) {
         verify_expected_errors([&](size_t i) { return statuses[i]; });
       }
 
@@ -1733,10 +1736,10 @@ class NonBatchedOpsStressTest : public StressTest {
     MaybeUseOlderTimestampForRangeScan(thread, read_ts_str, read_ts_slice,
                                        ro_copy);
 
-    if (fault_fs_guard) {
-      fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
+    if (db_fault_injection_fs_) {
+      db_fault_injection_fs_->GetAndResetInjectedThreadLocalErrorCount(
           FaultInjectionIOType::kRead);
-      fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
+      db_fault_injection_fs_->GetAndResetInjectedThreadLocalErrorCount(
           FaultInjectionIOType::kMetadataRead);
       SharedState::ignore_read_error = false;
     }
@@ -1798,11 +1801,11 @@ class NonBatchedOpsStressTest : public StressTest {
     }
 
     int injected_error_count = 0;
-    if (fault_fs_guard) {
+    if (db_fault_injection_fs_) {
       injected_error_count = GetMinInjectedErrorCount(
-          fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
+          db_fault_injection_fs_->GetAndResetInjectedThreadLocalErrorCount(
               FaultInjectionIOType::kRead),
-          fault_fs_guard->GetAndResetInjectedThreadLocalErrorCount(
+          db_fault_injection_fs_->GetAndResetInjectedThreadLocalErrorCount(
               FaultInjectionIOType::kMetadataRead));
       if (!SharedState::ignore_read_error && injected_error_count > 0 &&
           s.ok()) {
@@ -1811,9 +1814,9 @@ class NonBatchedOpsStressTest : public StressTest {
         MutexLock l(thread->shared->GetMutex());
         fprintf(stderr, "Didn't get expected error from PrefixScan\n");
         fprintf(stderr, "Callstack that injected the fault\n");
-        fault_fs_guard->PrintInjectedThreadLocalErrorBacktrace(
+        db_fault_injection_fs_->PrintInjectedThreadLocalErrorBacktrace(
             FaultInjectionIOType::kRead);
-        fault_fs_guard->PrintInjectedThreadLocalErrorBacktrace(
+        db_fault_injection_fs_->PrintInjectedThreadLocalErrorBacktrace(
             FaultInjectionIOType::kMetadataRead);
         std::terminate();
       }
@@ -1880,10 +1883,10 @@ class NonBatchedOpsStressTest : public StressTest {
 
     if (FLAGS_verify_before_write) {
       // Temporarily disable error injection for preparation
-      if (fault_fs_guard) {
-        fault_fs_guard->DisableThreadLocalErrorInjection(
+      if (db_fault_injection_fs_) {
+        db_fault_injection_fs_->DisableThreadLocalErrorInjection(
             FaultInjectionIOType::kRead);
-        fault_fs_guard->DisableThreadLocalErrorInjection(
+        db_fault_injection_fs_->DisableThreadLocalErrorInjection(
             FaultInjectionIOType::kMetadataRead);
       }
 
@@ -1894,10 +1897,10 @@ class NonBatchedOpsStressTest : public StressTest {
           /* msg_prefix */ "Pre-Put Get verification", from_db, s);
 
       // Enable back error injection disabled for preparation
-      if (fault_fs_guard) {
-        fault_fs_guard->EnableThreadLocalErrorInjection(
+      if (db_fault_injection_fs_) {
+        db_fault_injection_fs_->EnableThreadLocalErrorInjection(
             FaultInjectionIOType::kRead);
-        fault_fs_guard->EnableThreadLocalErrorInjection(
+        db_fault_injection_fs_->EnableThreadLocalErrorInjection(
             FaultInjectionIOType::kMetadataRead);
       }
       if (!res) {
@@ -2000,7 +2003,7 @@ class NonBatchedOpsStressTest : public StressTest {
               &commit_bypass_memtable);
         }
       }
-      UpdateIfInitialWriteFails(db_stress_env, s, &initial_write_s,
+      UpdateIfInitialWriteFails(raw_env, s, &initial_write_s,
                                 &initial_wal_write_may_succeed,
                                 &wait_for_recover_start_time);
 
@@ -2031,7 +2034,7 @@ class NonBatchedOpsStressTest : public StressTest {
       }
     } else {
       PrintWriteRecoveryWaitTimeIfNeeded(
-          db_stress_env, initial_write_s, initial_wal_write_may_succeed,
+          raw_env, initial_write_s, initial_wal_write_may_succeed,
           wait_for_recover_start_time, "TestPut");
       pending_expected_value.Commit();
       thread->stats.AddBytesForWrites(1, sz);
@@ -2107,7 +2110,7 @@ class NonBatchedOpsStressTest : public StressTest {
               &commit_bypass_memtable);
         }
         UpdateIfInitialWriteFails(
-            db_stress_env, s, &initial_write_s, &initial_wal_write_may_succeed,
+            raw_env, s, &initial_write_s, &initial_wal_write_may_succeed,
             &wait_for_recover_start_time, commit_bypass_memtable);
       } while (!s.ok() && IsErrorInjectedAndRetryable(s) &&
                initial_wal_write_may_succeed);
@@ -2137,7 +2140,7 @@ class NonBatchedOpsStressTest : public StressTest {
         }
       } else {
         PrintWriteRecoveryWaitTimeIfNeeded(
-            db_stress_env, initial_write_s, initial_wal_write_may_succeed,
+            raw_env, initial_write_s, initial_wal_write_may_succeed,
             wait_for_recover_start_time, "TestDelete");
         pending_expected_value.Commit();
         thread->stats.AddDeletes(1);
@@ -2179,7 +2182,7 @@ class NonBatchedOpsStressTest : public StressTest {
               &commit_bypass_memtable);
         }
         UpdateIfInitialWriteFails(
-            db_stress_env, s, &initial_write_s, &initial_wal_write_may_succeed,
+            raw_env, s, &initial_write_s, &initial_wal_write_may_succeed,
             &wait_for_recover_start_time, commit_bypass_memtable);
       } while (!s.ok() && IsErrorInjectedAndRetryable(s) &&
                initial_wal_write_may_succeed);
@@ -2209,7 +2212,7 @@ class NonBatchedOpsStressTest : public StressTest {
         }
       } else {
         PrintWriteRecoveryWaitTimeIfNeeded(
-            db_stress_env, initial_write_s, initial_wal_write_may_succeed,
+            raw_env, initial_write_s, initial_wal_write_may_succeed,
             wait_for_recover_start_time, "TestDelete");
         pending_expected_value.Commit();
         thread->stats.AddSingleDeletes(1);
@@ -2274,7 +2277,7 @@ class NonBatchedOpsStressTest : public StressTest {
       } else {
         s = db_->DeleteRange(write_opts, cfh, key, end_key);
       }
-      UpdateIfInitialWriteFails(db_stress_env, s, &initial_write_s,
+      UpdateIfInitialWriteFails(raw_env, s, &initial_write_s,
                                 &initial_wal_write_may_succeed,
                                 &wait_for_recover_start_time);
     } while (!s.ok() && IsErrorInjectedAndRetryable(s) &&
@@ -2302,7 +2305,7 @@ class NonBatchedOpsStressTest : public StressTest {
       }
     } else {
       PrintWriteRecoveryWaitTimeIfNeeded(
-          db_stress_env, initial_write_s, initial_wal_write_may_succeed,
+          raw_env, initial_write_s, initial_wal_write_may_succeed,
           wait_for_recover_start_time, "TestDeleteRange");
       for (PendingExpectedValue& pending_expected_value :
            pending_expected_values) {
@@ -2325,11 +2328,11 @@ class NonBatchedOpsStressTest : public StressTest {
         FLAGS_test_ingest_standalone_range_deletion_one_in);
     std::vector<std::string> external_files;
     const std::string sst_filename =
-        FLAGS_db + "/." + std::to_string(thread->tid) + ".sst";
+        GetDbPath() + "/." + std::to_string(thread->tid) + ".sst";
     external_files.push_back(sst_filename);
     std::string standalone_rangedel_filename;
     if (test_standalone_range_deletion) {
-      standalone_rangedel_filename = FLAGS_db + "/." +
+      standalone_rangedel_filename = GetDbPath() + "/." +
                                      std::to_string(thread->tid) +
                                      "_standalone_rangedel.sst";
       external_files.push_back(standalone_rangedel_filename);
@@ -2338,28 +2341,28 @@ class NonBatchedOpsStressTest : public StressTest {
     std::ostringstream ingest_options_oss;
 
     // Temporarily disable error injection for preparation
-    if (fault_fs_guard) {
-      fault_fs_guard->DisableThreadLocalErrorInjection(
+    if (db_fault_injection_fs_) {
+      db_fault_injection_fs_->DisableThreadLocalErrorInjection(
           FaultInjectionIOType::kMetadataRead);
-      fault_fs_guard->DisableThreadLocalErrorInjection(
+      db_fault_injection_fs_->DisableThreadLocalErrorInjection(
           FaultInjectionIOType::kMetadataWrite);
     }
 
     for (const auto& filename : external_files) {
-      if (db_stress_env->FileExists(filename).ok()) {
+      if (raw_env->FileExists(filename).ok()) {
         // Maybe we terminated abnormally before, so cleanup to give this file
         // ingestion a clean slate
-        s = db_stress_env->DeleteFile(filename);
+        s = raw_env->DeleteFile(filename);
       }
       if (!s.ok()) {
         return;
       }
     }
 
-    if (fault_fs_guard) {
-      fault_fs_guard->EnableThreadLocalErrorInjection(
+    if (db_fault_injection_fs_) {
+      db_fault_injection_fs_->EnableThreadLocalErrorInjection(
           FaultInjectionIOType::kMetadataRead);
-      fault_fs_guard->EnableThreadLocalErrorInjection(
+      db_fault_injection_fs_->EnableThreadLocalErrorInjection(
           FaultInjectionIOType::kMetadataWrite);
     }
 
@@ -3485,8 +3488,11 @@ class NonBatchedOpsStressTest : public StressTest {
   }
 };
 
-StressTest* CreateNonBatchedOpsStressTest() {
-  return new NonBatchedOpsStressTest();
+StressTest* CreateNonBatchedOpsStressTest(int db_index,
+                                          const std::string& db_path,
+                                          const std::string& ev_path,
+                                          const std::string& sec_path) {
+  return new NonBatchedOpsStressTest(db_index, db_path, ev_path, sec_path);
 }
 
 }  // namespace ROCKSDB_NAMESPACE

docs/_posts/2026-05-19-resumable-remote-compaction.markdown

@@ -0,0 +1,55 @@
+---
+title: Resumable Remote Compaction
+layout: post
+author: hx235
+category: blog
+---
+
+## Background
+
+RocksDB can offload compaction work to remote workers through the `CompactionService` API. In this model, the **primary RocksDB instance** selects the input files and sends a serialized `CompactionServiceInput` to a worker; the **remote worker** runs `DB::OpenAndCompact()`, writes output SSTs to `output_directory`, and returns a serialized `CompactionServiceResult` that the primary RocksDB instance installs into its LSM tree. See the [Remote Compaction wiki](https://github.com/facebook/rocksdb/wiki/Remote-Compaction) for the full architecture. This lets operators scale compaction throughput with stateless workers while keeping the primary RocksDB instance's CPU and I/O available for serving reads and writes. However, remote compaction jobs can be long-running—sometimes processing hundreds of gigabytes of input. When a worker crashes, gets preempted, or times out, the entire compaction must restart from scratch, wasting all output produced before the interruption and increasing compaction debt on the primary RocksDB instance.
+
+## How Resumable Remote Compaction Works
+
+Resumable remote compaction introduces a **checkpoint-and-resume** mechanism. During a compaction, the worker periodically saves its progress to the `output_directory`. If the compaction is interrupted, a subsequent call to `OpenAndCompact()` with the same output directory can pick up from the last checkpoint rather than starting over.
+
+### Checkpointing
+
+After each output SST file is completed, the worker persists a progress checkpoint to a **compaction progress file** in the output directory `output_directory`. The checkpoint records which internal key to resume from and the metadata of all completed output files. Progress records use **delta encoding**—each record only contains files completed since the last checkpoint—to keep serialization cost linear.
+
+![Checkpointing overview](/static/images/resumable-remote-compaction/checkpointing-overview.svg)
+
+The worker skips checkpointing at boundaries where resuming could be unsafe or requires complicated handling: when range deletions span the file boundary or when adjacent output files share the same user key. These constraints ensure that resuming produces the same results as if the compaction was not interrupted.
+
+### Resuming
+
+When `OpenAndCompact()` is called with `allow_resumption = true`, it scans the output directory for a valid progress file. If one is found, it loads the checkpointed state, seeks the input iterator to the recorded resume key, restores the output file state, and continues compaction from that point. If the progress file is corrupted or missing, the system falls back to a fresh compaction by cleaning the directory.
+
+![Resume flow](/static/images/resumable-remote-compaction/resume-flow.svg)
+
+## How to Enable It
+
+On the primary RocksDB instance, set a `CompactionService` implementation on the DB options. On the remote worker, pass `allow_resumption = true` in `OpenAndCompactOptions` when calling `DB::OpenAndCompact()`. The `output_directory` must be the same across retries for resumption to work—each retry call with the same directory will automatically detect and resume from the previous checkpoint. The `REMOTE_COMPACT_RESUMED_BYTES` statistics ticker tracks the total bytes of output files reused from a previous interrupted run, giving visibility into how much work resumption saved.
+
+```cpp
+// Primary RocksDB instance
+DBOptions db_options;
+db_options.compaction_service = std::make_shared<MyCompactionService>();
+
+// Remote worker
+OpenAndCompactOptions options;
+options.allow_resumption = true;
+
+std::string result;
+Status s = DB::OpenAndCompact(
+    options,
+    db_path,             // source database path
+    output_directory,    // where output SSTs and progress are stored
+    compaction_input,    // serialized CompactionServiceInput
+    &result,             // serialized CompactionServiceResult
+    override_options);
+```
+
+## Future Work
+
+Today this feature targets remote compaction. The same checkpoint-and-resume mechanism could also support **local compaction** after a crash. The core persistence and resume logic is already in `CompactionJob`; the remaining work is to integrate it with local compaction scheduling and recovery.

docs/static/images/resumable-remote-compaction/checkpointing-overview.svg

@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1000 390" style="background-color: #fafafa;">
+  <defs>
+    <marker id="cp-arrowhead" markerWidth="10" markerHeight="7" refX="10" refY="3.5" orient="auto">
+      <polygon points="0 0, 10 3.5, 0 7" fill="#355070" />
+    </marker>
+    <filter id="cp-shadow" x="-20%" y="-20%" width="140%" height="140%">
+      <feDropShadow dx="2" dy="2" stdDeviation="2" flood-color="#00000020"/>
+    </filter>
+  </defs>
+
+  <text x="500" y="36" font-family="Arial, sans-serif" font-size="22" font-weight="bold" text-anchor="middle" fill="#25364d">
+    Checkpointing
+  </text>
+
+  <g transform="translate(50,80)">
+    <rect x="0" y="0" width="360" height="220" rx="18" fill="#eef5ff" stroke="#9cb9dd" stroke-width="2" filter="url(#cp-shadow)"/>
+    <text x="180" y="38" font-family="Arial, sans-serif" font-size="18" font-weight="bold" text-anchor="middle" fill="#25364d">
+      During DB::OpenAndCompact()
+    </text>
+
+    <g transform="translate(28,86)">
+      <rect x="0" y="0" width="90" height="72" rx="12" fill="#daf3e5" stroke="#4f9d69" stroke-width="2"/>
+      <text x="45" y="30" font-family="Arial, sans-serif" font-size="15" font-weight="bold" text-anchor="middle" fill="#214e32">1.sst</text>
+      <text x="45" y="51" font-family="Arial, sans-serif" font-size="12" text-anchor="middle" fill="#214e32">complete</text>
+    </g>
+
+    <g transform="translate(136,86)">
+      <rect x="0" y="0" width="90" height="72" rx="12" fill="#daf3e5" stroke="#4f9d69" stroke-width="2"/>
+      <text x="45" y="30" font-family="Arial, sans-serif" font-size="15" font-weight="bold" text-anchor="middle" fill="#214e32">2.sst</text>
+      <text x="45" y="51" font-family="Arial, sans-serif" font-size="12" text-anchor="middle" fill="#214e32">complete</text>
+    </g>
+
+    <g transform="translate(244,86)">
+      <rect x="0" y="0" width="90" height="72" rx="12" fill="#fff1d6" stroke="#d08a2e" stroke-width="2" stroke-dasharray="6 4"/>
+      <text x="45" y="28" font-family="Arial, sans-serif" font-size="15" font-weight="bold" text-anchor="middle" fill="#7a5015">3.sst</text>
+      <text x="45" y="49" font-family="Arial, sans-serif" font-size="12" text-anchor="middle" fill="#7a5015">in progress</text>
+    </g>
+  </g>
+
+  <line x1="432" y1="190" x2="558" y2="190" stroke="#355070" stroke-width="3" marker-end="url(#cp-arrowhead)"/>
+  <text x="495" y="168" font-family="Arial, sans-serif" font-size="13" font-weight="bold" text-anchor="middle" fill="#25364d">
+    After each completed SST
+  </text>
+
+  <g transform="translate(590,80)">
+    <rect x="0" y="0" width="360" height="240" rx="18" fill="#fff8e8" stroke="#d4a24a" stroke-width="2" filter="url(#cp-shadow)"/>
+    <text x="180" y="38" font-family="Arial, sans-serif" font-size="18" font-weight="bold" text-anchor="middle" fill="#6e4a16">
+      Compaction Progress File
+    </text>
+    <g transform="translate(26,76)">
+      <rect x="0" y="0" width="308" height="72" rx="12" fill="#fffdf8" stroke="#d4a24a" stroke-width="1.5"/>
+      <text x="154" y="20" font-family="Courier New, monospace" font-size="12" text-anchor="middle" fill="#4e3a19">record 1</text>
+      <text x="154" y="38" font-family="Courier New, monospace" font-size="12" text-anchor="middle" fill="#4e3a19">resume_from = key_after_1.sst</text>
+      <text x="154" y="56" font-family="Courier New, monospace" font-size="12" text-anchor="middle" fill="#4e3a19">new_files += [1.sst metadata]</text>
+    </g>
+
+    <g transform="translate(26,158)">
+      <rect x="0" y="0" width="308" height="72" rx="12" fill="#fffdf8" stroke="#d4a24a" stroke-width="1.5"/>
+      <text x="154" y="20" font-family="Courier New, monospace" font-size="12" text-anchor="middle" fill="#4e3a19">record 2</text>
+      <text x="154" y="38" font-family="Courier New, monospace" font-size="12" text-anchor="middle" fill="#4e3a19">resume_from = key_after_2.sst</text>
+      <text x="154" y="56" font-family="Courier New, monospace" font-size="12" text-anchor="middle" fill="#4e3a19">new_files += [2.sst metadata]</text>
+    </g>
+  </g>
+</svg>

docs/static/images/resumable-remote-compaction/resume-flow.svg

@@ -0,0 +1,97 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1000 720" style="background-color: #fafafa;">
+  <defs>
+    <marker id="rf-arrowhead" markerWidth="10" markerHeight="7" refX="10" refY="3.5" orient="auto">
+      <polygon points="0 0, 10 3.5, 0 7" fill="#355070" />
+    </marker>
+    <filter id="rf-shadow" x="-20%" y="-20%" width="140%" height="140%">
+      <feDropShadow dx="2" dy="2" stdDeviation="2" flood-color="#00000020"/>
+    </filter>
+  </defs>
+
+  <text x="500" y="36" font-family="Arial, sans-serif" font-size="22" font-weight="bold" text-anchor="middle" fill="#25364d">
+    Resuming a Remote Compaction
+  </text>
+
+  <g transform="translate(275,70)">
+    <rect x="0" y="0" width="450" height="64" rx="18" fill="#edf4ff" stroke="#9cb9dd" stroke-width="2" filter="url(#rf-shadow)"/>
+    <text x="225" y="28" font-family="Arial, sans-serif" font-size="18" font-weight="bold" text-anchor="middle" fill="#25364d">
+      OpenAndCompact(allow_resumption = true)
+    </text>
+    <text x="225" y="48" font-family="Arial, sans-serif" font-size="12" text-anchor="middle" fill="#5b6573">
+      Retry with the same output_directory
+    </text>
+  </g>
+
+  <line x1="500" y1="134" x2="500" y2="180" stroke="#355070" stroke-width="3" marker-end="url(#rf-arrowhead)"/>
+
+  <g transform="translate(275,190)">
+    <rect x="0" y="0" width="450" height="78" rx="18" fill="#edf4ff" stroke="#9cb9dd" stroke-width="2" filter="url(#rf-shadow)"/>
+    <text x="225" y="30" font-family="Arial, sans-serif" font-size="18" font-weight="bold" text-anchor="middle" fill="#25364d">
+      Scan output_directory
+    </text>
+    <text x="225" y="52" font-family="Arial, sans-serif" font-size="12" text-anchor="middle" fill="#5b6573">
+      Inspect the progress file and any existing SST files
+    </text>
+  </g>
+
+  <line x1="500" y1="268" x2="500" y2="316" stroke="#355070" stroke-width="3" marker-end="url(#rf-arrowhead)"/>
+
+  <polygon points="500,316 650,386 500,456 350,386" fill="#fff8e8" stroke="#d4a24a" stroke-width="2" filter="url(#rf-shadow)"/>
+  <text x="500" y="392" font-family="Arial, sans-serif" font-size="18" font-weight="bold" text-anchor="middle" fill="#6e4a16">
+    Valid progress file?
+  </text>
+
+  <line x1="350" y1="386" x2="220" y2="386" stroke="#355070" stroke-width="3" marker-end="url(#rf-arrowhead)"/>
+  <text x="308" y="372" font-family="Arial, sans-serif" font-size="13" font-weight="bold" text-anchor="middle" fill="#25364d">No</text>
+
+  <g transform="translate(40,344)">
+    <rect x="0" y="0" width="240" height="72" rx="18" fill="#fff0f0" stroke="#d97070" stroke-width="2" filter="url(#rf-shadow)"/>
+    <text x="120" y="30" font-family="Arial, sans-serif" font-size="17" font-weight="bold" text-anchor="middle" fill="#7f2323">
+      Start fresh
+    </text>
+    <text x="120" y="50" font-family="Arial, sans-serif" font-size="12" text-anchor="middle" fill="#7f2323">
+      Clean output_directory
+    </text>
+  </g>
+
+  <line x1="650" y1="386" x2="780" y2="386" stroke="#355070" stroke-width="3" marker-end="url(#rf-arrowhead)"/>
+  <text x="694" y="372" font-family="Arial, sans-serif" font-size="13" font-weight="bold" text-anchor="middle" fill="#25364d">Yes</text>
+
+  <g transform="translate(720,310)">
+    <rect x="0" y="0" width="230" height="106" rx="18" fill="#eef8ef" stroke="#7db28e" stroke-width="2" filter="url(#rf-shadow)"/>
+    <text x="115" y="30" font-family="Arial, sans-serif" font-size="17" font-weight="bold" text-anchor="middle" fill="#24503a">
+      Resume from checkpoint
+    </text>
+    <text x="115" y="52" font-family="Arial, sans-serif" font-size="12" text-anchor="middle" fill="#24503a">
+      Load progress state
+    </text>
+    <text x="115" y="70" font-family="Arial, sans-serif" font-size="12" text-anchor="middle" fill="#24503a">
+      Seek the iterator to the resume key
+    </text>
+    <text x="115" y="88" font-family="Arial, sans-serif" font-size="12" text-anchor="middle" fill="#24503a">
+      Restore output file state
+    </text>
+  </g>
+
+  <line x1="160" y1="416" x2="160" y2="560" stroke="#355070" stroke-width="3" marker-end="url(#rf-arrowhead)"/>
+  <line x1="835" y1="416" x2="835" y2="560" stroke="#355070" stroke-width="3" marker-end="url(#rf-arrowhead)"/>
+  <line x1="160" y1="560" x2="355" y2="560" stroke="#355070" stroke-width="3"/>
+  <line x1="835" y1="560" x2="645" y2="560" stroke="#355070" stroke-width="3"/>
+
+  <g transform="translate(355,526)">
+    <rect x="0" y="0" width="290" height="50" rx="18" fill="#edf4ff" stroke="#9cb9dd" stroke-width="2" filter="url(#rf-shadow)"/>
+    <text x="145" y="31" font-family="Arial, sans-serif" font-size="18" font-weight="bold" text-anchor="middle" fill="#25364d">
+      Continue compaction
+    </text>
+  </g>
+
+  <line x1="500" y1="576" x2="500" y2="646" stroke="#355070" stroke-width="3" marker-end="url(#rf-arrowhead)"/>
+
+  <g transform="translate(310,656)">
+    <rect x="0" y="0" width="380" height="44" rx="16" fill="#eef8ef" stroke="#7db28e" stroke-width="2" filter="url(#rf-shadow)"/>
+    <text x="190" y="28" font-family="Arial, sans-serif" font-size="17" font-weight="bold" text-anchor="middle" fill="#24503a">
+      Return CompactionServiceResult
+    </text>
+  </g>
+</svg>

env/env_test.cc

@@ -295,6 +295,46 @@ class EnvPosixTestWithParam
     }
   }
 
+  // ReserveThreads() returns the number of threads observed waiting at that
+  // instant. When this test runs in parallel with many other CPU-heavy tests,
+  // worker-thread scheduling can vary enough that the sync points prove key
+  // transitions were reached before waiting-thread accounting has fully
+  // settled. Use this helper only for positive exact-reservation checks; it
+  // releases any partial reservation before retrying so the retry does not
+  // perturb test state.
+  testing::AssertionResult ReserveThreadsEventually(int expected, int requested,
+                                                    Env::Priority priority,
+                                                    int wait_micros) {
+    constexpr int kRetryMicros = 1000;
+    int last_reserved = -1;
+    for (int waited_micros = 0; waited_micros <= wait_micros;
+         waited_micros += kRetryMicros) {
+      int reserved = env_->ReserveThreads(requested, priority);
+      if (reserved == expected) {
+        return testing::AssertionSuccess();
+      }
+      if (reserved > 0) {
+        int released = env_->ReleaseThreads(reserved, priority);
+        if (released != reserved) {
+          return testing::AssertionFailure()
+                 << "ReserveThreads(" << requested << ") returned " << reserved
+                 << ", but ReleaseThreads(" << reserved << ") released "
+                 << released;
+        }
+      }
+      if (reserved > expected) {
+        return testing::AssertionFailure()
+               << "ReserveThreads(" << requested << ") returned " << reserved
+               << ", more than expected " << expected;
+      }
+      last_reserved = reserved;
+      Env::Default()->SleepForMicroseconds(kRetryMicros);
+    }
+    return testing::AssertionFailure()
+           << "ReserveThreads(" << requested << ") returned " << last_reserved
+           << " after waiting " << wait_micros << "us, expected " << expected;
+  }
+
   ~EnvPosixTestWithParam() override { WaitThreadPoolsEmpty(); }
 };
 
@@ -1022,7 +1062,7 @@ TEST_P(EnvPosixTestWithParam, ReserveThreads) {
   TEST_SYNC_POINT("EnvTest::ReserveThreads:2");
   TEST_SYNC_POINT("EnvTest::ReserveThreads:3");
   // Reserve 2 threads
-  ASSERT_EQ(2, env_->ReserveThreads(2, Env::Priority::HIGH));
+  ASSERT_TRUE(ReserveThreadsEventually(2, 2, Env::Priority::HIGH, kWaitMicros));
 
   // Schedule 3 tasks. Task 0 running (in this context, doing
   // SleepingBackgroundTask); Task 1, 2 waiting; 3 reserved threads.
@@ -1051,7 +1091,7 @@ TEST_P(EnvPosixTestWithParam, ReserveThreads) {
   // Add sync point to ensure the 4th thread starts
   TEST_SYNC_POINT("EnvTest::ReserveThreads:4");
   // As the thread pool is expanded, we can reserve one more thread
-  ASSERT_EQ(1, env_->ReserveThreads(3, Env::Priority::HIGH));
+  ASSERT_TRUE(ReserveThreadsEventually(1, 3, Env::Priority::HIGH, kWaitMicros));
   // No more threads can be reserved
   ASSERT_EQ(0, env_->ReserveThreads(3, Env::Priority::HIGH));
 
@@ -1070,7 +1110,7 @@ TEST_P(EnvPosixTestWithParam, ReserveThreads) {
   // Add sync point to ensure the number of waiting threads increases
   TEST_SYNC_POINT("EnvTest::ReserveThreads:5");
   // 1 more thread can be reserved
-  ASSERT_EQ(1, env_->ReserveThreads(3, Env::Priority::HIGH));
+  ASSERT_TRUE(ReserveThreadsEventually(1, 3, Env::Priority::HIGH, kWaitMicros));
   // 2 reserved threads now
 
   // Currently, two threads are blocked since the number of waiting

env/fs_posix.cc

@@ -1114,10 +1114,10 @@ class PosixFileSystem : public FileSystem {
         struct io_uring_cqe* cqe = nullptr;
         ssize_t ret = io_uring_wait_cqe(iu, &cqe);
         if (ret) {
-          fprintf(stderr, "Poll: io_uring_wait_cqe failed: %ld", (long)ret);
           if (ret == -EINTR || ret == -EAGAIN) {
             continue;  // Retry
           }
+          fprintf(stderr, "Poll: io_uring_wait_cqe failed: %ld\n", (long)ret);
           abort();
         }
 
@@ -1207,10 +1207,11 @@ class PosixFileSystem : public FileSystem {
         struct io_uring_cqe* cqe = nullptr;
         ssize_t ret = io_uring_wait_cqe(iu, &cqe);
         if (ret) {
-          fprintf(stderr, "AbortIO: io_uring_wait_cqe failed: %ld", (long)ret);
           if (ret == -EINTR || ret == -EAGAIN) {
             continue;  // Retry
           }
+          fprintf(stderr, "AbortIO: io_uring_wait_cqe failed: %ld\n",
+                  (long)ret);
           abort();
         }
         assert(cqe != nullptr);

file/file_prefetch_buffer.cc

@@ -103,7 +103,7 @@ Status FilePrefetchBuffer::Read(BufferInfo* buf, const IOOptions& opts,
   } else {
     to_buf = buf->buffer_.BufferStart() + aligned_useful_len;
     s = reader->Read(opts, start_offset + aligned_useful_len, read_len, &result,
-                     to_buf, /*aligned_buf=*/nullptr);
+                     to_buf);
   }
 
 #ifndef NDEBUG
@@ -165,7 +165,7 @@ Status FilePrefetchBuffer::ReadAsync(BufferInfo* buf, const IOOptions& opts,
     // Fall back to synchronous read so the buffer is populated inline
     // and callers proceed transparently.
     s = reader->Read(opts, start_offset, read_len, &result,
-                     buf->buffer_.BufferStart(), /*aligned_buf=*/nullptr);
+                     buf->buffer_.BufferStart());
     if (s.ok()) {
       buf->buffer_.Size(buf->CurrentSize() + result.size());
       if (usage_ == FilePrefetchBufferUsage::kUserScanPrefetch) {

file/file_prefetch_buffer.h

@@ -527,7 +527,9 @@ class FilePrefetchBuffer {
     read_req.offset = offset;
     read_req.len = n;
     read_req.scratch = nullptr;
-    IOStatus s = reader->MultiRead(opts, &read_req, 1, nullptr);
+    AlignedBuffer direct_io_buffer;
+    AlignedBufferAllocationContext direct_io_context{&direct_io_buffer};
+    IOStatus s = reader->MultiRead(opts, &read_req, 1, &direct_io_context);
     if (!s.ok()) {
       return s;
     }

file/random_access_file_reader.cc

@@ -16,6 +16,7 @@
 #include "monitoring/histogram.h"
 #include "monitoring/iostats_context_imp.h"
 #include "port/port.h"
+#include "rocksdb/io_status.h"
 #include "table/format.h"
 #include "test_util/sync_point.h"
 #include "util/random.h"
@@ -128,11 +129,17 @@ IOStatus RandomAccessFileReader::Create(
   return io_s;
 }
 
-IOStatus RandomAccessFileReader::Read(const IOOptions& opts, uint64_t offset,
-                                      size_t n, Slice* result, char* scratch,
-                                      AlignedBuf* aligned_buf,
-                                      IODebugContext* dbg) const {
-  (void)aligned_buf;
+IOStatus RandomAccessFileReader::Read(
+    const IOOptions& opts, uint64_t offset, size_t n, Slice* result,
+    char* scratch, AlignedBufferAllocationContext* direct_io_buffer_context,
+    IODebugContext* dbg) const {
+  AlignedBuffer* direct_io_buffer = direct_io_buffer_context != nullptr
+                                        ? direct_io_buffer_context->buffer
+                                        : nullptr;
+  const AlignedBuffer::Allocator* direct_io_allocator =
+      direct_io_buffer_context != nullptr ? direct_io_buffer_context->allocator
+                                          : nullptr;
+  assert(direct_io_buffer_context == nullptr || direct_io_buffer != nullptr);
   const Env::IOPriority rate_limiter_priority = opts.rate_limiter_priority;
 
   TEST_SYNC_POINT_CALLBACK("RandomAccessFileReader::Read", nullptr);
@@ -151,7 +158,7 @@ IOStatus RandomAccessFileReader::Read(const IOOptions& opts, uint64_t offset,
   uint64_t elapsed = 0;
   size_t alignment = file_->GetRequiredBufferAlignment();
   bool is_aligned = false;
-  if (scratch != nullptr) {
+  if (direct_io_buffer == nullptr && scratch != nullptr) {
     // Check if offset, length and buffer are aligned.
     is_aligned = (offset & (alignment - 1)) == 0 &&
                  (n & (alignment - 1)) == 0 &&
@@ -172,17 +179,25 @@ IOStatus RandomAccessFileReader::Read(const IOOptions& opts, uint64_t offset,
       size_t read_size =
           Roundup(static_cast<size_t>(offset + n), alignment) - aligned_offset;
       AlignedBuffer buf;
-      buf.Alignment(alignment);
-      buf.AllocateNewBuffer(read_size);
-      while (buf.CurrentSize() < read_size) {
+      AlignedBuffer* aligned_read_buffer =
+          direct_io_buffer != nullptr ? direct_io_buffer : &buf;
+      aligned_read_buffer->Alignment(alignment);
+      Status allocate_status = aligned_read_buffer->AllocateNewBuffer(
+          read_size, direct_io_allocator);
+      if (!allocate_status.ok()) {
+        io_s = status_to_io_status(std::move(allocate_status));
+      }
+      char* aligned_scratch = aligned_read_buffer->BufferStart();
+      size_t current_size = 0;
+      while (io_s.ok() && current_size < read_size) {
         size_t allowed;
         if (rate_limiter_priority != Env::IO_TOTAL &&
             rate_limiter_ != nullptr) {
           allowed = rate_limiter_->RequestToken(
-              buf.Capacity() - buf.CurrentSize(), buf.Alignment(),
-              rate_limiter_priority, stats_, RateLimiter::OpType::kRead);
+              read_size - current_size, alignment, rate_limiter_priority,
+              stats_, RateLimiter::OpType::kRead);
         } else {
-          assert(buf.CurrentSize() == 0);
+          assert(current_size == 0);
           allowed = read_size;
         }
         Slice tmp;
@@ -191,7 +206,7 @@ IOStatus RandomAccessFileReader::Read(const IOOptions& opts, uint64_t offset,
         uint64_t orig_offset = 0;
         if (ShouldNotifyListeners()) {
           start_ts = FileOperationInfo::StartNow();
-          orig_offset = aligned_offset + buf.CurrentSize();
+          orig_offset = aligned_offset + current_size;
         }
 
         {
@@ -201,8 +216,8 @@ IOStatus RandomAccessFileReader::Read(const IOOptions& opts, uint64_t offset,
           // one iteration of this loop, so we don't need to check and adjust
           // the opts.timeout before calling file_->Read
           assert(!opts.timeout.count() || allowed == read_size);
-          io_s = file_->Read(aligned_offset + buf.CurrentSize(), allowed, opts,
-                             &tmp, buf.Destination(), dbg);
+          io_s = file_->Read(aligned_offset + current_size, allowed, opts, &tmp,
+                             aligned_scratch + current_size, dbg);
         }
         if (ShouldNotifyListeners()) {
           auto finish_ts = FileOperationInfo::FinishNow();
@@ -214,19 +229,22 @@ IOStatus RandomAccessFileReader::Read(const IOOptions& opts, uint64_t offset,
           }
         }
 
-        buf.Size(buf.CurrentSize() + tmp.size());
+        current_size += tmp.size();
+        if (direct_io_buffer == nullptr) {
+          buf.Size(current_size);
+        }
         if (!io_s.ok() || tmp.size() < allowed) {
           break;
         }
       }
       size_t res_len = 0;
-      if (io_s.ok() && offset_advance < buf.CurrentSize()) {
-        res_len = std::min(buf.CurrentSize() - offset_advance, n);
-        if (aligned_buf == nullptr) {
-          buf.Read(scratch, offset_advance, res_len);
+      if (io_s.ok() && offset_advance < current_size) {
+        res_len = std::min(current_size - offset_advance, n);
+        if (direct_io_buffer != nullptr) {
+          scratch = aligned_scratch + offset_advance;
         } else {
-          scratch = buf.BufferStart() + offset_advance;
-          *aligned_buf = buf.Release();
+          assert(scratch != nullptr);
+          buf.Read(scratch, offset_advance, res_len);
         }
       }
       *result = Slice(scratch, res_len);
@@ -335,13 +353,18 @@ bool TryMerge(FSReadRequest* dest, const FSReadRequest& src) {
   return true;
 }
 
-IOStatus RandomAccessFileReader::MultiRead(const IOOptions& opts,
-                                           FSReadRequest* read_reqs,
-                                           size_t num_reqs,
-                                           AlignedBuf* aligned_buf,
-                                           IODebugContext* dbg) const {
-  (void)aligned_buf;  // suppress warning of unused variable in LITE mode
+IOStatus RandomAccessFileReader::MultiRead(
+    const IOOptions& opts, FSReadRequest* read_reqs, size_t num_reqs,
+    AlignedBufferAllocationContext* direct_io_buffer_context,
+    IODebugContext* dbg) const {
   assert(num_reqs > 0);
+  AlignedBuffer* direct_io_buffer = direct_io_buffer_context != nullptr
+                                        ? direct_io_buffer_context->buffer
+                                        : nullptr;
+  const AlignedBuffer::Allocator* direct_io_allocator =
+      direct_io_buffer_context != nullptr ? direct_io_buffer_context->allocator
+                                          : nullptr;
+  assert(direct_io_buffer != nullptr);
 
 #ifndef NDEBUG
   for (size_t i = 0; i < num_reqs - 1; ++i) {
@@ -403,16 +426,20 @@ IOStatus RandomAccessFileReader::MultiRead(const IOOptions& opts,
       for (const auto& r : aligned_reqs) {
         total_len += r.len;
       }
-      AlignedBuffer buf;
-      buf.Alignment(alignment);
-      buf.AllocateNewBuffer(total_len);
-      char* scratch = buf.BufferStart();
-      for (auto& r : aligned_reqs) {
-        r.scratch = scratch;
-        scratch += r.len;
+      direct_io_buffer->Alignment(alignment);
+      Status allocate_status =
+          direct_io_buffer->AllocateNewBuffer(total_len, direct_io_allocator);
+      if (!allocate_status.ok()) {
+        io_s = status_to_io_status(std::move(allocate_status));
+      }
+      if (io_s.ok()) {
+        char* scratch = direct_io_buffer->BufferStart();
+        for (auto& r : aligned_reqs) {
+          r.scratch = scratch;
+          scratch += r.len;
+        }
       }
 
-      *aligned_buf = buf.Release();
       fs_reqs = aligned_reqs.data();
       num_fs_reqs = aligned_reqs.size();
     }
@@ -422,7 +449,7 @@ IOStatus RandomAccessFileReader::MultiRead(const IOOptions& opts,
       start_ts = FileOperationInfo::StartNow();
     }
 
-    {
+    if (io_s.ok()) {
       IOSTATS_CPU_TIMER_GUARD(cpu_read_nanos, clock_);
       if (rate_limiter_priority != Env::IO_TOTAL && rate_limiter_ != nullptr) {
         // TODO: ideally we should call `RateLimiter::RequestToken()` for
@@ -455,7 +482,7 @@ IOStatus RandomAccessFileReader::MultiRead(const IOOptions& opts,
       RecordInHistogram(stats_, MULTIGET_IO_BATCH_SIZE, num_fs_reqs);
     }
 
-    if (use_direct_io()) {
+    if (use_direct_io() && io_s.ok()) {
       // Populate results in the unaligned read requests.
       size_t aligned_i = 0;
       for (size_t i = 0; i < num_reqs; i++) {
@@ -481,19 +508,20 @@ IOStatus RandomAccessFileReader::MultiRead(const IOOptions& opts,
       }
     }
 
+    const bool overall_io_ok = io_s.ok();
     for (size_t i = 0; i < num_reqs; ++i) {
+      const IOStatus& req_status = overall_io_ok ? read_reqs[i].status : io_s;
+      const size_t result_size = overall_io_ok ? read_reqs[i].result.size() : 0;
       if (ShouldNotifyListeners()) {
         auto finish_ts = FileOperationInfo::FinishNow();
-        NotifyOnFileReadFinish(read_reqs[i].offset, read_reqs[i].result.size(),
-                               start_ts, finish_ts, read_reqs[i].status);
+        NotifyOnFileReadFinish(read_reqs[i].offset, result_size, start_ts,
+                               finish_ts, req_status);
       }
-      if (!read_reqs[i].status.ok()) {
-        NotifyOnIOError(read_reqs[i].status, FileOperationType::kRead,
-                        file_name(), read_reqs[i].result.size(),
-                        read_reqs[i].offset);
+      if (!req_status.ok()) {
+        NotifyOnIOError(req_status, FileOperationType::kRead, file_name(),
+                        result_size, read_reqs[i].offset);
       }
-      RecordIOStats(stats_, file_temperature_, is_last_level_,
-                    read_reqs[i].result.size());
+      RecordIOStats(stats_, file_temperature_, is_last_level_, result_size);
     }
     SetPerfLevel(prev_perf_level);
   }
@@ -517,16 +545,26 @@ IOStatus RandomAccessFileReader::PrepareIOOptions(const ReadOptions& ro,
 
 // Notes for when direct_io is enabled:
 // Unless req.offset, req.len, req.scratch are all already aligned,
-// RandomAccessFileReader will creats aligned requests and aligned buffer for
-// the request. User should only provide either req.scratch or aligned_buf. If
-// only req.scratch is provided, result will be copied from allocated aligned
-// buffer to req.scratch. If only alignd_buf is provided, it will be set to
-// the ailgned buf allocated by RandomAccessFileReader and saves a copy.
+// RandomAccessFileReader creates an aligned request and aligned buffer for the
+// request. If direct_io_buffer_context is provided, its buffer owns the aligned
+// backing storage and its optional allocator is used only for this allocation.
+// Otherwise, callers should provide either req.scratch or aligned_buf. If only
+// req.scratch is provided, the result is copied from the allocated aligned
+// buffer to req.scratch. If only aligned_buf is provided, it is set to the
+// aligned buffer allocated by RandomAccessFileReader and saves a copy.
 IOStatus RandomAccessFileReader::ReadAsync(
     FSReadRequest& req, const IOOptions& opts,
     std::function<void(FSReadRequest&, void*)> cb, void* cb_arg,
     void** io_handle, IOHandleDeleter* del_fn, AlignedBuf* aligned_buf,
-    IODebugContext* dbg) {
+    IODebugContext* dbg,
+    AlignedBufferAllocationContext* direct_io_buffer_context) {
+  AlignedBuffer* direct_io_buffer = direct_io_buffer_context != nullptr
+                                        ? direct_io_buffer_context->buffer
+                                        : nullptr;
+  const AlignedBuffer::Allocator* direct_io_allocator =
+      direct_io_buffer_context != nullptr ? direct_io_buffer_context->allocator
+                                          : nullptr;
+  assert(direct_io_buffer_context == nullptr || direct_io_buffer != nullptr);
   IOStatus s;
   TEST_SYNC_POINT_CALLBACK("RandomAccessFileReader::ReadAsync:InjectStatus",
                            &s);
@@ -546,7 +584,8 @@ IOStatus RandomAccessFileReader::ReadAsync(
   }
 
   size_t alignment = file_->GetRequiredBufferAlignment();
-  bool is_aligned = (req.offset & (alignment - 1)) == 0 &&
+  bool is_aligned = direct_io_buffer == nullptr &&
+                    (req.offset & (alignment - 1)) == 0 &&
                     (req.len & (alignment - 1)) == 0 &&
                     (uintptr_t(req.scratch) & (alignment - 1)) == 0;
   read_async_info->is_aligned_ = is_aligned;
@@ -556,21 +595,27 @@ IOStatus RandomAccessFileReader::ReadAsync(
     FSReadRequest aligned_req = Align(req, alignment);
     aligned_req.status.PermitUncheckedError();
 
-    // Allocate aligned buffer.
-    read_async_info->buf_.Alignment(alignment);
-    read_async_info->buf_.AllocateNewBuffer(aligned_req.len);
-
-    // Set rem fields in aligned FSReadRequest.
-    aligned_req.scratch = read_async_info->buf_.BufferStart();
+    AlignedBuffer* aligned_read_buffer =
+        direct_io_buffer != nullptr ? direct_io_buffer : &read_async_info->buf_;
+    aligned_read_buffer->Alignment(alignment);
+    Status allocate_status = aligned_read_buffer->AllocateNewBuffer(
+        aligned_req.len, direct_io_allocator);
+    if (!allocate_status.ok()) {
+      delete read_async_info;
+      return status_to_io_status(std::move(allocate_status));
+    }
+    aligned_req.scratch = aligned_read_buffer->BufferStart();
 
     // Set user provided fields to populate back in callback.
     read_async_info->user_scratch_ = req.scratch;
     read_async_info->user_aligned_buf_ = aligned_buf;
+    read_async_info->direct_io_buffer_ = direct_io_buffer;
     read_async_info->user_len_ = req.len;
     read_async_info->user_offset_ = req.offset;
     read_async_info->user_result_ = req.result;
 
-    assert(read_async_info->buf_.CurrentSize() == 0);
+    assert(direct_io_buffer != nullptr ||
+           read_async_info->buf_.CurrentSize() == 0);
 
     StopWatch sw(clock_, stats_, hist_type_,
                  GetFileReadHistograms(stats_, opts.io_activity),
@@ -618,20 +663,25 @@ void RandomAccessFileReader::ReadAsyncCallback(FSReadRequest& req,
     user_req.result = req.result;
     user_req.status = req.status;
 
-    read_async_info->buf_.Size(read_async_info->buf_.CurrentSize() +
-                               req.result.size());
+    size_t current_size = req.result.size();
+    if (read_async_info->direct_io_buffer_ == nullptr) {
+      read_async_info->buf_.Size(read_async_info->buf_.CurrentSize() +
+                                 req.result.size());
+      current_size = read_async_info->buf_.CurrentSize();
+    }
 
     size_t offset_advance_len = static_cast<size_t>(
         /*offset_passed_by_user=*/read_async_info->user_offset_ -
         /*aligned_offset=*/req.offset);
 
     size_t res_len = 0;
-    if (req.status.ok() &&
-        offset_advance_len < read_async_info->buf_.CurrentSize()) {
-      res_len =
-          std::min(read_async_info->buf_.CurrentSize() - offset_advance_len,
-                   read_async_info->user_len_);
-      if (read_async_info->user_aligned_buf_ == nullptr) {
+    if (req.status.ok() && offset_advance_len < current_size) {
+      res_len = std::min(current_size - offset_advance_len,
+                         read_async_info->user_len_);
+      if (read_async_info->direct_io_buffer_ != nullptr) {
+        user_req.scratch = read_async_info->direct_io_buffer_->BufferStart() +
+                           offset_advance_len;
+      } else if (read_async_info->user_aligned_buf_ == nullptr) {
         // Copy the data into user's scratch.
 // Clang analyzer assumes that it will take use_direct_io() == false in
 // ReadAsync and use_direct_io() == true in Callback which cannot be true.

file/random_access_file_reader.h

@@ -9,6 +9,7 @@
 
 #pragma once
 #include <atomic>
+#include <functional>
 #include <sstream>
 #include <string>
 
@@ -27,6 +28,15 @@ class SystemClock;
 
 using AlignedBuf = FSAllocationPtr;
 
+struct AlignedBufferAllocationContext {
+  // `allocator` is intentionally not owned. RandomAccessFileReader invokes it
+  // only while allocating `buffer`, before Read/ReadAsync/MultiRead returns.
+  // For ReadAsync, `buffer` is still referenced by the completion callback and
+  // must outlive the async operation.
+  AlignedBuffer* buffer = nullptr;
+  const AlignedBuffer::Allocator* allocator = nullptr;
+};
+
 // Align the request r according to alignment and return the aligned result.
 FSReadRequest Align(const FSReadRequest& r, size_t alignment);
 
@@ -97,6 +107,7 @@ class RandomAccessFileReader {
           start_time_(start_time),
           user_scratch_(nullptr),
           user_aligned_buf_(nullptr),
+          direct_io_buffer_(nullptr),
           user_offset_(0),
           user_len_(0),
           is_aligned_(false) {}
@@ -108,6 +119,10 @@ class RandomAccessFileReader {
     // Below fields stores the parameters passed by caller in case of direct_io.
     char* user_scratch_;
     AlignedBuf* user_aligned_buf_;
+    // Raw pointer to caller-owned direct-I/O storage used when forming the
+    // user-visible result in ReadAsyncCallback. The caller must keep it alive
+    // until the async operation completes or is cancelled.
+    AlignedBuffer* direct_io_buffer_;
     uint64_t user_offset_;
     size_t user_len_;
     Slice user_result_;
@@ -157,23 +172,28 @@ class RandomAccessFileReader {
   // 1. if using mmap, result is stored in a buffer other than scratch;
   // 2. if not using mmap, result is stored in the buffer starting from scratch.
   //
-  // In direct IO mode, an aligned buffer is allocated internally.
-  // 1. If aligned_buf is null, then results are copied to the buffer
-  // starting from scratch;
-  // 2. Otherwise, scratch is not used and can be null, the aligned_buf owns
-  // the internally allocated buffer on return, and the result refers to a
-  // region in aligned_buf.
-  IOStatus Read(const IOOptions& opts, uint64_t offset, size_t n, Slice* result,
-                char* scratch, AlignedBuf* aligned_buf,
-                IODebugContext* dbg = nullptr) const;
+  // In direct IO mode, if direct_io_buffer_context is provided then it
+  // allocates the aligned buffer and the result refers to a region in
+  // direct_io_buffer_context->buffer. Otherwise, results are returned in
+  // scratch; unaligned reads use an internal aligned buffer and copy the
+  // requested subrange to scratch.
+  IOStatus Read(
+      const IOOptions& opts, uint64_t offset, size_t n, Slice* result,
+      char* scratch,
+      AlignedBufferAllocationContext* direct_io_buffer_context = nullptr,
+      IODebugContext* dbg = nullptr) const;
 
   // REQUIRES:
   // num_reqs > 0, reqs do not overlap, and offsets in reqs are increasing.
-  // In non-direct IO mode, aligned_buf should be null;
-  // In direct IO mode, aligned_buf stores the aligned buffer allocated inside
-  // MultiRead, the result Slices in reqs refer to aligned_buf.
+  // MultiRead uses direct_io_buffer_context to allocate the aligned buffer in
+  // direct IO mode. The result Slices in reqs refer to
+  // direct_io_buffer_context->buffer, so callers must keep it alive while those
+  // Slices are used. Callers should pass a default-constructed AlignedBuffer
+  // when default heap-backed allocation is sufficient. direct_io_buffer_context
+  // is ignored in non-direct IO mode.
   IOStatus MultiRead(const IOOptions& opts, FSReadRequest* reqs,
-                     size_t num_reqs, AlignedBuf* aligned_buf,
+                     size_t num_reqs,
+                     AlignedBufferAllocationContext* direct_io_buffer_context,
                      IODebugContext* dbg = nullptr) const;
 
   IOStatus Prefetch(const IOOptions& opts, uint64_t offset, size_t n,
@@ -190,10 +210,12 @@ class RandomAccessFileReader {
   IOStatus PrepareIOOptions(const ReadOptions& ro, IOOptions& opts,
                             IODebugContext* dbg = nullptr) const;
 
-  IOStatus ReadAsync(FSReadRequest& req, const IOOptions& opts,
-                     std::function<void(FSReadRequest&, void*)> cb,
-                     void* cb_arg, void** io_handle, IOHandleDeleter* del_fn,
-                     AlignedBuf* aligned_buf, IODebugContext* dbg = nullptr);
+  IOStatus ReadAsync(
+      FSReadRequest& req, const IOOptions& opts,
+      std::function<void(FSReadRequest&, void*)> cb, void* cb_arg,
+      void** io_handle, IOHandleDeleter* del_fn, AlignedBuf* aligned_buf,
+      IODebugContext* dbg = nullptr,
+      AlignedBufferAllocationContext* direct_io_buffer_context = nullptr);
 
   void ReadAsyncCallback(FSReadRequest& req, void* cb_arg);
 };

file/random_access_file_reader_test.cc

@@ -81,15 +81,91 @@ TEST_F(RandomAccessFileReaderTest, ReadDirectIO) {
   size_t offset = page_size / 2;
   size_t len = page_size / 3;
   Slice result;
-  AlignedBuf buf;
   for (Env::IOPriority rate_limiter_priority : {Env::IO_LOW, Env::IO_TOTAL}) {
     IOOptions io_opts;
     io_opts.rate_limiter_priority = rate_limiter_priority;
-    ASSERT_OK(r->Read(io_opts, offset, len, &result, nullptr, &buf));
+    AlignedBuffer direct_io_buffer;
+    AlignedBufferAllocationContext direct_io_context{&direct_io_buffer};
+    ASSERT_OK(r->Read(io_opts, offset, len, &result, nullptr,
+                      &direct_io_context,
+                      /*dbg=*/nullptr));
     ASSERT_EQ(result.ToString(), content.substr(offset, len));
   }
 }
 
+TEST_F(RandomAccessFileReaderTest, ReadDirectIOCopiesToScratch) {
+  std::string fname = "read-direct-io-copies-to-scratch";
+  Random rand(0);
+  std::string content = rand.RandomString(kDefaultPageSize);
+  Write(fname, content);
+
+  FileOptions opts;
+  opts.use_direct_reads = true;
+  std::unique_ptr<RandomAccessFileReader> r;
+  Read(fname, opts, &r);
+  ASSERT_TRUE(r->use_direct_io());
+
+  const size_t page_size = r->file()->GetRequiredBufferAlignment();
+  size_t offset = page_size / 2;
+  size_t len = page_size / 3;
+  std::string scratch(len, '\0');
+  Slice result;
+  ASSERT_OK(r->Read(IOOptions(), offset, len, &result, scratch.data(),
+                    /*direct_io_buffer=*/nullptr, /*dbg=*/nullptr));
+  ASSERT_EQ(result.data(), scratch.data());
+  ASSERT_EQ(result.ToString(), content.substr(offset, len));
+}
+
+TEST_F(RandomAccessFileReaderTest, ReadDirectIOUsesExternalBuffer) {
+  std::string fname = "read-direct-io-external-buffer";
+  Random rand(0);
+  std::string content = rand.RandomString(kDefaultPageSize);
+  Write(fname, content);
+
+  FileOptions opts;
+  opts.use_direct_reads = true;
+  std::unique_ptr<RandomAccessFileReader> r;
+  Read(fname, opts, &r);
+  ASSERT_TRUE(r->use_direct_io());
+
+  const size_t page_size = r->file()->GetRequiredBufferAlignment();
+  const size_t offset = page_size / 4;
+  const size_t len = page_size / 2;
+
+  AlignedBuffer external_storage;
+  int allocations = 0;
+  size_t requested_size = 0;
+  size_t requested_alignment = 0;
+  AlignedBuffer::Allocator allocator =
+      [&](size_t size, size_t alignment,
+          AlignedBuffer::ExternalAllocation* out) {
+        ++allocations;
+        requested_size = size;
+        requested_alignment = alignment;
+        external_storage.Alignment(alignment);
+        external_storage.AllocateNewBuffer(size);
+        out->data = external_storage.BufferStart();
+        out->size = external_storage.Capacity();
+        out->owner =
+            FSAllocationPtr(external_storage.BufferStart(), [](void*) {});
+        return Status::OK();
+      };
+
+  AlignedBuffer direct_io_buffer;
+  AlignedBufferAllocationContext direct_io_context{&direct_io_buffer,
+                                                   &allocator};
+  Slice result;
+  ASSERT_OK(r->Read(IOOptions(), offset, len, &result, /*scratch=*/nullptr,
+                    &direct_io_context, /*dbg=*/nullptr));
+  ASSERT_EQ(result.ToString(), content.substr(offset, len));
+
+  ASSERT_EQ(allocations, 1);
+  ASSERT_EQ(requested_alignment, page_size);
+  ASSERT_EQ(requested_size, page_size);
+  ASSERT_EQ(direct_io_buffer.BufferStart(), external_storage.BufferStart());
+  ASSERT_EQ(result.data(), external_storage.BufferStart() + offset);
+}
+
 TEST_F(RandomAccessFileReaderTest, MultiReadDirectIO) {
   std::vector<FSReadRequest> aligned_reqs;
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
@@ -146,10 +222,11 @@ TEST_F(RandomAccessFileReaderTest, MultiReadDirectIO) {
     std::vector<FSReadRequest> reqs;
     reqs.push_back(std::move(r0));
     reqs.push_back(std::move(r1));
-    AlignedBuf aligned_buf;
+    AlignedBuffer direct_io_buffer;
+    AlignedBufferAllocationContext direct_io_context{&direct_io_buffer};
     IODebugContext dbg;
-    ASSERT_OK(r->MultiRead(IOOptions(), reqs.data(), reqs.size(), &aligned_buf,
-                           &dbg));
+    ASSERT_OK(r->MultiRead(IOOptions(), reqs.data(), reqs.size(),
+                           &direct_io_context, &dbg));
 
     AssertResult(content, reqs);
 
@@ -192,10 +269,11 @@ TEST_F(RandomAccessFileReaderTest, MultiReadDirectIO) {
     reqs.push_back(std::move(r0));
     reqs.push_back(std::move(r1));
     reqs.push_back(std::move(r2));
-    AlignedBuf aligned_buf;
+    AlignedBuffer direct_io_buffer;
+    AlignedBufferAllocationContext direct_io_context{&direct_io_buffer};
     IODebugContext dbg;
-    ASSERT_OK(r->MultiRead(IOOptions(), reqs.data(), reqs.size(), &aligned_buf,
-                           &dbg));
+    ASSERT_OK(r->MultiRead(IOOptions(), reqs.data(), reqs.size(),
+                           &direct_io_context, &dbg));
 
     AssertResult(content, reqs);
 
@@ -238,10 +316,11 @@ TEST_F(RandomAccessFileReaderTest, MultiReadDirectIO) {
     reqs.push_back(std::move(r0));
     reqs.push_back(std::move(r1));
     reqs.push_back(std::move(r2));
-    AlignedBuf aligned_buf;
+    AlignedBuffer direct_io_buffer;
+    AlignedBufferAllocationContext direct_io_context{&direct_io_buffer};
     IODebugContext dbg;
-    ASSERT_OK(r->MultiRead(IOOptions(), reqs.data(), reqs.size(), &aligned_buf,
-                           &dbg));
+    ASSERT_OK(r->MultiRead(IOOptions(), reqs.data(), reqs.size(),
+                           &direct_io_context, &dbg));
 
     AssertResult(content, reqs);
 
@@ -276,10 +355,11 @@ TEST_F(RandomAccessFileReaderTest, MultiReadDirectIO) {
     std::vector<FSReadRequest> reqs;
     reqs.push_back(std::move(r0));
     reqs.push_back(std::move(r1));
-    AlignedBuf aligned_buf;
+    AlignedBuffer direct_io_buffer;
+    AlignedBufferAllocationContext direct_io_context{&direct_io_buffer};
     IODebugContext dbg;
-    ASSERT_OK(r->MultiRead(IOOptions(), reqs.data(), reqs.size(), &aligned_buf,
-                           &dbg));
+    ASSERT_OK(r->MultiRead(IOOptions(), reqs.data(), reqs.size(),
+                           &direct_io_context, &dbg));
 
     AssertResult(content, reqs);
 
@@ -299,6 +379,71 @@ TEST_F(RandomAccessFileReaderTest, MultiReadDirectIO) {
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
 }
 
+TEST_F(RandomAccessFileReaderTest, MultiReadDirectIOUsesExternalBuffer) {
+  std::string fname = "multi-read-direct-io-external-buffer";
+  Random rand(0);
+  std::string content = rand.RandomString(3 * kDefaultPageSize);
+  Write(fname, content);
+
+  FileOptions opts;
+  opts.use_direct_reads = true;
+  std::unique_ptr<RandomAccessFileReader> r;
+  Read(fname, opts, &r);
+  ASSERT_TRUE(r->use_direct_io());
+
+  const size_t page_size = r->file()->GetRequiredBufferAlignment();
+  FSReadRequest r0;
+  r0.offset = page_size / 4;
+  r0.len = page_size / 2;
+  r0.scratch = nullptr;
+
+  FSReadRequest r1;
+  r1.offset = 2 * page_size + page_size / 4;
+  r1.len = page_size / 2;
+  r1.scratch = nullptr;
+
+  std::vector<FSReadRequest> reqs;
+  reqs.push_back(std::move(r0));
+  reqs.push_back(std::move(r1));
+
+  AlignedBuffer external_storage;
+  int allocations = 0;
+  size_t requested_size = 0;
+  size_t requested_alignment = 0;
+  AlignedBuffer::Allocator allocator =
+      [&](size_t size, size_t alignment,
+          AlignedBuffer::ExternalAllocation* out) {
+        ++allocations;
+        requested_size = size;
+        requested_alignment = alignment;
+        external_storage.Alignment(alignment);
+        external_storage.AllocateNewBuffer(size);
+        out->data = external_storage.BufferStart();
+        out->size = external_storage.Capacity();
+        out->owner =
+            FSAllocationPtr(external_storage.BufferStart(), [](void*) {});
+        return Status::OK();
+      };
+
+  AlignedBuffer direct_io_buffer;
+  AlignedBufferAllocationContext direct_io_context{&direct_io_buffer,
+                                                   &allocator};
+  ASSERT_OK(r->MultiRead(IOOptions(), reqs.data(), reqs.size(),
+                         &direct_io_context, /*dbg=*/nullptr));
+  AssertResult(content, reqs);
+
+  ASSERT_EQ(allocations, 1);
+  ASSERT_EQ(requested_alignment, page_size);
+  ASSERT_EQ(requested_size, 2 * page_size);
+  ASSERT_EQ(direct_io_buffer.BufferStart(), external_storage.BufferStart());
+  const char* storage_begin = external_storage.BufferStart();
+  const char* storage_end = storage_begin + external_storage.Capacity();
+  for (const auto& req : reqs) {
+    ASSERT_GE(req.result.data(), storage_begin);
+    ASSERT_LE(req.result.data() + req.result.size(), storage_end);
+  }
+}
+
 TEST(FSReadRequest, Align) {
   FSReadRequest r;
   r.offset = 2000;

folly.mk

@@ -98,7 +98,31 @@ endif  # FMT_SOURCE_PATH
 	PLATFORM_LDFLAGS += -lglog
 endif
 
-FOLLY_COMMIT_HASH = 3b0eed0a128c484a2b077546c3e08c824a311357
+FOLLY_COMMIT_HASH = 548b16da0b3cc887d69cdb6ae06496ad8a2a9276
+FOLLY_GETDEPS_CACHE_DIR = /tmp/rocksdb-getdeps-cache
+
+define restore_folly_getdeps_downloads
+	@cd third-party/folly && \
+		DOWNLOAD_DIR=`$(PYTHON) build/fbcode_builder/getdeps.py show-inst-dir | sed 's|/installed/.*|/downloads|'` && \
+		mkdir -p "$$DOWNLOAD_DIR" && \
+		CACHE_DIR="$(FOLLY_GETDEPS_CACHE_DIR)" && \
+		mkdir -p "$$CACHE_DIR" && \
+		echo "Restoring cached downloads..." && \
+		if ls "$$CACHE_DIR"/*.tar.gz "$$CACHE_DIR"/*.tar.xz "$$CACHE_DIR"/*.zip >/dev/null 2>&1; then \
+			cp -n "$$CACHE_DIR"/*.tar.gz "$$CACHE_DIR"/*.tar.xz "$$CACHE_DIR"/*.zip "$$DOWNLOAD_DIR/" 2>/dev/null || true; \
+		fi && \
+		echo "Handling known unreliable downloads with fallback mirrors..." && \
+		$(PYTHON) ../../build_tools/getdeps_fallback_mirror.py "$$DOWNLOAD_DIR" "$$CACHE_DIR" build/fbcode_builder/manifests
+endef
+
+define cache_folly_getdeps_downloads
+	@cd third-party/folly && \
+		DOWNLOAD_DIR=`$(PYTHON) build/fbcode_builder/getdeps.py show-inst-dir | sed 's|/installed/.*|/downloads|'` && \
+		CACHE_DIR="$(FOLLY_GETDEPS_CACHE_DIR)" && \
+		if ls "$$DOWNLOAD_DIR"/*.tar.gz "$$DOWNLOAD_DIR"/*.tar.xz "$$DOWNLOAD_DIR"/*.zip >/dev/null 2>&1; then \
+			cp -n "$$DOWNLOAD_DIR"/*.tar.gz "$$DOWNLOAD_DIR"/*.tar.xz "$$DOWNLOAD_DIR"/*.zip "$$CACHE_DIR/" 2>/dev/null || true; \
+		fi
+endef
 
 # For public CI runs, checkout folly in a way that can build with RocksDB.
 # This is mostly intended as a test-only simulation of Meta-internal folly
@@ -117,26 +141,11 @@ checkout_folly:
 	@# const mismatch
 	perl -pi -e 's/: environ/: (const char**)(environ)/' third-party/folly/folly/Subprocess.cpp
 	@# Restore cached downloads and handle unreliable mirrors with fallback
-	@cd third-party/folly && \
-		DOWNLOAD_DIR=`$(PYTHON) build/fbcode_builder/getdeps.py show-inst-dir | sed 's|/installed/.*|/downloads|'` && \
-		mkdir -p "$$DOWNLOAD_DIR" && \
-		CACHE_DIR="/tmp/rocksdb-getdeps-cache" && \
-		mkdir -p "$$CACHE_DIR" && \
-		echo "Restoring cached downloads..." && \
-		if ls "$$CACHE_DIR"/*.tar.gz "$$CACHE_DIR"/*.tar.xz "$$CACHE_DIR"/*.zip >/dev/null 2>&1; then \
-			cp -n "$$CACHE_DIR"/*.tar.gz "$$CACHE_DIR"/*.tar.xz "$$CACHE_DIR"/*.zip "$$DOWNLOAD_DIR/" 2>/dev/null || true; \
-		fi && \
-		echo "Handling known unreliable downloads with fallback mirrors..." && \
-		$(PYTHON) ../../build_tools/getdeps_fallback_mirror.py "$$DOWNLOAD_DIR" "$$CACHE_DIR" build/fbcode_builder/manifests
+	$(restore_folly_getdeps_downloads)
 	@# NOTE: boost and fmt source will be needed for any build including `USE_FOLLY_LITE` builds as those depend on those headers
 	cd third-party/folly && GETDEPS_USE_WGET=1 $(PYTHON) build/fbcode_builder/getdeps.py fetch boost && GETDEPS_USE_WGET=1 $(PYTHON) build/fbcode_builder/getdeps.py fetch fmt
 	@# Update cache with any new downloads
-	@cd third-party/folly && \
-		DOWNLOAD_DIR=`$(PYTHON) build/fbcode_builder/getdeps.py show-inst-dir | sed 's|/installed/.*|/downloads|'` && \
-		CACHE_DIR="/tmp/rocksdb-getdeps-cache" && \
-		if ls "$$DOWNLOAD_DIR"/*.tar.gz "$$DOWNLOAD_DIR"/*.tar.xz "$$DOWNLOAD_DIR"/*.zip >/dev/null 2>&1; then \
-			cp -n "$$DOWNLOAD_DIR"/*.tar.gz "$$DOWNLOAD_DIR"/*.tar.xz "$$DOWNLOAD_DIR"/*.zip "$$CACHE_DIR/" 2>/dev/null || true; \
-		fi
+	$(cache_folly_getdeps_downloads)
 
 CXX_M_FLAGS = $(filter -m%, $(CXXFLAGS))
 
@@ -155,6 +164,8 @@ build_folly:
 		echo "Please run checkout_folly first"; \
 		false; \
 	fi
+	@# Restore fallback archives after the cleanup above removes downloads.
+	$(restore_folly_getdeps_downloads)
 	cd third-party/folly && \
 		CXXFLAGS=" $(CXX_M_FLAGS) -DHAVE_CXX11_ATOMIC " GETDEPS_USE_WGET=1 $(PYTHON) build/fbcode_builder/getdeps.py build $(FOLLY_BUILD_FLAGS)
 	@# In the folly build, glog and gflags are only built as dynamic libraries,

include/rocksdb/blob_file_partition_strategy.h

@@ -3,10 +3,6 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
 
-// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.

include/rocksdb/c.h

@@ -1220,11 +1220,15 @@ rocksdb_block_based_options_set_data_block_index_type(
 enum {
   rocksdb_block_based_table_index_block_search_type_binary = 0,
   rocksdb_block_based_table_index_block_search_type_interpolation = 1,
+  rocksdb_block_based_table_index_block_search_type_auto = 2,
 };
 extern ROCKSDB_LIBRARY_API void
 rocksdb_block_based_options_set_index_block_search_type(
     rocksdb_block_based_table_options_t*, int);  // uses one of the above enums
 extern ROCKSDB_LIBRARY_API void
+rocksdb_block_based_options_set_uniform_cv_threshold(
+    rocksdb_block_based_table_options_t*, double);
+extern ROCKSDB_LIBRARY_API void
 rocksdb_block_based_options_set_data_block_hash_ratio(
     rocksdb_block_based_table_options_t* options, double v);
 // rocksdb_block_based_options_set_hash_index_allow_collision()
@@ -1990,6 +1994,12 @@ extern ROCKSDB_LIBRARY_API void rocksdb_options_set_memtable_huge_page_size(
 extern ROCKSDB_LIBRARY_API size_t
 rocksdb_options_get_memtable_huge_page_size(rocksdb_options_t*);
 
+extern ROCKSDB_LIBRARY_API void
+rocksdb_options_set_memtable_batch_lookup_optimization(rocksdb_options_t*,
+                                                       unsigned char);
+extern ROCKSDB_LIBRARY_API unsigned char
+rocksdb_options_get_memtable_batch_lookup_optimization(rocksdb_options_t*);
+
 extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_successive_merges(
     rocksdb_options_t*, size_t);
 extern ROCKSDB_LIBRARY_API size_t
@@ -2215,7 +2225,8 @@ enum {
   rocksdb_filter_block_read_byte,
   rocksdb_compression_dict_block_read_byte,
   rocksdb_metadata_block_read_byte,
-  rocksdb_total_metric_count = 85
+  rocksdb_blob_cache_read_byte,
+  rocksdb_total_metric_count = 86
 };
 
 extern ROCKSDB_LIBRARY_API void rocksdb_set_perf_level(int);
@@ -2395,6 +2406,11 @@ extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_async_io(
     rocksdb_readoptions_t*, unsigned char);
 extern ROCKSDB_LIBRARY_API unsigned char rocksdb_readoptions_get_async_io(
     rocksdb_readoptions_t*);
+extern ROCKSDB_LIBRARY_API void
+rocksdb_readoptions_set_optimize_multiget_for_io(rocksdb_readoptions_t*,
+                                                 unsigned char);
+extern ROCKSDB_LIBRARY_API unsigned char
+rocksdb_readoptions_get_optimize_multiget_for_io(rocksdb_readoptions_t*);
 extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_timestamp(
     rocksdb_readoptions_t*, const char* ts, size_t tslen);
 extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_iter_start_ts(

include/rocksdb/cleanable.h

@@ -107,6 +107,7 @@ class SharedCleanablePtr {
   Cleanable* operator->();
   // Get as raw pointer to Cleanable
   Cleanable* get();
+  const Cleanable* get() const;
 
   // Creates a (virtual) copy of this SharedCleanablePtr and registers its
   // destruction with target, so that the cleanups registered with the

include/rocksdb/convenience.h

@@ -432,9 +432,33 @@ Status GetOptionsFromString(const ConfigOptions& config_options,
                             const Options& base_options,
                             const std::string& opts_str, Options* new_options);
 
+// StringToMap parses a serialized options string into a map. Each
+// resulting map value is in a self-contained form (it can be embedded
+// directly in a `key=value;` context -- e.g. SetOptions -- without further
+// escaping). Specifically: nested braced values from the input are
+// preserved with their outer braces. Permissive: accepts both braced and
+// unbraced forms; values not requiring braces are returned as-is.
+// Example:
+//   "filter_policy={id=ribbonfilter:10;bloom_before_level=-1};block_size=4096"
+// produces:
+//   {filter_policy -> "{id=ribbonfilter:10;bloom_before_level=-1}",
+//    block_size    -> "4096"}
 Status StringToMap(const std::string& opts_str,
                    std::unordered_map<std::string, std::string>* opts_map);
 
+// MapToString is the inverse of StringToMap: a naive `key=value;` join.
+// Each map value must already be in self-contained form (as returned by
+// StringToMap) -- i.e. simple text or a single balanced `{...}` block.
+// Values from StringToMap satisfy this property, so the round-trip
+//
+//   StringToMap(MapToString(StringToMap(s))) == StringToMap(s)
+//
+// holds. Callers building a map by hand are responsible for ensuring
+// values are self-contained; raw values containing `;` or starting with
+// `{` without matching `}` won't round-trip.
+Status MapToString(const std::unordered_map<std::string, std::string>& opts_map,
+                   std::string* opts_str);
+
 // Request stopping background work, if wait is true wait until it's done
 void CancelAllBackgroundWork(DB* db, bool wait = false);
 

include/rocksdb/io_dispatcher.h

@@ -152,6 +152,11 @@ class BlockBasedTable;
 struct JobOptions {
   uint64_t io_coalesce_threshold = 16 * 1024;
   ReadOptions read_options;
+
+  // True when IOJob::block_handles is already sorted by file offset. Callers
+  // that can guarantee sorted order may set this to let IODispatcher skip
+  // defensive sorting before cache lookup and coalescing.
+  bool block_handles_are_sorted = false;
 };
 
 class IOJob {
@@ -255,6 +260,17 @@ class ReadSet {
   Status PollAndProcessAsyncIO(
       const std::shared_ptr<AsyncIOState>& async_state);
 
+  // Release memory budget acquired for a prefetched block.
+  void ReleasePrefetchMemory(size_t block_index);
+
+  // Stop tracking one block from an in-flight async IO request. If this was
+  // the last block using the request, abort and delete the IO handle before the
+  // async state is released.
+  void ReleaseAsyncIOForBlock(size_t block_index);
+
+  // Delete a completed or aborted async IO handle exactly once.
+  void DeleteAsyncIOHandle(const std::shared_ptr<AsyncIOState>& async_state);
+
   // Perform synchronous read for a specific block
   Status SyncRead(size_t block_index);
 

include/rocksdb/listener.h

@@ -964,6 +964,17 @@ class EventListener : public Customizable {
   virtual void OnBackgroundJobPressureChanged(
       DB* /*db*/, const BackgroundJobPressure& /*pressure*/) {}
 
+  // A callback function for RocksDB which will be called once when a DB begins
+  // shutting down, before background work cancellation publishes the DB's
+  // shutdown state. This callback can also fire during cleanup of a failed
+  // DB::Open() attempt, in which case the DB pointer refers to the DB instance
+  // that failed to open and was never returned to the application.
+  //
+  // Background work may still be running when this callback fires, and other
+  // listener callbacks may still run concurrently or afterward. Implementations
+  // should not call blocking DB APIs or run for an extended period of time.
+  virtual void OnDBShutdownBegin(DB* /*db*/) {}
+
   ~EventListener() override {}
 };
 

include/rocksdb/options.h

@@ -52,6 +52,7 @@ class MergeOperator;
 class Snapshot;
 class MemTableRepFactory;
 class RateLimiter;
+class ReadScopedBlockBufferProvider;
 class Slice;
 class Statistics;
 class InternalKeyComparator;
@@ -1003,6 +1004,13 @@ struct DBOptions {
   //
   // This option is mutable with SetDBOptions(), taking effect on the next
   // manifest write (e.g. completed DB compaction or flush).
+  //
+  // For MANIFEST write batches containing foreground operations like external
+  // file ingestion/import, DeleteFilesInRange, CreateColumnFamily, and
+  // DropColumnFamily, the effective limit is relaxed by 25% to reduce the
+  // likelihood of user operations blocking on MANIFEST rotation.
+  // Background-only batches (flush and compaction) use the configured or
+  // auto-tuned limit directly.
   uint64_t max_manifest_file_size = 1024 * 1024 * 1024;
 
   // If true, on DB close, read back the entire MANIFEST file and validate
@@ -2379,6 +2387,30 @@ struct ReadOptions {
   // reads.
   const UserDefinedIndexFactory* table_index_factory = nullptr;
 
+  // EXPERIMENTAL: Optional non-owning provider for data-block storage pinned by
+  // scans using this ReadOptions. Applications that set it are attempting
+  // advanced performance optimizations and are responsible for ensuring the
+  // provider outlives the iterator/read scope and any provider-backed data that
+  // can remain pinned by that scope.
+  //
+  // This is a raw pointer rather than a shared_ptr because ReadOptions is
+  // copied through stack frames and iterator internals; a shared_ptr would add
+  // refcount overhead to those copies. An internal shadow ReadOptions that
+  // strips ownership would add maintenance overhead for this advanced option.
+  //
+  // Current support is limited to block-based table iterators and MultiScan
+  // data-block reads, and is ignored when mmap reads are enabled. When set,
+  // supported scan reads bypass the data-block cache and use provider-backed
+  // final data-block memory. RocksDB may still use ordinary temporary scratch
+  // for serialized block bytes, such as when a block may be compressed. When
+  // unset, scans use the normal RocksDB data-block backing for the table: use
+  // the configured block cache when present, otherwise use RocksDB-owned block
+  // memory. Index/filter blocks keep their normal block-cache behavior.
+  //
+  // TODO: Extend support to point lookups (Get/MultiGet) once those paths can
+  // preserve provider-backed block ownership.
+  ReadScopedBlockBufferProvider* read_scoped_block_buffer_provider = nullptr;
+
   // *** END options only relevant to iterators or scans ***
 
   // *** BEGIN options for RocksDB internal use only ***

include/rocksdb/perf_context.h

@@ -114,6 +114,7 @@ struct PerfContextBase {
   uint64_t iter_read_bytes;      // bytes for keys/vals decoded by iterator
 
   uint64_t blob_cache_hit_count;  // total number of blob cache hits
+  uint64_t blob_cache_read_byte;  // total bytes read from blob cache
   uint64_t blob_read_count;       // total number of blob reads (with IO)
   uint64_t blob_read_byte;        // total number of bytes from blob reads
   uint64_t blob_read_time;        // total nanos spent on blob reads

include/rocksdb/table.h

@@ -23,6 +23,7 @@
 #include <unordered_map>
 
 #include "rocksdb/cache.h"
+#include "rocksdb/cleanable.h"
 #include "rocksdb/customizable.h"
 #include "rocksdb/env.h"
 #include "rocksdb/options.h"
@@ -46,6 +47,67 @@ struct ConfigOptions;
 struct EnvOptions;
 class UserDefinedIndexFactory;
 
+// Hook for providing read-scoped storage for data blocks read from SST files.
+// This is configured through C++ options rather than OPTIONS files.
+//
+// Current support is limited to block-based table iterator scans and MultiScan
+// data-block reads. Reads using mmap ignore this provider and use normal
+// RocksDB block backing.
+//
+// The provider backs final data-block contents pinned by the scan. RocksDB may
+// still use ordinary temporary scratch for serialized block bytes, such as when
+// reading a block that may be compressed before decompressing or copying the
+// final data block into provider-backed storage.
+//
+// This is separate from MemoryAllocator because each allocation needs a
+// per-lease cleanup handle that RocksDB can attach to pinned blocks/slices, and
+// direct-I/O reads that use provider-backed read buffers need the requested
+// alignment to be passed to the provider.
+//
+// TODO: Extend support to point lookups (Get/MultiGet) once those paths can
+// preserve provider-backed block ownership.
+//
+// Requirements:
+// - If the same provider instance is shared by multiple concurrently active
+//   readers, `Allocate()` must be safe to call concurrently.
+// - `Lease::data` must point to at least `size` bytes of writable contiguous
+//   memory that remains valid until every copy of `Lease::cleanup` is reset.
+// - `Lease::data` must be aligned to `alignment` bytes, where `alignment` is a
+//   power of two and `1` means no special alignment requirement.
+// - When `alignment` is greater than 1, `Lease::size` must also be a multiple
+//   of `alignment` so it can be used as direct-I/O backing storage.
+// - `Lease::cleanup` must be non-null on success. Its cleanup may run on any
+//   thread that releases the last RocksDB reference. If the cleanup touches
+//   provider or other shared state, it must synchronize with Allocate() and
+//   other provider cleanup callbacks.
+// - After `Allocate()` succeeds, RocksDB releases `Lease::cleanup` on all
+//   paths, including later I/O or decompression failure.
+class ReadScopedBlockBufferProvider {
+ public:
+  // A Lease hands one writable backing allocation from the provider to
+  // RocksDB. For a provider-backed block, the final BlockContents data points
+  // into this allocation and RocksDB attaches `cleanup` to the resulting Blocks
+  // and any slices pinned from them. File-read scratch, copying, and
+  // decompression choices are implementation details outside this contract.
+  //
+  // The provider controls allocation reclamation. RocksDB keeps the data valid
+  // by copying `cleanup`; the provider must not reuse or release `data` until
+  // every copy of `cleanup` has been reset. `size` is the usable backing
+  // allocation size and may be larger than the requested allocation size. For
+  // direct-I/O reads, `size` must be a multiple of the requested alignment.
+  struct Lease {
+    // Writable contiguous memory for the loaded block.
+    char* data = nullptr;
+    size_t size = 0;
+    // Reclaims `data` after all derived pinned key/value slices are released.
+    SharedCleanablePtr cleanup;
+  };
+
+  virtual ~ReadScopedBlockBufferProvider() = default;
+
+  virtual Status Allocate(size_t size, size_t alignment, Lease* out) = 0;
+};
+
 // Types of checksums to use for checking integrity of logical blocks within
 // files. All checksums currently use 32 bits of checking power (1 in 4B
 // chance of failing to detect random corruption). Traditionally, the actual

include/rocksdb/utilities/options_type.h

@@ -453,11 +453,13 @@ class OptionTypeInfo {
                                        const std::string& value, void* addr) {
           std::map<std::string, std::string> map;
           Status s;
+          // Permit `{k1=v1;k2=v2}` (as serialized) or bare `k1=v1;k2=v2`.
+          std::string stripped = OptionTypeInfo::StripOuterBraces(value);
           for (size_t start = 0, end = 0;
-               s.ok() && start < value.size() && end != std::string::npos;
+               s.ok() && start < stripped.size() && end != std::string::npos;
                start = end + 1) {
             std::string token;
-            s = OptionTypeInfo::NextToken(value, item_separator, start, &end,
+            s = OptionTypeInfo::NextToken(stripped, item_separator, start, &end,
                                           &token);
             if (s.ok() && !token.empty()) {
               size_t pos = token.find(kv_separator);
@@ -947,6 +949,14 @@ class OptionTypeInfo {
   static Status NextToken(const std::string& opts, char delimiter, size_t start,
                           size_t* end, std::string* token);
 
+  // Strips a single layer of outer "{" / "}" wrapping iff the leading '{'
+  // pairs with the trailing '}'. This permissively unwraps values that
+  // could legitimately be either braced or bare (e.g. `{a:b:c}` and
+  // `a:b:c` are accepted equivalently by list-style parsers). The check
+  // is done at brace depth: `{a}:{b}` is left as-is because the leading
+  // '{' closes before the end of the string.
+  static std::string StripOuterBraces(const std::string& value);
+
   constexpr static const char* kIdPropName() { return "id"; }
   constexpr static const char* kIdPropSuffix() { return ".id"; }
 
@@ -995,12 +1005,16 @@ Status ParseArray(const ConfigOptions& config_options,
 
   ConfigOptions copy = config_options;
   copy.ignore_unsupported_options = false;
+  // Permit the caller to pass either bare `item1:item2` or wrapped
+  // `{item1:item2}`. Either form parses identically.
+  std::string stripped = OptionTypeInfo::StripOuterBraces(value);
   size_t i = 0, start = 0, end = 0;
-  for (; status.ok() && i < kSize && start < value.size() &&
+  for (; status.ok() && i < kSize && start < stripped.size() &&
          end != std::string::npos;
        i++, start = end + 1) {
     std::string token;
-    status = OptionTypeInfo::NextToken(value, separator, start, &end, &token);
+    status =
+        OptionTypeInfo::NextToken(stripped, separator, start, &end, &token);
     if (status.ok()) {
       status = elem_info.Parse(copy, name, token, &((*result)[i]));
       if (config_options.ignore_unsupported_options &&
@@ -1137,11 +1151,15 @@ Status ParseVector(const ConfigOptions& config_options,
   // object is valid or not.
   ConfigOptions copy = config_options;
   copy.ignore_unsupported_options = false;
+  // Permit the caller to pass either bare `item1:item2` or wrapped
+  // `{item1:item2}`. Either form parses identically.
+  std::string stripped = OptionTypeInfo::StripOuterBraces(value);
   for (size_t start = 0, end = 0;
-       status.ok() && start < value.size() && end != std::string::npos;
+       status.ok() && start < stripped.size() && end != std::string::npos;
        start = end + 1) {
     std::string token;
-    status = OptionTypeInfo::NextToken(value, separator, start, &end, &token);
+    status =
+        OptionTypeInfo::NextToken(stripped, separator, start, &end, &token);
     if (status.ok()) {
       T elem;
       status = elem_info.Parse(copy, name, token, &elem);

include/rocksdb/utilities/sorted_run_builder.h

@@ -3,11 +3,6 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
 
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
 #pragma once
 
 #include <memory>

include/rocksdb/utilities/transaction.h

@@ -234,6 +234,11 @@ class Transaction {
   // TransactionOptions.expiration. Status::TxnNotPrepared() may be returned if
   // TransactionOptions.skip_prepare is false and Prepare is not called on this
   // transaction before Commit.
+  //
+  // For TransactionDB transactions using 2PC, if Commit() returns non-OK after
+  // Prepare() succeeds, the transaction may still need to be resolved. Unless
+  // the application resolves it another way, call Rollback() before destroying
+  // the transaction or closing the DB.
   virtual Status Commit() = 0;
 
   // In addition to Commit(), also creates a snapshot of the db after all
@@ -259,7 +264,16 @@ class Transaction {
       std::shared_ptr<const Snapshot>* snapshot = nullptr);
 
   // Discard all batched writes in this transaction.
-  // FIXME: what happens if this isn't called before destruction?
+  //
+  // Transactions should normally be completed with Commit() or Rollback()
+  // before destruction and before closing the DB. Destroying a Transaction
+  // object releases its in-memory resources, but it is not a substitute for
+  // resolving a prepared transaction.
+  //
+  // Rollback() can return write errors when it needs to write rollback state,
+  // such as for prepared transactions. If it returns a retryable I/O error,
+  // recover the DB (for example by calling DB::Resume() when applicable) and
+  // retry Rollback() before closing the DB.
   virtual Status Rollback() = 0;
 
   // Records the state of the transaction for future calls to

include/rocksdb/version.h

@@ -12,8 +12,8 @@
 // NOTE: in 'main' development branch, this should be the *next*
 // minor or major version number planned for release.
 #define ROCKSDB_MAJOR 11
-#define ROCKSDB_MINOR 3
-#define ROCKSDB_PATCH 2
+#define ROCKSDB_MINOR 4
+#define ROCKSDB_PATCH 3
 
 // Make it easy to do conditional compilation based on version checks, i.e.
 // #if ROCKSDB_VERSION_GE(4, 5, 6)

java/rocksjni/jni_perf_context.cc

@@ -348,6 +348,18 @@ jlong Java_org_rocksdb_PerfContext_getBlobCacheHitCount(JNIEnv*, jobject,
   return perf_context->blob_cache_hit_count;
 }
 
+/*
+ * Class:     org_rocksdb_PerfContext
+ * Method:    getBlobCacheReadByte
+ * Signature: (J)J
+ */
+jlong Java_org_rocksdb_PerfContext_getBlobCacheReadByte(JNIEnv*, jobject,
+                                                        jlong jpc_handle) {
+  ROCKSDB_NAMESPACE::PerfContext* perf_context =
+      reinterpret_cast<ROCKSDB_NAMESPACE::PerfContext*>(jpc_handle);
+  return perf_context->blob_cache_read_byte;
+}
+
 /*
  * Class:     org_rocksdb_PerfContext
  * Method:    getBlobReadCount

java/src/main/java/org/rocksdb/PerfContext.java

@@ -184,6 +184,13 @@ public class PerfContext extends RocksObject {
     return getBlobCacheHitCount(nativeHandle_);
   }
 
+  /**
+   * @return total number of bytes read from blob cache
+   */
+  public long getBlobCacheReadByte() {
+    return getBlobCacheReadByte(nativeHandle_);
+  }
+
   /**
    * @return total number of blob reads (with IO)
    */
@@ -693,6 +700,7 @@ public class PerfContext extends RocksObject {
   private native long getMultigetReadBytes(final long handle);
   private native long getIterReadBytes(final long handle);
   private native long getBlobCacheHitCount(final long handle);
+  private native long getBlobCacheReadByte(final long handle);
   private native long getBlobReadCount(final long handle);
   private native long getBlobReadByte(final long handle);
   private native long getBlobReadTime(final long handle);

monitoring/perf_context.cc

@@ -78,6 +78,7 @@ struct PerfContextByLevelInt {
   defCmd(multiget_read_bytes)                      \
   defCmd(iter_read_bytes)                          \
   defCmd(blob_cache_hit_count)                     \
+  defCmd(blob_cache_read_byte)                     \
   defCmd(blob_read_count)                          \
   defCmd(blob_read_byte)                           \
   defCmd(blob_read_time)                           \

options/db_options.cc

@@ -557,11 +557,13 @@ static std::unordered_map<std::string, OptionTypeInfo>
             embedded.ignore_unsupported_options = true;
             std::vector<std::shared_ptr<EventListener>> listeners;
             Status s;
+            // Permit `{item1:item2}` or bare `item1:item2`.
+            std::string stripped = OptionTypeInfo::StripOuterBraces(value);
             for (size_t start = 0, end = 0;
-                 s.ok() && start < value.size() && end != std::string::npos;
+                 s.ok() && start < stripped.size() && end != std::string::npos;
                  start = end + 1) {
               std::string token;
-              s = OptionTypeInfo::NextToken(value, ':', start, &end, &token);
+              s = OptionTypeInfo::NextToken(stripped, ':', start, &end, &token);
               if (s.ok() && !token.empty()) {
                 std::shared_ptr<EventListener> listener;
                 s = EventListener::CreateFromString(embedded, token, &listener);

options/options_helper.cc

@@ -770,6 +770,11 @@ Status StringToMap(const std::string& opts_str,
   // Example:
   //   opts_str = "write_buffer_size=1024;max_write_buffer_number=2;"
   //              "nested_opt={opt1=1;opt2=2};max_bytes_for_level_base=100"
+  //
+  // Each value in the resulting map can be substituted directly into a
+  // `key=value;` context (e.g. SetOptions) and re-parsed without ambiguity:
+  // values that were wrapped in `{...}` in the input keep their wrapping,
+  // so embedded `;` characters stay escaped.
   size_t pos = 0;
   std::string opts = trim(opts_str);
   // If the input string starts and ends with "{...}", strip off the brackets
@@ -790,23 +795,78 @@ Status StringToMap(const std::string& opts_str,
       return Status::InvalidArgument("Empty key found");
     }
 
+    // Locate value start (after '=' and any whitespace).
+    size_t value_start = eq_pos + 1;
+    while (value_start < opts.size() && isspace(opts[value_start])) {
+      ++value_start;
+    }
+
     std::string value;
-    Status s = OptionTypeInfo::NextToken(opts, ';', eq_pos + 1, &pos, &value);
-    if (!s.ok()) {
-      return s;
-    } else {
-      (*opts_map)[key] = value;
-      if (pos == std::string::npos) {
-        break;
-      } else {
-        pos++;
+    if (value_start < opts.size() && opts[value_start] == '{') {
+      // Braced value: extract the entire `{...}` substring INCLUDING braces
+      // so that the value can be re-emitted as-is in a `key=value;` context.
+      int count = 1;
+      size_t brace_pos = value_start + 1;
+      while (brace_pos < opts.size() && count > 0) {
+        if (opts[brace_pos] == '{') {
+          ++count;
+        } else if (opts[brace_pos] == '}') {
+          --count;
+        }
+        if (count > 0) {
+          ++brace_pos;
+        }
       }
+      if (count != 0) {
+        return Status::InvalidArgument(
+            "Mismatched curly braces for nested options");
+      }
+      value = opts.substr(value_start, brace_pos - value_start + 1);
+      pos = brace_pos + 1;
+      // Allow whitespace, then either delimiter or end.
+      while (pos < opts.size() && isspace(opts[pos])) {
+        ++pos;
+      }
+      if (pos < opts.size() && opts[pos] != ';') {
+        return Status::InvalidArgument("Unexpected chars after nested options");
+      }
+    } else {
+      // Non-braced: scan to the next ';'. The value may contain '=' (only
+      // the first '=' separates key from value).
+      size_t end = opts.find(';', value_start);
+      if (end == std::string::npos) {
+        value = trim(opts.substr(value_start));
+        pos = std::string::npos;
+      } else {
+        value = trim(opts.substr(value_start, end - value_start));
+        pos = end;
+      }
+    }
+
+    (*opts_map)[key] = value;
+    if (pos == std::string::npos) {
+      break;
+    } else {
+      pos++;
     }
   }
 
   return Status::OK();
 }
 
+Status MapToString(const std::unordered_map<std::string, std::string>& opts_map,
+                   std::string* opts_str) {
+  assert(opts_str);
+  opts_str->clear();
+  for (const auto& [key, value] : opts_map) {
+    opts_str->append(key);
+    opts_str->append("=");
+    opts_str->append(value);
+    opts_str->append(";");
+  }
+  return Status::OK();
+}
+
 Status GetStringFromDBOptions(std::string* opt_string,
                               const DBOptions& db_options,
                               const std::string& delimiter) {
@@ -1053,6 +1113,32 @@ Status OptionTypeInfo::NextToken(const std::string& opts, char delimiter,
   return Status::OK();
 }
 
+std::string OptionTypeInfo::StripOuterBraces(const std::string& value) {
+  if (value.size() < 2 || value.front() != '{' || value.back() != '}') {
+    return value;
+  }
+  // Verify the leading '{' actually pairs with the trailing '}'.
+  // For `{a:b}` the leading brace closes only at the trailing `}` (one
+  // matching pair, OK to strip). For `{a}:{b}` the leading brace closes
+  // at the first inner `}`, so the leading and trailing braces are
+  // independent -- leave the value alone. We strip at most one layer:
+  // each layer of `{}` corresponds to one level of nesting that the
+  // encoder added for that level, and the typed parser at this level
+  // wants to peel exactly its own layer.
+  int depth = 0;
+  for (size_t i = 0; i + 1 < value.size(); ++i) {
+    if (value[i] == '{') {
+      ++depth;
+    } else if (value[i] == '}') {
+      --depth;
+      if (depth == 0) {
+        return value;  // outer braces are independent
+      }
+    }
+  }
+  return value.substr(1, value.size() - 2);
+}
+
 Status OptionTypeInfo::Parse(const ConfigOptions& config_options,
                              const std::string& opt_name,
                              const std::string& value, void* opt_ptr) const {
@@ -1071,7 +1157,13 @@ Status OptionTypeInfo::Parse(const ConfigOptions& config_options,
       copy.invoke_prepare_options = false;
       void* opt_addr = GetOffset(opt_ptr);
       return parse_func_(copy, opt_name, opt_value, opt_addr);
-    } else if (ParseOptionHelper(GetOffset(opt_ptr), type_, opt_value)) {
+    } else if (ParseOptionHelper(GetOffset(opt_ptr), type_,
+                                 StripOuterBraces(opt_value))) {
+      // Scalar types (int, bool, enum, string, ...). StringToMap now
+      // preserves outer braces in nested values, so a hand-crafted
+      // `key={42}` (or `db_log_dir={/tmp}`) lands here with a wrap
+      // that scalar parsers don't expect; permissively strip one
+      // level so braced scalar input still parses as before.
       return Status::OK();
     } else if (IsConfigurable()) {
       // The option is <config>.<name>

options/options_test.cc

@@ -1938,11 +1938,13 @@ TEST_F(OptionsTest, StringToMapTest) {
   ASSERT_EQ(opts_map["k2"], "");
   ASSERT_TRUE(opts_map.find("k3") != opts_map.end());
   ASSERT_EQ(opts_map["k3"], "");
-  // Regular nested options
+  // Regular nested options. Braces are preserved on values that came in
+  // braced, so the value is self-contained for direct embedding via
+  // `key=value;` and round-trips through StringToMap/MapToString.
   opts_map.clear();
   ASSERT_OK(StringToMap("k1=v1;k2={nk1=nv1;nk2=nv2};k3=v3", &opts_map));
   ASSERT_EQ(opts_map["k1"], "v1");
-  ASSERT_EQ(opts_map["k2"], "nk1=nv1;nk2=nv2");
+  ASSERT_EQ(opts_map["k2"], "{nk1=nv1;nk2=nv2}");
   ASSERT_EQ(opts_map["k3"], "v3");
   // Multi-level nested options
   opts_map.clear();
@@ -1951,34 +1953,35 @@ TEST_F(OptionsTest, StringToMapTest) {
                   "k3={nk1={nnk1={nnnk1=nnnv1;nnnk2;nnnv2}}};k4=v4",
                   &opts_map));
   ASSERT_EQ(opts_map["k1"], "v1");
-  ASSERT_EQ(opts_map["k2"], "nk1=nv1;nk2={nnk1=nnk2}");
-  ASSERT_EQ(opts_map["k3"], "nk1={nnk1={nnnk1=nnnv1;nnnk2;nnnv2}}");
+  ASSERT_EQ(opts_map["k2"], "{nk1=nv1;nk2={nnk1=nnk2}}");
+  ASSERT_EQ(opts_map["k3"], "{nk1={nnk1={nnnk1=nnnv1;nnnk2;nnnv2}}}");
   ASSERT_EQ(opts_map["k4"], "v4");
   // Garbage inside curly braces
   opts_map.clear();
   ASSERT_OK(StringToMap("k1=v1;k2={dfad=};k3={=};k4=v4", &opts_map));
   ASSERT_EQ(opts_map["k1"], "v1");
-  ASSERT_EQ(opts_map["k2"], "dfad=");
-  ASSERT_EQ(opts_map["k3"], "=");
+  ASSERT_EQ(opts_map["k2"], "{dfad=}");
+  ASSERT_EQ(opts_map["k3"], "{=}");
   ASSERT_EQ(opts_map["k4"], "v4");
   // Empty nested options
   opts_map.clear();
   ASSERT_OK(StringToMap("k1=v1;k2={};", &opts_map));
   ASSERT_EQ(opts_map["k1"], "v1");
-  ASSERT_EQ(opts_map["k2"], "");
+  ASSERT_EQ(opts_map["k2"], "{}");
   opts_map.clear();
   ASSERT_OK(StringToMap("k1=v1;k2={{{{}}}{}{}};", &opts_map));
   ASSERT_EQ(opts_map["k1"], "v1");
-  ASSERT_EQ(opts_map["k2"], "{{{}}}{}{}");
-  // With random spaces
+  ASSERT_EQ(opts_map["k2"], "{{{{}}}{}{}}");
+  // With random spaces. Internal whitespace is preserved verbatim now
+  // (we keep the original substring of the value rather than trimming).
   opts_map.clear();
   ASSERT_OK(
       StringToMap("  k1 =  v1 ; k2= {nk1=nv1; nk2={nnk1=nnk2}}  ; "
                   "k3={  {   } }; k4= v4  ",
                   &opts_map));
   ASSERT_EQ(opts_map["k1"], "v1");
-  ASSERT_EQ(opts_map["k2"], "nk1=nv1; nk2={nnk1=nnk2}");
-  ASSERT_EQ(opts_map["k3"], "{   }");
+  ASSERT_EQ(opts_map["k2"], "{nk1=nv1; nk2={nnk1=nnk2}}");
+  ASSERT_EQ(opts_map["k3"], "{  {   } }");
   ASSERT_EQ(opts_map["k4"], "v4");
 
   // Empty key
@@ -2008,6 +2011,165 @@ TEST_F(OptionsTest, StringToMapTest) {
   ASSERT_NOK(StringToMap("k1=v1;k2={{dfdl}adfa}{}", &opts_map));
 }
 
+TEST_F(OptionsTest, EmptyBracedVectorAndBracedScalarTest) {
+  // An empty braced list value (`key={}`) must parse as an empty
+  // collection, not as a vector with one empty element. StringToMap
+  // preserves the braces, so the typed vector parser sees `{}`; its
+  // outer-brace strip needs to handle the size-2 case.
+  ColumnFamilyOptions cf_opts;
+  ConfigOptions cfg;
+  ASSERT_OK(GetColumnFamilyOptionsFromString(
+      cfg, ColumnFamilyOptions(), "compression_per_level={};", &cf_opts));
+  EXPECT_TRUE(cf_opts.compression_per_level.empty());
+
+  // A scalar value wrapped in `{}` (e.g. hand-crafted `key={42}`) must
+  // still parse. StringToMap preserves the braces, so the scalar
+  // dispatch in OptionTypeInfo::Parse permissively strips one outer
+  // layer.
+  DBOptions db_opts;
+  ASSERT_OK(GetDBOptionsFromString(cfg, DBOptions(), "max_open_files={42};",
+                                   &db_opts));
+  EXPECT_EQ(db_opts.max_open_files, 42);
+
+  ASSERT_OK(GetDBOptionsFromString(cfg, DBOptions(),
+                                   "create_if_missing={true};", &db_opts));
+  EXPECT_TRUE(db_opts.create_if_missing);
+}
+
+TEST_F(OptionsTest, MapToStringTest) {
+  using Map = std::unordered_map<std::string, std::string>;
+  std::string s;
+
+  // Simple values.
+  Map simple{{"a", "1"}, {"b", "hello"}};
+  ASSERT_OK(MapToString(simple, &s));
+  Map reparsed;
+  ASSERT_OK(StringToMap(s, &reparsed));
+  ASSERT_EQ(reparsed, simple);
+
+  // A value already in self-contained braced form (as returned by
+  // StringToMap) passes through unchanged. No extra wrapping is needed
+  // for round-trip.
+  Map already_braced{{"k", "{a=b;c=d}"}};
+  ASSERT_OK(MapToString(already_braced, &s));
+  ASSERT_EQ(s, "k={a=b;c=d};");
+  Map reparsed_braced;
+  ASSERT_OK(StringToMap(s, &reparsed_braced));
+  ASSERT_EQ(reparsed_braced, already_braced);
+}
+
+TEST_F(OptionsTest, StringToMapMapToStringRoundTripTest) {
+  using Map = std::unordered_map<std::string, std::string>;
+  // Pull a single entry out of a StringToMap-ed map and re-embed it
+  // directly into a `key=value;` SetOptions-style string. The pulled
+  // value must round-trip without any extra escaping by the caller --
+  // this is the property that lets users compose option strings from
+  // map entries safely.
+  const std::string original =
+      "filter_policy={id=ribbonfilter:10:-1;bloom_before_level=-1;};"
+      "block_size=4096;cache_index_and_filter_blocks=true";
+
+  Map parsed;
+  ASSERT_OK(StringToMap(original, &parsed));
+  // The filter_policy value retains its braces -- it's self-contained.
+  EXPECT_EQ(parsed["filter_policy"],
+            "{id=ribbonfilter:10:-1;bloom_before_level=-1;}");
+  EXPECT_EQ(parsed["block_size"], "4096");
+  EXPECT_EQ(parsed["cache_index_and_filter_blocks"], "true");
+
+  // Pull a single entry out and embed it in a fresh `key=value;`
+  // string. No extra wrapping required by the caller.
+  const std::string embedded =
+      "filter_policy=" + parsed["filter_policy"] + ";other_opt=42";
+  Map reparsed;
+  ASSERT_OK(StringToMap(embedded, &reparsed));
+  EXPECT_EQ(reparsed["filter_policy"], parsed["filter_policy"]);
+  EXPECT_EQ(reparsed["other_opt"], "42");
+
+  // Whole-map round-trip via MapToString.
+  std::string regenerated;
+  ASSERT_OK(MapToString(parsed, &regenerated));
+  Map reparsed2;
+  ASSERT_OK(StringToMap(regenerated, &reparsed2));
+  EXPECT_EQ(reparsed2, parsed);
+}
+
+TEST_F(OptionsTest, FullOptionsStringToMapRoundTripTest) {
+  // Round-trip a populated DBOptions and CFOptions through the
+  // StringToMap -> MapToString path. This catches any custom option
+  // serializer that emits a value containing the StringToMap delimiter
+  // (';') without enclosing it in '{}' -- such a value would survive
+  // StringToMap as a self-contained entry but corrupt subsequent
+  // re-serialization. Existing typed tests
+  // (ColumnFamilyOptionsSerialization, DBOptionsSerialization) cover the
+  // typed Configurable round-trip; this test specifically exercises the
+  // public StringToMap + MapToString path that callers use to compose,
+  // mutate, and re-emit option strings.
+  Random rnd(607);
+  ConfigOptions config_options;
+  config_options.input_strings_escaped = false;
+
+  // ---- ColumnFamilyOptions ----
+  Options options;
+  ColumnFamilyOptions base_cf_opt;
+  base_cf_opt.comparator = test::BytewiseComparatorWithU64TsWrapper();
+  test::RandomInitCFOptions(&base_cf_opt, options, &rnd);
+  std::string cf_str;
+  ASSERT_OK(
+      GetStringFromColumnFamilyOptions(config_options, base_cf_opt, &cf_str));
+
+  std::unordered_map<std::string, std::string> cf_map;
+  ASSERT_OK(StringToMap(cf_str, &cf_map));
+  // Each entry must be embeddable as-is in a `key=value;` context.
+  for (const auto& [k, v] : cf_map) {
+    std::string solo;
+    solo.append(k).append("=").append(v).append(";");
+    std::unordered_map<std::string, std::string> solo_map;
+    ASSERT_OK(StringToMap(solo, &solo_map))
+        << "single-entry round-trip failed for " << k << ": " << v;
+    ASSERT_EQ(solo_map.size(), 1u) << k << "=" << v;
+    ASSERT_EQ(solo_map[k], v) << "single-entry value mismatch for " << k;
+  }
+  // Whole-map MapToString must reproduce a string parseable into a CF
+  // options object equivalent to the original.
+  std::string cf_round;
+  ASSERT_OK(MapToString(cf_map, &cf_round));
+  ColumnFamilyOptions cf_back;
+  ASSERT_OK(GetColumnFamilyOptionsFromString(
+      config_options, ColumnFamilyOptions(), cf_round, &cf_back));
+  ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt,
+                                                  cf_back));
+
+  // ---- DBOptions ----
+  DBOptions base_db_opt;
+  test::RandomInitDBOptions(&base_db_opt, &rnd);
+  std::string db_str;
+  ASSERT_OK(GetStringFromDBOptions(config_options, base_db_opt, &db_str));
+
+  std::unordered_map<std::string, std::string> db_map;
+  ASSERT_OK(StringToMap(db_str, &db_map));
+  for (const auto& [k, v] : db_map) {
+    std::string solo;
+    solo.append(k).append("=").append(v).append(";");
+    std::unordered_map<std::string, std::string> solo_map;
+    ASSERT_OK(StringToMap(solo, &solo_map))
+        << "single-entry round-trip failed for " << k << ": " << v;
+    ASSERT_EQ(solo_map.size(), 1u) << k << "=" << v;
+    ASSERT_EQ(solo_map[k], v) << "single-entry value mismatch for " << k;
+  }
+  std::string db_round;
+  ASSERT_OK(MapToString(db_map, &db_round));
+  DBOptions db_back;
+  ASSERT_OK(
+      GetDBOptionsFromString(config_options, DBOptions(), db_round, &db_back));
+  ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(config_options, base_db_opt,
+                                                  db_back));
+
+  // RandomInitCFOptions allocates a raw CompactionFilter*; the
+  // ColumnFamilyOptions destructor doesn't own it.
+  delete base_cf_opt.compaction_filter;
+}
+
 TEST_F(OptionsTest, StringToMapRandomTest) {
   std::unordered_map<std::string, std::string> opts_map;
   // Make sure segfault is not hit by semi-random strings
@@ -2028,6 +2190,15 @@ TEST_F(OptionsTest, StringToMapRandomTest) {
         str[pos] = ' ';
         Status s = StringToMap(str, &opts_map);
         ASSERT_TRUE(s.ok() || s.IsInvalidArgument());
+        if (s.ok()) {
+          // Round-trip: StringToMap entries are self-contained, so
+          // MapToString of them must parse back to the same map.
+          std::string regen;
+          ASSERT_OK(MapToString(opts_map, &regen));
+          std::unordered_map<std::string, std::string> reparsed;
+          ASSERT_OK(StringToMap(regen, &reparsed));
+          ASSERT_EQ(reparsed, opts_map) << "regen=" << regen;
+        }
         opts_map.clear();
       }
     }
@@ -2047,8 +2218,22 @@ TEST_F(OptionsTest, StringToMapRandomTest) {
     }
     Status s = StringToMap(str, &opts_map);
     ASSERT_TRUE(s.ok() || s.IsInvalidArgument());
+    if (s.ok()) {
+      std::string regen;
+      ASSERT_OK(MapToString(opts_map, &regen));
+      std::unordered_map<std::string, std::string> reparsed;
+      ASSERT_OK(StringToMap(regen, &reparsed));
+      ASSERT_EQ(reparsed, opts_map) << "regen=" << regen;
+    }
     s = StringToMap("name=" + str, &opts_map);
     ASSERT_TRUE(s.ok() || s.IsInvalidArgument());
+    if (s.ok()) {
+      std::string regen;
+      ASSERT_OK(MapToString(opts_map, &regen));
+      std::unordered_map<std::string, std::string> reparsed;
+      ASSERT_OK(StringToMap(regen, &reparsed));
+      ASSERT_EQ(reparsed, opts_map) << "regen=" << regen;
+    }
     opts_map.clear();
   }
 }

src.mk

@@ -506,6 +506,7 @@ TEST_MAIN_SOURCES =                                                     \
   db/db_clip_test.cc                                                    \
   db/db_dynamic_level_test.cc                                           \
   db/db_encryption_test.cc                                              \
+  db/db_etc2_test.cc                                                    \
   db/db_etc3_test.cc                                                    \
   db/db_flush_test.cc                                                   \
   db/db_follower_test.cc						                                    \
@@ -533,7 +534,6 @@ TEST_MAIN_SOURCES =                                                     \
   db/db_table_properties_test.cc                                        \
   db/db_tailing_iter_test.cc                                            \
   db/db_test.cc                                                         \
-  db/db_test2.cc                                                        \
   db/db_logical_block_size_cache_test.cc                                \
   db/db_universal_compaction_test.cc                                    \
   db/db_wal_test.cc                                                     \

table/block_based/block_based_table_iterator.cc

@@ -1077,6 +1077,9 @@ void BlockBasedTableIterator::Prepare(const MultiScanArgs* multiscan_opts) {
       multiscan_opts->io_coalesce_threshold;
   job->job_options.read_options = read_options_;
   job->job_options.read_options.async_io = multiscan_opts->use_async_io;
+  // CollectBlockHandles walks sorted scan ranges through the table index, whose
+  // key order matches data block offset order.
+  job->job_options.block_handles_are_sorted = true;
 
   std::shared_ptr<ReadSet> read_set;
   // IODispatcher should be provided by DBIter::Prepare() to enable sharing
@@ -1171,14 +1174,24 @@ Status BlockBasedTableIterator::CollectBlockHandles(
 
     if (index_iter_->Valid()) {
       // Handle the last block when its separator is equal or larger than limit
-      if (check_overlap &&
-          scan_block_handles->back() == index_iter_->value().handle) {
-        // Skip adding the current block since it's already in the list
-      } else {
-        scan_block_handles->push_back(index_iter_->value().handle);
-        data_block_separators->push_back(index_iter_->user_key().ToString());
+      IndexValue index_value = index_iter_->value();
+      bool add_block = true;
+      if (scan_opt.range.limit.has_value() &&
+          !index_value.first_internal_key.empty()) {
+        Slice first_user_key = ExtractUserKey(index_value.first_internal_key);
+        add_block = user_comparator_.CompareWithoutTimestamp(
+                        first_user_key, /*a_has_ts=*/true,
+                        *scan_opt.range.limit, /*b_has_ts=*/false) < 0;
+      }
+      if (add_block) {
+        if (check_overlap && scan_block_handles->back() == index_value.handle) {
+          // Skip adding the current block since it's already in the list
+        } else {
+          scan_block_handles->push_back(index_value.handle);
+          data_block_separators->push_back(index_iter_->user_key().ToString());
+        }
+        ++num_blocks;
       }
-      ++num_blocks;
     }
     block_index_ranges_per_scan->emplace_back(
         scan_block_handles->size() - num_blocks, scan_block_handles->size());

table/block_based/block_based_table_reader.cc

@@ -87,6 +87,125 @@ CacheAllocationPtr CopyBufferToHeap(MemoryAllocator* allocator, Slice& buf) {
 }
 }  // namespace
 
+Status AllocateReadScopedBlockBuffer(
+    ReadScopedBlockBufferProvider& block_buffer_provider, size_t requested_size,
+    size_t requested_alignment, ReadScopedBlockBufferProvider::Lease* lease) {
+  assert(lease != nullptr);
+
+  Status s = block_buffer_provider.Allocate(requested_size, requested_alignment,
+                                            lease);
+  if (!s.ok()) {
+    return s;
+  }
+  if (lease->data == nullptr || lease->size < requested_size ||
+      lease->cleanup.get() == nullptr) {
+    return Status::InvalidArgument(
+        "ReadScopedBlockBufferProvider returned an invalid lease");
+  }
+  if (requested_alignment > 1 && (reinterpret_cast<uintptr_t>(lease->data) &
+                                  (requested_alignment - 1)) != 0) {
+    return Status::InvalidArgument(
+        "ReadScopedBlockBufferProvider returned a misaligned lease");
+  }
+  return Status::OK();
+}
+
+Status AllocateReadScopedAlignedBuffer(
+    ReadScopedBlockBufferProvider& block_buffer_provider, size_t size,
+    size_t alignment, ReadScopedBlockBufferProvider::Lease* lease,
+    AlignedBuffer::ExternalAllocation* out) {
+  if (out == nullptr) {
+    return Status::InvalidArgument(
+        "AlignedBuffer allocation output is nullptr");
+  }
+  Status s = AllocateReadScopedBlockBuffer(block_buffer_provider, size,
+                                           alignment, lease);
+  if (!s.ok()) {
+    return s;
+  }
+
+  SharedCleanablePtr owner_cleanup = lease->cleanup;
+  out->data = lease->data;
+  out->size = lease->size;
+  out->owner = FSAllocationPtr(
+      lease->data, [owner_cleanup = std::move(owner_cleanup)](void*) mutable {
+        owner_cleanup.Reset();
+      });
+  return Status::OK();
+}
+
+AlignedBuffer::Allocator MakeReadScopedAlignedBufferAllocator(
+    ReadScopedBlockBufferProviderRef block_buffer_provider,
+    ReadScopedBlockBufferProvider::Lease* lease) {
+  assert(block_buffer_provider.has_value());
+  assert(lease != nullptr);
+  return [block_buffer_provider, lease](
+             size_t size, size_t alignment,
+             AlignedBuffer::ExternalAllocation* out) -> Status {
+    assert(block_buffer_provider.has_value());
+    return AllocateReadScopedAlignedBuffer(block_buffer_provider->get(), size,
+                                           alignment, lease, out);
+  };
+}
+
+ReadScopedBlockBufferProviderRef GetReadScopedBlockBufferProvider(
+    const ReadOptions& ro, bool allow_mmap_reads) {
+  if (allow_mmap_reads || ro.read_scoped_block_buffer_provider == nullptr) {
+    return std::nullopt;
+  }
+  return std::ref(*ro.read_scoped_block_buffer_provider);
+}
+
+bool ShouldUseDataBlockCacheForIterator(
+    const BlockBasedTableOptions& table_options, const ReadOptions& ro,
+    bool allow_mmap_reads) {
+  if (GetReadScopedBlockBufferProvider(ro, allow_mmap_reads).has_value()) {
+    // Provider-backed scan reads use caller-owned read-scope storage as the
+    // data-block backing, so they intentionally skip both lookup and insertion
+    // in the shared data-block cache.
+    return false;
+  }
+  return table_options.block_cache != nullptr;
+}
+
+Status CopyBufferToHeapBlockContents(Slice src, size_t data_size,
+                                     MemoryAllocator* allocator,
+                                     BlockContents* out_contents) {
+  assert(out_contents != nullptr);
+  assert(data_size <= src.size());
+
+  *out_contents = BlockContents(CopyBufferToHeap(allocator, src), data_size);
+#ifndef NDEBUG
+  out_contents->has_trailer = src.size() > data_size;
+#endif
+  return Status::OK();
+}
+
+Status CopyBufferToReadScopedBlockContents(
+    Slice src, size_t data_size,
+    ReadScopedBlockBufferProvider& block_buffer_provider,
+    BlockContents* out_contents) {
+  assert(out_contents != nullptr);
+  assert(data_size <= src.size());
+
+  ReadScopedBlockBufferProvider::Lease read_scoped_lease;
+  Status s = AllocateReadScopedBlockBuffer(block_buffer_provider, src.size(), 1,
+                                           &read_scoped_lease);
+  if (!s.ok()) {
+    return s;
+  }
+
+  memcpy(read_scoped_lease.data, src.data(), src.size());
+  out_contents->data = Slice(read_scoped_lease.data, data_size);
+  out_contents->cleanup = std::move(read_scoped_lease.cleanup);
+  out_contents->backing_size = read_scoped_lease.size;
+  out_contents->AssertSingleOwner();
+#ifndef NDEBUG
+  out_contents->has_trailer = src.size() > data_size;
+#endif
+  return Status::OK();
+}
+
 // Explicitly instantiate templates for each "blocklike" type we use (and
 // before implicit specialization).
 // This makes it possible to keep the template definitions in the .cc file.
@@ -204,7 +323,8 @@ Status ReadAndParseBlockFromFile(
     BlockCreateContext& create_context, bool maybe_compressed,
     UnownedPtr<Decompressor> decomp,
     const PersistentCacheOptions& cache_options,
-    MemoryAllocator* memory_allocator, bool for_compaction, bool async_read) {
+    MemoryAllocator* memory_allocator, bool for_compaction, bool async_read,
+    ReadScopedBlockBufferProviderRef block_buffer_provider = std::nullopt) {
   assert(result);
 
   BlockContents contents;
@@ -212,7 +332,7 @@ Status ReadAndParseBlockFromFile(
       file, prefetch_buffer, footer, options, handle, &contents, ioptions,
       /*do_uncompress*/ maybe_compressed, maybe_compressed,
       TBlocklike::kBlockType, decomp, cache_options, memory_allocator, nullptr,
-      for_compaction);
+      for_compaction, block_buffer_provider);
   Status s;
   // If prefetch_buffer is not allocated, it will fallback to synchronous
   // reading of block contents.
@@ -1715,7 +1835,14 @@ Status BlockBasedTable::LookupAndPinBlocksInCache(
   BlockCacheInterface<TBlocklike> block_cache{
       rep_->table_options.block_cache.get()};
 
-  assert(block_cache);
+  TEST_SYNC_POINT("BlockBasedTable::LookupAndPinBlocksInCache:Start");
+
+  if (!block_cache) {
+    // Callers can reach here after data-block cache use has been disabled by
+    // options such as read-scoped provider-backed scans. Treat the lookup as a
+    // cache miss rather than requiring every caller to special-case null cache.
+    return Status::OK();
+  }
 
   Status s;
   CachableEntry<DecompressorDict> cached_dict;
@@ -2096,9 +2223,8 @@ WithBlocklikeCheck<Status, TBlocklike> BlockBasedTable::RetrieveBlock(
   assert(out_parsed_block);
   assert(out_parsed_block->IsEmpty());
 
-  Status s;
   if (use_cache) {
-    s = MaybeReadBlockAndLoadToCache(
+    Status s = MaybeReadBlockAndLoadToCache(
         prefetch_buffer, ro, handle, decomp, for_compaction, out_parsed_block,
         get_context, lookup_context,
         /*contents=*/nullptr, async_read, use_block_cache_for_lookup);
@@ -2124,16 +2250,22 @@ WithBlocklikeCheck<Status, TBlocklike> BlockBasedTable::RetrieveBlock(
   const bool maybe_compressed =
       BlockTypeMaybeCompressed(TBlocklike::kBlockType) && rep_->decompressor;
   std::unique_ptr<TBlocklike> block;
+  Status s;
 
   {
     Histograms histogram =
         for_compaction ? READ_BLOCK_COMPACTION_MICROS : READ_BLOCK_GET_MICROS;
     StopWatch sw(rep_->ioptions.clock, rep_->ioptions.stats, histogram);
+    ReadScopedBlockBufferProviderRef block_buffer_provider =
+        (!use_cache && TBlocklike::kBlockType == BlockType::kData)
+            ? GetReadScopedBlockBufferProvider(ro,
+                                               rep_->ioptions.allow_mmap_reads)
+            : std::nullopt;
     s = ReadAndParseBlockFromFile(
         rep_->file.get(), prefetch_buffer, rep_->footer, ro, handle, &block,
         rep_->ioptions, rep_->create_context, maybe_compressed, decomp,
         rep_->persistent_cache_options, GetMemoryAllocator(rep_->table_options),
-        for_compaction, async_read);
+        for_compaction, async_read, block_buffer_provider);
 
     if (get_context) {
       switch (TBlocklike::kBlockType) {
@@ -3388,9 +3520,9 @@ void BlockBasedTable::DumpBlockChecksumInfo(const BlockHandle& block_handle,
   IODebugContext dbg;
   IOStatus io_s = rep_->file->PrepareIOOptions(read_options, opts, &dbg);
   if (io_s.ok()) {
-    io_s = rep_->file->Read(opts, block_handle.offset(),
-                            block_size_with_trailer, &raw_block_slice,
-                            raw_block.get(), /*aligned_buf=*/nullptr, &dbg);
+    io_s =
+        rep_->file->Read(opts, block_handle.offset(), block_size_with_trailer,
+                         &raw_block_slice, raw_block.get(), nullptr, &dbg);
   }
   if (io_s.ok() && raw_block_slice.size() == block_size_with_trailer) {
     const char* data = raw_block_slice.data();

table/block_based/block_based_table_reader.h

@@ -33,6 +33,7 @@
 #include "table/table_reader.h"
 #include "table/two_level_iterator.h"
 #include "trace_replay/block_cache_tracer.h"
+#include "util/aligned_buffer.h"
 #include "util/atomic.h"
 #include "util/cast_util.h"
 #include "util/coro_utils.h"
@@ -57,6 +58,55 @@ class GetContext;
 
 using KVPairBlock = std::vector<std::pair<std::string, std::string>>;
 
+// Calls the provider and checks that a successful allocation returned a lease
+// satisfying RocksDB's size, alignment, data pointer, and cleanup contract.
+Status AllocateReadScopedBlockBuffer(
+    ReadScopedBlockBufferProvider& block_buffer_provider, size_t requested_size,
+    size_t requested_alignment, ReadScopedBlockBufferProvider::Lease* lease);
+
+// Allocates a read-scoped provider lease and exposes it as an AlignedBuffer
+// external allocation. The AlignedBuffer owner keeps a cleanup reference so the
+// provider allocation remains valid while direct I/O can write into it.
+Status AllocateReadScopedAlignedBuffer(
+    ReadScopedBlockBufferProvider& block_buffer_provider, size_t size,
+    size_t alignment, ReadScopedBlockBufferProvider::Lease* lease,
+    AlignedBuffer::ExternalAllocation* out);
+
+// Returns an allocator whose direct-I/O allocation comes from the read-scoped
+// provider. The caller must keep `lease` alive until the file reader
+// synchronously invokes the allocator.
+AlignedBuffer::Allocator MakeReadScopedAlignedBufferAllocator(
+    ReadScopedBlockBufferProviderRef block_buffer_provider,
+    ReadScopedBlockBufferProvider::Lease* lease);
+
+// Resolves the read-scoped provider configured for this read. Returns
+// std::nullopt when no provider is configured, or when mmap reads are enabled
+// because mmap file reads can ignore RocksDB-provided scratch buffers.
+ReadScopedBlockBufferProviderRef GetReadScopedBlockBufferProvider(
+    const ReadOptions& ro, bool allow_mmap_reads);
+
+// Centralizes the data-block cache decision for iterator and MultiScan paths.
+// A read-scoped provider skips data-block cache lookup and insertion because it
+// supplies alternate read-scope backing for supported data-block reads.
+bool ShouldUseDataBlockCacheForIterator(
+    const BlockBasedTableOptions& table_options, const ReadOptions& ro,
+    bool allow_mmap_reads);
+
+// Copies block bytes into RocksDB-owned heap storage and records that ownership
+// in BlockContents. `src` may include a block trailer while `data_size` is the
+// payload size.
+Status CopyBufferToHeapBlockContents(Slice src, size_t data_size,
+                                     MemoryAllocator* allocator,
+                                     BlockContents* out_contents);
+
+// Copies block bytes into read-scoped provider storage and records that
+// cleanup in BlockContents. `src` may include a block trailer while
+// `data_size` is the payload size.
+Status CopyBufferToReadScopedBlockContents(
+    Slice src, size_t data_size,
+    ReadScopedBlockBufferProvider& block_buffer_provider,
+    BlockContents* out_contents);
+
 // Reader class for BlockBasedTable format.
 // For the format of BlockBasedTable refer to
 // https://github.com/facebook/rocksdb/wiki/Rocksdb-BlockBasedTable-Format.

table/block_based/block_based_table_reader_impl.h

@@ -90,10 +90,14 @@ TBlockIter* BlockBasedTable::NewDataBlockIterator(
                         lookup_context, for_compaction, /* use_cache */ true,
                         async_read, use_block_cache_for_lookup);
     } else {
-      s = RetrieveBlock(
-          prefetch_buffer, ro, handle, decomp, &block.As<IterBlocklike>(),
-          get_context, lookup_context, for_compaction,
-          /* use_cache */ true, async_read, use_block_cache_for_lookup);
+      const bool use_cache =
+          block_type != BlockType::kData ||
+          ShouldUseDataBlockCacheForIterator(rep_->table_options, ro,
+                                             rep_->ioptions.allow_mmap_reads);
+      s = RetrieveBlock(prefetch_buffer, ro, handle, decomp,
+                        &block.As<IterBlocklike>(), get_context, lookup_context,
+                        for_compaction, use_cache, async_read,
+                        use_block_cache_for_lookup && use_cache);
     }
   }
 

table/block_based/block_based_table_reader_sync_and_async.h

@@ -4,6 +4,7 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
 
+#include "util/aligned_buffer.h"
 #include "util/async_file_reader.h"
 #include "util/coro_utils.h"
 
@@ -134,7 +135,8 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::RetrieveMultipleBlocks)
     read_reqs.emplace_back(std::move(req));
   }
 
-  AlignedBuf direct_io_buf;
+  AlignedBuffer direct_io_buffer;
+  AlignedBufferAllocationContext direct_io_context{&direct_io_buffer};
   {
     IOOptions opts;
     IODebugContext dbg;
@@ -144,11 +146,11 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::RetrieveMultipleBlocks)
       if (file->use_direct_io()) {
 #endif  // WITH_COROUTINES
         s = file->MultiRead(opts, &read_reqs[0], read_reqs.size(),
-                            &direct_io_buf, &dbg);
+                            &direct_io_context, &dbg);
 #if defined(WITH_COROUTINES)
       } else {
         co_await batch->context()->reader().MultiReadAsync(
-            file, opts, &read_reqs[0], read_reqs.size(), &direct_io_buf, &dbg);
+            file, opts, &read_reqs[0], read_reqs.size(), &dbg);
       }
 #endif  // WITH_COROUTINES
     }

table/block_based/block_based_table_reader_test.cc

@@ -1083,6 +1083,71 @@ class BlockBasedTableReaderMultiScanTest : public BlockBasedTableReaderTest {
   }
 };
 
+class BlockBasedTableReaderMultiScanFirstInternalKeyTest
+    : public BlockBasedTableReaderMultiScanTest {
+ public:
+  void SetUp() override {
+    BlockBasedTableReaderMultiScanTest::SetUp();
+    options_.persist_user_defined_timestamps = persist_udt_;
+  }
+
+ protected:
+  Slice ScanBoundary(const std::string& internal_key) const {
+    const size_t ts_sz = options_.comparator->timestamp_size();
+    if (ts_sz == 0) {
+      return ExtractUserKey(internal_key);
+    }
+    return ExtractUserKeyAndStripTimestamp(internal_key, ts_sz);
+  }
+
+  bool UsesFirstInternalKey() const {
+    return GetParam().index_type ==
+           BlockBasedTableOptions::IndexType::kBinarySearchWithFirstKey;
+  }
+
+  void SetMaxReadTimestamp(ReadOptions* read_opts, std::string* read_ts_storage,
+                           Slice* read_ts) const {
+    if (options_.comparator->timestamp_size() > 0) {
+      read_ts_storage->assign(options_.comparator->timestamp_size(), '\xff');
+      *read_ts = Slice(*read_ts_storage);
+      read_opts->timestamp = read_ts;
+    }
+  }
+
+  std::unique_ptr<InternalIterator> NewTableIterator(
+      BlockBasedTable* table, const ReadOptions& read_opts) const {
+    return std::unique_ptr<InternalIterator>(table->NewIterator(
+        read_opts, options_.prefix_extractor.get(), /*arena=*/nullptr,
+        /*skip_filters=*/false, TableReaderCaller::kUncategorized));
+  }
+
+  bool UserKeyLess(const Slice& lhs, const Slice& rhs) const {
+    return options_.comparator->CompareWithoutTimestamp(
+               lhs, /*a_has_ts=*/false, rhs, /*b_has_ts=*/false) < 0;
+  }
+
+  void CreateAndOpenTable(
+      const std::string& table_name,
+      const std::vector<std::pair<std::string, std::string>>& kv,
+      std::unique_ptr<BlockBasedTable>* table) {
+    ioptions_ = std::make_unique<ImmutableOptions>(options_);
+    CreateTable(table_name, *ioptions_, compression_type_, kv,
+                compression_parallel_threads_, compression_dict_bytes_);
+
+    FileOptions foptions;
+    foptions.use_direct_reads = use_direct_reads_;
+    internal_comparator_ =
+        std::make_unique<InternalKeyComparator>(options_.comparator);
+    NewBlockBasedTableReader(foptions, *ioptions_, *internal_comparator_,
+                             table_name, table,
+                             true /* prefetch_index_and_filter_in_cache */,
+                             nullptr /* status */, persist_udt_);
+  }
+
+  std::unique_ptr<ImmutableOptions> ioptions_;
+  std::unique_ptr<InternalKeyComparator> internal_comparator_;
+};
+
 class BlockBasedTableReaderMultiScanAsyncIOTest
     : public BlockBasedTableReaderMultiScanTest {};
 
@@ -1623,6 +1688,262 @@ TEST_P(BlockBasedTableReaderMultiScanTest, MultiScanUnpinPreviousBlocks) {
   }
 }
 
+TEST_P(BlockBasedTableReaderMultiScanFirstInternalKeyTest,
+       MultiScanLimitAtBlockBoundarySkipsNextBlockWithFirstKey) {
+  std::vector<std::pair<std::string, std::string>> kv =
+      BlockBasedTableReaderBaseTest::GenerateKVMap(
+          4 /* num_block */, true /* mixed_with_human_readable_string_value */,
+          comparator_->timestamp_size(), same_key_diff_ts_, comparator_);
+  std::string table_name =
+      "BlockBasedTableReaderTest_FirstInternalKeyBoundary" +
+      CompressionTypeToString(compression_type_) + "_" +
+      std::to_string(static_cast<int>(GetParam().index_type));
+
+  std::unique_ptr<BlockBasedTable> table;
+  CreateAndOpenTable(table_name, kv, &table);
+
+  ReadOptions read_opts;
+  std::string read_ts_storage;
+  Slice read_ts;
+  SetMaxReadTimestamp(&read_opts, &read_ts_storage, &read_ts);
+
+  std::unique_ptr<InternalIterator> iter =
+      NewTableIterator(table.get(), read_opts);
+
+  MultiScanArgs scan_options(options_.comparator);
+  scan_options.insert(ScanBoundary(kv[0].first),
+                      ScanBoundary(kv[kEntriesPerBlock].first));
+  iter->Prepare(&scan_options);
+  ASSERT_OK(iter->status());
+
+  auto* bbiter = dynamic_cast<BlockBasedTableIterator*>(iter.get());
+  ASSERT_TRUE(bbiter);
+  ASSERT_TRUE(bbiter->TEST_IsBlockPinnedByMultiScan(0));
+  if (UsesFirstInternalKey()) {
+    ASSERT_FALSE(bbiter->TEST_IsBlockPinnedByMultiScan(1));
+  } else {
+    ASSERT_TRUE(bbiter->TEST_IsBlockPinnedByMultiScan(1));
+  }
+}
+
+TEST_P(BlockBasedTableReaderMultiScanFirstInternalKeyTest,
+       MultiScanLimitInsideBlockKeepsContainingBlock) {
+  std::vector<std::pair<std::string, std::string>> kv =
+      BlockBasedTableReaderBaseTest::GenerateKVMap(
+          4 /* num_block */, true /* mixed_with_human_readable_string_value */,
+          comparator_->timestamp_size(), same_key_diff_ts_, comparator_);
+  std::string table_name =
+      "BlockBasedTableReaderTest_FirstInternalKeyInsideBlock" +
+      CompressionTypeToString(compression_type_) + "_" +
+      std::to_string(static_cast<int>(GetParam().index_type));
+
+  std::unique_ptr<BlockBasedTable> table;
+  CreateAndOpenTable(table_name, kv, &table);
+
+  ReadOptions read_opts;
+  std::string read_ts_storage;
+  Slice read_ts;
+  SetMaxReadTimestamp(&read_opts, &read_ts_storage, &read_ts);
+
+  std::unique_ptr<InternalIterator> iter =
+      NewTableIterator(table.get(), read_opts);
+
+  MultiScanArgs scan_options(options_.comparator);
+  scan_options.insert(ScanBoundary(kv[0].first),
+                      ScanBoundary(kv[kEntriesPerBlock + 1].first));
+  iter->Prepare(&scan_options);
+  ASSERT_OK(iter->status());
+
+  auto* bbiter = dynamic_cast<BlockBasedTableIterator*>(iter.get());
+  ASSERT_TRUE(bbiter);
+  ASSERT_TRUE(bbiter->TEST_IsBlockPinnedByMultiScan(0));
+  ASSERT_TRUE(bbiter->TEST_IsBlockPinnedByMultiScan(1));
+  ASSERT_FALSE(bbiter->TEST_IsBlockPinnedByMultiScan(2));
+}
+
+TEST_P(BlockBasedTableReaderMultiScanFirstInternalKeyTest,
+       MultiScanUnboundedRangeIncludesAllCandidateBlocks) {
+  std::vector<std::pair<std::string, std::string>> kv =
+      BlockBasedTableReaderBaseTest::GenerateKVMap(
+          4 /* num_block */, true /* mixed_with_human_readable_string_value */,
+          comparator_->timestamp_size(), same_key_diff_ts_, comparator_);
+  std::string table_name =
+      "BlockBasedTableReaderTest_FirstInternalKeyUnbounded" +
+      CompressionTypeToString(compression_type_) + "_" +
+      std::to_string(static_cast<int>(GetParam().index_type));
+
+  std::unique_ptr<BlockBasedTable> table;
+  CreateAndOpenTable(table_name, kv, &table);
+
+  ReadOptions read_opts;
+  std::string read_ts_storage;
+  Slice read_ts;
+  SetMaxReadTimestamp(&read_opts, &read_ts_storage, &read_ts);
+
+  std::unique_ptr<InternalIterator> iter =
+      NewTableIterator(table.get(), read_opts);
+
+  MultiScanArgs scan_options(options_.comparator);
+  scan_options.insert(ScanBoundary(kv[0].first));
+  iter->Prepare(&scan_options);
+  ASSERT_OK(iter->status());
+
+  auto* bbiter = dynamic_cast<BlockBasedTableIterator*>(iter.get());
+  ASSERT_TRUE(bbiter);
+  for (int block = 0; block < 4; ++block) {
+    ASSERT_TRUE(bbiter->TEST_IsBlockPinnedByMultiScan(block)) << block;
+  }
+  ASSERT_FALSE(bbiter->TEST_IsBlockPinnedByMultiScan(4));
+}
+
+TEST_P(BlockBasedTableReaderMultiScanFirstInternalKeyTest,
+       MultiScanLimitBeyondLastBlockIncludesAllCandidateBlocks) {
+  std::vector<std::pair<std::string, std::string>> kv =
+      BlockBasedTableReaderBaseTest::GenerateKVMap(
+          4 /* num_block */, true /* mixed_with_human_readable_string_value */,
+          comparator_->timestamp_size(), same_key_diff_ts_, comparator_);
+  std::string table_name =
+      "BlockBasedTableReaderTest_FirstInternalKeyBeyondLast" +
+      CompressionTypeToString(compression_type_) + "_" +
+      std::to_string(static_cast<int>(GetParam().index_type));
+
+  std::unique_ptr<BlockBasedTable> table;
+  CreateAndOpenTable(table_name, kv, &table);
+
+  ReadOptions read_opts;
+  std::string read_ts_storage;
+  Slice read_ts;
+  SetMaxReadTimestamp(&read_opts, &read_ts_storage, &read_ts);
+
+  std::unique_ptr<InternalIterator> iter =
+      NewTableIterator(table.get(), read_opts);
+
+  Slice last_key = ScanBoundary(kv.back().first);
+  std::string limit_after_last = last_key.ToString();
+  limit_after_last.push_back('\xff');
+  if (!UserKeyLess(last_key, limit_after_last)) {
+    limit_after_last.clear();
+  }
+  ASSERT_TRUE(UserKeyLess(last_key, limit_after_last));
+
+  MultiScanArgs scan_options(options_.comparator);
+  scan_options.insert(ScanBoundary(kv[0].first), limit_after_last);
+  iter->Prepare(&scan_options);
+  ASSERT_OK(iter->status());
+
+  auto* bbiter = dynamic_cast<BlockBasedTableIterator*>(iter.get());
+  ASSERT_TRUE(bbiter);
+  for (int block = 0; block < 4; ++block) {
+    ASSERT_TRUE(bbiter->TEST_IsBlockPinnedByMultiScan(block)) << block;
+  }
+  ASSERT_FALSE(bbiter->TEST_IsBlockPinnedByMultiScan(4));
+}
+
+TEST_P(BlockBasedTableReaderMultiScanFirstInternalKeyTest,
+       MultiScanRangeBeforeFirstBlockPrunesAllBlocksWithFirstKey) {
+  std::vector<std::pair<std::string, std::string>> kv =
+      BlockBasedTableReaderBaseTest::GenerateKVMap(
+          4 /* num_block */, true /* mixed_with_human_readable_string_value */,
+          comparator_->timestamp_size(), same_key_diff_ts_, comparator_);
+  std::string table_name =
+      "BlockBasedTableReaderTest_FirstInternalKeyBeforeFirst" +
+      CompressionTypeToString(compression_type_) + "_" +
+      std::to_string(static_cast<int>(GetParam().index_type));
+
+  std::unique_ptr<BlockBasedTable> table;
+  CreateAndOpenTable(table_name, kv, &table);
+
+  ReadOptions read_opts;
+  std::string read_ts_storage;
+  Slice read_ts;
+  SetMaxReadTimestamp(&read_opts, &read_ts_storage, &read_ts);
+
+  std::unique_ptr<InternalIterator> iter =
+      NewTableIterator(table.get(), read_opts);
+
+  Slice first_key = ScanBoundary(kv[0].first);
+  std::string start_before_first;
+  std::string limit_before_first(1, '\0');
+  if (!UserKeyLess(start_before_first, limit_before_first) ||
+      !UserKeyLess(limit_before_first, first_key)) {
+    start_before_first.assign(1, '\xff');
+    limit_before_first.assign(1, '\xfe');
+  }
+  ASSERT_TRUE(UserKeyLess(start_before_first, limit_before_first));
+  ASSERT_TRUE(UserKeyLess(limit_before_first, first_key));
+
+  MultiScanArgs scan_options(options_.comparator);
+  scan_options.insert(start_before_first, limit_before_first);
+  iter->Prepare(&scan_options);
+  ASSERT_OK(iter->status());
+
+  auto* bbiter = dynamic_cast<BlockBasedTableIterator*>(iter.get());
+  ASSERT_TRUE(bbiter);
+  if (UsesFirstInternalKey()) {
+    for (int block = 0; block < 4; ++block) {
+      ASSERT_FALSE(bbiter->TEST_IsBlockPinnedByMultiScan(block)) << block;
+    }
+  } else {
+    ASSERT_TRUE(bbiter->TEST_IsBlockPinnedByMultiScan(0));
+    for (int block = 1; block < 4; ++block) {
+      ASSERT_FALSE(bbiter->TEST_IsBlockPinnedByMultiScan(block)) << block;
+    }
+  }
+}
+
+TEST_P(BlockBasedTableReaderMultiScanFirstInternalKeyTest,
+       MultiScanAdjacentRangeKeepsBlockNeededBySecondRange) {
+  std::vector<std::pair<std::string, std::string>> kv =
+      BlockBasedTableReaderBaseTest::GenerateKVMap(
+          4 /* num_block */, true /* mixed_with_human_readable_string_value */,
+          comparator_->timestamp_size(), same_key_diff_ts_, comparator_);
+  std::string table_name =
+      "BlockBasedTableReaderTest_FirstInternalKeyAdjacent" +
+      CompressionTypeToString(compression_type_) + "_" +
+      std::to_string(static_cast<int>(GetParam().index_type));
+
+  std::unique_ptr<BlockBasedTable> table;
+  CreateAndOpenTable(table_name, kv, &table);
+
+  ReadOptions read_opts;
+  std::string read_ts_storage;
+  Slice read_ts;
+  SetMaxReadTimestamp(&read_opts, &read_ts_storage, &read_ts);
+
+  std::unique_ptr<InternalIterator> iter =
+      NewTableIterator(table.get(), read_opts);
+
+  MultiScanArgs scan_options(options_.comparator);
+  scan_options.insert(ScanBoundary(kv[0].first),
+                      ScanBoundary(kv[kEntriesPerBlock].first));
+  scan_options.insert(ScanBoundary(kv[kEntriesPerBlock].first),
+                      ScanBoundary(kv[2 * kEntriesPerBlock].first));
+  iter->Prepare(&scan_options);
+  ASSERT_OK(iter->status());
+
+  auto* bbiter = dynamic_cast<BlockBasedTableIterator*>(iter.get());
+  ASSERT_TRUE(bbiter);
+  ASSERT_TRUE(bbiter->TEST_IsBlockPinnedByMultiScan(0));
+  ASSERT_TRUE(bbiter->TEST_IsBlockPinnedByMultiScan(1));
+  if (UsesFirstInternalKey()) {
+    ASSERT_FALSE(bbiter->TEST_IsBlockPinnedByMultiScan(2));
+  } else {
+    ASSERT_TRUE(bbiter->TEST_IsBlockPinnedByMultiScan(2));
+  }
+
+  iter->Seek(kv[0].first);
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_OK(iter->status());
+  ASSERT_EQ(iter->key(), kv[0].first);
+  ASSERT_EQ(iter->value(), kv[0].second);
+
+  iter->Seek(kv[kEntriesPerBlock].first);
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_OK(iter->status());
+  ASSERT_EQ(iter->key(), kv[kEntriesPerBlock].first);
+  ASSERT_EQ(iter->value(), kv[kEntriesPerBlock].second);
+}
+
 // Regression test for assertion failure when re-seeking to an already-exhausted
 // scan range. This can happen when MergingIterator re-seeks a child iterator
 // after all scan ranges have been consumed (e.g., due to range tombstone
@@ -1984,6 +2305,42 @@ struct BlockBasedTableReaderTestParamBuilder {
   std::vector<size_t> super_block_alignment_space_overhead_ratios;
 };
 
+std::vector<BlockBasedTableReaderTestParam>
+GenerateFirstInternalKeyMultiScanParameters() {
+  std::vector<BlockBasedTableReaderTestParam> params;
+  auto add_param = [&params](BlockBasedTableOptions::IndexType index_type,
+                             test::UserDefinedTimestampTestMode udt_test_mode,
+                             const Comparator* comparator) {
+    params.emplace_back(
+        kNoCompression, false /* use_direct_reads */, index_type,
+        false /* no_block_cache */, udt_test_mode,
+        1 /* compression_parallel_threads */, 0 /* compression_dict_bytes */,
+        false /* same_key_diff_ts */, comparator, true /* fill_cache */,
+        false /* use_async_io */, false /* block_align */,
+        0 /* super_block_alignment_size */,
+        128 /* super_block_alignment_space_overhead_ratio */);
+  };
+  for (const auto index_type :
+       {BlockBasedTableOptions::IndexType::kBinarySearch,
+        BlockBasedTableOptions::IndexType::kBinarySearchWithFirstKey}) {
+    add_param(index_type, test::UserDefinedTimestampTestMode::kNone,
+              BytewiseComparator());
+    add_param(index_type, test::UserDefinedTimestampTestMode::kNone,
+              ReverseBytewiseComparator());
+    add_param(index_type,
+              test::UserDefinedTimestampTestMode::kStripUserDefinedTimestamp,
+              test::BytewiseComparatorWithU64TsWrapper());
+    add_param(index_type,
+              test::UserDefinedTimestampTestMode::kStripUserDefinedTimestamp,
+              test::ReverseBytewiseComparatorWithU64TsWrapper());
+    add_param(index_type, test::UserDefinedTimestampTestMode::kNormal,
+              test::BytewiseComparatorWithU64TsWrapper());
+    add_param(index_type, test::UserDefinedTimestampTestMode::kNormal,
+              test::ReverseBytewiseComparatorWithU64TsWrapper());
+  }
+  return params;
+}
+
 std::vector<bool> IOUringFlags() {
 #ifdef ROCKSDB_IOURING_PRESENT
   return {false, true};
@@ -2015,6 +2372,11 @@ INSTANTIATE_TEST_CASE_P(
                                               ReverseBytewiseComparator()})
                             .build()));
 
+INSTANTIATE_TEST_CASE_P(
+    BlockBasedTableReaderMultiScanFirstInternalKeyTest,
+    BlockBasedTableReaderMultiScanFirstInternalKeyTest,
+    ::testing::ValuesIn(GenerateFirstInternalKeyMultiScanParameters()));
+
 INSTANTIATE_TEST_CASE_P(
     BlockBasedTableReaderGetTest, BlockBasedTableReaderGetTest,
     ::testing::ValuesIn(BlockBasedTableReaderTestParamBuilder()

table/block_based/block_util.h

@@ -3,11 +3,6 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
 
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
 #pragma once
 
 #include <algorithm>

table/block_fetcher.cc

@@ -24,6 +24,7 @@
 #include "table/block_based/reader_common.h"
 #include "table/format.h"
 #include "table/persistent_cache_helper.h"
+#include "util/aligned_buffer.h"
 #include "util/compression.h"
 #include "util/stop_watch.h"
 
@@ -151,8 +152,17 @@ inline bool BlockFetcher::TryGetSerializedBlockFromPersistentCache() {
 
 inline void BlockFetcher::PrepareBufferForBlockFromFile() {
   // cache miss read from device
-  if ((do_uncompress_ || ioptions_.allow_mmap_reads) &&
-      block_size_with_trailer_ < kDefaultStackBufferSize) {
+  if (block_buffer_provider_.has_value() && !maybe_compressed_) {
+    Status s = AllocateReadScopedBlockBuffer(block_buffer_provider_->get(),
+                                             block_size_with_trailer_, 1,
+                                             &read_scoped_buf_lease_);
+    if (!s.ok()) {
+      io_status_ = status_to_io_status(std::move(s));
+      return;
+    }
+    used_buf_ = read_scoped_buf_lease_.data;
+  } else if ((do_uncompress_ || ioptions_.allow_mmap_reads) &&
+             block_size_with_trailer_ < kDefaultStackBufferSize) {
     // If we've got a small enough chunk of data, read it in to the
     // trivially allocated stack buffer instead of needing a full malloc()
     //
@@ -165,9 +175,11 @@ inline void BlockFetcher::PrepareBufferForBlockFromFile() {
     // stack buffer, the cost of guessing incorrectly here is one extra memcpy.
     //
     // When `do_uncompress_` is true, we expect the uncompression step will
-    // allocate heap memory for the final result. However this expectation will
-    // be wrong if the block turns out to already be uncompressed, which we
-    // won't know for sure until after reading it.
+    // allocate memory for the final result, using the read-scoped provider if
+    // one is configured. However this expectation will be wrong if the block
+    // turns out to already be uncompressed, which we won't know for sure until
+    // after reading it. In that case provider-backed reads copy the block into
+    // provider storage in `GetBlockContents()`.
     //
     // When `ioptions_.allow_mmap_reads` is true, we do not expect the file
     // reader to use the scratch buffer at all, but instead return a pointer
@@ -231,14 +243,29 @@ inline void BlockFetcher::CopyBufferToCompressedBuf() {
 //    is not compressed
 // 3. heap_buf_ if the block is not compressed
 // 4. compressed_buf_ if the block is compressed
-// 5. direct_io_buf_ if direct IO is enabled or
+// 5. direct_io_buffer_ if direct IO is enabled or
 // 6. underlying file_system scratch is used (FSReadRequest.fs_scratch).
 //
-// After - After this method, if the block is compressed, it should be in
-// compressed_buf_ and heap_buf_ points to compressed_buf_, otherwise should be
-// in heap_buf_.
+// After - After this method, compressed blocks should be in compressed_buf_ and
+// heap_buf_ points to compressed_buf_. Uncompressed blocks should be recorded
+// in *contents_ with either heap ownership or read-scoped cleanup ownership.
 inline void BlockFetcher::GetBlockContents() {
-  if (slice_.data() != used_buf_) {
+  if (read_scoped_buf_lease_.cleanup.get() != nullptr &&
+      compression_type() == kNoCompression) {
+    contents_->data = Slice(slice_.data(), block_size_);
+    contents_->cleanup = std::move(read_scoped_buf_lease_.cleanup);
+    contents_->backing_size = read_scoped_buf_lease_.size;
+    contents_->AssertSingleOwner();
+  } else if (block_buffer_provider_.has_value() &&
+             compression_type() == kNoCompression) {
+    Status s = CopyBufferToReadScopedBlockContents(
+        Slice(slice_.data(), block_size_with_trailer_), block_size_,
+        block_buffer_provider_->get(), contents_);
+    if (!s.ok()) {
+      io_status_ = status_to_io_status(std::move(s));
+      return;
+    }
+  } else if (slice_.data() != used_buf_) {
     // the slice content is not the buffer provided
     *contents_ = BlockContents(Slice(slice_.data(), block_size_));
   } else {
@@ -253,7 +280,7 @@ inline void BlockFetcher::GetBlockContents() {
       } else {
         heap_buf_ = std::move(compressed_buf_);
       }
-    } else if (direct_io_buf_.get() != nullptr || use_fs_scratch_) {
+    } else if (direct_io_buffer_.BufferStart() != nullptr || use_fs_scratch_) {
       if (compression_type() == kNoCompression) {
         CopyBufferToHeapBuf();
       } else {
@@ -281,13 +308,21 @@ void BlockFetcher::ReadBlock(bool retry) {
   // Actual file read
   if (io_status_.ok()) {
     if (file_->use_direct_io()) {
+      AlignedBuffer::Allocator direct_io_allocator;
+      if (block_buffer_provider_.has_value() && !maybe_compressed_) {
+        direct_io_allocator = MakeReadScopedAlignedBufferAllocator(
+            block_buffer_provider_, &read_scoped_buf_lease_);
+      }
+      AlignedBufferAllocationContext direct_io_context{
+          &direct_io_buffer_,
+          direct_io_allocator ? &direct_io_allocator : nullptr};
       PERF_TIMER_GUARD(block_read_time);
       PERF_CPU_TIMER_GUARD(
           block_read_cpu_time,
           ioptions_.env ? ioptions_.env->GetSystemClock().get() : nullptr);
       io_status_ =
           file_->Read(opts, handle_.offset(), block_size_with_trailer_, &slice_,
-                      /*scratch=*/nullptr, &direct_io_buf_, &dbg);
+                      /*scratch=*/nullptr, &direct_io_context, &dbg);
       PERF_COUNTER_ADD(block_read_count, 1);
       used_buf_ = const_cast<char*>(slice_.data());
     } else if (use_fs_scratch_) {
@@ -298,8 +333,10 @@ void BlockFetcher::ReadBlock(bool retry) {
       read_req.offset = handle_.offset();
       read_req.len = block_size_with_trailer_;
       read_req.scratch = nullptr;
+      AlignedBuffer direct_io_buffer;
+      AlignedBufferAllocationContext direct_io_context{&direct_io_buffer};
       io_status_ = file_->MultiRead(opts, &read_req, /*num_reqs=*/1,
-                                    /*AlignedBuf* =*/nullptr, &dbg);
+                                    &direct_io_context, &dbg);
       PERF_COUNTER_ADD(block_read_count, 1);
 
       slice_ = Slice(read_req.result.data(), read_req.result.size());
@@ -307,6 +344,9 @@ void BlockFetcher::ReadBlock(bool retry) {
     } else {
       // It allocates/assign used_buf_
       PrepareBufferForBlockFromFile();
+      if (!io_status_.ok()) {
+        return;
+      }
 
       PERF_TIMER_GUARD(block_read_time);
       PERF_CPU_TIMER_GUARD(
@@ -316,7 +356,7 @@ void BlockFetcher::ReadBlock(bool retry) {
       io_status_ =
           file_->Read(opts, handle_.offset(), /*size*/ block_size_with_trailer_,
                       /*result*/ &slice_, /*scratch*/ used_buf_,
-                      /*aligned_buf=*/nullptr, &dbg);
+                      /*direct_io_buffer=*/nullptr, &dbg);
       PERF_COUNTER_ADD(block_read_count, 1);
 #ifndef NDEBUG
       if (slice_.data() == &stack_buf_[0]) {
@@ -380,7 +420,7 @@ void BlockFetcher::ReadBlock(bool retry) {
     }
   } else {
     ReleaseFileSystemProvidedBuffer(&read_req);
-    direct_io_buf_.reset();
+    direct_io_buffer_.Release();
     compressed_buf_.reset();
     heap_buf_.reset();
     used_buf_ = nullptr;
@@ -423,7 +463,8 @@ IOStatus BlockFetcher::ReadBlockContents() {
     slice_.size_ = block_size_;
     decomp_args_.compressed_data = slice_;
     io_status_ = status_to_io_status(DecompressSerializedBlock(
-        decomp_args_, *decompressor_, contents_, ioptions_, memory_allocator_));
+        decomp_args_, *decompressor_, contents_, ioptions_, memory_allocator_,
+        block_buffer_provider_));
 #ifndef NDEBUG
     num_heap_buf_memcpy_++;
 #endif
@@ -477,9 +518,9 @@ IOStatus BlockFetcher::ReadAsyncBlockContents() {
           // Process the compressed block without trailer
           slice_.size_ = block_size_;
           decomp_args_.compressed_data = slice_;
-          io_status_ = status_to_io_status(
-              DecompressSerializedBlock(decomp_args_, *decompressor_, contents_,
-                                        ioptions_, memory_allocator_));
+          io_status_ = status_to_io_status(DecompressSerializedBlock(
+              decomp_args_, *decompressor_, contents_, ioptions_,
+              memory_allocator_, block_buffer_provider_));
 #ifndef NDEBUG
           num_heap_buf_memcpy_++;
 #endif

table/block_fetcher.h

@@ -39,19 +39,18 @@ namespace ROCKSDB_NAMESPACE {
 
 class BlockFetcher {
  public:
-  BlockFetcher(RandomAccessFileReader* file,
-               FilePrefetchBuffer* prefetch_buffer,
-               const Footer& footer /* ref retained */,
-               const ReadOptions& read_options,
-               const BlockHandle& handle /* ref retained */,
-               BlockContents* contents,
-               const ImmutableOptions& ioptions /* ref retained */,
-               bool do_uncompress, bool maybe_compressed, BlockType block_type,
-               UnownedPtr<Decompressor> decompressor,
-               const PersistentCacheOptions& cache_options /* ref retained */,
-               MemoryAllocator* memory_allocator = nullptr,
-               MemoryAllocator* memory_allocator_compressed = nullptr,
-               bool for_compaction = false)
+  BlockFetcher(
+      RandomAccessFileReader* file, FilePrefetchBuffer* prefetch_buffer,
+      const Footer& footer /* ref retained */, const ReadOptions& read_options,
+      const BlockHandle& handle /* ref retained */, BlockContents* contents,
+      const ImmutableOptions& ioptions /* ref retained */, bool do_uncompress,
+      bool maybe_compressed, BlockType block_type,
+      UnownedPtr<Decompressor> decompressor,
+      const PersistentCacheOptions& cache_options /* ref retained */,
+      MemoryAllocator* memory_allocator = nullptr,
+      MemoryAllocator* memory_allocator_compressed = nullptr,
+      bool for_compaction = false,
+      ReadScopedBlockBufferProviderRef block_buffer_provider = std::nullopt)
       : file_(file),
         prefetch_buffer_(prefetch_buffer),
         footer_(footer),
@@ -68,6 +67,8 @@ class BlockFetcher {
         cache_options_(cache_options),
         memory_allocator_(memory_allocator),
         memory_allocator_compressed_(memory_allocator_compressed),
+        block_buffer_provider_(
+            ioptions.allow_mmap_reads ? std::nullopt : block_buffer_provider),
         for_compaction_(for_compaction) {
     io_status_.PermitUncheckedError();  // TODO(AR) can we improve on this?
     if (CheckFSFeatureSupport(ioptions_.fs.get(), FSSupportedOps::kFSBuffer)) {
@@ -129,12 +130,18 @@ class BlockFetcher {
   const PersistentCacheOptions& cache_options_;
   MemoryAllocator* memory_allocator_;
   MemoryAllocator* memory_allocator_compressed_;
+  // Optional provider used when the fetched block should be backed by
+  // read-scoped storage instead of RocksDB heap memory.
+  ReadScopedBlockBufferProviderRef block_buffer_provider_;
   IOStatus io_status_;
   Slice slice_;
   char* used_buf_ = nullptr;
-  AlignedBuf direct_io_buf_;
+  AlignedBuffer direct_io_buffer_;
   CacheAllocationPtr heap_buf_;
   CacheAllocationPtr compressed_buf_;
+  // Provider lease backing `used_buf_`; moved into BlockContents once the block
+  // is parsed so the provider allocation lives as long as the block does.
+  ReadScopedBlockBufferProvider::Lease read_scoped_buf_lease_;
   char stack_buf_[kDefaultStackBufferSize];
   bool got_from_prefetch_buffer_ = false;
   bool for_compaction_ = false;

table/format.cc

@@ -529,7 +529,7 @@ static Status ReadFooterFromFileInternal(
   }
 
   std::array<char, Footer::kMaxEncodedLength + 1> footer_buf;
-  AlignedBuf internal_buf;
+  AlignedBuffer direct_io_buffer;
   Slice footer_input;
   uint64_t read_offset = (expected_file_size > Footer::kMaxEncodedLength)
                              ? expected_file_size - Footer::kMaxEncodedLength
@@ -545,11 +545,12 @@ static Status ReadFooterFromFileInternal(
                                          Footer::kMaxEncodedLength,
                                          &footer_input, nullptr)) {
     if (file->use_direct_io()) {
+      AlignedBufferAllocationContext direct_io_context{&direct_io_buffer};
       s = file->Read(opts, read_offset, Footer::kMaxEncodedLength,
-                     &footer_input, nullptr, &internal_buf);
+                     &footer_input, nullptr, &direct_io_context);
     } else {
       s = file->Read(opts, read_offset, Footer::kMaxEncodedLength,
-                     &footer_input, footer_buf.data(), nullptr);
+                     &footer_input, footer_buf.data());
     }
     if (!s.ok()) {
       return s;
@@ -683,10 +684,11 @@ uint32_t ComputeBuiltinChecksumWithLastByte(ChecksumType type, const char* data,
   }
 }
 
-Status DecompressBlockData(Decompressor::Args& args, Decompressor& decompressor,
-                           BlockContents* out_contents,
-                           const ImmutableOptions& ioptions,
-                           MemoryAllocator* allocator) {
+Status DecompressBlockData(
+    Decompressor::Args& args, Decompressor& decompressor,
+    BlockContents* out_contents, const ImmutableOptions& ioptions,
+    MemoryAllocator* allocator,
+    ReadScopedBlockBufferProviderRef block_buffer_provider) {
   assert(args.compression_type != kNoCompression && "Invalid compression type");
 
   StopWatchNano timer(ioptions.clock,
@@ -696,13 +698,35 @@ Status DecompressBlockData(Decompressor::Args& args, Decompressor& decompressor,
   if (UNLIKELY(!s.ok())) {
     return s;
   }
-  CacheAllocationPtr ubuf = AllocateBlock(args.uncompressed_size, allocator);
-  s = decompressor.DecompressBlock(args, ubuf.get());
-  if (UNLIKELY(!s.ok())) {
-    return s;
-  }
+  if (block_buffer_provider.has_value()) {
+    // Compressed provider path: allocate the final uncompressed block backing
+    // from the provider, then decompress directly into it.
+    ReadScopedBlockBufferProvider::Lease read_scoped_lease;
+    s = AllocateReadScopedBlockBuffer(block_buffer_provider->get(),
+                                      args.uncompressed_size, 1,
+                                      &read_scoped_lease);
+    if (UNLIKELY(!s.ok())) {
+      return s;
+    }
 
-  *out_contents = BlockContents(std::move(ubuf), args.uncompressed_size);
+    s = decompressor.DecompressBlock(args, read_scoped_lease.data);
+    if (UNLIKELY(!s.ok())) {
+      return s;
+    }
+
+    out_contents->data = Slice(read_scoped_lease.data, args.uncompressed_size);
+    out_contents->cleanup = std::move(read_scoped_lease.cleanup);
+    out_contents->backing_size = read_scoped_lease.size;
+  } else {
+    CacheAllocationPtr ubuf = AllocateBlock(args.uncompressed_size, allocator);
+
+    s = decompressor.DecompressBlock(args, ubuf.get());
+    if (UNLIKELY(!s.ok())) {
+      return s;
+    }
+
+    *out_contents = BlockContents(std::move(ubuf), args.uncompressed_size);
+  }
 
   if (ShouldReportDetailedTime(ioptions.env, ioptions.stats)) {
     RecordTimeToHistogram(ioptions.stats, DECOMPRESSION_TIMES_NANOS,
@@ -733,32 +757,42 @@ Status DecompressBlockData(const char* data, size_t size, CompressionType type,
   args.compression_type = type;
   args.working_area = working_area;
   return DecompressBlockData(args, decompressor, out_contents, ioptions,
-                             allocator);
+                             allocator, std::nullopt);
 }
 
-Status DecompressSerializedBlock(const char* data, size_t size,
-                                 CompressionType type,
-                                 Decompressor& decompressor,
-                                 BlockContents* out_contents,
-                                 const ImmutableOptions& ioptions,
-                                 MemoryAllocator* allocator) {
+Status DecompressSerializedBlock(
+    const char* data, size_t size, CompressionType type,
+    Decompressor& decompressor, BlockContents* out_contents,
+    const ImmutableOptions& ioptions, MemoryAllocator* allocator,
+    ReadScopedBlockBufferProviderRef block_buffer_provider) {
   assert(data[size] != kNoCompression);
   assert(data[size] == static_cast<char>(type));
-  return DecompressBlockData(data, size, type, decompressor, out_contents,
-                             ioptions, allocator);
+  Decompressor::Args args;
+  args.compressed_data = Slice(data, size);
+  args.compression_type = type;
+  return DecompressBlockData(args, decompressor, out_contents, ioptions,
+                             allocator, block_buffer_provider);
 }
 
-Status DecompressSerializedBlock(Decompressor::Args& args,
-                                 Decompressor& decompressor,
-                                 BlockContents* out_contents,
-                                 const ImmutableOptions& ioptions,
-                                 MemoryAllocator* allocator) {
+Status DecompressBlockData(Decompressor::Args& args, Decompressor& decompressor,
+                           BlockContents* out_contents,
+                           const ImmutableOptions& ioptions,
+                           MemoryAllocator* allocator) {
+  return DecompressBlockData(args, decompressor, out_contents, ioptions,
+                             allocator, std::nullopt);
+}
+
+Status DecompressSerializedBlock(
+    Decompressor::Args& args, Decompressor& decompressor,
+    BlockContents* out_contents, const ImmutableOptions& ioptions,
+    MemoryAllocator* allocator,
+    ReadScopedBlockBufferProviderRef block_buffer_provider) {
   assert(args.compressed_data.data()[args.compressed_data.size()] !=
          kNoCompression);
   assert(args.compressed_data.data()[args.compressed_data.size()] ==
          static_cast<char>(args.compression_type));
   return DecompressBlockData(args, decompressor, out_contents, ioptions,
-                             allocator);
+                             allocator, block_buffer_provider);
 }
 
 // Replace the contents of db_host_id with the actual hostname, if db_host_id

table/format.h

@@ -10,7 +10,10 @@
 #pragma once
 
 #include <array>
+#include <cassert>
 #include <cstdint>
+#include <functional>
+#include <optional>
 #include <string>
 
 #include "file/file_prefetch_buffer.h"
@@ -19,6 +22,7 @@
 #include "options/cf_options.h"
 #include "port/malloc.h"
 #include "port/port.h"  // noexcept
+#include "rocksdb/cleanable.h"
 #include "rocksdb/slice.h"
 #include "rocksdb/status.h"
 #include "rocksdb/table.h"
@@ -29,6 +33,9 @@ namespace ROCKSDB_NAMESPACE {
 class RandomAccessFile;
 struct ReadOptions;
 
+using ReadScopedBlockBufferProviderRef =
+    std::optional<std::reference_wrapper<ReadScopedBlockBufferProvider>>;
+
 bool ShouldReportDetailedTime(Env* env, Statistics* stats);
 
 // the length of the magic number in bytes.
@@ -404,6 +411,11 @@ struct BlockContents {
   // Points to block payload (without trailer)
   Slice data;
   CacheAllocationPtr allocation;
+  // Optional shared lifetime token for block bytes not owned through
+  // `allocation`, such as bytes backed by a read-scoped provider.
+  SharedCleanablePtr cleanup;
+  // Usable byte size of the provider/external backing allocation.
+  size_t backing_size = 0;
 
 #ifndef NDEBUG
   // Whether there is a known trailer after what is pointed to by `data`.
@@ -411,6 +423,10 @@ struct BlockContents {
   bool has_trailer = false;
 #endif  // NDEBUG
 
+  void AssertSingleOwner() const {
+    assert(allocation.get() == nullptr || cleanup.get() == nullptr);
+  }
+
   BlockContents() {}
 
   // Does not take ownership of the underlying data bytes.
@@ -427,10 +443,14 @@ struct BlockContents {
   }
 
   // Returns whether the object has ownership of the underlying data bytes.
-  bool own_bytes() const { return allocation.get() != nullptr; }
+  bool own_bytes() const {
+    AssertSingleOwner();
+    return allocation.get() != nullptr || cleanup.get() != nullptr;
+  }
 
   // The additional memory space taken by the block data.
   size_t usable_size() const {
+    AssertSingleOwner();
     // FIXME: doesn't account for possible block trailer
     if (allocation.get() != nullptr) {
       auto allocator = allocation.get_deleter().allocator;
@@ -442,12 +462,15 @@ struct BlockContents {
 #else
       return data.size();
 #endif  // ROCKSDB_MALLOC_USABLE_SIZE
+    } else if (cleanup.get() != nullptr) {
+      return backing_size;
     } else {
       return 0;  // no extra memory is occupied by the data
     }
   }
 
   size_t ApproximateMemoryUsage() const {
+    AssertSingleOwner();
     return usable_size() + sizeof(*this);
   }
 
@@ -456,9 +479,12 @@ struct BlockContents {
   BlockContents& operator=(BlockContents&& other) {
     data = std::move(other.data);
     allocation = std::move(other.allocation);
+    cleanup = std::move(other.cleanup);
+    backing_size = other.backing_size;
 #ifndef NDEBUG
     has_trailer = other.has_trailer;
 #endif  // NDEBUG
+    AssertSingleOwner();
     return *this;
   }
 };
@@ -467,18 +493,17 @@ struct BlockContents {
 // must be compressed and include a trailer beyond `size`. A new buffer is
 // allocated with the given allocator (or default) and the uncompressed
 // contents are returned in `out_contents`. Statistics updated.
-Status DecompressSerializedBlock(const char* data, size_t size,
-                                 CompressionType type,
-                                 Decompressor& decompressor,
-                                 BlockContents* out_contents,
-                                 const ImmutableOptions& ioptions,
-                                 MemoryAllocator* allocator = nullptr);
+Status DecompressSerializedBlock(
+    const char* data, size_t size, CompressionType type,
+    Decompressor& decompressor, BlockContents* out_contents,
+    const ImmutableOptions& ioptions, MemoryAllocator* allocator = nullptr,
+    ReadScopedBlockBufferProviderRef block_buffer_provider = std::nullopt);
 
-Status DecompressSerializedBlock(Decompressor::Args& args,
-                                 Decompressor& decompressor,
-                                 BlockContents* out_contents,
-                                 const ImmutableOptions& ioptions,
-                                 MemoryAllocator* allocator = nullptr);
+Status DecompressSerializedBlock(
+    Decompressor::Args& args, Decompressor& decompressor,
+    BlockContents* out_contents, const ImmutableOptions& ioptions,
+    MemoryAllocator* allocator = nullptr,
+    ReadScopedBlockBufferProviderRef block_buffer_provider = std::nullopt);
 
 // This is a variant of DecompressSerializedBlock that does not expect a
 // block trailer beyond `size`. (CompressionType is passed in.)

tools/check_format_compatible.sh

@@ -19,7 +19,7 @@
 #  LONG_TEST=1 - Test all branches known to build for this test, rather than
 #    the default of randomly sampling the branches that aren't the oldest in
 #    each set.
-#  USE_SSH=1 - Connect to GitHub with ssh instead of https 
+#  USE_SSH=1 - Connect to GitHub with ssh instead of https
 
 if ! git diff-index --quiet HEAD; then
   echo "You have uncommitted changes. Aborting."
@@ -148,7 +148,7 @@ EOF
 
 # To check for DB forward compatibility with loading options (old version
 # reading data from new), as well as backward compatibility
-declare -a db_forward_with_options_refs=("10.4.fb" "10.5.fb" "10.6.fb" "10.7.fb" "10.8.fb" "10.9.fb" "10.10.fb" "10.11.fb" "11.0.fb" "11.1.fb 11.2.fb")
+declare -a db_forward_with_options_refs=("10.4.fb" "10.5.fb" "10.6.fb" "10.7.fb" "10.8.fb" "10.9.fb" "10.10.fb" "10.11.fb" "11.0.fb" "11.1.fb" "11.2.fb" "11.3.fb")
 # To check for DB forward compatibility without loading options (in addition
 # to the "with loading options" set), as well as backward compatibility
 declare -a db_forward_no_options_refs=() # N/A at the moment
@@ -174,6 +174,28 @@ declare -a bak_forward_refs=("${db_forward_no_options_refs[@]}" "${db_forward_wi
 # (when format_version=2 became the default)
 declare -a db_backward_only_refs=("4.6.fb" "4.7.fb" "4.8.fb" "4.9.fb" "4.10.fb" "${bak_backward_only_refs[@]}")
 
+validate_release_refs()
+{
+  local list_name="$1"
+  local checkout_ref
+  shift
+  for checkout_ref in "$@"
+  do
+    if [[ ! "$checkout_ref" =~ ^[0-9]+\.[0-9]+\.fb$ ]]; then
+      echo "Invalid ref in ${list_name}: '${checkout_ref}'" >&2
+      exit 1
+    fi
+  done
+}
+
+validate_release_refs "db_backward_only_refs" "${db_backward_only_refs[@]}"
+validate_release_refs "db_forward_no_options_refs" "${db_forward_no_options_refs[@]}"
+validate_release_refs "db_forward_with_options_refs" "${db_forward_with_options_refs[@]}"
+validate_release_refs "ext_backward_only_refs" "${ext_backward_only_refs[@]}"
+validate_release_refs "ext_forward_refs" "${ext_forward_refs[@]}"
+validate_release_refs "bak_backward_only_refs" "${bak_backward_only_refs[@]}"
+validate_release_refs "bak_forward_refs" "${bak_forward_refs[@]}"
+
 if [ "$SHORT_TEST" ]; then
   # Use only the first (if exists) of each list
   db_backward_only_refs=(${db_backward_only_refs[0]})
@@ -190,20 +212,20 @@ declare -a checkout_refs=()
 # Always include first release on each list
 for checkout_ref in "${db_backward_only_refs[0]}" "${db_forward_no_options_refs[0]}" "${db_forward_with_options_refs[0]}" "${ext_backward_only_refs[0]}" "${ext_forward_refs[0]}" "${bak_backward_only_refs[0]}" "${bak_forward_refs[0]}"
 do
-  if [ ! -e $db_test_dir/$checkout_ref ]; then
-    mkdir -p $db_test_dir/$checkout_ref
-    checkout_refs+=($checkout_ref)
+  if [ ! -e "$db_test_dir/$checkout_ref" ]; then
+    mkdir -p "$db_test_dir/$checkout_ref"
+    checkout_refs+=("$checkout_ref")
   fi
 done
 # Maybe include rest if not short test
 if [ "$SHORT_TEST" == "" ]; then
   for checkout_ref in "${db_backward_only_refs[@]}" "${db_forward_no_options_refs[@]}" "${db_forward_with_options_refs[@]}" "${ext_backward_only_refs[@]}" "${ext_forward_refs[@]}" "${bak_backward_only_refs[@]}" "${bak_forward_refs[@]}"
   do
-    if [ ! -e $db_test_dir/$checkout_ref ]; then
-      mkdir -p $db_test_dir/$checkout_ref
+    if [ ! -e "$db_test_dir/$checkout_ref" ]; then
+      mkdir -p "$db_test_dir/$checkout_ref"
       # Randomly sample remaining releases, unless long test
       if [ "$LONG_TEST" ] || [ "$(($RANDOM % 3))" == "0" ]; then
-        checkout_refs+=($checkout_ref)
+        checkout_refs+=("$checkout_ref")
       fi
     fi
   done

tools/db_bench_tool.cc

@@ -2912,6 +2912,44 @@ class Benchmark {
   std::shared_ptr<const SliceTransform> prefix_extractor_;
   DBWithColumnFamilies db_;
   std::vector<DBWithColumnFamilies> multi_dbs_;
+  //
+  // === DB lifetime protocol ===
+  //
+  // Two RAII guards protect db_, multi_dbs_, and secondary_update_thread_:
+  //
+  //  - DbUseGuard:           held by any thread that reads or holds a raw
+  //                          DBWithColumnFamilies* derived from db_ /
+  //                          multi_dbs_, for the entire scope of use. Takes a
+  //                          read lock on db_lifecycle_rwlock_.
+  //                          STRICT enforcement at the SelectDBWithCfh entry
+  //                          point (assert on holds_db_use_guard_).
+  //                          Direct field accesses elsewhere (e.g.
+  //                          db_.db->NewIterator) are protected by the outer
+  //                          DbUseGuard that ThreadBody installs for workers,
+  //                          or by main-thread serialization -- but are not
+  //                          mechanically asserted per-site.
+  //
+  //  - DbStateMutationGuard: held by any thread that mutates db_, multi_dbs_,
+  //                          or secondary_update_thread_. Takes a write lock
+  //                          on db_lifecycle_rwlock_, then stops the secondary
+  //                          update thread before mutation may proceed.
+  //                          STRICT enforcement at DeleteDBs (assert on
+  //                          holds_db_state_mutation_guard_). Direct calls
+  //                          to DBWithColumnFamilies::DeleteDBs() and
+  //                          multi_dbs_.clear() in the fresh-DB reopen branch
+  //                          are protected by being inside the same guard
+  //                          scope, but not mechanically asserted.
+  //
+  // Three thread-locals, each with a single meaning:
+  //  - holds_db_use_guard_            : in DbUseGuard scope
+  //  - holds_db_state_mutation_guard_ : in DbStateMutationGuard scope
+  //  - is_secondary_update_thread_    : this thread is the secondary updater
+  //
+  // ErrorExit consults the first and third: a thread in *either* state
+  // cannot safely run the mutation cleanup path (self-wait or self-join
+  // respectively), so it takes std::_Exit(1) instead.
+  //
+  port::RWMutex db_lifecycle_rwlock_;
   int64_t num_;
   int key_size_;
   int user_timestamp_size_;
@@ -3502,6 +3540,11 @@ class Benchmark {
   }
 
   void DeleteDBs() {
+    // Caller MUST hold DbStateMutationGuard (proven by ownership flag).
+    // Note: direct calls to db_.DeleteDBs() / multi_dbs_[i].DeleteDBs() in
+    // the fresh-DB reopen branch bypass this wrapper; they're protected by
+    // being inside DbStateMutationGuard but are not mechanically asserted.
+    assert(holds_db_state_mutation_guard_);
     db_.DeleteDBs();
     for (auto& dbwcf : multi_dbs_) {
       dbwcf.DeleteDBs();
@@ -3509,7 +3552,10 @@ class Benchmark {
   }
 
   ~Benchmark() {
-    DeleteDBs();
+    {
+      DbStateMutationGuard mutation(this);
+      DeleteDBs();
+    }
     if (cache_.get() != nullptr) {
       // Clear cache reference first
       open_options_.write_buffer_manager.reset();
@@ -3643,7 +3689,19 @@ class Benchmark {
   }
 
   void ErrorExit() {
-    DeleteDBs();
+    if (holds_db_use_guard_ || is_secondary_update_thread_) {
+      // Neither kind of DB-user context can safely run the mutation cleanup
+      // path:
+      //   - DbUseGuard holder would self-wait on the write lock
+      //   - secondary update thread would self-join via
+      //     StopSecondaryUpdateThread()
+      // Terminate immediately and let the OS reclaim resources.
+      std::_Exit(1);
+    }
+    {
+      DbStateMutationGuard mutation(this);
+      DeleteDBs();
+    }
     db_bench_exit(1);
   }
 
@@ -4010,6 +4068,7 @@ class Benchmark {
       }
 
       if (fresh_db) {
+        DbStateMutationGuard mutation(this);
         if (FLAGS_use_existing_db) {
           fprintf(stdout, "%-12s : skipped (--use_existing_db is true)\n",
                   name.c_str());
@@ -4137,11 +4196,7 @@ class Benchmark {
       }
     }
 
-    if (secondary_update_thread_) {
-      secondary_update_stopped_.store(1, std::memory_order_relaxed);
-      secondary_update_thread_->join();
-      secondary_update_thread_.reset();
-    }
+    StopSecondaryUpdateThread();
 
     if (name != "replay" && FLAGS_trace_file != "") {
       Status s = db_.db->EndTrace();
@@ -4179,6 +4234,83 @@ class Benchmark {
   std::unique_ptr<port::Thread> secondary_update_thread_;
   std::atomic<int> secondary_update_stopped_{0};
   uint64_t secondary_db_updates_ = 0;
+  void StopSecondaryUpdateThread() {
+    if (secondary_update_thread_) {
+      secondary_update_stopped_.store(1, std::memory_order_relaxed);
+      secondary_update_thread_->join();
+      secondary_update_thread_.reset();
+      secondary_update_stopped_.store(0, std::memory_order_relaxed);
+    }
+  }
+
+  // True iff the current thread is inside a DbUseGuard scope.
+  // Set/cleared only by DbUseGuard. Strict guard ownership.
+  static thread_local bool holds_db_use_guard_;
+
+  // True iff the current thread is inside a DbStateMutationGuard scope.
+  // Set/cleared only by DbStateMutationGuard.
+  static thread_local bool holds_db_state_mutation_guard_;
+
+  // True iff the current thread is the secondary update thread.
+  // The secondary thread reads its captured DBWithColumnFamilies* without
+  // going through DbUseGuard or SelectDBWithCfh. This flag exists so
+  // ErrorExit can route it to _Exit(1) without self-joining via
+  // StopSecondaryUpdateThread.
+  static thread_local bool is_secondary_update_thread_;
+
+  class DbUseGuard {
+   public:
+    explicit DbUseGuard(Benchmark* bm)
+        : lock_(CheckDbUsePreconditionsAndGetLock(bm)) {
+      holds_db_use_guard_ = true;
+    }
+    ~DbUseGuard() {
+      assert(holds_db_use_guard_);
+      holds_db_use_guard_ = false;
+    }
+    DbUseGuard(const DbUseGuard&) = delete;
+    DbUseGuard& operator=(const DbUseGuard&) = delete;
+    DbUseGuard(DbUseGuard&&) = delete;
+    DbUseGuard& operator=(DbUseGuard&&) = delete;
+
+   private:
+    static port::RWMutex* CheckDbUsePreconditionsAndGetLock(Benchmark* bm) {
+      assert(!holds_db_use_guard_);
+      return &bm->db_lifecycle_rwlock_;
+    }
+
+    ReadLock lock_;
+  };
+
+  class DbStateMutationGuard {
+   public:
+    explicit DbStateMutationGuard(Benchmark* bm)
+        : bm_(bm), lock_(CheckDbMutationPreconditionsAndGetLock(bm)) {
+      bm_->StopSecondaryUpdateThread();
+      holds_db_state_mutation_guard_ = true;
+    }
+    ~DbStateMutationGuard() {
+      assert(holds_db_state_mutation_guard_);
+      holds_db_state_mutation_guard_ = false;
+    }
+    DbStateMutationGuard(const DbStateMutationGuard&) = delete;
+    DbStateMutationGuard& operator=(const DbStateMutationGuard&) = delete;
+    DbStateMutationGuard(DbStateMutationGuard&&) = delete;
+    DbStateMutationGuard& operator=(DbStateMutationGuard&&) = delete;
+
+   private:
+    static port::RWMutex* CheckDbMutationPreconditionsAndGetLock(
+        Benchmark* bm) {
+      assert(!holds_db_state_mutation_guard_);
+      assert(!holds_db_use_guard_);
+      assert(!is_secondary_update_thread_);
+      return &bm->db_lifecycle_rwlock_;
+    }
+
+    Benchmark* bm_;
+    WriteLock lock_;
+  };
+
   struct ThreadArg {
     Benchmark* bm;
     SharedState* shared;
@@ -4204,7 +4336,10 @@ class Benchmark {
     SetPerfLevel(static_cast<PerfLevel>(shared->perf_level));
     perf_context.EnablePerLevelPerfContext();
     thread->stats.Start(thread->tid);
-    (arg->bm->*(arg->method))(thread);
+    {
+      DbUseGuard db_guard(arg->bm);
+      (arg->bm->*(arg->method))(thread);
+    }
     if (FLAGS_perf_level > ROCKSDB_NAMESPACE::PerfLevel::kDisable) {
       thread->stats.AddMessage(std::string("PERF_CONTEXT:\n") +
                                get_perf_context()->ToString());
@@ -5382,6 +5517,7 @@ class Benchmark {
       if (s.ok() && FLAGS_secondary_update_interval > 0) {
         secondary_update_thread_.reset(new port::Thread(
             [this](int interval, DBWithColumnFamilies* _db) {
+              is_secondary_update_thread_ = true;
               while (0 == secondary_update_stopped_.load(
                               std::memory_order_relaxed)) {
                 Status secondary_update_status =
@@ -5694,6 +5830,10 @@ class Benchmark {
   }
 
   DBWithColumnFamilies* SelectDBWithCfh(uint64_t rand_int) {
+    // Caller MUST hold DbUseGuard (proven by ownership flag). The secondary
+    // update thread does not call this function; it uses its captured
+    // DBWithColumnFamilies* directly.
+    assert(holds_db_use_guard_);
     if (db_.db != nullptr) {
       return &db_;
     } else {
@@ -9485,6 +9625,10 @@ class Benchmark {
   }
 };
 
+thread_local bool Benchmark::holds_db_use_guard_ = false;
+thread_local bool Benchmark::holds_db_state_mutation_guard_ = false;
+thread_local bool Benchmark::is_secondary_update_thread_ = false;
+
 int db_bench_tool(int argc, char** argv, ToolHooks& hooks) {
   ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ConfigOptions config_options;