Update version and HISTORY.md for 11.4.3 release

Summary:
Pull Request resolved: https://github.com/facebook/rocksdb/pull/14881

Read-only DB instances can share a directory with a live writer, so their view of live files is only a snapshot from the time they opened. After read-only DB open began setting `opened_successfully_` for its intended open-success semantics, that state had an unintended side effect: the common close path treated a successful read-only open like a read-write open and ran obsolete-file cleanup. If the writer created or made files live after the read-only handle opened, that cleanup could use the stale read-only live set and delete files still needed by the writer.

This change records whether a `DBImpl` was opened read-only and skips close-time obsolete-file cleanup for read-only DBs instead of simply keeping `opened_successfully_=false`, which could be confusing and easily mistaken in the future again.

For completeness I also updated other callsites of `opened_successfully_`, but those should not be real bugs as ReadOnly DBs do not run flushes/compactions or write to WALs.

Reviewed By: xingbowang

Differential Revision: D109622445

fbshipit-source-id: d7be4b20fce86ccb218a63ac6f5b707b316aac79

.github/actions/pre-steps/action.yml

@@ -2,6 +2,13 @@ name: pre-steps
 runs:
   using: composite
   steps:
+  - name: Dump ulimits for diagnostics
+    run: |
+      echo "=== ulimits (soft) ==="
+      ulimit -a -S || true
+      echo "=== ulimits (hard) ==="
+      ulimit -a -H || true
+    shell: bash
   - name: Install lld linker for faster builds
     run: apt-get update -y && apt-get install -y lld 2>/dev/null || true
     shell: bash

.github/actions/windows-build-steps/action.yml

@@ -3,7 +3,7 @@ inputs:
   suite-run:
     description: Comma-separated list of test suites to run (empty to skip C++ tests)
     required: false
-    default: arena_test,db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test
+    default: arena_test,db_basic_test,db_test,db_etc2_test,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test
   run-java:
     description: Whether to run Java tests
     required: false

.github/workflows/ai-review-analysis.yml

@@ -531,7 +531,9 @@ jobs:
               core.warning(
                 `Could not parse classifier output: ${error.message}`);
             }
-            const budget = label === 'simple' ? '16000' : '32000';
+            // Keep Claude's thinking budget below the action's effective
+            // max_tokens so the API has room for the final review text.
+            const budget = label === 'simple' ? '16000' : '24000';
             core.info(
               `Complexity classified as: ${label} ` +
               `-> MAX_THINKING_TOKENS=${budget}`
@@ -551,15 +553,28 @@ jobs:
         run: |
           if [ -z "${OPENAI_API_KEY}" ]; then
             echo "OPENAI_API_KEY not configured, skipping Codex review"
+            printf 'complex\n' > codex-complexity-output.txt
             exit 0
           fi
           mkdir -p "${CODEX_HOME}"
+          set +e
           codex exec \
             -m "${CODEX_MODEL}" \
             -c model_reasoning_effort="medium" \
             --dangerously-bypass-approvals-and-sandbox \
             --output-last-message codex-complexity-output.txt \
             - < /tmp/complexity_prompt.txt > codex-complexity.log 2>&1
+          exit_code=$?
+          set -e
+          if [ "${exit_code}" -ne 0 ]; then
+            echo "Codex complexity classification failed with exit code ${exit_code}"
+            echo "Defaulting to complex review budget"
+            if [ -s codex-complexity.log ]; then
+              echo "Last 100 lines of codex-complexity.log:"
+              tail -100 codex-complexity.log
+            fi
+            printf 'complex\n' > codex-complexity-output.txt
+          fi
 
       - name: Pick thinking budget (Codex)
         if: >-
@@ -1102,9 +1117,19 @@ jobs:
             }
             const override = (process.env.OVERRIDE || '').trim();
             if (override && /^\d+$/.test(override)) {
-              core.info(`Using thinking_budget override: ${override}`);
+              const maxSafeBudget = 24000;
+              const requested = parseInt(override, 10);
+              const budget = Math.min(requested, maxSafeBudget).toString();
+              if (budget !== override) {
+                core.info(
+                  `Clamping thinking_budget override from ${override} ` +
+                  `to ${budget} so max_tokens can exceed thinking tokens.`
+                );
+              } else {
+                core.info(`Using thinking_budget override: ${budget}`);
+              }
               core.setOutput('label', 'override');
-              core.setOutput('tokens', override);
+              core.setOutput('tokens', budget);
               return;
             }
             let label = 'complex';
@@ -1122,7 +1147,9 @@ jobs:
               core.warning(
                 `Could not parse classifier output: ${error.message}`);
             }
-            const budget = label === 'simple' ? '16000' : '32000';
+            // Keep Claude's thinking budget below the action's effective
+            // max_tokens so the API has room for the final review text.
+            const budget = label === 'simple' ? '16000' : '24000';
             core.info(
               `Complexity classified as: ${label} ` +
               `-> MAX_THINKING_TOKENS=${budget}`
@@ -1140,13 +1167,30 @@ jobs:
           CODEX_HOME: /tmp/codex-home
           CODEX_MODEL: ${{ inputs.selected_model != '' && inputs.selected_model || inputs.default_model }}
         run: |
+          if [ -z "${OPENAI_API_KEY}" ]; then
+            echo "OPENAI_API_KEY not configured, skipping Codex review"
+            printf 'complex\n' > codex-complexity-output.txt
+            exit 0
+          fi
           mkdir -p "${CODEX_HOME}"
+          set +e
           codex exec \
             -m "${CODEX_MODEL}" \
             -c model_reasoning_effort="medium" \
             --dangerously-bypass-approvals-and-sandbox \
             --output-last-message codex-complexity-output.txt \
             - < /tmp/complexity_prompt.txt > codex-complexity.log 2>&1
+          exit_code=$?
+          set -e
+          if [ "${exit_code}" -ne 0 ]; then
+            echo "Codex complexity classification failed with exit code ${exit_code}"
+            echo "Defaulting to complex review budget"
+            if [ -s codex-complexity.log ]; then
+              echo "Last 100 lines of codex-complexity.log:"
+              tail -100 codex-complexity.log
+            fi
+            printf 'complex\n' > codex-complexity-output.txt
+          fi
 
       - name: Pick thinking budget (Codex)
         if: steps.request.outputs.skip != 'true' && inputs.provider == 'codex'

.github/workflows/claude-review.yml

@@ -34,7 +34,7 @@ on:
           - claude-sonnet-4-6
         default: claude-opus-4-6
       thinking_budget:
-        description: Override MAX_THINKING_TOKENS (blank = auto-classify)
+        description: Override MAX_THINKING_TOKENS (blank = auto-classify, capped at 24000)
         required: false
         type: string
         default: ""

.github/workflows/pr-jobs.yml

@@ -373,14 +373,17 @@ jobs:
           crash_duration: 240
     container:
       image: ghcr.io/facebook/rocksdb_ubuntu:22.1
-      options: --shm-size=16gb
+      options: --shm-size=16gb --ulimit nofile=1048576:1048576
     steps:
     - uses: actions/checkout@v4.1.0
     - uses: "./.github/actions/pre-steps"
     - uses: "./.github/actions/setup-ccache"
       with:
         cache-key-prefix: mini-crashtest
-    - run: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=${{ matrix.crash_duration }} --max_key=2500000' ${{ matrix.crash_test_target }}
+    - run: |
+        ulimit -S -n $(ulimit -Hn) # Raise open files soft limit to hard limit
+        echo "Updated ulimit -Hn: $(ulimit -Hn) -Sn: $(ulimit -Sn)"
+        make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=${{ matrix.crash_duration }} --max_key=2500000' ${{ matrix.crash_test_target }}
     - uses: "./.github/actions/teardown-ccache"
       if: always()
     - uses: "./.github/actions/post-steps"
@@ -534,7 +537,7 @@ jobs:
           suite_run: db_test
           run_java: "false"
         - test_shard: other
-          suite_run: arena_test,db_basic_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test
+          suite_run: arena_test,db_basic_test,db_etc2_test,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test
           run_java: "false"
         - test_shard: java
           suite_run: ""

.gitignore

@@ -52,7 +52,7 @@ GRTAGS
 GTAGS
 rocksdb_dump
 rocksdb_undump
-db_test2
+db_etc2_test
 trace_analyzer
 block_cache_trace_analyzer
 io_tracer_parser

BUCK

@@ -41,9 +41,6 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "db/blob/blob_source.cc",
         "db/blob/blob_write_batch_transformer.cc",
         "db/blob/prefetch_buffer_collection.cc",
-        "db/blog/blog_format.cc",
-        "db/blog/blog_reader.cc",
-        "db/blog/blog_writer.cc",
         "db/builder.cc",
         "db/c.cc",
         "db/coalescing_iterator.cc",
@@ -4595,18 +4592,6 @@ cpp_unittest_wrapper(name="block_test",
             extra_compiler_flags=[])
 
 
-cpp_unittest_wrapper(name="blog_format_test",
-            srcs=["db/blog/blog_format_test.cc"],
-            deps=[":rocksdb_test_lib"],
-            extra_compiler_flags=[])
-
-
-cpp_unittest_wrapper(name="blog_writer_test",
-            srcs=["db/blog/blog_writer_test.cc"],
-            deps=[":rocksdb_test_lib"],
-            extra_compiler_flags=[])
-
-
 cpp_unittest_wrapper(name="bloom_test",
             srcs=["util/bloom_test.cc"],
             deps=[":rocksdb_test_lib"],
@@ -4883,6 +4868,12 @@ cpp_unittest_wrapper(name="db_encryption_test",
             extra_compiler_flags=[])
 
 
+cpp_unittest_wrapper(name="db_etc2_test",
+            srcs=["db/db_etc2_test.cc"],
+            deps=[":rocksdb_test_lib"],
+            extra_compiler_flags=[])
+
+
 cpp_unittest_wrapper(name="db_etc3_test",
             srcs=["db/db_etc3_test.cc"],
             deps=[":rocksdb_test_lib"],
@@ -5039,12 +5030,6 @@ cpp_unittest_wrapper(name="db_test",
             extra_compiler_flags=[])
 
 
-cpp_unittest_wrapper(name="db_test2",
-            srcs=["db/db_test2.cc"],
-            deps=[":rocksdb_test_lib"],
-            extra_compiler_flags=[])
-
-
 cpp_unittest_wrapper(name="db_universal_compaction_test",
             srcs=["db/db_universal_compaction_test.cc"],
             deps=[":rocksdb_test_lib"],

CLAUDE.md

@@ -209,14 +209,48 @@ The following patterns emerged as frequent sources of review feedback:
 ## Important tips
 
 ### Build system
-* There are 3 build system. Make, CMake, BUCK(meta internal).
+* There are 3 build system. Make for git clones, BUCK (meta internal) for hg
+  clones, and CMake for some special cases.
 * When a new .cc file is added, update Makefile, CMakeLists.txt, src.mk, BUCK.
 * Don't manually edit BUCK file, after updating src.mk, run
     /usr/local/bin/python3 buckifier/buckify_rocksdb.py to update it
-* Use make to build and run the test. CMake and BUCK are not used locally.
-* Use `make dbg` command to build all of the unit test in debug mode.
 * For -j in make command, use the number of CPU cores to decide it.
 
+### When to run `make clean` (avoid mixing build modes)
+
+The Makefile does **not** track build mode, so object files from a prior
+build are silently reused even when compiled with different flags, leading
+to confusing linker errors, sanitizer false negatives, ODR violations, or
+"phantom" bugs.
+
+Run `make clean` before switching any of these:
+* **`ASSERT_STATUS_CHECKED=1` ↔ unset** — changes the `Status` class layout (ABI break).
+* **Sanitizer builds** — toggling any of `COMPILE_WITH_ASAN=1`,
+    `COMPILE_WITH_UBSAN=1`, `COMPILE_WITH_TSAN=1` on/off.
+* `DEBUG_LEVEL=0` (release) ↔ `DEBUG_LEVEL=1` (debug, default for `make dbg`).
+* Different compilers, `OPT` levels, or other flags affecting codegen/ABI.
+
+**Notable exception:** `DEBUG_LEVEL=2` can be safely mixed with
+`DEBUG_LEVEL=1` — rebuild a subset of files with `DEBUG_LEVEL=2` to get
+extra/more accurate runtime checks for those files without a full clean.
+
+When in doubt, `make clean` is cheap insurance compared to chasing a
+phantom bug.
+
+### Source checks
+* Run `make check-sources` before committing. This catches non-ASCII
+    characters in source files and other source-level issues that CI will
+    reject. In particular, **do not use Unicode characters** (em dashes,
+    smart quotes, etc.) in comments or strings -- use ASCII equivalents
+    (`--` instead of em dash, `'` instead of smart quote, etc.).
+
+### RTTI and dynamic_cast
+* Production code and `db_stress` must build in **release mode
+    (`-fno-rtti`)**. Do not use `dynamic_cast` anywhere except unit tests.
+    Use `static_cast_with_check` from `util/cast_util.h` (validates with
+    `dynamic_cast` in debug builds, plain `static_cast` in release).
+* Unit tests (`*_test.cc`) are built in debug mode with RTTI enabled.
+
 ### Unit Test
 * After all of the unit tests are added, review them and try to extract common
     reusable utility functions to reduce code duplication due to copy past between

CMakeLists.txt

@@ -654,6 +654,8 @@ if(USE_FOLLY)
     FMT_INST_PATH)
     exec_program(ls ARGS -d ${FOLLY_INST_PATH}/../gflags* OUTPUT_VARIABLE
     GFLAGS_INST_PATH)
+    exec_program(ls ARGS -d ${FOLLY_INST_PATH}/../glog* OUTPUT_VARIABLE
+    GLOG_INST_PATH)
     set(Boost_DIR ${BOOST_INST_PATH}/lib/cmake/Boost-1.83.0)
     if(EXISTS ${FMT_INST_PATH}/lib64)
       set(fmt_DIR ${FMT_INST_PATH}/lib64/cmake/fmt)
@@ -661,6 +663,42 @@ if(USE_FOLLY)
       set(fmt_DIR ${FMT_INST_PATH}/lib/cmake/fmt)
     endif()
     set(gflags_DIR ${GFLAGS_INST_PATH}/lib/cmake/gflags)
+    if(EXISTS ${GLOG_INST_PATH}/lib64/cmake/glog)
+      set(GLOG_CMAKE_DIR ${GLOG_INST_PATH}/lib64/cmake/glog)
+    elseif(EXISTS ${GLOG_INST_PATH}/lib/cmake/glog)
+      set(GLOG_CMAKE_DIR ${GLOG_INST_PATH}/lib/cmake/glog)
+    endif()
+    if(NOT GLOG_CMAKE_DIR)
+      find_library(GETDEPS_GLOG_LIBRARY NAMES glogd glog
+        PATHS ${GLOG_INST_PATH}/lib64 ${GLOG_INST_PATH}/lib
+        NO_DEFAULT_PATH)
+      if(NOT GETDEPS_GLOG_LIBRARY OR
+         NOT EXISTS ${GLOG_INST_PATH}/include/glog/logging.h)
+        message(FATAL_ERROR "Could not find getdeps glog under "
+          "${GLOG_INST_PATH}")
+      endif()
+      set(GLOG_CMAKE_DIR ${CMAKE_CURRENT_BINARY_DIR}/getdeps-glog-cmake)
+      file(MAKE_DIRECTORY ${GLOG_CMAKE_DIR})
+      file(WRITE ${GLOG_CMAKE_DIR}/glog-config.cmake
+        "if(NOT TARGET glog::glog)\n"
+        "  add_library(glog::glog UNKNOWN IMPORTED)\n"
+        "  set_target_properties(glog::glog PROPERTIES\n"
+        "    IMPORTED_LOCATION \"${GETDEPS_GLOG_LIBRARY}\"\n"
+        "    INTERFACE_INCLUDE_DIRECTORIES \"${GLOG_INST_PATH}/include\")\n"
+        "endif()\n"
+        "set(glog_FOUND TRUE)\n"
+        "set(Glog_FOUND TRUE)\n"
+        "set(GLOG_FOUND TRUE)\n"
+        "set(GLOG_LIBRARY \"${GETDEPS_GLOG_LIBRARY}\")\n"
+        "set(GLOG_LIBRARIES \"${GETDEPS_GLOG_LIBRARY}\")\n"
+        "set(GLOG_INCLUDE_DIR \"${GLOG_INST_PATH}/include\")\n")
+    endif()
+    set(glog_DIR ${GLOG_CMAKE_DIR})
+    set(Glog_DIR ${GLOG_CMAKE_DIR})
+    # Unlike gflags, glog may also be installed system-wide. Pre-resolve the
+    # getdeps copy so folly-config.cmake cannot fall back to another version.
+    find_package(glog CONFIG REQUIRED PATHS ${GLOG_CMAKE_DIR}
+                 NO_DEFAULT_PATH)
 
     exec_program(sed ARGS -i 's/gflags_shared//g'
     ${FOLLY_INST_PATH}/lib/cmake/folly/folly-targets.cmake)
@@ -743,9 +781,6 @@ set(SOURCES
         db/blob/blob_source.cc
         db/blob/blob_write_batch_transformer.cc
         db/blob/prefetch_buffer_collection.cc
-        db/blog/blog_format.cc
-        db/blog/blog_reader.cc
-        db/blog/blog_writer.cc
         db/builder.cc
         db/c.cc
         db/coalescing_iterator.cc
@@ -1434,8 +1469,6 @@ if(WITH_TESTS)
         db/blob/db_blob_compaction_test.cc
         db/blob/db_blob_corruption_test.cc
         db/blob/db_blob_index_test.cc
-        db/blog/blog_format_test.cc
-        db/blog/blog_writer_test.cc
         db/column_family_test.cc
         db/compact_files_test.cc
         db/compaction/clipping_iterator_test.cc
@@ -1458,6 +1491,7 @@ if(WITH_TESTS)
         db/db_clip_test.cc
         db/db_dynamic_level_test.cc
         db/db_encryption_test.cc
+        db/db_etc2_test.cc
         db/db_etc3_test.cc
         db/db_flush_test.cc
         db/db_inplace_update_test.cc
@@ -1481,7 +1515,6 @@ if(WITH_TESTS)
         db/db_table_properties_test.cc
         db/db_tailing_iter_test.cc
         db/db_test.cc
-        db/db_test2.cc
         db/db_logical_block_size_cache_test.cc
         db/db_universal_compaction_test.cc
         db/db_wal_test.cc

HISTORY.md

@@ -1,6 +1,63 @@
 # Rocksdb Change Log
 > NOTE: Entries for next release do not go here. Follow instructions in `unreleased_history/README.txt`
 
+## 11.4.3 (06/24/2026)
+### Bug Fixes
+* Fixed a bug where closing a read-only DB instance could delete live SST files created by a concurrent read-write DB sharing the same directory.
+
+## 11.4.2 (06/08/2026)
+### Public API Changes
+* Added experimental read-scoped block buffer provider API, configured through `ReadOptions::read_scoped_block_buffer_provider`, for supported block-based table iterator scans and MultiScan reads to use caller-provided read-scoped storage for final data-block contents. When configured, supported provider-backed scan data-block reads bypass the data-block cache. The provider is ignored when mmap reads are enabled. RocksDB may still use ordinary temporary scratch for serialized block bytes, such as when a block may be compressed. Get and MultiGet do not currently provide API guarantees for this provider.
+
+### Bug Fixes
+* Fixed a use after free bug in Multiscan when async IO is used
+
+## 11.4.1 (06/05/2026)
+* No-op patch used to sync with internal fb version.
+
+## 11.4.0 (06/02/2026)
+### Public API Changes
+* Added `rocksdb_options_set_memtable_batch_lookup_optimization()` and `rocksdb_options_get_memtable_batch_lookup_optimization()` to the C API, exposing the existing `AdvancedColumnFamilyOptions::memtable_batch_lookup_optimization` field. This allows C API users (and downstream language bindings) to enable the skip-list memtable's batch-lookup optimization for `MultiGet`, which caches the search path between consecutive keys and reduces per-key cost from O(log N) to O(log d) where d is the distance between consecutive keys.
+* Added `rocksdb_readoptions_set_optimize_multiget_for_io()` and `rocksdb_readoptions_get_optimize_multiget_for_io()` to the C API, exposing the existing `ReadOptions::optimize_multiget_for_io` field. This allows C API users (and downstream language bindings) to opt out of the multi-level parallel MultiGet path, which only takes effect when the library is built with `USE_COROUTINES`.
+* Added `rocksdb_block_based_table_index_block_search_type_auto` enum constant and the `rocksdb_block_based_options_set_uniform_cv_threshold()` setter to the C API. The constant exposes `BlockSearchType::kAuto`, and the setter exposes `BlockBasedTableOptions::uniform_cv_threshold`. Both are required for `kAuto` index-block search to take effect from the C API: without setting `uniform_cv_threshold >= 0`, the per-block `is_uniform` footer bit is never written, and `kAuto` falls back to binary search at read time.
+* Added `EventListener::OnDBShutdownBegin`, a callback that fires once when a DB begins shutdown, including when RocksDB cleans up a failed `DB::Open()` attempt.
+* Added a new PerfContext counter `blob_cache_read_byte` for blob cache bytes read.
+
+### Behavior Changes
+* Remote compaction now falls back to local compaction when the primary cannot deserialize a successful `CompactionServiceResult` before installing any remote output files.
+* StringToMap (rocksdb/convenience.h) now preserves the outer braces of nested values so each map entry is in self-contained form and can be embedded directly into another `key=value;` string (e.g. SetOptions) without losing inner ';' delimiters. A new symmetric MapToString utility is provided.
+
+### Bug Fixes
+* Fixed a bug where `AbortAllCompactions()` could leave automatic compaction work unscheduled when aborting before the background worker picked it, causing compaction and write stalls until DB restart.
+* Fix a bug where db had a false positive compaction corruption error due to remote compaction service result with `has_accurate_num_input_records=false` not being serialized.
+* Fixed WritePrepared TransactionDB cleanup after retryable commit or rollback write failures so prepared transactions remain rollbackable instead of leaving unresolved prepared state.
+* Fix a rare corruption bug for tiered compaction that incorrectly moved last level range tombstones into proximal level. This corruption error is surfaced when force_consistency_checks is enabled.
+
+### Performance Improvements
+* Reduce the likelihood of user-facing metadata operations blocking on MANIFEST rotation when file creation is slow, such as on remote storage. MANIFEST write batches containing foreground edits from `CreateColumnFamily()`, `DropColumnFamily()`, `CreateColumnFamilyWithImport()`, `IngestExternalFile()`, and `DeleteFilesInRanges()` now get a relaxed file size threshold for triggering MANIFEST rotation; background-only MANIFEST write batches, such as compaction and flush, continue to use the normal threshold. This change should make auto-tuning manifest file size more attractive (see `max_manifest_file_size` and `max_manifest_space_amp_pct` options).
+
+## 11.3.0 (05/15/2026)
+### New Features
+* Add experimental DB option `async_wal_precreate` to precreate the next WAL file in a background thread and reduce foreground WAL rotation latency. The option is sanitized to false when WAL recycling is enabled.
+* Added a new `EventListener::OnCompactionPreCommit` callback that fires after a compaction job finishes but before its input files are released (i.e. while `FileMetaData::being_compacted` is still true). Listeners that maintain bookkeeping of which files are currently being compacted can clean up such state in this new callback to avoid races with concurrent compaction picking, where another thread might pick up the same files for a new compaction immediately after `being_compacted` is flipped back to false but before `OnCompactionCompleted` fires. The default implementation is a no-op so this is not a breaking change.
+* Add mutable DBOption `optimize_manifest_for_recovery` (default false). When enabled, RocksDB can reduce recovery work after a clean shutdown, which may lower DB::Open latency on warm reopens.
+* Added public utility APIs `ParseCompressionNameForDisplay()` to convert `TableProperties::compression_name` into a human-readable compression name for both legacy and format_version 7+ SST metadata, including custom `CompressionManager`-provided display names for custom compression types.
+* Add `reuse_manifest_on_open` DBOption (default false). When enabled, DB::Open reuses the existing MANIFEST file for append instead of creating a fresh one, avoiding the cost of serializing the entire database state into a new MANIFEST on the first post-open write. To prevent this feature from interfering with manifest file size auto-tuning, an extra forward-compatible field is now always added to the MANIFEST (to track the last "compacted" size).
+
+### Behavior Changes
+* Read-only open with `error_if_wal_file_exists=true` now tolerates empty WAL files so empty precreated WALs do not prevent inspection.
+* WriteCommitted TransactionDB now matches WritePrepared and WriteUnprepared compaction filtering in both single- and two-write-queue modes: a compaction filter's FilterMergeOperand will not be invoked on merge operands at or below the latest published sequence number.
+
+### Bug Fixes
+* Fixed blob-backed wide-column merge reads to preserve correct status
+propagation and resolution across memtable, read-only, and secondary DB
+paths.
+* Fixed a bug where `DB::GetCreationTimeOfOldestFile()` could return inaccurate results instead of the real creation time when called shortly after opening a legacy DB (one whose manifest lacks `file_creation_time`) with `open_files_async = true`. The API now waits for background SST file loading to complete only when needed; modern DBs are unaffected.
+* Fixed merge reads against wide-column/blob-backed base values to preserve precise failure statuses, including `GetMergeOperands()` and direct-write memtable reads.
+* Fix bug in range tombstone synthesis that covers live keys added during an IngestExternalFile
+* Reject the empty string as a column family name in `DB::CreateColumnFamily` / `DB::CreateColumnFamilies`. Previously such calls returned OK and a usable handle, but the column family was not persisted in the manifest, so any data written to it was silently lost on DB reopen.
+* Fixed a bug where a WriteCommitted TransactionDB using commit-bypass WBWI ingestion could drop an entry that is still visible at the published sequence boundary.
+
 ## 11.2.0 (04/18/2026)
 ### New Features
 * Added experimental `DBOptions::fast_sst_open` option. When enabled, RocksDB retrieves opaque file system metadata for SST files after flush, compaction, and external file ingestion, persists it in the MANIFEST, and passes it back to the file system on subsequent file opens to accelerate DB open time.

Makefile

@@ -327,6 +327,9 @@ missing_make_config_paths := $(shell				\
 $(foreach path, $(missing_make_config_paths), \
 	$(warning Warning: $(path) does not exist))
 
+# This (the first rule) must depend on "all".
+default: all
+
 ifeq ($(PLATFORM), OS_AIX)
 # no debug info
 else ifneq ($(PLATFORM), IOS)
@@ -384,6 +387,10 @@ endif
 # TSAN doesn't work well with jemalloc. If we're compiling with TSAN, we should use regular malloc.
 ifdef COMPILE_WITH_TSAN
 	DISABLE_JEMALLOC=1
+	# Use a suppressions file instead of the process-wide TSAN default
+	# suppressions hook, which belongs to the final application.
+	TSAN_OPTIONS?=suppressions=$(CURDIR)/tools/tsan_suppressions.txt
+	export TSAN_OPTIONS
 	EXEC_LDFLAGS += -fsanitize=thread
 	PLATFORM_CCFLAGS += -fsanitize=thread -fPIC -DFOLLY_SANITIZE_THREAD
 	PLATFORM_CXXFLAGS += -fsanitize=thread -fPIC -DFOLLY_SANITIZE_THREAD
@@ -481,9 +488,6 @@ ifdef ROCKSDB_MODIFY_NPHASH
   PLATFORM_CXXFLAGS += -DROCKSDB_MODIFY_NPHASH=1
 endif
 
-# This (the first rule) must depend on "all".
-default: all
-
 WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare -Wshadow \
   -Wunused-parameter
 
@@ -883,23 +887,43 @@ coverage: clean
 parallel_tests = $(patsubst %,parallel_%,$(PARALLEL_TEST))
 .PHONY: gen_parallel_tests $(parallel_tests)
 # Shard size controls how many test cases run per process. The actual number
-# of shards per binary is: min(ceil(test_count / GTEST_SHARD_SIZE), NCORES * 8)
+# of shards per binary is: min(ceil(test_count / shard_size), NCORES * 8)
 # This adapts to machine size: many small shards on beefy machines, fewer
 # larger shards on CI (typically 2-4 cores).
 GTEST_SHARD_SIZE ?= 10
 NCORES ?= $(shell nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)
 
+# Per-binary overrides for GTEST_SHARD_SIZE, used to chop up binaries whose
+# individual test cases are slow enough that the default of 10 leaves one
+# shard on the critical path of "make check". Format: whitespace-separated
+# tokens "BINARY:SIZE". Sizes were tuned from `make suggest-slow-tests`
+# output to keep max shard time under ~20s on a beefy box. Reducing too far
+# adds per-shard fixed overhead, so don't use 1 unless individual tests are
+# >5s.
+SHARD_SIZE_OVERRIDES ?= \
+  point_lock_manager_stress_test:1 \
+  external_sst_file_test:1 \
+  external_sst_file_basic_test:2 \
+  db_test:3 \
+  compaction_service_test:1
+
 $(parallel_tests):
 	$(AM_V_at)TEST_BINARY=$(patsubst parallel_%,%,$@); \
   TEST_COUNT=` \
     (./$$TEST_BINARY --gtest_list_tests 2>/dev/null || echo "  list_failure") \
     | grep -c '^ '`; \
 	if [ "$$TEST_COUNT" -le 0 ]; then TEST_COUNT=1; fi; \
+	SHARD_SIZE=$(GTEST_SHARD_SIZE); \
+	for o in $(SHARD_SIZE_OVERRIDES); do \
+		case "$$o" in \
+			"$$TEST_BINARY":*) SHARD_SIZE=$${o#*:} ;; \
+		esac; \
+	done; \
 	MAX_SHARDS=$$(( $(NCORES) * 8 )); \
-	NUM_SHARDS=$$(( (TEST_COUNT + $(GTEST_SHARD_SIZE) - 1) / $(GTEST_SHARD_SIZE) )); \
+	NUM_SHARDS=$$(( (TEST_COUNT + SHARD_SIZE - 1) / SHARD_SIZE )); \
 	if [ "$$NUM_SHARDS" -gt "$$MAX_SHARDS" ]; then NUM_SHARDS=$$MAX_SHARDS; fi; \
 	if [ "$$NUM_SHARDS" -le 0 ]; then NUM_SHARDS=1; fi; \
-	echo "  Generating $$NUM_SHARDS shards for $$TEST_BINARY ($$TEST_COUNT tests)"; \
+	echo "  Generating $$NUM_SHARDS shards for $$TEST_BINARY ($$TEST_COUNT tests, shard_size=$$SHARD_SIZE)"; \
 	SHARD_IDX=0; \
 	while [ "$$SHARD_IDX" -lt "$$NUM_SHARDS" ]; do \
 		if [ -n "$(CI_TOTAL_SHARDS)" ] && [ $$(( $$SHARD_IDX % $(CI_TOTAL_SHARDS) )) -ne $(CI_SHARD_INDEX) ]; then \
@@ -925,31 +949,64 @@ gen_parallel_tests:
 	$(AM_V_at)$(FIND) t -type f -name 'run-*' -exec rm -f {} \;
 	$(MAKE) $(parallel_tests)
 
-# Reorder input lines (which are one per test) so that the
-# longest-running tests appear first in the output.
-# Do this by prefixing each selected name with its duration,
-# sort the resulting names, and remove the leading numbers.
-# FIXME: the "100" we prepend is a fake time, for now.
-# FIXME: squirrel away timings from each run and use them
-# (when present) on subsequent runs to order these tests.
+# Reorder input lines (one per test or per shard) so the longest-running
+# tests/shards start first under "make check" parallel scheduling. The trick
+# is to prefix matched names with a fake duration (100), sort numerically
+# descending, then strip the prefix. Slow-matched tests therefore come ahead
+# of all unmatched tests; ordering within each group is unspecified.
 #
-# Without this reordering, these two tests would happen to start only
-# after almost all other tests had completed, thus adding 100 seconds
-# to the duration of parallel "make check".  That's the difference
-# between 4 minutes (old) and 2m20s (new).
+# Why this matters: gnu_parallel hands out jobs in input order. If a slow
+# test starts only after most others have finished, the run is bottlenecked
+# on its tail. Front-loading slow tests overlaps them with the bulk of
+# faster ones for much better wall-clock time.
 #
-# 152.120 PASS t/DBTest.FileCreationRandomFailure
-# 107.816 PASS t/DBTest.EncodeDecompressedBlockSizeTest
+# Maintenance:
+#   1) Run `make check` so LOG is up-to-date.
+#   2) Run `make suggest-slow-tests` to see candidate slow binaries.
+#   3) Add binaries with high per-shard time (slow critical path) or high
+#      total time across many shards (need early queueing for fan-out).
+# Each alternative is wrapped in `^.*NAME.*$$` so the *whole input line* is
+# captured into $$1; then `s,(...),100 $$1,` prepends "100 " to the line.
 #
-# With sharded test execution, prioritize binaries known to be slow.
-# These generate many shards and should start early for good load balancing.
+# Tiers below are based on observed timings (see suggest-slow-tests).
+# Tier 1: max single-shard time >= 30s (critical-path bottlenecks).
+# Tier 2: max single-shard time 15-30s.
+# Tier 3: huge total time across many tiny shards; front-loading them keeps
+#         the tail of the run busy while big shards finish.
 slow_test_regexp = \
-	^.*block_based_table_reader_test.*$$|^.*table_test.*$$|^.*block_test.*$$|^.*write_prepared_transaction_test.*$$|^.*transaction_test.*$$|^.*external_sst_file_test.*$$|^.*db_wal_test.*$$|^.*db_with_timestamp_basic_test.*$$|^.*db_test-.*$$
+	^.*point_lock_manager_stress_test.*$$|^.*db_test.*$$|^.*external_sst_file_test.*$$|^.*compaction_service_test.*$$|^.*corruption_test.*$$|^.*comparator_db_test.*$$|^.*external_sst_file_basic_test.*$$|^.*rate_limiter_test.*$$|^.*db_compaction_test.*$$|^.*write_prepared_transaction_test.*$$|^.*db_merge_operator_test.*$$|\
+	^.*db_dynamic_level_test.*$$|^.*db_bloom_filter_test.*$$|^.*error_handler_fs_test.*$$|^.*merge_helper_test.*$$|^.*transaction_test.*$$|^.*db_kv_checksum_test.*$$|^.*inlineskiplist_test.*$$|\
+	^.*db_with_timestamp_basic_test.*$$|^.*table_test.*$$|^.*db_wal_test.*$$|^.*block_based_table_reader_test.*$$|^.*block_test.*$$
 prioritize_long_running_tests =						\
   perl -pe 's,($(slow_test_regexp)),100 $$1,'				\
     | sort -k1,1gr							\
     | sed 's/^[.0-9]* //'
 
+# Helper: print binaries observed to be slow in the last `make check` run.
+# Use this to decide whether `slow_test_regexp` above needs updating.
+# Aggregates per-binary across shards; flags any binary whose max single-shard
+# time is >= 20s OR whose total time is >= 200s.
+.PHONY: suggest-slow-tests
+suggest-slow-tests:
+	@if [ ! -s LOG ]; then \
+	  echo "No (or empty) LOG file. Run 'make check' first." >&2; exit 1; \
+	fi
+	@bash -c '$(quoted_perl_command)' < LOG | awk '\
+	  { \
+	    t=$$1; name=$$3; \
+	    sub(/^t\/run-/, "", name); \
+	    sub(/-shard-[0-9]+$$/, "", name); \
+	    total[name] += t; \
+	    if (t > max[name]) max[name] = t; \
+	    count[name]++; \
+	  } \
+	  END { \
+	    printf "%8s %8s %5s  %s\n", "TOTAL_s", "MAX_s", "SHRDS", "BINARY"; \
+	    for (n in total) \
+	      if (max[n] >= 20 || total[n] >= 200) \
+	        printf "%8.1f %8.1f %5d  %s\n", total[n], max[n], count[n], n; \
+	  }' | (read header; echo "$$header"; sort -k2,2gr)
+
 # "make check" uses
 # Run with "make J=1 check" to disable parallelism in "make check".
 # Run with "make J=200% check" to run two parallel jobs per core.
@@ -986,7 +1043,7 @@ check_0:
 		        awk -v s=$(CI_SHARD_INDEX) -v n=$(CI_TOTAL_SHARDS) '(NR-1)%n==s'; \
 		      else cat; fi; \
 		fi; \
-		find t -name 'run-*' -print; \
+		$(FIND) t -name 'run-*' -print; \
 	} \
 	  | $(prioritize_long_running_tests)				\
 	  | grep -E '$(tests-regexp)'					\
@@ -1008,7 +1065,7 @@ valgrind_check_0:
 	  '  run "make watch-log" in a separate window' '';		\
 	{								\
 	  printf './%s\n' $(filter-out $(PARALLEL_TEST) %skiplist_test options_settable_test, $(TESTS));		\
-	  find t -name 'run-*' -print; \
+	  $(FIND) t -name 'run-*' -print; \
 	}								\
 	  | $(prioritize_long_running_tests)				\
 	  | grep -E '$(tests-regexp)'					\
@@ -1226,11 +1283,17 @@ clean-not-downloaded: clean-ext-libraries-bin clean-rocks clean-not-downloaded-r
 
 clean-rocks:
 # Not practical to exactly match all versions/variants in naming (e.g. debug or not)
-	rm -f ${LIBNAME}*.so* ${LIBNAME}*.a
-	rm -f $(BENCHMARKS) $(TOOLS) $(TESTS) $(PARALLEL_TEST) $(MICROBENCHS)
-	rm -rf $(CLEAN_FILES) ios-x86 ios-arm scan_build_report
-	$(FIND) . -name "*.[oda]" -exec rm -f {} \;
-	$(FIND) . -type f \( -name "*.gcda" -o -name "*.gcno" \) -exec rm -f {} \;
+	@echo Removing link targets
+	@rm -f ${LIBNAME}*.so* ${LIBNAME}*.a $(BENCHMARKS) $(TOOLS) $(TESTS) $(PARALLEL_TEST) $(MICROBENCHS)
+	@echo Finding and cleaning other files
+	@rm -rf $(CLEAN_FILES) ios-x86 ios-arm scan_build_report
+	@if $(FIND) Makefile -regextype awk &> /dev/null; then \
+		$(FIND) . -regextype awk \
+			-regex '.*/([.].*|third-party)' -prune -o \
+			-type f -regex '.*[.]([oda]|gcda|gcno)' -exec rm -f {} \; ; \
+	else \
+		$(FIND) . -name "*.[oda]" -exec rm -f {} \; ; \
+	fi
 
 clean-rocksjava: clean-rocks
 	rm -rf jl jls
@@ -1243,7 +1306,7 @@ clean-ext-libraries-all:
 	rm -rf bzip2* snappy* zlib* lz4* zstd*
 
 clean-ext-libraries-bin:
-	find . -maxdepth 1 -type d \( -name bzip2\* -or -name snappy\* -or -name zlib\* -or -name lz4\* -or -name zstd\* \) -prune -exec rm -rf {} \;
+	$(FIND) . -maxdepth 1 -type d \( -name bzip2\* -or -name snappy\* -or -name zlib\* -or -name lz4\* -or -name zstd\* \) -prune -exec rm -rf {} \;
 
 tags:
 	ctags -R .
@@ -1521,7 +1584,7 @@ db_open_with_config_test: $(OBJ_DIR)/db/db_open_with_config_test.o $(TEST_LIBRAR
 db_test: $(OBJ_DIR)/db/db_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
-db_test2: $(OBJ_DIR)/db/db_test2.o $(TEST_LIBRARY) $(LIBRARY)
+db_etc2_test: $(OBJ_DIR)/db/db_etc2_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
 db_etc3_test: $(OBJ_DIR)/db/db_etc3_test.o $(TEST_LIBRARY) $(LIBRARY)
@@ -1536,12 +1599,6 @@ db_logical_block_size_cache_test: $(OBJ_DIR)/db/db_logical_block_size_cache_test
 db_blob_index_test: $(OBJ_DIR)/db/blob/db_blob_index_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
-blog_format_test: $(OBJ_DIR)/db/blog/blog_format_test.o $(TEST_LIBRARY) $(LIBRARY)
-	$(AM_LINK)
-
-blog_writer_test: $(OBJ_DIR)/db/blog/blog_writer_test.o $(TEST_LIBRARY) $(LIBRARY)
-	$(AM_LINK)
-
 db_block_cache_test: $(OBJ_DIR)/db/db_block_cache_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 

blog_format_spec.html

@@ -1,318 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-<meta charset="UTF-8">
-<title>Blog File Format Specification - RocksDB</title>
-<style>
-  body { font-family: 'Segoe UI', Arial, sans-serif; max-width: 960px; margin: 40px auto; padding: 0 20px; color: #222; line-height: 1.6; }
-  h1 { border-bottom: 3px solid #333; padding-bottom: 8px; }
-  h2 { border-bottom: 2px solid #999; padding-bottom: 4px; margin-top: 40px; }
-  h3 { margin-top: 30px; }
-  pre { background: #f4f4f4; border: 1px solid #ddd; border-radius: 4px; padding: 12px 16px; overflow-x: auto; font-size: 13px; line-height: 1.5; }
-  code { background: #f0f0f0; padding: 1px 5px; border-radius: 3px; font-size: 13px; }
-  table { border-collapse: collapse; width: 100%; margin: 16px 0; }
-  th, td { border: 1px solid #bbb; padding: 8px 12px; text-align: left; vertical-align: top; }
-  th { background: #e8e8e8; font-weight: 600; }
-  .diagram { background: #fafafa; border: 1px solid #ccc; border-radius: 4px; padding: 16px; margin: 16px 0; font-family: 'Courier New', monospace; font-size: 13px; white-space: pre; overflow-x: auto; line-height: 1.4; }
-  .note { background: #fff8e1; border-left: 4px solid #ffc107; padding: 10px 14px; margin: 14px 0; }
-  .decision { background: #e8f5e9; border-left: 4px solid #4caf50; padding: 10px 14px; margin: 14px 0; }
-</style>
-</head>
-<body>
-
-<h1>"Blog" File Format Specification</h1>
-<p><em>A unified file format for WAL, Blob, and related data in RocksDB</em></p>
-<p>Status: Implementation in progress &nbsp;|&nbsp; Format name: "blog" (portmanteau of "blob" + "log")</p>
-
-<!-- ======================================================================= -->
-<h2>Vocabulary</h2>
-
-<table>
-<tr><th>Term</th><th>Scope</th><th>Definition</th></tr>
-<tr><td><strong>Record</strong></td><td>Blog file</td><td>A top-level unit in the blog file body: blob record, WriteBatch record, preamble-start record, footer record.</td></tr>
-<tr><td><strong>Operation (op)</strong></td><td>WriteBatch</td><td>An entry within a WriteBatch: Put op, Delete op, Merge op, etc.</td></tr>
-<tr><td><strong>Compact format</strong></td><td>Record</td><td>A record with varint length &le; 3 bytes. Omits the type byte (type from header).</td></tr>
-<tr><td><strong>Full format</strong></td><td>Record</td><td>A record with varint length &gt; 3 bytes. Includes explicit type, pre-payload compression type, and prefix checksum.</td></tr>
-</table>
-
-<!-- ======================================================================= -->
-<h2 id="s1">File-Level Structure</h2>
-
-<p>A blog file consists of three regions: a <strong>fixed header</strong> with a <strong>property section</strong>, a <strong>body</strong> of records, and an optional <strong>footer</strong> composed of footer records.</p>
-
-<div class="diagram">
-+---------------------------------------------------------------------+
-|                         FILE HEADER (40 bytes fixed)                 |
-|  96-bit magic, schema version, checksum type, compact record type,  |
-|  flags, escape sequence, reserved, property section size, checksum  |
-+---------------------------------------------------------------------+
-|                    HEADER PROPERTY SECTION                           |
-|  Named key-value pairs + 5-byte trailer (compression + checksum)    |
-|  Followed by arbitrarily long padding (0x00 or 0xFF)                |
-+---------------------------------------------------------------------+
-|                         FILE BODY                                    |
-|  Sequence of records, each starting with the escape sequence.       |
-|  Record types: blob, WriteBatch, preamble-start, footer records.    |
-+---------------------------------------------------------------------+
-|                    FILE FOOTER (optional)                            |
-|  Footer index records (0xFD), footer properties record (0xFE),      |
-|  footer locator record (0xFF) -- must be last.                      |
-+---------------------------------------------------------------------+</div>
-
-<h3>Fixed Header (40 bytes)</h3>
-
-<table>
-<tr><th>Offset</th><th>Size</th><th>Field</th></tr>
-<tr><td>0</td><td>12</td><td>File magic: <code>kBlogFileMagic</code> -- opaque 12 bytes ("Blog" ASCII + XXH3_64bits("Blog"))</td></tr>
-<tr><td>12</td><td>1</td><td>Schema version (currently 0). Newer versions are rejected by older readers.</td></tr>
-<tr><td>13</td><td>1</td><td>Checksum type (<code>ChecksumType</code> enum: 0=none, 1=CRC32c, 4=XXH3). Configured via <code>DBOptions::blog_checksum</code>.</td></tr>
-<tr><td>14</td><td>1</td><td>Compact record type (<code>BlogRecordType</code> -- type for all compact-format records)</td></tr>
-<tr><td>15</td><td>1</td><td>Flags: bit 0 = <code>kBlogFileRecycled</code> (file may contain trailing stale data)</td></tr>
-<tr><td>16</td><td>10</td><td>Escape sequence: bytes [0-5] random (byte[0] not 0x00/0xFF), bytes [6-9] = lower32(XXH3_64bits_withSeed(bytes[0-5], kBlogEscapeSeqSeed)). First 4 bytes also serve as incarnation ID (little-endian via DecodeFixed32).</td></tr>
-<tr><td>26</td><td>6</td><td>Reserved (zero)</td></tr>
-<tr><td>32</td><td>4</td><td>Property section size (uint32 LE) -- includes 5-byte trailer</td></tr>
-<tr><td>36</td><td>4</td><td>Fixed header checksum -- covers bytes [0, 36)</td></tr>
-</table>
-
-<h3>Header Property Section</h3>
-
-<p>Immediately follows the fixed header. Contains named key-value pairs encoded as:</p>
-<pre>[name_len: varint32] [name: name_len bytes] [value_len: varint32] [value: value_len bytes] ...</pre>
-<p>Followed by a 5-byte trailer: <code>[compression_type: 1B][checksum: 4B]</code> (same as block-based table). The checksum covers the property data + compression_type, with a context checksum modifier using the incarnation ID at offset <code>kBlogFileFixedHeaderSize</code>.</p>
-
-<p>After the property section, arbitrarily long padding (0x00 or 0xFF, differing from the last meaningful byte) aligns the first record's escape sequence. This supports stronger alignment requirements (flush/sync boundaries) or recycling files of all zeros.</p>
-
-<h3>Property Name Convention</h3>
-<table>
-<tr><th>First letter</th><th>Meaning</th></tr>
-<tr><td>Uppercase (e.g. <code>"CompressionCompatibilityName"</code>)</td><td><strong>Required</strong> -- reader must recognize or reject the file</td></tr>
-<tr><td>Lowercase (e.g. <code>"role"</code>)</td><td><strong>Ignorable</strong> -- reader may skip unrecognized properties</td></tr>
-</table>
-
-<h3>Known Header Properties</h3>
-<table>
-<tr><th>Name</th><th>Required?</th><th>Description</th></tr>
-<tr><td><code>"CompressionCompatibilityName"</code></td><td>Required (if any compression)</td><td>Compression manager compatibility name (e.g. "builtin_v2")</td></tr>
-<tr><td><code>"WriteBatchStreamingCompressionType"</code></td><td>Required (if streaming active)</td><td>2-digit hex of CompressionType for streaming WriteBatch compression (e.g. "07" for ZSTD). Presence means WriteBatch records with <code>kStreamingCompressionSentinel</code> (0x7F) trailer are part of a cross-record streaming context.</td></tr>
-<tr><td><code>"TimestampSize"</code></td><td>Required (if &gt;0)</td><td>Size of user timestamps</td></tr>
-<tr><td><code>"PredecessorWalInfo"</code></td><td>Required (if tracking)</td><td>Predecessor WAL metadata for verification</td></tr>
-<tr><td><code>"role"</code></td><td>Ignorable</td><td>File purpose hint: "wal", "blob", "rlog"</td></tr>
-<tr><td><code>"compressionSettings"</code></td><td>Ignorable</td><td>Human-readable compression settings for debugging</td></tr>
-<tr><td><code>"creationTime"</code></td><td>Ignorable</td><td>Unix timestamp of file creation</td></tr>
-<tr><td><code>"creator"</code></td><td>Ignorable</td><td>Host/process identifier for debugging</td></tr>
-</table>
-
-<!-- ======================================================================= -->
-<h2 id="s2">Escape Sequences</h2>
-
-<div class="decision">
-<strong>Decision:</strong> 10-byte escape sequence: 6 random bytes + 4 derived bytes. First byte not 0x00 nor 0xFF (distinguishable from padding). Bytes [6-9] = lower32(XXH3_64bits_withSeed(bytes[0-5], kBlogEscapeSeqSeed)), enabling heuristic detection without knowing the header.
-</div>
-
-<p>Escape sequences must be <strong>32-bit aligned</strong> in the file. The first 4 bytes (little-endian) serve as the <strong>incarnation ID</strong> for context checksums.</p>
-
-<h3>Primary Roles</h3>
-<ul>
-  <li><strong>WAL recycling detection:</strong> When recovery encounters undecodable data, scan for our escape sequence. If not found, the data is stale/recycled (different random sequence). The <code>kBlogFileRecycled</code> flag in the header indicates recycling is expected.</li>
-  <li><strong>Non-linear recovery:</strong> After corruption, scan forward for the next escape sequence to resume recovery (<code>kSkipAnyCorruptedRecords</code>).</li>
-  <li><strong>Record boundary discovery:</strong> Find record boundaries from any file position.</li>
-</ul>
-
-<!-- ======================================================================= -->
-<h2 id="s3">Record Types and Formats</h2>
-
-<h3>Record Type Values</h3>
-<table>
-<tr><th>Value</th><th>Name</th><th>Usage</th></tr>
-<tr><td colspan="3"><em>Body record types (0x01-0x7F)</em></td></tr>
-<tr><td>0x01</td><td><code>kBlogBlobRecord</code></td><td>Blob data (typically a separated value)</td></tr>
-<tr><td>0x02</td><td><code>kBlogPreambleStartRecord</code></td><td>Preamble marker (stub)</td></tr>
-<tr><td>0x03</td><td><code>kBlogWriteBatchRecord</code></td><td>Serialized WriteBatch</td></tr>
-<tr><td>0x04-0x7F</td><td></td><td><em>Reserved for future body record types</em></td></tr>
-<tr><td colspan="3"><em>Footer record types (0xF0-0xFF, in expected order of occurrence with gaps)</em></td></tr>
-<tr><td>0xF0</td><td><code>kBlogFooterIndexRecord</code></td><td>Large footer metadata (sparse index)</td></tr>
-<tr><td>0xF1-0xF3</td><td></td><td><em>Reserved for future footer data records</em></td></tr>
-<tr><td>0xF4-0xF7</td><td></td><td><em>Reserved for future footer data records</em></td></tr>
-<tr><td>0xF8</td><td><code>kBlogFooterPropertiesRecord</code></td><td>Small footer metadata (named properties)</td></tr>
-<tr><td>0xF9-0xFB</td><td></td><td><em>Reserved</em></td></tr>
-<tr><td>0xFC</td><td><code>kBlogFooterFileChecksumInfo</code></td><td>Full file checksum state dump. Allows inferring the file checksum from just reading the tail. Placeholder for future use.</td></tr>
-<tr><td>0xFD-0xFE</td><td></td><td><em>Reserved</em></td></tr>
-<tr><td>0xFF</td><td><code>kBlogFooterLocatorRecord</code></td><td>Footer locator with relative offsets. Must be the last record in the file.</td></tr>
-</table>
-
-<h3>Length Semantics</h3>
-<ul>
-  <li><code>length = 0</code>: No payload and no 5-byte trailer. Record is escape_seq + prefix fields + padding.</li>
-  <li><code>length &gt; 0</code>: Payload of <code>length</code> bytes followed by 5-byte trailer.</li>
-  <li><code>length = 2^63 - 1</code>: Unspecified size. Reader scans forward for next escape sequence.</li>
-</ul>
-
-<h3>Compact Record Format (varint length &le; 2 bytes, default record type)</h3>
-<div class="diagram">
-[escape_seq: 10B] [varint length: 1-2B]
-  [payload: length bytes]
-  [compression_type: 1B] [checksum: 4B]
-  [padding: 0+ B]</div>
-<p>No type byte -- type comes from the header's <code>compact_record_type</code> field. Used for payloads &lt; 16 KiB.</p>
-
-<h3>Full Record Format (varint length &gt; 2 bytes, or forced)</h3>
-<div class="diagram">
-[escape_seq: 10B] [varint length: 3+B] [type: 1B] [compression_type: 1B]
-  [prefix_checksum: 4B]
-  [payload: length bytes]     (if length &gt; 0)
-  [compression_type: 1B] [checksum: 4B]  (if length &gt; 0)
-  [padding: 0+ B]</div>
-<p>The <code>prefix_checksum</code> covers (varint + type + compression_type), with a context checksum modifier for defense-in-depth. Validates the length before attempting a potentially large read. The pre-payload <code>compression_type</code> enables streaming decompression without seeking to the trailer.</p>
-
-<p>Records that don't match the compact_record_type (e.g. footer records, preamble-start) always use full format by encoding the varint with &ge; 3 bytes (irregular encoding).</p>
-
-<p>All checksums in blog files (fixed header, property section trailer, record prefix, record trailer) use context checksum modifiers incorporating the incarnation ID, providing defense-in-depth for misplaced data detection even with <code>kNoChecksum</code>.</p>
-
-<!-- ======================================================================= -->
-<h2 id="s4">Compression</h2>
-
-<h3>Per-Record Compression</h3>
-<p>Each record's trailer <code>compression_type</code> byte indicates the compression algorithm used for that record's payload. The actual type comes from <code>Compressor::CompressBlock</code> output, not the configured type. This is the same approach used by SST block-based table builders.</p>
-
-<h3>Streaming Compression (WriteBatch records only)</h3>
-<p>When the header property <code>WriteBatchStreamingCompressionType</code> is set, WriteBatch record payloads may use streaming compression across records. Such records use <code>kStreamingCompressionSentinel</code> (0x7F) as their trailer's compression_type byte, indicating "compressed using the streaming algorithm declared in the header."</p>
-
-<p>A record with <code>kNoCompression</code> (0x00) in the trailer remains individually uncompressed even when streaming is active, enabling dynamic on/off switching based on compression effectiveness.</p>
-
-<p>Non-WriteBatch records (blob records, footer records) are always outside the streaming context and use per-record compression, keeping blob data randomly readable.</p>
-
-<div class="note">
-Currently only ZSTD is supported for streaming compression. The <code>WriteBatchStreamingCompressionType</code> property is required (uppercase first letter), so older readers that don't understand it will reject the file rather than misinterpret streaming-compressed payloads.
-</div>
-
-<!-- ======================================================================= -->
-<h2 id="s5">Padding Scheme</h2>
-
-<p>Padding fills the gap between a record's trailer (or the header property section) and the next escape sequence. The padding byte value is chosen to <strong>differ from the last meaningful byte</strong>, enabling unambiguous scanning.</p>
-
-<table>
-<tr><th>Last meaningful byte</th><th>Pad with</th><th>Mandatory?</th></tr>
-<tr><td><code>0x00</code></td><td><code>0xFF</code></td><td>Yes -- at least 1 byte required</td></tr>
-<tr><td><code>0xFF</code></td><td><code>0x00</code></td><td>Yes -- at least 1 byte required</td></tr>
-<tr><td>Anything else</td><td><code>0x00</code> (writer's choice)</td><td>Only if needed for alignment</td></tr>
-</table>
-
-<p>Padding may be arbitrarily long for stronger alignment requirements (e.g. flush/sync boundaries) or for recycling a file of all zeros. The reader skips padding by scanning forward over 0x00/0xFF bytes in 4-byte-aligned chunks until finding a byte that is neither, which is the first byte of the next escape sequence (guaranteed to be neither 0x00 nor 0xFF).</p>
-
-<!-- ======================================================================= -->
-<h2 id="s6">Context Checksums</h2>
-
-<div class="decision">
-<strong>Decision (adopted from SST format_version=6):</strong> Each record's checksum incorporates a 32-bit <strong>incarnation ID</strong> (first 4 bytes of the escape sequence). Uses <code>ChecksumModifierForContext(incarnation_id, record_offset)</code> -- the same mechanism as block-based SST files.
-</div>
-
-<p>Context checksums provide defense-in-depth for recycling detection and general data integrity verification with zero per-record overhead.</p>
-
-<p>The checksum type is configured from the CF's <code>BlockBasedTableOptions::checksum</code> setting (for blob files) or the default CF's setting (for WAL files). This is a reasonable but hopefully temporary way of configuring blog file internal checksums; a dedicated option may be added later.</p>
-
-<!-- ======================================================================= -->
-<h2 id="s7">Footer</h2>
-
-<p>The footer is composed of <strong>regular records</strong> using the full record format, maintaining simple forward parsability. A file is cleanly sealed when a valid footer locator record is the last record. Footer records appear in the order listed below.</p>
-
-<ol>
-  <li><strong>Footer index records</strong> (0xF0): Large metadata like sparse indexes for range queries. Zero or more per file.</li>
-  <li><strong>Footer properties record</strong> (0xF8): Named properties (same encoding as header properties, but the full record format's own trailer provides integrity). At most one per file.</li>
-  <li><strong>Footer file checksum info</strong> (0xFC): Placeholder for future use. Would contain a dump of the full file checksum state, allowing the file checksum to be inferred from just reading the tail of the file.</li>
-  <li><strong>Footer locator record</strong> (0xFF): Must be the <strong>last record</strong>. Contains relative offsets (in 4-byte units) to other footer records, sequential relative to each other in reverse order.</li>
-</ol>
-
-<h3>Footer Locator Payload</h3>
-<pre>[record_type: 1B] [relative_offset_in_4B_units: uint32 LE] ...</pre>
-<p>First entry: distance from locator's escape_seq backward to the preceding footer record. Subsequent: from the previous entry backward to the next.</p>
-
-<h3>Known Footer Properties</h3>
-<table>
-<tr><th>Name</th><th>Description</th></tr>
-<tr><td><code>"blobCount"</code></td><td>Number of blob records</td></tr>
-<tr><td><code>"totalBlobBytes"</code></td><td>Total uncompressed blob payload bytes</td></tr>
-<tr><td><code>"totalBlobBytesCompressed"</code></td><td>Total compressed blob payload bytes on disk</td></tr>
-<tr><td><code>"totalWriteBatchBytes"</code></td><td>Total uncompressed WriteBatch payload bytes</td></tr>
-<tr><td><code>"totalWriteBatchBytesCompressed"</code></td><td>Total compressed WriteBatch payload bytes on disk</td></tr>
-<tr><td><code>"minSequence"</code></td><td>Minimum sequence number</td></tr>
-<tr><td><code>"maxSequence"</code></td><td>Maximum sequence number</td></tr>
-</table>
-
-<!-- ======================================================================= -->
-<h2 id="s8">WAL Recovery Modes</h2>
-
-<p>The escape sequence is the <strong>primary mechanism</strong> for detecting WAL recycling and for non-linear data recovery. The <code>kBlogFileRecycled</code> flag in the header indicates the file may contain trailing stale data from a previous incarnation.</p>
-
-<table>
-<tr>
-  <th style="width:22%">Mode</th>
-  <th style="width:48%">Behavior on Undecodable Data</th>
-  <th style="width:30%">Escape Sequence Role</th>
-</tr>
-<tr>
-  <td><code>kAbsoluteConsistency</code></td>
-  <td>Error on any failure, including incomplete trailing records. If recycled and no escape sequence found in trailing data, accept as clean end.</td>
-  <td>Distinguishes stale recycled data from corruption.</td>
-</tr>
-<tr>
-  <td><code>kTolerateCorruptedTailRecords</code></td>
-  <td>Incomplete trailing record (Incomplete status): tolerate, stop at last complete record. Corruption: scan for escape sequence. Not found = clean end (recycled or preallocated). Found with valid record beyond = mid-file corruption (error).</td>
-  <td>Distinguishes tail damage from mid-file corruption.</td>
-</tr>
-<tr>
-  <td><code>kPointInTimeRecovery</code> (default)</td>
-  <td>Stop at first inconsistency. Incomplete trailing records tolerated. Recycled trailing data accepted.</td>
-  <td>Diagnostic improvement.</td>
-</tr>
-<tr>
-  <td><code>kSkipAnyCorruptedRecords</code></td>
-  <td>On corruption, scan forward for next escape sequence. If found, try to decode and verify; resume if valid. If not found, stop.</td>
-  <td>Essential for record discovery after corruption.</td>
-</tr>
-</table>
-
-<!-- ======================================================================= -->
-<h2 id="s9">WriteBatch Blob Preamble</h2>
-
-<p>A <strong>blob preamble</strong> is a sequence of blob records preceded by a preamble-start record (type 0x02, full format, length=0) and followed by the associated WriteBatch record. The blob records contain separated values referenced by <code>file_number = 0</code> blob references in the WriteBatch.</p>
-
-<p>This functionality is stubbed in the current implementation: the preamble-start record type is defined and can be written/read, but the full preamble flow (in-memory serialization, blob reference encoding) is deferred.</p>
-
-<!-- ======================================================================= -->
-<h2 id="s10">Replication Log Column Family (ShallowL0Store)</h2>
-
-<p>Deferred. The blog format supports a "rlog" role hint for replication log storage, with sparse indexes in footer records and in-memory index maintenance. See design notes in the original proposal.</p>
-
-<!-- ======================================================================= -->
-<h2 id="s11">Implementation Status</h2>
-
-<h3>Resolved (from original Open Questions)</h3>
-<table>
-<tr><th>Question</th><th>Resolution</th></tr>
-<tr><td>Escape sequence size</td><td>10 bytes: 6 random + 4 derived. First byte not 0x00/0xFF.</td></tr>
-<tr><td>Header/footer encoding</td><td>40-byte fixed header + named property section with 5-byte trailer. Footer uses regular full-format records: index (0xF0), properties (0xF8), file checksum info (0xFC, placeholder), locator (0xFF). Numbered in expected occurrence order with gaps for future types.</td></tr>
-<tr><td>Preamble-start record format</td><td>Full format, length=0, type 0x02. Minimal -- just the type tag.</td></tr>
-<tr><td>WriteBatch record framing</td><td>Same compact/full format as blob records, type 0x03.</td></tr>
-<tr><td>Compact vs full format</td><td>Compact: varint &le; 2B (payload &lt; 16 KiB), matches compact_record_type, no type byte. Full: varint &gt; 2B (possibly irregular), includes type + pre-payload compression_type + prefix_checksum with context modifier.</td></tr>
-<tr><td>Checksum context</td><td>All checksums (fixed header, property section, record prefix, record trailer) use context checksum modifiers with the incarnation ID, ensuring nonzero stored values even with <code>kNoChecksum</code>.</td></tr>
-<tr><td>Schema version</td><td>Starts at 0. Newer versions are rejected by older readers (returns <code>NotSupported</code>). Checksum is verified before schema version is checked.</td></tr>
-<tr><td>Streaming compression sentinel</td><td><code>kStreamingCompressionSentinel = 0x7F</code> defined in <code>CompressionType</code> enum (compression_type.h). Used in blog record trailers for streaming-compressed WriteBatch payloads.</td></tr>
-</table>
-
-<h3>DB Options</h3>
-<ul>
-  <li><code>use_blog_format_for_wals</code> (bool, default false): New WAL files use blog format.</li>
-  <li><code>use_blog_format_for_blobs</code> (bool, default false): New blob files use blog format.</li>
-  <li><code>blog_checksum</code> (<code>ChecksumType</code>, default <code>kXXH3</code>): Checksum algorithm for blog format files. <code>ChecksumType</code> enum moved to <code>include/rocksdb/checksum_type.h</code> (from <code>table.h</code>) for use by both table and DB options.</li>
-</ul>
-<p>All are immutable <code>DBOptions</code>. Existing legacy WAL and blob files are always readable regardless of these settings.</p>
-
-<h3>Compression Manager Integration</h3>
-<ul>
-  <li><strong>Blog blob files</strong> use the CF's configured compression manager (via <code>MutableCFOptions::compression_manager</code>). Legacy blob files continue to use the builtin manager for backward compatibility.</li>
-  <li><strong>Blog WAL files</strong> use per-record compression via <code>Compressor::CompressBlock</code>. The actual <code>CompressionType</code> from <code>CompressBlock</code> output is stored in the record trailer.</li>
-  <li>Streaming compression infrastructure exists (<code>WriteBatchStreamingCompressionType</code> property, <code>kStreamingCompressionSentinel</code> 0x7F trailer value) but is not yet enabled by default.</li>
-</ul>
-
-</body>
-</html>

build_tools/check-sources.sh

@@ -37,7 +37,7 @@ if [ "$?" != "1" ]; then
   BAD=1
 fi
 
-git grep -n -P "[\x80-\xFF]" -- ':!docs' ':!*.md'
+LC_ALL=C git grep -n $'[\x80-\xff]' -- ':!docs' ':!*.md' ':!.github'
 if [ "$?" != "1" ]; then
   echo '^^^^ Use only ASCII characters in source files'
   BAD=1

build_tools/getdeps_fallback_mirror.py

@@ -1,62 +1,188 @@
 #!/usr/bin/env python3
+#  Copyright (c) Meta Platforms, Inc. and affiliates.
+#  This source code is licensed under both the GPLv2 (found in the COPYING file in the root directory)
+#  and the Apache 2.0 License (found in the LICENSE.Apache file in the root directory).
 """
 Pre-download packages with unreliable mirrors using fallback mirrors.
 Reads package info from folly's getdeps manifest files.
 """
-import sys
-import os
-import hashlib
-import subprocess
 import configparser
+import hashlib
+import os
+import shutil
+import sys
+import urllib.request
+
+DOWNLOAD_TIMEOUT_SECONDS = 120
+DOWNLOAD_CHUNK_BYTES = 64 * 1024
+MAX_DOWNLOAD_BYTES = 50 * 1024 * 1024
+
+MIRROR_FALLBACKS = {
+    "ftpmirror.gnu.org/gnu/": [
+        "https://mirrors.kernel.org/gnu/",
+        "https://ftpmirror.gnu.org/gnu/",
+        "https://ftp.gnu.org/gnu/",
+    ],
+    "ftp.gnu.org/gnu/": [
+        "https://mirrors.kernel.org/gnu/",
+        "https://ftpmirror.gnu.org/gnu/",
+        "https://ftp.gnu.org/gnu/",
+    ],
+}
+
+# These packages must have URLs matching MIRROR_FALLBACKS; other packages are
+# left for getdeps.py's normal download path.
+PACKAGES_TO_CHECK = ("autoconf", "automake", "libtool", "libiberty")
+
 
 def sha256_file(path):
     """Calculate SHA256 hash of a file."""
     h = hashlib.sha256()
     try:
-        with open(path, 'rb') as f:
-            for chunk in iter(lambda: f.read(65536), b''):
+        with open(path, "rb") as f:
+            for chunk in iter(lambda: f.read(65536), b""):
                 h.update(chunk)
         return h.hexdigest()
     except Exception:
         return None
 
+
 def parse_manifest(manifest_path):
     """Parse a getdeps manifest file to extract download info."""
-    config = configparser.ConfigParser()
+    # folly manifests can contain bare keys in sections unrelated to downloads.
+    config = configparser.ConfigParser(allow_no_value=True, interpolation=None)
     try:
-        config.read(manifest_path)
-        if 'download' in config:
-            return {
-                'url': config['download'].get('url', ''),
-                'sha256': config['download'].get('sha256', ''),
-            }
-    except Exception:
-        pass
+        with open(manifest_path, encoding="utf-8") as manifest_file:
+            config.read_file(manifest_file)
+    except Exception as ex:
+        print(f"  {os.path.basename(manifest_path)}: WARNING - parse failed: {ex}")
+        return None
+
+    if "download" in config:
+        return {
+            "url": config["download"].get("url", ""),
+            "sha256": config["download"].get("sha256", ""),
+        }
     return None
 
+
+def file_size(path):
+    try:
+        return os.path.getsize(path)
+    except Exception:
+        return None
+
+
 def get_fallback_mirrors(url):
     """Get fallback mirror URLs for a given URL."""
-    # Fallback mirror patterns for known unreliable hosts
-    mirror_fallbacks = {
-        "ftp.gnu.org/gnu/": [
-            "https://mirrors.kernel.org/gnu/",
-            "https://ftpmirror.gnu.org/gnu/",
-            "https://ftp.gnu.org/gnu/",
-        ],
-        "ftpmirror.gnu.org/gnu/": [
-            "https://mirrors.kernel.org/gnu/",
-            "https://ftpmirror.gnu.org/gnu/",
-            "https://ftp.gnu.org/gnu/",
-        ],
-    }
-
-    for pattern, mirrors in mirror_fallbacks.items():
+    for pattern, mirrors in MIRROR_FALLBACKS.items():
         if pattern in url:
             # Extract the path after the pattern
             path_start = url.find(pattern) + len(pattern)
             path = url[path_start:]
             return [mirror + path for mirror in mirrors]
-    return [url]  # No fallback, use original
+    return []
+
+
+def download_url(url, filepath):
+    """Download URL to filepath without leaving partial files behind."""
+    tmp_filepath = filepath + ".tmp"
+    if os.path.exists(tmp_filepath):
+        os.remove(tmp_filepath)
+
+    request = urllib.request.Request(
+        url, headers={"User-Agent": "rocksdb-getdeps-fallback/1.0"}
+    )
+    try:
+        with urllib.request.urlopen(
+            request, timeout=DOWNLOAD_TIMEOUT_SECONDS
+        ) as response, open(tmp_filepath, "wb") as output:
+            copied = 0
+            while True:
+                chunk = response.read(DOWNLOAD_CHUNK_BYTES)
+                if not chunk:
+                    break
+
+                copied += len(chunk)
+                if copied > MAX_DOWNLOAD_BYTES:
+                    raise Exception(
+                        f"download exceeds {MAX_DOWNLOAD_BYTES} bytes"
+                    )
+                output.write(chunk)
+        os.replace(tmp_filepath, filepath)
+    finally:
+        if os.path.exists(tmp_filepath):
+            os.remove(tmp_filepath)
+
+
+def prepare_download(package, info, download_dir, cache_dir):
+    url = info["url"]
+    expected_sha256 = info["sha256"]
+    mirrors = get_fallback_mirrors(url)
+    if not mirrors:
+        return False
+
+    if not expected_sha256:
+        print(f"  {package}: WARNING - skipped fallback without sha256")
+        return False
+
+    # getdeps uses format: {package}-{filename}
+    filename = f"{package}-{os.path.basename(url)}"
+    filepath = os.path.join(download_dir, filename)
+    cache_path = os.path.join(cache_dir, filename)
+
+    # Check if already valid.
+    actual_sha256 = sha256_file(filepath) if os.path.exists(filepath) else None
+    if actual_sha256 == expected_sha256:
+        print(f"  {filename}: OK (already downloaded)")
+        return True
+    if actual_sha256 is not None:
+        print(
+            f"  {filename}: WARNING - removing invalid download "
+            f"sha256={actual_sha256}"
+        )
+        os.remove(filepath)
+
+    # The cache is only an opportunistic single-build accelerator; callers
+    # should not share it across concurrent builds without external locking.
+    actual_sha256 = sha256_file(cache_path) if os.path.exists(cache_path) else None
+    if actual_sha256 == expected_sha256:
+        print(f"  {filename}: OK (from cache)")
+        shutil.copy2(cache_path, filepath)
+        return True
+    if actual_sha256 is not None:
+        print(
+            f"  {filename}: WARNING - removing invalid cache "
+            f"sha256={actual_sha256}"
+        )
+        os.remove(cache_path)
+
+    # Try fallback mirrors.
+    for mirror_url in mirrors:
+        print(f"  {filename}: trying {mirror_url}...")
+        try:
+            download_url(mirror_url, filepath)
+        except Exception as ex:
+            print(f"  {filename}: WARNING - download failed: {ex}")
+            continue
+
+        actual_sha256 = sha256_file(filepath)
+        if actual_sha256 == expected_sha256:
+            size = file_size(filepath)
+            print(f"  {filename}: OK (downloaded, {size} bytes)")
+            shutil.copy2(filepath, cache_path)
+            return True
+
+        size = file_size(filepath)
+        print(
+            f"  {filename}: WARNING - sha256 mismatch from {mirror_url}: "
+            f"expected={expected_sha256} actual={actual_sha256} size={size}"
+        )
+        os.remove(filepath)
+
+    print(f"  {filename}: WARNING - all mirrors failed")
+    return False
+
 
 def main():
     if len(sys.argv) != 4:
@@ -64,60 +190,39 @@ def main():
         sys.exit(1)
 
     download_dir, cache_dir, manifests_dir = sys.argv[1], sys.argv[2], sys.argv[3]
+    os.makedirs(download_dir, exist_ok=True)
+    os.makedirs(cache_dir, exist_ok=True)
 
-    # Packages known to have unreliable mirrors
-    packages_to_check = ["autoconf", "automake", "libtool"]
-
-    for package in packages_to_check:
+    checked = 0
+    ready = 0
+    for package in PACKAGES_TO_CHECK:
         manifest_path = os.path.join(manifests_dir, package)
-        if not os.path.exists(manifest_path):
+        if not os.path.isfile(manifest_path):
             continue
 
         info = parse_manifest(manifest_path)
-        if not info or not info['url'] or not info['sha256']:
+        if not info or not info["url"]:
             continue
 
-        # Determine filename from URL
-        url = info['url']
-        expected_sha256 = info['sha256']
-        url_filename = os.path.basename(url)
-
-        # getdeps uses format: {package}-{filename}
-        filename = f"{package}-{url_filename}"
-        filepath = os.path.join(download_dir, filename)
-        cache_path = os.path.join(cache_dir, filename)
-
-        # Check if already valid
-        if os.path.exists(filepath) and sha256_file(filepath) == expected_sha256:
-            print(f"  {filename}: OK (already downloaded)")
+        if not info["sha256"]:
+            print(f"  {package}: WARNING - skipped fallback without sha256")
             continue
 
-        # Check cache
-        if os.path.exists(cache_path) and sha256_file(cache_path) == expected_sha256:
-            print(f"  {filename}: OK (from cache)")
-            subprocess.run(['cp', cache_path, filepath], check=True)
+        if not get_fallback_mirrors(info["url"]):
+            print(
+                f"  {package}: WARNING - skipped fallback without known mirror "
+                f"for {info['url']}"
+            )
             continue
 
-        # Try fallback mirrors
-        mirrors = get_fallback_mirrors(url)
-        downloaded = False
-        for mirror_url in mirrors:
-            print(f"  {filename}: trying {mirror_url}...")
-            try:
-                subprocess.run(['wget', '-q', '-O', filepath, mirror_url], check=True, timeout=120)
-                if sha256_file(filepath) == expected_sha256:
-                    print(f"  {filename}: OK (downloaded)")
-                    subprocess.run(['cp', filepath, cache_path], check=False)
-                    downloaded = True
-                    break
-                else:
-                    os.remove(filepath)
-            except Exception:
-                if os.path.exists(filepath):
-                    os.remove(filepath)
+        checked += 1
+        try:
+            if prepare_download(package, info, download_dir, cache_dir):
+                ready += 1
+        except Exception as ex:
+            print(f"  {package}: WARNING - fallback preparation failed: {ex}")
 
-        if not downloaded:
-            print(f"  {filename}: WARNING - all mirrors failed")
+    print(f"  fallback mirror downloads ready: {ready}/{checked}")
 
 if __name__ == "__main__":
     main()

crash_test.mk

@@ -7,6 +7,13 @@ DB_STRESS_CMD?=./db_stress
 
 include common.mk
 
+ifdef COMPILE_WITH_TSAN
+	# Keep direct `make -f crash_test.mk COMPILE_WITH_TSAN=1 ...` runs
+	# aligned with the main Makefile's TSAN runtime options.
+	TSAN_OPTIONS?=suppressions=$(CURDIR)/tools/tsan_suppressions.txt
+	export TSAN_OPTIONS
+endif
+
 CRASHTEST_MAKE=$(MAKE) -f crash_test.mk
 CRASHTEST_PY=$(PYTHON) -u tools/db_crashtest.py --stress_cmd=$(DB_STRESS_CMD) --cleanup_cmd='$(DB_CLEANUP_CMD)' --destroy_db_initially=1
 

db/blob/blob_file_builder.cc

@@ -14,8 +14,6 @@
 #include "db/blob/blob_log_format.h"
 #include "db/blob/blob_log_writer.h"
 #include "db/blob/blob_source.h"
-#include "db/blog/blog_format.h"
-#include "db/blog/blog_writer.h"
 #include "db/event_helpers.h"
 #include "db/version_set.h"
 #include "file/filename.h"
@@ -69,13 +67,10 @@ BlobFileBuilder::BlobFileBuilder(
       min_blob_size_(mutable_cf_options->min_blob_size),
       blob_file_size_(mutable_cf_options->blob_file_size),
       blob_compression_type_(mutable_cf_options->blob_compression_type),
-      // Blog format uses the CF's compression manager; legacy format must
-      // use the builtin for compatibility.
-      blob_compression_manager_(immutable_options->use_blog_format_for_blobs &&
-                                        mutable_cf_options->compression_manager
-                                    ? mutable_cf_options->compression_manager
-                                    : GetBuiltinV2CompressionManager()),
-      blob_compressor_(blob_compression_manager_->GetCompressor(
+      // TODO with schema change: support custom compression manager and options
+      // such as max_compressed_bytes_per_kb
+      // NOTE: returns nullptr for kNoCompression
+      blob_compressor_(GetBuiltinV2CompressionManager()->GetCompressor(
           mutable_cf_options->blob_compression_opts, blob_compression_type_)),
       blob_compressor_wa_(blob_compressor_
                               ? blob_compressor_->ObtainWorkingArea()
@@ -128,10 +123,8 @@ Status BlobFileBuilder::Add(const Slice& key, const Slice& value,
   Slice blob = value;
   GrowableBuffer compressed_blob;
 
-  if (!blog_writer_) {
-    // Legacy format: compress before writing (compression type is implicit
-    // from the configured blob_compression_type_).
-    const Status s = LegacyCompressBlob(&blob, &compressed_blob);
+  {
+    const Status s = CompressBlobIfNeeded(&blob, &compressed_blob);
     if (!s.ok()) {
       return s;
     }
@@ -139,13 +132,10 @@ Status BlobFileBuilder::Add(const Slice& key, const Slice& value,
 
   uint64_t blob_file_number = 0;
   uint64_t blob_offset = 0;
-  uint64_t blob_on_disk_size = 0;
-  CompressionType actual_compression_type = blob_compression_type_;
 
   {
     const Status s =
-        WriteBlobToFile(key, blob, &blob_file_number, &blob_offset,
-                        &blob_on_disk_size, &actual_compression_type);
+        WriteBlobToFile(key, blob, &blob_file_number, &blob_offset);
     if (!s.ok()) {
       return s;
     }
@@ -168,8 +158,8 @@ Status BlobFileBuilder::Add(const Slice& key, const Slice& value,
     }
   }
 
-  BlobIndex::EncodeBlob(blob_index, blob_file_number, blob_offset,
-                        blob_on_disk_size, actual_compression_type);
+  BlobIndex::EncodeBlob(blob_index, blob_file_number, blob_offset, blob.size(),
+                        blob_compression_type_);
 
   return Status::OK();
 }
@@ -182,9 +172,7 @@ Status BlobFileBuilder::Finish() {
   return CloseBlobFile();
 }
 
-bool BlobFileBuilder::IsBlobFileOpen() const {
-  return !!writer_ || !!blog_writer_;
-}
+bool BlobFileBuilder::IsBlobFileOpen() const { return !!writer_; }
 
 Status BlobFileBuilder::OpenBlobFileIfNeeded() {
   if (IsBlobFileOpen()) {
@@ -243,73 +231,37 @@ Status BlobFileBuilder::OpenBlobFileIfNeeded() {
       immutable_options_->file_checksum_gen_factory.get(),
       tmp_set.Contains(FileType::kBlobFile), false));
 
-  blob_file_number_ = blob_file_number;
+  constexpr bool do_flush = false;
 
-  if (immutable_options_->use_blog_format_for_blobs) {
-    // Blog format path
-    BlogFileHeader blog_header;
-    blog_header.checksum_type = immutable_options_->blog_checksum;
-    blog_header.compact_record_type = kBlogBlobRecord;
-    blog_header.GenerateFromUniqueId(db_id_, db_session_id_, blob_file_number);
-    blog_header.SetProperty(kBlogPropRole, "blob");
-    if (!immutable_options_->db_host_id.empty()) {
-      blog_header.SetProperty(kBlogPropDbHostId,
-                              immutable_options_->db_host_id);
-    }
-    blog_header.SetUint32Property(kBlogPropColumnFamilyId, column_family_id_);
-    if (!column_family_name_.empty()) {
-      blog_header.SetProperty(kBlogPropColumnFamilyName, column_family_name_);
-    }
-    if (blob_compressor_) {
-      blog_header.SetProperty(kBlogPropCompressionCompatibilityName,
-                              blob_compression_manager_->CompatibilityName());
-    }
+  std::unique_ptr<BlobLogWriter> blob_log_writer(new BlobLogWriter(
+      std::move(file_writer), immutable_options_->clock, statistics,
+      blob_file_number, immutable_options_->use_fsync, do_flush));
 
-    blog_writer_ =
-        std::make_unique<BlogFileWriter>(std::move(file_writer), blog_header);
+  constexpr bool has_ttl = false;
+  constexpr ExpirationRange expiration_range;
 
-    Status s = blog_writer_->WriteHeader(*write_options_);
+  BlobLogHeader header(column_family_id_, blob_compression_type_, has_ttl,
+                       expiration_range);
+
+  {
+    Status s = blob_log_writer->WriteHeader(*write_options_, header);
 
     TEST_SYNC_POINT_CALLBACK(
-        "BlobFileBuilder::OpenBlobFileIfNeeded:WriteBlogHeader", &s);
+        "BlobFileBuilder::OpenBlobFileIfNeeded:WriteHeader", &s);
 
     if (!s.ok()) {
       return s;
     }
-  } else {
-    // Legacy blob log format path
-    constexpr bool do_flush = false;
-
-    std::unique_ptr<BlobLogWriter> blob_log_writer(
-        new BlobLogWriter(std::move(file_writer), immutable_options_->clock,
-                          statistics, immutable_options_->use_fsync, do_flush));
-
-    constexpr bool has_ttl = false;
-    constexpr ExpirationRange expiration_range;
-
-    BlobLogHeader header(column_family_id_, blob_compression_type_, has_ttl,
-                         expiration_range);
-
-    {
-      Status s = blob_log_writer->WriteHeader(*write_options_, header);
-
-      TEST_SYNC_POINT_CALLBACK(
-          "BlobFileBuilder::OpenBlobFileIfNeeded:WriteHeader", &s);
-
-      if (!s.ok()) {
-        return s;
-      }
-    }
-
-    writer_ = std::move(blob_log_writer);
   }
 
+  writer_ = std::move(blob_log_writer);
+
   assert(IsBlobFileOpen());
 
   return Status::OK();
 }
 
-Status BlobFileBuilder::LegacyCompressBlob(
+Status BlobFileBuilder::CompressBlobIfNeeded(
     Slice* blob, GrowableBuffer* compressed_blob) const {
   assert(blob);
   assert(compressed_blob);
@@ -338,48 +290,17 @@ Status BlobFileBuilder::LegacyCompressBlob(
   return Status::OK();
 }
 
-Status BlobFileBuilder::WriteBlobToFile(
-    const Slice& key, const Slice& blob, uint64_t* blob_file_number,
-    uint64_t* blob_offset, uint64_t* blob_on_disk_size,
-    CompressionType* actual_compression_type) {
+Status BlobFileBuilder::WriteBlobToFile(const Slice& key, const Slice& blob,
+                                        uint64_t* blob_file_number,
+                                        uint64_t* blob_offset) {
   assert(IsBlobFileOpen());
   assert(blob_file_number);
   assert(blob_offset);
-  assert(blob_on_disk_size);
-  assert(actual_compression_type);
 
-  Status s;
+  uint64_t key_offset = 0;
 
-  if (blog_writer_) {
-    // Blog format: compress here using CompressBlock (like SST) so we
-    // capture the actual compression type used in the record trailer.
-    CompressionType actual_type = kNoCompression;
-    GrowableBuffer compressed;
-    if (blob_compressor_) {
-      StopWatch stop_watch(immutable_options_->clock, immutable_options_->stats,
-                           BLOB_DB_COMPRESSION_MICROS);
-      size_t max_compressed_size = blob.size();
-      compressed.ResetForSize(max_compressed_size);
-      s = blob_compressor_->CompressBlock(blob, compressed.data(),
-                                          &compressed.MutableSize(),
-                                          &actual_type, &blob_compressor_wa_);
-      if (!s.ok()) {
-        return s;
-      }
-    }
-    Slice write_data =
-        (actual_type != kNoCompression) ? Slice(compressed) : blob;
-    s = blog_writer_->AddBlobRecord(*write_options_, write_data, actual_type,
-                                    blob_offset);
-    *blob_on_disk_size = write_data.size();
-    *actual_compression_type = actual_type;
-  } else {
-    uint64_t key_offset = 0;
-    s = writer_->AddRecord(*write_options_, key, blob, &key_offset,
-                           blob_offset);
-    *blob_on_disk_size = blob.size();  // legacy: already compressed by caller
-    *actual_compression_type = blob_compression_type_;
-  }
+  Status s =
+      writer_->AddRecord(*write_options_, key, blob, &key_offset, blob_offset);
 
   TEST_SYNC_POINT_CALLBACK("BlobFileBuilder::WriteBlobToFile:AddRecord", &s);
 
@@ -387,16 +308,10 @@ Status BlobFileBuilder::WriteBlobToFile(
     return s;
   }
 
-  *blob_file_number = blob_file_number_;
+  *blob_file_number = writer_->get_log_number();
 
   ++blob_count_;
-  if (blog_writer_) {
-    // Blog format: blob_bytes_ tracks uncompressed payload size only
-    // (no legacy record header, no key).
-    blob_bytes_ += blob.size();
-  } else {
-    blob_bytes_ += BlobLogRecord::kHeaderSize + key.size() + blob.size();
-  }
+  blob_bytes_ += BlobLogRecord::kHeaderSize + key.size() + blob.size();
 
   return Status::OK();
 }
@@ -404,60 +319,22 @@ Status BlobFileBuilder::WriteBlobToFile(
 Status BlobFileBuilder::CloseBlobFile() {
   assert(IsBlobFileOpen());
 
+  BlobLogFooter footer;
+  footer.blob_count = blob_count_;
+
   std::string checksum_method;
   std::string checksum_value;
-  Status s;
 
-  if (blog_writer_) {
-    // Blog format: write footer properties and locator records.
-    BlogFileFooterProperties footer_props;
-    footer_props.SetBlobCount(blob_count_);
-    footer_props.SetTotalBlobBytes(blob_bytes_);
+  Status s = writer_->AppendFooter(*write_options_, footer, &checksum_method,
+                                   &checksum_value);
 
-    uint64_t props_offset = blog_writer_->current_offset();
-    s = blog_writer_->AddFooterPropertiesRecord(*write_options_, footer_props);
-
-    if (s.ok()) {
-      BlogFileFooterLocator locator;
-      uint64_t locator_offset = blog_writer_->current_offset();
-      locator.entries.push_back(
-          {kBlogFooterPropertiesRecord,
-           static_cast<uint32_t>((locator_offset - props_offset) / 4)});
-      s = blog_writer_->AddFooterLocatorRecord(*write_options_, locator);
-    }
-
-    if (s.ok()) {
-      s = blog_writer_->Sync(*write_options_);
-    }
-    if (s.ok()) {
-      s = blog_writer_->Close(*write_options_);
-    }
-    if (s.ok()) {
-      std::string method = blog_writer_->file()->GetFileChecksumFuncName();
-      if (method != kUnknownFileChecksumFuncName) {
-        checksum_method = std::move(method);
-      }
-      std::string value = blog_writer_->file()->GetFileChecksum();
-      if (value != kUnknownFileChecksum) {
-        checksum_value = std::move(value);
-      }
-    }
-  } else {
-    BlobLogFooter footer;
-    footer.blob_count = blob_count_;
-
-    s = writer_->LegacyAppendFooterAndClose(*write_options_, footer,
-                                            &checksum_method, &checksum_value);
-  }
-
-  TEST_SYNC_POINT_CALLBACK(
-      "BlobFileBuilder::WriteBlobToFile:LegacyAppendFooterAndClose", &s);
+  TEST_SYNC_POINT_CALLBACK("BlobFileBuilder::WriteBlobToFile:AppendFooter", &s);
 
   if (!s.ok()) {
     return s;
   }
 
-  const uint64_t blob_file_number = blob_file_number_;
+  const uint64_t blob_file_number = writer_->get_log_number();
 
   if (blob_callback_) {
     s = blob_callback_->OnBlobFileCompleted(
@@ -479,7 +356,6 @@ Status BlobFileBuilder::CloseBlobFile() {
                  blob_count_, blob_bytes_);
 
   writer_.reset();
-  blog_writer_.reset();
   blob_count_ = 0;
   blob_bytes_ = 0;
 
@@ -489,8 +365,7 @@ Status BlobFileBuilder::CloseBlobFile() {
 Status BlobFileBuilder::CloseBlobFileIfNeeded() {
   assert(IsBlobFileOpen());
 
-  const WritableFileWriter* const file_writer =
-      blog_writer_ ? blog_writer_->file() : writer_->file();
+  const WritableFileWriter* const file_writer = writer_->file();
   assert(file_writer);
 
   if (file_writer->GetFileSize() < blob_file_size_) {
@@ -509,13 +384,13 @@ void BlobFileBuilder::Abandon(const Status& s) {
     // Blob files. So we can ignore the below error.
     blob_callback_
         ->OnBlobFileCompleted(blob_file_paths_->back(), column_family_name_,
-                              job_id_, blob_file_number_, creation_reason_, s,
-                              "", "", blob_count_, blob_bytes_)
+                              job_id_, writer_->get_log_number(),
+                              creation_reason_, s, "", "", blob_count_,
+                              blob_bytes_)
         .PermitUncheckedError();
   }
 
   writer_.reset();
-  blog_writer_.reset();
   blob_count_ = 0;
   blob_bytes_ = 0;
 }

db/blob/blob_file_builder.h

@@ -34,8 +34,6 @@ class BlobLogWriter;
 class IOTracer;
 class BlobFileCompletionCallback;
 
-class BlogFileWriter;
-
 class BlobFileBuilder {
  public:
   BlobFileBuilder(VersionSet* versions, FileSystem* fs,
@@ -80,11 +78,10 @@ class BlobFileBuilder {
  private:
   bool IsBlobFileOpen() const;
   Status OpenBlobFileIfNeeded();
-  Status LegacyCompressBlob(Slice* blob, GrowableBuffer* compressed_blob) const;
+  Status CompressBlobIfNeeded(Slice* blob,
+                              GrowableBuffer* compressed_blob) const;
   Status WriteBlobToFile(const Slice& key, const Slice& blob,
-                         uint64_t* blob_file_number, uint64_t* blob_offset,
-                         uint64_t* blob_on_disk_size,
-                         CompressionType* actual_compression_type);
+                         uint64_t* blob_file_number, uint64_t* blob_offset);
   Status CloseBlobFile();
   Status CloseBlobFileIfNeeded();
 
@@ -97,7 +94,6 @@ class BlobFileBuilder {
   uint64_t min_blob_size_;
   uint64_t blob_file_size_;
   CompressionType blob_compression_type_;
-  std::shared_ptr<CompressionManager> blob_compression_manager_;
   std::unique_ptr<Compressor> blob_compressor_;
   mutable Compressor::ManagedWorkingArea blob_compressor_wa_;
   PrepopulateBlobCache prepopulate_blob_cache_;
@@ -115,8 +111,6 @@ class BlobFileBuilder {
   std::vector<std::string>* blob_file_paths_;
   std::vector<BlobFileAddition>* blob_file_additions_;
   std::unique_ptr<BlobLogWriter> writer_;
-  std::unique_ptr<BlogFileWriter> blog_writer_;
-  uint64_t blob_file_number_ = 0;
   uint64_t blob_count_;
   uint64_t blob_bytes_;
 };

db/blob/blob_file_builder_test.cc

@@ -598,7 +598,7 @@ INSTANTIATE_TEST_CASE_P(
         "BlobFileBuilder::OpenBlobFileIfNeeded:NewWritableFile",
         "BlobFileBuilder::OpenBlobFileIfNeeded:WriteHeader",
         "BlobFileBuilder::WriteBlobToFile:AddRecord",
-        "BlobFileBuilder::WriteBlobToFile:LegacyAppendFooterAndClose"}));
+        "BlobFileBuilder::WriteBlobToFile:AppendFooter"}));
 
 TEST_P(BlobFileBuilderIOErrorTest, IOError) {
   // Simulate an I/O error during the specified step of Add()

db/blob/blob_file_cache_test.cc

@@ -48,7 +48,8 @@ void WriteBlobFile(uint32_t column_family_id,
   constexpr bool do_flush = false;
 
   BlobLogWriter blob_log_writer(std::move(file_writer), immutable_options.clock,
-                                statistics, use_fsync, do_flush);
+                                statistics, blob_file_number, use_fsync,
+                                do_flush);
 
   constexpr bool has_ttl = false;
   constexpr ExpirationRange expiration_range;
@@ -76,8 +77,8 @@ void WriteBlobFile(uint32_t column_family_id,
   std::string checksum_method;
   std::string checksum_value;
 
-  ASSERT_OK(blob_log_writer.LegacyAppendFooterAndClose(
-      WriteOptions(), footer, &checksum_method, &checksum_value));
+  ASSERT_OK(blob_log_writer.AppendFooter(WriteOptions(), footer,
+                                         &checksum_method, &checksum_value));
 }
 
 }  // anonymous namespace
@@ -221,7 +222,8 @@ TEST_F(BlobFileCacheTest, RefreshBlobFileReaderPrefersLargestObservedFileSize) {
   constexpr bool use_fsync = false;
   constexpr bool do_flush = false;
   BlobLogWriter blob_log_writer(std::move(file_writer), immutable_options.clock,
-                                statistics, use_fsync, do_flush);
+                                statistics, blob_file_number, use_fsync,
+                                do_flush);
 
   constexpr bool has_ttl = false;
   constexpr ExpirationRange expiration_range;

db/blob/blob_file_partition_manager.cc

@@ -214,10 +214,10 @@ Status BlobFilePartitionManager::OpenNewBlobFile(Partition* partition,
 
   // This only drains WritableFileWriter's buffered bytes so readers can see
   // each appended record promptly. Durability still comes from SyncAllOpenFiles
-  // or LegacyAppendFooterAndClose(), both of which call Sync().
+  // or AppendFooter(), both of which call Sync().
   constexpr bool kDoFlushEachRecord = true;
   auto blob_log_writer = std::make_unique<BlobLogWriter>(
-      std::move(file_writer), clock_, statistics_, use_fsync_,
+      std::move(file_writer), clock_, statistics_, blob_file_number, use_fsync_,
       kDoFlushEachRecord);
 
   constexpr bool has_ttl = false;
@@ -300,8 +300,8 @@ Status BlobFilePartitionManager::FinalizeBlobFile(
 
   std::string checksum_method;
   std::string checksum_value;
-  Status s = writer->LegacyAppendFooterAndClose(
-      write_options, footer, &checksum_method, &checksum_value);
+  Status s = writer->AppendFooter(write_options, footer, &checksum_method,
+                                  &checksum_value);
   if (!s.ok()) {
     return s;
   }
@@ -786,13 +786,19 @@ Status BlobFilePartitionManager::ResolveBlobDirectWriteIndex(
     }
   }
 
-  Status s = version != nullptr ? Status::Corruption("Invalid blob file number")
-                                : Status::NotFound();
-
   if (blob_file_cache == nullptr) {
-    return s;
+    return version != nullptr ? Status::Corruption("Invalid blob file number")
+                              : Status::NotFound();
   }
 
+  if (read_options.read_tier == kBlockCacheTier) {
+    // The direct-write fallback below may need to open the blob file reader,
+    // which `kBlockCacheTier` forbids. Keep the normal Version-backed path
+    // above eligible for cache-only hits.
+    return Status::Incomplete("Cannot read blob(s): no disk I/O allowed");
+  }
+
+  Status s;
   CacheHandleGuard<BlobFileReader> reader;
   s = blob_file_cache->GetBlobFileReader(read_options, blob_idx.file_number(),
                                          &reader,

db/blob/blob_file_reader.cc

@@ -5,14 +5,11 @@
 
 #include "db/blob/blob_file_reader.h"
 
-#include <algorithm>
 #include <cassert>
-#include <cstring>
 #include <string>
 
 #include "db/blob/blob_contents.h"
 #include "db/blob/blob_log_format.h"
-#include "db/blog/blog_format.h"
 #include "file/file_prefetch_buffer.h"
 #include "file/filename.h"
 #include "monitoring/statistics_impl.h"
@@ -23,6 +20,7 @@
 #include "table/format.h"
 #include "table/multiget_context.h"
 #include "test_util/sync_point.h"
+#include "util/aligned_buffer.h"
 #include "util/compression.h"
 #include "util/stop_watch.h"
 
@@ -54,30 +52,37 @@ Status BlobFileReader::Create(
 
   Statistics* const statistics = immutable_options.stats;
 
-  // Construct with defaults; ReadHeader will detect blog vs legacy format
-  // and populate the appropriate fields.
-  blob_file_reader->reset(
-      new BlobFileReader(std::move(file_reader), file_size, kNoCompression,
-                         /*decompressor=*/nullptr, immutable_options.clock,
-                         statistics, /*has_footer=*/false));
+  CompressionType compression_type = kNoCompression;
 
   {
     const Status s =
-        (*blob_file_reader)->ReadHeader(read_options, column_family_id);
+        ReadHeader(file_reader.get(), read_options, column_family_id,
+                   statistics, &compression_type);
     if (!s.ok()) {
-      blob_file_reader->reset();
       return s;
     }
   }
 
   if (!skip_footer_validation) {
-    const Status s = (*blob_file_reader)->ReadFooter(read_options);
+    const Status s =
+        ReadFooter(file_reader.get(), read_options, file_size, statistics);
     if (!s.ok()) {
-      blob_file_reader->reset();
       return s;
     }
   }
 
+  std::shared_ptr<Decompressor> decompressor;
+  if (compression_type != kNoCompression) {
+    // The blob format has always used compression format 2
+    decompressor = GetBuiltinV2CompressionManager()->GetDecompressorOptimizeFor(
+        compression_type);
+  }
+
+  blob_file_reader->reset(
+      new BlobFileReader(std::move(file_reader), file_size, compression_type,
+                         std::move(decompressor), immutable_options.clock,
+                         statistics, !skip_footer_validation));
+
   return Status::OK();
 }
 
@@ -151,28 +156,27 @@ Status BlobFileReader::OpenFile(
   return Status::OK();
 }
 
-Status BlobFileReader::ReadHeader(const ReadOptions& read_options,
-                                  uint32_t column_family_id) {
-  assert(file_reader_);
-
-  // Read ~4 KiB from the start to cover both legacy (30B) and blog (40B
-  // fixed + property section) headers in a single I/O. This is enough for
-  // typical blog headers; only if the property section exceeds ~4 KiB do
-  // we need a second read.
-  constexpr size_t kInitialReadSize = 4096;
-  const size_t read_size =
-      std::min(static_cast<uint64_t>(kInitialReadSize), file_size_);
+Status BlobFileReader::ReadHeader(const RandomAccessFileReader* file_reader,
+                                  const ReadOptions& read_options,
+                                  uint32_t column_family_id,
+                                  Statistics* statistics,
+                                  CompressionType* compression_type) {
+  assert(file_reader);
+  assert(compression_type);
 
   Slice header_slice;
   Buffer buf;
-  AlignedBuf aligned_buf;
+  AlignedBuffer direct_io_buffer;
 
   {
     TEST_SYNC_POINT("BlobFileReader::ReadHeader:ReadFromFile");
 
+    constexpr uint64_t read_offset = 0;
+    constexpr size_t read_size = BlobLogHeader::kSize;
+
     const Status s =
-        ReadFromFile(file_reader_.get(), read_options, 0, read_size,
-                     statistics_, &header_slice, &buf, &aligned_buf);
+        ReadFromFile(file_reader, read_options, read_offset, read_size,
+                     statistics, &header_slice, &buf, &direct_io_buffer);
     if (!s.ok()) {
       return s;
     }
@@ -181,58 +185,10 @@ Status BlobFileReader::ReadHeader(const ReadOptions& read_options,
                              &header_slice);
   }
 
-  // Detect blog format by checking the 12-byte magic.
-  if (BlogFileHeader::IsBlogFormat(header_slice.data(), header_slice.size())) {
-    // Try to decode the blog header from the buffer we already read.
-    // DecodeFromBuffer handles the case where the property section
-    // extends beyond the buffer.
-    BlogFileHeader blog_header;
-    Slice buf_slice = header_slice;
-    size_t additional_needed = 0;
-    Status s = blog_header.DecodeFromBuffer(&buf_slice, &additional_needed);
-
-    if (s.IsIncomplete()) {
-      // Need more data. Read the rest and retry.
-      Slice extra_slice;
-      Buffer extra_buf;
-      AlignedBuf extra_aligned;
-      s = ReadFromFile(file_reader_.get(), read_options, header_slice.size(),
-                       additional_needed, statistics_, &extra_slice, &extra_buf,
-                       &extra_aligned);
-      if (!s.ok()) {
-        return s;
-      }
-      std::string extended;
-      extended.append(header_slice.data(), header_slice.size());
-      extended.append(extra_slice.data(), extra_slice.size());
-      Slice extended_slice(extended);
-      s = blog_header.DecodeFromBuffer(&extended_slice, nullptr);
-    }
-
-    if (!s.ok()) {
-      return s;
-    }
-
-    is_blog_format_ = true;
-    blog_checksum_type_ = blog_header.checksum_type;
-    blog_incarnation_id_ = blog_header.incarnation_id();
-    memcpy(blog_escape_sequence_, blog_header.escape_sequence,
-           kBlogEscapeSequenceSize);
-    compression_type_ = kNoCompression;
-    decompressor_ = GetBuiltinV2CompressionManager()->GetDecompressor();
-    return Status::OK();
-  }
-
-  // Legacy blob format. The initial read already covers the 30-byte header.
-  if (header_slice.size() < BlobLogHeader::kSize) {
-    return Status::Corruption("Blob file header too small");
-  }
-
-  Slice legacy_slice(header_slice.data(), BlobLogHeader::kSize);
   BlobLogHeader header;
 
   {
-    const Status s = header.DecodeFrom(legacy_slice);
+    const Status s = header.DecodeFrom(header_slice);
     if (!s.ok()) {
       return s;
     }
@@ -248,76 +204,30 @@ Status BlobFileReader::ReadHeader(const ReadOptions& read_options,
     return Status::Corruption("Column family ID mismatch");
   }
 
-  compression_type_ = header.compression;
-  if (compression_type_ != kNoCompression) {
-    decompressor_ =
-        GetBuiltinV2CompressionManager()->GetDecompressorOptimizeFor(
-            compression_type_);
-  }
+  *compression_type = header.compression;
 
   return Status::OK();
 }
 
-Status BlobFileReader::ReadFooter(const ReadOptions& read_options) {
-  assert(file_reader_);
-
-  if (is_blog_format_) {
-    // Blog format: verify the file was cleanly sealed by scanning backward
-    // from EOF for the last record's escape sequence. Read the trailing
-    // ~4 KiB in a single I/O and scan within the buffer.
-    if (file_size_ < kBlogFileFixedHeaderSize + kBlogEscapeSequenceSize) {
-      return Status::Corruption("Blog blob file too small for footer");
-    }
-
-    constexpr size_t kTrailingReadSize = 4096;
-    const uint64_t min_offset = kBlogFileFixedHeaderSize;
-    // Round trailing_offset up to 4-byte alignment so buffer positions
-    // and file positions share the same alignment grid.
-    const uint64_t trailing_offset =
-        (file_size_ -
-         std::min(static_cast<uint64_t>(kTrailingReadSize),
-                  file_size_ - min_offset) +
-         3) &
-        ~uint64_t{3};
-    const size_t trailing_size =
-        static_cast<size_t>(file_size_ - trailing_offset);
-
-    Slice trailing_slice;
-    Buffer trailing_buf;
-    AlignedBuf trailing_aligned;
-    Status s = ReadFromFile(file_reader_.get(), read_options, trailing_offset,
-                            trailing_size, statistics_, &trailing_slice,
-                            &trailing_buf, &trailing_aligned);
-    if (!s.ok()) {
-      return s;
-    }
-
-    if (VerifyBlogFooterLocator(trailing_slice.data(), trailing_size,
-                                trailing_offset, blog_escape_sequence_,
-                                blog_checksum_type_, blog_incarnation_id_)) {
-      has_footer_ = true;
-      return Status::OK();
-    }
-
-    return Status::Corruption("Blog blob file: no footer locator record found");
-  }
-
-  // Legacy blob format.
-  assert(file_size_ >= BlobLogHeader::kSize + BlobLogFooter::kSize);
+Status BlobFileReader::ReadFooter(const RandomAccessFileReader* file_reader,
+                                  const ReadOptions& read_options,
+                                  uint64_t file_size, Statistics* statistics) {
+  assert(file_size >= BlobLogHeader::kSize + BlobLogFooter::kSize);
+  assert(file_reader);
 
   Slice footer_slice;
   Buffer buf;
-  AlignedBuf aligned_buf;
+  AlignedBuffer direct_io_buffer;
 
   {
     TEST_SYNC_POINT("BlobFileReader::ReadFooter:ReadFromFile");
 
-    const uint64_t read_offset = file_size_ - BlobLogFooter::kSize;
+    const uint64_t read_offset = file_size - BlobLogFooter::kSize;
     constexpr size_t read_size = BlobLogFooter::kSize;
 
     const Status s =
-        ReadFromFile(file_reader_.get(), read_options, read_offset, read_size,
-                     statistics_, &footer_slice, &buf, &aligned_buf);
+        ReadFromFile(file_reader, read_options, read_offset, read_size,
+                     statistics, &footer_slice, &buf, &direct_io_buffer);
     if (!s.ok()) {
       return s;
     }
@@ -341,7 +251,6 @@ Status BlobFileReader::ReadFooter(const ReadOptions& read_options) {
     return Status::Corruption("Unexpected TTL blob file");
   }
 
-  has_footer_ = true;
   return Status::OK();
 }
 
@@ -349,10 +258,11 @@ Status BlobFileReader::ReadFromFile(const RandomAccessFileReader* file_reader,
                                     const ReadOptions& read_options,
                                     uint64_t read_offset, size_t read_size,
                                     Statistics* statistics, Slice* slice,
-                                    Buffer* buf, AlignedBuf* aligned_buf) {
+                                    Buffer* buf,
+                                    AlignedBuffer* direct_io_buffer) {
   assert(slice);
   assert(buf);
-  assert(aligned_buf);
+  assert(direct_io_buffer);
 
   assert(file_reader);
 
@@ -369,15 +279,15 @@ Status BlobFileReader::ReadFromFile(const RandomAccessFileReader* file_reader,
 
   if (file_reader->use_direct_io()) {
     constexpr char* scratch = nullptr;
+    AlignedBufferAllocationContext direct_io_context{direct_io_buffer};
 
     s = file_reader->Read(io_options, read_offset, read_size, slice, scratch,
-                          aligned_buf, &dbg);
+                          &direct_io_context, &dbg);
   } else {
     buf->reset(new char[read_size]);
-    constexpr AlignedBuf* aligned_scratch = nullptr;
 
     s = file_reader->Read(io_options, read_offset, read_size, slice, buf->get(),
-                          aligned_scratch, &dbg);
+                          nullptr, &dbg);
   }
 
   if (!s.ok()) {
@@ -415,43 +325,33 @@ Status BlobFileReader::GetBlob(
     std::unique_ptr<BlobContents>* result, uint64_t* bytes_read) const {
   assert(result);
 
-  // Compute the read region. Both formats store the blob value at `offset`
-  // with size `value_size`. The difference:
-  // - Legacy: if verifying checksums, also read the record header (before
-  //   the value) which contains key + CRC.
-  // - Blog: if verifying checksums, also read the 5-byte trailer (after
-  //   the value) which contains compression_type + checksum.
-  uint64_t adjustment_before = 0;  // bytes before the value to include
-  uint64_t adjustment_after = 0;   // bytes after the value to include
+  const uint64_t key_size = user_key.size();
 
-  if (is_blog_format_) {
-    if (read_options.verify_checksums) {
-      adjustment_after = kBlogBlockTrailerSize;
-    }
-  } else {
-    const uint64_t key_size = user_key.size();
-    if (!IsValidBlobOffset(offset, key_size, value_size, file_size_,
-                           has_footer_)) {
-      return Status::Corruption("Invalid blob offset");
-    }
-    if (compression_type != compression_type_) {
-      return Status::Corruption("Compression type mismatch when reading blob");
-    }
-    if (read_options.verify_checksums) {
-      adjustment_before =
-          BlobLogRecord::CalculateAdjustmentForRecordHeader(key_size);
-    }
+  if (!IsValidBlobOffset(offset, key_size, value_size, file_size_,
+                         has_footer_)) {
+    return Status::Corruption("Invalid blob offset");
   }
 
-  assert(offset >= adjustment_before);
-  const uint64_t record_offset = offset - adjustment_before;
-  const uint64_t record_size =
-      adjustment_before + value_size + adjustment_after;
+  if (compression_type != compression_type_) {
+    return Status::Corruption("Compression type mismatch when reading blob");
+  }
+
+  // Note: if verify_checksum is set, we read the entire blob record to be able
+  // to perform the verification; otherwise, we just read the blob itself. Since
+  // the offset in BlobIndex actually points to the blob value, we need to make
+  // an adjustment in the former case.
+  const uint64_t adjustment =
+      read_options.verify_checksums
+          ? BlobLogRecord::CalculateAdjustmentForRecordHeader(key_size)
+          : 0;
+  assert(offset >= adjustment);
+
+  const uint64_t record_offset = offset - adjustment;
+  const uint64_t record_size = value_size + adjustment;
 
-  // Read the data: try prefetch buffer first, then fall back to file read.
   Slice record_slice;
   Buffer buf;
-  AlignedBuf aligned_buf;
+  AlignedBuffer direct_io_buffer;
 
   bool prefetched = false;
 
@@ -481,7 +381,7 @@ Status BlobFileReader::GetBlob(
     const Status s =
         ReadFromFile(file_reader_.get(), read_options, record_offset,
                      static_cast<size_t>(record_size), statistics_,
-                     &record_slice, &buf, &aligned_buf);
+                     &record_slice, &buf, &direct_io_buffer);
     if (!s.ok()) {
       return s;
     }
@@ -490,30 +390,17 @@ Status BlobFileReader::GetBlob(
   TEST_SYNC_POINT_CALLBACK("BlobFileReader::GetBlob:TamperWithResult",
                            &record_slice);
 
-  // Verify checksum (format-specific).
-  CompressionType actual_comp_type = compression_type;
   if (read_options.verify_checksums) {
-    if (is_blog_format_) {
-      const Status s = VerifyBlogRecordTrailer(
-          blog_checksum_type_, record_slice.data(),
-          static_cast<size_t>(value_size), blog_incarnation_id_, offset,
-          &actual_comp_type);
-      if (!s.ok()) {
-        return s;
-      }
-    } else {
-      const Status s = VerifyBlob(record_slice, user_key, value_size);
-      if (!s.ok()) {
-        return s;
-      }
+    const Status s = VerifyBlob(record_slice, user_key, value_size);
+    if (!s.ok()) {
+      return s;
     }
   }
 
-  // Extract the value slice (format-specific offset within the read region).
-  const Slice value_slice(record_slice.data() + adjustment_before, value_size);
+  const Slice value_slice(record_slice.data() + adjustment, value_size);
 
   {
-    const Status s = UncompressBlobIfNeeded(value_slice, actual_comp_type,
+    const Status s = UncompressBlobIfNeeded(value_slice, compression_type,
                                             decompressor_.get(), allocator,
                                             clock_, statistics_, result);
     if (!s.ok()) {
@@ -592,7 +479,7 @@ void BlobFileReader::MultiGetBlob(
   }
 
   Buffer buf;
-  AlignedBuf aligned_buf;
+  AlignedBuffer direct_io_buffer;
 
   Status s;
   bool direct_io = file_reader_->use_direct_io();
@@ -615,8 +502,9 @@ void BlobFileReader::MultiGetBlob(
   IODebugContext dbg;
   s = file_reader_->PrepareIOOptions(read_options, opts, &dbg);
   if (s.ok()) {
+    AlignedBufferAllocationContext direct_io_context{&direct_io_buffer};
     s = file_reader_->MultiRead(opts, read_reqs.data(), read_reqs.size(),
-                                direct_io ? &aligned_buf : nullptr, &dbg);
+                                &direct_io_context, &dbg);
   }
   if (!s.ok()) {
     for (auto& req : read_reqs) {

db/blob/blob_file_reader.h

@@ -93,15 +93,14 @@ class BlobFileReader {
                          std::unique_ptr<RandomAccessFileReader>* file_reader,
                          bool skip_footer_size_check);
 
-  // Read and validate the file header. Detects blog format vs legacy format,
-  // setting is_blog_format_ and related fields accordingly. For legacy format,
-  // validates column_family_id and sets compression_type_.
-  Status ReadHeader(const ReadOptions& read_options, uint32_t column_family_id);
+  static Status ReadHeader(const RandomAccessFileReader* file_reader,
+                           const ReadOptions& read_options,
+                           uint32_t column_family_id, Statistics* statistics,
+                           CompressionType* compression_type);
 
-  // Validate the file footer. For legacy format, reads and validates the
-  // BlobLogFooter. For blog format, scans backward for the footer locator
-  // record to verify the file was cleanly sealed.
-  Status ReadFooter(const ReadOptions& read_options);
+  static Status ReadFooter(const RandomAccessFileReader* file_reader,
+                           const ReadOptions& read_options, uint64_t file_size,
+                           Statistics* statistics);
 
   using Buffer = std::unique_ptr<char[]>;
 
@@ -109,7 +108,7 @@ class BlobFileReader {
                              const ReadOptions& read_options,
                              uint64_t read_offset, size_t read_size,
                              Statistics* statistics, Slice* slice, Buffer* buf,
-                             AlignedBuf* aligned_buf);
+                             AlignedBuffer* direct_io_buffer);
 
   static Status VerifyBlob(const Slice& record_slice, const Slice& user_key,
                            uint64_t value_size);
@@ -130,13 +129,6 @@ class BlobFileReader {
   Statistics* statistics_;
   // False when the reader was opened before the blob file footer was written.
   bool has_footer_;
-  // True when the file uses blog format instead of legacy blob log format.
-  bool is_blog_format_ = false;
-  // For blog format: fields from the file header used for record checksum
-  // verification and footer validation.
-  uint32_t blog_incarnation_id_ = 0;
-  ChecksumType blog_checksum_type_ = kNoChecksum;
-  char blog_escape_sequence_[10] = {};  // kBlogEscapeSequenceSize
 };
 
 }  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_reader_test.cc

@@ -57,7 +57,8 @@ void WriteBlobFile(const ImmutableOptions& immutable_options,
   constexpr bool do_flush = false;
 
   BlobLogWriter blob_log_writer(std::move(file_writer), immutable_options.clock,
-                                statistics, use_fsync, do_flush);
+                                statistics, blob_file_number, use_fsync,
+                                do_flush);
 
   BlobLogHeader header(column_family_id, compression, has_ttl,
                        expiration_range_header);
@@ -97,8 +98,8 @@ void WriteBlobFile(const ImmutableOptions& immutable_options,
 
   std::string checksum_method;
   std::string checksum_value;
-  ASSERT_OK(blob_log_writer.LegacyAppendFooterAndClose(
-      WriteOptions(), footer, &checksum_method, &checksum_value));
+  ASSERT_OK(blob_log_writer.AppendFooter(WriteOptions(), footer,
+                                         &checksum_method, &checksum_value));
 }
 
 // Creates a test blob file with a single blob in it. Note: this method
@@ -464,7 +465,7 @@ TEST_F(BlobFileReaderTest, Malformed) {
 
     BlobLogWriter blob_log_writer(std::move(file_writer),
                                   immutable_options.clock, statistics,
-                                  use_fsync, do_flush);
+                                  blob_file_number, use_fsync, do_flush);
 
     BlobLogHeader header(column_family_id, kNoCompression, has_ttl,
                          expiration_range);

db/blob/blob_log_writer.cc

@@ -20,10 +20,12 @@ namespace ROCKSDB_NAMESPACE {
 
 BlobLogWriter::BlobLogWriter(std::unique_ptr<WritableFileWriter>&& dest,
                              SystemClock* clock, Statistics* statistics,
-                             bool use_fs, bool do_flush, uint64_t boffset)
+                             uint64_t log_number, bool use_fs, bool do_flush,
+                             uint64_t boffset)
     : dest_(std::move(dest)),
       clock_(clock),
       statistics_(statistics),
+      log_number_(log_number),
       block_offset_(boffset),
       use_fsync_(use_fs),
       do_flush_(do_flush),
@@ -72,9 +74,10 @@ Status BlobLogWriter::WriteHeader(const WriteOptions& write_options,
   return s;
 }
 
-Status BlobLogWriter::LegacyAppendFooterAndClose(
-    const WriteOptions& write_options, BlobLogFooter& footer,
-    std::string* checksum_method, std::string* checksum_value) {
+Status BlobLogWriter::AppendFooter(const WriteOptions& write_options,
+                                   BlobLogFooter& footer,
+                                   std::string* checksum_method,
+                                   std::string* checksum_value) {
   assert(block_offset_ != 0);
   assert(last_elem_type_ == kEtFileHdr || last_elem_type_ == kEtRecord);
 

db/blob/blob_log_writer.h

@@ -32,8 +32,8 @@ class BlobLogWriter {
   // "*dest" must be initially empty.
   // "*dest" must remain live while this BlobLogWriter is in use.
   BlobLogWriter(std::unique_ptr<WritableFileWriter>&& dest, SystemClock* clock,
-                Statistics* statistics, bool use_fsync, bool do_flush,
-                uint64_t boffset = 0);
+                Statistics* statistics, uint64_t log_number, bool use_fsync,
+                bool do_flush, uint64_t boffset = 0);
   // No copying allowed
   BlobLogWriter(const BlobLogWriter&) = delete;
   BlobLogWriter& operator=(const BlobLogWriter&) = delete;
@@ -56,10 +56,9 @@ class BlobLogWriter {
                             const Slice& val, uint64_t* key_offset,
                             uint64_t* blob_offset);
 
-  Status LegacyAppendFooterAndClose(const WriteOptions& write_options,
-                                    BlobLogFooter& footer,
-                                    std::string* checksum_method,
-                                    std::string* checksum_value);
+  Status AppendFooter(const WriteOptions& write_options, BlobLogFooter& footer,
+                      std::string* checksum_method,
+                      std::string* checksum_value);
 
   Status WriteHeader(const WriteOptions& write_options, BlobLogHeader& header);
 
@@ -67,12 +66,15 @@ class BlobLogWriter {
 
   const WritableFileWriter* file() const { return dest_.get(); }
 
+  uint64_t get_log_number() const { return log_number_; }
+
   Status Sync(const WriteOptions& write_options);
 
  private:
   std::unique_ptr<WritableFileWriter> dest_;
   SystemClock* clock_;
   Statistics* statistics_;
+  uint64_t log_number_;
   uint64_t block_offset_;  // Current offset in block
   bool use_fsync_;
   bool do_flush_;

db/blob/blob_source.cc

@@ -74,6 +74,7 @@ Status BlobSource::GetBlobFromCache(
     assert(cached_blob->GetValue());
 
     PERF_COUNTER_ADD(blob_cache_hit_count, 1);
+    PERF_COUNTER_ADD(blob_cache_read_byte, cached_blob->GetValue()->size());
     RecordTick(statistics_, BLOB_DB_CACHE_HIT);
     RecordTick(statistics_, BLOB_DB_CACHE_BYTES_READ,
                cached_blob->GetValue()->size());

db/blob/blob_source_test.cc

@@ -59,7 +59,8 @@ void WriteBlobFile(const ImmutableOptions& immutable_options,
   constexpr bool do_flush = false;
 
   BlobLogWriter blob_log_writer(std::move(file_writer), immutable_options.clock,
-                                statistics, use_fsync, do_flush);
+                                statistics, blob_file_number, use_fsync,
+                                do_flush);
 
   BlobLogHeader header(column_family_id, compression, has_ttl,
                        expiration_range_header);
@@ -99,8 +100,8 @@ void WriteBlobFile(const ImmutableOptions& immutable_options,
 
   std::string checksum_method;
   std::string checksum_value;
-  ASSERT_OK(blob_log_writer.LegacyAppendFooterAndClose(
-      WriteOptions(), footer, &checksum_method, &checksum_value));
+  ASSERT_OK(blob_log_writer.AppendFooter(WriteOptions(), footer,
+                                         &checksum_method, &checksum_value));
 }
 
 }  // anonymous namespace
@@ -228,6 +229,7 @@ TEST_F(BlobSourceTest, GetBlobsFromCache) {
     // Retrieved the blob cache num_blobs * 3 times via TEST_BlobInCache,
     // GetBlob, and TEST_BlobInCache.
     ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, 0);
+    ASSERT_EQ((int)get_perf_context()->blob_cache_read_byte, 0);
     ASSERT_EQ((int)get_perf_context()->blob_read_count, num_blobs);
     ASSERT_EQ((int)get_perf_context()->blob_read_byte, total_bytes);
     ASSERT_GE((int)get_perf_context()->blob_checksum_time, 0);
@@ -261,6 +263,8 @@ TEST_F(BlobSourceTest, GetBlobsFromCache) {
       blob_bytes += blob_sizes[i];
       total_bytes += bytes_read;
       ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, i);
+      ASSERT_EQ((int)get_perf_context()->blob_cache_read_byte,
+                blob_bytes - blob_sizes[i]);
       ASSERT_EQ((int)get_perf_context()->blob_read_count, i + 1);
       ASSERT_EQ((int)get_perf_context()->blob_read_byte, total_bytes);
 
@@ -268,11 +272,13 @@ TEST_F(BlobSourceTest, GetBlobsFromCache) {
                                                blob_offsets[i]));
 
       ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, i + 1);
+      ASSERT_EQ((int)get_perf_context()->blob_cache_read_byte, blob_bytes);
       ASSERT_EQ((int)get_perf_context()->blob_read_count, i + 1);
       ASSERT_EQ((int)get_perf_context()->blob_read_byte, total_bytes);
     }
 
     ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, num_blobs);
+    ASSERT_EQ((int)get_perf_context()->blob_cache_read_byte, blob_bytes);
     ASSERT_EQ((int)get_perf_context()->blob_read_count, num_blobs);
     ASSERT_EQ((int)get_perf_context()->blob_read_byte, total_bytes);
 
@@ -311,6 +317,7 @@ TEST_F(BlobSourceTest, GetBlobsFromCache) {
     // Retrieved the blob cache num_blobs * 3 times via TEST_BlobInCache,
     // GetBlob, and TEST_BlobInCache.
     ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, num_blobs * 3);
+    ASSERT_EQ((int)get_perf_context()->blob_cache_read_byte, blob_bytes * 3);
     ASSERT_EQ((int)get_perf_context()->blob_read_count, 0);  // without i/o
     ASSERT_EQ((int)get_perf_context()->blob_read_byte, 0);   // without i/o
 
@@ -689,6 +696,8 @@ TEST_F(BlobSourceTest, MultiGetBlobsFromMultiFiles) {
     // TEST_BlobInCache.
     ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count,
               num_blobs * blob_files);
+    ASSERT_EQ((int)get_perf_context()->blob_cache_read_byte,
+              blob_value_bytes * blob_files);
     ASSERT_EQ((int)get_perf_context()->blob_read_count,
               num_blobs * blob_files);  // blocking i/o
     ASSERT_EQ((int)get_perf_context()->blob_read_byte,
@@ -749,6 +758,8 @@ TEST_F(BlobSourceTest, MultiGetBlobsFromMultiFiles) {
     // via MultiGetBlob and TEST_BlobInCache.
     ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count,
               num_blobs * blob_files * 2);
+    ASSERT_EQ((int)get_perf_context()->blob_cache_read_byte,
+              blob_value_bytes * blob_files * 2);
     ASSERT_EQ((int)get_perf_context()->blob_read_count,
               0);  // blocking i/o
     ASSERT_EQ((int)get_perf_context()->blob_read_byte,

db/blob/blob_write_batch_transformer.cc

@@ -278,7 +278,7 @@ Status BlobWriteBatchTransformer::MergeCF(uint32_t column_family_id,
 Status BlobWriteBatchTransformer::PutBlobIndexCF(uint32_t column_family_id,
                                                  const Slice& key,
                                                  const Slice& value) {
-  // Already a blob index — pass through unchanged.
+  // Already a blob index -- pass through unchanged.
   return WriteBatchInternal::PutBlobIndex(output_batch_, column_family_id, key,
                                           value);
 }

db/blob/blob_write_batch_transformer.h

@@ -35,7 +35,7 @@ struct BlobDirectWriteSettings {
   CompressionType compression_type = kNoCompression;
   // Compression options for newly written blob records.
   CompressionOptions compression_opts;
-  // Raw pointer — the Cache is owned by ColumnFamilyOptions and outlives all
+  // Raw pointer -- the Cache is owned by ColumnFamilyOptions and outlives all
   // settings snapshots. Using raw avoids 2 atomic ref-count ops per Put().
   Cache* blob_cache = nullptr;
   // Blob-cache prepopulation policy for direct-write records.

db/blob/db_blob_basic_test.cc

@@ -1620,6 +1620,47 @@ TEST_P(DBBlobBasicIOErrorTest, GetBlob_IOError) {
   SyncPoint::GetInstance()->ClearAllCallBacks();
 }
 
+TEST_P(DBBlobBasicIOErrorTest, GetEntityMergeWithBlobBaseIOError) {
+  // Goal: verify GetEntity preserves injected blob-read IOErrors when merge
+  // reads a blob-backed base value, instead of laundering them into Corruption.
+  // The test writes a blob-backed base value plus a merge operand, then injects
+  // an IOError at blob read time and checks both GetEntity and Get see it.
+  Options options;
+  options.env = fault_injection_env_.get();
+  options.enable_blob_files = true;
+  options.min_blob_size = 0;
+  options.merge_operator = MergeOperators::CreateStringAppendOperator();
+
+  Reopen(options);
+
+  constexpr char key[] = "key";
+  constexpr char base_value[] = "base_value";
+
+  ASSERT_OK(Put(key, base_value));
+  ASSERT_OK(Flush());
+
+  ASSERT_OK(Merge(key, "merge_operand"));
+  ASSERT_OK(Flush());
+
+  SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* /* arg */) {
+    fault_injection_env_->SetFilesystemActive(false,
+                                              Status::IOError(sync_point_));
+  });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  PinnableWideColumns entity_result;
+  Status s = db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), key,
+                            &entity_result);
+  ASSERT_TRUE(s.IsIOError()) << "Expected IOError but got: " << s.ToString();
+
+  PinnableSlice get_result;
+  s = db_->Get(ReadOptions(), db_->DefaultColumnFamily(), key, &get_result);
+  ASSERT_TRUE(s.IsIOError()) << "Expected IOError but got: " << s.ToString();
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+}
+
 TEST_P(DBBlobBasicIOErrorMultiGetTest, MultiGetBlobs_IOError) {
   Options options = GetDefaultOptions();
   options.env = fault_injection_env_.get();

db/blob/db_blob_index_test.cc

@@ -128,6 +128,16 @@ class DBBlobIndexTest : public DBTestBase {
         columns, s, is_blob_index, value_found);
   }
 
+  bool MaybeResolveMemtableBlobValueForTest(const Slice& key,
+                                            const BlobFetcher* blob_fetcher,
+                                            PinnableSlice* value,
+                                            PinnableWideColumns* columns,
+                                            Status* s, bool* is_blob_index,
+                                            bool* value_found = nullptr) {
+    return DBImpl::MaybeResolveMemtableBlobValue(
+        key, blob_fetcher, value, columns, s, is_blob_index, value_found);
+  }
+
   Options GetTestOptions() {
     Options options;
     options.env = CurrentOptions().env;
@@ -271,6 +281,55 @@ TEST_F(DBBlobIndexTest,
   ASSERT_EQ(static_cast<char>(BlobIndex::Type::kUnknown), blob_index.front());
 }
 
+TEST_F(DBBlobIndexTest,
+       MaybeResolveMemtableBlobValueWithoutFetcherFailsClosed) {
+  // Goal: if a readonly/secondary memtable hit produces a blob-backed payload
+  // but no BlobFetcher is available, the helper must fail closed instead of
+  // handing raw blob-index bytes back to the caller as if they were the value.
+  std::string blob_index;
+  BlobIndex::EncodeBlob(&blob_index, /*file_number=*/123, /*offset=*/456,
+                        /*size=*/789, kNoCompression);
+
+  PinnableSlice value;
+  value.GetSelf()->assign(blob_index.data(), blob_index.size());
+  value.PinSelf();
+
+  Status s = Status::OK();
+  bool is_blob_index = true;
+  ASSERT_TRUE(MaybeResolveMemtableBlobValueForTest(
+      Slice("key"), /*blob_fetcher=*/nullptr, &value, /*columns=*/nullptr, &s,
+      &is_blob_index));
+
+  ASSERT_TRUE(s.IsNotSupported()) << s.ToString();
+  ASSERT_TRUE(value.empty());
+  ASSERT_FALSE(is_blob_index);
+}
+
+TEST_F(DBBlobIndexTest, ReadOnlyGetImplReturnsBlobIndexWhenRequested) {
+  // Goal: cover the internal read-only GetImpl contract when the caller
+  // explicitly asks for raw blob-index bytes via `is_blob_index`. Recovery
+  // keeps the blob index in the memtable, and the read-only path must preserve
+  // the encoded index instead of eagerly resolving or rejecting it.
+  Options options = GetTestOptions();
+
+  DestroyAndReopen(options);
+
+  std::string blob_index;
+  BlobIndex::EncodeInlinedTTL(&blob_index, /*expiration=*/9876543210, "blob");
+
+  WriteBatch batch;
+  ASSERT_OK(PutBlobIndex(&batch, "blob_key", blob_index));
+  ASSERT_OK(Write(&batch));
+
+  Close();
+  options.avoid_flush_during_recovery = true;
+  ASSERT_OK(ReadOnlyReopen(options));
+
+  bool is_blob_index = false;
+  ASSERT_EQ(blob_index, GetImpl("blob_key", &is_blob_index));
+  ASSERT_TRUE(is_blob_index);
+}
+
 class PlainBlobValueFilterV3 : public CompactionFilter {
  public:
   PlainBlobValueFilterV3(std::atomic<int>* filter_call_count,
@@ -2138,7 +2197,7 @@ TEST_F(DBBlobIndexTest, EntityBlobFilterV3Remove) {
       db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(), key, columns));
   ASSERT_OK(Flush());
 
-  // Compact to trigger the filter — entity should be removed
+  // Compact to trigger the filter -- entity should be removed
   ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
 
   // Key should be gone

db/blog/blog_format.cc

@@ -1,621 +0,0 @@
-//  Copyright (c) Meta Platforms, Inc. and affiliates.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blog/blog_format.h"
-
-#include <cassert>
-#include <cstring>
-
-#include "env/unique_id_gen.h"
-#include "table/format.h"
-#include "table/unique_id_impl.h"
-#include "util/coding.h"
-#include "util/fastrange.h"
-#include "util/xxhash.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-// --- Property encoding/decoding ---
-
-void EncodeBlogProperties(const BlogPropertyMap& props, std::string* dst) {
-  for (const auto& [name, value] : props) {
-    PutLengthPrefixedSlice(dst, name);
-    PutLengthPrefixedSlice(dst, value);
-  }
-}
-
-Status DecodeBlogProperties(Slice* input, BlogPropertyMap* props) {
-  while (input->size() > 0) {
-    Slice name_slice;
-    Slice value_slice;
-    if (!GetLengthPrefixedSlice(input, &name_slice)) {
-      return Status::Corruption("Blog properties: truncated property name");
-    }
-    if (!GetLengthPrefixedSlice(input, &value_slice)) {
-      return Status::Corruption("Blog properties: truncated property value");
-    }
-    props->emplace_back(name_slice.ToString(), value_slice.ToString());
-  }
-  return Status::OK();
-}
-
-// --- BlogFileHeader ---
-
-uint32_t BlogFileHeader::incarnation_id() const {
-  return DecodeFixed32(escape_sequence);
-}
-
-void BlogFileHeader::EncodeTo(std::string* dst) const {
-  // --- Fixed header (40 bytes) ---
-  // Bytes [0-11]: File magic
-  dst->append(kBlogFileMagic, kBlogFileMagicSize);
-  // Byte [12]: Schema version
-  dst->push_back(static_cast<char>(schema_version));
-  // Byte [13]: Checksum type
-  dst->push_back(static_cast<char>(checksum_type));
-  // Byte [14]: Compact record type
-  dst->push_back(static_cast<char>(compact_record_type));
-  // Byte [15]: Flags
-  dst->push_back(static_cast<char>(flags));
-  // Bytes [16-25]: Escape sequence
-  dst->append(escape_sequence, kBlogEscapeSequenceSize);
-  // Bytes [26-31]: Reserved
-  for (int i = 0; i < 6; ++i) {
-    dst->push_back(0);
-  }
-
-  // Encode properties into a temporary buffer to measure size.
-  std::string prop_buf;
-  EncodeBlogProperties(properties, &prop_buf);
-  char prop_comp_type = static_cast<char>(kNoCompression);
-
-  // Build property section: [properties][compression_type][checksum]
-  uint32_t prop_checksum = ComputeBuiltinChecksumWithLastByte(
-      checksum_type, prop_buf.data(), prop_buf.size(), prop_comp_type);
-  prop_checksum +=
-      ChecksumModifierForContext(incarnation_id(), kBlogFileFixedHeaderSize);
-  std::string prop_section = std::move(prop_buf);
-  prop_section.push_back(prop_comp_type);
-  PutFixed32(&prop_section, prop_checksum);
-
-  // Bytes [32-35]: Property section size (trailer included, no padding)
-  PutFixed32(dst, static_cast<uint32_t>(prop_section.size()));
-
-  // Bytes [36-39]: Fixed header checksum (covers bytes [0, 36))
-  // Uses context checksum with incarnation_id at offset 0 for
-  // defense-in-depth (misplaced data detection).
-  assert(dst->size() == 36);
-  uint32_t header_checksum =
-      ComputeBuiltinChecksum(checksum_type, dst->data(), dst->size()) +
-      ChecksumModifierForContext(incarnation_id(), 0);
-  PutFixed32(dst, header_checksum);
-
-  // Property section follows the fixed header
-  assert(dst->size() == kBlogFileFixedHeaderSize);
-  dst->append(prop_section);
-}
-
-Status BlogFileHeader::DecodeFrom(Slice* input) {
-  if (input->size() < kBlogFileFixedHeaderSize) {
-    return Status::Corruption("Blog header too short: have " +
-                              std::to_string(input->size()) + " bytes, need " +
-                              std::to_string(kBlogFileFixedHeaderSize));
-  }
-
-  const char* p = input->data();
-
-  // Verify magic
-  if (memcmp(p, kBlogFileMagic, kBlogFileMagicSize) != 0) {
-    return Status::Corruption("Blog header: bad magic (first 4 bytes: 0x" +
-                              Slice(p, 4).ToString(true) + ")");
-  }
-
-  // Parse fields needed for checksum computation before verifying.
-  checksum_type = static_cast<ChecksumType>(p[13]);
-  memcpy(escape_sequence, p + 16, kBlogEscapeSequenceSize);
-
-  // Verify fixed header checksum FIRST, before trusting any other fields.
-  uint32_t prop_section_size = DecodeFixed32(p + 32);
-  uint32_t stored_header_checksum = DecodeFixed32(p + 36);
-  uint32_t computed_header_checksum =
-      ComputeBuiltinChecksum(checksum_type, p, 36) +
-      ChecksumModifierForContext(incarnation_id(), 0);
-  if (stored_header_checksum != computed_header_checksum) {
-    return Status::Corruption(
-        "Blog header: fixed header checksum mismatch (stored: 0x" +
-        Slice(p + 36, 4).ToString(true) + ", computed: 0x" +
-        Slice(reinterpret_cast<const char*>(&computed_header_checksum), 4)
-            .ToString(true) +
-        ")");
-  }
-
-  // Now that checksum is verified, validate the remaining fields.
-  schema_version = static_cast<uint8_t>(p[12]);
-  if (schema_version > kBlogCurrentSchemaVersion) {
-    return Status::NotSupported("Blog header: schema version " +
-                                std::to_string(schema_version) +
-                                " is newer than supported version " +
-                                std::to_string(kBlogCurrentSchemaVersion));
-  }
-  compact_record_type = static_cast<BlogRecordType>(p[14]);
-  flags = static_cast<uint8_t>(p[15]);
-
-  if (!VerifyEscapeSequence(escape_sequence)) {
-    return Status::Corruption(
-        "Blog header: escape sequence derived bytes mismatch (first byte: 0x" +
-        Slice(escape_sequence, 1).ToString(true) + ")");
-  }
-
-  // Advance past fixed header
-  input->remove_prefix(kBlogFileFixedHeaderSize);
-
-  // Read property section
-  if (input->size() < prop_section_size) {
-    return Status::Corruption("Blog header: truncated property section (have " +
-                              std::to_string(input->size()) + " bytes, need " +
-                              std::to_string(prop_section_size) + ")");
-  }
-
-  if (prop_section_size < kBlogBlockTrailerSize) {
-    return Status::Corruption("Blog header: property section too small (" +
-                              std::to_string(prop_section_size) +
-                              " bytes, min " +
-                              std::to_string(kBlogBlockTrailerSize) + ")");
-  }
-
-  // Property section layout: [properties][comp_type:1B][checksum:4B]
-  const char* prop_data = input->data();
-  size_t prop_data_size = prop_section_size - kBlogBlockTrailerSize;
-  char prop_comp_type = prop_data[prop_data_size];
-  uint32_t stored_prop_checksum = DecodeFixed32(prop_data + prop_data_size + 1);
-  uint32_t computed_prop_checksum = ComputeBuiltinChecksumWithLastByte(
-      checksum_type, prop_data, prop_data_size, prop_comp_type);
-  computed_prop_checksum +=
-      ChecksumModifierForContext(incarnation_id(), kBlogFileFixedHeaderSize);
-  if (stored_prop_checksum != computed_prop_checksum) {
-    return Status::Corruption(
-        "Blog header: property section checksum mismatch (prop_size=" +
-        std::to_string(prop_data_size) + ", comp_type=0x" +
-        Slice(&prop_comp_type, 1).ToString(true) + ")");
-  }
-
-  // Decode properties
-  Slice prop_slice(prop_data, prop_data_size);
-  properties.clear();
-  Status s = DecodeBlogProperties(&prop_slice, &properties);
-  if (!s.ok()) {
-    return s;
-  }
-
-  // Check for unrecognized required properties
-  for (const auto& [name, value] : properties) {
-    if (IsBlogRequiredProperty(name)) {
-      if (name != kBlogPropCompressionCompatibilityName &&
-          name != kBlogPropTimestampSize &&
-          name != kBlogPropPredecessorWalInfo &&
-          name != kBlogPropWriteBatchStreamingCompressionType) {
-        return Status::NotSupported(
-            "Blog header: unrecognized required property: \"" + name + "\"");
-      }
-    }
-  }
-
-  input->remove_prefix(prop_section_size);
-  return Status::OK();
-}
-
-void BlogFileHeader::SetEscapeSequenceFromU64(uint64_t v) {
-  // Byte 0: derived from top 4 bytes via FastRange32 into [1, 254],
-  // avoiding 0x00 and 0xFF (which are padding bytes).
-  uint32_t top = static_cast<uint32_t>(v >> 32);
-  escape_sequence[0] = static_cast<char>(FastRange32(top, 254) + 1);
-  assert(escape_sequence[0] != '\0');
-  assert(escape_sequence[0] != '\xFF');
-
-  // Bytes 1-5: lowest 5 bytes as-is (little-endian).
-  EncodeFixed32(escape_sequence + 1, static_cast<uint32_t>(v));
-  escape_sequence[5] = static_cast<char>(v >> 32);
-  // (byte at index 5 overlaps with top byte used for FastRange32, but
-  // that's fine -- it's just input entropy, not the derived byte 0)
-
-  // Bytes 6-9: XXH3-derived suffix for heuristic detection. This would enable
-  // us to scan through a file missing its header and, with some hashing effort,
-  // detect escape sequences with high accuracy (expect 1 false positive per
-  // ~16GB; further validation/pruning can be done trying to parse records
-  // and/or picking the most common candidate escape sequence).
-  uint64_t derived = XXH3_64bits_withSeed(
-      escape_sequence, kBlogEscapeSeqRandomPartSize, kBlogEscapeSeqSeed);
-  EncodeFixed32(escape_sequence + kBlogEscapeSeqRandomPartSize,
-                static_cast<uint32_t>(derived));
-}
-
-void BlogFileHeader::GenerateFromUniqueId(const std::string& db_id,
-                                          const std::string& db_session_id,
-                                          uint64_t file_number) {
-  if (db_id.empty() || db_session_id.empty() || file_number == 0) {
-    GenerateRandomFields();
-    return;
-  }
-
-  // Derive escape sequence from the same unique ID scheme as SST files.
-  UniqueId64x2 unique_id;
-  Status s =
-      GetSstInternalUniqueId(db_id, db_session_id, file_number, &unique_id);
-  if (!s.ok()) {
-    GenerateRandomFields();
-    return;
-  }
-
-  // The internal unique ID is intentionally not fully random/mixed. Mix it
-  // so that it is.
-  InternalUniqueIdToExternal(&unique_id);
-
-  SetEscapeSequenceFromU64(unique_id[0]);
-
-  // Store the inputs as ignorable header properties for debugging/tracking.
-  SetProperty(kBlogPropDbId, db_id);
-  SetProperty(kBlogPropDbSessionId, db_session_id);
-  SetUint64Property(kBlogPropFileNumber, file_number);
-}
-
-void BlogFileHeader::GenerateRandomFields() {
-  uint64_t rand_a;
-  uint64_t rand_b;
-  GenerateRawUniqueId(&rand_a, &rand_b);
-  // Use rand_a for escape sequence, rand_b is unused extra entropy.
-  (void)rand_b;
-  SetEscapeSequenceFromU64(rand_a);
-}
-
-bool BlogFileHeader::VerifyEscapeSequence(const char* seq) {
-  uint8_t first = static_cast<uint8_t>(seq[0]);
-  if (first == 0x00 || first == 0xFF) {
-    return false;
-  }
-  // Verify derived part
-  uint64_t derived = XXH3_64bits_withSeed(seq, kBlogEscapeSeqRandomPartSize,
-                                          kBlogEscapeSeqSeed);
-  uint32_t expected = static_cast<uint32_t>(derived);
-  uint32_t actual = DecodeFixed32(seq + kBlogEscapeSeqRandomPartSize);
-  return expected == actual;
-}
-
-void BlogFileHeader::SetProperty(const std::string& name,
-                                 const std::string& value) {
-  for (auto& [k, v] : properties) {
-    if (k == name) {
-      v = value;
-      return;
-    }
-  }
-  properties.emplace_back(name, value);
-}
-
-std::string BlogFileHeader::GetProperty(const std::string& name) const {
-  for (const auto& [k, v] : properties) {
-    if (k == name) {
-      return v;
-    }
-  }
-  return "";
-}
-
-bool BlogFileHeader::HasProperty(const std::string& name) const {
-  for (const auto& [k, v] : properties) {
-    if (k == name) {
-      return true;
-    }
-  }
-  return false;
-}
-
-void BlogFileHeader::SetUint64Property(const std::string& name,
-                                       uint64_t value) {
-  std::string encoded;
-  PutVarint64(&encoded, value);
-  SetProperty(name, encoded);
-}
-
-void BlogFileHeader::SetUint32Property(const std::string& name,
-                                       uint32_t value) {
-  std::string encoded;
-  PutVarint32(&encoded, value);
-  SetProperty(name, encoded);
-}
-
-bool BlogFileHeader::GetUint64Property(const std::string& name,
-                                       uint64_t* value) const {
-  std::string raw = GetProperty(name);
-  if (raw.empty()) {
-    return false;
-  }
-  Slice s(raw);
-  return GetVarint64(&s, value);
-}
-
-bool BlogFileHeader::GetUint32Property(const std::string& name,
-                                       uint32_t* value) const {
-  std::string raw = GetProperty(name);
-  if (raw.empty()) {
-    return false;
-  }
-  Slice s(raw);
-  return GetVarint32(&s, value);
-}
-
-Status BlogFileHeader::DecodeFromBuffer(Slice* buffer,
-                                        size_t* additional_bytes_needed) {
-  if (buffer->size() < kBlogFileFixedHeaderSize) {
-    return Status::Corruption("Blog header: buffer too small for fixed header");
-  }
-
-  // Check if the full header (fixed + property section) fits in the buffer.
-  uint32_t prop_section_size = DecodeFixed32(buffer->data() + 32);
-  size_t total_header_size = kBlogFileFixedHeaderSize + prop_section_size;
-
-  if (buffer->size() < total_header_size) {
-    if (additional_bytes_needed) {
-      *additional_bytes_needed = total_header_size - buffer->size();
-    }
-    return Status::Incomplete(
-        "Blog header: need more data for property section",
-        std::to_string(total_header_size));
-  }
-
-  // Full header is available. Decode it.
-  Slice header_slice(buffer->data(), total_header_size);
-  Status s = DecodeFrom(&header_slice);
-  if (s.ok()) {
-    buffer->remove_prefix(total_header_size);
-  }
-  return s;
-}
-
-// --- Footer locator verification ---
-
-bool VerifyBlogFooterLocator(const char* buffer, size_t buffer_size,
-                             uint64_t buffer_file_offset,
-                             const char* expected_escape_seq,
-                             ChecksumType checksum_type,
-                             uint32_t incarnation_id) {
-  // Scan backward in 4-byte steps for the escape sequence.
-  // Buffer must start at a 4-byte-aligned file offset.
-  for (size_t pos = buffer_size; pos >= 4; pos -= 4) {
-    size_t esc_start = pos - 4;
-    uint8_t first_byte = static_cast<uint8_t>(buffer[esc_start]);
-    if (first_byte == 0x00 || first_byte == 0xFF) {
-      continue;  // padding byte
-    }
-    // Check if this is our escape sequence
-    if (esc_start + kBlogEscapeSequenceSize > buffer_size ||
-        memcmp(buffer + esc_start, expected_escape_seq,
-               kBlogEscapeSequenceSize) != 0) {
-      continue;  // not our escape sequence
-    }
-
-    // Found escape sequence. Parse the full-format record that follows.
-    // Full format: [escape_seq:10B] [varint length:4+B] [type:1B]
-    //              [compression_type:1B] [prefix_checksum:4B]
-    //              [payload:length B] [comp:1B] [checksum:4B]
-    const char* rec = buffer + esc_start + kBlogEscapeSequenceSize;
-    size_t remaining = buffer_size - esc_start - kBlogEscapeSequenceSize;
-
-    // Parse varint length (must be >= 4 bytes for full format)
-    Slice varint_slice(rec, remaining);
-    uint64_t length = 0;
-    const char* varint_end = GetVarint64Ptr(rec, rec + remaining, &length);
-    if (!varint_end) {
-      continue;  // truncated varint
-    }
-    size_t varint_len = static_cast<size_t>(varint_end - rec);
-    if (varint_len <= kBlogCompactVarintMaxBytes) {
-      continue;  // not full format
-    }
-
-    // Need type(1) + comp(1) + prefix_checksum(4) after varint
-    size_t after_varint = remaining - varint_len;
-    if (after_varint < 6) {
-      continue;  // truncated prefix
-    }
-
-    BlogRecordType type =
-        static_cast<BlogRecordType>(static_cast<uint8_t>(varint_end[0]));
-    if (type != kBlogFooterLocatorRecord) {
-      continue;  // not a footer locator
-    }
-
-    // Verify prefix checksum over (varint + type + compression_type)
-    size_t prefix_data_size = varint_len + 2;  // varint + type + comp
-    uint64_t esc_file_offset = buffer_file_offset + esc_start;
-    uint32_t stored_prefix_checksum = DecodeFixed32(varint_end + 2);
-    uint32_t computed_prefix_checksum =
-        ComputeBuiltinChecksum(checksum_type, rec, prefix_data_size) +
-        ChecksumModifierForContext(incarnation_id, esc_file_offset);
-    if (stored_prefix_checksum != computed_prefix_checksum) {
-      continue;  // prefix checksum mismatch
-    }
-
-    // If length > 0, verify the payload trailer checksum
-    if (length > 0) {
-      size_t after_prefix = after_varint - 6;  // after type+comp+prefix_cksum
-      if (after_prefix < length + kBlogBlockTrailerSize) {
-        continue;  // truncated payload or trailer
-      }
-      const char* payload = varint_end + 6;
-      uint64_t payload_file_offset = buffer_file_offset + esc_start +
-                                     kBlogEscapeSequenceSize + varint_len + 6;
-      CompressionType actual_comp;
-      Status s = VerifyBlogRecordTrailer(
-          checksum_type, payload, static_cast<size_t>(length), incarnation_id,
-          payload_file_offset, &actual_comp);
-      if (!s.ok()) {
-        continue;  // trailer checksum mismatch
-      }
-    }
-
-    // Valid footer locator record found.
-    return true;
-  }
-  return false;
-}
-
-// --- BlogFileFooterLocator ---
-
-void BlogFileFooterLocator::EncodeTo(std::string* dst) const {
-  for (const auto& entry : entries) {
-    dst->push_back(static_cast<char>(entry.record_type));
-    PutFixed32(dst, entry.relative_offset_4B);
-  }
-}
-
-Status BlogFileFooterLocator::DecodeFrom(const Slice& input) {
-  Slice s = input;
-  entries.clear();
-  while (s.size() >= 5) {  // 1 byte type + 4 bytes offset
-    entries.push_back(
-        {.record_type = static_cast<BlogRecordType>(static_cast<uint8_t>(s[0])),
-         .relative_offset_4B = DecodeFixed32(s.data() + 1)});
-    s.remove_prefix(5);
-  }
-  if (s.size() != 0) {
-    return Status::Corruption(
-        "Blog footer locator: trailing bytes after entries");
-  }
-  return Status::OK();
-}
-
-// --- BlogFileFooterProperties ---
-
-void BlogFileFooterProperties::SetBlobCount(uint64_t count) {
-  std::string val;
-  PutVarint64(&val, count);
-  properties.emplace_back("blobCount", std::move(val));
-}
-
-void BlogFileFooterProperties::SetTotalBlobBytes(uint64_t bytes) {
-  std::string val;
-  PutVarint64(&val, bytes);
-  properties.emplace_back("totalBlobBytes", std::move(val));
-}
-
-void BlogFileFooterProperties::SetSequenceRange(uint64_t min_seq,
-                                                uint64_t max_seq) {
-  std::string min_val;
-  PutVarint64(&min_val, min_seq);
-  properties.emplace_back("minSequence", std::move(min_val));
-  std::string max_val;
-  PutVarint64(&max_val, max_seq);
-  properties.emplace_back("maxSequence", std::move(max_val));
-}
-
-void BlogFileFooterProperties::EncodeTo(std::string* dst) const {
-  EncodeBlogProperties(properties, dst);
-}
-
-Status BlogFileFooterProperties::DecodeFrom(const Slice& input) {
-  Slice s = input;
-  properties.clear();
-  return DecodeBlogProperties(&s, &properties);
-}
-
-// --- Padding ---
-
-void ComputeBlogPaddingParams(uint8_t last_meaningful_byte,
-                              size_t current_offset, uint8_t* pad_byte,
-                              size_t* pad_count) {
-  // Determine how many bytes needed to reach next 4-byte alignment
-  size_t remainder = current_offset % 4;
-  size_t needed = (remainder == 0) ? 0 : (4 - remainder);
-
-  // Choose pad byte to differ from last_meaningful_byte
-  if (last_meaningful_byte == 0x00) {
-    *pad_byte = 0xFF;
-    // Must have at least 1 byte of padding to distinguish from checksum
-    if (needed == 0) {
-      needed = 4;
-    }
-  } else if (last_meaningful_byte == 0xFF) {
-    *pad_byte = 0x00;
-    if (needed == 0) {
-      needed = 4;
-    }
-  } else {
-    *pad_byte = 0x00;  // Writer's choice; 0x00 is conventional
-  }
-  *pad_count = needed;
-}
-
-void ComputeBlogPadding(uint8_t last_meaningful_byte, size_t current_offset,
-                        std::string* dst) {
-  uint8_t pad_byte;
-  size_t pad_count;
-  ComputeBlogPaddingParams(last_meaningful_byte, current_offset, &pad_byte,
-                           &pad_count);
-  dst->append(pad_count, static_cast<char>(pad_byte));
-}
-
-// --- Checksum ---
-
-uint32_t ComputeBlogRecordChecksum(ChecksumType type, const char* data,
-                                   size_t data_size, char compression_type_byte,
-                                   uint32_t base_context_checksum,
-                                   uint64_t record_offset) {
-  uint32_t checksum = ComputeBuiltinChecksumWithLastByte(type, data, data_size,
-                                                         compression_type_byte);
-  checksum += ChecksumModifierForContext(base_context_checksum, record_offset);
-  return checksum;
-}
-
-// --- Record trailer verification ---
-
-Status VerifyBlogRecordTrailer(ChecksumType checksum_type, const char* payload,
-                               size_t payload_size, uint32_t incarnation_id,
-                               uint64_t payload_file_offset,
-                               CompressionType* actual_compression_type) {
-  // Trailer is the 5 bytes immediately after the payload:
-  // [compression_type: 1B][checksum: 4B]
-  const char* trailer = payload + payload_size;
-  char comp_byte = trailer[0];
-  uint32_t stored_checksum = DecodeFixed32(trailer + 1);
-  uint32_t computed_checksum =
-      ComputeBlogRecordChecksum(checksum_type, payload, payload_size, comp_byte,
-                                incarnation_id, payload_file_offset);
-  if (stored_checksum != computed_checksum) {
-    return Status::Corruption("Blog record checksum mismatch");
-  }
-  if (actual_compression_type) {
-    *actual_compression_type =
-        static_cast<CompressionType>(static_cast<uint8_t>(comp_byte));
-  }
-  return Status::OK();
-}
-
-// --- Irregular varint ---
-
-void PutBlogIrregularVarint64(std::string* dst, uint64_t value,
-                              size_t min_encoded_bytes) {
-  // Encode as standard varint, then pad with trailing zero-data bytes
-  // to reach min_encoded_bytes. The result decodes to the same value.
-  char buf[10];
-  char* end = EncodeVarint64(buf, value);
-  size_t natural_len = static_cast<size_t>(end - buf);
-
-  if (natural_len >= min_encoded_bytes) {
-    dst->append(buf, natural_len);
-    return;
-  }
-
-  // Set the continuation bit on what was the terminal byte, then append
-  // (extra - 1) continuation bytes (0x80) and one terminal byte (0x00).
-  size_t extra = min_encoded_bytes - natural_len;
-  buf[natural_len - 1] |= static_cast<char>(0x80);
-  dst->append(buf, natural_len);
-  for (size_t i = 0; i + 1 < extra; ++i) {
-    dst->push_back(static_cast<char>(0x80));
-  }
-  dst->push_back(static_cast<char>(0x00));
-}
-
-}  // namespace ROCKSDB_NAMESPACE

db/blog/blog_format.h

@@ -1,299 +0,0 @@
-//  Copyright (c) Meta Platforms, Inc. and affiliates.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <cstdint>
-#include <string>
-#include <utility>
-#include <vector>
-
-#include "rocksdb/rocksdb_namespace.h"
-#include "rocksdb/slice.h"
-#include "rocksdb/status.h"
-#include "rocksdb/table.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-// Record types used in blog file body and footer.
-// Types 0x01-0x7F are for body records.
-// Types 0xF0-0xFF are for footer records (in expected order of occurrence,
-// with gaps for future additions).
-enum BlogRecordType : uint8_t {
-  kBlogBlobRecord = 0x01,
-  kBlogPreambleStartRecord = 0x02,
-  kBlogWriteBatchRecord = 0x03,
-  // 0x04-0x7F: reserved for future body record types
-
-  kBlogFooterIndexRecord = 0xF0,  // sparse index or similar large metadata
-  // 0xF1-0xF3: reserved for future footer data records
-  kBlogFooterPropertiesRecord = 0xF8,  // small metadata (named properties)
-  // 0xF9-0xFB: reserved
-  kBlogFooterFileChecksumInfo = 0xFC,  // full file checksum state dump, so
-                                       // the file checksum can be inferred
-                                       // from just reading the tail
-  // 0xFD-0xFE: reserved
-  kBlogFooterLocatorRecord = 0xFF,  // relative offsets to other footer
-                                    // records; must be last
-};
-
-// --- Constants ---
-
-// 12-byte file magic: "Blog" (ASCII) followed by XXH3_64bits("Blog", 4).
-// Used as an opaque identifier for blog format files.
-static constexpr char kBlogFileMagic[12] = {
-    '\x42', '\x6C', '\x6F', '\x67',                                 // "Blog"
-    '\x9F', '\xFC', '\x41', '\x52', '\xF7', '\xF2', '\x2C', '\x49'  // XXH3_64
-};
-static constexpr size_t kBlogFileMagicSize = 12;
-
-// Fixed header size in bytes (before property section).
-static constexpr size_t kBlogFileFixedHeaderSize = 40;
-
-// Escape sequence: 6 random bytes + 4 derived bytes.
-static constexpr size_t kBlogEscapeSequenceSize = 10;
-static constexpr size_t kBlogEscapeSeqRandomPartSize = 6;
-
-// Seed for deriving bytes [6-9] of escape sequence via
-// XXH3_64bits_withSeed(random_6_bytes, 6, kBlogEscapeSeqSeed).
-// Chosen arbitrarily; must never change once the format is released.
-static constexpr uint64_t kBlogEscapeSeqSeed =
-    0x52636B73426C6F67ULL;  // "RcksBlg\0"
-
-// Sentinel for unspecified-size records. Encodes to a 9-byte varint,
-// which always triggers the full record format.
-static constexpr uint64_t kBlogUnspecifiedSize = (uint64_t{1} << 63) - 1;
-
-// Maximum payload size that encodes to a compact-format varint (<= 2 bytes).
-// varint of 16383 (0x3FFF) uses 2 bytes; 16384 uses 3 bytes.
-// Records larger than this get full format with prefix checksum protection.
-static constexpr uint64_t kBlogMaxCompactPayloadSize = (1u << 14) - 1;
-
-// Compact format varint threshold: varints of this many bytes or fewer
-// use compact format (no type byte, no prefix checksum).
-static constexpr size_t kBlogCompactVarintMaxBytes = 2;
-
-// Block trailer size: 1 byte compression_type + 4 bytes checksum.
-static constexpr size_t kBlogBlockTrailerSize = 5;
-
-// kStreamingCompressionSentinel (0x7F) is defined in compression_type.h
-// and used in blog record trailers to indicate streaming compression.
-// A record with kNoCompression in the trailer remains individually
-// uncompressed even when streaming is active (dynamic on/off).
-
-// --- Named Properties ---
-
-// Named property: string key -> string value.
-// Case convention: Uppercase first letter = required (reader must recognize
-// or reject the file). Lowercase first letter = ignorable.
-using BlogPropertyMap = std::vector<std::pair<std::string, std::string>>;
-
-// Encode properties as a sequence of length-prefixed name/value pairs.
-// No count prefix; section is terminated by reaching the end of the slice.
-void EncodeBlogProperties(const BlogPropertyMap& props, std::string* dst);
-
-// Decode properties from a slice, consuming all bytes.
-Status DecodeBlogProperties(Slice* input, BlogPropertyMap* props);
-
-// Returns true if a property name starts with an uppercase letter,
-// meaning the reader must recognize it or reject the file.
-inline bool IsBlogRequiredProperty(const Slice& name) {
-  return name.size() > 0 && name[0] >= 'A' && name[0] <= 'Z';
-}
-
-// Known header property names
-static constexpr const char* kBlogPropCompressionCompatibilityName =
-    "CompressionCompatibilityName";
-static constexpr const char* kBlogPropTimestampSize = "TimestampSize";
-static constexpr const char* kBlogPropPredecessorWalInfo = "PredecessorWalInfo";
-// Required (uppercase) when streaming compression is active for WriteBatch
-// records. Value is a 2-digit hex encoding of the CompressionType used
-// (e.g. "07" for kZSTD). The presence of this property means WriteBatch
-// record payloads with kStreamingCompressionSentinel in their trailer
-// are part of a streaming compression context that spans across records.
-static constexpr const char* kBlogPropWriteBatchStreamingCompressionType =
-    "WriteBatchStreamingCompressionType";
-static constexpr const char* kBlogPropRole = "role";
-static constexpr const char* kBlogPropCompressionSettings =
-    "compressionSettings";
-static constexpr const char* kBlogPropCreationTime = "creationTime";
-static constexpr const char* kBlogPropCreator = "creator";
-// Ignorable properties for file identity and diagnostics.
-static constexpr const char* kBlogPropDbId = "dbId";
-static constexpr const char* kBlogPropDbSessionId = "dbSessionId";
-static constexpr const char* kBlogPropFileNumber = "fileNumber";
-static constexpr const char* kBlogPropDbHostId = "dbHostId";
-static constexpr const char* kBlogPropColumnFamilyId = "columnFamilyId";
-static constexpr const char* kBlogPropColumnFamilyName = "columnFamilyName";
-
-// --- BlogFileHeader ---
-
-// Bit flags for BlogFileHeader::flags (byte [15] in the fixed header).
-enum BlogFileHeaderFlags : uint8_t {
-  kBlogFileRecycled = 0x01,  // File may contain trailing stale data from a
-                             // previous incarnation. Recovery should tolerate
-                             // trailing data that doesn't match our escape
-                             // sequence (recycled/stale, not corruption).
-};
-
-static constexpr uint8_t kBlogCurrentSchemaVersion = 0;
-
-struct BlogFileHeader {
-  uint8_t schema_version = kBlogCurrentSchemaVersion;
-  ChecksumType checksum_type = kXXH3;
-  BlogRecordType compact_record_type = kBlogBlobRecord;
-  uint8_t flags = 0;  // BlogFileHeaderFlags, byte [15] in fixed header
-  // 10-byte escape sequence: bytes [0-5] random (byte[0] not 0x00/0xFF),
-  // bytes [6-9] = lower32(XXH3_64bits_withSeed(bytes[0-5],
-  // kBlogEscapeSeqSeed)).
-  char escape_sequence[kBlogEscapeSequenceSize] = {};
-
-  bool is_recycled() const { return flags & kBlogFileRecycled; }
-
-  BlogPropertyMap properties;
-
-  // Incarnation ID = first 4 bytes of escape_sequence (little-endian via
-  // DecodeFixed32). Guaranteed nonzero since byte[0] is not 0x00.
-  uint32_t incarnation_id() const;
-
-  // Encode the full header (fixed prefix + property section + checksums)
-  // into dst.
-  void EncodeTo(std::string* dst) const;
-
-  // Decode from input. Consumes the bytes read.
-  Status DecodeFrom(Slice* input);
-
-  // Generate escape_sequence from the file's unique ID (db_id, db_session_id,
-  // file_number), using the same scheme as SST internal unique IDs. Falls back
-  // to GenerateRandomFields() if any input is empty/zero. Also sets the
-  // ignorable dbId/dbSessionId/fileNumber header properties.
-  void GenerateFromUniqueId(const std::string& db_id,
-                            const std::string& db_session_id,
-                            uint64_t file_number);
-
-  // Generate escape_sequence from random bytes. Appropriate when unique ID
-  // inputs are not available.
-  void GenerateRandomFields();
-
- private:
-  // Populate escape_sequence from a 64 bits of entropy.
-  void SetEscapeSequenceFromU64(uint64_t v);
-
- public:
-  // Verify that the escape sequence's derived bytes match.
-  static bool VerifyEscapeSequence(const char* seq);
-
-  // Check if a buffer starts with the blog file magic number.
-  static bool IsBlogFormat(const char* data, size_t size) {
-    return size >= kBlogFileMagicSize &&
-           memcmp(data, kBlogFileMagic, kBlogFileMagicSize) == 0;
-  }
-
-  // Decode a blog file header from a buffer that may or may not contain
-  // the full header. Returns OK and consumes the header bytes on success.
-  // If the buffer contains the fixed header but not the full property
-  // section, returns the number of additional bytes needed in
-  // *additional_bytes_needed and returns Status::Incomplete().
-  // This enables single-read header parsing from a ~4 KiB buffer.
-  Status DecodeFromBuffer(Slice* buffer, size_t* additional_bytes_needed);
-
-  // Raw property helpers (string values)
-  void SetProperty(const std::string& name, const std::string& value);
-  std::string GetProperty(const std::string& name) const;
-
-  // Typed property setters/getters for properties that aren't natively strings.
-  void SetUint64Property(const std::string& name, uint64_t value);
-  void SetUint32Property(const std::string& name, uint32_t value);
-  bool GetUint64Property(const std::string& name, uint64_t* value) const;
-  bool GetUint32Property(const std::string& name, uint32_t* value) const;
-  bool HasProperty(const std::string& name) const;
-};
-
-// --- Footer types ---
-
-// Entry in a footer locator record. Offsets are in units of 4 bytes,
-// sequential relative to each other in reverse order of footer records.
-// First entry: offset from locator's escape_seq to the preceding footer
-// record's escape_seq. Subsequent entries: offset from the previous
-// entry's escape_seq to the next earlier footer record's escape_seq.
-// Verify that a blog file's trailing data contains a valid footer locator
-// record. Scans backward from the end of the buffer for the escape sequence,
-// then parses and verifies the full record (varint length, type, prefix
-// checksum, payload, trailer checksum). The buffer must start at a 4-byte-
-// aligned file offset. Returns true if a valid footer locator record is found.
-bool VerifyBlogFooterLocator(const char* buffer, size_t buffer_size,
-                             uint64_t buffer_file_offset,
-                             const char* expected_escape_seq,
-                             ChecksumType checksum_type,
-                             uint32_t incarnation_id);
-
-struct BlogFooterLocatorEntry {
-  BlogRecordType record_type;
-  uint32_t relative_offset_4B;
-};
-
-struct BlogFileFooterLocator {
-  std::vector<BlogFooterLocatorEntry> entries;
-
-  // Encode as the payload of a footer locator record.
-  void EncodeTo(std::string* dst) const;
-  Status DecodeFrom(const Slice& input);
-};
-
-struct BlogFileFooterProperties {
-  BlogPropertyMap properties;
-
-  // Convenience setters that encode values into the property map.
-  void SetBlobCount(uint64_t count);
-  void SetTotalBlobBytes(uint64_t bytes);
-  void SetSequenceRange(uint64_t min_seq, uint64_t max_seq);
-
-  // Encode as named properties (no 5-byte trailer; that comes from
-  // the enclosing full record format).
-  void EncodeTo(std::string* dst) const;
-  Status DecodeFrom(const Slice& input);
-};
-
-// --- Padding helpers ---
-
-// Compute padding needed to reach the next 4-byte-aligned offset,
-// choosing the pad byte value to differ from last_meaningful_byte.
-// Appends padding bytes to dst.
-void ComputeBlogPadding(uint8_t last_meaningful_byte, size_t current_offset,
-                        std::string* dst);
-
-// Same as above, but returns the pad byte value and count without
-// allocating. For use on hot paths.
-void ComputeBlogPaddingParams(uint8_t last_meaningful_byte,
-                              size_t current_offset, uint8_t* pad_byte,
-                              size_t* pad_count);
-
-// --- Checksum helpers ---
-
-// Compute a context-aware checksum over data + compression_type byte,
-// suitable for a blog record trailer. Uses the same algorithm and context
-// checksum approach as block-based SST files.
-uint32_t ComputeBlogRecordChecksum(ChecksumType type, const char* data,
-                                   size_t data_size, char compression_type_byte,
-                                   uint32_t base_context_checksum,
-                                   uint64_t record_offset);
-
-// Verify a blog record's 5-byte trailer (compression_type + checksum).
-// payload points to the record payload of payload_size bytes; the trailer
-// immediately follows. Returns OK if checksum matches, Corruption otherwise.
-// On success, *actual_compression_type is set from the trailer.
-Status VerifyBlogRecordTrailer(ChecksumType checksum_type, const char* payload,
-                               size_t payload_size, uint32_t incarnation_id,
-                               uint64_t payload_file_offset,
-                               CompressionType* actual_compression_type);
-
-// Encode an irregular varint: a value encoded with more bytes than the
-// minimum required, ensuring varint length > 3 bytes to trigger full
-// record format. The value is encoded correctly and will decode to the
-// same value via standard GetVarint64.
-void PutBlogIrregularVarint64(std::string* dst, uint64_t value,
-                              size_t min_encoded_bytes);
-
-}  // namespace ROCKSDB_NAMESPACE

db/blog/blog_format_test.cc

@@ -1,539 +0,0 @@
-//  Copyright (c) Meta Platforms, Inc. and affiliates.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blog/blog_format.h"
-
-#include <cstdint>
-#include <cstring>
-#include <string>
-
-#include "table/format.h"
-#include "test_util/testharness.h"
-#include "util/coding.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class BlogFormatTest : public testing::Test {};
-
-// --- Property encoding/decoding ---
-
-TEST_F(BlogFormatTest, EncodeDecodePropertiesEmpty) {
-  BlogPropertyMap props;
-  std::string encoded;
-  EncodeBlogProperties(props, &encoded);
-  ASSERT_EQ(encoded.size(), 0u);
-
-  BlogPropertyMap decoded;
-  Slice input(encoded);
-  ASSERT_OK(DecodeBlogProperties(&input, &decoded));
-  ASSERT_TRUE(decoded.empty());
-}
-
-TEST_F(BlogFormatTest, EncodeDecodePropertiesMultiple) {
-  BlogPropertyMap props = {
-      {"CompressionCompatibilityName", "zstd"},
-      {"role", "wal"},
-      {"compressionSettings", "level=3"},
-  };
-  std::string encoded;
-  EncodeBlogProperties(props, &encoded);
-  ASSERT_GT(encoded.size(), 0u);
-
-  BlogPropertyMap decoded;
-  Slice input(encoded);
-  ASSERT_OK(DecodeBlogProperties(&input, &decoded));
-  ASSERT_EQ(decoded.size(), 3u);
-  ASSERT_EQ(decoded[0].first, "CompressionCompatibilityName");
-  ASSERT_EQ(decoded[0].second, "zstd");
-  ASSERT_EQ(decoded[1].first, "role");
-  ASSERT_EQ(decoded[1].second, "wal");
-  ASSERT_EQ(decoded[2].first, "compressionSettings");
-  ASSERT_EQ(decoded[2].second, "level=3");
-}
-
-TEST_F(BlogFormatTest, DecodePropertiesTruncated) {
-  // Write a name but no value
-  std::string encoded;
-  PutLengthPrefixedSlice(&encoded, "key");
-  // Truncate before value
-  BlogPropertyMap decoded;
-  Slice input(encoded);
-  ASSERT_TRUE(DecodeBlogProperties(&input, &decoded).IsCorruption());
-}
-
-// --- IsBlogRequiredProperty ---
-
-TEST_F(BlogFormatTest, IsBlogRequiredProperty) {
-  ASSERT_TRUE(IsBlogRequiredProperty("CompressionCompatibilityName"));
-  ASSERT_TRUE(IsBlogRequiredProperty("TimestampSize"));
-  ASSERT_TRUE(IsBlogRequiredProperty("A"));
-  ASSERT_FALSE(IsBlogRequiredProperty("role"));
-  ASSERT_FALSE(IsBlogRequiredProperty("compressionSettings"));
-  ASSERT_FALSE(IsBlogRequiredProperty("a"));
-  ASSERT_FALSE(IsBlogRequiredProperty(""));
-}
-
-// --- Escape sequence generation ---
-
-TEST_F(BlogFormatTest, GenerateRandomFieldsConstraints) {
-  // Generate many escape sequences and verify constraints
-  for (int i = 0; i < 100; ++i) {
-    BlogFileHeader header;
-    header.GenerateRandomFields();
-
-    // First byte must not be 0x00 or 0xFF
-    uint8_t first = static_cast<uint8_t>(header.escape_sequence[0]);
-    ASSERT_NE(first, 0x00u) << "iteration " << i;
-    ASSERT_NE(first, 0xFFu) << "iteration " << i;
-
-    // Derived bytes must match
-    ASSERT_TRUE(BlogFileHeader::VerifyEscapeSequence(header.escape_sequence))
-        << "iteration " << i;
-  }
-}
-
-TEST_F(BlogFormatTest, IncarnationIdNonzero) {
-  BlogFileHeader header;
-  header.GenerateRandomFields();
-  // incarnation_id is first 4 bytes little-endian; since byte[0] is not 0,
-  // the incarnation_id must be nonzero.
-  ASSERT_NE(header.incarnation_id(), 0u);
-}
-
-TEST_F(BlogFormatTest, VerifyEscapeSequenceRejectsInvalid) {
-  char seq[kBlogEscapeSequenceSize] = {};
-  // All zeros: first byte is 0x00 -> invalid
-  ASSERT_FALSE(BlogFileHeader::VerifyEscapeSequence(seq));
-
-  // Valid first byte but wrong derived part
-  seq[0] = 0x42;
-  seq[1] = 0x13;
-  // Derived part is random garbage, likely won't match
-  ASSERT_FALSE(BlogFileHeader::VerifyEscapeSequence(seq));
-}
-
-TEST_F(BlogFormatTest, HeuristicEscapeSequenceDetection) {
-  // Generate a valid escape sequence and verify we can detect it
-  // heuristically (by checking derived bytes against random part).
-  BlogFileHeader header;
-  header.GenerateRandomFields();
-
-  // VerifyEscapeSequence does exactly this heuristic check
-  ASSERT_TRUE(BlogFileHeader::VerifyEscapeSequence(header.escape_sequence));
-
-  // Corrupt one random byte -> should fail verification
-  char corrupted[kBlogEscapeSequenceSize];
-  memcpy(corrupted, header.escape_sequence, kBlogEscapeSequenceSize);
-  corrupted[2] ^= 0x42;
-  // Might still pass if the corruption happens to produce a valid derived
-  // part, but astronomically unlikely
-  // (We don't ASSERT_FALSE here because of the tiny probability)
-}
-
-// --- BlogFileHeader encode/decode ---
-
-TEST_F(BlogFormatTest, HeaderEncodeDecodeMinimal) {
-  BlogFileHeader header;
-  header.checksum_type = kXXH3;
-  header.compact_record_type = kBlogWriteBatchRecord;
-  header.GenerateRandomFields();
-
-  std::string encoded;
-  header.EncodeTo(&encoded);
-  ASSERT_GE(encoded.size(), kBlogFileFixedHeaderSize);
-
-  BlogFileHeader decoded;
-  Slice input(encoded);
-  ASSERT_OK(decoded.DecodeFrom(&input));
-
-  ASSERT_EQ(decoded.schema_version, header.schema_version);
-  ASSERT_EQ(decoded.checksum_type, header.checksum_type);
-  ASSERT_EQ(decoded.compact_record_type, header.compact_record_type);
-  ASSERT_EQ(memcmp(decoded.escape_sequence, header.escape_sequence,
-                   kBlogEscapeSequenceSize),
-            0);
-  ASSERT_TRUE(decoded.properties.empty());
-  ASSERT_EQ(input.size(), 0u);  // all bytes consumed
-}
-
-TEST_F(BlogFormatTest, HeaderEncodeDecodeWithProperties) {
-  BlogFileHeader header;
-  header.checksum_type = kCRC32c;
-  header.compact_record_type = kBlogBlobRecord;
-  header.GenerateRandomFields();
-  header.SetProperty("CompressionCompatibilityName", "lz4");
-  header.SetProperty("role", "blob");
-  header.SetProperty("compressionSettings", "level=1");
-
-  std::string encoded;
-  header.EncodeTo(&encoded);
-
-  BlogFileHeader decoded;
-  Slice input(encoded);
-  ASSERT_OK(decoded.DecodeFrom(&input));
-
-  ASSERT_EQ(decoded.GetProperty("CompressionCompatibilityName"), "lz4");
-  ASSERT_EQ(decoded.GetProperty("role"), "blob");
-  ASSERT_EQ(decoded.GetProperty("compressionSettings"), "level=1");
-  ASSERT_EQ(input.size(), 0u);
-}
-
-TEST_F(BlogFormatTest, HeaderRejectsUnknownRequiredProperty) {
-  BlogFileHeader header;
-  header.GenerateRandomFields();
-  header.SetProperty("FutureRequired", "something");
-
-  std::string encoded;
-  header.EncodeTo(&encoded);
-
-  BlogFileHeader decoded;
-  Slice input(encoded);
-  Status s = decoded.DecodeFrom(&input);
-  ASSERT_TRUE(s.IsNotSupported()) << s.ToString();
-}
-
-TEST_F(BlogFormatTest, HeaderRejectsNewerSchemaVersion) {
-  BlogFileHeader header;
-  header.GenerateRandomFields();
-
-  std::string encoded;
-  header.EncodeTo(&encoded);
-
-  // Tamper with schema version byte (offset 12) to a future version.
-  // Must also recompute the fixed header checksum (bytes [36-39]) with
-  // context modifier using the incarnation_id.
-  encoded[12] = static_cast<char>(kBlogCurrentSchemaVersion + 1);
-  uint32_t new_checksum =
-      ComputeBuiltinChecksum(header.checksum_type, encoded.data(), 36) +
-      ChecksumModifierForContext(header.incarnation_id(), 0);
-  EncodeFixed32(&encoded[36], new_checksum);
-
-  BlogFileHeader decoded;
-  Slice input(encoded);
-  Status s = decoded.DecodeFrom(&input);
-  ASSERT_TRUE(s.IsNotSupported()) << s.ToString();
-  ASSERT_NE(s.ToString().find("newer than supported"), std::string::npos)
-      << s.ToString();
-}
-
-TEST_F(BlogFormatTest, HeaderCurrentSchemaVersionAccepted) {
-  BlogFileHeader header;
-  header.GenerateRandomFields();
-  ASSERT_EQ(header.schema_version, kBlogCurrentSchemaVersion);
-
-  std::string encoded;
-  header.EncodeTo(&encoded);
-
-  BlogFileHeader decoded;
-  Slice input(encoded);
-  ASSERT_OK(decoded.DecodeFrom(&input));
-  ASSERT_EQ(decoded.schema_version, kBlogCurrentSchemaVersion);
-}
-
-TEST_F(BlogFormatTest, TypedPropertyRoundTrip) {
-  BlogFileHeader header;
-  header.GenerateRandomFields();
-  header.SetUint64Property("fileNumber", 12345678ULL);
-  header.SetUint32Property("columnFamilyId", 42);
-
-  std::string encoded;
-  header.EncodeTo(&encoded);
-
-  BlogFileHeader decoded;
-  Slice input(encoded);
-  ASSERT_OK(decoded.DecodeFrom(&input));
-
-  uint64_t fn = 0;
-  ASSERT_TRUE(decoded.GetUint64Property("fileNumber", &fn));
-  ASSERT_EQ(fn, 12345678ULL);
-
-  uint32_t cf_id = 0;
-  ASSERT_TRUE(decoded.GetUint32Property("columnFamilyId", &cf_id));
-  ASSERT_EQ(cf_id, 42u);
-
-  // Missing property returns false
-  uint64_t missing = 0;
-  ASSERT_FALSE(decoded.GetUint64Property("nonexistent", &missing));
-}
-
-TEST_F(BlogFormatTest, HeaderAcceptsUnknownIgnorableProperty) {
-  BlogFileHeader header;
-  header.GenerateRandomFields();
-  header.SetProperty("futureIgnorable", "something");
-
-  std::string encoded;
-  header.EncodeTo(&encoded);
-
-  BlogFileHeader decoded;
-  Slice input(encoded);
-  ASSERT_OK(decoded.DecodeFrom(&input));
-  ASSERT_EQ(decoded.GetProperty("futureIgnorable"), "something");
-}
-
-TEST_F(BlogFormatTest, HeaderChecksumCorruption) {
-  BlogFileHeader header;
-  header.GenerateRandomFields();
-
-  std::string encoded;
-  header.EncodeTo(&encoded);
-
-  // Corrupt a byte in the fixed header
-  encoded[14] ^= 0xFF;
-
-  BlogFileHeader decoded;
-  Slice input(encoded);
-  ASSERT_TRUE(decoded.DecodeFrom(&input).IsCorruption());
-}
-
-TEST_F(BlogFormatTest, HeaderPropertyChecksumCorruption) {
-  BlogFileHeader header;
-  header.GenerateRandomFields();
-  header.SetProperty("role", "wal");
-
-  std::string encoded;
-  header.EncodeTo(&encoded);
-
-  // Corrupt a byte in the property section (after fixed header)
-  if (encoded.size() > kBlogFileFixedHeaderSize + 2) {
-    encoded[kBlogFileFixedHeaderSize + 2] ^= 0xFF;
-  }
-
-  BlogFileHeader decoded;
-  Slice input(encoded);
-  Status s = decoded.DecodeFrom(&input);
-  ASSERT_TRUE(s.IsCorruption()) << s.ToString();
-}
-
-TEST_F(BlogFormatTest, HeaderSetPropertyOverwrites) {
-  BlogFileHeader header;
-  header.SetProperty("role", "wal");
-  ASSERT_EQ(header.GetProperty("role"), "wal");
-
-  header.SetProperty("role", "blob");
-  ASSERT_EQ(header.GetProperty("role"), "blob");
-  // Should not have duplicate entries
-  int count = 0;
-  for (const auto& [k, v] : header.properties) {
-    if (k == "role") {
-      ++count;
-    }
-  }
-  ASSERT_EQ(count, 1);
-}
-
-TEST_F(BlogFormatTest, HeaderMagicValues) {
-  BlogFileHeader header;
-  header.GenerateRandomFields();
-
-  std::string encoded;
-  header.EncodeTo(&encoded);
-
-  // Verify magic bytes
-  ASSERT_GE(encoded.size(), kBlogFileMagicSize);
-  ASSERT_EQ(memcmp(encoded.data(), kBlogFileMagic, kBlogFileMagicSize), 0);
-}
-
-// --- Footer locator ---
-
-TEST_F(BlogFormatTest, FooterLocatorEncodeDecodeEmpty) {
-  BlogFileFooterLocator locator;
-
-  std::string encoded;
-  locator.EncodeTo(&encoded);
-  ASSERT_EQ(encoded.size(), 0u);
-
-  BlogFileFooterLocator decoded;
-  ASSERT_OK(decoded.DecodeFrom(Slice(encoded)));
-  ASSERT_TRUE(decoded.entries.empty());
-}
-
-TEST_F(BlogFormatTest, FooterLocatorEncodeDecodeMultiple) {
-  BlogFileFooterLocator locator;
-  locator.entries.push_back({kBlogFooterPropertiesRecord, 50});
-  locator.entries.push_back({kBlogFooterIndexRecord, 85});
-
-  std::string encoded;
-  locator.EncodeTo(&encoded);
-  ASSERT_EQ(encoded.size(), 10u);  // 2 * (1 + 4)
-
-  BlogFileFooterLocator decoded;
-  ASSERT_OK(decoded.DecodeFrom(Slice(encoded)));
-  ASSERT_EQ(decoded.entries.size(), 2u);
-  ASSERT_EQ(decoded.entries[0].record_type, kBlogFooterPropertiesRecord);
-  ASSERT_EQ(decoded.entries[0].relative_offset_4B, 50u);
-  ASSERT_EQ(decoded.entries[1].record_type, kBlogFooterIndexRecord);
-  ASSERT_EQ(decoded.entries[1].relative_offset_4B, 85u);
-}
-
-// --- Footer properties ---
-
-TEST_F(BlogFormatTest, FooterPropertiesEncodeDecodeRoundTrip) {
-  BlogFileFooterProperties props;
-  props.SetBlobCount(42);
-  props.SetTotalBlobBytes(123456);
-  props.SetSequenceRange(100, 200);
-
-  std::string encoded;
-  props.EncodeTo(&encoded);
-
-  BlogFileFooterProperties decoded;
-  ASSERT_OK(decoded.DecodeFrom(Slice(encoded)));
-  ASSERT_EQ(decoded.properties.size(), 4u);
-}
-
-// --- Padding ---
-
-TEST_F(BlogFormatTest, PaddingLastByteZero) {
-  // When last byte is 0x00, must pad with 0xFF, at least 1 byte
-  std::string pad;
-  // At offset 0 (already aligned), still need at least 1 byte
-  ComputeBlogPadding(0x00, 0, &pad);
-  ASSERT_GE(pad.size(), 1u);
-  for (char c : pad) {
-    ASSERT_EQ(static_cast<uint8_t>(c), 0xFFu);
-  }
-  // Result must be 4-byte aligned
-  ASSERT_EQ(pad.size() % 4, 0u);
-}
-
-TEST_F(BlogFormatTest, PaddingLastByteFF) {
-  // When last byte is 0xFF, must pad with 0x00, at least 1 byte
-  std::string pad;
-  ComputeBlogPadding(0xFF, 0, &pad);
-  ASSERT_GE(pad.size(), 1u);
-  for (char c : pad) {
-    ASSERT_EQ(static_cast<uint8_t>(c), 0x00u);
-  }
-  ASSERT_EQ(pad.size() % 4, 0u);
-}
-
-TEST_F(BlogFormatTest, PaddingLastByteOther) {
-  // When last byte is something else, pad only if needed for alignment
-  std::string pad;
-  ComputeBlogPadding(0x42, 0, &pad);
-  // Already aligned, no padding needed
-  ASSERT_EQ(pad.size(), 0u);
-
-  pad.clear();
-  ComputeBlogPadding(0x42, 1, &pad);
-  // Need 3 bytes to reach alignment
-  ASSERT_EQ(pad.size(), 3u);
-  for (char c : pad) {
-    ASSERT_EQ(static_cast<uint8_t>(c), 0x00u);
-  }
-}
-
-TEST_F(BlogFormatTest, PaddingAlignment) {
-  for (size_t offset = 0; offset < 8; ++offset) {
-    std::string pad;
-    ComputeBlogPadding(0x42, offset, &pad);
-    size_t after = offset + pad.size();
-    ASSERT_EQ(after % 4, 0u) << "offset=" << offset;
-  }
-}
-
-// --- Irregular varint ---
-
-TEST_F(BlogFormatTest, IrregularVarintZero) {
-  std::string encoded;
-  PutBlogIrregularVarint64(&encoded, 0, 4);
-  ASSERT_GE(encoded.size(), 4u);
-
-  // Must decode back to 0
-  Slice input(encoded);
-  uint64_t value;
-  ASSERT_TRUE(GetVarint64(&input, &value));
-  ASSERT_EQ(value, 0u);
-}
-
-TEST_F(BlogFormatTest, IrregularVarintSmallValue) {
-  std::string encoded;
-  PutBlogIrregularVarint64(&encoded, 42, 4);
-  ASSERT_GE(encoded.size(), 4u);
-
-  Slice input(encoded);
-  uint64_t value;
-  ASSERT_TRUE(GetVarint64(&input, &value));
-  ASSERT_EQ(value, 42u);
-}
-
-TEST_F(BlogFormatTest, IrregularVarintLargeValue) {
-  // A value that naturally needs 3 bytes, padded to 5
-  uint64_t original = 100000;
-  std::string encoded;
-  PutBlogIrregularVarint64(&encoded, original, 5);
-  ASSERT_GE(encoded.size(), 5u);
-
-  Slice input(encoded);
-  uint64_t value;
-  ASSERT_TRUE(GetVarint64(&input, &value));
-  ASSERT_EQ(value, original);
-}
-
-TEST_F(BlogFormatTest, IrregularVarintNaturallyLargeEnough) {
-  // A value that naturally needs 5+ bytes should not be padded further
-  uint64_t original = 1ULL << 35;
-  std::string encoded;
-  PutBlogIrregularVarint64(&encoded, original, 4);
-  // Natural encoding is 6 bytes, already >= 4
-  ASSERT_EQ(encoded.size(), 6u);
-
-  Slice input(encoded);
-  uint64_t value;
-  ASSERT_TRUE(GetVarint64(&input, &value));
-  ASSERT_EQ(value, original);
-}
-
-TEST_F(BlogFormatTest, UnspecifiedSizeVarint) {
-  std::string encoded;
-  PutVarint64(&encoded, kBlogUnspecifiedSize);
-  // 2^63 - 1 should encode to 9 bytes (63 bits / 7 = 9)
-  ASSERT_EQ(encoded.size(), 9u);
-
-  Slice input(encoded);
-  uint64_t value;
-  ASSERT_TRUE(GetVarint64(&input, &value));
-  ASSERT_EQ(value, kBlogUnspecifiedSize);
-}
-
-// --- Checksum ---
-
-TEST_F(BlogFormatTest, ComputeBlogRecordChecksumBasic) {
-  const char* data = "hello world";
-  size_t size = strlen(data);
-
-  // With no context (incarnation_id = 0), should match standard checksum
-  uint32_t cksum = ComputeBlogRecordChecksum(
-      kXXH3, data, size, static_cast<char>(kNoCompression), 0, 0);
-  uint32_t expected = ComputeBuiltinChecksumWithLastByte(
-      kXXH3, data, size, static_cast<char>(kNoCompression));
-  ASSERT_EQ(cksum, expected);
-}
-
-TEST_F(BlogFormatTest, ComputeBlogRecordChecksumWithContext) {
-  const char* data = "hello world";
-  size_t size = strlen(data);
-
-  uint32_t cksum1 = ComputeBlogRecordChecksum(
-      kXXH3, data, size, static_cast<char>(kNoCompression), 0x12345678, 100);
-  uint32_t cksum2 = ComputeBlogRecordChecksum(
-      kXXH3, data, size, static_cast<char>(kNoCompression), 0x12345678, 200);
-
-  // Different offsets should produce different checksums
-  ASSERT_NE(cksum1, cksum2);
-
-  // Different incarnation IDs should produce different checksums
-  uint32_t cksum3 = ComputeBlogRecordChecksum(
-      kXXH3, data, size, static_cast<char>(kNoCompression), 0xABCDEF01, 100);
-  ASSERT_NE(cksum1, cksum3);
-}
-
-}  // namespace ROCKSDB_NAMESPACE
-
-int main(int argc, char** argv) {
-  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}

db/blog/blog_reader.cc

@@ -1,388 +0,0 @@
-//  Copyright (c) Meta Platforms, Inc. and affiliates.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blog/blog_reader.h"
-
-#include <cassert>
-#include <cstring>
-
-#include "table/format.h"
-#include "util/coding.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-BlogFileReader::BlogFileReader(std::unique_ptr<SequentialFileReader>&& file,
-                               Reporter* reporter, bool verify_checksums)
-    : seq_file_(std::move(file)),
-      reporter_(reporter),
-      verify_checksums_(verify_checksums) {}
-
-Status BlogFileReader::ReadHeader(BlogFileHeader* header) {
-  assert(!header_read_);
-
-  // Read the fixed header to get property_section_size
-  std::string fixed_buf;
-  Slice fixed_slice;
-  Status s = ReadBytes(kBlogFileFixedHeaderSize, &fixed_slice, &fixed_buf);
-  if (!s.ok()) {
-    return s;
-  }
-
-  // Peek at property_section_size from bytes [32-35]
-  uint32_t prop_section_size = DecodeFixed32(fixed_slice.data() + 32);
-
-  // Read the property section
-  std::string prop_buf;
-  Slice prop_slice;
-  s = ReadBytes(prop_section_size, &prop_slice, &prop_buf);
-  if (!s.ok()) {
-    return s;
-  }
-
-  // Combine into a single buffer for decoding
-  std::string full_header;
-  full_header.append(fixed_slice.data(), fixed_slice.size());
-  full_header.append(prop_slice.data(), prop_slice.size());
-
-  Slice input(full_header);
-  s = header->DecodeFrom(&input);
-  if (!s.ok()) {
-    return s;
-  }
-
-  header_ = *header;
-  header_read_ = true;
-
-  // Skip padding after header to reach the first record's escape sequence.
-  // Padding may be arbitrarily long (for stronger alignment or recycled
-  // files of all zeros). EOF here is fine (header-only file, no records).
-  s = SkipPadding();
-  if (!s.ok() && !s.IsNotFound()) {
-    return s;
-  }
-  return Status::OK();
-}
-
-Status BlogFileReader::ReadRecord(BlogRecordType* type, Slice* payload,
-                                  std::string* scratch, uint64_t* record_offset,
-                                  CompressionType* record_compression_type) {
-  if (!header_read_) {
-    return Status::Corruption("Blog reader: header not read");
-  }
-  if (eof_) {
-    return Status::NotFound();
-  }
-
-  Status s;
-  scratch->clear();
-
-  // Read escape sequence. If we have a pending prefix from header padding
-  // skip or inter-record padding skip, use it as the first 4 bytes.
-  std::string esc_buf;
-  if (!pending_escape_prefix_.empty()) {
-    esc_buf = std::move(pending_escape_prefix_);
-    pending_escape_prefix_.clear();
-    Slice rest_slice;
-    std::string rest_buf;
-    s = ReadBytes(kBlogEscapeSequenceSize - 4, &rest_slice, &rest_buf);
-    if (!s.ok()) {
-      eof_ = true;
-      // Short read after we had a pending prefix = incomplete trailing record
-      return Status::Incomplete("Blog reader: incomplete escape sequence");
-    }
-    esc_buf.append(rest_slice.data(), rest_slice.size());
-  } else {
-    Slice esc_slice;
-    s = ReadBytes(kBlogEscapeSequenceSize, &esc_slice, &esc_buf);
-    if (!s.ok()) {
-      eof_ = true;
-      return Status::NotFound();  // clean EOF
-    }
-    if (esc_slice.data() != esc_buf.data()) {
-      esc_buf.assign(esc_slice.data(), esc_slice.size());
-    }
-  }
-
-  // Verify escape sequence matches
-  if (memcmp(esc_buf.data(), header_.escape_sequence,
-             kBlogEscapeSequenceSize) != 0) {
-    return ReportCorruption(kBlogEscapeSequenceSize,
-                            "escape sequence mismatch");
-  }
-
-  uint64_t rec_offset = offset_ - kBlogEscapeSequenceSize;
-  if (record_offset) {
-    *record_offset = rec_offset;
-  }
-  last_record_offset_ = rec_offset;
-
-  // Read varint length. scratch receives the raw varint bytes for use
-  // in prefix checksum verification (avoids fragile re-encoding).
-  uint64_t length = 0;
-  s = ReadVarint64(&length, scratch);
-  if (!s.ok()) {
-    eof_ = true;
-    return Status::Incomplete("Blog reader: incomplete varint");
-  }
-  size_t varint_len = scratch->size();
-
-  bool is_full_format = (varint_len > kBlogCompactVarintMaxBytes);
-
-  BlogRecordType rec_type;
-  std::string prefix_rest_buf;
-
-  if (is_full_format) {
-    // Full format: read type + compression_type + prefix_checksum
-    Slice prefix_rest_slice;
-    s = ReadBytes(6, &prefix_rest_slice, &prefix_rest_buf);  // 1+1+4
-    if (!s.ok()) {
-      eof_ = true;
-      return Status::Incomplete("Blog reader: incomplete full record prefix");
-    }
-
-    rec_type =
-        static_cast<BlogRecordType>(static_cast<uint8_t>(prefix_rest_slice[0]));
-
-    if (verify_checksums_) {
-      // Build the prefix data from the original varint bytes (in scratch)
-      // plus type and compression_type bytes.
-      scratch->push_back(prefix_rest_slice[0]);  // type
-      scratch->push_back(prefix_rest_slice[1]);  // comp_type
-
-      uint32_t stored_prefix_checksum =
-          DecodeFixed32(prefix_rest_slice.data() + 2);
-      uint32_t computed_prefix_checksum =
-          ComputeBuiltinChecksum(header_.checksum_type, scratch->data(),
-                                 scratch->size()) +
-          ChecksumModifierForContext(header_.incarnation_id(), rec_offset);
-      if (stored_prefix_checksum != computed_prefix_checksum) {
-        return ReportCorruption(varint_len + 6, "prefix checksum mismatch");
-      }
-    }
-  } else {
-    rec_type = header_.compact_record_type;
-  }
-
-  *type = rec_type;
-
-  if (length == 0) {
-    payload->clear();
-    if (record_compression_type) {
-      *record_compression_type = kNoCompression;
-    }
-    s = SkipPadding();
-    if (!s.ok() && !s.IsNotFound()) {
-      return s;
-    }
-    return Status::OK();
-  }
-
-  if (length == kBlogUnspecifiedSize) {
-    return ReportCorruption(0, "unspecified-size records not yet implemented");
-  }
-
-  // Read payload
-  scratch->clear();
-  Slice payload_slice;
-  s = ReadBytes(static_cast<size_t>(length), &payload_slice, scratch);
-  if (!s.ok()) {
-    eof_ = true;
-    return Status::Incomplete("Blog reader: incomplete payload");
-  }
-  if (payload_slice.data() != scratch->data()) {
-    scratch->assign(payload_slice.data(), payload_slice.size());
-  }
-
-  // Read 5-byte trailer
-  Slice trailer_slice;
-  std::string trailer_buf;
-  s = ReadBytes(kBlogBlockTrailerSize, &trailer_slice, &trailer_buf);
-  if (!s.ok()) {
-    eof_ = true;
-    return Status::Incomplete("Blog reader: incomplete trailer");
-  }
-
-  char trailer_comp_type = trailer_slice[0];
-  uint32_t stored_checksum = DecodeFixed32(trailer_slice.data() + 1);
-
-  if (verify_checksums_) {
-    uint32_t computed_checksum = ComputeBlogRecordChecksum(
-        header_.checksum_type, scratch->data(), scratch->size(),
-        trailer_comp_type, header_.incarnation_id(),
-        offset_ - kBlogBlockTrailerSize - length);
-    if (stored_checksum != computed_checksum) {
-      return ReportCorruption(
-          static_cast<size_t>(length) + kBlogBlockTrailerSize,
-          "record checksum mismatch");
-    }
-  }
-
-  *payload = Slice(*scratch);
-  if (record_compression_type) {
-    *record_compression_type =
-        static_cast<CompressionType>(static_cast<uint8_t>(trailer_comp_type));
-  }
-
-  // Skip padding to next record.
-  s = SkipPadding();
-  if (!s.ok() && !s.IsNotFound()) {
-    return s;
-  }
-
-  return Status::OK();
-}
-
-Status BlogFileReader::ScanForEscapeSequence(uint64_t* found_offset) {
-  // Scan forward 4 bytes at a time (escape sequences are 32-bit aligned).
-  // Check first 4 bytes of escape sequence, then verify remaining 6 bytes.
-  uint32_t first4;
-  memcpy(&first4, header_.escape_sequence, 4);
-
-  std::string buf;
-  // Align to 4-byte boundary first
-  size_t remainder = offset_ % 4;
-  if (remainder != 0) {
-    size_t skip = 4 - remainder;
-    Slice skip_slice;
-    Status s = ReadBytes(skip, &skip_slice, &buf);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  while (!eof_) {
-    Slice chunk_slice;
-    Status s = ReadBytes(4, &chunk_slice, &buf);
-    if (!s.ok()) {
-      if (eof_) {
-        return Status::NotFound("escape sequence not found before EOF");
-      }
-      return s;
-    }
-
-    uint32_t candidate;
-    memcpy(&candidate, chunk_slice.data(), 4);
-    if (candidate == first4) {
-      // Potential match - verify remaining 6 bytes
-      uint64_t candidate_offset = offset_ - 4;
-      Slice rest_slice;
-      std::string rest_buf;
-      s = ReadBytes(kBlogEscapeSequenceSize - 4, &rest_slice, &rest_buf);
-      if (!s.ok()) {
-        return s;
-      }
-
-      if (memcmp(rest_slice.data(), header_.escape_sequence + 4,
-                 kBlogEscapeSequenceSize - 4) == 0) {
-        *found_offset = candidate_offset;
-        return Status::OK();
-      }
-      // False match, continue scanning. But we consumed 6 extra bytes that
-      // might not be aligned. Re-align.
-      remainder = offset_ % 4;
-      if (remainder != 0) {
-        size_t skip = 4 - remainder;
-        Slice skip_s;
-        s = ReadBytes(skip, &skip_s, &buf);
-        if (!s.ok()) {
-          return s;
-        }
-      }
-    }
-  }
-  return Status::NotFound("escape sequence not found before EOF");
-}
-
-Status BlogFileReader::SkipPadding() {
-  // Skip arbitrarily long padding (0x00 or 0xFF bytes). Padding is
-  // 32-bit aligned. Since the escape sequence's first byte is guaranteed
-  // to be neither 0x00 nor 0xFF, we align to 4 bytes then scan 4-byte
-  // chunks until we find one that starts with a non-padding byte.
-  size_t remainder = offset_ % 4;
-  if (remainder != 0) {
-    Slice skip_slice;
-    std::string skip_buf;
-    Status s = ReadBytes(4 - remainder, &skip_slice, &skip_buf);
-    if (!s.ok()) {
-      if (eof_) {
-        return Status::NotFound();
-      }
-      return s;
-    }
-  }
-  while (true) {
-    Slice chunk_slice;
-    std::string chunk_buf;
-    Status s = ReadBytes(4, &chunk_slice, &chunk_buf);
-    if (!s.ok()) {
-      if (eof_) {
-        return Status::NotFound();
-      }
-      return s;
-    }
-    uint8_t first = static_cast<uint8_t>(chunk_slice[0]);
-    if (first != 0x00 && first != 0xFF) {
-      // Found the start of the next escape sequence.
-      pending_escape_prefix_.assign(chunk_slice.data(), 4);
-      return Status::OK();
-    }
-  }
-}
-
-Status BlogFileReader::ReadBytes(size_t n, Slice* result,
-                                 std::string* scratch) {
-  scratch->resize(n);
-  Slice read_result;
-  IOStatus ios =
-      seq_file_->Read(n, &read_result, scratch->data(), Env::IO_TOTAL);
-  if (!ios.ok()) {
-    return static_cast<Status>(ios);
-  }
-  if (read_result.size() < n) {
-    eof_ = true;
-    // Return Incomplete -- the caller (ReadRecord) determines whether
-    // this is an acceptable incomplete tail vs. genuine corruption.
-    return Status::Incomplete("Blog reader", "unexpected EOF");
-  }
-  offset_ += n;
-  *result = Slice(scratch->data(), n);
-  return Status::OK();
-}
-
-// FIXME? Consider a better encoding that could be processed more efficiently
-Status BlogFileReader::ReadVarint64(uint64_t* value, std::string* scratch) {
-  // Varints can be up to 10 bytes. Read one byte at a time.
-  // The raw varint bytes are appended to scratch so the caller can use
-  // them for checksum verification without re-encoding.
-  scratch->clear();
-  std::string byte_buf;
-  for (int i = 0; i < 10; ++i) {
-    Slice byte_slice;
-    Status s = ReadBytes(1, &byte_slice, &byte_buf);
-    if (!s.ok()) {
-      return s;
-    }
-    scratch->push_back(byte_slice[0]);
-    // Check if this is the terminal byte (high bit not set)
-    if ((static_cast<uint8_t>(byte_slice[0]) & 0x80) == 0) {
-      Slice varint_slice(*scratch);
-      if (!GetVarint64(&varint_slice, value)) {
-        return ReportCorruption(scratch->size(), "invalid varint");
-      }
-      return Status::OK();
-    }
-  }
-  return ReportCorruption(10, "varint too long");
-}
-
-Status BlogFileReader::ReportCorruption(size_t bytes, const std::string& msg) {
-  Status s = Status::Corruption("Blog reader", msg);
-  if (reporter_) {
-    reporter_->Corruption(bytes, s);
-  }
-  return s;
-}
-
-}  // namespace ROCKSDB_NAMESPACE

db/blog/blog_reader.h

@@ -1,101 +0,0 @@
-//  Copyright (c) Meta Platforms, Inc. and affiliates.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <cstdint>
-#include <memory>
-#include <string>
-
-#include "db/blog/blog_format.h"
-#include "file/random_access_file_reader.h"
-#include "file/sequence_file_reader.h"
-#include "rocksdb/slice.h"
-#include "rocksdb/status.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-// Reads blog-format files sequentially (WAL replay) or at random offsets
-// (blob lookups). Supports footer reading by backward scanning.
-class BlogFileReader {
- public:
-  // Error reporter, same pattern as log::Reader::Reporter.
-  class Reporter {
-   public:
-    virtual ~Reporter() = default;
-    virtual void Corruption(size_t bytes, const Status& status) = 0;
-  };
-
-  // Construct for sequential reading (WAL replay, forward scan).
-  BlogFileReader(std::unique_ptr<SequentialFileReader>&& file,
-                 Reporter* reporter, bool verify_checksums);
-
-  ~BlogFileReader() = default;
-
-  // Take ownership of a reporter (e.g., an adapter wrapping another type).
-  void SetOwnedReporter(std::unique_ptr<Reporter> reporter) {
-    owned_reporter_ = std::move(reporter);
-    reporter_ = owned_reporter_.get();
-  }
-
-  // Read and validate the file header. Must be called first.
-  Status ReadHeader(BlogFileHeader* header);
-
-  // Read the next record sequentially. Returns:
-  //   OK - record read successfully
-  //   NotFound - clean EOF (no more records, only padding/zeros remain)
-  //   Incomplete - partial record at tail (crash mid-write)
-  //   Corruption - data integrity error
-  // On OK, *type is the record type and *payload contains the payload.
-  // *scratch is used as backing storage for payload.
-  Status ReadRecord(BlogRecordType* type, Slice* payload, std::string* scratch,
-                    uint64_t* record_offset = nullptr,
-                    CompressionType* record_compression_type = nullptr);
-
-  // Scan forward from the current position for the next occurrence of
-  // the file's escape sequence. Used for WAL recovery and unspecified-
-  // size record parsing.
-  // On success, *found_offset is the file offset of the escape sequence.
-  Status ScanForEscapeSequence(uint64_t* found_offset);
-
-  bool IsEOF() const { return eof_; }
-  uint64_t LastRecordOffset() const { return last_record_offset_; }
-  uint64_t current_offset() const { return offset_; }
-  const BlogFileHeader& header() const { return header_; }
-
- private:
-  // Skip arbitrarily long padding (0x00 or 0xFF bytes) after a record or
-  // header. Aligns to 4-byte boundary first, then scans 4-byte chunks
-  // until finding one whose first byte is not 0x00/0xFF. Stashes that
-  // chunk in pending_escape_prefix_ for the next ReadRecord call.
-  // Returns OK on success, NotFound on clean EOF (no more records),
-  // or an error Status on I/O failure.
-  Status SkipPadding();
-
-  // Read exactly n bytes sequentially. Returns Corruption on short read.
-  Status ReadBytes(size_t n, Slice* result, std::string* scratch);
-
-  // Read a varint64 from the sequential stream.
-  Status ReadVarint64(uint64_t* value, std::string* scratch);
-
-  // Report corruption and return a Corruption status.
-  Status ReportCorruption(size_t bytes, const std::string& msg);
-
-  std::unique_ptr<SequentialFileReader> seq_file_;
-  Reporter* reporter_;
-  std::unique_ptr<Reporter> owned_reporter_;  // optional, for adapter lifetime
-  bool verify_checksums_;
-  BlogFileHeader header_;
-  bool header_read_ = false;
-  bool eof_ = false;
-  uint64_t offset_ = 0;
-  uint64_t last_record_offset_ = 0;
-
-  // First 4 bytes of escape sequence consumed during header padding skip.
-  // ReadRecord uses these as the beginning of the first escape sequence.
-  std::string pending_escape_prefix_;
-};
-
-}  // namespace ROCKSDB_NAMESPACE

db/blog/blog_writer.cc

@@ -1,320 +0,0 @@
-//  Copyright (c) Meta Platforms, Inc. and affiliates.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blog/blog_writer.h"
-
-#include <cassert>
-
-#include "table/format.h"
-#include "util/coding.h"
-#include "util/string_util.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-BlogFileWriter::BlogFileWriter(std::unique_ptr<WritableFileWriter>&& dest,
-                               const BlogFileHeader& header, bool manual_flush)
-    : dest_(std::move(dest)), header_(header), manual_flush_(manual_flush) {}
-
-BlogFileWriter::~BlogFileWriter() = default;
-
-IOStatus BlogFileWriter::WriteHeader(const WriteOptions& wo) {
-  assert(!header_written_);
-  std::string buf;
-  header_.EncodeTo(&buf);
-  // Pad after header properties using the same padding scheme as records.
-  uint8_t last_byte = static_cast<uint8_t>(buf.back());
-  ComputeBlogPadding(last_byte, buf.size(), &buf);
-  IOStatus s = EmitBytes(wo, buf);
-  if (s.ok()) {
-    header_written_ = true;
-  }
-  if (s.ok() && !manual_flush_) {
-    IOOptions opts;
-    s = WritableFileWriter::PrepareIOOptions(wo, opts);
-    if (s.ok()) {
-      s = dest_->Flush(opts);
-    }
-  }
-  return s;
-}
-
-IOStatus BlogFileWriter::AddBlobRecord(const WriteOptions& wo,
-                                       const Slice& payload,
-                                       CompressionType comp_type,
-                                       uint64_t* blob_offset) {
-  return AddRecord(wo, kBlogBlobRecord, payload, comp_type, blob_offset);
-}
-
-IOStatus BlogFileWriter::AddWriteBatchRecord(const WriteOptions& wo,
-                                             const Slice& wb_data,
-                                             CompressionType comp_type) {
-  // Stub: WriteBatch record writing is not yet integrated (WAL integration
-  // deferred). The record format is the same as blob records with type 0x03.
-  return AddRecord(wo, kBlogWriteBatchRecord, wb_data, comp_type, nullptr);
-}
-
-IOStatus BlogFileWriter::AddPreambleStartRecord(const WriteOptions& wo) {
-  return AddRecord(wo, kBlogPreambleStartRecord, Slice(), kNoCompression,
-                   nullptr, /*force_full=*/true);
-}
-
-IOStatus BlogFileWriter::AddFooterIndexRecord(const WriteOptions& wo,
-                                              const Slice& index_data) {
-  return AddRecord(wo, kBlogFooterIndexRecord, index_data, kNoCompression,
-                   nullptr, /*force_full=*/true);
-}
-
-IOStatus BlogFileWriter::AddFooterPropertiesRecord(
-    const WriteOptions& wo, const BlogFileFooterProperties& props) {
-  std::string payload;
-  props.EncodeTo(&payload);
-  return AddRecord(wo, kBlogFooterPropertiesRecord, payload, kNoCompression,
-                   nullptr, /*force_full=*/true);
-}
-
-IOStatus BlogFileWriter::AddFooterLocatorRecord(
-    const WriteOptions& wo, const BlogFileFooterLocator& locator) {
-  std::string payload;
-  locator.EncodeTo(&payload);
-  return AddRecord(wo, kBlogFooterLocatorRecord, payload, kNoCompression,
-                   nullptr, /*force_full=*/true);
-}
-
-IOStatus BlogFileWriter::WriteBuffer(const WriteOptions& wo) {
-  IOOptions opts;
-  IOStatus s = WritableFileWriter::PrepareIOOptions(wo, opts);
-  if (s.ok()) {
-    s = dest_->Flush(opts);
-  }
-  return s;
-}
-
-IOStatus BlogFileWriter::Sync(const WriteOptions& wo, bool use_fsync) {
-  IOOptions opts;
-  IOStatus s = WritableFileWriter::PrepareIOOptions(wo, opts);
-  if (s.ok()) {
-    s = dest_->Sync(opts, use_fsync);
-  }
-  return s;
-}
-
-IOStatus BlogFileWriter::Close(const WriteOptions& wo) {
-  IOOptions opts;
-  IOStatus s = WritableFileWriter::PrepareIOOptions(wo, opts);
-  if (s.ok()) {
-    s = dest_->Close(opts);
-  }
-  return s;
-}
-
-IOStatus BlogFileWriter::AddRecord(const WriteOptions& wo, BlogRecordType type,
-                                   const Slice& payload,
-                                   CompressionType comp_type,
-                                   uint64_t* payload_offset, bool force_full) {
-  assert(header_written_);
-
-  // Determine varint length of payload size
-  uint16_t varint_len = VarintLength(payload.size());
-
-  // Use compact format when:
-  // 1. varint fits in <= kBlogCompactVarintMaxBytes (payload < ~16 KiB)
-  // 2. record type matches the header's compact_record_type
-  // 3. not forced to full format (e.g. footer/preamble records)
-  bool use_compact =
-      !force_full && (varint_len <= kBlogCompactVarintMaxBytes) &&
-      (type == header_.compact_record_type) && (payload.size() > 0);
-
-  IOStatus s;
-  if (use_compact) {
-    s = EmitCompactRecord(wo, payload, comp_type, payload_offset);
-  } else {
-    s = EmitFullRecord(wo, type, payload, comp_type, payload_offset);
-  }
-
-  if (s.ok() && !manual_flush_) {
-    IOOptions opts;
-    s = WritableFileWriter::PrepareIOOptions(wo, opts);
-    if (s.ok()) {
-      s = dest_->Flush(opts);
-    }
-  }
-  return s;
-}
-
-IOStatus BlogFileWriter::EmitCompactRecord(const WriteOptions& wo,
-                                           const Slice& payload,
-                                           CompressionType comp_type,
-                                           uint64_t* payload_offset) {
-  // Compact format:
-  //   [escape_seq: 10B] [varint length: 1-3B]
-  //   [payload: length bytes]
-  //   [compression_type: 1B] [checksum: 4B]
-  //   [padding: 0+B]
-  assert(payload.size() > 0);  // length=0 compact records not expected
-
-  std::string prefix;
-  // Escape sequence
-  prefix.append(header_.escape_sequence, kBlogEscapeSequenceSize);
-  // Varint length
-  PutVarint64(&prefix, payload.size());
-
-  IOStatus s = EmitBytes(wo, prefix);
-  if (!s.ok()) {
-    return s;
-  }
-
-  // Record payload offset for caller
-  if (payload_offset) {
-    *payload_offset = offset_;
-  }
-
-  // Payload
-  s = EmitBytes(wo, payload);
-  if (!s.ok()) {
-    return s;
-  }
-
-  // 5-byte trailer: compression_type + checksum
-  char trailer[kBlogBlockTrailerSize];
-  trailer[0] = static_cast<char>(comp_type);
-  uint32_t checksum = ComputeBlogRecordChecksum(
-      header_.checksum_type, payload.data(), payload.size(), trailer[0],
-      header_.incarnation_id(),
-      offset_ - payload.size());  // record offset = start of payload
-  EncodeFixed32(trailer + 1, checksum);
-  s = EmitBytes(wo, trailer, kBlogBlockTrailerSize);
-  if (!s.ok()) {
-    return s;
-  }
-
-  // Padding (stack-allocated, no heap allocation)
-  uint8_t last_byte = static_cast<uint8_t>(trailer[kBlogBlockTrailerSize - 1]);
-  uint8_t pad_byte;
-  size_t pad_count;
-  ComputeBlogPaddingParams(last_byte, offset_, &pad_byte, &pad_count);
-  if (pad_count > 0) {
-    char pad_buf[4];
-    memset(pad_buf, pad_byte, pad_count);
-    s = EmitBytes(wo, pad_buf, pad_count);
-  }
-  return s;
-}
-
-IOStatus BlogFileWriter::EmitFullRecord(const WriteOptions& wo,
-                                        BlogRecordType type,
-                                        const Slice& payload,
-                                        CompressionType comp_type,
-                                        uint64_t* payload_offset) {
-  // Full format:
-  //   [escape_seq: 10B] [varint length: 4+B] [type: 1B] [compression_type: 1B]
-  //   [prefix_checksum: 4B]
-  //   (if length > 0):
-  //     [payload: length bytes]
-  //     [compression_type: 1B] [checksum: 4B]
-  //   [padding: 0+B]
-
-  std::string prefix;
-  // Escape sequence
-  prefix.append(header_.escape_sequence, kBlogEscapeSequenceSize);
-
-  // Varint length (must be > kBlogCompactVarintMaxBytes to trigger full format)
-  constexpr size_t kFullVarintMinBytes = kBlogCompactVarintMaxBytes + 1;
-  size_t varint_start = prefix.size();
-  if (payload.size() == 0) {
-    PutBlogIrregularVarint64(&prefix, 0, kFullVarintMinBytes);
-  } else {
-    uint16_t natural_len = VarintLength(payload.size());
-    if (natural_len >= kFullVarintMinBytes) {
-      PutVarint64(&prefix, payload.size());
-    } else {
-      PutBlogIrregularVarint64(&prefix, payload.size(), kFullVarintMinBytes);
-    }
-  }
-
-  // Type byte
-  prefix.push_back(static_cast<char>(type));
-  // Pre-payload compression type
-  prefix.push_back(static_cast<char>(comp_type));
-
-  // Prefix checksum covers (varint length + type + compression_type),
-  // with context modifier for defense-in-depth.
-  const char* prefix_data = prefix.data() + varint_start;
-  size_t prefix_data_size = prefix.size() - varint_start;
-  uint32_t prefix_checksum =
-      ComputeBuiltinChecksum(header_.checksum_type, prefix_data,
-                             prefix_data_size) +
-      ChecksumModifierForContext(header_.incarnation_id(), offset_);
-  PutFixed32(&prefix, prefix_checksum);
-
-  IOStatus s = EmitBytes(wo, prefix);
-  if (!s.ok()) {
-    return s;
-  }
-
-  // Track the last meaningful byte for padding computation.
-  // EncodeFixed32 is little-endian: last byte written = (value >> 24) & 0xFF.
-  uint8_t last_byte;
-
-  if (payload.size() > 0) {
-    // Record payload offset for caller
-    if (payload_offset) {
-      *payload_offset = offset_;
-    }
-
-    // Payload
-    s = EmitBytes(wo, payload);
-    if (!s.ok()) {
-      return s;
-    }
-
-    // 5-byte trailer: compression_type + checksum
-    char trailer[kBlogBlockTrailerSize];
-    trailer[0] = static_cast<char>(comp_type);
-    uint32_t checksum = ComputeBlogRecordChecksum(
-        header_.checksum_type, payload.data(), payload.size(), trailer[0],
-        header_.incarnation_id(), offset_ - payload.size());
-    EncodeFixed32(trailer + 1, checksum);
-    s = EmitBytes(wo, trailer, kBlogBlockTrailerSize);
-    if (!s.ok()) {
-      return s;
-    }
-
-    last_byte = static_cast<uint8_t>(trailer[kBlogBlockTrailerSize - 1]);
-  } else {
-    // No payload, no trailer. Last meaningful byte is the last byte of
-    // prefix_checksum (little-endian).
-    last_byte = static_cast<uint8_t>(prefix_checksum >> 24);
-  }
-
-  uint8_t pad_byte;
-  size_t pad_count;
-  ComputeBlogPaddingParams(last_byte, offset_, &pad_byte, &pad_count);
-  if (pad_count > 0) {
-    char pad_buf[4];
-    memset(pad_buf, pad_byte, pad_count);
-    s = EmitBytes(wo, pad_buf, pad_count);
-  }
-  return s;
-}
-
-IOStatus BlogFileWriter::EmitBytes(const WriteOptions& wo, const Slice& data) {
-  return EmitBytes(wo, data.data(), data.size());
-}
-
-IOStatus BlogFileWriter::EmitBytes(const WriteOptions& wo, const char* data,
-                                   size_t len) {
-  IOOptions opts;
-  IOStatus s = WritableFileWriter::PrepareIOOptions(wo, opts);
-  if (!s.ok()) {
-    return s;
-  }
-  s = dest_->Append(opts, Slice(data, len));
-  if (s.ok()) {
-    offset_ += len;
-  }
-  return s;
-}
-
-}  // namespace ROCKSDB_NAMESPACE

db/blog/blog_writer.h

@@ -1,95 +0,0 @@
-//  Copyright (c) Meta Platforms, Inc. and affiliates.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <cstdint>
-#include <memory>
-#include <string>
-
-#include "db/blog/blog_format.h"
-#include "file/writable_file_writer.h"
-#include "rocksdb/io_status.h"
-#include "rocksdb/options.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-// Writes blog-format files (WAL or blob). The caller constructs a
-// BlogFileHeader, calls WriteHeader(), then adds records via
-// AddBlobRecord/AddWriteBatchRecord, and optionally appends footer records
-// before closing.
-//
-// Format selection (compact vs full) is automatic based on payload size and
-// record type. See blog_format.h for the wire format details.
-class BlogFileWriter {
- public:
-  BlogFileWriter(std::unique_ptr<WritableFileWriter>&& dest,
-                 const BlogFileHeader& header, bool manual_flush = false);
-  ~BlogFileWriter();
-
-  // Write the file header. Must be called exactly once, before any records.
-  IOStatus WriteHeader(const WriteOptions& wo);
-
-  // Write a blob record. Selects compact or full format automatically.
-  // On success, *blob_offset is the file offset of the payload start.
-  IOStatus AddBlobRecord(const WriteOptions& wo, const Slice& payload,
-                         CompressionType comp_type, uint64_t* blob_offset);
-
-  // Write a WriteBatch record. Selects compact or full format automatically.
-  IOStatus AddWriteBatchRecord(const WriteOptions& wo, const Slice& wb_data,
-                               CompressionType comp_type = kNoCompression);
-
-  // Write a preamble-start record (stub). Uses full format with length=0.
-  IOStatus AddPreambleStartRecord(const WriteOptions& wo);
-
-  // Write a footer index record (full format).
-  IOStatus AddFooterIndexRecord(const WriteOptions& wo,
-                                const Slice& index_data);
-
-  // Write a footer properties record (full format).
-  IOStatus AddFooterPropertiesRecord(const WriteOptions& wo,
-                                     const BlogFileFooterProperties& props);
-
-  // Write a footer locator record (full format). Must be the last record.
-  IOStatus AddFooterLocatorRecord(const WriteOptions& wo,
-                                  const BlogFileFooterLocator& locator);
-
-  IOStatus WriteBuffer(const WriteOptions& wo);
-  IOStatus Sync(const WriteOptions& wo, bool use_fsync = false);
-  IOStatus Close(const WriteOptions& wo);
-
-  uint64_t current_offset() const { return offset_; }
-  WritableFileWriter* file() { return dest_.get(); }
-  const BlogFileHeader& header() const { return header_; }
-
- private:
-  // Write a record. Chooses compact format when varint fits in <= 3 bytes
-  // and type matches compact_record_type, unless force_full is true.
-  IOStatus AddRecord(const WriteOptions& wo, BlogRecordType type,
-                     const Slice& payload, CompressionType comp_type,
-                     uint64_t* payload_offset, bool force_full = false);
-
-  // Emit a compact-format record.
-  IOStatus EmitCompactRecord(const WriteOptions& wo, const Slice& payload,
-                             CompressionType comp_type,
-                             uint64_t* payload_offset);
-
-  // Emit a full-format record.
-  IOStatus EmitFullRecord(const WriteOptions& wo, BlogRecordType type,
-                          const Slice& payload, CompressionType comp_type,
-                          uint64_t* payload_offset);
-
-  // Append raw bytes to the file and advance offset_.
-  IOStatus EmitBytes(const WriteOptions& wo, const Slice& data);
-  IOStatus EmitBytes(const WriteOptions& wo, const char* data, size_t len);
-
-  std::unique_ptr<WritableFileWriter> dest_;
-  BlogFileHeader header_;
-  uint64_t offset_ = 0;
-  bool manual_flush_;
-  bool header_written_ = false;
-};
-
-}  // namespace ROCKSDB_NAMESPACE

db/blog/blog_writer_test.cc

@@ -1,452 +0,0 @@
-//  Copyright (c) Meta Platforms, Inc. and affiliates.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blog/blog_writer.h"
-
-#include <string>
-#include <vector>
-
-#include "db/blog/blog_reader.h"
-#include "file/sequence_file_reader.h"
-#include "file/writable_file_writer.h"
-#include "rocksdb/write_buffer_manager.h"
-#include "test_util/testharness.h"
-#include "test_util/testutil.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class BlogWriterTest : public testing::Test {
- protected:
-  // Shared contents buffer: StringSink writes here, StringSource reads from it.
-  Slice reader_contents_;
-  test::StringSink* sink_;
-
-  BlogWriterTest() : sink_(new test::StringSink(&reader_contents_)) {}
-
-  // Create a writer with the given header.
-  std::unique_ptr<BlogFileWriter> MakeWriter(const BlogFileHeader& header) {
-    std::unique_ptr<FSWritableFile> sink_holder(sink_);
-    std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
-        std::move(sink_holder), "" /* file name */, FileOptions()));
-    return std::make_unique<BlogFileWriter>(std::move(file_writer), header);
-  }
-
-  std::unique_ptr<BlogFileReader> MakeReaderWithReporter(
-      BlogFileReader::Reporter* r) {
-    return MakeReaderImpl(r);
-  }
-
-  // Create a reader over the data written so far.
-  std::unique_ptr<BlogFileReader> MakeReader() {
-    return MakeReaderImpl(nullptr);
-  }
-
-  std::unique_ptr<BlogFileReader> MakeReaderImpl(
-      BlogFileReader::Reporter* reporter) {
-    // Create an FSSequentialFile that reads from reader_contents_
-    class MemSequentialFile : public FSSequentialFile {
-     public:
-      Slice& contents_;
-      size_t pos_ = 0;
-      explicit MemSequentialFile(Slice& contents) : contents_(contents) {}
-      IOStatus Read(size_t n, const IOOptions& /*opts*/, Slice* result,
-                    char* scratch, IODebugContext* /*dbg*/) override {
-        size_t avail = contents_.size() - pos_;
-        if (n > avail) {
-          n = avail;
-        }
-        if (n > 0) {
-          memcpy(scratch, contents_.data() + pos_, n);
-          pos_ += n;
-        }
-        *result = Slice(scratch, n);
-        return IOStatus::OK();
-      }
-      IOStatus Skip(uint64_t n) override {
-        pos_ += static_cast<size_t>(n);
-        return IOStatus::OK();
-      }
-    };
-
-    auto* source = new MemSequentialFile(reader_contents_);
-    std::unique_ptr<FSSequentialFile> source_holder(source);
-    std::unique_ptr<SequentialFileReader> file_reader(
-        new SequentialFileReader(std::move(source_holder), "" /* file name */));
-    return std::make_unique<BlogFileReader>(std::move(file_reader), reporter,
-                                            true /* verify_checksums */);
-  }
-
-  // Helper to create a default header for blob files.
-  BlogFileHeader MakeBlobHeader() {
-    BlogFileHeader header;
-    header.checksum_type = kXXH3;
-    header.compact_record_type = kBlogBlobRecord;
-    header.GenerateRandomFields();
-    return header;
-  }
-
-  // Helper to create a default header for WAL files.
-  BlogFileHeader MakeWalHeader() {
-    BlogFileHeader header;
-    header.checksum_type = kXXH3;
-    header.compact_record_type = kBlogWriteBatchRecord;
-    header.GenerateRandomFields();
-    header.SetProperty("role", "wal");
-    return header;
-  }
-};
-
-TEST_F(BlogWriterTest, WriteAndReadHeader) {
-  auto header = MakeBlobHeader();
-  auto writer = MakeWriter(header);
-
-  WriteOptions wo;
-  ASSERT_OK(writer->WriteHeader(wo));
-
-  auto reader = MakeReader();
-  BlogFileHeader read_header;
-  ASSERT_OK(reader->ReadHeader(&read_header));
-
-  ASSERT_EQ(read_header.schema_version, header.schema_version);
-  ASSERT_EQ(read_header.checksum_type, header.checksum_type);
-  ASSERT_EQ(read_header.compact_record_type, header.compact_record_type);
-  ASSERT_EQ(memcmp(read_header.escape_sequence, header.escape_sequence,
-                   kBlogEscapeSequenceSize),
-            0);
-}
-
-TEST_F(BlogWriterTest, WriteSingleBlobRecord) {
-  auto header = MakeBlobHeader();
-  auto writer = MakeWriter(header);
-
-  WriteOptions wo;
-  ASSERT_OK(writer->WriteHeader(wo));
-
-  std::string blob_data(100, 'A');
-  uint64_t blob_offset = 0;
-  ASSERT_OK(writer->AddBlobRecord(wo, blob_data, kNoCompression, &blob_offset));
-  ASSERT_GT(blob_offset, 0u);
-
-  // Verify alignment: offset after record should be 4-byte aligned
-  ASSERT_EQ(writer->current_offset() % 4, 0u);
-
-  // Read it back
-  auto reader = MakeReader();
-  BlogFileHeader read_header;
-  ASSERT_OK(reader->ReadHeader(&read_header));
-
-  BlogRecordType type;
-  Slice payload;
-  std::string scratch;
-  uint64_t rec_offset;
-  ASSERT_OK(reader->ReadRecord(&type, &payload, &scratch, &rec_offset));
-  ASSERT_EQ(type, kBlogBlobRecord);
-  ASSERT_EQ(payload.size(), 100u);
-  ASSERT_EQ(payload.ToString(), blob_data);
-}
-
-TEST_F(BlogWriterTest, WriteMultipleBlobRecords) {
-  auto header = MakeBlobHeader();
-  auto writer = MakeWriter(header);
-
-  WriteOptions wo;
-  ASSERT_OK(writer->WriteHeader(wo));
-
-  std::vector<std::string> blobs = {"hello", std::string(1000, 'X'), "short",
-                                    std::string(500, 'Y')};
-  for (const auto& blob : blobs) {
-    uint64_t offset;
-    ASSERT_OK(writer->AddBlobRecord(wo, blob, kNoCompression, &offset));
-  }
-
-  auto reader = MakeReader();
-  BlogFileHeader rh;
-  ASSERT_OK(reader->ReadHeader(&rh));
-
-  for (const auto& expected : blobs) {
-    BlogRecordType type;
-    Slice payload;
-    std::string scratch;
-    ASSERT_OK(reader->ReadRecord(&type, &payload, &scratch));
-    ASSERT_EQ(type, kBlogBlobRecord);
-    ASSERT_EQ(payload.ToString(), expected);
-  }
-
-  // No more records
-  BlogRecordType type;
-  Slice payload;
-  std::string scratch;
-  ASSERT_TRUE(reader->ReadRecord(&type, &payload, &scratch).IsNotFound());
-}
-
-TEST_F(BlogWriterTest, WriteWriteBatchRecord) {
-  auto header = MakeWalHeader();
-  auto writer = MakeWriter(header);
-
-  WriteOptions wo;
-  ASSERT_OK(writer->WriteHeader(wo));
-
-  std::string wb_data(200, 'W');
-  ASSERT_OK(writer->AddWriteBatchRecord(wo, wb_data));
-
-  auto reader = MakeReader();
-  BlogFileHeader rh;
-  ASSERT_OK(reader->ReadHeader(&rh));
-
-  BlogRecordType type;
-  Slice payload;
-  std::string scratch;
-  ASSERT_OK(reader->ReadRecord(&type, &payload, &scratch));
-  ASSERT_EQ(type, kBlogWriteBatchRecord);
-  ASSERT_EQ(payload.ToString(), wb_data);
-}
-
-TEST_F(BlogWriterTest, CompactVsFullFormat) {
-  auto header = MakeBlobHeader();
-  auto writer = MakeWriter(header);
-
-  WriteOptions wo;
-  ASSERT_OK(writer->WriteHeader(wo));
-
-  // Small blob (< 2 MiB) should use compact format (kBlogBlobRecord matches
-  // compact_record_type)
-  std::string small_blob(100, 'S');
-  uint64_t offset1;
-  ASSERT_OK(writer->AddBlobRecord(wo, small_blob, kNoCompression, &offset1));
-
-  // Large blob (> 2 MiB) should use full format
-  std::string large_blob(3 * 1024 * 1024, 'L');
-  uint64_t offset2;
-  ASSERT_OK(writer->AddBlobRecord(wo, large_blob, kNoCompression, &offset2));
-
-  // Read both back
-  auto reader = MakeReader();
-  BlogFileHeader rh;
-  ASSERT_OK(reader->ReadHeader(&rh));
-
-  BlogRecordType type;
-  Slice payload;
-  std::string scratch;
-
-  ASSERT_OK(reader->ReadRecord(&type, &payload, &scratch));
-  ASSERT_EQ(type, kBlogBlobRecord);
-  ASSERT_EQ(payload.size(), small_blob.size());
-
-  ASSERT_OK(reader->ReadRecord(&type, &payload, &scratch));
-  ASSERT_EQ(type, kBlogBlobRecord);
-  ASSERT_EQ(payload.size(), large_blob.size());
-  ASSERT_EQ(payload, Slice(large_blob));
-}
-
-TEST_F(BlogWriterTest, PreambleStartRecord) {
-  auto header = MakeWalHeader();
-  auto writer = MakeWriter(header);
-
-  WriteOptions wo;
-  ASSERT_OK(writer->WriteHeader(wo));
-  uint64_t pre_offset = writer->current_offset();
-  ASSERT_EQ(pre_offset % 4, 0u) << "Pre-record offset not aligned";
-  ASSERT_OK(writer->AddPreambleStartRecord(wo));
-  uint64_t post_offset = writer->current_offset();
-  ASSERT_EQ(post_offset % 4, 0u) << "Post-record offset not aligned";
-
-  // Use a reporter to capture errors
-  class TestReporter : public BlogFileReader::Reporter {
-   public:
-    std::string last_msg;
-    void Corruption(size_t /*bytes*/, const Status& s) override {
-      last_msg = s.ToString();
-    }
-  };
-  TestReporter reporter;
-
-  auto reader = MakeReaderWithReporter(&reporter);
-  BlogFileHeader rh;
-  ASSERT_OK(reader->ReadHeader(&rh));
-
-  BlogRecordType type;
-  Slice payload;
-  std::string scratch;
-  Status rs = reader->ReadRecord(&type, &payload, &scratch);
-  if (!rs.ok()) {
-    fprintf(stderr, "ReadRecord failed: %s, Reporter: %s, EOF=%d\n",
-            rs.ToString().c_str(), reporter.last_msg.c_str(), reader->IsEOF());
-  }
-  ASSERT_OK(rs);
-  ASSERT_EQ(type, kBlogPreambleStartRecord);
-  ASSERT_TRUE(payload.empty());
-}
-
-TEST_F(BlogWriterTest, FooterRecords) {
-  auto header = MakeBlobHeader();
-  auto writer = MakeWriter(header);
-
-  WriteOptions wo;
-  ASSERT_OK(writer->WriteHeader(wo));
-
-  // Write a blob record first
-  std::string blob_data(50, 'B');
-  uint64_t blob_offset;
-  ASSERT_OK(writer->AddBlobRecord(wo, blob_data, kNoCompression, &blob_offset));
-
-  // Write footer properties
-  uint64_t props_offset = writer->current_offset();
-  BlogFileFooterProperties props;
-  props.SetBlobCount(1);
-  props.SetTotalBlobBytes(50);
-  ASSERT_OK(writer->AddFooterPropertiesRecord(wo, props));
-
-  // Write footer locator
-  BlogFileFooterLocator locator;
-  uint64_t locator_offset = writer->current_offset();
-  locator.entries.push_back(
-      {kBlogFooterPropertiesRecord,
-       static_cast<uint32_t>((locator_offset - props_offset) / 4)});
-  ASSERT_OK(writer->AddFooterLocatorRecord(wo, locator));
-
-  // Read all records
-  auto reader = MakeReader();
-  BlogFileHeader rh;
-  ASSERT_OK(reader->ReadHeader(&rh));
-
-  BlogRecordType type;
-  Slice payload;
-  std::string scratch;
-
-  // Blob record
-  ASSERT_OK(reader->ReadRecord(&type, &payload, &scratch));
-  ASSERT_EQ(type, kBlogBlobRecord);
-
-  // Footer properties record
-  ASSERT_OK(reader->ReadRecord(&type, &payload, &scratch));
-  ASSERT_EQ(type, kBlogFooterPropertiesRecord);
-  BlogFileFooterProperties read_props;
-  ASSERT_OK(read_props.DecodeFrom(payload));
-  ASSERT_EQ(read_props.properties.size(), 2u);
-
-  // Footer locator record
-  ASSERT_OK(reader->ReadRecord(&type, &payload, &scratch));
-  ASSERT_EQ(type, kBlogFooterLocatorRecord);
-  BlogFileFooterLocator read_locator;
-  ASSERT_OK(read_locator.DecodeFrom(payload));
-  ASSERT_EQ(read_locator.entries.size(), 1u);
-  ASSERT_EQ(read_locator.entries[0].record_type, kBlogFooterPropertiesRecord);
-}
-
-TEST_F(BlogWriterTest, ChecksumCorruptionDetected) {
-  auto header = MakeBlobHeader();
-  auto writer = MakeWriter(header);
-
-  WriteOptions wo;
-  ASSERT_OK(writer->WriteHeader(wo));
-
-  std::string blob_data(100, 'C');
-  uint64_t blob_offset;
-  ASSERT_OK(writer->AddBlobRecord(wo, blob_data, kNoCompression, &blob_offset));
-
-  // Corrupt one byte in the blob payload
-  std::string& contents = sink_->contents_;
-  ASSERT_GT(contents.size(), blob_offset + 10);
-  contents[blob_offset + 10] ^= 0xFF;
-
-  // Reading should detect the corruption
-  auto reader = MakeReader();
-  BlogFileHeader rh;
-  ASSERT_OK(reader->ReadHeader(&rh));
-
-  BlogRecordType type;
-  Slice payload;
-  std::string scratch;
-  ASSERT_TRUE(reader->ReadRecord(&type, &payload, &scratch).IsCorruption());
-}
-
-TEST_F(BlogWriterTest, MixedRecordTypes) {
-  // Use blob compact type but also write WriteBatch records (full format)
-  auto header = MakeBlobHeader();
-  auto writer = MakeWriter(header);
-
-  WriteOptions wo;
-  ASSERT_OK(writer->WriteHeader(wo));
-
-  // Blob (compact format)
-  std::string blob(50, 'B');
-  uint64_t blob_offset;
-  ASSERT_OK(writer->AddBlobRecord(wo, blob, kNoCompression, &blob_offset));
-
-  // WriteBatch (full format since it doesn't match compact_record_type)
-  std::string wb(60, 'W');
-  ASSERT_OK(writer->AddWriteBatchRecord(wo, wb));
-
-  // Another blob (compact)
-  std::string blob2(70, 'D');
-  uint64_t blob2_offset;
-  ASSERT_OK(writer->AddBlobRecord(wo, blob2, kNoCompression, &blob2_offset));
-
-  // Read all back
-  auto reader = MakeReader();
-  BlogFileHeader rh;
-  ASSERT_OK(reader->ReadHeader(&rh));
-
-  BlogRecordType type;
-  Slice payload;
-  std::string scratch;
-
-  ASSERT_OK(reader->ReadRecord(&type, &payload, &scratch));
-  ASSERT_EQ(type, kBlogBlobRecord);
-  ASSERT_EQ(payload.size(), 50u);
-
-  ASSERT_OK(reader->ReadRecord(&type, &payload, &scratch));
-  ASSERT_EQ(type, kBlogWriteBatchRecord);
-  ASSERT_EQ(payload.size(), 60u);
-
-  ASSERT_OK(reader->ReadRecord(&type, &payload, &scratch));
-  ASSERT_EQ(type, kBlogBlobRecord);
-  ASSERT_EQ(payload.size(), 70u);
-
-  ASSERT_TRUE(reader->ReadRecord(&type, &payload, &scratch).IsNotFound());
-}
-
-TEST_F(BlogWriterTest, AlignmentInvariant) {
-  auto header = MakeBlobHeader();
-  auto writer = MakeWriter(header);
-
-  WriteOptions wo;
-  ASSERT_OK(writer->WriteHeader(wo));
-
-  // Write records of various sizes and verify alignment is maintained
-  for (int size = 1; size <= 200; ++size) {
-    std::string data(size, static_cast<char>(size & 0xFF));
-    uint64_t offset;
-    ASSERT_OK(writer->AddBlobRecord(wo, data, kNoCompression, &offset));
-    ASSERT_EQ(writer->current_offset() % 4, 0u)
-        << "Alignment violated after record of size " << size;
-  }
-}
-
-TEST_F(BlogWriterTest, HeaderWithProperties) {
-  auto header = MakeBlobHeader();
-  header.SetProperty("CompressionCompatibilityName", "zstd");
-  header.SetProperty("role", "blob");
-  header.SetProperty("compressionSettings", "level=3");
-  auto writer = MakeWriter(header);
-
-  WriteOptions wo;
-  ASSERT_OK(writer->WriteHeader(wo));
-
-  auto reader = MakeReader();
-  BlogFileHeader rh;
-  ASSERT_OK(reader->ReadHeader(&rh));
-  ASSERT_EQ(rh.GetProperty("CompressionCompatibilityName"), "zstd");
-  ASSERT_EQ(rh.GetProperty("role"), "blob");
-  ASSERT_EQ(rh.GetProperty("compressionSettings"), "level=3");
-}
-
-}  // namespace ROCKSDB_NAMESPACE
-
-int main(int argc, char** argv) {
-  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}

db/c.cc

@@ -3771,6 +3771,11 @@ void rocksdb_block_based_options_set_index_block_search_type(
       static_cast<BlockBasedTableOptions::BlockSearchType>(v);
 }
 
+void rocksdb_block_based_options_set_uniform_cv_threshold(
+    rocksdb_block_based_table_options_t* options, double v) {
+  options->rep.uniform_cv_threshold = v;
+}
+
 void rocksdb_block_based_options_set_data_block_hash_ratio(
     rocksdb_block_based_table_options_t* options, double v) {
   options->rep.data_block_hash_table_util_ratio = v;
@@ -5286,6 +5291,15 @@ size_t rocksdb_options_get_recycle_log_file_num(rocksdb_options_t* opt) {
   return opt->rep.recycle_log_file_num;
 }
 
+void rocksdb_options_set_async_wal_precreate(rocksdb_options_t* opt,
+                                             unsigned char v) {
+  opt->rep.async_wal_precreate = v;
+}
+
+unsigned char rocksdb_options_get_async_wal_precreate(rocksdb_options_t* opt) {
+  return opt->rep.async_wal_precreate;
+}
+
 void rocksdb_options_set_soft_pending_compaction_bytes_limit(
     rocksdb_options_t* opt, size_t v) {
   opt->rep.soft_pending_compaction_bytes_limit = v;
@@ -5389,6 +5403,16 @@ size_t rocksdb_options_get_memtable_huge_page_size(rocksdb_options_t* opt) {
   return opt->rep.memtable_huge_page_size;
 }
 
+void rocksdb_options_set_memtable_batch_lookup_optimization(
+    rocksdb_options_t* opt, unsigned char v) {
+  opt->rep.memtable_batch_lookup_optimization = v;
+}
+
+unsigned char rocksdb_options_get_memtable_batch_lookup_optimization(
+    rocksdb_options_t* opt) {
+  return opt->rep.memtable_batch_lookup_optimization;
+}
+
 void rocksdb_options_set_hash_skip_list_rep(rocksdb_options_t* opt,
                                             size_t bucket_count,
                                             int32_t skiplist_height,
@@ -5820,6 +5844,8 @@ uint64_t rocksdb_perfcontext_metric(rocksdb_perfcontext_t* context,
       return rep->number_async_seek;
     case rocksdb_blob_cache_hit_count:
       return rep->blob_cache_hit_count;
+    case rocksdb_blob_cache_read_byte:
+      return rep->blob_cache_read_byte;
     case rocksdb_blob_read_count:
       return rep->blob_read_count;
     case rocksdb_blob_read_byte:
@@ -6238,6 +6264,16 @@ unsigned char rocksdb_readoptions_get_async_io(rocksdb_readoptions_t* opt) {
   return opt->rep.async_io;
 }
 
+void rocksdb_readoptions_set_optimize_multiget_for_io(
+    rocksdb_readoptions_t* opt, unsigned char v) {
+  opt->rep.optimize_multiget_for_io = v;
+}
+
+unsigned char rocksdb_readoptions_get_optimize_multiget_for_io(
+    rocksdb_readoptions_t* opt) {
+  return opt->rep.optimize_multiget_for_io;
+}
+
 void rocksdb_readoptions_set_timestamp(rocksdb_readoptions_t* opt,
                                        const char* ts, size_t tslen) {
   if (ts == nullptr) {

db/c_test.c

@@ -888,6 +888,9 @@ int main(int argc, char** argv) {
   rocksdb_block_based_options_set_block_cache(table_options, cache);
   rocksdb_block_based_options_set_data_block_index_type(table_options, 1);
   rocksdb_block_based_options_set_data_block_hash_ratio(table_options, 0.75);
+  rocksdb_block_based_options_set_index_block_search_type(
+      table_options, rocksdb_block_based_table_index_block_search_type_auto);
+  rocksdb_block_based_options_set_uniform_cv_threshold(table_options, 0.2);
   rocksdb_block_based_options_set_top_level_index_pinning_tier(table_options,
                                                                1);
   rocksdb_block_based_options_set_partition_pinning_tier(table_options, 2);
@@ -2812,6 +2815,15 @@ int main(int argc, char** argv) {
     rocksdb_options_set_memtable_huge_page_size(o, 25);
     CheckCondition(25 == rocksdb_options_get_memtable_huge_page_size(o));
 
+    // memtable_batch_lookup_optimization defaults to 0; flip to 1 to verify
+    // the setter is wired through to the underlying C++ ColumnFamilyOptions
+    // field.
+    CheckCondition(0 ==
+                   rocksdb_options_get_memtable_batch_lookup_optimization(o));
+    rocksdb_options_set_memtable_batch_lookup_optimization(o, 1);
+    CheckCondition(1 ==
+                   rocksdb_options_get_memtable_batch_lookup_optimization(o));
+
     rocksdb_options_set_max_successive_merges(o, 26);
     CheckCondition(26 == rocksdb_options_get_max_successive_merges(o));
 
@@ -2869,6 +2881,9 @@ int main(int argc, char** argv) {
     rocksdb_options_set_track_and_verify_wals_in_manifest(o, 42);
     CheckCondition(1 ==
                    rocksdb_options_get_track_and_verify_wals_in_manifest(o));
+    CheckCondition(0 == rocksdb_options_get_async_wal_precreate(o));
+    rocksdb_options_set_async_wal_precreate(o, 1);
+    CheckCondition(1 == rocksdb_options_get_async_wal_precreate(o));
 
     /* Blob Options */
     rocksdb_options_set_enable_blob_files(o, 1);
@@ -2995,6 +3010,8 @@ int main(int argc, char** argv) {
                    rocksdb_options_get_memtable_prefix_bloom_size_ratio(copy));
     CheckCondition(24 == rocksdb_options_get_max_compaction_bytes(copy));
     CheckCondition(25 == rocksdb_options_get_memtable_huge_page_size(copy));
+    CheckCondition(
+        1 == rocksdb_options_get_memtable_batch_lookup_optimization(copy));
     CheckCondition(26 == rocksdb_options_get_max_successive_merges(copy));
     CheckCondition(27 == rocksdb_options_get_bloom_locality(copy));
     CheckCondition(1 == rocksdb_options_get_inplace_update_support(copy));
@@ -3429,6 +3446,12 @@ int main(int argc, char** argv) {
     rocksdb_readoptions_set_async_io(ro, 1);
     CheckCondition(1 == rocksdb_readoptions_get_async_io(ro));
 
+    // optimize_multiget_for_io defaults to true (1); flip to 0 to verify the
+    // setter is wired through to the underlying C++ ReadOptions field.
+    CheckCondition(1 == rocksdb_readoptions_get_optimize_multiget_for_io(ro));
+    rocksdb_readoptions_set_optimize_multiget_for_io(ro, 0);
+    CheckCondition(0 == rocksdb_readoptions_get_optimize_multiget_for_io(ro));
+
     rocksdb_readoptions_destroy(ro);
   }
 

db/column_family_test.cc

@@ -734,6 +734,42 @@ TEST_P(ColumnFamilyTest, AddDrop) {
               std::vector<std::string>({"default", "four", "three"}));
 }
 
+TEST_P(ColumnFamilyTest, EmptyNameRejected) {
+  // Creating a column family with an empty-string name should be rejected with
+  // InvalidArgument. Empty strings are reserved by various RocksDB APIs and
+  // serialization formats to mean "unknown/unspecified column family" (e.g.
+  // TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, table
+  // properties).
+  //
+  // Prior behavior was silently broken: CreateColumnFamily("") would return
+  // OK and a usable handle, but the CF was not persisted in the manifest --
+  // data written to it was lost on DB reopen.
+  Open();
+  ColumnFamilyHandle* handle = nullptr;
+  Status s =
+      db_->CreateColumnFamily(column_family_options_, std::string(), &handle);
+  ASSERT_TRUE(s.IsInvalidArgument()) << s.ToString();
+  ASSERT_EQ(handle, nullptr);
+
+  // Same via the bulk-by-names API. The first valid name should still get
+  // created; the failure should come on the empty one.
+  std::vector<ColumnFamilyHandle*> handles;
+  s = db_->CreateColumnFamilies(column_family_options_,
+                                {"ok_name", "", "another"}, &handles);
+  ASSERT_TRUE(s.IsInvalidArgument()) << s.ToString();
+  // Clean up the partially-created handle ("ok_name") before the failure.
+  ASSERT_EQ(handles.size(), 1U);
+  ASSERT_OK(db_->DropColumnFamily(handles[0]));
+  ASSERT_OK(db_->DestroyColumnFamilyHandle(handles[0]));
+  handles.clear();
+
+  // Same via the bulk-by-descriptor API.
+  s = db_->CreateColumnFamilies(
+      {ColumnFamilyDescriptor("", column_family_options_)}, &handles);
+  ASSERT_TRUE(s.IsInvalidArgument()) << s.ToString();
+  ASSERT_TRUE(handles.empty());
+}
+
 TEST_P(ColumnFamilyTest, BulkAddDrop) {
   constexpr int kNumCF = 1000;
   ColumnFamilyOptions cf_options;

db/compaction/compaction.h

@@ -454,6 +454,18 @@ class Compaction {
     return notify_on_compaction_completion_;
   }
 
+  // Called by DBImpl::NotifyOnCompactionPreCommit to ensure the
+  // OnCompactionPreCommit listener callback fires at most once per
+  // compaction lifecycle, even though the notify helper is invoked at multiple
+  // potential ReleaseCompactionFiles() sites for safety.
+  void SetNotifyOnCompactionPreCommitCalled() {
+    notify_on_compaction_pre_commit_called_ = true;
+  }
+
+  bool WasNotifyOnCompactionPreCommitCalled() const {
+    return notify_on_compaction_pre_commit_called_;
+  }
+
   static constexpr int kInvalidLevel = -1;
 
   // Evaluate proximal output level. If the compaction supports
@@ -626,6 +638,12 @@ class Compaction {
   // begin.
   bool notify_on_compaction_completion_;
 
+  // Tracks whether OnCompactionPreCommit has already fired for this
+  // compaction. The notify helper is called from multiple potential
+  // ReleaseCompactionFiles() call sites; this flag guarantees that listeners
+  // observe the callback exactly once per compaction.
+  bool notify_on_compaction_pre_commit_called_ = false;
+
   // Enable/disable GC collection for blobs during compaction.
   bool enable_blob_garbage_collection_;
 

db/compaction/compaction_iterator.h

@@ -75,7 +75,7 @@ class CompactionBlobResolver : public WideColumnBlobResolver {
   bool track_resolve_error_ = false;
 
   // Cache for resolved blob values to avoid re-fetching. Uses a vector of
-  // (column_index, PinnableSlice) pairs — typical entities have few blob
+  // (column_index, PinnableSlice) pairs -- typical entities have few blob
   // columns (<5), making linear scan cheaper than hash map overhead.
   std::vector<std::pair<size_t, std::unique_ptr<PinnableSlice>>>
       resolved_cache_;

db/compaction/compaction_job.cc

@@ -2065,10 +2065,18 @@ Status CompactionJob::FinishCompactionOutputFile(
     std::pair<SequenceNumber, SequenceNumber> keep_seqno_range{
         0, kMaxSequenceNumber};
     if (sub_compact->compaction->SupportsPerKeyPlacement()) {
+      // Point entries are routed to proximal output only when their seqno is
+      // strictly greater than `proximal_after_seqno_`. Range tombstones use a
+      // [lower, upper) filter, so split them at the next seqno to preserve the
+      // same boundary.
+      SequenceNumber range_del_split_seqno = proximal_after_seqno_;
+      if (range_del_split_seqno < kMaxSequenceNumber) {
+        range_del_split_seqno++;
+      }
       if (outputs.IsProximalLevel()) {
-        keep_seqno_range.first = proximal_after_seqno_;
+        keep_seqno_range.first = range_del_split_seqno;
       } else {
-        keep_seqno_range.second = proximal_after_seqno_;
+        keep_seqno_range.second = range_del_split_seqno;
       }
     }
     CompactionIterationStats range_del_out_stats;

db/compaction/compaction_picker_fifo.cc

@@ -251,7 +251,7 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction(
     // When using blob-aware sizing, use proportional estimation (same
     // principle as EstimateTotalDataForSST): each SST "owns"
     // effective_size / num_files of total data. This is an approximation
-    // — individual SSTs may reference different amounts of blob data,
+    // -- individual SSTs may reference different amounts of blob data,
     // but uniform distribution is a reasonable estimate for FIFO dropping.
     uint64_t remaining_size = effective_size;
     const uint64_t num_files = last_level_files.size();
@@ -477,12 +477,12 @@ Compaction* FIFOCompactionPicker::PickIntraL0Compaction(
     if (fifo_opts.max_data_files_size == 0) {
       ROCKS_LOG_BUFFER(
           log_buffer,
-          "[%s] FIFO kv-ratio compaction: skipping — "
+          "[%s] FIFO kv-ratio compaction: skipping -- "
           "max_data_files_size is 0, cannot compute target file size. ",
           cf_name.c_str());
     } else if (fifo_opts.max_data_files_size < fifo_opts.max_table_files_size) {
       ROCKS_LOG_BUFFER(log_buffer,
-                       "[%s] FIFO kv-ratio compaction: skipping — "
+                       "[%s] FIFO kv-ratio compaction: skipping -- "
                        "max_data_files_size (%" PRIu64
                        ") < max_table_files_size "
                        "(%" PRIu64 ").",
@@ -547,7 +547,7 @@ Compaction* FIFOCompactionPicker::PickRatioBasedIntraL0Compaction(
   for (int level = 1; level < vstorage->num_levels(); ++level) {
     if (!vstorage->LevelFiles(level).empty()) {
       ROCKS_LOG_BUFFER(log_buffer,
-                       "[%s] FIFO kv-ratio compaction: skipping — non-L0 "
+                       "[%s] FIFO kv-ratio compaction: skipping -- non-L0 "
                        "level %d still has %" ROCKSDB_PRIszt
                        " files (migration in progress)",
                        cf_name.c_str(), level,
@@ -587,7 +587,7 @@ Compaction* FIFOCompactionPicker::PickRatioBasedIntraL0Compaction(
   // once per flush or compaction completion, so no caching is needed.
   uint64_t target = 0;
   if (mutable_cf_options.max_compaction_bytes > 0) {
-    // User explicitly set max_compaction_bytes — use it as target
+    // User explicitly set max_compaction_bytes -- use it as target
     target = mutable_cf_options.max_compaction_bytes;
   } else {
     // Auto-calculate from capacity and observed SST/blob ratio
@@ -642,7 +642,7 @@ Compaction* FIFOCompactionPicker::PickRatioBasedIntraL0Compaction(
   //   of linear) write amplification.
 
   // Build tier boundaries from smallest to largest.
-  // Stop at 10KB minimum — SST files of most workloads are larger than
+  // Stop at 10KB minimum -- SST files of most workloads are larger than
   // this, so lower boundaries would only waste CPU scanning L0 files.
   // Files smaller than the lowest boundary simply merge at that boundary.
   static constexpr uint64_t kMinTierBoundary = 10 * 1024;  // 10KB
@@ -651,7 +651,7 @@ Compaction* FIFOCompactionPicker::PickRatioBasedIntraL0Compaction(
     boundaries.push_back(b);
   }
   if (boundaries.empty()) {
-    // target itself is below kMinTierBoundary — use target as the
+    // target itself is below kMinTierBoundary -- use target as the
     // sole boundary so we can still compact at the target size.
     boundaries.push_back(target);
   }
@@ -668,7 +668,7 @@ Compaction* FIFOCompactionPicker::PickRatioBasedIntraL0Compaction(
         continue;
       }
 
-      // Found a file < boundary — collect contiguous batch
+      // Found a file < boundary -- collect contiguous batch
       std::vector<FileMetaData*> batch;
       uint64_t accumulated = 0;
       size_t pos = scan;
@@ -713,7 +713,7 @@ Compaction* FIFOCompactionPicker::PickRatioBasedIntraL0Compaction(
         return c;
       }
 
-      // This batch wasn't enough — advance past it
+      // This batch wasn't enough -- advance past it
       scan = pos;
     }
   }

db/compaction/compaction_picker_test.cc

@@ -1101,7 +1101,7 @@ TEST_F(CompactionPickerTest, ReadTriggeredSkipsLastLevel) {
   Add(0, 1U, "150", "200", kFileSize, 0, 500, 550);
   Add(4, 3U, "301", "350", kFileSize, 0, 101, 150);
 
-  // File 3 is at the last non-empty level — should NOT be marked for
+  // File 3 is at the last non-empty level -- should NOT be marked for
   // read-triggered compaction. Bottommost file cleanup is handled
   // separately by ComputeBottommostFilesMarkedForCompaction().
   file_map_[3U].first->stats.num_collapsible_entry_reads_sampled.store(

db/compaction/compaction_service_job.cc

@@ -189,14 +189,24 @@ CompactionJob::ProcessKeyValueCompactionWithCompactionService(
     return compaction_status;
   }
 
-  // CompactionServiceJobStatus::kSuccess was returned, but somehow we failed to
-  // read the result. Consider this as an installation failure
   if (!s.ok()) {
-    sub_compact->status = s;
+    // Wait() returned kSuccess, but the primary host could not deserialize the
+    // remote result before importing any output files into this DB. The remote
+    // output is still isolated in the service-managed staging directory, so it
+    // is safe to fall back to local compaction for the same job and notify the
+    // service via OnInstallation(kUseLocal).
+    assert(sub_compact->status.ok());
+    assert(sub_compact->GetMutableCompactionOutputs().empty());
+    assert(sub_compact->GetMutableProximalOutputs().empty());
+    ROCKS_LOG_WARN(db_options_.info_log,
+                   "[%s] [JOB %d] Failed to parse remote compaction result "
+                   "(%s), falling back to local compaction",
+                   compaction->column_family_data()->GetName().c_str(), job_id_,
+                   s.ToString().c_str());
     compaction_result.status.PermitUncheckedError();
     db_options_.compaction_service->OnInstallation(
-        response.scheduled_job_id, CompactionServiceJobStatus::kFailure);
-    return CompactionServiceJobStatus::kFailure;
+        response.scheduled_job_id, CompactionServiceJobStatus::kUseLocal);
+    return CompactionServiceJobStatus::kUseLocal;
   }
   sub_compact->status = compaction_result.status;
 
@@ -641,6 +651,10 @@ static std::unordered_map<std::string, OptionTypeInfo>
         {"cpu_micros",
          {offsetof(struct CompactionJobStats, cpu_micros), OptionType::kUInt64T,
           OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
+        {"has_accurate_num_input_records",
+         {offsetof(struct CompactionJobStats, has_accurate_num_input_records),
+          OptionType::kBoolean, OptionVerificationType::kNormal,
+          OptionTypeFlags::kNone}},
         {"num_input_records",
          {offsetof(struct CompactionJobStats, num_input_records),
           OptionType::kUInt64T, OptionVerificationType::kNormal,
@@ -846,11 +860,16 @@ static std::unordered_map<std::string, OptionTypeInfo>
          {offsetof(struct InternalStats::CompactionStats, count),
           OptionType::kUInt64T, OptionVerificationType::kNormal,
           OptionTypeFlags::kNone}},
-        {"counts", OptionTypeInfo::Array<
-                       int, static_cast<int>(CompactionReason::kNumOfReasons)>(
-                       offsetof(struct InternalStats::CompactionStats, counts),
-                       OptionVerificationType::kNormal, OptionTypeFlags::kNone,
-                       {0, OptionType::kInt})},
+        {"counts",
+         OptionTypeInfo::Array<
+             /* In release 11.2, a new compaction reason was added. This broken
+              * the reader and writer. To unblock release 11.1, we temporarily
+              * reduce the count array size to the old one. TODO add a proper
+              * serialization and deserialization method. */
+             int, static_cast<int>(CompactionReason::kNumOfReasons) - 1>(
+             offsetof(struct InternalStats::CompactionStats, counts),
+             OptionVerificationType::kNormal, OptionTypeFlags::kNone,
+             {0, OptionType::kInt})},
 };
 
 static std::unordered_map<std::string, OptionTypeInfo>

db/compaction/compaction_service_test.cc

@@ -127,6 +127,7 @@ class MyTestCompactionService : public CompactionService {
 
   void OnInstallation(const std::string& /*scheduled_job_id*/,
                       CompactionServiceJobStatus status) override {
+    installation_callback_count_.fetch_add(1);
     final_updated_status_ = status;
   }
 
@@ -167,6 +168,8 @@ class MyTestCompactionService : public CompactionService {
     return final_updated_status_.load();
   }
 
+  int GetOnInstallationCount() { return installation_callback_count_.load(); }
+
  protected:
   InstrumentedMutex mutex_;
   const std::string db_path_;
@@ -195,6 +198,7 @@ class MyTestCompactionService : public CompactionService {
   std::vector<std::shared_ptr<EventListener>> listeners_;
   std::vector<std::shared_ptr<TablePropertiesCollectorFactory>>
       table_properties_collector_factories_;
+  std::atomic_int installation_callback_count_{0};
   std::atomic<CompactionServiceJobStatus> final_updated_status_{
       CompactionServiceJobStatus::kUseLocal};
 
@@ -939,6 +943,37 @@ TEST_F(CompactionServiceTest, VerifyInputRecordCount) {
   SyncPoint::GetInstance()->ClearAllCallBacks();
 }
 
+TEST_F(CompactionServiceTest, InaccurateInputRecordCount) {
+  Options options = CurrentOptions();
+  options.disable_auto_compactions = true;
+  ReopenWithCompactionService(&options);
+  GenerateTestData();
+
+  auto my_cs = GetCompactionService();
+
+  std::string start_str = Key(15);
+  std::string end_str = Key(45);
+  Slice start(start_str);
+  Slice end(end_str);
+  uint64_t comp_num = my_cs->GetCompactionNum();
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "CompactionServiceCompactionJob::Run:0", [&](void* arg) {
+        CompactionServiceResult* compaction_result =
+            *(static_cast<CompactionServiceResult**>(arg));
+        ASSERT_TRUE(compaction_result != nullptr);
+        compaction_result->stats.has_accurate_num_input_records = false;
+        compaction_result->stats.num_input_records = 0;
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &start, &end));
+  ASSERT_GE(my_cs->GetCompactionNum(), comp_num + 1);
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+}
+
 TEST_F(CompactionServiceTest, EmptyResult) {
   Options options = CurrentOptions();
   options.disable_auto_compactions = true;
@@ -1448,7 +1483,7 @@ TEST_F(CompactionServiceTest, FailedToStart) {
   ASSERT_TRUE(s.IsIncomplete());
 }
 
-TEST_F(CompactionServiceTest, InvalidResult) {
+TEST_F(CompactionServiceTest, InvalidResultFallsBackToLocal) {
   Options options = CurrentOptions();
   options.disable_auto_compactions = true;
   ReopenWithCompactionService(&options);
@@ -1456,15 +1491,22 @@ TEST_F(CompactionServiceTest, InvalidResult) {
   GenerateTestData();
 
   auto my_cs = GetCompactionService();
+  ASSERT_EQ(0, my_cs->GetOnInstallationCount());
   my_cs->OverrideWaitResult("Invalid Str");
+  int compaction_num_before = my_cs->GetCompactionNum();
 
   std::string start_str = Key(15);
   std::string end_str = Key(45);
   Slice start(start_str);
   Slice end(end_str);
+  // The remote result is malformed before any installation starts, so the
+  // primary should recover by rerunning the compaction locally for this job.
   Status s = db_->CompactRange(CompactRangeOptions(), &start, &end);
-  ASSERT_FALSE(s.ok());
-  ASSERT_EQ(CompactionServiceJobStatus::kFailure,
+  ASSERT_OK(s);
+  ASSERT_GE(my_cs->GetCompactionNum(), compaction_num_before + 1);
+  VerifyTestData();
+  ASSERT_EQ(1, my_cs->GetOnInstallationCount());
+  ASSERT_EQ(CompactionServiceJobStatus::kUseLocal,
             my_cs->GetFinalCompactionServiceJobStatus());
 }
 

db/compaction/tiered_compaction_test.cc

@@ -2897,6 +2897,61 @@ TEST_P(PrecludeLastLevelTest, RangeDelsCauseFileEndpointsToOverlap) {
   Close();
 }
 
+TEST_F(PrecludeLastLevelTestBase,
+       RangeDelAtProximalSeqnoBoundaryStaysInLastLevel) {
+  constexpr int kNumLevels = 7;
+
+  Options options = CurrentOptions();
+  options.env = mock_env_.get();
+  options.num_levels = kNumLevels;
+  options.disable_auto_compactions = true;
+  DestroyAndReopen(options);
+  Defer close_db([&] { Close(); });
+
+  ASSERT_OK(Put(Key(2), "old"));
+  ASSERT_OK(Put(Key(12), "old"));
+  ManagedSnapshot snapshot(db_.get());
+  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(2),
+                             Key(12)));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(6);
+
+  const int l5_file_keys[][2] = {{0, 4}, {5, 9}};
+  for (const auto& l5_file : l5_file_keys) {
+    ASSERT_OK(Put(Key(l5_file[0]), "hot"));
+    ASSERT_OK(Put(Key(l5_file[1]), "hot"));
+    ASSERT_OK(Flush());
+    MoveFilesToLevel(5);
+  }
+  ASSERT_EQ("0,0,0,0,0,2,1", FilesPerLevel());
+
+  SyncPoint::GetInstance()->SetCallBack(
+      "CompactionJob::PrepareTimes():preclude_last_level_min_seqno",
+      [](void* arg) { *static_cast<SequenceNumber*>(arg) = 0; });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  ASSERT_OK(db_->SetOptions({{"preclude_last_level_data_seconds", "10000"}}));
+
+  auto l5_files = GetLevelFileMetadatas(5);
+  auto l6_files = GetLevelFileMetadatas(6);
+  ASSERT_EQ(2, l5_files.size());
+  ASSERT_EQ(1, l6_files.size());
+
+  ASSERT_OK(db_->CompactFiles(CompactionOptions(),
+                              {MakeTableFileName(l5_files[1]->fd.GetNumber()),
+                               MakeTableFileName(l6_files[0]->fd.GetNumber())},
+                              6));
+
+  uint64_t l6_range_deletions = 0;
+  for (auto* f : GetLevelFileMetadatas(5)) {
+    ASSERT_EQ(0, f->num_range_deletions);
+  }
+  for (auto* f : GetLevelFileMetadatas(6)) {
+    l6_range_deletions += f->num_range_deletions;
+  }
+  ASSERT_GT(l6_range_deletions, 0);
+}
+
 // Tests DBIter::GetProperty("rocksdb.iterator.write-time") return a data's
 // approximate write unix time.
 class IteratorWriteTimeTest

db/db_basic_test.cc

@@ -102,6 +102,862 @@ TEST_F(DBBasicTest, OpenWhenOpen) {
   ASSERT_TRUE(strstr(s.getState(), "lock ") != nullptr);
 }
 
+namespace {
+// Helper that captures per-branch SkippedNoopEdit counts for tests.
+struct RecoveryOptimizationCounters {
+  std::atomic<int> setup_dbid{0};
+  std::atomic<int> per_cf{0};
+  std::atomic<int> wal_deletion{0};
+  std::atomic<int> next_file_number{0};
+
+  void Install() {
+    auto* sp = ROCKSDB_NAMESPACE::SyncPoint::GetInstance();
+    sp->SetCallBack("DBImpl::Recovery:SkippedNoopEdit:SetupDBId",
+                    [this](void*) { setup_dbid.fetch_add(1); });
+    sp->SetCallBack("DBImpl::Recovery:SkippedNoopEdit:PerCF",
+                    [this](void*) { per_cf.fetch_add(1); });
+    sp->SetCallBack("DBImpl::Recovery:SkippedNoopEdit:WalDeletion",
+                    [this](void*) { wal_deletion.fetch_add(1); });
+    sp->SetCallBack("DBImpl::Recovery:SkippedNoopEdit:NextFileNumber",
+                    [this](void*) { next_file_number.fetch_add(1); });
+    sp->EnableProcessing();
+  }
+  void Uninstall() {
+    auto* sp = ROCKSDB_NAMESPACE::SyncPoint::GetInstance();
+    sp->DisableProcessing();
+    sp->ClearAllCallBacks();
+  }
+};
+}  // namespace
+
+// optimize_manifest_for_recovery=true: a clean reopen of a flushed DB must
+// append fewer individual records to the MANIFEST than the default-off path.
+// Verified by counting AddRecord calls (one per VersionEdit written to the
+// MANIFEST log).
+TEST_F(DBBasicTest,
+       OptimizeManifestForRecoveryReducesManifestWritesOnCleanReopen) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  auto* sp = ROCKSDB_NAMESPACE::SyncPoint::GetInstance();
+
+  // Measure MANIFEST records with optimize_manifest_for_recovery OFF (default).
+  DestroyAndReopen(options);
+  ASSERT_OK(Put("k1", "v1"));
+  ASSERT_OK(Flush());
+  Close();
+
+  std::atomic<int> records_off{0};
+  std::atomic<int> next_file_skips_off{0};
+  sp->SetCallBack("VersionSet::ProcessManifestWrites:AddRecord",
+                  [&](void*) { records_off.fetch_add(1); });
+  sp->SetCallBack("DBImpl::Recovery:SkippedNoopEdit:NextFileNumber",
+                  [&](void*) { next_file_skips_off.fetch_add(1); });
+  sp->EnableProcessing();
+  Reopen(options);
+  sp->DisableProcessing();
+  sp->ClearAllCallBacks();
+  ASSERT_EQ("v1", Get("k1"));
+  int off_count = records_off.load();
+  ASSERT_GT(off_count, 0);
+  ASSERT_EQ(0, next_file_skips_off.load());
+  Close();
+
+  // Measure MANIFEST records with optimize_manifest_for_recovery ON.
+  options.optimize_manifest_for_recovery = true;
+  DestroyAndReopen(options);
+  ASSERT_OK(Put("k2", "v2"));
+  ASSERT_OK(Flush());
+  Close();
+
+  std::atomic<int> records_on{0};
+  std::atomic<int> next_file_skips_on{0};
+  sp->SetCallBack("VersionSet::ProcessManifestWrites:AddRecord",
+                  [&](void*) { records_on.fetch_add(1); });
+  sp->SetCallBack("DBImpl::Recovery:SkippedNoopEdit:NextFileNumber",
+                  [&](void*) { next_file_skips_on.fetch_add(1); });
+  sp->EnableProcessing();
+  Reopen(options);
+  sp->DisableProcessing();
+  sp->ClearAllCallBacks();
+  ASSERT_EQ("v2", Get("k2"));
+  int on_count = records_on.load();
+  ASSERT_GT(next_file_skips_on.load(), 0);
+  ASSERT_LT(on_count, off_count);
+}
+
+// When the per-CF log_number actually advances, the per-CF skip must NOT
+// fire -- the edit carries real information and must be emitted.
+TEST_F(DBBasicTest, OptimizeManifestForRecoveryEmitsPerCFWhenLogAdvances) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.optimize_manifest_for_recovery = true;
+  DestroyAndReopen(options);
+  ASSERT_OK(Put("k", "v"));
+  // Write data and close WITHOUT flushing -- the WAL has un-replayed
+  // records, so on reopen recovery flushes the memtable and the per-CF
+  // edit's log_number must advance past the prior WAL.
+  Close();
+
+  RecoveryOptimizationCounters counters;
+  counters.Install();
+  Reopen(options);
+  counters.Uninstall();
+
+  ASSERT_EQ(0, counters.per_cf.load());
+  ASSERT_EQ("v", Get("k"));
+}
+
+// With track_and_verify_wals_in_manifest=true, the wal_deletion edit
+// must STILL be emitted (the DeleteWalsBefore record is required).
+TEST_F(DBBasicTest, OptimizeManifestForRecoveryPreservesWalTracking) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.optimize_manifest_for_recovery = true;
+  options.track_and_verify_wals_in_manifest = true;
+  DestroyAndReopen(options);
+  ASSERT_OK(Put("k", "v"));
+  ASSERT_OK(Flush());
+  Close();
+
+  RecoveryOptimizationCounters counters;
+  counters.Install();
+  Reopen(options);
+  counters.Uninstall();
+
+  ASSERT_EQ(0, counters.wal_deletion.load());
+}
+
+// best_efforts_recovery requires a fresh MANIFEST + CURRENT to be
+// produced on every open (the salvage contract). Even with the option
+// on, none of the SkippedNoopEdit branches must fire under
+// best_efforts_recovery -- otherwise CURRENT can be left missing or
+// stale (regression caught by DBBasicTest.RecoverWithNoCurrentFile and
+// DBTest2.BestEffortsRecoveryWithSstUniqueIdVerification under an
+// option-on default).
+TEST_F(DBBasicTest, OptimizeManifestForRecoveryDisabledByBestEffortsRecovery) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.optimize_manifest_for_recovery = true;
+  options.best_efforts_recovery = true;
+  DestroyAndReopen(options);
+  ASSERT_OK(Put("k", "v"));
+  ASSERT_OK(Flush());
+  Close();
+
+  RecoveryOptimizationCounters counters;
+  counters.Install();
+  Reopen(options);
+  counters.Uninstall();
+
+  ASSERT_EQ(0, counters.setup_dbid.load());
+  ASSERT_EQ(0, counters.per_cf.load());
+  ASSERT_EQ(0, counters.wal_deletion.load());
+  ASSERT_EQ(0, counters.next_file_number.load());
+  ASSERT_EQ("v", Get("k"));
+}
+
+// Multi-column-family coverage: with one CF flushed and another not,
+// the un-flushed CF's edit must still be emitted.
+TEST_F(DBBasicTest, OptimizeManifestForRecoveryMultiCF) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.optimize_manifest_for_recovery = true;
+  CreateAndReopenWithCF({"pikachu", "raichu"}, options);
+  ASSERT_OK(Put(0, "k", "v0"));
+  ASSERT_OK(Put(1, "k", "v1"));
+  ASSERT_OK(Put(2, "k", "v2"));
+  ASSERT_OK(Flush(0));
+  // CF 1 and 2 keep dirty memtables -- recovery must emit their per-CF
+  // edits (log_number advance from WAL replay).
+  Close();
+
+  RecoveryOptimizationCounters counters;
+  counters.Install();
+  ReopenWithColumnFamilies({"default", "pikachu", "raichu"}, options);
+  counters.Uninstall();
+
+  ASSERT_EQ("v0", Get(0, "k"));
+  ASSERT_EQ("v1", Get(1, "k"));
+  ASSERT_EQ("v2", Get(2, "k"));
+}
+
+// MaybeUpdateNextFileNumber's seed value (next_file_number_) makes the
+// post-loop comparison trivially true on every clean recovery, causing a
+// no-op SetNextFile edit to be appended to the MANIFEST. With
+// optimize_manifest_for_recovery=true, that emission is gated and
+// the SkippedNoopEdit:NextFileNumber sync point fires in its place.
+TEST_F(DBBasicTest, OptimizeManifestForRecoverySkipsNextFileNumber) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.optimize_manifest_for_recovery = true;
+  DestroyAndReopen(options);
+  ASSERT_OK(Put("k", "v"));
+  ASSERT_OK(Flush());
+  Close();
+
+  RecoveryOptimizationCounters counters;
+  counters.Install();
+  Reopen(options);
+  counters.Uninstall();
+
+  ASSERT_EQ(1, counters.next_file_number.load());
+}
+
+// With option=true and a synthetic on-disk file whose number is at or
+// above next_file_number_, MaybeUpdateNextFileNumber MUST emit the
+// SetNextFile edit and advance the counter past the synthetic number.
+TEST_F(DBBasicTest,
+       OptimizeManifestForRecoveryEmitsNextFileNumberWhenJustified) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.optimize_manifest_for_recovery = true;
+  DestroyAndReopen(options);
+  ASSERT_OK(Put("k", "v"));
+  ASSERT_OK(Flush());
+
+  const uint64_t next_before_close =
+      dbfull()->GetVersionSet()->current_next_file_number();
+  Close();
+
+  // Drop an empty .sst with a number well above next_file_number into
+  // the DB dir. MaybeUpdateNextFileNumber must observe it and advance.
+  const uint64_t synthetic_number = next_before_close + 100;
+  const std::string synthetic_sst =
+      dbname_ + "/" + MakeTableFileName("", synthetic_number);
+  ASSERT_OK(WriteStringToFile(env_, "" /*data*/, synthetic_sst,
+                              /*should_sync=*/true));
+
+  RecoveryOptimizationCounters counters;
+  counters.Install();
+  Reopen(options);
+  counters.Uninstall();
+
+  ASSERT_EQ(0, counters.next_file_number.load());
+  ASSERT_GT(dbfull()->GetVersionSet()->current_next_file_number(),
+            synthetic_number);
+}
+
+// optimize_manifest_for_recovery=true: after a clean Put + Flush + Close, the
+// next Open's min_log_number_to_keep equals (max_wal_before_close + 1),
+// proving the close-time MANIFEST write took effect.
+TEST_F(DBBasicTest, OptimizeManifestForRecoveryAdvancesWalMarkersOnClose) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.optimize_manifest_for_recovery = true;
+  DestroyAndReopen(options);
+  ASSERT_OK(Put("k", "v"));
+  ASSERT_OK(Flush());
+
+  uint64_t max_wal_before_close = 0;
+  auto* sp = ROCKSDB_NAMESPACE::SyncPoint::GetInstance();
+  sp->SetCallBack("DBImpl::CloseHelper:CapturedMaxWal", [&](void* arg) {
+    max_wal_before_close = *static_cast<uint64_t*>(arg);
+  });
+  sp->EnableProcessing();
+  Close();
+  sp->DisableProcessing();
+  sp->ClearAllCallBacks();
+  ASSERT_GT(max_wal_before_close, 0u);
+
+  Reopen(options);
+  ASSERT_EQ(max_wal_before_close + 1,
+            dbfull()->GetVersionSet()->min_log_number_to_keep());
+  ASSERT_EQ("v", Get("k"));
+}
+
+// Shared-option matrix: default-off and disable-before-close should both fall
+// back to recovery-time MANIFEST work, while enable-through-close should avoid
+// MANIFEST appends on a clean reopen.
+TEST_F(DBBasicTest, OptimizeManifestForRecoveryCleanReopenMatrix) {
+  struct TestCase {
+    const char* name;
+    bool enable_on_open;
+    bool disable_before_close;
+    bool expect_close_write;
+  };
+  const TestCase test_cases[] = {
+      {"default_off", false, false, false},
+      {"enabled", true, false, true},
+      {"disabled_before_close", true, true, false},
+  };
+
+  auto* sp = ROCKSDB_NAMESPACE::SyncPoint::GetInstance();
+  for (const auto& test_case : test_cases) {
+    SCOPED_TRACE(test_case.name);
+    Options options = CurrentOptions();
+    options.create_if_missing = true;
+    options.optimize_manifest_for_recovery = test_case.enable_on_open;
+    DestroyAndReopen(options);
+    ASSERT_OK(Put("k", test_case.name));
+    ASSERT_OK(Flush());
+
+    if (test_case.disable_before_close) {
+      ASSERT_OK(dbfull()->SetDBOptions(
+          {{"optimize_manifest_for_recovery", "false"}}));
+    }
+
+    std::atomic<int> entered{0};
+    sp->SetCallBack("DBImpl::CloseHelper:WriteWalMarkersOnCloseEntered",
+                    [&](void*) { entered.fetch_add(1); });
+    sp->EnableProcessing();
+    Close();
+    sp->DisableProcessing();
+    sp->ClearAllCallBacks();
+
+    std::atomic<int> records{0};
+    RecoveryOptimizationCounters counters;
+    counters.Install();
+    sp->SetCallBack("VersionSet::ProcessManifestWrites:AddRecord",
+                    [&](void*) { records.fetch_add(1); });
+    Reopen(options);
+    counters.Uninstall();
+
+    if (test_case.expect_close_write) {
+      ASSERT_EQ(1, entered.load());
+      ASSERT_EQ(0, records.load());
+      ASSERT_GT(counters.per_cf.load(), 0);
+    } else {
+      ASSERT_EQ(0, entered.load());
+      ASSERT_GT(records.load(), 0);
+    }
+    ASSERT_EQ(test_case.name, Get("k"));
+    Close();
+  }
+}
+
+// allow_2pc=true: even with the option on, the close-time write must NOT
+// advance MinLogNumberToKeep / DeleteWalsBefore (2pc requires the WAL to
+// remain replayable for uncommitted prepared transactions).
+TEST_F(DBBasicTest, OptimizeManifestForRecoveryRespectsTwoPC) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.optimize_manifest_for_recovery = true;
+  options.allow_2pc = true;
+  DestroyAndReopen(options);
+  ASSERT_OK(Put("k", "v"));
+  ASSERT_OK(Flush());
+
+  const uint64_t min_log_before_close =
+      dbfull()->GetVersionSet()->min_log_number_to_keep();
+  Close();
+  Reopen(options);
+  ASSERT_EQ(min_log_before_close,
+            dbfull()->GetVersionSet()->min_log_number_to_keep());
+  ASSERT_EQ("v", Get("k"));
+}
+
+// Mixed-CF emptiness: on reopen, recovery should still have MANIFEST work to
+// do for the dirty CF while skipping per-CF recovery edits for the empty CFs
+// whose markers were persisted at close.
+TEST_F(DBBasicTest, OptimizeManifestForRecoverySkipsNonEmptyCF) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.optimize_manifest_for_recovery = true;
+  CreateAndReopenWithCF({"empty_cf", "dirty_cf"}, options);
+  ASSERT_OK(Put(1, "k", "v1"));
+  ASSERT_OK(Flush(1));
+  ASSERT_OK(Put(2, "k", "v2"));
+  Close();
+
+  RecoveryOptimizationCounters counters;
+  std::atomic<int> records{0};
+  auto* sp = ROCKSDB_NAMESPACE::SyncPoint::GetInstance();
+  counters.Install();
+  sp->SetCallBack("VersionSet::ProcessManifestWrites:AddRecord",
+                  [&](void*) { records.fetch_add(1); });
+  ReopenWithColumnFamilies({"default", "empty_cf", "dirty_cf"}, options);
+  counters.Uninstall();
+
+  ASSERT_GT(counters.per_cf.load(), 0);
+  ASSERT_GT(records.load(), 0);
+  ASSERT_EQ("v1", Get(1, "k"));
+  ASSERT_EQ("v2", Get(2, "k"));
+}
+
+// track_and_verify_wals_in_manifest=true: the close-time write must leave
+// recovery with only the mandatory WAL-tracking MANIFEST work. The per-CF
+// recovery edits should still skip.
+TEST_F(DBBasicTest, OptimizeManifestForRecoveryEmitsWalDeletionWhenTracking) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.optimize_manifest_for_recovery = true;
+  options.track_and_verify_wals_in_manifest = true;
+  DestroyAndReopen(options);
+  ASSERT_OK(Put("k", "v"));
+  ASSERT_OK(Flush());
+  Close();
+
+  RecoveryOptimizationCounters counters;
+  std::atomic<int> records{0};
+  auto* sp = ROCKSDB_NAMESPACE::SyncPoint::GetInstance();
+  counters.Install();
+  sp->SetCallBack("VersionSet::ProcessManifestWrites:AddRecord",
+                  [&](void*) { records.fetch_add(1); });
+  Reopen(options);
+  counters.Uninstall();
+
+  ASSERT_GT(counters.per_cf.load(), 0);
+  ASSERT_EQ(1, records.load());
+  ASSERT_GT(dbfull()->GetVersionSet()->GetWalSet().GetMinWalNumberToKeep(), 0u);
+  ASSERT_EQ("v", Get("k"));
+}
+
+// Regression for the close-time marker path: if a new WAL is created while an
+// otherwise-empty user CF stays empty, Close() must reserve the next file
+// number before persisting SetLogNumber(cur_wal + 1) for that CF.
+TEST_F(DBBasicTest, OptimizeManifestForRecoveryImmediateCloseAfterWarmReopen) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.optimize_manifest_for_recovery = true;
+  CreateAndReopenWithCF({"empty_cf"}, options);
+
+  ASSERT_OK(Put(0, "k", "v"));
+  const uint64_t wal_before_switch = dbfull()->TEST_LogfileNumber();
+  ASSERT_OK(dbfull()->TEST_SwitchMemtable());
+  ASSERT_GT(dbfull()->TEST_LogfileNumber(), wal_before_switch);
+
+  const uint64_t expected_new_log_num = dbfull()->TEST_LogfileNumber() + 1;
+  ASSERT_EQ(expected_new_log_num,
+            dbfull()->GetVersionSet()->current_next_file_number());
+
+  Close();
+  ReopenWithColumnFamilies({"default", "empty_cf"}, options);
+
+  ASSERT_GT(dbfull()->GetVersionSet()->current_next_file_number(),
+            expected_new_log_num);
+  ASSERT_EQ("v", Get(0, "k"));
+}
+
+// Dropped-CF safety: dropped column families must NOT have markers written for
+// them at close time (the !IsDropped() guard protects against attaching the
+// global edit to a dropped CF, which would later trip MANIFEST replay
+// assertions).
+TEST_F(DBBasicTest, OptimizeManifestForRecoverySkipsDroppedCF) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.optimize_manifest_for_recovery = true;
+  CreateAndReopenWithCF({"to_drop", "keeper"}, options);
+  ASSERT_OK(Put(1, "k", "v1"));
+  ASSERT_OK(Put(2, "k", "v2"));
+  ASSERT_OK(Flush(1));
+  ASSERT_OK(Flush(2));
+  ASSERT_OK(db_->DropColumnFamily(handles_[1]));
+  Close();
+
+  ReopenWithColumnFamilies({"default", "keeper"}, options);
+  ASSERT_EQ("v2", Get(1, "k"));
+}
+
+// reuse_manifest_on_open=true: the next LogAndApply after Recover must
+// append to the existing MANIFEST file instead of allocating a fresh
+// one. Force a write after reopen and verify the MANIFEST file number
+// stays unchanged.
+TEST_F(DBBasicTest, ReuseManifestOnOpenAppendsToExistingFile) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.reuse_manifest_on_open = true;
+  DestroyAndReopen(options);
+  ASSERT_OK(Put("k", "v"));
+  ASSERT_OK(Flush());
+  Close();
+
+  Reopen(options);
+  const uint64_t manifest_after_reopen =
+      dbfull()->TEST_Current_Manifest_FileNo();
+  ASSERT_OK(Put("k2", "v2"));
+  ASSERT_OK(Flush());
+  const uint64_t manifest_after_flush =
+      dbfull()->TEST_Current_Manifest_FileNo();
+
+  EXPECT_EQ(manifest_after_reopen, manifest_after_flush);
+  EXPECT_EQ("v", Get("k"));
+  EXPECT_EQ("v2", Get("k2"));
+}
+
+// Default off: ReopenManifestForAppend must NOT be invoked.
+TEST_F(DBBasicTest, ReuseManifestOnOpenDefaultOffSkipsReopen) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  // reuse_manifest_on_open defaults to false
+  DestroyAndReopen(options);
+  ASSERT_OK(Put("k", "v"));
+  ASSERT_OK(Flush());
+  Close();
+
+  std::atomic<int> reopened{0};
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "VersionSet::ReopenManifestForAppend:Reopened",
+      [&](void* /*arg*/) { reopened.fetch_add(1); });
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  Reopen(options);
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
+
+  ASSERT_EQ(0, reopened.load());
+}
+
+// Regression for the WritableFileWriter::filesize_=0 bug fixed via
+// constructor-time initial_file_size: after ReopenManifestForAppend binds a
+// writer to the existing MANIFEST, GetFileSize() must return the on-disk
+// size, not 0.
+// (If it returned 0, the size-limit check in ProcessManifestWrites would
+// compare against 0 and Close-time Truncate could shrink the file.)
+TEST_F(DBBasicTest, ReuseManifestOnOpenAdoptsOnDiskSize) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.reuse_manifest_on_open = true;
+  DestroyAndReopen(options);
+  ASSERT_OK(Put("k", "v"));
+  ASSERT_OK(Flush());
+  Close();
+
+  // Capture the on-disk MANIFEST size before reopen.
+  Reopen(options);
+  const uint64_t manifest_no = dbfull()->TEST_Current_Manifest_FileNo();
+  const std::string manifest_path = DescriptorFileName(dbname_, manifest_no);
+  uint64_t on_disk_size = 0;
+  ASSERT_OK(env_->GetFileSize(manifest_path, &on_disk_size));
+  ASSERT_GT(on_disk_size, 0u);
+
+  // Force a write to flush the writer's buffer; verify Close doesn't
+  // shrink the file (which would happen if Truncate(filesize_=0) ran).
+  ASSERT_OK(Put("k2", "v2"));
+  ASSERT_OK(Flush());
+  Close();
+
+  uint64_t after_close_size = 0;
+  ASSERT_OK(env_->GetFileSize(manifest_path, &after_close_size));
+  EXPECT_GE(after_close_size, on_disk_size);
+}
+
+// MANIFEST rotation continues to work under reuse: when the file grows
+// past tuned_max_manifest_file_size_, ProcessManifestWrites must rotate
+// to a fresh MANIFEST (same as legacy). Use the rotation SyncPoint to
+// observe rotation directly, since tuned_max_manifest_file_size_ is
+// auto-derived and not directly = max_manifest_file_size.
+TEST_F(DBBasicTest, ReuseManifestOnOpenStillRotatesOnSizeCap) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.reuse_manifest_on_open = true;
+  options.max_manifest_file_size = 1;
+  options.max_manifest_space_amp_pct = 0;  // disable amp-based tuning
+  DestroyAndReopen(options);
+  ASSERT_OK(Put("k", "v"));
+  ASSERT_OK(Flush());
+  Close();
+
+  std::atomic<int> rotations{0};
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "VersionSet::ProcessManifestWrites:BeforeNewManifest",
+      [&](void* /*arg*/) { rotations.fetch_add(1); });
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  Reopen(options);
+  ASSERT_OK(Put("k2", "v2"));
+  ASSERT_OK(Flush());
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
+
+  // The Flush after Reopen must have triggered a rotation given the
+  // tiny size cap -- proves the size-driven rotation path still runs
+  // when descriptor_log_ is bound by reuse.
+  EXPECT_GT(rotations.load(), 0);
+  EXPECT_EQ("v", Get("k"));
+  EXPECT_EQ("v2", Get("k2"));
+}
+
+// Multi-CF reuse: append-mode MANIFEST must correctly handle edits
+// from multiple CFs after reopen.
+TEST_F(DBBasicTest, ReuseManifestOnOpenMultiCF) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.reuse_manifest_on_open = true;
+  CreateAndReopenWithCF({"alpha", "beta"}, options);
+  ASSERT_OK(Put(0, "k", "v0"));
+  ASSERT_OK(Put(1, "k", "v1"));
+  ASSERT_OK(Put(2, "k", "v2"));
+  ASSERT_OK(Flush(0));
+  ASSERT_OK(Flush(1));
+  ASSERT_OK(Flush(2));
+  Close();
+
+  ReopenWithColumnFamilies({"default", "alpha", "beta"}, options);
+  // Trigger more writes post-reopen on each CF -- these get appended to
+  // the reused MANIFEST.
+  ASSERT_OK(Put(0, "k2", "v0b"));
+  ASSERT_OK(Put(1, "k2", "v1b"));
+  ASSERT_OK(Put(2, "k2", "v2b"));
+  ASSERT_OK(Flush(0));
+  ASSERT_OK(Flush(1));
+  ASSERT_OK(Flush(2));
+
+  EXPECT_EQ("v0", Get(0, "k"));
+  EXPECT_EQ("v1", Get(1, "k"));
+  EXPECT_EQ("v2", Get(2, "k"));
+  EXPECT_EQ("v0b", Get(0, "k2"));
+  EXPECT_EQ("v1b", Get(1, "k2"));
+  EXPECT_EQ("v2b", Get(2, "k2"));
+}
+
+// Disabled under best_efforts_recovery: that mode rebuilds CURRENT and
+// MANIFEST as the side-effect of LogAndApplyForRecovery emitting an
+// edit; reusing the prior MANIFEST contradicts the salvage contract.
+TEST_F(DBBasicTest, ReuseManifestOnOpenDisabledByBestEffortsRecovery) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.reuse_manifest_on_open = true;
+  options.best_efforts_recovery = true;
+  DestroyAndReopen(options);
+  ASSERT_OK(Put("k", "v"));
+  ASSERT_OK(Flush());
+  Close();
+
+  std::atomic<int> reopened{0};
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "VersionSet::ReopenManifestForAppend:Reopened",
+      [&](void* /*arg*/) { reopened.fetch_add(1); });
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  Reopen(options);
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
+
+  ASSERT_EQ(0, reopened.load());
+  ASSERT_EQ("v", Get("k"));
+}
+
+// Full-stack composition: optimize_manifest_for_recovery plus
+// reuse_manifest_on_open. Verifies that Close writes recovery markers,
+// Reopen skips clean-recovery MANIFEST edits, and the MANIFEST is reused
+// instead of recreated for the next metadata update.
+TEST_F(DBBasicTest, ReuseManifestOnOpenFullStackComposition) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.optimize_manifest_for_recovery = true;
+  options.reuse_manifest_on_open = true;
+  DestroyAndReopen(options);
+  ASSERT_OK(Put("k", "v"));
+  ASSERT_OK(Flush());
+  Close();
+
+  // Capture the MANIFEST file number before reopen.
+  std::atomic<int> reopened{0};
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "VersionSet::ReopenManifestForAppend:Reopened",
+      [&](void* /*arg*/) { reopened.fetch_add(1); });
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  Reopen(options);
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
+
+  // All three composing: reuse fired, data is intact, no fresh MANIFEST.
+  EXPECT_EQ(1, reopened.load());
+  EXPECT_EQ("v", Get("k"));
+}
+
+// Regression test for WAL recovery while publishing a fresh MANIFEST. The test
+// stores SSTs in a separate DB path, injects failure after CURRENT points at
+// the new MANIFEST, and simulates crash cleanup; the recovered SST must survive
+// because the synced MANIFEST references it.
+TEST_F(DBBasicTest, RecoverySstDirSyncedBeforeFreshManifestPublish) {
+  auto fault_fs = std::make_shared<FaultInjectionTestFS>(env_->GetFileSystem());
+  std::unique_ptr<Env> fault_env(NewCompositeEnv(fault_fs));
+
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.disable_auto_compactions = true;
+  options.env = fault_env.get();
+  options.reuse_manifest_on_open = false;
+  options.db_paths.emplace_back(dbname_ + "_2", 1ULL << 30);
+
+  DestroyAndReopen(options);
+  ASSERT_OK(Put("base", "value"));
+  ASSERT_OK(Flush());
+
+  ASSERT_OK(Put("recovered", "value"));
+  ASSERT_OK(db_->FlushWAL(true));
+  Close();
+
+  fault_fs->ResetState();
+
+  std::atomic<bool> fail_after_current_publish{true};
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "VersionSet::ProcessManifestWrites:AfterSetCurrentFile", [&](void* arg) {
+        if (fail_after_current_publish.exchange(false)) {
+          ASSERT_NE(nullptr, arg);
+          IOStatus* io_s = static_cast<IOStatus*>(arg);
+          ASSERT_OK(*io_s);
+          *io_s = IOStatus::IOError("injected current publish aftermath");
+        }
+      });
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  Status s = TryReopen(options);
+  ASSERT_TRUE(s.IsIOError()) << s.ToString();
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
+
+  ASSERT_OK(fault_fs->DeleteFilesCreatedAfterLastDirSync(IOOptions(), nullptr));
+
+  s = TryReopen(options);
+  ASSERT_OK(s);
+  ASSERT_EQ("value", Get("base"));
+  ASSERT_EQ("value", Get("recovered"));
+  Close();
+}
+
+// Regression test for WAL recovery while appending to a reused MANIFEST. The
+// first reopen forces recovery to create an SST and then fail after MANIFEST
+// sync. The simulated crash cleanup deletes files without a prior directory
+// sync; the recovered SST must survive because the synced MANIFEST references
+// it.
+TEST_F(DBBasicTest,
+       ReuseManifestOnOpenSyncsRecoverySstDirBeforeManifestAppend) {
+  auto fault_fs = std::make_shared<FaultInjectionTestFS>(env_->GetFileSystem());
+  std::unique_ptr<Env> fault_env(NewCompositeEnv(fault_fs));
+
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.disable_auto_compactions = true;
+  options.env = fault_env.get();
+  options.reuse_manifest_on_open = true;
+
+  DestroyAndReopen(options);
+  ASSERT_OK(Put("base", "value"));
+  ASSERT_OK(Flush());
+
+  ASSERT_OK(Put("recovered", "value"));
+  ASSERT_OK(db_->FlushWAL(true));
+  Close();
+
+  fault_fs->ResetState();
+
+  std::atomic<bool> fail_after_manifest_sync{true};
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "VersionSet::ProcessManifestWrites:AfterSyncManifest", [&](void* arg) {
+        if (fail_after_manifest_sync.exchange(false)) {
+          ASSERT_NE(nullptr, arg);
+          IOStatus* io_s = static_cast<IOStatus*>(arg);
+          ASSERT_OK(*io_s);
+          *io_s = IOStatus::IOError("injected manifest sync aftermath");
+        }
+      });
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  Status s = TryReopen(options);
+  ASSERT_TRUE(s.IsIOError()) << s.ToString();
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
+
+  ASSERT_OK(fault_fs->DeleteFilesCreatedAfterLastDirSync(IOOptions(), nullptr));
+
+  s = TryReopen(options);
+  ASSERT_OK(s);
+  ASSERT_EQ("value", Get("base"));
+  ASSERT_EQ("value", Get("recovered"));
+  Close();
+}
+
+// Direct unit test for WritableFileWriter's initial_file_size parameter:
+// verifies the visible size accessors report the existing on-disk size
+// immediately, rather than the constructor's zero default.
+TEST_F(DBBasicTest, WritableFileWriterInitialFileSizeAdoptsExistingSize) {
+  Env* env = Env::Default();
+  std::string fname = test::PerThreadDBPath("set_file_size_test");
+  ASSERT_OK(env->CreateDirIfMissing(test::TmpDir(env)));
+
+  // Create a file with some bytes, close it.
+  {
+    std::unique_ptr<WritableFile> raw;
+    ASSERT_OK(env->NewWritableFile(fname, &raw, EnvOptions()));
+    ASSERT_OK(raw->Append("hello world"));
+    ASSERT_OK(raw->Close());
+  }
+
+  // Reopen and wrap in a WritableFileWriter without initial_file_size --
+  // GetFileSize() should be 0 (the constructor's default).
+  std::unique_ptr<FSWritableFile> fs_file;
+  ASSERT_OK(env->GetFileSystem()->ReopenWritableFile(
+      fname, FileOptions(), &fs_file, /*dbg=*/nullptr));
+  std::unique_ptr<WritableFileWriter> writer(
+      new WritableFileWriter(std::move(fs_file), fname, FileOptions()));
+  EXPECT_EQ(0u, writer->GetFileSize());
+
+  // Reopen again and seed the writer's size accounting from the existing
+  // bytes. GetFileSize and GetFlushedSize must reflect it immediately.
+  ASSERT_OK(env->GetFileSystem()->ReopenWritableFile(
+      fname, FileOptions(), &fs_file, /*dbg=*/nullptr));
+  writer.reset(new WritableFileWriter(
+      std::move(fs_file), fname, FileOptions(),
+      /*clock=*/nullptr, /*io_tracer=*/nullptr, /*stats=*/nullptr,
+      Histograms::HISTOGRAM_ENUM_MAX, /*listeners=*/{},
+      /*file_checksum_gen_factory=*/nullptr,
+      /*perform_data_verification=*/false,
+      /*buffered_data_with_checksum=*/false,
+      /*initial_file_size=*/11));
+  EXPECT_EQ(11u, writer->GetFileSize());
+  EXPECT_EQ(11u, writer->GetFlushedSize());
+
+  ASSERT_OK(env->DeleteFile(fname));
+}
+
+// Tail corruption: appending garbage bytes to the MANIFEST after a
+// clean close must prevent reuse -- the physical size exceeds the
+// last valid record end.
+TEST_F(DBBasicTest, ReuseManifestOnOpenSkipsOnTailCorruption) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.reuse_manifest_on_open = true;
+  DestroyAndReopen(options);
+  ASSERT_OK(Put("k", "v"));
+  ASSERT_OK(Flush());
+  Close();
+
+  // Find the MANIFEST file and append garbage to it.
+  std::string manifest_path;
+  {
+    std::vector<std::string> files;
+    ASSERT_OK(env_->GetChildren(dbname_, &files));
+    for (const auto& f : files) {
+      uint64_t number;
+      FileType type;
+      if (ParseFileName(f, &number, &type) && type == kDescriptorFile) {
+        manifest_path = dbname_ + "/" + f;
+        break;
+      }
+    }
+  }
+  ASSERT_FALSE(manifest_path.empty());
+  {
+    std::string contents;
+    ASSERT_OK(ReadFileToString(env_, manifest_path, &contents));
+    contents.append("garbage!");
+    ASSERT_OK(WriteStringToFile(env_, contents, manifest_path));
+  }
+
+  std::atomic<int> reopened{0};
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "VersionSet::ReopenManifestForAppend:Reopened",
+      [&](void* /*arg*/) { reopened.fetch_add(1); });
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  Reopen(options);
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
+
+  ASSERT_EQ(0, reopened.load());
+  ASSERT_EQ("v", Get("k"));
+}
+
 TEST_F(DBBasicTest, EnableDirectIOWithZeroBuf) {
   if (!IsDirectIOSupported()) {
     ROCKSDB_GTEST_BYPASS("Direct IO not supported");

db/db_block_cache_test.cc

@@ -542,7 +542,7 @@ TEST_F(DBBlockCacheTest, WarmCacheWithDataBlocksDuringCompaction) {
   EXPECT_TRUE(tracking_cache->HasPriority(Cache::Priority::BOTTOM));
   EXPECT_FALSE(tracking_cache->HasPriority(Cache::Priority::LOW));
 
-  // Compaction output is in cache — reads should have zero misses.
+  // Compaction output is in cache -- reads should have zero misses.
   auto data_miss_before =
       options.statistics->getTickerCount(BLOCK_CACHE_DATA_MISS);
   ASSERT_EQ(value + "2", Get("key"));

db/db_bloom_filter_test.cc

@@ -2022,7 +2022,11 @@ TEST_F(DBBloomFilterTest, MutableFilterPolicy) {
   double expected_bpk = 10.0;
   // Other configs to try
   std::vector<std::pair<std::string, double>> configs = {
-      {"ribbonfilter:10:-1", 7.0}, {"bloomfilter:5", 5.0}, {"nullptr", 0.0}};
+      {"ribbonfilter:10:-1", 7.0},
+      {"bloomfilter:5", 5.0},
+      {"nullptr", 0.0},
+      // As serialized in OPTIONS file
+      {"{id=ribbonfilter:12:-1;bloom_before_level=-1;}", 8.4}};
 
   table_options.cache_index_and_filter_blocks = true;
   options.table_factory.reset(NewBlockBasedTableFactory(table_options));

db/db_compaction_abort_test.cc

@@ -427,6 +427,38 @@ TEST_F(DBCompactionAbortTest, AbortAutomaticCompaction) {
   }
 }
 
+TEST_F(DBCompactionAbortTest, AbortScheduledAutomaticCompactionBeforePick) {
+  // The goal is to cover an automatic compaction that has been scheduled, but
+  // aborts before the worker picks and removes a column family from the
+  // compaction queue. This pins the scheduler bookkeeping invariant that
+  // ResumeAllCompactions() must be able to schedule that still-queued work.
+  constexpr int kCompactionTrigger = 4;
+  constexpr int kNumL0Files = kCompactionTrigger + 1;
+  Options options = GetOptionsWithStats();
+  options.level0_file_num_compaction_trigger = kCompactionTrigger;
+  options.max_background_compactions = 1;
+  options.max_subcompactions = 1;
+  options.disable_auto_compactions = false;
+  Reopen(options);
+
+  SyncPointAbortHelper helper("BackgroundCallCompaction:0");
+  helper.Setup(dbfull());
+
+  PopulateData(/*num_files=*/kNumL0Files, /*keys_per_file=*/100,
+               /*value_size=*/1000);
+  helper.CleanupAndWait();
+
+  const uint64_t compact_write_bytes_before_resume =
+      stats_->getTickerCount(COMPACT_WRITE_BYTES);
+  dbfull()->ResumeAllCompactions();
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+
+  ASSERT_GT(stats_->getTickerCount(COMPACT_WRITE_BYTES),
+            compact_write_bytes_before_resume);
+  ASSERT_LT(NumTableFilesAtLevel(0), kNumL0Files);
+  VerifyDataIntegrity(/*num_keys=*/100);
+}
+
 TEST_F(DBCompactionAbortTest, AbortAndVerifyNoOutputFiles) {
   Options options = CurrentOptions();
   options.level0_file_num_compaction_trigger = 4;

db/db_compaction_test.cc

@@ -9057,11 +9057,10 @@ class DBCompactionTestBlobError
   std::string sync_point_;
 };
 
-INSTANTIATE_TEST_CASE_P(
-    DBCompactionTestBlobError, DBCompactionTestBlobError,
-    ::testing::ValuesIn(std::vector<std::string>{
-        "BlobFileBuilder::WriteBlobToFile:AddRecord",
-        "BlobFileBuilder::WriteBlobToFile:LegacyAppendFooterAndClose"}));
+INSTANTIATE_TEST_CASE_P(DBCompactionTestBlobError, DBCompactionTestBlobError,
+                        ::testing::ValuesIn(std::vector<std::string>{
+                            "BlobFileBuilder::WriteBlobToFile:AddRecord",
+                            "BlobFileBuilder::WriteBlobToFile:AppendFooter"}));
 
 TEST_P(DBCompactionTestBlobError, CompactionError) {
   Options options = CurrentOptions();
@@ -11834,7 +11833,7 @@ TEST_F(DBCompactionTest, RoundRobinCleanCutWithSharedBoundary) {
 // 2. A post-verification step fails (injected here via sync point), setting
 //    compact_->status to error while each subcompaction's status stays OK.
 // 3. SubcompactionState::Cleanup checks individual status (OK) and skips
-//    ReleaseObsolete — the cache entries leak.
+//    ReleaseObsolete -- the cache entries leak.
 // 4. FaultInjectionTestFS injects metadata read errors, causing GetChildren
 //    to fail in FindObsoleteFiles.
 // 5. Close()'s FindObsoleteFiles also fails to find the orphan for the same
@@ -11865,7 +11864,7 @@ TEST_F(DBCompactionTest, LeakedTableCacheEntryOnCompactionFailure) {
   // fail while individual subcompaction statuses stay OK (so Cleanup skips
   // ReleaseObsolete). The filesystem deactivation makes GetChildren fail
   // in FindObsoleteFiles, preventing the backstop from evicting the leaked
-  // cache entries — matching the crash test's metadata read fault injection.
+  // cache entries -- matching the crash test's metadata read fault injection.
   std::atomic<bool> inject_error{true};
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
       "CompactionJob::Run():AfterVerifyOutputFiles", [&](void* arg) {
@@ -11877,7 +11876,7 @@ TEST_F(DBCompactionTest, LeakedTableCacheEntryOnCompactionFailure) {
   // Enable metadata read fault injection on the bg compaction thread after
   // the compaction job finishes but before FindObsoleteFiles runs. This
   // makes GetChildren fail (metadata read), matching crash test's
-  // --open_metadata_read_fault_one_in=8. Only metadata reads fail —
+  // --open_metadata_read_fault_one_in=8. Only metadata reads fail --
   // logging and other IO operations continue normally.
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
       "BackgroundCallCompaction:1", [&](void*) {
@@ -11890,7 +11889,7 @@ TEST_F(DBCompactionTest, LeakedTableCacheEntryOnCompactionFailure) {
 
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
 
-  // Trigger compaction — fails after VerifyOutputFiles.
+  // Trigger compaction -- fails after VerifyOutputFiles.
   Status s = dbfull()->TEST_CompactRange(0, nullptr, nullptr);
   ASSERT_NOK(s);
 
@@ -11972,7 +11971,7 @@ TEST_F(DBCompactionTest, LeakedTableCacheEntryOnInstallFailure) {
 
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
 
-  // Trigger compaction — Run() succeeds but Install() fails.
+  // Trigger compaction -- Run() succeeds but Install() fails.
   Status s = dbfull()->TEST_CompactRange(0, nullptr, nullptr);
   ASSERT_NOK(s);
 
@@ -12033,7 +12032,7 @@ TEST_F(DBCompactionTest, ObsoleteFileTableCacheEntryWithConcurrentRef) {
       TableCache::Lookup(table_cache, target_file_number);
   ASSERT_NE(concurrent_handle, nullptr);
 
-  // Call ReleaseObsolete directly — this is what PurgeObsoleteFiles calls
+  // Call ReleaseObsolete directly -- this is what PurgeObsoleteFiles calls
   // when a file becomes obsolete. Pass nullptr for the handle to make
   // ReleaseObsolete do its own Lookup internally.
   TableCache::ReleaseObsolete(table_cache, target_file_number,
@@ -12050,6 +12049,88 @@ TEST_F(DBCompactionTest, ObsoleteFileTableCacheEntryWithConcurrentRef) {
   ASSERT_EQ(leaked, nullptr);
 }
 
+// Regression test for the atomic flush path leaking table cache entries when
+// InstallMemtableAtomicFlushResults (MANIFEST write) fails.
+// 1. atomic_flush is enabled with two column families.
+// 2. Both CFs have data, a flush is triggered.
+// 3. FlushJob::Run() succeeds for both (files are added to table cache by
+//    BuildTable), but write_manifest=false so no install happens in Run().
+// 4. The combined MANIFEST write (InstallMemtableAtomicFlushResults) fails
+//    due to injected I/O error.
+// 5. Without the fix, the files remain in the table cache but are not in
+//    any Version, causing TEST_VerifyNoObsoleteFilesCached to fire during
+//    Close() (especially when metadata_read_fault_one_in disables the
+//    FindObsoleteFiles backstop).
+TEST_F(DBCompactionTest, LeakedTableCacheEntryOnAtomicFlushInstallFailure) {
+  auto fault_fs = std::make_shared<FaultInjectionTestFS>(env_->GetFileSystem());
+  std::unique_ptr<Env> fault_env(NewCompositeEnv(fault_fs));
+
+  Options options = CurrentOptions();
+  options.env = fault_env.get();
+  options.atomic_flush = true;
+  options.create_if_missing = true;
+  options.disable_auto_compactions = true;
+  // Enough buffer so auto-flush doesn't trigger prematurely.
+  options.write_buffer_size = 64 << 20;
+  DestroyAndReopen(options);
+
+  // Create a second column family.
+  CreateAndReopenWithCF({"pikachu"}, options);
+  ASSERT_EQ(2, handles_.size());
+
+  // Write data to both CFs.
+  ASSERT_OK(Put(0, "key0", std::string(1024, 'a')));
+  ASSERT_OK(Put(1, "key1", std::string(1024, 'b')));
+
+  // Inject a write error during the combined MANIFEST write that happens
+  // inside InstallMemtableAtomicFlushResults (via LogAndApply).
+  std::atomic<bool> inject_error{true};
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "VersionSet::LogAndApply:WriteManifest", [&](void*) {
+        if (inject_error.exchange(false)) {
+          fault_fs->SetThreadLocalErrorContext(
+              FaultInjectionIOType::kWrite, /*seed=*/0, /*one_in=*/1,
+              /*retryable=*/false, /*has_data_loss=*/false);
+          fault_fs->EnableThreadLocalErrorInjection(
+              FaultInjectionIOType::kWrite);
+        }
+      });
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  // Trigger atomic flush - Run() succeeds but MANIFEST write fails.
+  FlushOptions flush_opts;
+  flush_opts.wait = true;
+  Status s = db_->Flush(flush_opts, handles_);
+  ASSERT_NOK(s);
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
+
+  // Disable the write fault so Close() can proceed.
+  fault_fs->DisableThreadLocalErrorInjection(FaultInjectionIOType::kWrite);
+
+  // Enable metadata read fault injection on the main thread so that
+  // Close()'s FindObsoleteFiles full-scan backstop cannot find the orphan
+  // files (simulating metadata_read_fault_one_in from the stress test).
+  fault_fs->SetThreadLocalErrorContext(
+      FaultInjectionIOType::kMetadataRead, /*seed=*/0, /*one_in=*/1,
+      /*retryable=*/false, /*has_data_loss=*/false);
+  fault_fs->EnableThreadLocalErrorInjection(
+      FaultInjectionIOType::kMetadataRead);
+
+  // Destroy column family handles before closing.
+  for (auto h : handles_) {
+    ASSERT_OK(db_->DestroyColumnFamilyHandle(h));
+  }
+  handles_.clear();
+
+  // Close the DB. Without the fix, TEST_VerifyNoObsoleteFilesCached fires.
+  s = db_->Close();
+  ASSERT_OK(s);
+  db_ = nullptr;
+}
+
 TEST_F(DBCompactionTest, VerifyFileChecksumOnCompactionOutput) {
   Options options = CurrentOptions();
   options.disable_auto_compactions = true;

db/db_etc2_test.cc

@@ -35,7 +35,7 @@ namespace ROCKSDB_NAMESPACE {
 
 class DBTest2 : public DBTestBase {
  public:
-  DBTest2() : DBTestBase("db_test2", /*env_do_fsync=*/true) {}
+  DBTest2() : DBTestBase("db_etc2_test", /*env_do_fsync=*/true) {}
 };
 
 TEST_F(DBTest2, OpenForReadOnly) {
@@ -107,7 +107,7 @@ TEST_F(DBTest2, ReadOnlyDBWalInDbPathInitialized) {
   ASSERT_OK(Put("key2", "value2"));
   Close();
 
-  // Reopen as read-only — wal_in_db_path_ must be properly initialized.
+  // Reopen as read-only -- wal_in_db_path_ must be properly initialized.
   // Before the fix, closing this DB would read an uninitialized bool in
   // DeleteObsoleteFileImpl, which UBSan catches as undefined behavior.
   std::unique_ptr<DB> db_ptr;
@@ -115,7 +115,7 @@ TEST_F(DBTest2, ReadOnlyDBWalInDbPathInitialized) {
   std::string value;
   ASSERT_OK(db_ptr->Get(ReadOptions(), "key1", &value));
   ASSERT_EQ("value1", value);
-  // Close the read-only DB — this triggers PurgeObsoleteFiles which reads
+  // Close the read-only DB -- this triggers PurgeObsoleteFiles which reads
   // wal_in_db_path_. Under UBSan, an uninitialized bool here would fail.
   db_ptr.reset();
 
@@ -7510,6 +7510,65 @@ TEST_F(DBTest2, GetLatestSeqAndTsForKey) {
   ASSERT_EQ(0, options.statistics->getTickerCount(GET_HIT_L0));
 }
 
+TEST_F(DBTest2,
+       GetLatestSequenceForKeyFromHistoryWithBlobBackedWideColumnEntity) {
+  // Goal: exercise the memtable history lookup in GetLatestSequenceForKey()
+  // after blob direct write stores a blob-backed V2 entity in a flushed
+  // memtable. Using cache_only=true ensures the lookup succeeds only if the
+  // history path can resolve the wide-column base value under the newer merge.
+  Destroy(last_options_);
+
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.enable_blob_files = true;
+  options.enable_blob_direct_write = true;
+  options.min_blob_size = 50;
+  options.allow_concurrent_memtable_write = false;
+  options.blob_direct_write_partitions = 1;
+  options.max_write_buffer_size_to_maintain = 64 << 10;
+  options.merge_operator = MergeOperators::CreateStringAppendOperator("|");
+
+  Reopen(options);
+
+  const std::string key = "history_blob_entity";
+  const std::string default_value(100, 'd');
+  const std::string merge_operand = "suffix";
+  WideColumns columns{{kDefaultWideColumnName, default_value},
+                      {"meta", "inline"}};
+
+  ASSERT_OK(
+      db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(), key, columns));
+  ASSERT_OK(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), key,
+                       merge_operand));
+  const SequenceNumber expected_seq = dbfull()->GetLatestSequenceNumber();
+
+  ASSERT_OK(Flush());
+  ASSERT_FALSE(GetBlobFileNumbers().empty());
+
+  uint64_t num_immutable_memtables = 0;
+  ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kNumImmutableMemTable,
+                                  &num_immutable_memtables));
+  ASSERT_EQ(num_immutable_memtables, 0);
+
+  auto* cfhi = static_cast_with_check<ColumnFamilyHandleImpl>(
+      dbfull()->DefaultColumnFamily());
+  assert(cfhi);
+  assert(cfhi->cfd());
+  SuperVersion* sv = cfhi->cfd()->GetSuperVersion();
+
+  SequenceNumber seq = kMaxSequenceNumber;
+  bool found_record_for_key = false;
+  bool is_blob_index = false;
+  const Status s = dbfull()->GetLatestSequenceForKey(
+      sv, key, /*cache_only=*/true, /*lower_bound_seq=*/0, &seq,
+      /*timestamp=*/nullptr, &found_record_for_key, &is_blob_index);
+
+  ASSERT_OK(s);
+  ASSERT_TRUE(found_record_for_key);
+  ASSERT_EQ(expected_seq, seq);
+  ASSERT_FALSE(is_blob_index);
+}
+
 #if defined(ZSTD)
 TEST_F(DBTest2, ZSTDChecksum) {
   // Verify that corruption during decompression is caught.
@@ -8173,7 +8232,7 @@ TEST_F(DBTest2, FastSstOpenDisableAfterMetadataPersisted) {
   ASSERT_EQ(1, test_fs->GetMetadataRetrievedCount());
   db.reset();
 
-  // Reopen with fast_sst_open DISABLED — metadata is in the MANIFEST
+  // Reopen with fast_sst_open DISABLED -- metadata is in the MANIFEST
   // but should NOT be passed to the filesystem
   options.fast_sst_open = false;
   test_fs->ResetCounters();

db/db_etc3_test.cc

@@ -1,9 +1,12 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
 //  This source code is licensed under both the GPLv2 (found in the
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
 
 #include "db/db_test_util.h"
+#include "rocksdb/convenience.h"
+#include "rocksdb/metadata.h"
+#include "rocksdb/sst_file_writer.h"
 
 namespace ROCKSDB_NAMESPACE {
 
@@ -43,50 +46,98 @@ TEST_F(DBEtc3Test, AutoTuneManifestSize) {
   ASSERT_EQ(DBOptions{}.max_manifest_space_amp_pct, 500);
 
   Options options = CurrentOptions();
-  ASSERT_OK(db_->SetOptions({{"level0_file_num_compaction_trigger", "20"}}));
+  // Test setup: create many flushed files. Keep level compaction semantics so
+  // DeleteFilesInRanges can remove non-L0 files, but prevent automatic
+  // compactions and write stalls from adding unrelated behavior.
+  options.disable_auto_compactions = true;
+  options.level0_slowdown_writes_trigger = 100;
+  options.level0_stop_writes_trigger = 200;
 
-  // Use large column family names to essentially control the amount of payload
-  // data needed for the manifest file. Drop manifest entries don't include the
-  // CF name so are small.
+  // Test strategy: use large column family names to control the rough amount
+  // of payload added to the MANIFEST. Drop manifest entries do not include the
+  // CF name, so they are small.
+  //
+  // Most CF helper calls piggy-back a background manifest write so the main
+  // auto-tuning phases continue to test the unrelaxed background threshold
+  // even though CF manipulation itself is foreground. Phase-specific foreground
+  // checks disable that piggy-backed background write.
   uint64_t prev_manifest_num = 0, cur_manifest_num = 0;
   std::deque<ColumnFamilyHandle*> handles;
   int counter = 5;
-  auto AddCfFn = [&]() {
+  auto UpdateManifestNumsFrom = [&](uint64_t before_manifest_num) {
+    prev_manifest_num = before_manifest_num;
+    cur_manifest_num = dbfull()->TEST_Current_Manifest_FileNo();
+  };
+  auto BackgroundManifestWriteFn = [&]() {
+    uint64_t before_manifest_num = cur_manifest_num;
+    ASSERT_OK(Put("x", std::to_string(counter++)));
+    ASSERT_OK(Flush());
+    UpdateManifestNumsFrom(before_manifest_num);
+  };
+  auto AddCfFn = [&](bool include_background_manifest_write = true) {
+    uint64_t before_manifest_num = cur_manifest_num;
     std::string name = "cf" + std::to_string(counter++);
     name.resize(1000, 'a');
     ASSERT_OK(db_->CreateColumnFamily(options, name, &handles.emplace_back()));
-    prev_manifest_num = cur_manifest_num;
     cur_manifest_num = dbfull()->TEST_Current_Manifest_FileNo();
+    if (include_background_manifest_write) {
+      BackgroundManifestWriteFn();
+    }
+    UpdateManifestNumsFrom(before_manifest_num);
   };
-  auto DropCfFn = [&]() {
+  auto DropCfFn = [&](bool include_background_manifest_write = true) {
+    uint64_t before_manifest_num = cur_manifest_num;
     ASSERT_OK(db_->DropColumnFamily(handles.front()));
     ASSERT_OK(db_->DestroyColumnFamilyHandle(handles.front()));
     handles.pop_front();
-    prev_manifest_num = cur_manifest_num;
-    cur_manifest_num = dbfull()->TEST_Current_Manifest_FileNo();
-  };
-  auto TrivialManifestWriteFn = [&]() {
-    ASSERT_OK(Put("x", std::to_string(counter++)));
-    ASSERT_OK(Flush());
-    prev_manifest_num = cur_manifest_num;
     cur_manifest_num = dbfull()->TEST_Current_Manifest_FileNo();
+    if (include_background_manifest_write) {
+      BackgroundManifestWriteFn();
+    }
+    UpdateManifestNumsFrom(before_manifest_num);
   };
 
+  // ---- Phase 1: foreground threshold relaxation is bounded ----
+  //
+  // Foreground operations should only get about 25% extra headroom, not an
+  // unbounded threshold. With 1000-byte CF names and a 3000-byte normal limit,
+  // the relaxed limit should allow the first four foreground-only CF additions
+  // but require rotation on the fifth.
+  options.max_manifest_file_size = 3000;
+  options.max_manifest_space_amp_pct = 0;
+  DestroyAndReopen(options);
+  cur_manifest_num = dbfull()->TEST_Current_Manifest_FileNo();
+  prev_manifest_num = cur_manifest_num;
+  for (int i = 1; i <= 4; ++i) {
+    AddCfFn(/*include_background_manifest_write=*/false);
+    ASSERT_EQ(prev_manifest_num, cur_manifest_num);
+  }
+  AddCfFn(/*include_background_manifest_write=*/false);
+  ASSERT_LT(prev_manifest_num, cur_manifest_num);
+  while (!handles.empty()) {
+    ASSERT_OK(db_->DestroyColumnFamilyHandle(handles.front()));
+    handles.pop_front();
+  }
+
+  // ---- Phase 2: no auto-tuning means frequent rotation ----
+  //
   options.max_manifest_file_size = 1000000;
   options.max_manifest_space_amp_pct = 0;  // no auto-tuning yet
   DestroyAndReopen(options);
+  cur_manifest_num = dbfull()->TEST_Current_Manifest_FileNo();
+  prev_manifest_num = cur_manifest_num;
 
-  // With the generous (minimum) maximum manifest size, should not be rotated
+  // With the generous minimum manifest size, should not be rotated.
   AddCfFn();
   AddCfFn();
   AddCfFn();
   ASSERT_EQ(prev_manifest_num, cur_manifest_num);
 
-  // Change options for small max and (still) no auto-tuning
+  // Lower the minimum while still disabling auto-tuning.
   ASSERT_OK(db_->SetDBOptions({{"max_manifest_file_size", "3000"}}));
 
   // Takes effect on the next manifest write
-  TrivialManifestWriteFn();
+  BackgroundManifestWriteFn();
   ASSERT_LT(prev_manifest_num, cur_manifest_num);
 
   // Now we have to rewrite the whole manifest on each write because the
@@ -97,28 +148,31 @@ TEST_F(DBEtc3Test, AutoTuneManifestSize) {
   ASSERT_LT(prev_manifest_num, cur_manifest_num);
   AddCfFn();
   ASSERT_LT(prev_manifest_num, cur_manifest_num);
-  TrivialManifestWriteFn();
+  BackgroundManifestWriteFn();
   ASSERT_LT(prev_manifest_num, cur_manifest_num);
 
+  // ---- Phase 3: auto-tuning raises the background threshold ----
+  //
   // Enabling auto-tuning should fix this, immediately for next manifest writes.
-  // This will allow up to double-ish the size of the compacted manifest,
-  // which last should have been 4000 + some bytes.
+  // This will allow up to roughly double the size of the compacted manifest,
+  // which now includes CF entries plus the piggy-backed background writes.
   ASSERT_EQ(handles.size(), 4U);
-  ASSERT_OK(db_->SetDBOptions({{"max_manifest_space_amp_pct", "105"}}));
+  ASSERT_OK(db_->SetDBOptions({{"max_manifest_space_amp_pct", "95"}}));
 
-  // After 9 CF names should be enough to rotate the manifest
-  for (int i = 1; i <= 5; ++i) {
+  // Auto-tuning lets several more CF names accumulate in the MANIFEST before
+  // the piggy-backed background write crosses the threshold.
+  for (int i = 1; i <= 3; ++i) {
     if ((i % 2) == 1) {
       DropCfFn();
     }
     AddCfFn();
     ASSERT_EQ(prev_manifest_num, cur_manifest_num);
   }
-  TrivialManifestWriteFn();
+  AddCfFn();
   ASSERT_LT(prev_manifest_num, cur_manifest_num);
 
-  // We now have a different last compacted manifest size, should be
-  // able to go beyond 9 CFs named in manifest this time.
+  // We now have a different last compacted manifest size, so the next
+  // threshold should be based on the newly compacted MANIFEST.
   ASSERT_EQ(handles.size(), 6U);
 
   DropCfFn();
@@ -128,11 +182,10 @@ TEST_F(DBEtc3Test, AutoTuneManifestSize) {
     AddCfFn();
     ASSERT_EQ(prev_manifest_num, cur_manifest_num);
   }
-  // We've written 10 named CFs to the manifest. We should be able to
-  // dynamically change the auto-tuning still based on the last "compacted"
-  // manifest size of 7000 + some bytes.
+  // We should be able to dynamically change the auto-tuning still based on
+  // the last "compacted" manifest size.
   ASSERT_OK(db_->SetDBOptions({{"max_manifest_space_amp_pct", "51"}}));
-  TrivialManifestWriteFn();
+  BackgroundManifestWriteFn();
   ASSERT_LT(prev_manifest_num, cur_manifest_num);
   // And the "compacted" manifest size has reset again, so should be changed
   // again sooner.
@@ -141,13 +194,179 @@ TEST_F(DBEtc3Test, AutoTuneManifestSize) {
     AddCfFn();
     ASSERT_EQ(prev_manifest_num, cur_manifest_num);
   }
-  // Enough for manifest change
-  AddCfFn();
+
+  // ---- Phase 4: foreground operations use relaxed threshold ----
+  //
+  // The current MANIFEST is now large enough for the next background manifest
+  // write to rotate it, but still small enough for foreground operations to use
+  // their 25% extra headroom. Assert that each foreground operation stays on
+  // the same MANIFEST, then verify the next background write rotates.
+  const std::string external_files_dir = dbname_ + "/external_files";
+  ASSERT_OK(env_->CreateDirIfMissing(external_files_dir));
+  auto WriteExternalSstFile = [&](const std::string& file_name,
+                                  const std::string& key,
+                                  const std::string& value) {
+    const std::string file_path = external_files_dir + "/" + file_name;
+    Status ignored = env_->DeleteFile(file_path);
+    ignored.PermitUncheckedError();
+
+    SstFileWriter sst_file_writer(EnvOptions(), options);
+    ASSERT_OK(sst_file_writer.Open(file_path));
+    ASSERT_OK(sst_file_writer.Put(key, value));
+    ASSERT_OK(sst_file_writer.Finish());
+  };
+  auto IngestExternalFileForegroundManifestWriteFn =
+      [&](std::string* ingested_key) {
+        uint64_t before_manifest_num = cur_manifest_num;
+        const std::string key = "z" + std::to_string(counter++);
+        const std::string value = "v" + std::to_string(counter++);
+        const std::string file_name =
+            "ingest_" + std::to_string(counter++) + ".sst";
+        const std::string file_path = external_files_dir + "/" + file_name;
+        WriteExternalSstFile(file_name, key, value);
+
+        ASSERT_OK(
+            db_->IngestExternalFile({file_path}, IngestExternalFileOptions()));
+        ASSERT_EQ(value, Get(key));
+        *ingested_key = key;
+        Status ignored = env_->DeleteFile(file_path);
+        ignored.PermitUncheckedError();
+        UpdateManifestNumsFrom(before_manifest_num);
+      };
+  auto CreateColumnFamilyWithImportForegroundManifestWriteFn = [&]() {
+    uint64_t before_manifest_num = cur_manifest_num;
+    const std::string cf_name = "import_cf" + std::to_string(counter++);
+    const std::string key = "import" + std::to_string(counter++);
+    const std::string value = "v" + std::to_string(counter++);
+    const std::string file_name =
+        "import_" + std::to_string(counter++) + ".sst";
+    const std::string file_path = external_files_dir + "/" + file_name;
+    WriteExternalSstFile(file_name, key, value);
+
+    LiveFileMetaData file_metadata;
+    file_metadata.name = file_name;
+    file_metadata.db_path = external_files_dir;
+    file_metadata.smallest_seqno = 0;
+    file_metadata.largest_seqno = 0;
+    file_metadata.level = 0;
+    ExportImportFilesMetaData metadata;
+    metadata.files.push_back(file_metadata);
+    metadata.db_comparator_name = options.comparator->Name();
+
+    ColumnFamilyHandle* import_handle = nullptr;
+    ASSERT_OK(db_->CreateColumnFamilyWithImport(options, cf_name,
+                                                ImportColumnFamilyOptions(),
+                                                metadata, &import_handle));
+    ASSERT_NE(import_handle, nullptr);
+    handles.push_back(import_handle);
+
+    std::string result;
+    ASSERT_OK(db_->Get(ReadOptions(), import_handle, key, &result));
+    ASSERT_EQ(value, result);
+    Status ignored = env_->DeleteFile(file_path);
+    ignored.PermitUncheckedError();
+    UpdateManifestNumsFrom(before_manifest_num);
+  };
+  auto DeleteFilesInRangesForegroundManifestWriteFn =
+      [&](const std::string& key) {
+        uint64_t before_manifest_num = cur_manifest_num;
+        const std::string limit = key + "\xff";
+        std::vector<RangeOpt> ranges;
+        ranges.emplace_back(key, limit);
+        ASSERT_OK(DeleteFilesInRanges(db_.get(), db_->DefaultColumnFamily(),
+                                      ranges.data(), ranges.size(),
+                                      /*include_end=*/false));
+
+        std::string result;
+        ASSERT_TRUE(db_->Get(ReadOptions(), key, &result).IsNotFound());
+        UpdateManifestNumsFrom(before_manifest_num);
+      };
+
+  // Column family manipulation.
+  AddCfFn(/*include_background_manifest_write=*/false);
+  ASSERT_EQ(prev_manifest_num, cur_manifest_num);
+
+  // SetOptions should not write to the MANIFEST. If that regresses, the write
+  // should be treated as a background write and rotate here.
+  {
+    ASSERT_OK(db_->SetOptions({{"level0_slowdown_writes_trigger", "101"}}));
+    UpdateManifestNumsFrom(cur_manifest_num);
+  }
+  ASSERT_EQ(prev_manifest_num, cur_manifest_num);
+
+  // External file ingestion.
+  std::string ingested_key;
+  IngestExternalFileForegroundManifestWriteFn(&ingested_key);
+  ASSERT_EQ(prev_manifest_num, cur_manifest_num);
+
+  // Imported column family creation.
+  CreateColumnFamilyWithImportForegroundManifestWriteFn();
+  ASSERT_EQ(prev_manifest_num, cur_manifest_num);
+
+  // Physical file deletion by range.
+  DeleteFilesInRangesForegroundManifestWriteFn(ingested_key);
+  ASSERT_EQ(prev_manifest_num, cur_manifest_num);
+
+  // Background flush.
+  BackgroundManifestWriteFn();
   ASSERT_LT(prev_manifest_num, cur_manifest_num);
 
+  // ---- Phase 5: persisted compacted manifest size survives close/reopen ----
+  // Close with CFs still live. Reopen with reuse_manifest_on_open so the
+  // manifest is NOT rewritten from scratch. The persisted compacted size
+  // should be loaded and used for auto-tuning.
+  // At this point we have 8 CF handles plus default.
+  ASSERT_EQ(handles.size(), 8U);
+
+  // Collect CF names for reopen, then release handles (Close needs this)
+  std::vector<std::string> cf_names = {"default"};
+  for (auto* h : handles) {
+    cf_names.push_back(h->GetName());
+  }
+  for (auto* h : handles) {
+    ASSERT_OK(db_->DestroyColumnFamilyHandle(h));
+  }
+  handles.clear();
+
+  Close();
+  // Use a max_manifest_file_size below the reused manifest size. Auto-tuning
+  // with the persisted compacted size at 200% amp keeps the tuned threshold
+  // high enough. Without persistence, the threshold would be
+  // max(max_manifest_file_size, 0 * anything) = max_manifest_file_size.
+  //
+  // With max_manifest_file_size set to 3000, missing persisted compacted size
+  // would keep tuned = 3000 and the first AddCf would rotate because the
+  // reused manifest is already larger. With persisted compacted size loaded,
+  // the tuned threshold is based on the compacted size, so no rotation until
+  // the manifest grows well beyond the minimum.
+  options.max_manifest_file_size = 3000;
+  options.max_manifest_space_amp_pct = 200;
+  options.reuse_manifest_on_open = true;
+  uint64_t manifest_num_before_reopen = cur_manifest_num;
+  ReopenWithColumnFamilies(cf_names, options);
+  cur_manifest_num = dbfull()->TEST_Current_Manifest_FileNo();
+
+  // With persistence: the manifest file number should NOT have changed
+  // during reopen, because the persisted compacted size keeps the tuned
+  // threshold high enough. Without persistence, last_compacted = 0, so
+  // tuned = max_manifest_file_size = 3000, and the first LogAndApply
+  // during Open rotates the manifest because it is already larger than 3000.
+  ASSERT_EQ(manifest_num_before_reopen, cur_manifest_num);
+
+  // Adding CFs should still not trigger rotation because the tuned threshold
+  // from the persisted compacted size exceeds the current manifest size.
+  for (int i = 1; i <= 4; ++i) {
+    AddCfFn();
+    ASSERT_EQ(prev_manifest_num, cur_manifest_num);
+  }
+  for (int i = 1; i <= 4; ++i) {
+    AddCfFn();
+    ASSERT_EQ(prev_manifest_num, cur_manifest_num);
+  }
+
   // Wrap up
   while (!handles.empty()) {
-    DropCfFn();
+    DropCfFn(/*include_background_manifest_write=*/false);
   }
 }
 

db/db_flush_test.cc

@@ -1852,6 +1852,8 @@ TEST_F(DBFlushTest, MemPurgeCorrectLogNumberAndSSTFileCreation) {
     }
   }
 
+  ASSERT_OK(WaitForFlushCallbacks());
+
   // Check that there was at least one mempurge
   uint32_t expected_min_mempurge_count = 1;
   // Check that there was no SST files created during flush.
@@ -1872,6 +1874,8 @@ TEST_F(DBFlushTest, MemPurgeCorrectLogNumberAndSSTFileCreation) {
     ASSERT_EQ(Get(key), value);
   }
 
+  ASSERT_OK(WaitForFlushCallbacks());
+
   // Check that there was at least one SST files created during flush.
   expected_sst_count = 1;
   EXPECT_GE(sst_count.load(), expected_sst_count);
@@ -1891,6 +1895,9 @@ TEST_F(DBFlushTest, MemPurgeCorrectLogNumberAndSSTFileCreation) {
   // Extra check of database consistency.
   ASSERT_EQ(Get(key), value);
 
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
+
   Close();
 }
 
@@ -2507,11 +2514,10 @@ class DBFlushTestBlobError : public DBFlushTest,
   std::string sync_point_;
 };
 
-INSTANTIATE_TEST_CASE_P(
-    DBFlushTestBlobError, DBFlushTestBlobError,
-    ::testing::ValuesIn(std::vector<std::string>{
-        "BlobFileBuilder::WriteBlobToFile:AddRecord",
-        "BlobFileBuilder::WriteBlobToFile:LegacyAppendFooterAndClose"}));
+INSTANTIATE_TEST_CASE_P(DBFlushTestBlobError, DBFlushTestBlobError,
+                        ::testing::ValuesIn(std::vector<std::string>{
+                            "BlobFileBuilder::WriteBlobToFile:AddRecord",
+                            "BlobFileBuilder::WriteBlobToFile:AppendFooter"}));
 
 TEST_P(DBFlushTestBlobError, FlushError) {
   Options options;
@@ -3884,7 +3890,7 @@ TEST_F(DBFlushTest, LeakedTableCacheEntryOnFlushInstallFailure) {
 
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
 
-  // Trigger flush — BuildTable succeeds but LogAndApply fails.
+  // Trigger flush -- BuildTable succeeds but LogAndApply fails.
   Status s = Flush();
   ASSERT_NOK(s);
 

db/db_impl/db_impl.cc

@@ -15,6 +15,7 @@
 
 #include <cinttypes>
 #include <cstdio>
+#include <deque>
 #include <map>
 #include <memory>
 #include <optional>
@@ -29,6 +30,7 @@
 
 #include "db/arena_wrapped_db_iter.h"
 #include "db/attribute_group_iterator_impl.h"
+#include "db/blob/blob_fetcher.h"
 #include "db/blob/blob_file_partition_manager.h"
 #include "db/blob/blob_index.h"
 #include "db/builder.h"
@@ -173,6 +175,7 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname,
                const bool seq_per_batch, const bool batch_per_txn,
                bool read_only)
     : dbname_(dbname),
+      read_only_(read_only),
       own_info_log_(options.info_log == nullptr),
       initial_db_options_(SanitizeOptions(dbname, options, read_only,
                                           &init_logger_creation_s_)),
@@ -481,6 +484,33 @@ void DBImpl::WaitForBackgroundWork() {
   }
 }
 
+void DBImpl::WaitForAsyncFileOpen() {
+  if (!immutable_db_options_.open_files_async) {
+    return;
+  }
+  InstrumentedMutexLock l(&mutex_);
+  TEST_SYNC_POINT("DBImpl::WaitForAsyncFileOpen::BeforeWait");
+  while (bg_async_file_open_state_ == AsyncFileOpenState::kScheduled &&
+         !shutting_down_.load(std::memory_order_acquire)) {
+    bg_cv_.Wait();
+  }
+}
+
+void DBImpl::NotifyOnDBShutdownBegin() {
+  mutex_.AssertHeld();
+  if (shutdown_notification_sent_ || immutable_db_options_.listeners.empty()) {
+    return;
+  }
+  shutdown_notification_sent_ = true;
+
+  // release lock while notifying events
+  mutex_.Unlock();
+  for (const auto& listener : immutable_db_options_.listeners) {
+    listener->OnDBShutdownBegin(this);
+  }
+  mutex_.Lock();
+}
+
 // Will lock the mutex_,  will wait for completion if wait is true
 void DBImpl::CancelAllBackgroundWork(bool wait) {
   ROCKS_LOG_INFO(immutable_db_options_.info_log,
@@ -527,6 +557,9 @@ void DBImpl::CancelAllBackgroundWork(bool wait) {
     immutable_db_options_.compaction_service->CancelAwaitingJobs();
   }
 
+  if (!already_shutting_down) {
+    NotifyOnDBShutdownBegin();
+  }
   shutting_down_.store(true, std::memory_order_release);
   bg_cv_.SignalAll();
   if (!wait) {
@@ -636,6 +669,106 @@ void DBImpl::UnregisterBlobDirectWriteColumnFamily() {
   assert(false);
 }
 
+Status DBImpl::MaybeWriteWalMarkersToManifestOnClose() {
+  mutex_.AssertHeld();
+  if (!mutable_db_options_.optimize_manifest_for_recovery ||
+      !opened_successfully_ || read_only_ || versions_ == nullptr ||
+      logs_.empty()) {
+    return Status::OK();
+  }
+
+  TEST_SYNC_POINT("DBImpl::CloseHelper:WriteWalMarkersOnCloseEntered");
+  const ReadOptions read_options(Env::IOActivity::kUnknown);
+  const WriteOptions write_options(Env::IOActivity::kUnknown);
+  uint64_t max_wal_number = logs_.back().number;
+  TEST_SYNC_POINT_CALLBACK("DBImpl::CloseHelper:CapturedMaxWal",
+                           &max_wal_number);
+  const uint64_t new_log_num = max_wal_number + 1;
+
+  // std::deque keeps VersionEdit pointers stable across emplace_back.
+  std::deque<VersionEdit> edits;
+  autovector<ColumnFamilyData*> cfds;
+  autovector<autovector<VersionEdit*>> edit_lists;
+  VersionEdit global_edit;
+  bool persist_global_edit = false;
+  bool reserved_log_number = false;
+
+  bool all_cfs_empty = true;
+  for (auto* cfd : *versions_->GetColumnFamilySet()) {
+    if (cfd->IsDropped()) {
+      continue;
+    }
+    // cfd->IsEmpty() checks both mem and imm. An immutable memtable queued for
+    // flush still pins an older WAL; advancing past it would drop unflushed
+    // records on the next open.
+    if (!cfd->IsEmpty()) {
+      all_cfs_empty = false;
+      continue;
+    }
+    if (new_log_num > cfd->GetLogNumber()) {
+      if (!reserved_log_number) {
+        const uint64_t next_file_number = versions_->current_next_file_number();
+        if (next_file_number <= new_log_num) {
+          versions_->FetchAddFileNumber(new_log_num + 1 - next_file_number);
+          global_edit.SetNextFile(versions_->current_next_file_number());
+          persist_global_edit = true;
+        }
+        reserved_log_number = true;
+      }
+      edits.emplace_back();
+      VersionEdit& edit = edits.back();
+      edit.SetLogNumber(new_log_num);
+      edit.SetColumnFamily(cfd->GetID());
+      cfds.push_back(cfd);
+      edit_lists.emplace_back();
+      edit_lists.back().push_back(&edit);
+    }
+  }
+
+  if (all_cfs_empty && !allow_2pc()) {
+    if (new_log_num > versions_->min_log_number_to_keep()) {
+      global_edit.SetMinLogNumberToKeep(new_log_num);
+      persist_global_edit = true;
+    }
+    // Mirror the monotonicity guard in db_impl_files.cc:
+    // PrecomputeWalDeletionEdit emits DeleteWalsBefore only when the
+    // marker actually advances the WalSet's existing min. Skipping
+    // this guard would let a stale-replay edit regress WalSet state.
+    if (immutable_db_options_.track_and_verify_wals_in_manifest &&
+        new_log_num > versions_->GetWalSet().GetMinWalNumberToKeep()) {
+      global_edit.DeleteWalsBefore(new_log_num);
+      persist_global_edit = true;
+    }
+  }
+
+  if (persist_global_edit) {
+    ColumnFamilyData* default_cfd =
+        versions_->GetColumnFamilySet()->GetDefault();
+    if (default_cfd && !default_cfd->IsDropped()) {
+      bool attached_to_existing = false;
+      for (size_t i = 0; i < cfds.size(); ++i) {
+        if (cfds[i] == default_cfd) {
+          edit_lists[i].push_back(&global_edit);
+          attached_to_existing = true;
+          break;
+        }
+      }
+      if (!attached_to_existing) {
+        cfds.push_back(default_cfd);
+        edit_lists.emplace_back();
+        edit_lists.back().push_back(&global_edit);
+      }
+    }
+  }
+
+  if (cfds.empty()) {
+    return Status::OK();
+  }
+  return versions_->LogAndApply(cfds, read_options, write_options, edit_lists,
+                                &mutex_, directories_.GetDbDir(),
+                                /*new_descriptor_log=*/false);
+}
+
 Status DBImpl::CloseHelper() {
   // Guarantee that there is no background error recovery in progress before
   // continuing with the shutdown
@@ -676,12 +809,27 @@ Status DBImpl::CloseHelper() {
          bg_flush_scheduled_ || bg_purge_scheduled_ ||
          bg_pressure_callback_in_progress_ ||
          bg_async_file_open_state_ == AsyncFileOpenState::kScheduled ||
+         async_wal_precreate_state_ == AsyncWALPrecreateState::kScheduled ||
          pending_purge_obsolete_files_ ||
          error_handler_.IsRecoveryInProgress()) {
     TEST_SYNC_POINT("DBImpl::~DBImpl:WaitJob");
     bg_cv_.Wait();
   }
 
+  // Release any opened-but-unpublished WAL writer after the in-flight worker
+  // has published its result. Clear the DB-owned async slot while holding
+  // mutex_, but destroy the detached writer after dropping mutex_ because
+  // log::Writer / WritableFileWriter destruction can flush and close the file.
+  // The file itself can be left behind as an empty future WAL; recovery already
+  // tolerates it and marks its file number used if observed.
+  UnpublishedWAL unused_async_wal = std::move(async_wal_precreate_wal_);
+  async_wal_precreate_state_ = AsyncWALPrecreateState::kNotScheduled;
+  if (unused_async_wal.writer) {
+    mutex_.Unlock();
+    unused_async_wal.Reset();
+    mutex_.Lock();
+  }
+
   // Ensure subclasses don't forget to schedule async file opening
   assert(!immutable_db_options_.open_files_async || !opened_successfully_ ||
          bg_async_file_open_state_ != AsyncFileOpenState::kNotScheduled);
@@ -727,7 +875,7 @@ Status DBImpl::CloseHelper() {
   // manifest file), it is not able to identify live files correctly. As a
   // result, all "live" files can get deleted by accident. However, corrupted
   // manifest is recoverable by RepairDB().
-  if (opened_successfully_) {
+  if (opened_successfully_ && !read_only_) {
     JobContext job_context(next_job_id_.fetch_add(1));
     FindObsoleteFiles(&job_context, true);
 
@@ -740,6 +888,17 @@ Status DBImpl::CloseHelper() {
     job_context.Clean();
     mutex_.Lock();
   }
+  // Best-effort warm-reopen optimization: after obsolete-file cleanup and
+  // before logs_.clear(), persist close-time WAL markers that can reduce
+  // recovery work on the next open. This matters most when some CFs have no
+  // unflushed writes and/or the newest WAL is empty, since recovery would
+  // otherwise append marker-only MANIFEST edits on reopen.
+  Status manifest_s = MaybeWriteWalMarkersToManifestOnClose();
+  if (!manifest_s.ok()) {
+    ROCKS_LOG_WARN(immutable_db_options_.info_log,
+                   "optimize_manifest_for_recovery close-time write failed: %s",
+                   manifest_s.ToString().c_str());
+  }
   {
     InstrumentedMutexLock lock(&wal_write_mutex_);
     for (auto l : wals_to_free_) {
@@ -2748,6 +2907,148 @@ Status DBImpl::ResolveDirectWriteWideColumns(const ReadOptions& read_options,
   return status;
 }
 
+bool DBImpl::MaybeResolveMemtableBlobValue(const Slice& key,
+                                           const BlobFetcher* blob_fetcher,
+                                           PinnableSlice* value,
+                                           PinnableWideColumns* columns,
+                                           Status* s, bool* is_blob_index,
+                                           bool* value_found) {
+  if (!s->ok() || (!value && !columns)) {
+    return false;
+  }
+
+  auto reset_outputs = [&]() {
+    if (value != nullptr) {
+      value->Reset();
+    }
+    if (columns != nullptr) {
+      columns->Reset();
+    }
+  };
+  auto clear_blob_state = [&]() {
+    if (is_blob_index != nullptr) {
+      *is_blob_index = false;
+    }
+  };
+
+  const bool needs_plain_value_resolution =
+      is_blob_index != nullptr && *is_blob_index;
+  const bool needs_wide_column_resolution =
+      columns != nullptr && !columns->unresolved_blob_column_indices_.empty();
+  if (!needs_plain_value_resolution && !needs_wide_column_resolution) {
+    return false;
+  }
+
+  if (blob_fetcher == nullptr) {
+    reset_outputs();
+    *s = Status::NotSupported(
+        "Encountered blob-backed memtable value without blob fetcher.");
+    clear_blob_state();
+    return true;
+  }
+
+  if (needs_plain_value_resolution) {
+    Slice blob_index_slice;
+    std::string blob_index_storage;
+    if (value != nullptr) {
+      if (value->size() > 0) {
+        blob_index_slice = Slice(value->data(), value->size());
+      } else {
+        blob_index_slice = Slice(*(value->GetSelf()));
+      }
+    } else {
+      assert(columns != nullptr);
+
+      const WideColumns& plain_value_columns = columns->columns();
+      assert(plain_value_columns.size() == 1);
+      assert(plain_value_columns.front().name() == kDefaultWideColumnName);
+      blob_index_slice = plain_value_columns.front().value();
+    }
+
+    PinnableSlice resolved_value;
+    PinnableSlice* target = value != nullptr ? value : &resolved_value;
+    if (value != nullptr) {
+      // BlobIndex::DecodeFrom can retain Slices into the encoded bytes for
+      // inlined blob indices, so take an owned copy before resetting `value`
+      // and reusing the same PinnableSlice as the output target.
+      blob_index_storage.assign(blob_index_slice.data(),
+                                blob_index_slice.size());
+      blob_index_slice = Slice(blob_index_storage);
+      value->Reset();
+    }
+
+    *s = blob_fetcher->FetchBlob(key, blob_index_slice,
+                                 nullptr /* prefetch_buffer */, target,
+                                 nullptr /* bytes_read */);
+    if (s->ok() && columns != nullptr) {
+      columns->SetPlainValue(std::move(*target));
+    } else if (!s->ok()) {
+      reset_outputs();
+      if (s->IsIncomplete() && value_found != nullptr) {
+        *value_found = false;
+      }
+    }
+
+    clear_blob_state();
+    return true;
+  }
+
+  assert(columns != nullptr);
+
+  std::string resolved_entity;
+  bool resolved = false;
+  *s = WideColumnSerialization::ResolveEntityBlobColumns(
+      columns->value_, key, blob_fetcher, nullptr /* prefetch_buffers */,
+      resolved_entity, resolved, nullptr /* total_bytes_read */,
+      nullptr /* num_blobs_resolved */);
+  if (s->ok()) {
+    assert(resolved);
+    if (resolved) {
+      *s = columns->SetWideColumnValue(std::move(resolved_entity));
+    }
+  }
+  if (!s->ok()) {
+    reset_outputs();
+    if (s->IsIncomplete() && value_found != nullptr) {
+      *value_found = false;
+    }
+  }
+
+  clear_blob_state();
+  return true;
+}
+
+void DBImpl::PostprocessMemtableValueRead(
+    const Slice& key, const std::string* timestamp,
+    bool resolve_blob_backed_memtable_value,
+    const BlobFetcher* memtable_blob_fetcher, PinnableSlice* value,
+    PinnableWideColumns* columns, Status* s, bool* is_blob_index,
+    bool* value_found) {
+  if (resolve_blob_backed_memtable_value) {
+    std::string blob_lookup_key_storage;
+    const bool value_resolved = MaybeResolveMemtableBlobValue(
+        GetBlobLookupUserKey(key, timestamp, &blob_lookup_key_storage),
+        memtable_blob_fetcher, value, columns, s, is_blob_index, value_found);
+    if (!value_resolved && value != nullptr && s->ok()) {
+      value->PinSelf();
+    }
+    return;
+  }
+
+  if (s->ok()) {
+    if (value != nullptr) {
+      value->PinSelf();
+    }
+  } else {
+    if (value != nullptr) {
+      value->Reset();
+    }
+    if (columns != nullptr) {
+      columns->Reset();
+    }
+  }
+}
+
 bool DBImpl::MaybeResolveDirectWriteValue(
     const ReadOptions& read_options, const Slice& key,
     bool resolve_direct_write_value, const Version* current,
@@ -2963,6 +3264,14 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key,
   }
   const bool resolve_direct_write_value =
       partition_mgr != nullptr && (is_blob_ptr == &is_blob_index);
+  std::optional<BlobFetcher> memtable_blob_fetcher;
+  if (partition_mgr != nullptr) {
+    memtable_blob_fetcher.emplace(sv->current, read_options,
+                                  cfd->blob_file_cache(),
+                                  /*allow_write_path_fallback=*/true);
+  }
+  const BlobFetcher* memtable_blob_fetcher_ptr =
+      memtable_blob_fetcher ? &*memtable_blob_fetcher : nullptr;
   std::string blob_lookup_key_storage;
   auto get_blob_lookup_key = [&]() -> Slice {
     return GetBlobLookupUserKey(key, timestamp, &blob_lookup_key_storage);
@@ -2990,19 +3299,21 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key,
                        get_impl_options.columns, timestamp, &s, &merge_context,
                        &max_covering_tombstone_seq, read_options,
                        false /* immutable_memtable */,
-                       get_impl_options.callback, is_blob_ptr)) {
+                       get_impl_options.callback, is_blob_ptr,
+                       /*do_merge=*/true, memtable_blob_fetcher_ptr)) {
         done = true;
         maybe_resolve_memtable_value();
 
         RecordTick(stats_, MEMTABLE_HIT);
       } else if ((s.ok() || s.IsMergeInProgress()) &&
-                 sv->imm->Get(
-                     lkey,
-                     get_impl_options.value ? get_impl_options.value->GetSelf()
-                                            : nullptr,
-                     get_impl_options.columns, timestamp, &s, &merge_context,
-                     &max_covering_tombstone_seq, read_options,
-                     get_impl_options.callback, is_blob_ptr)) {
+                 sv->imm->Get(lkey,
+                              get_impl_options.value
+                                  ? get_impl_options.value->GetSelf()
+                                  : nullptr,
+                              get_impl_options.columns, timestamp, &s,
+                              &merge_context, &max_covering_tombstone_seq,
+                              read_options, get_impl_options.callback,
+                              is_blob_ptr, memtable_blob_fetcher_ptr)) {
         done = true;
         maybe_resolve_memtable_value();
 
@@ -3014,14 +3325,14 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key,
       if (sv->mem->Get(lkey, /*value=*/nullptr, /*columns=*/nullptr,
                        /*timestamp=*/nullptr, &s, &merge_context,
                        &max_covering_tombstone_seq, read_options,
-                       false /* immutable_memtable */, nullptr, nullptr,
-                       false)) {
+                       false /* immutable_memtable */, nullptr, nullptr, false,
+                       memtable_blob_fetcher_ptr)) {
         done = true;
         RecordTick(stats_, MEMTABLE_HIT);
       } else if ((s.ok() || s.IsMergeInProgress()) &&
-                 sv->imm->GetMergeOperands(lkey, &s, &merge_context,
-                                           &max_covering_tombstone_seq,
-                                           read_options)) {
+                 sv->imm->GetMergeOperands(
+                     lkey, &s, &merge_context, &max_covering_tombstone_seq,
+                     read_options, memtable_blob_fetcher_ptr)) {
         done = true;
         RecordTick(stats_, MEMTABLE_HIT);
       }
@@ -3717,6 +4028,14 @@ Status DBImpl::MultiGetImpl(
       (*sorted_keys)[start_key]->column_family);
   ColumnFamilyData* cfd = cfh->cfd();
   auto* partition_mgr = cfd->blob_partition_manager();
+  std::optional<BlobFetcher> memtable_blob_fetcher;
+  if (partition_mgr != nullptr) {
+    memtable_blob_fetcher.emplace(super_version->current, read_options,
+                                  cfd->blob_file_cache(),
+                                  /*allow_write_path_fallback=*/true);
+  }
+  const BlobFetcher* memtable_blob_fetcher_ptr =
+      memtable_blob_fetcher ? &*memtable_blob_fetcher : nullptr;
   // Clear the timestamps for returning results so that we can distinguish
   // between tombstone or key that has never been written
   for (size_t i = start_key; i < start_key + num_keys; ++i) {
@@ -3763,9 +4082,11 @@ Status DBImpl::MultiGetImpl(
          has_unpersisted_data_.load(std::memory_order_relaxed));
     if (!skip_memtable) {
       super_version->mem->MultiGet(read_options, &range, callback,
-                                   false /* immutable_memtable */);
+                                   false /* immutable_memtable */,
+                                   memtable_blob_fetcher_ptr);
       if (!range.empty()) {
-        super_version->imm->MultiGet(read_options, &range, callback);
+        super_version->imm->MultiGet(read_options, &range, callback,
+                                     memtable_blob_fetcher_ptr);
       }
       if (!range.empty()) {
         uint64_t left = range.KeysLeft();
@@ -4142,9 +4463,21 @@ Status DBImpl::CreateColumnFamilyImpl(const ReadOptions& read_options,
                                       const std::string& column_family_name,
                                       ColumnFamilyHandle** handle) {
   options_mutex_.AssertHeld();
-  Status s;
   *handle = nullptr;
 
+  // The empty string is reserved as a "no/unknown column family name" sentinel
+  // in various APIs and serialization formats (e.g.
+  // TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, table
+  // properties), so reject it to avoid ambiguity. Note: prior to this check,
+  // CreateColumnFamily("") would silently "succeed" but the CF was not
+  // persisted in the manifest, causing data loss across DB reopens.
+  if (column_family_name.empty()) {
+    return Status::InvalidArgument(
+        "Column family name cannot be the empty string");
+  }
+
+  Status s;
+
   DBOptions db_options =
       BuildDBOptions(immutable_db_options_, mutable_db_options_);
   s = ColumnFamilyData::ValidateOptions(db_options, cf_options);
@@ -5522,6 +5855,7 @@ Status DBImpl::DeleteFilesInRanges(ColumnFamilyHandle* column_family,
   }
 
   VersionEdit edit;
+  edit.MarkForegroundOperation();
   std::set<FileMetaData*> deleted_files;
   JobContext job_context(next_job_id_.fetch_add(1), true);
   {
@@ -6200,12 +6534,20 @@ Status DBImpl::GetLatestSequenceForKey(
 
   *seq = kMaxSequenceNumber;
   *found_record_for_key = false;
+  std::optional<BlobFetcher> memtable_blob_fetcher;
+  if (cfd->blob_partition_manager() != nullptr) {
+    memtable_blob_fetcher.emplace(sv->current, read_options,
+                                  cfd->blob_file_cache(),
+                                  /*allow_write_path_fallback=*/true);
+  }
+  const BlobFetcher* memtable_blob_fetcher_ptr =
+      memtable_blob_fetcher ? &*memtable_blob_fetcher : nullptr;
 
   // Check if there is a record for this key in the latest memtable
   sv->mem->Get(lkey, /*value=*/nullptr, /*columns=*/nullptr, timestamp, &s,
                &merge_context, &max_covering_tombstone_seq, seq, read_options,
                false /* immutable_memtable */, nullptr /*read_callback*/,
-               is_blob_index);
+               is_blob_index, /*do_merge=*/true, memtable_blob_fetcher_ptr);
 
   if (!(s.ok() || s.IsNotFound() || s.IsMergeInProgress())) {
     // unexpected error reading memtable.
@@ -6238,7 +6580,8 @@ Status DBImpl::GetLatestSequenceForKey(
   // Check if there is a record for this key in the immutable memtables
   sv->imm->Get(lkey, /*value=*/nullptr, /*columns=*/nullptr, timestamp, &s,
                &merge_context, &max_covering_tombstone_seq, seq, read_options,
-               nullptr /*read_callback*/, is_blob_index);
+               nullptr /*read_callback*/, is_blob_index,
+               memtable_blob_fetcher_ptr);
 
   if (!(s.ok() || s.IsNotFound() || s.IsMergeInProgress())) {
     // unexpected error reading memtable.
@@ -6271,7 +6614,7 @@ Status DBImpl::GetLatestSequenceForKey(
   sv->imm->GetFromHistory(lkey, /*value=*/nullptr, /*columns=*/nullptr,
                           timestamp, &s, &merge_context,
                           &max_covering_tombstone_seq, seq, read_options,
-                          is_blob_index);
+                          is_blob_index, memtable_blob_fetcher_ptr);
 
   if (!(s.ok() || s.IsNotFound() || s.IsMergeInProgress())) {
     // unexpected error reading memtable.
@@ -6623,7 +6966,9 @@ Status DBImpl::IngestExternalFiles(
         assert(!cfd->IsDropped());
         cfds_to_commit.push_back(cfd);
         autovector<VersionEdit*> edit_list;
-        edit_list.push_back(ingestion_jobs[i].edit());
+        auto* edit = ingestion_jobs[i].edit();
+        edit->MarkForegroundOperation();
+        edit_list.push_back(edit);
         edit_lists.push_back(edit_list);
         ++num_entries;
       }
@@ -6638,7 +6983,6 @@ Status DBImpl::IngestExternalFiles(
       }
       status =
           versions_->LogAndApply(cfds_to_commit, read_options, write_options,
-
                                  edit_lists, &mutex_, directories_.GetDbDir());
       // It is safe to update VersionSet last seqno here after LogAndApply since
       // LogAndApply persists last sequence number from VersionEdits,
@@ -6771,6 +7115,7 @@ Status DBImpl::CreateColumnFamilyWithImport(
 
   SuperVersionContext dummy_sv_ctx(/* create_superversion */ true);
   VersionEdit dummy_edit;
+  dummy_edit.MarkForegroundOperation();
   uint64_t next_file_number = 0;
   std::unique_ptr<std::list<uint64_t>::iterator> pending_output_elem;
   {
@@ -6828,9 +7173,10 @@ Status DBImpl::CreateColumnFamilyWithImport(
 
       // Install job edit [Mutex will be unlocked here]
       if (status.ok()) {
-        status = versions_->LogAndApply(cfd, read_options, write_options,
-                                        import_job.edit(), &mutex_,
-                                        directories_.GetDbDir());
+        auto* edit = import_job.edit();
+        edit->MarkForegroundOperation();
+        status = versions_->LogAndApply(cfd, read_options, write_options, edit,
+                                        &mutex_, directories_.GetDbDir());
         if (status.ok()) {
           InstallSuperVersionForConfigChange(cfd, &sv_context);
         }
@@ -7261,6 +7607,7 @@ Status DBImpl::ReserveFileNumbersBeforeIngestion(
   // reuse the file number that has already assigned to the internal file,
   // and this will overwrite the external file. To protect the external
   // file, we have to make sure the file number will never being reused.
+  dummy_edit.MarkForegroundOperation();
   s = versions_->LogAndApply(cfd, read_options, write_options, &dummy_edit,
                              &mutex_, directories_.GetDbDir());
   if (s.ok()) {
@@ -7273,6 +7620,7 @@ Status DBImpl::ReserveFileNumbersBeforeIngestion(
 Status DBImpl::GetCreationTimeOfOldestFile(uint64_t* creation_time) {
   if (mutable_db_options_.max_open_files == -1) {
     uint64_t oldest_time = std::numeric_limits<uint64_t>::max();
+    std::once_flag waited_for_async_open;
     for (auto cfd : *versions_->GetColumnFamilySet()) {
       if (!cfd->IsDropped()) {
         uint64_t ctime;
@@ -7280,6 +7628,17 @@ Status DBImpl::GetCreationTimeOfOldestFile(uint64_t* creation_time) {
           SuperVersion* sv = GetAndRefSuperVersion(cfd);
           Version* version = sv->current;
           version->GetCreationTimeOfOldestFile(&ctime);
+          // For modern DBs, manifest carries file_creation_time and the
+          // first call returns the real value. We only need to wait for
+          // BGWorkAsyncFileOpen on legacy DBs whose manifest lacks
+          // file_creation_time -- those rely on the pinned reader, which is
+          // null until async file open completes.
+          if (ctime == 0 && immutable_db_options_.open_files_async) {
+            std::call_once(waited_for_async_open, [&]() {
+              WaitForAsyncFileOpen();
+              version->GetCreationTimeOfOldestFile(&ctime);
+            });
+          }
           ReturnAndCleanupSuperVersion(cfd, sv);
         }
 

db/db_impl/db_impl.h

@@ -15,6 +15,8 @@
 #include <limits>
 #include <list>
 #include <map>
+#include <memory>
+#include <optional>
 #include <set>
 #include <string>
 #include <unordered_map>
@@ -1096,6 +1098,7 @@ class DBImpl : public DB {
     superversions_to_free_queue_.push_back(sv);
   }
 
+  void EnableTrackPublishedSeqInSnapshotContext();
   void SetSnapshotChecker(SnapshotChecker* snapshot_checker);
 
   // Fill JobContext with snapshot information needed by flush and compaction.
@@ -1371,6 +1374,7 @@ class DBImpl : public DB {
 
  protected:
   const std::string dbname_;
+  const bool read_only_;
   // TODO(peterd): unify with VersionSet::db_id_
   std::string db_id_;
   // db_session_id_ is an identifier that gets reset
@@ -1452,6 +1456,13 @@ class DBImpl : public DB {
   std::atomic<int> next_job_id_ = 1;
 
   std::atomic<bool> shutting_down_ = false;
+  // Protected by mutex_. This is separate from shutting_down_ because
+  // OnDBShutdownBegin must fire before shutting_down_ is published, and the
+  // notification releases mutex_ while invoking listeners. Marking that the
+  // notification has started prevents a concurrent or reentrant
+  // CancelAllBackgroundWork() call from firing it again during that unlocked
+  // callback window.
+  bool shutdown_notification_sent_ = false;
 
   // No new background jobs can be queued if true. This is used to prevent new
   // background jobs from being queued after WaitForCompact() completes waiting
@@ -1526,6 +1537,16 @@ class DBImpl : public DB {
                                    const Status& st,
                                    const CompactionJobStats& job_stats,
                                    int job_id);
+  // Fires the OnCompactionPreCommit listener callback. Mirrors
+  // NotifyOnCompactionCompleted but is invoked before ReleaseCompactionFiles()
+  // (i.e. while input files still have being_compacted == true). Idempotent:
+  // safe to call from multiple potential release sites; only fires once per
+  // compaction, and only if NotifyOnCompactionBegin previously fired.
+  void NotifyOnCompactionPreCommit(ColumnFamilyData* cfd, Compaction* c,
+                                   const Status& st,
+                                   const CompactionJobStats& job_stats,
+                                   int job_id);
+  void NotifyOnDBShutdownBegin();
   void NotifyOnMemTableSealed(ColumnFamilyData* cfd,
                               const MemTableInfo& mem_table_info);
 
@@ -1854,6 +1875,11 @@ class DBImpl : public DB {
   // Background work function for async file opening.
   static void BGWorkAsyncFileOpen(void* arg);
 
+  // Block the until any in-flight async file open work has
+  // completed. No-op when open_files_async is false. Returns early if
+  // shutdown begins.
+  void WaitForAsyncFileOpen();
+
   void InvokeWalFilterIfNeededOnColumnFamilyToWalNumberMap();
 
   // Return true to proceed with current WAL record whose content is stored in
@@ -1869,6 +1895,7 @@ class DBImpl : public DB {
 
  private:
   friend class DB;
+  friend class DBImplReadOnly;
   friend class DBImplSecondary;
   friend class ErrorHandler;
   friend class InternalStats;
@@ -2567,10 +2594,15 @@ class DBImpl : public DB {
   // Helper function to perform trivial move by updating manifest metadata
   // without rewriting data files. This is called when IsTrivialMove() is true.
   // REQUIRES: mutex held
-  // Returns: Status of the trivial move operation
-  Status PerformTrivialMove(Compaction& c, LogBuffer* log_buffer,
-                            bool& compaction_released, size_t& moved_files,
-                            size_t& moved_bytes);
+  // Populates the Compaction's edit with DeleteFile/AddFile entries for
+  // a trivial move (no data rewriting). Does not commit the edit.
+  void PrepareTrivialMoveEdit(Compaction& c, LogBuffer* log_buffer,
+                              size_t& moved_files, size_t& moved_bytes);
+
+  // REQUIRES: mutex held, PrepareTrivialMoveEdit already called
+  // Commits the trivial move by calling LogAndApply on the prepared edit.
+  // Returns: Status of the manifest write
+  Status CommitTrivialMove(Compaction& c, bool& compaction_released);
 
   // REQUIRES: mutex unlocked
   void TrackOrUntrackFiles(const std::vector<std::string>& existing_data_files,
@@ -2743,6 +2775,8 @@ class DBImpl : public DB {
 
   Status MaybeReleaseTimestampedSnapshotsAndCheck();
 
+  Status MaybeWriteWalMarkersToManifestOnClose();
+
   Status CloseHelper();
 
   void WaitForBackgroundWork();
@@ -2801,7 +2835,68 @@ class DBImpl : public DB {
   size_t GetWalPreallocateBlockSize(uint64_t write_buffer_size) const;
   Env::WriteLifeTimeHint CalculateWALWriteHint() { return Env::WLTH_SHORT; }
 
-  IOStatus CreateWAL(const WriteOptions& write_options, uint64_t log_file_num,
+  // Returns true when async WAL precreation is enabled and compatible with the
+  // active WAL strategy. WAL recycling already avoids file creation latency, so
+  // precreation is disabled when recycle_log_file_num is non-zero.
+  bool AsyncWALPrecreateEnabled() const;
+
+  // A WAL file that has a reserved file number and may have an opened writer,
+  // but has not been added to DBImpl's in-memory logical WAL tracking lists
+  // (logs_ and alive_wal_files_).
+  struct UnpublishedWAL {
+    uint64_t log_number = 0;
+    std::unique_ptr<log::Writer> writer;
+
+    UnpublishedWAL() = default;
+    UnpublishedWAL(const UnpublishedWAL&) = delete;
+    UnpublishedWAL& operator=(const UnpublishedWAL&) = delete;
+
+    UnpublishedWAL(UnpublishedWAL&& other) noexcept {
+      *this = std::move(other);
+    }
+    UnpublishedWAL& operator=(UnpublishedWAL&& other) noexcept {
+      if (this != &other) {
+        log_number = other.log_number;
+        writer = std::move(other.writer);
+        other.Reset();
+      }
+      return *this;
+    }
+
+    void Reset() {
+      log_number = 0;
+      writer.reset();
+    }
+  };
+
+  // Reserves the next WAL file number and schedules a HIGH-priority background
+  // task to precreate that WAL file. A precreated WAL is not a logical WAL
+  // until a foreground WAL rotation consumes it.
+  void MaybeScheduleAsyncWALPrecreate(size_t preallocate_block_size);
+
+  // Background task for opening the reserved future WAL and publishing the
+  // result under mutex_.
+  static void BGWorkAsyncWALPrecreate(void* arg);
+
+  // Waits for an in-flight async WAL precreation and returns a prepared WAL if
+  // one is available. If precreation failed, returns an empty WAL and lets the
+  // foreground rotation create the WAL synchronously. Caller must hold mutex_.
+  UnpublishedWAL WaitForAsyncWALPrecreate();
+
+  // Opens and preallocates a WAL writer without writing logical WAL records.
+  // Used by async WAL precreation and by synchronous WAL creation.
+  IOStatus CreateWALWriter(const DBOptions& db_options, uint64_t log_file_num,
+                           uint64_t recycle_log_number,
+                           size_t preallocate_block_size,
+                           UnpublishedWAL* new_wal);
+
+  // Starts an opened WAL file by writing the initial records required before it
+  // can be installed as the current WAL for foreground writes.
+  IOStatus StartWALFile(const WriteOptions& write_options,
+                        const PredecessorWALInfo& predecessor_wal_info,
+                        log::Writer* new_log);
+  IOStatus CreateWAL(const DBOptions& db_options,
+                     const WriteOptions& write_options, uint64_t log_file_num,
                      uint64_t recycle_log_number, size_t preallocate_block_size,
                      const PredecessorWALInfo& predecessor_wal_info,
                      log::Writer** new_log);
@@ -2944,6 +3039,29 @@ class DBImpl : public DB {
       bool resolve_direct_write_value, const Version* current,
       ColumnFamilyData* cfd, PinnableSlice* value, PinnableWideColumns* columns,
       Status* s, bool* is_blob_index, bool* value_found = nullptr);
+  // Resolves memtable read results that still carry blob references through
+  // either a raw blob-index payload in `value` or unresolved blob columns in
+  // `columns`. Unlike the direct-write helper above, this path only depends on
+  // a BlobFetcher and therefore works for read-only/secondary DBs.
+  static bool MaybeResolveMemtableBlobValue(const Slice& key,
+                                            const BlobFetcher* blob_fetcher,
+                                            PinnableSlice* value,
+                                            PinnableWideColumns* columns,
+                                            Status* s, bool* is_blob_index,
+                                            bool* value_found = nullptr);
+  // Completes read-only/secondary memtable Get()/GetEntity() hits by resolving
+  // blob-backed payloads when `resolve_blob_backed_memtable_value` is true,
+  // pinning plain values on success, and clearing outputs on error. When the
+  // caller explicitly requested raw blob indices via
+  // `GetImplOptions::is_blob_index`, this helper leaves that payload
+  // untouched. `memtable_blob_fetcher` may be null when blob support is
+  // disabled for the column family.
+  static void PostprocessMemtableValueRead(
+      const Slice& key, const std::string* timestamp,
+      bool resolve_blob_backed_memtable_value,
+      const BlobFetcher* memtable_blob_fetcher, PinnableSlice* value,
+      PinnableWideColumns* columns, Status* s, bool* is_blob_index,
+      bool* value_found = nullptr);
 
   template <typename IterType, typename ImplType,
             typename ErrorIteratorFuncType>
@@ -3274,6 +3392,28 @@ class DBImpl : public DB {
   AsyncFileOpenState bg_async_file_open_state_ =
       AsyncFileOpenState::kNotScheduled;
 
+  // State machine for the single async WAL precreation slot protected by
+  // mutex_. Background precreation failure returns to kNotScheduled; foreground
+  // rotation handles it the same as no prepared WAL and creates one
+  // synchronously. kScheduled owns a reserved file number; kReady owns an
+  // opened writer that has not been started or added to logical WAL tracking.
+  enum class AsyncWALPrecreateState : uint8_t {
+    kNotScheduled = 0,  // No WAL precreate work is in-flight or ready.
+    kScheduled,         // Background task owns creation of the reserved WAL.
+    kReady,             // Reserved WAL writer is open but not logically live.
+  };
+
+  // Protected by mutex_. Tracks at most one background precreated WAL. A
+  // precreated WAL is only reserved empty storage until SwitchMemtable()
+  // consumes it and installs it in DBImpl's in-memory logical WAL tracking
+  // lists (logs_ and alive_wal_files_).
+  AsyncWALPrecreateState async_wal_precreate_state_ =
+      AsyncWALPrecreateState::kNotScheduled;
+
+  // Reserved in-flight/ready precreated WAL. The writer is populated only while
+  // state is kReady.
+  UnpublishedWAL async_wal_precreate_wal_;
+
   std::deque<ManualCompactionState*> manual_compaction_dequeue_;
 
   // shall we disable deletion of obsolete files
@@ -3348,6 +3488,13 @@ class DBImpl : public DB {
   // Callback for compaction to check if a key is visible to a snapshot.
   // REQUIRES: mutex held
   std::unique_ptr<SnapshotChecker> snapshot_checker_;
+  // When set, InitSnapshotContext() appends GetLastPublishedSequence() to the
+  // job's snapshot_seqs (if not already present) so that flush/compaction
+  // preserves the published-sequence boundary even when no explicit user
+  // snapshot exists there. Unlike taking a real ManagedSnapshot, this just
+  // pins the seqno into the snapshot list: no allocation, no snapshot list
+  // mutation, and no SnapshotChecker.
+  bool track_published_seq_in_snapshot_context_ = false;
 
   // Callback for when the cached_recoverable_state_ is written to memtable
   // Only to be set during initialization

db/db_impl/db_impl_compaction_flush.cc

@@ -874,6 +874,24 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
         directories_.GetDbDir(), log_buffer);
   }
 
+  if (!s.ok()) {
+    // If the atomic flush's combined MANIFEST write failed, the output files
+    // are cached in the table cache (added by BuildTable during FlushJob::Run)
+    // but never installed into any Version. Evict them now so that
+    // TEST_VerifyNoObsoleteFilesCached does not fire during Close() when the
+    // FindObsoleteFiles full-scan backstop is disabled by
+    // metadata_read_fault_one_in.
+    for (int i = 0; i != num_cfs; ++i) {
+      if (exec_status[i].first && exec_status[i].second.ok() &&
+          !switched_to_mempurge[i] && file_meta[i].fd.GetFileSize() > 0) {
+        TableCache::ReleaseObsolete(
+            cfds[i]->table_cache()->get_cache().get(),
+            file_meta[i].fd.GetNumber(), nullptr /*handle*/,
+            all_mutable_cf_options[i].uncache_aggressiveness);
+      }
+    }
+  }
+
   for (int i = 0; i != num_cfs; ++i) {
     auto* mgr = cfds[i]->blob_partition_manager();
     if (mgr == nullptr || prepared_blob_generations[i] == 0) {
@@ -1563,9 +1581,8 @@ Status DBImpl::CompactFiles(const CompactionOptions& compact_options,
   return s;
 }
 
-Status DBImpl::PerformTrivialMove(Compaction& c, LogBuffer* log_buffer,
-                                  bool& compaction_released,
-                                  size_t& moved_files, size_t& moved_bytes) {
+void DBImpl::PrepareTrivialMoveEdit(Compaction& c, LogBuffer* log_buffer,
+                                    size_t& moved_files, size_t& moved_bytes) {
   mutex_.AssertHeld();
 
   ROCKS_LOG_BUFFER(log_buffer, "[%s] Moving %d files to level-%d\n",
@@ -1598,6 +1615,10 @@ Status DBImpl::PerformTrivialMove(Compaction& c, LogBuffer* log_buffer,
     }
     moved_files += c.num_input_files(l);
   }
+}
+
+Status DBImpl::CommitTrivialMove(Compaction& c, bool& compaction_released) {
+  mutex_.AssertHeld();
 
   // Install the new version
   const ReadOptions read_options(Env::IOActivity::kCompaction);
@@ -1722,8 +1743,8 @@ Status DBImpl::CompactFilesImpl(
     bool compaction_released = false;
     size_t moved_files = 0;
     size_t moved_bytes = 0;
-    Status status = PerformTrivialMove(
-        *c.get(), log_buffer, compaction_released, moved_files, moved_bytes);
+    PrepareTrivialMoveEdit(*c.get(), log_buffer, moved_files, moved_bytes);
+    Status status = CommitTrivialMove(*c.get(), compaction_released);
 
     if (status.ok()) {
       InstallSuperVersionAndScheduleWork(
@@ -1991,6 +2012,46 @@ void DBImpl::NotifyOnCompactionCompleted(
   // flush process.
 }
 
+void DBImpl::NotifyOnCompactionPreCommit(
+    ColumnFamilyData* cfd, Compaction* c, const Status& st,
+    const CompactionJobStats& compaction_job_stats, const int job_id) {
+  if (immutable_db_options_.listeners.size() == 0U) {
+    return;
+  }
+  mutex_.AssertHeld();
+  if (shutting_down_.load(std::memory_order_acquire)) {
+    return;
+  }
+
+  // Only fire if OnCompactionBegin has fired for this compaction.
+  if (c->ShouldNotifyOnCompactionCompleted() == false) {
+    return;
+  }
+  // Idempotency: ensure listeners observe at most one
+  // OnCompactionPreCommit per compaction lifecycle, even though the
+  // notify helper is invoked at multiple potential ReleaseCompactionFiles()
+  // sites for safety.
+  if (c->WasNotifyOnCompactionPreCommitCalled()) {
+    return;
+  }
+  c->SetNotifyOnCompactionPreCommitCalled();
+
+  int num_l0_files = cfd->current()->storage_info()->NumLevelFiles(0);
+  // release lock while notifying events
+  mutex_.Unlock();
+  TEST_SYNC_POINT("DBImpl::NotifyOnCompactionPreCommit::UnlockMutex");
+  {
+    CompactionJobInfo info{};
+    info.num_l0_files = num_l0_files;
+    BuildCompactionJobInfo(cfd, c, st, compaction_job_stats, job_id, &info);
+    for (const auto& listener : immutable_db_options_.listeners) {
+      listener->OnCompactionPreCommit(this, info);
+    }
+    info.status.PermitUncheckedError();
+  }
+  mutex_.Lock();
+}
+
 // REQUIREMENT: block all background work by calling PauseBackgroundWork()
 // before calling this function
 // TODO (hx235): Replace Status::NotSupported() with Status::Aborted() for
@@ -3162,7 +3223,7 @@ void DBImpl::ResumeAllCompactions() {
 void DBImpl::MaybeScheduleFlushOrCompaction() {
   mutex_.AssertHeld();
   TEST_SYNC_POINT("DBImpl::MaybeScheduleFlushOrCompaction:Start");
-  if (!opened_successfully_) {
+  if (!opened_successfully_ || read_only_) {
     // Compaction may introduce data race to DB open
     return;
   }
@@ -4006,6 +4067,13 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
       status = Status::ShutdownInProgress();
     } else if (compaction_aborted_.load(std::memory_order_acquire) > 0) {
       status = Status::Incomplete(Status::SubCode::kCompactionAborted);
+      if (!is_prepicked) {
+        // This automatic compaction was scheduled from compaction_queue_, but
+        // abort happened before PickCompactionFromQueue() could remove the
+        // queued CF. Restore the unscheduled count so ResumeAllCompactions()
+        // can schedule the still-queued work.
+        unscheduled_compactions_++;
+      }
     } else if (is_manual &&
                manual_compaction->canceled.load(std::memory_order_acquire)) {
       status = Status::Incomplete(Status::SubCode::kManualCompactionPaused);
@@ -4013,10 +4081,14 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
   } else {
     status = error_handler_.GetBGError();
     // If we get here, it means a hard error happened after this compaction
-    // was scheduled by MaybeScheduleFlushOrCompaction(), but before it got
-    // a chance to execute. Since we didn't pop a cfd from the compaction
-    // queue, increment unscheduled_compactions_
-    unscheduled_compactions_++;
+    // was scheduled by MaybeScheduleFlushOrCompaction(), but before it got a
+    // chance to execute. Since non-prepicked work consumed an unscheduled
+    // compaction credit without popping a cfd from the compaction queue,
+    // restore the credit here. Prepicked work was scheduled directly and did
+    // not consume unscheduled_compactions_.
+    if (!is_prepicked) {
+      unscheduled_compactions_++;
+    }
   }
 
   if (!status.ok()) {
@@ -4237,6 +4309,8 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
     for (const auto& f : *c->inputs(0)) {
       c->edit()->DeleteFile(c->level(), f->fd.GetNumber());
     }
+    NotifyOnCompactionPreCommit(c->column_family_data(), c.get(), status,
+                                compaction_job_stats, job_context->job_id);
     status = versions_->LogAndApply(
         c->column_family_data(), read_options, write_options, c->edit(),
         &mutex_, directories_.GetDbDir(),
@@ -4455,6 +4529,8 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
         ++out_file_metadata_it;
       }
 
+      NotifyOnCompactionPreCommit(c->column_family_data(), c.get(), status,
+                                  compaction_job_stats, job_context->job_id);
       status = versions_->LogAndApply(
           c->column_family_data(), read_options, write_options, c->edit(),
           &mutex_, directories_.GetDbDir(),
@@ -4531,8 +4607,12 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
     // Perform the trivial move
     size_t moved_files = 0;
     size_t moved_bytes = 0;
-    status = PerformTrivialMove(*c.get(), log_buffer, compaction_released,
-                                moved_files, moved_bytes);
+    // Populate the edit first so that CompactionJobInfo has output file
+    // info when OnCompactionPreCommit fires.
+    PrepareTrivialMoveEdit(*c.get(), log_buffer, moved_files, moved_bytes);
+    NotifyOnCompactionPreCommit(c->column_family_data(), c.get(), status,
+                                compaction_job_stats, job_context->job_id);
+    status = CommitTrivialMove(*c.get(), compaction_released);
     io_s = versions_->io_status();
     InstallSuperVersionAndScheduleWork(
         c->column_family_data(), job_context->superversion_contexts.data());
@@ -4655,6 +4735,10 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
       ReleaseOptionsFileNumber(min_options_file_number_elem);
     }
 
+    // Fire OnCompactionPreCommit before compaction_job.Install runs
+    // ReleaseCompactionFiles in its manifest write callback.
+    NotifyOnCompactionPreCommit(c->column_family_data(), c.get(), status,
+                                compaction_job_stats, job_context->job_id);
     status = compaction_job.Install(&compaction_released);
     io_s = compaction_job.io_status();
     if (status.ok()) {
@@ -4674,6 +4758,14 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
 
   if (c != nullptr) {
     if (!compaction_released) {
+      // Safety net: if any of the paths above did not fire the
+      // OnCompactionPreCommit notification (e.g. status was not OK and
+      // an early-out skipped the LogAndApply / Install), still fire the
+      // notify here while files are about to be released. The notify is
+      // idempotent and is a no-op if it has already fired (or if Begin never
+      // fired).
+      NotifyOnCompactionPreCommit(c->column_family_data(), c.get(), status,
+                                  compaction_job_stats, job_context->job_id);
       c->ReleaseCompactionFiles(status);
     } else {
 #ifndef NDEBUG
@@ -5114,6 +5206,11 @@ void DBImpl::MarkAsGrabbedForPurge(uint64_t file_number) {
   files_grabbed_for_purge_.insert(file_number);
 }
 
+void DBImpl::EnableTrackPublishedSeqInSnapshotContext() {
+  InstrumentedMutexLock l(&mutex_);
+  track_published_seq_in_snapshot_context_ = true;
+}
+
 void DBImpl::SetSnapshotChecker(SnapshotChecker* snapshot_checker) {
   InstrumentedMutexLock l(&mutex_);
   // snapshot_checker_ should only set once. If we need to set it multiple
@@ -5147,6 +5244,19 @@ void DBImpl::InitSnapshotContext(JobContext* job_context) {
   SequenceNumber earliest_write_conflict_snapshot = kMaxSequenceNumber;
   std::vector<SequenceNumber> snapshot_seqs =
       snapshots_.GetAll(&earliest_write_conflict_snapshot);
+
+  // The SnapshotChecker path above already adds a managed snapshot to
+  // snapshot_seqs.
+  if (track_published_seq_in_snapshot_context_ && !snapshot_checker) {
+    // Pin the published-sequence boundary into snapshot_seqs without taking
+    // a real snapshot. This prevents flush/compaction from collapsing a version
+    // visible at the published boundary into a newer unpublished seqno.
+    const SequenceNumber published_seq = GetLastPublishedSequence();
+    if (snapshot_seqs.empty() || snapshot_seqs.back() < published_seq) {
+      snapshot_seqs.push_back(published_seq);
+    }
+  }
+  TEST_SYNC_POINT("DBImpl::InitSnapshotContext:BeforeInit");
   job_context->InitSnapshotContext(
       snapshot_checker, std::move(managed_snapshot),
       earliest_write_conflict_snapshot, std::move(snapshot_seqs));

db/db_impl/db_impl_files.cc

@@ -1186,8 +1186,9 @@ std::set<std::string> DBImpl::CollectAllDBPaths() {
 
 Status DBImpl::MaybeUpdateNextFileNumber(RecoveryContext* recovery_ctx) {
   mutex_.AssertHeld();
-  uint64_t next_file_number = versions_->current_next_file_number();
-  uint64_t largest_file_number = next_file_number;
+  const uint64_t prev_next_file_number = versions_->current_next_file_number();
+  uint64_t largest_file_number = prev_next_file_number;
+  bool on_disk_file_advanced_counter = false;
   Status s;
   for (const auto& path : CollectAllDBPaths()) {
     std::vector<std::string> files;
@@ -1201,7 +1202,15 @@ Status DBImpl::MaybeUpdateNextFileNumber(RecoveryContext* recovery_ctx) {
       if (!ParseFileName(fname, &number, &type)) {
         continue;
       }
-      const std::string normalized_fpath = path + kFilePathSeparator + fname;
+      std::string normalized_fpath = path;
+      normalized_fpath += kFilePathSeparator;
+      normalized_fpath.append(fname);
+      // Use >= so a crashed-mid-allocation file at exactly
+      // prev_next_file_number triggers the advance -- otherwise the next
+      // NewFileNumber() would collide with it.
+      if (number >= prev_next_file_number) {
+        on_disk_file_advanced_counter = true;
+      }
       largest_file_number = std::max(largest_file_number, number);
       if ((type == kTableFile || type == kBlobFile)) {
         recovery_ctx->existing_data_files_.push_back(normalized_fpath);
@@ -1212,16 +1221,31 @@ Status DBImpl::MaybeUpdateNextFileNumber(RecoveryContext* recovery_ctx) {
     return s;
   }
 
-  if (largest_file_number >= next_file_number) {
-    versions_->next_file_number_.store(largest_file_number + 1);
-  }
+  // Legacy bug: initializing largest_file_number from prev_next_file_number
+  // made the post-loop check trivially true and emitted a noop SetNextFile
+  // on every recovery.
+  // Preserved verbatim by default (some tests pin file numbers); skipped
+  // only when the option is on AND we're not in salvage mode (which
+  // requires the MANIFEST rewrite this emit triggers).
+  const bool optimize_manifest_for_recovery =
+      mutable_db_options_.optimize_manifest_for_recovery &&
+      !immutable_db_options_.best_efforts_recovery;
 
-  VersionEdit edit;
-  edit.SetNextFile(versions_->next_file_number_.load());
-  assert(versions_->GetColumnFamilySet());
-  ColumnFamilyData* default_cfd = versions_->GetColumnFamilySet()->GetDefault();
-  assert(default_cfd);
-  recovery_ctx->UpdateVersionEdits(default_cfd, edit);
+  if (!optimize_manifest_for_recovery || on_disk_file_advanced_counter) {
+    if (largest_file_number >= prev_next_file_number) {
+      versions_->next_file_number_.store(largest_file_number + 1);
+    }
+    TEST_SYNC_POINT("DBImpl::MaybeUpdateNextFileNumber:EmitEdit");
+    VersionEdit edit;
+    edit.SetNextFile(versions_->next_file_number_.load());
+    assert(versions_->GetColumnFamilySet());
+    ColumnFamilyData* default_cfd =
+        versions_->GetColumnFamilySet()->GetDefault();
+    assert(default_cfd);
+    recovery_ctx->UpdateVersionEdits(default_cfd, edit);
+  } else {
+    TEST_SYNC_POINT("DBImpl::Recovery:SkippedNoopEdit:NextFileNumber");
+  }
   return s;
 }
 }  // namespace ROCKSDB_NAMESPACE

db/db_impl/db_impl_follower.cc

@@ -295,7 +295,8 @@ Status DB::OpenAsFollower(
   impl->versions_.reset(new ReactiveVersionSet(
       dbname, &impl->immutable_db_options_, impl->mutable_db_options_,
       impl->file_options_, impl->table_cache_.get(),
-      impl->write_buffer_manager_, &impl->write_controller_, impl->io_tracer_));
+      impl->write_buffer_manager_, &impl->write_controller_, impl->io_tracer_,
+      impl->db_id_, impl->db_session_id_));
   impl->column_family_memtables_.reset(
       new ColumnFamilyMemTablesImpl(impl->versions_->GetColumnFamilySet()));
   impl->wal_in_db_path_ = impl->immutable_db_options_.IsWalDirSameAsDBPath();

db/db_impl/db_impl_open.cc

@@ -21,6 +21,7 @@
 #include "file/writable_file_writer.h"
 #include "logging/logging.h"
 #include "monitoring/persistent_stats_history.h"
+#include "monitoring/statistics_impl.h"
 #include "monitoring/thread_status_util.h"
 #include "options/options_helper.h"
 #include "rocksdb/options.h"
@@ -122,6 +123,16 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src,
     result.recycle_log_file_num = 0;
   }
 
+  if (result.async_wal_precreate && result.recycle_log_file_num != 0) {
+    // Async WAL precreation reserves a future log number, while WAL recycling
+    // chooses from old WAL files. Keep the recycling behavior and disable only
+    // the async optimization.
+    result.async_wal_precreate = false;
+    ROCKS_LOG_WARN(result.info_log,
+                   "async_wal_precreate is disabled since "
+                   "recycle_log_file_num is non-zero");
+  }
+
   if (result.db_paths.size() == 0) {
     result.db_paths.emplace_back(dbname, std::numeric_limits<uint64_t>::max());
   } else if (result.wal_dir.empty()) {
@@ -680,8 +691,18 @@ Status DBImpl::Recover(
   } else if (immutable_db_options_.write_dbid_to_manifest && recovery_ctx) {
     VersionEdit edit;
     s = SetupDBId(write_options, read_only, is_new_db, is_retry, &edit);
-    recovery_ctx->UpdateVersionEdits(
-        versions_->GetColumnFamilySet()->GetDefault(), edit);
+    // best_efforts_recovery rebuilds CURRENT/MANIFEST as the side-effect
+    // of LogAndApplyForRecovery emitting an edit, so the optimization is
+    // disabled there (see optimize_manifest_for_recovery doc).
+    const bool optimize_manifest_for_recovery =
+        mutable_db_options_.optimize_manifest_for_recovery &&
+        !immutable_db_options_.best_efforts_recovery;
+    if (!optimize_manifest_for_recovery || edit.HasDbId()) {
+      recovery_ctx->UpdateVersionEdits(
+          versions_->GetColumnFamilySet()->GetDefault(), edit);
+    } else {
+      TEST_SYNC_POINT("DBImpl::Recovery:SkippedNoopEdit:SetupDBId");
+    }
   } else {
     s = SetupDBId(write_options, read_only, is_new_db, is_retry, nullptr);
   }
@@ -770,6 +791,10 @@ Status DBImpl::Recover(
       // otherwise, in the future, if WAL tracking is enabled again,
       // since the WALs deleted when WAL tracking is disabled are not persisted
       // into MANIFEST, WAL check may fail.
+      //
+      // Intentionally NOT gated by optimize_manifest_for_recovery:
+      // this edit is safety-critical for a future re-enable of WAL tracking,
+      // even though it can look noop-ish under the current session.
       VersionEdit edit;
       WalNumber max_wal_number =
           versions_->GetWalSet().GetWals().rbegin()->first;
@@ -785,9 +810,19 @@ Status DBImpl::Recover(
 
     if (!wal_files.empty()) {
       if (error_if_wal_file_exists) {
-        return Status::Corruption(
-            "The db was opened in readonly mode with error_if_wal_file_exists"
-            "flag but a WAL file already exists");
+        for (const auto& wal_file : wal_files) {
+          uint64_t bytes;
+          s = env_->GetFileSize(wal_file.second, &bytes);
+          if (!s.ok()) {
+            return s;
+          }
+          if (bytes > 0) {
+            return Status::Corruption(
+                "The db was opened in readonly mode with "
+                "error_if_wal_file_exists flag but a non-empty WAL file "
+                "already exists");
+          }
+        }
       } else if (error_if_data_exists_in_wals) {
         for (auto& wal_file : wal_files) {
           uint64_t bytes;
@@ -983,7 +1018,11 @@ Status DBImpl::InitPersistStatsColumnFamily() {
 
 Status DBImpl::LogAndApplyForRecovery(const RecoveryContext& recovery_ctx) {
   mutex_.AssertHeld();
-  assert(versions_->descriptor_log_ == nullptr);
+  // descriptor_log_ is normally null after Recover, but when
+  // reuse_manifest_on_open is set VersionSet::Recover may have already
+  // bound a log::Writer to the existing MANIFEST for append.
+  assert(versions_->descriptor_log_ == nullptr ||
+         immutable_db_options_.reuse_manifest_on_open);
   const ReadOptions read_options(Env::IOActivity::kDBOpen);
   const WriteOptions write_options(Env::IOActivity::kDBOpen);
 
@@ -1870,25 +1909,47 @@ Status DBImpl::MaybeFlushFinalMemtableOrRestoreActiveLogFiles(
       versions_->MarkFileNumberUsed(max_wal_number + 1);
       assert(recovery_ctx != nullptr);
 
+      const bool optimize_manifest_for_recovery =
+          mutable_db_options_.optimize_manifest_for_recovery &&
+          !immutable_db_options_.best_efforts_recovery;
       for (auto* cfd : *versions_->GetColumnFamilySet()) {
         auto iter = version_edits->find(cfd->GetID());
         assert(iter != version_edits->end());
-        recovery_ctx->UpdateVersionEdits(cfd, iter->second);
+        const VersionEdit& cf_edit = iter->second;
+        if (!optimize_manifest_for_recovery ||
+            cf_edit.ShouldEmitPerColumnFamilyRecoveryEdit(
+                cfd->GetLogNumber())) {
+          recovery_ctx->UpdateVersionEdits(cfd, cf_edit);
+        } else {
+          TEST_SYNC_POINT("DBImpl::Recovery:SkippedNoopEdit:PerCF");
+        }
       }
 
       if (flushed || !data_seen) {
+        const uint64_t new_min_log = max_wal_number + 1;
         VersionEdit wal_deletion;
+        bool emit_wal_deletion = false;
         if (immutable_db_options_.track_and_verify_wals_in_manifest) {
-          wal_deletion.DeleteWalsBefore(max_wal_number + 1);
+          // Determining whether DeleteWalsBefore actually shrinks WalSet
+          // membership requires WalSet state outside this site, so emit
+          // unconditionally (pre-existing behavior).
+          wal_deletion.DeleteWalsBefore(new_min_log);
+          emit_wal_deletion = true;
         }
-        if (!allow_2pc()) {
-          // In non-2pc mode, flushing the memtables of the column families
-          // means we can advance min_log_number_to_keep.
-          wal_deletion.SetMinLogNumberToKeep(max_wal_number + 1);
+        const bool min_log_advances =
+            new_min_log > versions_->min_log_number_to_keep();
+        if (!allow_2pc() &&
+            (!optimize_manifest_for_recovery || min_log_advances)) {
+          wal_deletion.SetMinLogNumberToKeep(new_min_log);
+          emit_wal_deletion = true;
         }
         assert(versions_->GetColumnFamilySet() != nullptr);
-        recovery_ctx->UpdateVersionEdits(
-            versions_->GetColumnFamilySet()->GetDefault(), wal_deletion);
+        if (!optimize_manifest_for_recovery || emit_wal_deletion) {
+          recovery_ctx->UpdateVersionEdits(
+              versions_->GetColumnFamilySet()->GetDefault(), wal_deletion);
+        } else {
+          TEST_SYNC_POINT("DBImpl::Recovery:SkippedNoopEdit:WalDeletion");
+        }
       }
     }
   }
@@ -2173,6 +2234,16 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
 
   constexpr int level = 0;
 
+  if (s.ok() && has_output) {
+    // Before publishing any recovery edit, make the new SST's directory entry
+    // durable. Reused MANIFESTs have no CURRENT update, and fresh MANIFEST
+    // publication should not be relied on to order the recovered SST entry.
+    s = GetDataDir(cfd, meta.fd.GetPathId())
+            ->FsyncWithDirOptions(
+                IOOptions(), nullptr,
+                DirFsyncOptions(DirFsyncOptions::FsyncReason::kNewFileSynced));
+  }
+
   if (s.ok() && has_output) {
     edit->AddFile(level, meta.fd.GetNumber(), meta.fd.GetPathId(),
                   meta.fd.GetFileSize(), meta.smallest, meta.largest,
@@ -2337,16 +2408,17 @@ Status DB::OpenAndTrimHistory(
   return s;
 }
 
-IOStatus DBImpl::CreateWAL(const WriteOptions& write_options,
-                           uint64_t log_file_num, uint64_t recycle_log_number,
-                           size_t preallocate_block_size,
-                           const PredecessorWALInfo& predecessor_wal_info,
-                           log::Writer** new_log) {
+IOStatus DBImpl::CreateWALWriter(const DBOptions& db_options,
+                                 uint64_t log_file_num,
+                                 uint64_t recycle_log_number,
+                                 size_t preallocate_block_size,
+                                 UnpublishedWAL* new_wal) {
+  assert(new_wal);
+  assert(new_wal->log_number == 0);
+  assert(new_wal->writer == nullptr);
   IOStatus io_s;
   std::unique_ptr<FSWritableFile> lfile;
 
-  DBOptions db_options =
-      BuildDBOptions(immutable_db_options_, mutable_db_options_);
   FileOptions opt_file_options =
       fs_->OptimizeForLogWrite(file_options_, db_options);
   opt_file_options.write_hint = CalculateWALWriteHint();
@@ -2384,21 +2456,162 @@ IOStatus DBImpl::CreateWAL(const WriteOptions& write_options,
         Histograms::HISTOGRAM_ENUM_MAX /* hist_type */, listeners, nullptr,
         tmp_set.Contains(FileType::kWalFile),
         tmp_set.Contains(FileType::kWalFile)));
-    *new_log = new log::Writer(std::move(file_writer), log_file_num,
-                               immutable_db_options_.recycle_log_file_num > 0,
-                               immutable_db_options_.manual_wal_flush,
-                               immutable_db_options_.wal_compression,
-                               immutable_db_options_.track_and_verify_wals);
-    io_s = (*new_log)->AddCompressionTypeRecord(write_options);
-    if (io_s.ok()) {
-      io_s = (*new_log)->MaybeAddPredecessorWALInfo(write_options,
-                                                    predecessor_wal_info);
-    }
+    new_wal->log_number = log_file_num;
+    new_wal->writer = std::make_unique<log::Writer>(
+        std::move(file_writer), log_file_num,
+        immutable_db_options_.recycle_log_file_num > 0,
+        immutable_db_options_.manual_wal_flush,
+        immutable_db_options_.wal_compression,
+        immutable_db_options_.track_and_verify_wals);
   }
 
   return io_s;
 }
 
+IOStatus DBImpl::StartWALFile(const WriteOptions& write_options,
+                              const PredecessorWALInfo& predecessor_wal_info,
+                              log::Writer* new_log) {
+  assert(new_log);
+  IOStatus io_s = new_log->AddCompressionTypeRecord(write_options);
+  TEST_SYNC_POINT_CALLBACK("DBImpl::StartWALFile:AfterCompressionTypeRecord",
+                           &io_s);
+  if (io_s.ok()) {
+    io_s = new_log->MaybeAddPredecessorWALInfo(write_options,
+                                               predecessor_wal_info);
+  }
+  return io_s;
+}
+
+IOStatus DBImpl::CreateWAL(const DBOptions& db_options,
+                           const WriteOptions& write_options,
+                           uint64_t log_file_num, uint64_t recycle_log_number,
+                           size_t preallocate_block_size,
+                           const PredecessorWALInfo& predecessor_wal_info,
+                           log::Writer** new_log) {
+  assert(new_log);
+  assert(*new_log == nullptr);
+  UnpublishedWAL wal;
+  IOStatus io_s = CreateWALWriter(db_options, log_file_num, recycle_log_number,
+                                  preallocate_block_size, &wal);
+  if (io_s.ok()) {
+    io_s = StartWALFile(write_options, predecessor_wal_info, wal.writer.get());
+  }
+  if (io_s.ok()) {
+    *new_log = wal.writer.release();
+  }
+  return io_s;
+}
+
+bool DBImpl::AsyncWALPrecreateEnabled() const {
+  return immutable_db_options_.async_wal_precreate &&
+         immutable_db_options_.recycle_log_file_num == 0;
+}
+
+struct AsyncWALPrecreateContext {
+  DBImpl* db = nullptr;
+  DBOptions db_options;
+  uint64_t log_number = 0;
+  size_t preallocate_block_size = 0;
+};
+
+void DBImpl::MaybeScheduleAsyncWALPrecreate(size_t preallocate_block_size) {
+  mutex_.AssertHeld();
+
+  if (!AsyncWALPrecreateEnabled() ||
+      shutting_down_.load(std::memory_order_acquire) ||
+      error_handler_.IsBGWorkStopped() ||
+      async_wal_precreate_state_ != AsyncWALPrecreateState::kNotScheduled) {
+    return;
+  }
+
+  async_wal_precreate_wal_.Reset();
+  async_wal_precreate_wal_.log_number = versions_->NewFileNumber();
+  async_wal_precreate_state_ = AsyncWALPrecreateState::kScheduled;
+
+  auto* ctx = new AsyncWALPrecreateContext();
+  ctx->db = this;
+  ctx->db_options = BuildDBOptions(immutable_db_options_, mutable_db_options_);
+  ctx->log_number = async_wal_precreate_wal_.log_number;
+  ctx->preallocate_block_size = preallocate_block_size;
+  env_->Schedule(&DBImpl::BGWorkAsyncWALPrecreate, ctx, Env::Priority::HIGH,
+                 nullptr);
+}
+
+DBImpl::UnpublishedWAL DBImpl::WaitForAsyncWALPrecreate() {
+  mutex_.AssertHeld();
+  UnpublishedWAL result;
+
+  if (!AsyncWALPrecreateEnabled()) {
+    return result;
+  }
+
+  TEST_SYNC_POINT("DBImpl::WaitForAsyncWALPrecreate:Begin");
+  bool waited = false;
+  uint64_t wait_start_micros = 0;
+  while (async_wal_precreate_state_ == AsyncWALPrecreateState::kScheduled) {
+    TEST_SYNC_POINT("DBImpl::WaitForAsyncWALPrecreate:BeforeWait");
+    if (!waited) {
+      waited = true;
+      wait_start_micros = immutable_db_options_.clock->NowMicros();
+      RecordTick(stats_, WAL_PRECREATE_WAITED);
+    }
+    bg_cv_.Wait();
+  }
+  if (waited) {
+    RecordTick(stats_, WAL_PRECREATE_WAIT_MICROS,
+               immutable_db_options_.clock->NowMicros() - wait_start_micros);
+  }
+
+  if (async_wal_precreate_state_ == AsyncWALPrecreateState::kReady) {
+    result = std::move(async_wal_precreate_wal_);
+    async_wal_precreate_state_ = AsyncWALPrecreateState::kNotScheduled;
+    RecordTick(stats_, WAL_PRECREATE_HIT);
+    return result;
+  }
+
+  assert(async_wal_precreate_state_ == AsyncWALPrecreateState::kNotScheduled);
+  RecordTick(stats_, WAL_PRECREATE_MISS);
+  return result;
+}
+
+void DBImpl::BGWorkAsyncWALPrecreate(void* arg) {
+  TEST_SYNC_POINT("DBImpl::BGWorkAsyncWALPrecreate:Start");
+  std::unique_ptr<AsyncWALPrecreateContext> ctx(
+      static_cast<AsyncWALPrecreateContext*>(arg));
+  DBImpl* db = ctx->db;
+
+  UnpublishedWAL new_wal;
+  IOStatus io_s = db->CreateWALWriter(ctx->db_options, ctx->log_number,
+                                      0 /* recycle_log_number */,
+                                      ctx->preallocate_block_size, &new_wal);
+  if (!io_s.ok()) {
+    RecordTick(db->stats_, WAL_PRECREATE_FAILED);
+    ROCKS_LOG_WARN(db->immutable_db_options_.info_log,
+                   "Async WAL precreate failed for #%" PRIu64 ": %s",
+                   ctx->log_number, io_s.ToString().c_str());
+  }
+
+  TEST_SYNC_POINT("DBImpl::BGWorkAsyncWALPrecreate:BeforePublish");
+  TEST_SYNC_POINT("DBImpl::BGWorkAsyncWALPrecreate:Publish");
+  {
+    InstrumentedMutexLock l(&db->mutex_);
+    if (db->async_wal_precreate_state_ == AsyncWALPrecreateState::kScheduled &&
+        db->async_wal_precreate_wal_.log_number == ctx->log_number) {
+      if (io_s.ok()) {
+        db->async_wal_precreate_wal_ = std::move(new_wal);
+        db->async_wal_precreate_state_ = AsyncWALPrecreateState::kReady;
+      } else {
+        // Async WAL precreation is best-effort. Clear the slot so foreground
+        // rotation falls back to normal synchronous WAL creation.
+        db->async_wal_precreate_wal_.Reset();
+        db->async_wal_precreate_state_ = AsyncWALPrecreateState::kNotScheduled;
+      }
+    }
+    db->bg_cv_.SignalAll();
+  }
+  TEST_SYNC_POINT("DBImpl::BGWorkAsyncWALPrecreate:Done");
+}
+
 void DBImpl::TrackExistingDataFiles(
     const std::vector<std::string>& existing_data_files) {
   TrackOrUntrackFiles(existing_data_files, /*track=*/true);
@@ -2494,8 +2707,10 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
     // created during DB open with predecessor WALs from previous DB session due
     // to `avoid_flush_during_recovery == true`. This can protect the last WAL
     // recovered.
-    s = impl->CreateWAL(write_options, new_log_number, 0 /*recycle_log_number*/,
-                        preallocate_block_size,
+    const DBOptions db_options_snapshot =
+        BuildDBOptions(impl->immutable_db_options_, impl->mutable_db_options_);
+    s = impl->CreateWAL(db_options_snapshot, write_options, new_log_number,
+                        0 /*recycle_log_number*/, preallocate_block_size,
                         PredecessorWALInfo() /* predecessor_wal_info */,
                         &new_log);
     if (s.ok()) {
@@ -2700,6 +2915,8 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
     if (impl->immutable_db_options_.open_files_async) {
       impl->ScheduleAsyncFileOpening();
     }
+    impl->MaybeScheduleAsyncWALPrecreate(
+        impl->GetWalPreallocateBlockSize(max_write_buffer_size));
     impl->mutex_.Unlock();
   }
 

db/db_impl/db_impl_readonly.cc

@@ -5,7 +5,10 @@
 
 #include "db/db_impl/db_impl_readonly.h"
 
+#include <optional>
+
 #include "db/arena_wrapped_db_iter.h"
+#include "db/blob/blob_fetcher.h"
 #include "db/db_impl/compacted_db_impl.h"
 #include "db/db_impl/db_impl.h"
 #include "db/manifest_ops.h"
@@ -63,13 +66,25 @@ Status DBImplReadOnly::GetImpl(const ReadOptions& read_options,
 
   const Comparator* ucmp = get_impl_options.column_family->GetComparator();
   assert(ucmp);
-  std::string* ts =
-      ucmp->timestamp_size() > 0 ? get_impl_options.timestamp : nullptr;
   SequenceNumber snapshot = versions_->LastSequence();
   GetWithTimestampReadCallback read_cb(snapshot);
   auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(
       get_impl_options.column_family);
   auto cfd = cfh->cfd();
+  bool is_blob_index = false;
+  bool* is_blob_ptr = get_impl_options.is_blob_index;
+  std::string timestamp_storage;
+  std::string* ts = nullptr;
+  if (ucmp->timestamp_size() > 0) {
+    ts = get_impl_options.timestamp != nullptr
+             ? get_impl_options.timestamp
+             : (get_impl_options.get_value ? &timestamp_storage : nullptr);
+  }
+  if (!is_blob_ptr && get_impl_options.get_value) {
+    is_blob_ptr = &is_blob_index;
+  }
+  const bool resolve_blob_backed_memtable_value =
+      get_impl_options.get_value && (is_blob_ptr == &is_blob_index);
   if (tracer_) {
     InstrumentedMutexLock lock(&trace_mutex_);
     if (tracer_) {
@@ -95,6 +110,18 @@ Status DBImplReadOnly::GetImpl(const ReadOptions& read_options,
   SequenceNumber max_covering_tombstone_seq = 0;
   LookupKey lkey(key, snapshot, read_options.timestamp);
   PERF_TIMER_STOP(get_snapshot_time);
+  std::optional<BlobFetcher> memtable_blob_fetcher;
+  if (cfd->ioptions().enable_blob_direct_write ||
+      cfd->GetLatestMutableCFOptions().enable_blob_files) {
+    // Recovered memtables can still contain older blob references after
+    // mutable blob-file settings change, so keep blob resolution available
+    // whenever either blob knob indicates it may be needed.
+    memtable_blob_fetcher.emplace(super_version->current, read_options,
+                                  cfd->blob_file_cache(),
+                                  /*allow_write_path_fallback=*/true);
+  }
+  const BlobFetcher* memtable_blob_fetcher_ptr =
+      memtable_blob_fetcher ? &*memtable_blob_fetcher : nullptr;
 
   // Look up starts here
   if (super_version->mem->Get(
@@ -102,11 +129,12 @@ Status DBImplReadOnly::GetImpl(const ReadOptions& read_options,
           get_impl_options.value ? get_impl_options.value->GetSelf() : nullptr,
           get_impl_options.columns, ts, &s, &merge_context,
           &max_covering_tombstone_seq, read_options,
-          false /* immutable_memtable */, &read_cb,
-          /*is_blob_index=*/nullptr, /*do_merge=*/get_impl_options.get_value)) {
-    if (get_impl_options.value) {
-      get_impl_options.value->PinSelf();
-    }
+          false /* immutable_memtable */, &read_cb, is_blob_ptr,
+          /*do_merge=*/get_impl_options.get_value, memtable_blob_fetcher_ptr)) {
+    DBImpl::PostprocessMemtableValueRead(
+        key, ts, resolve_blob_backed_memtable_value, memtable_blob_fetcher_ptr,
+        get_impl_options.value, get_impl_options.columns, &s, &is_blob_index,
+        get_impl_options.value_found);
     RecordTick(stats_, MEMTABLE_HIT);
   } else {
     PERF_TIMER_GUARD(get_from_output_files_time);

db/db_impl/db_impl_secondary.cc

@@ -6,8 +6,10 @@
 #include "db/db_impl/db_impl_secondary.h"
 
 #include <cinttypes>
+#include <optional>
 
 #include "db/arena_wrapped_db_iter.h"
+#include "db/blob/blob_fetcher.h"
 #include "db/log_reader.h"
 #include "db/log_writer.h"
 #include "db/merge_context.h"
@@ -103,10 +105,22 @@ Status DBImplSecondary::FindNewLogNumbers(std::vector<uint64_t>* logs) {
     return s;
   }
 
-  // if log_readers_ is non-empty, it means we have applied all logs with log
-  // numbers smaller than the smallest log in log_readers_, so there is no
-  // need to pass these logs to RecoverLogFiles
-  uint64_t log_number_min = 0;
+  // Drop readers only after MANIFEST replay says the WAL is obsolete. The
+  // primary can create a higher-number empty WAL before it stops appending to a
+  // lower-number current WAL, so seeing a higher-number WAL is not sufficient.
+  const uint64_t min_log_number_to_keep = versions_->min_log_number_to_keep();
+  for (auto iter = log_readers_.begin(); iter != log_readers_.end();) {
+    if (iter->first < min_log_number_to_keep) {
+      iter = log_readers_.erase(iter);
+    } else {
+      break;
+    }
+  }
+
+  // If log_readers_ is non-empty, it means we have applied all logs with log
+  // numbers smaller than the smallest retained log reader. Otherwise, manifest
+  // recovery tells us which older logs can be skipped.
+  uint64_t log_number_min = min_log_number_to_keep;
   if (!log_readers_.empty()) {
     log_number_min = log_readers_.begin()->first;
   }
@@ -319,12 +333,6 @@ Status DBImplSecondary::RecoverLogFiles(
       return status;
     }
   }
-  // remove logreaders from map after successfully recovering the WAL
-  if (log_readers_.size() > 1) {
-    auto erase_iter = log_readers_.begin();
-    std::advance(erase_iter, log_readers_.size() - 1);
-    log_readers_.erase(log_readers_.begin(), erase_iter);
-  }
   return status;
 }
 
@@ -363,13 +371,25 @@ Status DBImplSecondary::GetImpl(const ReadOptions& read_options,
 
   const Comparator* ucmp = get_impl_options.column_family->GetComparator();
   assert(ucmp);
-  std::string* ts =
-      ucmp->timestamp_size() > 0 ? get_impl_options.timestamp : nullptr;
   SequenceNumber snapshot = versions_->LastSequence();
   GetWithTimestampReadCallback read_cb(snapshot);
   auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(
       get_impl_options.column_family);
   auto cfd = cfh->cfd();
+  bool is_blob_index = false;
+  bool* is_blob_ptr = get_impl_options.is_blob_index;
+  std::string timestamp_storage;
+  std::string* ts = nullptr;
+  if (ucmp->timestamp_size() > 0) {
+    ts = get_impl_options.timestamp != nullptr
+             ? get_impl_options.timestamp
+             : (get_impl_options.get_value ? &timestamp_storage : nullptr);
+  }
+  if (!is_blob_ptr && get_impl_options.get_value) {
+    is_blob_ptr = &is_blob_index;
+  }
+  const bool resolve_blob_backed_memtable_value =
+      get_impl_options.get_value && (is_blob_ptr == &is_blob_index);
   if (tracer_) {
     InstrumentedMutexLock lock(&trace_mutex_);
     if (tracer_) {
@@ -395,6 +415,18 @@ Status DBImplSecondary::GetImpl(const ReadOptions& read_options,
   LookupKey lkey(key, snapshot, read_options.timestamp);
   PERF_TIMER_STOP(get_snapshot_time);
   bool done = false;
+  std::optional<BlobFetcher> memtable_blob_fetcher;
+  if (cfd->ioptions().enable_blob_direct_write ||
+      cfd->GetLatestMutableCFOptions().enable_blob_files) {
+    // Catch-up can rebuild older blob references into memtables after mutable
+    // blob-file settings change, so keep blob resolution available whenever
+    // either blob knob indicates it may be needed.
+    memtable_blob_fetcher.emplace(super_version->current, read_options,
+                                  cfd->blob_file_cache(),
+                                  /*allow_write_path_fallback=*/true);
+  }
+  const BlobFetcher* memtable_blob_fetcher_ptr =
+      memtable_blob_fetcher ? &*memtable_blob_fetcher : nullptr;
 
   // Look up starts here
   if (get_impl_options.get_value) {
@@ -404,12 +436,14 @@ Status DBImplSecondary::GetImpl(const ReadOptions& read_options,
                                    : nullptr,
             get_impl_options.columns, ts, &s, &merge_context,
             &max_covering_tombstone_seq, read_options,
-            false /* immutable_memtable */, &read_cb,
-            /*is_blob_index=*/nullptr, /*do_merge=*/true)) {
+            false /* immutable_memtable */, &read_cb, is_blob_ptr,
+            /*do_merge=*/true, memtable_blob_fetcher_ptr)) {
       done = true;
-      if (get_impl_options.value) {
-        get_impl_options.value->PinSelf();
-      }
+      DBImpl::PostprocessMemtableValueRead(
+          key, ts, resolve_blob_backed_memtable_value,
+          memtable_blob_fetcher_ptr, get_impl_options.value,
+          get_impl_options.columns, &s, &is_blob_index,
+          get_impl_options.value_found);
       RecordTick(stats_, MEMTABLE_HIT);
     } else if ((s.ok() || s.IsMergeInProgress()) &&
                super_version->imm->Get(
@@ -417,11 +451,14 @@ Status DBImplSecondary::GetImpl(const ReadOptions& read_options,
                    get_impl_options.value ? get_impl_options.value->GetSelf()
                                           : nullptr,
                    get_impl_options.columns, ts, &s, &merge_context,
-                   &max_covering_tombstone_seq, read_options, &read_cb)) {
+                   &max_covering_tombstone_seq, read_options, &read_cb,
+                   is_blob_ptr, memtable_blob_fetcher_ptr)) {
       done = true;
-      if (get_impl_options.value) {
-        get_impl_options.value->PinSelf();
-      }
+      DBImpl::PostprocessMemtableValueRead(
+          key, ts, resolve_blob_backed_memtable_value,
+          memtable_blob_fetcher_ptr, get_impl_options.value,
+          get_impl_options.columns, &s, &is_blob_index,
+          get_impl_options.value_found);
       RecordTick(stats_, MEMTABLE_HIT);
     }
   } else {
@@ -433,13 +470,14 @@ Status DBImplSecondary::GetImpl(const ReadOptions& read_options,
             get_impl_options.columns, ts, &s, &merge_context,
             &max_covering_tombstone_seq, read_options,
             false /* immutable_memtable */, &read_cb,
-            /*is_blob_index=*/nullptr, /*do_merge=*/false)) {
+            /*is_blob_index=*/nullptr, /*do_merge=*/false,
+            memtable_blob_fetcher_ptr)) {
       done = true;
       RecordTick(stats_, MEMTABLE_HIT);
     } else if ((s.ok() || s.IsMergeInProgress()) &&
-               super_version->imm->GetMergeOperands(lkey, &s, &merge_context,
-                                                    &max_covering_tombstone_seq,
-                                                    read_options)) {
+               super_version->imm->GetMergeOperands(
+                   lkey, &s, &merge_context, &max_covering_tombstone_seq,
+                   read_options, memtable_blob_fetcher_ptr)) {
       done = true;
       RecordTick(stats_, MEMTABLE_HIT);
     }
@@ -457,7 +495,7 @@ Status DBImplSecondary::GetImpl(const ReadOptions& read_options,
         ts, &s, &merge_context, &max_covering_tombstone_seq, &pinned_iters_mgr,
         /*value_found*/ nullptr,
         /*key_exists*/ nullptr, /*seq*/ nullptr, &read_cb, /*is_blob*/ nullptr,
-        /*do_merge*/ true);
+        /*do_merge=*/get_impl_options.get_value);
     RecordTick(stats_, MEMTABLE_MISS);
   }
   {
@@ -777,7 +815,8 @@ Status DBImplSecondary::OpenAsSecondaryImpl(
   impl->versions_.reset(new ReactiveVersionSet(
       dbname, &impl->immutable_db_options_, impl->mutable_db_options_,
       impl->file_options_, impl->table_cache_.get(),
-      impl->write_buffer_manager_, &impl->write_controller_, impl->io_tracer_));
+      impl->write_buffer_manager_, &impl->write_controller_, impl->io_tracer_,
+      impl->db_id_, impl->db_session_id_));
   impl->column_family_memtables_.reset(
       new ColumnFamilyMemTablesImpl(impl->versions_->GetColumnFamilySet()));
   impl->wal_in_db_path_ = impl->immutable_db_options_.IsWalDirSameAsDBPath();
@@ -1553,7 +1592,7 @@ Status DB::OpenAndCompact(
     }
   }
 
-  // 5. Open db As Secondary (skip WAL recovery — remote compaction only
+  // 5. Open db As Secondary (skip WAL recovery -- remote compaction only
   //    needs LSM state from MANIFEST, not memtable data from WAL replay)
   std::unique_ptr<DB> db;
   std::vector<ColumnFamilyHandle*> handles;

db/db_impl/db_impl_write.cc

@@ -16,6 +16,7 @@
 #include "db/error_handler.h"
 #include "db/event_helpers.h"
 #include "db/wide/wide_columns_helper.h"
+#include "file/filename.h"
 #include "logging/logging.h"
 #include "memtable/wbwi_memtable.h"
 #include "monitoring/perf_context_imp.h"
@@ -1546,6 +1547,10 @@ Status DBImpl::WriteImpl(
       // Note: if we are to resume after non-OK statuses we need to revisit how
       // we react to non-OK statuses here.
       if (w.status.ok()) {  // Don't publish a partial batch write
+        TEST_SYNC_POINT(
+            "DBImpl::WriteImpl:AfterWBWIIngestBeforeSetLastSequence:pause");
+        TEST_SYNC_POINT(
+            "DBImpl::WriteImpl:AfterWBWIIngestBeforeSetLastSequence:resume");
         versions_->SetLastSequence(last_sequence);
       }
     }
@@ -3103,6 +3108,7 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
   const WriteOptions write_options;
 
   log::Writer* new_log = nullptr;
+  UnpublishedWAL prepared_wal;
   MemTable* new_mem = nullptr;
   IOStatus io_s;
 
@@ -3130,8 +3136,16 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
       !wal_recycle_files_.empty() && IsFileDeletionsEnabled()) {
     recycle_log_number = wal_recycle_files_.front();
   }
-  uint64_t new_log_number =
-      creating_new_log ? versions_->NewFileNumber() : cur_wal_number_;
+  uint64_t new_log_number = cur_wal_number_;
+  if (creating_new_log) {
+    prepared_wal = WaitForAsyncWALPrecreate();
+    if (prepared_wal.writer) {
+      new_log_number = prepared_wal.log_number;
+    } else {
+      assert(prepared_wal.log_number == 0);
+      new_log_number = versions_->NewFileNumber();
+    }
+  }
   // For use outside of holding DB mutex
   const MutableCFOptions mutable_cf_options_copy =
       cfd->GetLatestMutableCFOptions();
@@ -3152,6 +3166,8 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
   // Log this later after lock release. It may be outdated, e.g., if background
   // flush happens before logging, but that should be ok.
   int num_imm_unflushed = cfd->imm()->NumNotFlushed();
+  const DBOptions db_options_snapshot =
+      BuildDBOptions(immutable_db_options_, mutable_db_options_);
   const auto preallocate_block_size =
       GetWalPreallocateBlockSize(mutable_cf_options_copy.write_buffer_size);
   mutex_.Unlock();
@@ -3167,13 +3183,35 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
     wal_write_mutex_.Unlock();
     // TODO: Write buffer size passed in should be max of all CF's instead
     // of mutable_cf_options.write_buffer_size.
-    io_s = CreateWAL(write_options, new_log_number, recycle_log_number,
-                     preallocate_block_size, info, &new_log);
+    if (prepared_wal.writer) {
+      io_s = StartWALFile(write_options, info, prepared_wal.writer.get());
+      if (io_s.ok()) {
+        new_log = prepared_wal.writer.release();
+      }
+    } else {
+      io_s =
+          CreateWAL(db_options_snapshot, write_options, new_log_number,
+                    recycle_log_number, preallocate_block_size, info, &new_log);
+    }
     TEST_SYNC_POINT_CALLBACK("DBImpl::SwitchMemtable:AfterCreateWAL", &io_s);
     if (s.ok()) {
       s = io_s;
     }
   }
+  if (!s.ok() && prepared_wal.writer) {
+    const uint64_t prepared_wal_number = prepared_wal.log_number;
+    const std::string prepared_wal_fname =
+        LogFileName(immutable_db_options_.GetWalDir(), prepared_wal_number);
+    prepared_wal.Reset();
+    Status delete_s = env_->DeleteFile(prepared_wal_fname);
+    if (!delete_s.ok() && !delete_s.IsNotFound()) {
+      ROCKS_LOG_WARN(immutable_db_options_.info_log,
+                     "Failed to delete unstarted async precreated WAL #%" PRIu64
+                     ": %s",
+                     prepared_wal_number, delete_s.ToString().c_str());
+    }
+    delete_s.PermitUncheckedError();
+  }
   if (s.ok()) {
     // FIXME: from the comment for GetEarliestSequenceNumber(), any key with
     //  seqno >= earliest_seqno should be in this or later memtable. This means
@@ -3355,6 +3393,7 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
   new_mem->Ref();
   cfd->SetMemtable(new_mem);
   InstallSuperVersionAndScheduleWork(cfd, &context->superversion_context);
+  MaybeScheduleAsyncWALPrecreate(preallocate_block_size);
 
   // Notify client that memtable is sealed, now that we have successfully
   // installed a new memtable

db/db_iter.cc

@@ -623,7 +623,7 @@ bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key) {
               PERF_COUNTER_ADD(internal_delete_skipped_count, 1);
               MarkMemtableForFlushForPerOpTrigger(mem_hidden_op_scanned);
               // Track contiguous tombstones for range conversion.
-              // Skip if outside seek prefix — the top-of-loop check
+              // Skip if outside seek prefix -- the top-of-loop check
               // flushed any pending run, but we must also avoid starting
               // a new run outside the prefix.
               if (min_tombstones_for_range_conversion_ > 0 &&
@@ -1097,7 +1097,7 @@ void DBIter::PrevInternal() {
     if (min_tombstones_for_range_conversion_ > 0 &&
         range_tomb_end_key_.Size() > 0 && timestamp_lb_ == nullptr) {
       if (!valid_ && found_visible && PrefixCheck(saved_key_without_ts)) {
-        // Key was deleted and is within the seek prefix — track it.
+        // Key was deleted and is within the seek prefix -- track it.
         TrackContiguousTombstone(saved_key_.GetUserKey(),
                                  /*always_update_first_key=*/true);
       } else if (valid_) {
@@ -1141,7 +1141,7 @@ void DBIter::PrevInternal() {
 // Sets ikey_ to the last visible entry's internal key.  When found_visible
 // is false, ikey_ is not updated and may contain stale data.
 // Sets found_visible to true if at least one entry passed the IsVisible()
-// check (seqno <= snapshot).  When false, no entry was visible — the key
+// check (seqno <= snapshot).  When false, no entry was visible -- the key
 // does not exist at this snapshot and should not be treated as a tombstone.
 // Returns false if an error occurred, and !status().ok() and !valid_.
 //
@@ -1196,7 +1196,7 @@ bool DBIter::FindValueForCurrentKey(bool& found_visible) {
       break;
     }
 
-    // Entry survived the visibility check — at least one visible version
+    // Entry survived the visibility check -- at least one visible version
     // exists for this user key.
     found_visible = true;
 
@@ -1480,7 +1480,9 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
     if (timestamp_lb_ != nullptr) {
       saved_key_.SetInternalKey(ikey);
     } else {
-      saved_key_.SetUserKey(ikey.user_key);
+      saved_key_.SetUserKey(
+          ikey.user_key,
+          !pin_thru_lifetime_ || !iter_.iter()->IsKeyPinned() /* copy */);
     }
 
     valid_ = true;

db/db_iter.h

@@ -589,7 +589,7 @@ class DBIter final : public Iterator {
 
   // If enough contiguous tombstones have been tracked, insert a range
   // tombstone [first_key, end_key) into the mutable memtable.
-  // end_key is the exclusive upper bound — typically the next live key.
+  // end_key is the exclusive upper bound -- typically the next live key.
   void MaybeInsertRangeTombstone(const Slice& end_key);
   void ResetContiguousTombstoneTracking() {
     contiguous_tombstone_count_ = 0;

db/db_iter_test.cc

@@ -216,12 +216,19 @@ class TestIterator : public InternalIterator {
   bool IsKeyPinned() const override { return true; }
   bool IsValuePinned() const override { return true; }
 
+  void Prepare(const MultiScanArgs* /*scan_opts*/) override {
+    ++prepare_call_count_;
+  }
+
+  size_t prepare_call_count() const { return prepare_call_count_; }
+
  private:
   bool initialized_;
   bool valid_;
   size_t sequence_number_;
   size_t iter_;
   size_t steps_ = 0;
+  size_t prepare_call_count_ = 0;
 
   InternalKeyComparator cmp;
   std::vector<std::pair<std::string, std::string>> data_;
@@ -243,6 +250,53 @@ class DBIteratorTest : public testing::Test {
   DBIteratorTest() : env_(Env::Default()) {}
 };
 
+TEST_F(DBIteratorTest, PrepareForwardsValidatedScanRanges) {
+  Options options;
+  ImmutableOptions ioptions = ImmutableOptions(options);
+  MutableCFOptions mutable_cf_options = MutableCFOptions(options);
+  TestIterator* internal_iter = new TestIterator(BytewiseComparator());
+  internal_iter->AddPut("a", "val_a");
+  internal_iter->AddPut("b", "val_b");
+  internal_iter->Finish();
+
+  ReadOptions ro;
+  std::unique_ptr<Iterator> db_iter(DBIter::NewIter(
+      env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
+      internal_iter, nullptr /* version */, 10 /* sequence */,
+      nullptr /* read_callback */, /*active_mem=*/nullptr));
+
+  MultiScanArgs scan_opts(BytewiseComparator());
+  scan_opts.insert("a", "c");
+
+  db_iter->Prepare(scan_opts);
+  ASSERT_OK(db_iter->status());
+  EXPECT_EQ(internal_iter->prepare_call_count(), 1);
+}
+
+TEST_F(DBIteratorTest, PrepareDoesNotForwardInvalidScanRanges) {
+  Options options;
+  ImmutableOptions ioptions = ImmutableOptions(options);
+  MutableCFOptions mutable_cf_options = MutableCFOptions(options);
+  TestIterator* internal_iter = new TestIterator(BytewiseComparator());
+  internal_iter->AddPut("a", "val_a");
+  internal_iter->AddPut("b", "val_b");
+  internal_iter->Finish();
+
+  ReadOptions ro;
+  std::unique_ptr<Iterator> db_iter(DBIter::NewIter(
+      env_, ro, ioptions, mutable_cf_options, BytewiseComparator(),
+      internal_iter, nullptr /* version */, 10 /* sequence */,
+      nullptr /* read_callback */, /*active_mem=*/nullptr));
+
+  MultiScanArgs scan_opts(BytewiseComparator());
+  scan_opts.insert("b", "d");
+  scan_opts.insert("a", "c");
+
+  db_iter->Prepare(scan_opts);
+  EXPECT_TRUE(db_iter->status().IsInvalidArgument());
+  EXPECT_EQ(internal_iter->prepare_call_count(), 0);
+}
+
 TEST_F(DBIteratorTest, DBIteratorPrevNext) {
   Options options;
   ImmutableOptions ioptions = ImmutableOptions(options);

db/db_iterator_test.cc

@@ -4151,7 +4151,7 @@ TEST_P(DBIteratorTest, PrefixSameAsStartSeekToNonInDomainKey) {
   ASSERT_OK(Put("abc2", "v2"));
   ASSERT_OK(Put("abc3", "v3"));
 
-  // Seek to "ab" (2 bytes) — out-of-domain for FixedPrefixTransform(3).
+  // Seek to "ab" (2 bytes) -- out-of-domain for FixedPrefixTransform(3).
   // ShouldSetPrefix returns false for out-of-domain targets, so no prefix
   // constraint is set. The seek should find "abc1" and iteration should
   // proceed without prefix_same_as_start enforcement.
@@ -4395,6 +4395,71 @@ TEST_P(DBMultiScanIteratorTest, RangeAcrossFiles) {
   iter.reset();
 }
 
+TEST_P(DBMultiScanIteratorTest, SortedRangesSkipIODispatcherSort) {
+  auto options = CurrentOptions();
+  options.target_file_size_base = 100 << 10;
+  options.compaction_style = kCompactionStyleUniversal;
+  options.num_levels = 50;
+  options.compression = kNoCompression;
+  DestroyAndReopen(options);
+
+  auto key = [](int i) {
+    std::stringstream ss;
+    ss << "k" << std::setw(5) << std::setfill('0') << i;
+    return ss.str();
+  };
+
+  auto rnd = Random::GetTLSInstance();
+  for (int i = 0; i < 100; ++i) {
+    ASSERT_OK(Put(key(i), rnd->RandomString(2 << 10)));
+  }
+  ASSERT_OK(Flush());
+
+  ASSERT_OK(db_->CompactRange({}, nullptr, nullptr));
+  ASSERT_EQ(2, NumTableFilesAtLevel(49));
+
+  int sort_count = 0;
+  SyncPoint::GetInstance()->SetCallBack(
+      "IODispatcherImpl::SubmitJob:SortBlockHandles",
+      [&](void* /*arg*/) { ++sort_count; });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  auto tracking_dispatcher = std::make_shared<TrackingIODispatcher>();
+  std::vector<std::string> key_ranges({key(10), key(90)});
+  MultiScanArgs scan_options(BytewiseComparator());
+  scan_options.io_dispatcher = tracking_dispatcher;
+  scan_options.use_async_io = false;
+  scan_options.insert(key_ranges[0], key_ranges[1]);
+
+  ReadOptions ro;
+  ro.fill_cache = GetParam();
+  ColumnFamilyHandle* cfh = dbfull()->DefaultColumnFamily();
+  std::unique_ptr<MultiScan> iter =
+      dbfull()->NewMultiScan(ro, cfh, scan_options);
+
+  try {
+    int i = 10;
+    for (auto range : *iter) {
+      for (auto it : range) {
+        ASSERT_EQ(it.first.ToString(), key(i));
+        ++i;
+      }
+    }
+    ASSERT_EQ(i, 90);
+  } catch (MultiScanException& ex) {
+    ASSERT_NOK(ex.status());
+    std::cerr << "Iterator returned status " << ex.what();
+    abort();
+  } catch (std::logic_error& ex) {
+    std::cerr << "Iterator returned logic error " << ex.what();
+    abort();
+  }
+
+  ASSERT_GT(tracking_dispatcher->GetReadSets().size(), 0);
+  EXPECT_EQ(sort_count, 0);
+  iter.reset();
+}
+
 TEST_P(DBMultiScanIteratorTest, FailureTest) {
   auto options = CurrentOptions();
   options.compression = kNoCompression;
@@ -5820,13 +5885,13 @@ TEST_P(ReadPathRangeTombstoneTest, ExhaustedIteratorWithBounds) {
   // Both directions encounter two tombstone runs (a-d and g-j).
   ASSERT_EQ(attempted_insert_ranges_.size(), 2);
   if (Forward()) {
-    // Forward: sees a-d tombstones first → live e terminates → [a, e).
-    // Then g-j → exhaustion past j, saved_key_="j" fallback → [g, j),
+    // Forward: sees a-d tombstones first -> live e terminates -> [a, e).
+    // Then g-j -> exhaustion past j, saved_key_="j" fallback -> [g, j),
     // covering g,h,i. j remains a point tombstone.
     AssertRange(0, "a", "e");
     AssertRange(1, "g", "j");
   } else {
-    // Reverse: sees j-g tombstones first, then f,e live, then d-a → exhausts.
+    // Reverse: sees j-g tombstones first, then f,e live, then d-a -> exhausts.
     AssertRange(0, "g", "z");
     AssertRange(1, "a", "e");
   }
@@ -5863,8 +5928,8 @@ TEST_P(ReadPathRangeTombstoneTest, ExhaustedIteratorNoBounds) {
   VerifyIteration({"e", "f"});
 
   if (Forward()) {
-    // Forward: a-d run terminates at live e → [a, e). g-j run exhausts
-    // past j → saved_key_ fallback → [g, j), covering g,h,i with j as a
+    // Forward: a-d run terminates at live e -> [a, e). g-j run exhausts
+    // past j -> saved_key_ fallback -> [g, j), covering g,h,i with j as a
     // point tombstone.
     ASSERT_EQ(attempted_insert_ranges_.size(), 2);
     AssertRange(0, "a", "e");
@@ -6061,7 +6126,7 @@ TEST_P(ReadPathRangeTombstoneTest, PrefixFilterDefaultReadOptions) {
 }
 
 TEST_P(ReadPathRangeTombstoneTest, PrefixFilterTotalOrderSeek) {
-  // total_order_seek=true disables prefix filtering — all files visible.
+  // total_order_seek=true disables prefix filtering -- all files visible.
   // The delete run spans from prefix 'b' into prefix 'c', terminated by
   // live key "cb" from L0. The tombstone [ba, cb) crosses prefix boundaries,
   // which is safe because total_order_seek makes all files visible.
@@ -6103,7 +6168,7 @@ TEST_P(ReadPathRangeTombstoneTest, PrefixFilterTotalOrderSeek) {
     ASSERT_OK(Flush());
 
     // Memtable: contiguous deletes spanning prefixes 'b' and 'c'.
-    // No live key between "bb" and "cb" — the run crosses prefix boundaries.
+    // No live key between "bb" and "cb" -- the run crosses prefix boundaries.
     ASSERT_OK(put("aa", "below"));
     ASSERT_OK(del("ba"));
     ASSERT_OK(del("bb"));
@@ -6311,7 +6376,7 @@ TEST_P(ReadPathRangeTombstoneTest, PrefixFilterOutOfDomainSeek) {
   ro.prefix_same_as_start = true;
   auto it = std::unique_ptr<Iterator>(db_->NewIterator(ro));
   if (Forward()) {
-    // Seek with a 1-byte key — out-of-domain for FixedPrefixTransform(4).
+    // Seek with a 1-byte key -- out-of-domain for FixedPrefixTransform(4).
     // prefix_same_as_start cannot set a prefix bound for this target.
     it->Seek("b");
     while (it->Valid()) {
@@ -6674,7 +6739,7 @@ TEST_P(ReadPathRangeTombstoneTest, ExhaustedWithUDT) {
 
   ASSERT_EQ(attempted_insert_ranges_.size(), 1);
   if (Forward()) {
-    // Forward exhaustion past h: saved_key_="h"+ts fallback → [e+ts, h+ts).
+    // Forward exhaustion past h: saved_key_="h"+ts fallback -> [e+ts, h+ts).
     AssertRange(0, std::string("e") + ts, std::string("h") + ts);
   } else {
     // Reverse exhaustion: range is [a+ts, e+ts).
@@ -6687,9 +6752,9 @@ TEST_P(ReadPathRangeTombstoneTest, ExhaustedWithUDT) {
 
 TEST_P(ReadPathRangeTombstoneTest, SeekForPrevTombstone) {
   // SeekForPrev lands directly on tombstones. No upper bound. Forward
-  // exhaustion past h falls back to saved_key_="h" → [e, h), covering
+  // exhaustion past h falls back to saved_key_="h" -> [e, h), covering
   // e,f,g with h as a point tombstone.
-  // Reverse: SeekForPrev("h") → Delete(h,g,f,e) → live "d" → range [e, h).
+  // Reverse: SeekForPrev("h") -> Delete(h,g,f,e) -> live "d" -> range [e, h).
   for (bool use_udt : {false, true}) {
     SCOPED_TRACE(use_udt ? "with UDT" : "without UDT");
     Options options = CurrentOptions();
@@ -6727,7 +6792,7 @@ TEST_P(ReadPathRangeTombstoneTest, SeekForPrevTombstone) {
         keys.push_back(iter->key().ToString());
       }
     } else {
-      // SeekForPrev("h") lands on Delete(h), traverses g,f,e → finds "d".
+      // SeekForPrev("h") lands on Delete(h), traverses g,f,e -> finds "d".
       for (iter->SeekForPrev("h"); iter->Valid(); iter->Prev()) {
         keys.push_back(iter->key().ToString());
       }
@@ -6738,7 +6803,7 @@ TEST_P(ReadPathRangeTombstoneTest, SeekForPrevTombstone) {
 
     ASSERT_EQ(attempted_insert_ranges_.size(), 1);
     if (Forward()) {
-      // Forward exhausts past h → saved_key_="h" fallback → [e, h),
+      // Forward exhausts past h -> saved_key_="h" fallback -> [e, h),
       // covering e,f,g with h as a point tombstone.
       if (use_udt) {
         AssertRange(0, std::string("e") + ts, std::string("h") + ts);
@@ -6761,11 +6826,11 @@ TEST_P(ReadPathRangeTombstoneTest, SeekForPrevTombstone) {
 
 TEST_P(ReadPathRangeTombstoneTest, UpperBoundTombstone) {
   // iterate_upper_bound lands past the data. Both directions see tombstones
-  // e-h. Forward exhausts naturally past h (no key in DB ≥ upper) so the
+  // e-h. Forward exhausts naturally past h (no key in DB >= upper) so the
   // forward path falls back to saved_key_ ("h") as the exclusive end_key,
-  // covering n-1 of n deletes — [e, h). The remaining tombstone for h
+  // covering n-1 of n deletes -- [e, h). The remaining tombstone for h
   // stays as a point delete. Reverse captures the live key "d" before the
-  // run and uses it via range_tomb_end_key_ — [e, i) when SeekToLast
+  // run and uses it via range_tomb_end_key_ -- [e, i) when SeekToLast
   // delegates to SeekForPrev("i").
   for (bool use_udt : {false, true}) {
     SCOPED_TRACE(use_udt ? "with UDT" : "without UDT");
@@ -6837,8 +6902,8 @@ TEST_P(ReadPathRangeTombstoneTest, UpperBoundTombstone) {
 
 TEST_P(ReadPathRangeTombstoneTest, LowerBoundTruncatesReverse) {
   // Keys a-j, delete a-h. Lower bound "e" truncates reverse iteration
-  // mid-tombstone-run. Forward: tombstones e-h ended by live key i → [e, i).
-  // Reverse: tombstones h,g,f,e then lower_bound hit → flush [e, i).
+  // mid-tombstone-run. Forward: tombstones e-h ended by live key i -> [e, i).
+  // Reverse: tombstones h,g,f,e then lower_bound hit -> flush [e, i).
   for (bool use_udt : {false, true}) {
     SCOPED_TRACE(use_udt ? "with UDT" : "without UDT");
     Options options = CurrentOptions();
@@ -6975,14 +7040,14 @@ TEST_P(ReadPathRangeTombstoneTest, InvisibleKeysDontBreakTombstoneRun) {
   ASSERT_OK(Put("f", "vf"));
   ASSERT_OK(Flush());
 
-  // Delete b, d — visible tombstones.
+  // Delete b, d -- visible tombstones.
   ASSERT_OK(Delete("b"));
   ASSERT_OK(Delete("d"));
 
   // Snapshot S.  At S: a(live), b(del), d(del), f(live).
   const Snapshot* snap = db_->GetSnapshot();
 
-  // Write c, e AFTER the snapshot — invisible at S.
+  // Write c, e AFTER the snapshot -- invisible at S.
   // At S the iterator sees: a(live), b(del), c(invis), d(del),
   //   e(invis), f(live).
   ASSERT_OK(Put("c", "vc"));
@@ -6994,7 +7059,7 @@ TEST_P(ReadPathRangeTombstoneTest, InvisibleKeysDontBreakTombstoneRun) {
   VerifyIteration({"a", "f"}, ro);
 
   // Invisible keys c and e should not break the tombstone run.
-  // 2 tombstones (b, d) ≥ threshold → range [b, f).
+  // 2 tombstones (b, d) >= threshold -> range [b, f).
   ASSERT_EQ(attempted_insert_ranges_.size(), 1);
   AssertRange(0, "b", "f");
 
@@ -7181,6 +7246,52 @@ TEST_P(ReadPathRangeTombstoneTest,
   db_->ReleaseSnapshot(snap);
 }
 
+// SeekForPrev uses the reverse path: PrevInternal -> FindValueForCurrentKey.
+// When a key has more versions than max_sequential_skip_in_iterations,
+// FindValueForCurrentKey delegates to FindValueForCurrentKeyUsingSeek which
+// re-seeks forward to the newest visible version. That function must preserve
+// key pinning by passing the correct `copy` parameter to
+// saved_key_.SetUserKey().
+TEST_P(DBIteratorTest, SeekForPrevKeyPinnedWithManyVersions) {
+  Options options = CurrentOptions();
+  options.max_sequential_skip_in_iterations = 8;
+  options.statistics = CreateDBStatistics();
+  DestroyAndReopen(options);
+
+  // 20 versions > max_sequential_skip (8), forcing the reseek path.
+  // No flush: keeps all versions alive in the memtable.
+  for (int i = 0; i < 20; i++) {
+    ASSERT_OK(Put("key1", "value" + std::to_string(i)));
+  }
+
+  ReadOptions ro;
+  ro.pin_data = true;
+
+  std::unique_ptr<Iterator> iter(NewIterator(ro));
+  std::string prop_value;
+
+  // Seek (forward) correctly reports key as pinned.
+  iter->Seek("key1");
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_OK(iter->GetProperty("rocksdb.iterator.is-key-pinned", &prop_value));
+  ASSERT_EQ("1", prop_value);
+
+  uint64_t reseeks_before =
+      options.statistics->getTickerCount(NUMBER_OF_RESEEKS_IN_ITERATION);
+
+  iter->SeekForPrev("key1");
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_EQ("key1", iter->key().ToString());
+
+  // Confirm FindValueForCurrentKeyUsingSeek was actually taken.
+  uint64_t reseeks_after =
+      options.statistics->getTickerCount(NUMBER_OF_RESEEKS_IN_ITERATION);
+  ASSERT_GT(reseeks_after, reseeks_before);
+
+  ASSERT_OK(iter->GetProperty("rocksdb.iterator.is-key-pinned", &prop_value));
+  ASSERT_EQ("1", prop_value);
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

db/db_options_test.cc

@@ -102,6 +102,40 @@ TEST_F(DBOptionsTest, ImmutableTrackAndVerifyWalsInManifest) {
   ASSERT_FALSE(s.ok());
 }
 
+TEST_F(DBOptionsTest, ImmutableAsyncWalPrecreate) {
+  Options options;
+  options.env = env_;
+  options.async_wal_precreate = true;
+
+  ImmutableDBOptions db_options(options);
+  ASSERT_TRUE(db_options.async_wal_precreate);
+
+  Reopen(options);
+  ASSERT_TRUE(dbfull()->GetDBOptions().async_wal_precreate);
+
+  Status s = dbfull()->SetDBOptions({{"async_wal_precreate", "false"}});
+  ASSERT_FALSE(s.ok());
+}
+
+TEST_F(DBOptionsTest, SanitizeAsyncWalPrecreateWithWalRecycle) {
+  // This checks the option-level contract for the incompatible WAL paths:
+  // SanitizeOptions() preserves WAL recycling and disables only the
+  // opportunistic async precreation path, so the immutable options reflect
+  // effective behavior.
+  Options options;
+  options.env = env_;
+  options.async_wal_precreate = true;
+  options.recycle_log_file_num = 1;
+
+  Options sanitized_options = SanitizeOptions(dbname_, options);
+  ASSERT_FALSE(sanitized_options.async_wal_precreate);
+  ASSERT_EQ(1U, sanitized_options.recycle_log_file_num);
+
+  options.recycle_log_file_num = 0;
+  sanitized_options = SanitizeOptions(dbname_, options);
+  ASSERT_TRUE(sanitized_options.async_wal_precreate);
+}
+
 TEST_F(DBOptionsTest, ImmutableVerifySstUniqueIdInManifest) {
   Options options;
   options.env = env_;

db/db_secondary_test.cc

@@ -7,9 +7,13 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
+#include "db/blob/blob_index.h"
 #include "db/db_impl/db_impl_secondary.h"
 #include "db/db_test_util.h"
 #include "db/db_with_timestamp_test_util.h"
+#include "db/wide/wide_column_test_util.h"
+#include "db/write_batch_internal.h"
+#include "file/filename.h"
 #include "port/stack_trace.h"
 #include "rocksdb/utilities/transaction_db.h"
 #include "test_util/sync_point.h"
@@ -357,8 +361,7 @@ TEST_F(DBSecondaryTest, GetMergeOperands) {
   const Status s = db_secondary_->GetMergeOperands(
       ReadOptions(), cfh, "k1", values.data(), &merge_operands_info,
       &number_of_operands);
-  ASSERT_NOK(s);
-  ASSERT_TRUE(s.IsMergeInProgress());
+  ASSERT_OK(s);
 
   ASSERT_EQ(number_of_operands, 4);
   ASSERT_EQ(values[0].ToString(), "v1");
@@ -367,6 +370,197 @@ TEST_F(DBSecondaryTest, GetMergeOperands) {
   ASSERT_EQ(values[3].ToString(), "v4");
 }
 
+TEST_F(DBSecondaryTest, GetMergeOperandsWithBlobBackedEntityDefaultColumn) {
+  // Goal: exercise the secondary read path with a blob-backed V2 entity base
+  // in SST and a newer merge operand applied through catch-up. The secondary
+  // DB must resolve the blob-backed default column for both Get() and
+  // GetMergeOperands() while combining the newer memtable merge with the older
+  // SST-backed base entity.
+  Options options = GetDefaultOptions();
+  options.create_if_missing = true;
+  options.enable_blob_files = true;
+  options.min_blob_size = 50;
+  options.disable_auto_compactions = true;
+  options.merge_operator = MergeOperators::CreateStringAppendOperator("|");
+  options.env = env_;
+  Reopen(options);
+
+  const std::string key = "secondary_blob_entity";
+  const std::string default_value(100, 'd');
+  const std::string large_value(120, 'l');
+  const std::string merge_operand = "suffix";
+  const std::string expected_merged = default_value + "|" + merge_operand;
+  WideColumns columns{{kDefaultWideColumnName, default_value},
+                      {"col_large", large_value},
+                      {"meta", "inline"}};
+
+  ASSERT_OK(
+      db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(), key, columns));
+  ASSERT_OK(Flush());
+  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+
+  options.max_open_files = -1;
+  OpenSecondary(options);
+  ASSERT_OK(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), key,
+                       merge_operand));
+  ASSERT_OK(db_secondary_->TryCatchUpWithPrimary());
+
+  auto* cfh = db_secondary_->DefaultColumnFamily();
+
+  {
+    PinnableSlice result;
+    ASSERT_OK(db_secondary_->Get(ReadOptions(), cfh, key, &result));
+    ASSERT_EQ(result, expected_merged);
+  }
+
+  {
+    GetMergeOperandsOptions get_merge_opts;
+    get_merge_opts.expected_max_number_of_operands = 2;
+
+    std::array<PinnableSlice, 2> merge_operands;
+    int number_of_operands = 0;
+
+    ASSERT_OK(db_secondary_->GetMergeOperands(
+        ReadOptions(), cfh, key, merge_operands.data(), &get_merge_opts,
+        &number_of_operands));
+    ASSERT_EQ(number_of_operands, 2);
+    ASSERT_EQ(merge_operands[0], default_value);
+    ASSERT_EQ(merge_operands[1], merge_operand);
+  }
+}
+
+TEST_F(DBSecondaryTest, GetImplReturnsBlobIndexWhenRequested) {
+  // Goal: cover the internal secondary GetImpl contract when the caller
+  // explicitly asks for raw blob-index bytes via `is_blob_index`. Catch-up
+  // rebuilds the blob index into the secondary memtable, and this path must
+  // preserve the encoded index instead of eagerly resolving or rejecting it.
+  Options options = GetDefaultOptions();
+  options.create_if_missing = true;
+  options.env = env_;
+
+  Reopen(options);
+
+  options.max_open_files = -1;
+  OpenSecondary(options);
+
+  std::string blob_index;
+  BlobIndex::EncodeInlinedTTL(&blob_index, /*expiration=*/9876543210, "blob");
+
+  WriteBatch batch;
+  ASSERT_OK(WriteBatchInternal::PutBlobIndex(
+      &batch, db_->DefaultColumnFamily()->GetID(), "blob_key", blob_index));
+  ASSERT_OK(db_->Write(WriteOptions(), &batch));
+  ASSERT_OK(db_secondary_->TryCatchUpWithPrimary());
+
+  PinnableSlice value;
+  bool is_blob_index = false;
+  DBImpl::GetImplOptions get_impl_options;
+  get_impl_options.column_family = db_secondary_->DefaultColumnFamily();
+  get_impl_options.value = &value;
+  get_impl_options.is_blob_index = &is_blob_index;
+
+  ASSERT_OK(db_secondary_full()->GetImpl(ReadOptions(), Slice("blob_key"),
+                                         get_impl_options));
+  ASSERT_TRUE(is_blob_index);
+  ASSERT_EQ(value, blob_index);
+}
+
+TEST_F(DBSecondaryTest,
+       GetAndGetEntityWithBlobBackedDefaultColumnDirectWriteMemtable) {
+  // Goal: cover the secondary memtable path after catch-up replays a blob
+  // direct-write entity from the primary WAL. The test checks both `Get()`,
+  // which must resolve the blob-backed default column, and `GetEntity()`,
+  // which must eagerly resolve all unresolved blob columns instead of exposing
+  // encoded blob indices to the caller.
+  Options options =
+      wide_column_test_util::GetDirectWriteOptions(GetDefaultOptions());
+  options.create_if_missing = true;
+  options.min_blob_size = 50;
+  options.env = env_;
+
+  Reopen(options);
+
+  options.max_open_files = -1;
+  OpenSecondary(options);
+
+  const std::string key = "secondary_direct_write_memtable_entity";
+  const std::string default_value =
+      wide_column_test_util::GenerateLargeValue(100, 'd');
+  const std::string large_value =
+      wide_column_test_util::GenerateLargeValue(120, 'l');
+  const std::string small_value = wide_column_test_util::GenerateSmallValue();
+  WideColumns columns{{kDefaultWideColumnName, default_value},
+                      {"col_large", large_value},
+                      {"col_small", small_value}};
+
+  ASSERT_OK(
+      db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(), key, columns));
+  ASSERT_OK(db_secondary_->TryCatchUpWithPrimary());
+
+  auto* cfh = db_secondary_->DefaultColumnFamily();
+
+  {
+    PinnableSlice result;
+    ASSERT_OK(db_secondary_->Get(ReadOptions(), cfh, key, &result));
+    ASSERT_EQ(result, default_value);
+  }
+
+  {
+    PinnableWideColumns result;
+    ASSERT_OK(db_secondary_->GetEntity(ReadOptions(), cfh, key, &result));
+    ASSERT_EQ(result.columns(), columns);
+  }
+}
+
+TEST_F(DBSecondaryTest, SecondaryDirectWriteMemtableBlobBlockCacheTier) {
+  // Goal: cover the secondary memtable path under kBlockCacheTier. Catch-up
+  // rebuilds the blob-backed direct-write entity from WAL, so serving Get() or
+  // GetEntity() would require blob I/O through the memtable resolution path,
+  // which must return Incomplete instead of issuing that I/O.
+  Options options =
+      wide_column_test_util::GetDirectWriteOptions(GetDefaultOptions());
+  options.create_if_missing = true;
+  options.min_blob_size = 50;
+  options.env = env_;
+
+  Reopen(options);
+
+  options.max_open_files = -1;
+  OpenSecondary(options);
+
+  const std::string key = "secondary_direct_write_memtable_block_cache_tier";
+  const std::string default_value =
+      wide_column_test_util::GenerateLargeValue(100, 'd');
+  const std::string large_value =
+      wide_column_test_util::GenerateLargeValue(120, 'l');
+  const std::string small_value = wide_column_test_util::GenerateSmallValue();
+  WideColumns columns{{kDefaultWideColumnName, default_value},
+                      {"col_large", large_value},
+                      {"col_small", small_value}};
+
+  ASSERT_OK(
+      db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(), key, columns));
+  ASSERT_OK(db_secondary_->TryCatchUpWithPrimary());
+
+  auto* cfh = db_secondary_->DefaultColumnFamily();
+  ReadOptions read_opts;
+  read_opts.read_tier = kBlockCacheTier;
+
+  {
+    PinnableSlice result;
+    const Status s = db_secondary_->Get(read_opts, cfh, key, &result);
+    ASSERT_TRUE(s.IsIncomplete()) << s.ToString();
+    ASSERT_TRUE(result.empty());
+  }
+
+  {
+    PinnableWideColumns result;
+    const Status s = db_secondary_->GetEntity(read_opts, cfh, key, &result);
+    ASSERT_TRUE(s.IsIncomplete()) << s.ToString();
+    ASSERT_TRUE(result.columns().empty());
+  }
+}
+
 TEST_F(DBSecondaryTest, InternalCompactionCompactedFiles) {
   Options options;
   options.env = env_;
@@ -729,6 +923,48 @@ TEST_F(DBSecondaryTest, OpenAsSecondaryWALTailing) {
   verify_db_func("new_foo_value_1", "new_bar_value");
 }
 
+TEST_F(DBSecondaryTest, CatchUpTailsCurrentWalWhenFutureWalExists) {
+  // Goal: secondary catch-up must keep tailing the current WAL even when a
+  // higher-number empty WAL is present. This reproduces the async WAL
+  // precreation shape by creating a future WAL before opening the secondary,
+  // then appending another write to the current WAL and catching up again.
+  Options options;
+  options.env = env_;
+  Reopen(options);
+
+  ASSERT_OK(Put("foo", "value0"));
+  ASSERT_OK(Flush());
+  ASSERT_OK(Put("foo", "value1"));
+  ASSERT_OK(db_->FlushWAL(/*sync=*/true));
+
+  std::unique_ptr<WalFile> current_wal;
+  ASSERT_OK(db_->GetCurrentWalFile(&current_wal));
+  ASSERT_NE(nullptr, current_wal);
+
+  const uint64_t future_wal_number = current_wal->LogNumber() + 1000;
+  std::unique_ptr<WritableFile> future_wal_file;
+  ASSERT_OK(env_->NewWritableFile(LogFileName(dbname_, future_wal_number),
+                                  &future_wal_file, EnvOptions()));
+  ASSERT_OK(future_wal_file->Close());
+
+  Options secondary_options;
+  secondary_options.env = env_;
+  secondary_options.max_open_files = -1;
+  OpenSecondary(secondary_options);
+
+  ReadOptions read_options;
+  std::string value;
+  ASSERT_OK(db_secondary_->Get(read_options, "foo", &value));
+  ASSERT_EQ("value1", value);
+
+  ASSERT_OK(Put("foo", "value2"));
+  ASSERT_OK(db_->FlushWAL(/*sync=*/true));
+
+  ASSERT_OK(db_secondary_->TryCatchUpWithPrimary());
+  ASSERT_OK(db_secondary_->Get(read_options, "foo", &value));
+  ASSERT_EQ("value2", value);
+}
+
 TEST_F(DBSecondaryTest, SecondaryTailingBug_ISSUE_8467) {
   Options options;
   options.env = env_;

db/db_test.cc

@@ -8071,6 +8071,41 @@ INSTANTIATE_TEST_CASE_P(OpenFilesAsync, OpenFilesAsyncTest,
                                            ::testing::Values(-1, 10),
                                            ::testing::Bool()));
 
+TEST_F(DBTest, ReadOnlyCloseDoesNotDeleteWriterFiles) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.disable_auto_compactions = true;
+  DestroyAndReopen(options);
+
+  ASSERT_OK(Put("before", "value"));
+  ASSERT_OK(Flush());
+
+  std::unique_ptr<DB> read_only_db;
+  ASSERT_OK(DB::OpenForReadOnly(options, dbname_, &read_only_db));
+
+  ASSERT_OK(Put("after", "value"));
+  ASSERT_OK(Flush());
+
+  std::vector<LiveFileMetaData> live_files;
+  db_->GetLiveFilesMetaData(&live_files);
+  ASSERT_GE(live_files.size(), 2);
+
+  std::string newest_sst_path;
+  uint64_t newest_file_number = 0;
+  for (const auto& live_file : live_files) {
+    if (live_file.file_number > newest_file_number) {
+      newest_file_number = live_file.file_number;
+      newest_sst_path = live_file.directory + "/" + live_file.relative_filename;
+    }
+  }
+  ASSERT_FALSE(newest_sst_path.empty());
+  ASSERT_OK(env_->FileExists(newest_sst_path));
+
+  read_only_db.reset();
+
+  ASSERT_OK(env_->FileExists(newest_sst_path));
+}
+
 // Test mix of races with async file open, reads, compactions
 TEST_P(OpenFilesAsyncTest, ConcurrentFileAccess) {
   Options options = CurrentOptions();
@@ -8203,6 +8238,98 @@ TEST_P(OpenFilesAsyncTest, BeforeRead) {
   }
 }
 
+// For modern DBs (manifest carries file_creation_time),
+// GetCreationTimeOfOldestFile returns the real value without waiting for async
+// file open.
+TEST_P(OpenFilesAsyncTest, GetCreationTimeOfOldestFileSkipsWaitForModernDB) {
+  if (max_open_files_ != -1) {
+    return;
+  }
+
+  Options options = CurrentOptions();
+  ASSERT_NO_FATAL_FAILURE(SetupData(options));
+
+  // If WaitForAsyncFileOpen is entered, the test fails -- modern DBs must
+  // never need the wait.
+  std::atomic<bool> waited{false};
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::WaitForAsyncFileOpen::BeforeWait",
+      [&](void* /*arg*/) { waited.store(true); });
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  OpenTestDB(options);
+
+  uint64_t creation_time = std::numeric_limits<uint64_t>::max();
+  ASSERT_OK(dbfull()->GetCreationTimeOfOldestFile(&creation_time));
+  EXPECT_GT(creation_time, 0);
+  EXPECT_LT(creation_time, std::numeric_limits<uint64_t>::max());
+  EXPECT_FALSE(waited.load());
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
+  Close();
+}
+
+// For legacy DBs (manifest lacks file_creation_time, value comes from the
+// pinned reader), GetCreationTimeOfOldestFile must block until
+// BGWorkAsyncFileOpen pins the readers; otherwise it returns 0 (the "info
+// unavailable" sentinel) instead of the real value.
+TEST_P(OpenFilesAsyncTest,
+       GetCreationTimeOfOldestFileBlocksOnAsyncOpenForLegacyDB) {
+  if (max_open_files_ != -1) {
+    return;
+  }
+
+  Options options = CurrentOptions();
+  ASSERT_NO_FATAL_FAILURE(SetupData(options));
+
+  // Simulate a legacy DB: force TryGetFileCreationTime to return
+  // kUnknownFileCreationTime until BGWorkAsyncFileOpen completes (i.e., what
+  // would happen if the manifest had no file_creation_time and the pinned
+  // reader was the only source).
+  std::atomic<bool> async_open_done{false};
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::BGWorkAsyncFileOpen:Done",
+      [&](void* /*arg*/) { async_open_done.store(true); });
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "Version::GetCreationTimeOfOldestFile::FileCreationTime", [&](void* arg) {
+        if (!async_open_done.load()) {
+          *static_cast<uint64_t*>(arg) = kUnknownFileCreationTime;
+        }
+      });
+
+  // Dependency chain: caller's first pass returns 0, enters
+  // WaitForAsyncFileOpen -> main thread confirms wait -> main thread releases
+  // async open -> background worker completes -> caller wakes -> second pass
+  // sees the real value.
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
+      {{"DBImpl::WaitForAsyncFileOpen::BeforeWait",
+        "OpenFilesAsyncTest::CreationTime::CallerBlocked"},
+       {"OpenFilesAsyncTest::CreationTime::CallerBlocked",
+        "OpenFilesAsyncTest::CreationTime::ReleaseAsyncOpen"},
+       {"OpenFilesAsyncTest::CreationTime::ReleaseAsyncOpen",
+        "DBImpl::BGWorkAsyncFileOpen::Start"}});
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  OpenTestDB(options);
+
+  uint64_t creation_time = std::numeric_limits<uint64_t>::max();
+  port::Thread caller([&]() {
+    ASSERT_OK(dbfull()->GetCreationTimeOfOldestFile(&creation_time));
+  });
+
+  TEST_SYNC_POINT("OpenFilesAsyncTest::CreationTime::CallerBlocked");
+  TEST_SYNC_POINT("OpenFilesAsyncTest::CreationTime::ReleaseAsyncOpen");
+  caller.join();
+
+  EXPECT_GT(creation_time, 0);
+  EXPECT_LT(creation_time, std::numeric_limits<uint64_t>::max());
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
+  Close();
+}
+
 TEST_P(OpenFilesAsyncTest, Shutdown) {
   Options options = CurrentOptions();
   ASSERT_NO_FATAL_FAILURE(SetupData(options));

db/db_test_util.cc

@@ -13,6 +13,7 @@
 #include "db/forward_iterator.h"
 #include "env/fs_readonly.h"
 #include "env/mock_env.h"
+#include "file/file_util.h"
 #include "port/lang.h"
 #include "rocksdb/cache.h"
 #include "rocksdb/convenience.h"
@@ -90,6 +91,8 @@ DBTestBase::DBTestBase(const std::string path, bool env_do_fsync)
   dbname_ = test::PerThreadDBPath(env_, path);
   alternative_wal_dir_ = dbname_ + "/wal";
   alternative_db_log_dir_ = dbname_ + "/db_log_dir";
+  EXPECT_OK(DestroyDir(env_, alternative_wal_dir_));
+  EXPECT_OK(DestroyDir(env_, alternative_db_log_dir_));
   auto options = CurrentOptions();
   options.env = env_;
   auto delete_options = options;
@@ -117,6 +120,8 @@ DBTestBase::~DBTestBase() {
   if (getenv("KEEP_DB")) {
     printf("DB is still at %s\n", dbname_.c_str());
   } else {
+    EXPECT_OK(DestroyDir(env_, alternative_wal_dir_));
+    EXPECT_OK(DestroyDir(env_, alternative_db_log_dir_));
     EXPECT_OK(DestroyDB(dbname_, options));
   }
   // Ensure SstFileManager (and its DeleteScheduler background thread) is

db/db_wal_test.cc

@@ -7,6 +7,8 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
+#include <algorithm>
+
 #include "db/db_test_util.h"
 #include "db/db_with_timestamp_test_util.h"
 #include "options/options_helper.h"
@@ -19,12 +21,62 @@
 #include "utilities/fault_injection_env.h"
 #include "utilities/fault_injection_fs.h"
 
+#if defined(OS_LINUX)
+#include <sys/statfs.h>
+#endif
+
+#if defined(OS_LINUX) && !defined(BTRFS_SUPER_MAGIC)
+#define BTRFS_SUPER_MAGIC 0x9123683E
+#endif
+
+#if defined(OS_LINUX) && !defined(TMPFS_MAGIC)
+#define TMPFS_MAGIC 0x01021994
+#endif
+
+#if defined(OS_LINUX) && !defined(OVERLAYFS_SUPER_MAGIC)
+#define OVERLAYFS_SUPER_MAGIC 0x794c7630
+#endif
+
+#if defined(OS_LINUX) && !defined(ZFS_SUPER_MAGIC)
+#define ZFS_SUPER_MAGIC 0x2fc12fc1
+#endif
+
 namespace ROCKSDB_NAMESPACE {
 class DBWALTestBase : public DBTestBase {
  protected:
   explicit DBWALTestBase(const std::string& dir_name)
       : DBTestBase(dir_name, /*env_do_fsync=*/true) {}
 
+  std::vector<uint64_t> ListWalNumbers() {
+    std::vector<uint64_t> wal_numbers;
+    std::vector<std::string> files;
+    EXPECT_OK(env_->GetChildren(dbname_, &files));
+    for (const auto& file : files) {
+      uint64_t number = 0;
+      FileType type = kWalFile;
+      if (ParseFileName(file, &number, &type) && type == kWalFile) {
+        wal_numbers.push_back(number);
+      }
+    }
+    std::sort(wal_numbers.begin(), wal_numbers.end());
+    return wal_numbers;
+  }
+
+  void CreateEmptyWal(uint64_t wal_number) {
+    std::unique_ptr<WritableFile> file;
+    ASSERT_OK(env_->NewWritableFile(LogFileName(dbname_, wal_number), &file,
+                                    EnvOptions()));
+    ASSERT_OK(file->Close());
+  }
+
+  void CreateWalWithContents(uint64_t wal_number, const Slice& contents) {
+    std::unique_ptr<WritableFile> file;
+    ASSERT_OK(env_->NewWritableFile(LogFileName(dbname_, wal_number), &file,
+                                    EnvOptions()));
+    ASSERT_OK(file->Append(contents));
+    ASSERT_OK(file->Close());
+  }
+
 #if defined(ROCKSDB_PLATFORM_POSIX)
  public:
 #if defined(ROCKSDB_FALLOCATE_PRESENT)
@@ -57,6 +109,31 @@ class DBWALTestBase : public DBTestBase {
     assert(err == 0);
     return sbuf.st_blocks * 512;
   }
+
+#if defined(ROCKSDB_FALLOCATE_PRESENT)
+  bool ShouldSkipAllocationCheck(const std::string& file_name) {
+    (void)file_name;
+#if defined(OS_LINUX)
+    struct statfs fs_stat;
+    if (statfs(file_name.c_str(), &fs_stat) == 0) {
+      if (fs_stat.f_type ==
+          static_cast<decltype(fs_stat.f_type)>(BTRFS_SUPER_MAGIC)) {
+        return true;
+      } else if (fs_stat.f_type ==
+                 static_cast<decltype(fs_stat.f_type)>(ZFS_SUPER_MAGIC)) {
+        return true;
+      } else if (fs_stat.f_type ==
+                 static_cast<decltype(fs_stat.f_type)>(TMPFS_MAGIC)) {
+        return true;
+      } else if (fs_stat.f_type ==
+                 static_cast<decltype(fs_stat.f_type)>(OVERLAYFS_SUPER_MAGIC)) {
+        return true;
+      }
+    }
+#endif
+    return false;
+  }
+#endif  // ROCKSDB_FALLOCATE_PRESENT
 #endif  // ROCKSDB_PLATFORM_POSIX
 };
 
@@ -900,11 +977,10 @@ class DBRecoveryTestBlobError
   std::string sync_point_;
 };
 
-INSTANTIATE_TEST_CASE_P(
-    DBRecoveryTestBlobError, DBRecoveryTestBlobError,
-    ::testing::ValuesIn(std::vector<std::string>{
-        "BlobFileBuilder::WriteBlobToFile:AddRecord",
-        "BlobFileBuilder::WriteBlobToFile:LegacyAppendFooterAndClose"}));
+INSTANTIATE_TEST_CASE_P(DBRecoveryTestBlobError, DBRecoveryTestBlobError,
+                        ::testing::ValuesIn(std::vector<std::string>{
+                            "BlobFileBuilder::WriteBlobToFile:AddRecord",
+                            "BlobFileBuilder::WriteBlobToFile:AppendFooter"}));
 
 TEST_P(DBRecoveryTestBlobError, RecoverWithBlobError) {
   // Write a value. Note that blob files are not actually enabled at this point.
@@ -2753,16 +2829,35 @@ TEST_F(DBWALTest, TruncateLastLogAfterRecoverWithoutFlush) {
   auto& file_before = log_files_before[0];
   ASSERT_LT(file_before->SizeFileBytes(), 1 * kKB);
   // The log file has preallocated space.
-  ASSERT_GE(GetAllocatedFileSize(dbname_ + file_before->PathName()),
-            preallocated_size);
+  {
+    std::string fname = dbname_ + file_before->PathName();
+    uint64_t allocated = GetAllocatedFileSize(fname);
+    if (ShouldSkipAllocationCheck(fname)) {
+      fprintf(stderr,
+              "Skipping preallocation check on this filesystem for %s\n",
+              fname.c_str());
+    } else if (allocated < preallocated_size) {
+      fprintf(stderr,
+              "Warning: allocated size (%lu) less than preallocated size (%lu) "
+              "for %s, skipping check. This may indicate the filesystem does "
+              "not support preallocation or does not report it.\n",
+              allocated, preallocated_size, fname.c_str());
+    } else {
+      ASSERT_GE(allocated, preallocated_size);
+    }
+  }
   Reopen(options);
   VectorLogPtr log_files_after;
   ASSERT_OK(dbfull()->GetSortedWalFiles(log_files_after));
   ASSERT_EQ(1, log_files_after.size());
   ASSERT_LT(log_files_after[0]->SizeFileBytes(), 1 * kKB);
   // The preallocated space should be truncated.
-  ASSERT_LT(GetAllocatedFileSize(dbname_ + file_before->PathName()),
-            preallocated_size);
+  {
+    std::string fname = dbname_ + file_before->PathName();
+    if (!ShouldSkipAllocationCheck(fname)) {
+      ASSERT_LT(GetAllocatedFileSize(fname), preallocated_size);
+    }
+  }
 }
 // Tests that we will truncate the preallocated space of the last log from
 // previous.
@@ -2789,8 +2884,23 @@ TEST_F(DBWALTest, TruncateLastLogAfterRecoverWithFlush) {
   ASSERT_EQ(1, log_files_before.size());
   auto& file_before = log_files_before[0];
   ASSERT_LT(file_before->SizeFileBytes(), 1 * kKB);
-  ASSERT_GE(GetAllocatedFileSize(dbname_ + file_before->PathName()),
-            preallocated_size);
+  {
+    std::string fname = dbname_ + file_before->PathName();
+    uint64_t allocated = GetAllocatedFileSize(fname);
+    if (ShouldSkipAllocationCheck(fname)) {
+      fprintf(stderr,
+              "Skipping preallocation check on this filesystem for %s\n",
+              fname.c_str());
+    } else if (allocated < preallocated_size) {
+      fprintf(stderr,
+              "Warning: allocated size (%lu) less than preallocated size (%lu) "
+              "for %s, skipping check. This may indicate the filesystem does "
+              "not support preallocation or does not report it.\n",
+              allocated, preallocated_size, fname.c_str());
+    } else {
+      ASSERT_GE(allocated, preallocated_size);
+    }
+  }
   // The log file has preallocated space.
   Close();
 
@@ -2808,8 +2918,12 @@ TEST_F(DBWALTest, TruncateLastLogAfterRecoverWithFlush) {
   // if  the process is in a crash loop, the log file may not get
   // deleted and thte preallocated space will keep accumulating. So we need
   // to ensure it gets trtuncated.
-  EXPECT_LT(GetAllocatedFileSize(dbname_ + file_before->PathName()),
-            preallocated_size);
+  {
+    std::string fname = dbname_ + file_before->PathName();
+    if (!ShouldSkipAllocationCheck(fname)) {
+      EXPECT_LT(GetAllocatedFileSize(fname), preallocated_size);
+    }
+  }
   TEST_SYNC_POINT(
       "DBWALTest::TruncateLastLogAfterRecoverWithFlush:AfterTruncate");
   reopen_thread.join();
@@ -2864,14 +2978,31 @@ TEST_F(DBWALTest, TruncateLastLogAfterRecoverWALEmpty) {
   log_file->PrepareWrite(0, 4096);
   log_file.reset();
 
-  ASSERT_GE(GetAllocatedFileSize(last_log), preallocated_size);
+  {
+    uint64_t allocated = GetAllocatedFileSize(last_log);
+    if (ShouldSkipAllocationCheck(last_log)) {
+      fprintf(stderr,
+              "Skipping preallocation check on this filesystem for %s\n",
+              last_log.c_str());
+    } else if (allocated < preallocated_size) {
+      fprintf(stderr,
+              "Warning: allocated size (%lu) less than preallocated size (%lu) "
+              "for %s, skipping check. This may indicate the filesystem does "
+              "not support preallocation or does not report it.\n",
+              allocated, preallocated_size, last_log.c_str());
+    } else {
+      ASSERT_GE(allocated, preallocated_size);
+    }
+  }
 
   port::Thread reopen_thread([&]() { Reopen(options); });
 
   TEST_SYNC_POINT(
       "DBWALTest::TruncateLastLogAfterRecoverWithFlush:AfterRecover");
   // The preallocated space should be truncated.
-  EXPECT_LT(GetAllocatedFileSize(last_log), preallocated_size);
+  if (!ShouldSkipAllocationCheck(last_log)) {
+    EXPECT_LT(GetAllocatedFileSize(last_log), preallocated_size);
+  }
   TEST_SYNC_POINT(
       "DBWALTest::TruncateLastLogAfterRecoverWithFlush:AfterTruncate");
   reopen_thread.join();
@@ -2913,8 +3044,23 @@ TEST_F(DBWALTest, ReadOnlyRecoveryNoTruncate) {
   auto& file_before = log_files_before[0];
   ASSERT_LT(file_before->SizeFileBytes(), 1 * kKB);
   // The log file has preallocated space.
-  auto db_size = GetAllocatedFileSize(dbname_ + file_before->PathName());
-  ASSERT_GE(db_size, preallocated_size);
+  std::string fname = dbname_ + file_before->PathName();
+  auto db_size = GetAllocatedFileSize(fname);
+  {
+    if (ShouldSkipAllocationCheck(fname)) {
+      fprintf(stderr,
+              "Skipping preallocation check on this filesystem for %s\n",
+              fname.c_str());
+    } else if (db_size < preallocated_size) {
+      fprintf(stderr,
+              "Warning: allocated size (%lu) less than preallocated size (%lu) "
+              "for %s, skipping check. This may indicate the filesystem does "
+              "not support preallocation or does not report it.\n",
+              db_size, preallocated_size, fname.c_str());
+    } else {
+      ASSERT_GE(db_size, preallocated_size);
+    }
+  }
   Close();
 
   // enable truncate and open DB as readonly, the file should not be truncated
@@ -2928,8 +3074,12 @@ TEST_F(DBWALTest, ReadOnlyRecoveryNoTruncate) {
   ASSERT_EQ(log_files_after[0]->PathName(), file_before->PathName());
   // The preallocated space should NOT be truncated.
   // the DB size is almost the same.
-  ASSERT_NEAR(GetAllocatedFileSize(dbname_ + file_before->PathName()), db_size,
-              db_size / 100);
+  {
+    std::string fname2 = dbname_ + file_before->PathName();
+    if (!ShouldSkipAllocationCheck(fname2)) {
+      ASSERT_NEAR(GetAllocatedFileSize(fname2), db_size, db_size / 100);
+    }
+  }
   SyncPoint::GetInstance()->DisableProcessing();
   SyncPoint::GetInstance()->ClearAllCallBacks();
 }
@@ -3072,6 +3222,284 @@ TEST_F(DBWALTest, EmptyWalReopenTest) {
   }
 }
 
+TEST_F(DBWALTest, AsyncWalPrecreateWaitsForPendingFile) {
+  Options options = CurrentOptions();
+  options.env = env_;
+  options.statistics = CreateDBStatistics();
+  options.async_wal_precreate = true;
+  options.recycle_log_file_num = 0;
+  options.write_buffer_size = 1024;
+  options.max_write_buffer_number = 4;
+
+  // Goal: cover the race where foreground WAL rotation reaches the switch
+  // before the background precreate publishes its result. The sync points hold
+  // the async worker after it has created the file but before it marks it
+  // ready, then verify the foreground writer waits instead of allocating a
+  // higher WAL number.
+  SyncPoint::GetInstance()->LoadDependency({
+      {"DBImpl::BGWorkAsyncWALPrecreate:BeforePublish",
+       "DBWALTest::AsyncWalPrecreateWaitsForPendingFile:WorkerBlocked"},
+      {"DBWALTest::AsyncWalPrecreateWaitsForPendingFile:AllowPublish",
+       "DBImpl::BGWorkAsyncWALPrecreate:Publish"},
+      {"DBImpl::WaitForAsyncWALPrecreate:BeforeWait",
+       "DBWALTest::AsyncWalPrecreateWaitsForPendingFile:ForegroundWaiting"},
+  });
+  SyncPoint::GetInstance()->EnableProcessing();
+  Defer cleanup_sync_point([] {
+    SyncPoint::GetInstance()->DisableProcessing();
+    SyncPoint::GetInstance()->ClearAllCallBacks();
+  });
+
+  Reopen(options);
+  TEST_SYNC_POINT(
+      "DBWALTest::AsyncWalPrecreateWaitsForPendingFile:WorkerBlocked");
+
+  ASSERT_OK(Put("fill", "value"));
+  Status writer_status;
+  port::Thread writer(
+      [&]() { writer_status = dbfull()->TEST_SwitchMemtable(); });
+
+  TEST_SYNC_POINT(
+      "DBWALTest::AsyncWalPrecreateWaitsForPendingFile:ForegroundWaiting");
+  TEST_SYNC_POINT(
+      "DBWALTest::AsyncWalPrecreateWaitsForPendingFile:AllowPublish");
+  writer.join();
+  ASSERT_OK(writer_status);
+  ASSERT_OK(Put("trigger", "value"));
+  ASSERT_EQ("value", Get("trigger"));
+  ASSERT_EQ(1, options.statistics->getTickerCount(WAL_PRECREATE_HIT));
+  ASSERT_EQ(1, options.statistics->getTickerCount(WAL_PRECREATE_WAITED));
+  ASSERT_EQ(0, options.statistics->getTickerCount(WAL_PRECREATE_MISS));
+}
+
+TEST_F(DBWALTest, AsyncWalPrecreateConsumesReadyFile) {
+  Options options = CurrentOptions();
+  options.env = env_;
+  options.statistics = CreateDBStatistics();
+  options.async_wal_precreate = true;
+  options.recycle_log_file_num = 0;
+  options.write_buffer_size = 1024;
+  options.max_write_buffer_number = 4;
+
+  // Goal: exercise the no-wait consumption path. The background task publishes
+  // a prepared future WAL before the foreground memtable switch, and the switch
+  // must install exactly that reserved WAL number instead of allocating
+  // another.
+  SyncPoint::GetInstance()->LoadDependency({
+      {"DBImpl::BGWorkAsyncWALPrecreate:Done",
+       "DBWALTest::AsyncWalPrecreateConsumesReadyFile:PrecreateDone"},
+  });
+  SyncPoint::GetInstance()->EnableProcessing();
+  Defer cleanup_sync_point([] {
+    SyncPoint::GetInstance()->DisableProcessing();
+    SyncPoint::GetInstance()->ClearAllCallBacks();
+  });
+
+  Reopen(options);
+  TEST_SYNC_POINT(
+      "DBWALTest::AsyncWalPrecreateConsumesReadyFile:PrecreateDone");
+
+  const std::vector<uint64_t> wal_numbers = ListWalNumbers();
+  ASSERT_GE(wal_numbers.size(), 2);
+  const uint64_t prepared_wal_number = wal_numbers.back();
+  ASSERT_NE(prepared_wal_number, dbfull()->TEST_GetCurrentLogNumber());
+
+  ASSERT_OK(Put("fill", "value"));
+  ASSERT_OK(dbfull()->TEST_SwitchMemtable());
+  ASSERT_EQ(prepared_wal_number, dbfull()->TEST_GetCurrentLogNumber());
+  ASSERT_EQ(1, options.statistics->getTickerCount(WAL_PRECREATE_HIT));
+  ASSERT_EQ(0, options.statistics->getTickerCount(WAL_PRECREATE_MISS));
+  ASSERT_EQ(0, options.statistics->getTickerCount(WAL_PRECREATE_WAITED));
+}
+
+TEST_F(DBWALTest, AsyncWalPrecreateRecoveryToleratesActualFutureWal) {
+  Options options = CurrentOptions();
+  options.env = env_;
+  options.async_wal_precreate = true;
+  options.recycle_log_file_num = 0;
+  options.track_and_verify_wals = true;
+  options.track_and_verify_wals_in_manifest = true;
+  options.avoid_flush_during_shutdown = true;
+  options.avoid_flush_during_recovery = true;
+
+  // Goal: cover the actual async precreation path, not a hand-forged empty WAL.
+  // The background task opens the future WAL, clean close releases the
+  // unpublished writer and leaves the empty future WAL behind, and recovery
+  // must tolerate that file while replaying the preceding WAL with real user
+  // data.
+  SyncPoint::GetInstance()->LoadDependency({
+      {"DBImpl::BGWorkAsyncWALPrecreate:Done",
+       "DBWALTest::AsyncWalPrecreateRecoveryToleratesActualFutureWal:"
+       "PrecreateDone"},
+  });
+  SyncPoint::GetInstance()->EnableProcessing();
+  Defer cleanup_sync_point([] {
+    SyncPoint::GetInstance()->DisableProcessing();
+    SyncPoint::GetInstance()->ClearAllCallBacks();
+  });
+
+  DestroyAndReopen(options);
+  TEST_SYNC_POINT(
+      "DBWALTest::AsyncWalPrecreateRecoveryToleratesActualFutureWal:"
+      "PrecreateDone");
+
+  const std::vector<uint64_t> wal_numbers = ListWalNumbers();
+  ASSERT_GE(wal_numbers.size(), 2);
+  const uint64_t future_wal_number = wal_numbers.back();
+  uint64_t future_wal_size = 0;
+  ASSERT_OK(env_->GetFileSize(LogFileName(dbname_, future_wal_number),
+                              &future_wal_size));
+  ASSERT_EQ(0, future_wal_size);
+
+  ASSERT_OK(Put("key", "value"));
+  Close();
+
+  ASSERT_OK(TryReopen(options));
+  ASSERT_EQ("value", Get("key"));
+  ASSERT_OK(Put("after-recovery", "value2"));
+  ASSERT_EQ("value2", Get("after-recovery"));
+}
+
+TEST_F(DBWALTest, AsyncWalPrecreateDeletesFileOnStartFailure) {
+  Options options = CurrentOptions();
+  options.env = env_;
+  options.statistics = CreateDBStatistics();
+  options.async_wal_precreate = true;
+  options.recycle_log_file_num = 0;
+  options.write_buffer_size = 1024;
+  options.max_write_buffer_number = 4;
+
+  // Goal: force failure after consuming a prepared WAL but before it becomes a
+  // logical WAL. The failed, unpublished file must be deleted and its writer
+  // destroyed outside mutex_.
+  SyncPoint::GetInstance()->LoadDependency({
+      {"DBImpl::BGWorkAsyncWALPrecreate:Done",
+       "DBWALTest::AsyncWalPrecreateDeletesFileOnStartFailure:PrecreateDone"},
+  });
+  SyncPoint::GetInstance()->EnableProcessing();
+  Defer cleanup_sync_point([] {
+    SyncPoint::GetInstance()->DisableProcessing();
+    SyncPoint::GetInstance()->ClearAllCallBacks();
+  });
+
+  Reopen(options);
+  TEST_SYNC_POINT(
+      "DBWALTest::AsyncWalPrecreateDeletesFileOnStartFailure:PrecreateDone");
+  SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::StartWALFile:AfterCompressionTypeRecord", [](void* arg) {
+        *static_cast<IOStatus*>(arg) =
+            IOStatus::IOError("injected async WAL start failure");
+      });
+  const std::vector<uint64_t> wal_numbers = ListWalNumbers();
+  ASSERT_GE(wal_numbers.size(), 2);
+  const uint64_t prepared_wal_number = wal_numbers.back();
+
+  ASSERT_OK(Put("fill", "value"));
+  ASSERT_NOK(dbfull()->TEST_SwitchMemtable());
+  ASSERT_TRUE(
+      env_->FileExists(LogFileName(dbname_, prepared_wal_number)).IsNotFound());
+  ASSERT_EQ(1, options.statistics->getTickerCount(WAL_PRECREATE_HIT));
+}
+
+TEST_F(DBWALTest, AsyncWalPrecreateFailureFallsBackToSyncCreation) {
+  Options options = CurrentOptions();
+  options.env = env_;
+  options.statistics = CreateDBStatistics();
+  options.async_wal_precreate = true;
+  options.recycle_log_file_num = 0;
+  options.write_buffer_size = 1024;
+  options.max_write_buffer_number = 4;
+
+  // Goal: fail the background precreate file open after normal DB open has
+  // created the current WAL. The foreground switch should observe no prepared
+  // WAL, create one synchronously, and expose both failure and miss counters.
+  SyncPoint::GetInstance()->LoadDependency({
+      {"DBImpl::BGWorkAsyncWALPrecreate:Done",
+       "DBWALTest::AsyncWalPrecreateFailureFallsBackToSyncCreation:"
+       "PrecreateDone"},
+  });
+  SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::Open:AfterDeleteFiles",
+      [&](void*) { env_->non_writable_count_ = 1; });
+  SyncPoint::GetInstance()->EnableProcessing();
+  Defer cleanup_sync_point([&] {
+    env_->non_writable_count_ = 0;
+    SyncPoint::GetInstance()->DisableProcessing();
+    SyncPoint::GetInstance()->ClearAllCallBacks();
+  });
+
+  Reopen(options);
+  TEST_SYNC_POINT(
+      "DBWALTest::AsyncWalPrecreateFailureFallsBackToSyncCreation:"
+      "PrecreateDone");
+  ASSERT_EQ(1, options.statistics->getTickerCount(WAL_PRECREATE_FAILED));
+
+  ASSERT_OK(Put("fill", "value"));
+  ASSERT_OK(dbfull()->TEST_SwitchMemtable());
+  ASSERT_EQ(1, options.statistics->getTickerCount(WAL_PRECREATE_MISS));
+}
+
+TEST_F(DBWALTest, ReadOnlyErrorIfWalFileExistsIgnoresEmptyWals) {
+  Options options = CurrentOptions();
+  options.env = env_;
+  DestroyAndReopen(options);
+  Close();
+
+  // Goal: preserve error_if_wal_file_exists for WALs containing data while
+  // allowing empty future WALs left by async precreation. This creates empty
+  // WAL files directly so the test isolates the read-only open contract.
+  const std::vector<uint64_t> existing_wal_numbers = ListWalNumbers();
+  const uint64_t empty_wal_number =
+      existing_wal_numbers.empty() ? 1 : existing_wal_numbers.back() + 1;
+  CreateEmptyWal(empty_wal_number);
+
+  DBOptions db_options(options);
+  std::vector<ColumnFamilyDescriptor> column_families;
+  column_families.emplace_back(kDefaultColumnFamilyName,
+                               ColumnFamilyOptions(options));
+  std::vector<ColumnFamilyHandle*> handles;
+  std::unique_ptr<DB> read_only_db;
+  ASSERT_OK(DB::OpenForReadOnly(db_options, dbname_, column_families, &handles,
+                                &read_only_db,
+                                true /* error_if_wal_file_exists */));
+  for (ColumnFamilyHandle* handle : handles) {
+    ASSERT_OK(read_only_db->DestroyColumnFamilyHandle(handle));
+  }
+  read_only_db.reset();
+  handles.clear();
+
+  CreateWalWithContents(empty_wal_number + 1, Slice("not empty"));
+  ASSERT_NOK(DB::OpenForReadOnly(db_options, dbname_, column_families, &handles,
+                                 &read_only_db,
+                                 true /* error_if_wal_file_exists */));
+}
+
+TEST_F(DBWALTest, RecoveryToleratesEmptyFutureWal) {
+  Options options = CurrentOptions();
+  options.env = env_;
+  options.track_and_verify_wals = true;
+  options.track_and_verify_wals_in_manifest = true;
+  options.avoid_flush_during_recovery = true;
+  DestroyAndReopen(options);
+
+  ASSERT_OK(Put("key", "value"));
+  Close();
+
+  const std::vector<uint64_t> wal_numbers = ListWalNumbers();
+  ASSERT_FALSE(wal_numbers.empty());
+  const uint64_t future_wal_number = wal_numbers.back() + 1;
+  CreateEmptyWal(future_wal_number);
+
+  // Goal: model a crash after async WAL precreation but before consumption.
+  // Recovery should process the zero-byte future WAL, mark its file number
+  // used, and keep replaying the real preceding WAL without reporting a WAL
+  // chain or MANIFEST-tracking error.
+  ASSERT_OK(TryReopen(options));
+  ASSERT_EQ("value", Get("key"));
+  ASSERT_OK(Put("after-recovery", "value2"));
+  ASSERT_EQ("value2", Get("after-recovery"));
+}
+
 TEST_F(DBWALTest, RecoveryFlushSwitchWALOnEmptyMemtable) {
   Options options = CurrentOptions();
   auto fault_fs = std::make_shared<FaultInjectionTestFS>(FileSystem::Default());

db/dbformat_test.cc

@@ -218,7 +218,7 @@ TEST_P(IterKeySwapTest, SwapAndDestroy) {
         b_use_secondary] = GetParam();
 
   std::string expected_a, expected_b;
-  // Storage for pinned keys — must outlive the IterKeys.
+  // Storage for pinned keys -- must outlive the IterKeys.
   std::string a_storage, b_storage;
   IterKey a;
   expected_a =
@@ -236,7 +236,7 @@ TEST_P(IterKeySwapTest, SwapAndDestroy) {
     // After swap: a has b's old data, b has a's old data.
     ASSERT_EQ(a.GetUserKey().ToString(), expected_b);
     ASSERT_EQ(b.GetUserKey().ToString(), expected_a);
-  }  // b destroyed here — must not corrupt a's data
+  }  // b destroyed here -- must not corrupt a's data
 
   // a must still hold valid data after b's destruction.
   ASSERT_EQ(a.GetUserKey().ToString(), expected_b);

db/listener_test.cc

@@ -31,6 +31,7 @@
 #include "util/mutexlock.h"
 #include "util/rate_limiter_impl.h"
 #include "util/string_util.h"
+#include "utilities/fault_injection_fs.h"
 #include "utilities/merge_operators.h"
 
 namespace ROCKSDB_NAMESPACE {
@@ -201,6 +202,194 @@ TEST_F(EventListenerTest, OnSingleDBCompactionTest) {
   }
 }
 
+// Listener that asserts OnCompactionPreCommit fires strictly between
+// OnCompactionBegin and OnCompactionCompleted, and that input files'
+// being_compacted flag is still true at that point.
+class TestCompactionPreCommitListener : public EventListener {
+ public:
+  explicit TestCompactionPreCommitListener(EventListenerTest* test)
+      : test_(test) {}
+
+  void OnCompactionBegin(DB* /*db*/, const CompactionJobInfo& ci) override {
+    std::lock_guard<std::mutex> lock(mutex_);
+    ++begin_count_;
+    last_begin_job_id_ = ci.job_id;
+    EXPECT_EQ(begin_count_, pre_commit_count_ + 1);
+    EXPECT_EQ(begin_count_, completed_count_ + 1);
+  }
+
+  void OnCompactionPreCommit(DB* db, const CompactionJobInfo& ci) override {
+    std::lock_guard<std::mutex> lock(mutex_);
+    ++pre_commit_count_;
+    // Must fire after Begin and before Completed for this compaction.
+    EXPECT_EQ(pre_commit_count_, begin_count_);
+    EXPECT_EQ(pre_commit_count_, completed_count_ + 1);
+    EXPECT_EQ(ci.job_id, last_begin_job_id_);
+    EXPECT_GT(ci.input_files.size(), 0U);
+
+    // Verify input files are still marked being_compacted.
+    std::vector<std::vector<FileMetaData>> files_by_level;
+    test_->dbfull()->TEST_GetFilesMetaData(test_->handles_[ci.cf_id],
+                                           &files_by_level);
+    EXPECT_EQ(test_->db_.get(), db);
+    for (const auto& info : ci.input_file_infos) {
+      bool found = false;
+      for (const auto& level_files : files_by_level) {
+        for (const auto& meta : level_files) {
+          if (meta.fd.GetNumber() == info.file_number) {
+            found = true;
+            EXPECT_TRUE(meta.being_compacted)
+                << "input file " << info.file_number
+                << " should still be being_compacted in "
+                   "OnCompactionPreCommit";
+          }
+        }
+      }
+      EXPECT_TRUE(found) << "input file " << info.file_number
+                         << " not found in DB";
+    }
+  }
+
+  void OnCompactionCompleted(DB* /*db*/, const CompactionJobInfo& ci) override {
+    std::lock_guard<std::mutex> lock(mutex_);
+    ++completed_count_;
+    // Must fire after the matching PreCommit.
+    EXPECT_EQ(completed_count_, pre_commit_count_);
+    EXPECT_EQ(completed_count_, begin_count_);
+    EXPECT_EQ(ci.job_id, last_begin_job_id_);
+  }
+
+  size_t BeginCount() {
+    std::lock_guard<std::mutex> lock(mutex_);
+    return begin_count_;
+  }
+  size_t PreCommitCount() {
+    std::lock_guard<std::mutex> lock(mutex_);
+    return pre_commit_count_;
+  }
+  size_t CompletedCount() {
+    std::lock_guard<std::mutex> lock(mutex_);
+    return completed_count_;
+  }
+
+ private:
+  EventListenerTest* test_;
+  std::mutex mutex_;
+  size_t begin_count_ = 0;
+  size_t pre_commit_count_ = 0;
+  size_t completed_count_ = 0;
+  int last_begin_job_id_ = -1;
+};
+
+TEST_F(EventListenerTest, OnCompactionPreCommitOrdering) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.compaction_style = kCompactionStyleLevel;
+  options.compression = kNoCompression;
+  options.level0_file_num_compaction_trigger = 4;
+  auto* listener = new TestCompactionPreCommitListener(this);
+  options.listeners.emplace_back(listener);
+
+  // Verify sync-point ordering: Begin -> PreCommit -> Completed.
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
+      {{"DBImpl::NotifyOnCompactionBegin::UnlockMutex",
+        "DBImpl::NotifyOnCompactionPreCommit::UnlockMutex"},
+       {"DBImpl::NotifyOnCompactionPreCommit::UnlockMutex",
+        "DBImpl::NotifyOnCompactionCompleted::UnlockMutex"}});
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  CreateAndReopenWithCF({"pikachu"}, options);
+
+  Random rnd(301);
+  for (int i = 0; i < 4; i++) {
+    for (int j = 0; j < 10; j++) {
+      ASSERT_OK(Put(1, rnd.RandomString(10), rnd.RandomString(10)));
+    }
+    ASSERT_OK(Flush(1));
+  }
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+
+  EXPECT_GT(listener->BeginCount(), 0U);
+  EXPECT_EQ(listener->PreCommitCount(), listener->BeginCount());
+  EXPECT_EQ(listener->CompletedCount(), listener->BeginCount());
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
+}
+
+class TestDBShutdownBeginListener : public EventListener {
+ public:
+  void OnDBShutdownBegin(DB* db) override {
+    std::lock_guard<std::mutex> lock(mutex_);
+    ++shutdown_count_;
+    last_db_ = db;
+  }
+
+  int ShutdownCount() const {
+    std::lock_guard<std::mutex> lock(mutex_);
+    return shutdown_count_;
+  }
+
+  DB* LastDB() const {
+    std::lock_guard<std::mutex> lock(mutex_);
+    return last_db_;
+  }
+
+ private:
+  mutable std::mutex mutex_;
+  int shutdown_count_ = 0;
+  DB* last_db_ = nullptr;
+};
+
+TEST_F(EventListenerTest, OnDBShutdownBeginOnceForCancelAndClose) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  auto listener = std::make_shared<TestDBShutdownBeginListener>();
+  options.listeners.emplace_back(listener);
+  DestroyAndReopen(options);
+
+  DB* db = db_.get();
+  dbfull()->CancelAllBackgroundWork(false);
+  EXPECT_EQ(1, listener->ShutdownCount());
+  EXPECT_EQ(db, listener->LastDB());
+
+  Close();
+  EXPECT_EQ(1, listener->ShutdownCount());
+}
+
+TEST_F(EventListenerTest, OnDBShutdownBeginOnFailedOpen) {
+  Close();
+
+  std::shared_ptr<FaultInjectionTestFS> fs(
+      new FaultInjectionTestFS(env_->GetFileSystem()));
+  std::unique_ptr<Env> env(NewCompositeEnv(fs));
+
+  Options options = CurrentOptions();
+  options.env = env.get();
+  options.create_if_missing = true;
+  auto listener = std::make_shared<TestDBShutdownBeginListener>();
+  options.listeners.emplace_back(listener);
+
+  SyncPoint::GetInstance()->SetCallBack(
+      "PersistRocksDBOptions:create",
+      [&](void* /*arg*/) { fs->SetFilesystemActive(false); });
+  SyncPoint::GetInstance()->SetCallBack(
+      "PersistRocksDBOptions:written",
+      [&](void* /*arg*/) { fs->SetFilesystemActive(true); });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  std::unique_ptr<DB> db;
+  Status s = DB::Open(options, dbname_, &db);
+  ASSERT_TRUE(s.IsIOError()) << s.ToString();
+  EXPECT_EQ(1, listener->ShutdownCount());
+  EXPECT_NE(nullptr, listener->LastDB());
+  EXPECT_EQ(nullptr, db.get());
+
+  fs->SetFilesystemActive(true);
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+}
+
 // This simple Listener can only handle one flush at a time.
 class TestFlushListener : public EventListener {
  public:
@@ -1696,7 +1885,7 @@ TEST_F(EventListenerTest, BackgroundJobPressure) {
                  Env::Priority::LOW);
   sleeping_task.WaitUntilSleeping();
 
-  // Phase 1: No pressure — 3 SST files (below slowdown trigger=4).
+  // Phase 1: No pressure -- 3 SST files (below slowdown trigger=4).
   for (int i = 0; i < 3; i++) {
     ASSERT_OK(Put("k" + std::to_string(i), std::string(100, 'x')));
     ASSERT_OK(Flush());
@@ -1714,7 +1903,7 @@ TEST_F(EventListenerTest, BackgroundJobPressure) {
   // MaybeScheduleFlushOrCompaction() runs before the pressure callback and
   // may schedule new flush work, making flush counts non-deterministic.
 
-  // Phase 2: Build pressure — flush past slowdown trigger (4 L0 SST files).
+  // Phase 2: Build pressure -- flush past slowdown trigger (4 L0 SST files).
   // Compaction is blocked, so L0 SST files pile up.
   listener->Reset();
   {
@@ -1750,11 +1939,11 @@ TEST_F(EventListenerTest, BackgroundJobPressure) {
   ASSERT_TRUE(found_compaction_scheduled);
   ASSERT_TRUE(found_high_proximity);
 
-  // Phase 3: Relieve pressure — unblock compaction, wait for completion.
+  // Phase 3: Relieve pressure -- unblock compaction, wait for completion.
   listener->Reset();
   sleeping_task.WakeUp();
   sleeping_task.WaitUntilDone();
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+  ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork());
 
   snapshots = listener->GetSnapshots();
   CheckInvariants(snapshots);

db/log_writer.cc

@@ -22,9 +22,10 @@ namespace ROCKSDB_NAMESPACE::log {
 
 Writer::Writer(std::unique_ptr<WritableFileWriter>&& dest, uint64_t log_number,
                bool recycle_log_files, bool manual_flush,
-               CompressionType compression_type, bool track_and_verify_wals)
+               CompressionType compression_type, bool track_and_verify_wals,
+               size_t initial_block_offset)
     : dest_(std::move(dest)),
-      block_offset_(0),
+      block_offset_(initial_block_offset),
       log_number_(log_number),
       recycle_log_files_(recycle_log_files),
       // Header size varies depending on whether we are recycling or not.

db/log_writer.h

@@ -75,15 +75,20 @@ namespace log {
 class Writer {
  public:
   // Create a writer that will append data to "*dest".
-  // "*dest" must be initially empty.
-  // "*dest" must remain live while this Writer is in use.
+  // "*dest" must remain live while this Writer is in use. By default
+  // "*dest" is expected to be empty (initial_block_offset = 0). When
+  // resuming append into an existing log file (e.g.
+  // VersionSet::ReopenManifestForAppend), pass the offset within the
+  // current 32 KiB block at which writes should resume so that record
+  // framing aligns to the existing block layout.
   // TODO(hx235): separate WAL related parameters from general `Reader`
   // parameters
   explicit Writer(std::unique_ptr<WritableFileWriter>&& dest,
                   uint64_t log_number, bool recycle_log_files,
                   bool manual_flush = false,
                   CompressionType compressionType = kNoCompression,
-                  bool track_and_verify_wals = false);
+                  bool track_and_verify_wals = false,
+                  size_t initial_block_offset = 0);
   // No copying allowed
   Writer(const Writer&) = delete;
   void operator=(const Writer&) = delete;

db/memtable.cc

@@ -15,6 +15,7 @@
 #include <memory>
 #include <optional>
 
+#include "db/blob/blob_fetcher.h"
 #include "db/blob/blob_file_partition_manager.h"
 #include "db/blob/blob_index.h"
 #include "db/dbformat.h"
@@ -70,6 +71,67 @@ Status GetDefaultColumnBlobIndexSlice(Slice entity, Slice* blob_index_slice) {
   return status;
 }
 
+Status PushWideColumnEntityDefaultOperand(const Slice& user_key,
+                                          const Slice& entity,
+                                          MergeContext* merge_context,
+                                          bool operand_pinned,
+                                          const BlobFetcher* blob_fetcher) {
+  assert(merge_context != nullptr);
+
+  Slice entity_ref = entity;
+  Slice value_of_default;
+  Status status = WideColumnSerialization::GetValueOfDefaultColumn(
+      entity_ref, value_of_default);
+  if (status.ok()) {
+    merge_context->PushOperand(value_of_default, operand_pinned);
+    return status;
+  }
+  if (!status.IsNotSupported()) {
+    return status;
+  }
+  if (blob_fetcher == nullptr) {
+    return Status::Corruption(
+        "Cannot resolve blob-backed default column without a blob fetcher");
+  }
+
+  PinnableSlice resolved_default;
+  bool resolved = false;
+  status = WideColumnSerialization::GetValueOfDefaultColumnResolvingBlobs(
+      entity, user_key, blob_fetcher, resolved_default, resolved);
+  if (status.ok()) {
+    // Resolved blob values are backed by this stack-local PinnableSlice, so
+    // copy them into MergeContext instead of pinning their storage.
+    merge_context->PushOperand(Slice(resolved_default), false);
+  }
+  return status;
+}
+
+Status MergeWithWideColumnEntityBaseValue(
+    const Slice& user_key, const Slice& entity,
+    const MergeOperator* merge_operator, MergeContext* merge_context,
+    Logger* logger, Statistics* statistics, SystemClock* clock,
+    std::string* value, PinnableWideColumns* columns,
+    const BlobFetcher* blob_fetcher) {
+  assert(merge_context != nullptr);
+
+  std::string resolved_entity;
+  Slice effective_entity;
+  Status status = WideColumnSerialization::ResolveEntityForMerge(
+      entity, user_key, blob_fetcher, nullptr /* prefetch_buffers */,
+      resolved_entity, effective_entity);
+  if (!status.ok()) {
+    return status;
+  }
+
+  // `op_failure_scope` (an output parameter) is not provided (set to nullptr)
+  // since a failure must be propagated regardless of its value.
+  return MergeHelper::TimedFullMerge(
+      merge_operator, user_key, MergeHelper::kWideBaseValue, effective_entity,
+      merge_context->GetOperands(), logger, statistics, clock,
+      /* update_num_ops_stats */ true, /* op_failure_scope */ nullptr, value,
+      columns);
+}
+
 }  // namespace
 
 ImmutableMemTableOptions::ImmutableMemTableOptions(
@@ -1254,6 +1316,7 @@ struct Saver {
   bool inplace_update_support;
   bool do_merge;
   SystemClock* clock;
+  const BlobFetcher* blob_fetcher;
 
   ReadCallback* callback_;
   bool* is_blob_index;
@@ -1424,27 +1487,17 @@ static bool SaveValue(void* arg, const char* entry) {
           // Preserve the value with the goal of returning it as part of
           // raw merge operands to the user
 
-          Slice value_of_default;
-          *(s->status) = WideColumnSerialization::GetValueOfDefaultColumn(
-              v, value_of_default);
-
-          if (s->status->ok()) {
-            merge_context->PushOperand(
-                value_of_default,
-                s->inplace_update_support == false /* operand_pinned */);
-          }
+          *(s->status) = PushWideColumnEntityDefaultOperand(
+              s->key->user_key(), v, merge_context,
+              s->inplace_update_support == false /* operand_pinned */,
+              s->blob_fetcher);
         } else if (*(s->merge_in_progress)) {
           assert(s->do_merge);
 
           if (s->value || s->columns) {
-            // `op_failure_scope` (an output parameter) is not provided (set
-            // to nullptr) since a failure must be propagated regardless of
-            // its value.
-            *(s->status) = MergeHelper::TimedFullMerge(
-                merge_operator, s->key->user_key(), MergeHelper::kWideBaseValue,
-                v, merge_context->GetOperands(), s->logger, s->statistics,
-                s->clock, /* update_num_ops_stats */ true,
-                /* op_failure_scope */ nullptr, s->value, s->columns);
+            *(s->status) = MergeWithWideColumnEntityBaseValue(
+                s->key->user_key(), v, merge_operator, merge_context, s->logger,
+                s->statistics, s->clock, s->value, s->columns, s->blob_fetcher);
           }
         } else if (s->value) {
           Slice value_of_default;
@@ -1520,7 +1573,8 @@ bool MemTable::Get(const LookupKey& key, std::string* value,
                    SequenceNumber* max_covering_tombstone_seq,
                    SequenceNumber* seq, const ReadOptions& read_opts,
                    bool immutable_memtable, ReadCallback* callback,
-                   bool* is_blob_index, bool do_merge) {
+                   bool* is_blob_index, bool do_merge,
+                   const BlobFetcher* blob_fetcher) {
   // The sequence number is updated synchronously in version_set.h
   if (IsEmpty()) {
     // Avoiding recording stats for speed.
@@ -1578,7 +1632,7 @@ bool MemTable::Get(const LookupKey& key, std::string* value,
     }
     GetFromTable(key, *max_covering_tombstone_seq, do_merge, callback,
                  is_blob_index, value, columns, timestamp, s, merge_context,
-                 seq, &found_final_value, &merge_in_progress);
+                 seq, &found_final_value, &merge_in_progress, blob_fetcher);
   }
 
   // No change to value, since we have not yet found a Put/Delete
@@ -1601,7 +1655,8 @@ void MemTable::GetFromTable(const LookupKey& key,
                             PinnableWideColumns* columns,
                             std::string* timestamp, Status* s,
                             MergeContext* merge_context, SequenceNumber* seq,
-                            bool* found_final_value, bool* merge_in_progress) {
+                            bool* found_final_value, bool* merge_in_progress,
+                            const BlobFetcher* blob_fetcher) {
   Saver saver;
   saver.status = s;
   saver.found_final_value = found_final_value;
@@ -1619,6 +1674,7 @@ void MemTable::GetFromTable(const LookupKey& key,
   saver.inplace_update_support = moptions_.inplace_update_support;
   saver.statistics = moptions_.statistics;
   saver.clock = clock_;
+  saver.blob_fetcher = blob_fetcher;
   saver.callback_ = callback;
   saver.is_blob_index = is_blob_index;
   saver.do_merge = do_merge;
@@ -1648,7 +1704,8 @@ Status MemTable::ValidateKey(const char* key, bool allow_data_in_errors) {
 }
 
 void MemTable::MultiGet(const ReadOptions& read_options, MultiGetRange* range,
-                        ReadCallback* callback, bool immutable_memtable) {
+                        ReadCallback* callback, bool immutable_memtable,
+                        const BlobFetcher* blob_fetcher) {
   // The sequence number is updated synchronously in version_set.h
   if (IsEmpty()) {
     // Avoiding recording stats for speed.
@@ -1740,6 +1797,7 @@ void MemTable::MultiGet(const ReadOptions& read_options, MultiGetRange* range,
       saver.inplace_update_support = moptions_.inplace_update_support;
       saver.statistics = moptions_.statistics;
       saver.clock = clock_;
+      saver.blob_fetcher = blob_fetcher;
       saver.callback_ = callback;
       saver.is_blob_index = &iter->is_blob_index;
       saver.do_merge = true;
@@ -1831,7 +1889,7 @@ void MemTable::MultiGet(const ReadOptions& read_options, MultiGetRange* range,
           *(iter->lkey), iter->max_covering_tombstone_seq, true, callback,
           &iter->is_blob_index, iter->value ? iter->value->GetSelf() : nullptr,
           iter->columns, iter->timestamp, iter->s, &(iter->merge_context),
-          &dummy_seq, &found_final_value, &merge_in_progress);
+          &dummy_seq, &found_final_value, &merge_in_progress, blob_fetcher);
 
       if (!found_final_value && merge_in_progress) {
         if (iter->s->ok()) {

db/memtable.h

@@ -39,6 +39,7 @@
 
 namespace ROCKSDB_NAMESPACE {
 
+class BlobFetcher;
 class BlobFilePartitionManager;
 struct FlushJobInfo;
 class Mutex;
@@ -236,25 +237,27 @@ class ReadOnlyMemTable {
                    SequenceNumber* max_covering_tombstone_seq,
                    SequenceNumber* seq, const ReadOptions& read_opts,
                    bool immutable_memtable, ReadCallback* callback = nullptr,
-                   bool* is_blob_index = nullptr, bool do_merge = true) = 0;
+                   bool* is_blob_index = nullptr, bool do_merge = true,
+                   const BlobFetcher* blob_fetcher = nullptr) = 0;
   bool Get(const LookupKey& key, std::string* value,
            PinnableWideColumns* columns, std::string* timestamp, Status* s,
            MergeContext* merge_context,
            SequenceNumber* max_covering_tombstone_seq,
            const ReadOptions& read_opts, bool immutable_memtable,
            ReadCallback* callback = nullptr, bool* is_blob_index = nullptr,
-           bool do_merge = true) {
+           bool do_merge = true, const BlobFetcher* blob_fetcher = nullptr) {
     SequenceNumber seq;
     return Get(key, value, columns, timestamp, s, merge_context,
                max_covering_tombstone_seq, &seq, read_opts, immutable_memtable,
-               callback, is_blob_index, do_merge);
+               callback, is_blob_index, do_merge, blob_fetcher);
   }
 
   // @param immutable_memtable Whether this memtable is immutable. Used
   // internally by NewRangeTombstoneIterator(). See comment above
   // NewRangeTombstoneIterator() for more detail.
   virtual void MultiGet(const ReadOptions& read_options, MultiGetRange* range,
-                        ReadCallback* callback, bool immutable_memtable) = 0;
+                        ReadCallback* callback, bool immutable_memtable,
+                        const BlobFetcher* blob_fetcher = nullptr) = 0;
 
   // Get total number of entries in the mem table.
   // REQUIRES: external synchronization to prevent simultaneous
@@ -320,7 +323,7 @@ class ReadOnlyMemTable {
   virtual uint64_t ApproximateOldestKeyTime() const = 0;
 
   // Inserts a range tombstone [start_key, end_key) that is logically redundant
-  // — it is derived from existing point tombstones observed during iteration
+  // -- it is derived from existing point tombstones observed during iteration
   // and does not delete any data that isn't already deleted. This is a
   // best-effort optimization. It allows future reads to skip iterating over
   // continuous single deletion tombstones.
@@ -688,10 +691,12 @@ class MemTable final : public ReadOnlyMemTable {
            SequenceNumber* max_covering_tombstone_seq, SequenceNumber* seq,
            const ReadOptions& read_opts, bool immutable_memtable,
            ReadCallback* callback = nullptr, bool* is_blob_index = nullptr,
-           bool do_merge = true) override;
+           bool do_merge = true,
+           const BlobFetcher* blob_fetcher = nullptr) override;
 
   void MultiGet(const ReadOptions& read_options, MultiGetRange* range,
-                ReadCallback* callback, bool immutable_memtable) override;
+                ReadCallback* callback, bool immutable_memtable,
+                const BlobFetcher* blob_fetcher = nullptr) override;
 
   // If `key` exists in current memtable with type value_type and the existing
   // value is at least as large as the new value, updates it in-place. Otherwise
@@ -991,7 +996,8 @@ class MemTable final : public ReadOnlyMemTable {
                     std::string* value, PinnableWideColumns* columns,
                     std::string* timestamp, Status* s,
                     MergeContext* merge_context, SequenceNumber* seq,
-                    bool* found_final_value, bool* merge_in_progress);
+                    bool* found_final_value, bool* merge_in_progress,
+                    const BlobFetcher* blob_fetcher);
 
   // Always returns non-null and assumes certain pre-checks (e.g.,
   // is_range_del_table_empty_) are done. This is only valid during the lifetime

db/memtable_list.cc

@@ -108,18 +108,19 @@ bool MemTableListVersion::Get(const LookupKey& key, std::string* value,
                               MergeContext* merge_context,
                               SequenceNumber* max_covering_tombstone_seq,
                               SequenceNumber* seq, const ReadOptions& read_opts,
-                              ReadCallback* callback, bool* is_blob_index) {
+                              ReadCallback* callback, bool* is_blob_index,
+                              const BlobFetcher* blob_fetcher) {
   return GetFromList(&memlist_, key, value, columns, timestamp, s,
                      merge_context, max_covering_tombstone_seq, seq, read_opts,
-                     callback, is_blob_index);
+                     callback, is_blob_index, blob_fetcher);
 }
 
 void MemTableListVersion::MultiGet(const ReadOptions& read_options,
-                                   MultiGetRange* range,
-                                   ReadCallback* callback) {
+                                   MultiGetRange* range, ReadCallback* callback,
+                                   const BlobFetcher* blob_fetcher) {
   for (auto memtable : memlist_) {
     memtable->MultiGet(read_options, range, callback,
-                       true /* immutable_memtable */);
+                       true /* immutable_memtable */, blob_fetcher);
     if (range->empty()) {
       return;
     }
@@ -128,12 +129,13 @@ void MemTableListVersion::MultiGet(const ReadOptions& read_options,
 
 bool MemTableListVersion::GetMergeOperands(
     const LookupKey& key, Status* s, MergeContext* merge_context,
-    SequenceNumber* max_covering_tombstone_seq, const ReadOptions& read_opts) {
+    SequenceNumber* max_covering_tombstone_seq, const ReadOptions& read_opts,
+    const BlobFetcher* blob_fetcher) {
   for (ReadOnlyMemTable* memtable : memlist_) {
     bool done = memtable->Get(
         key, /*value=*/nullptr, /*columns=*/nullptr, /*timestamp=*/nullptr, s,
         merge_context, max_covering_tombstone_seq, read_opts,
-        true /* immutable_memtable */, nullptr, nullptr, false);
+        true /* immutable_memtable */, nullptr, nullptr, false, blob_fetcher);
     if (done) {
       return true;
     }
@@ -145,10 +147,11 @@ bool MemTableListVersion::GetFromHistory(
     const LookupKey& key, std::string* value, PinnableWideColumns* columns,
     std::string* timestamp, Status* s, MergeContext* merge_context,
     SequenceNumber* max_covering_tombstone_seq, SequenceNumber* seq,
-    const ReadOptions& read_opts, bool* is_blob_index) {
+    const ReadOptions& read_opts, bool* is_blob_index,
+    const BlobFetcher* blob_fetcher) {
   return GetFromList(&memlist_history_, key, value, columns, timestamp, s,
                      merge_context, max_covering_tombstone_seq, seq, read_opts,
-                     nullptr /*read_callback*/, is_blob_index);
+                     nullptr /*read_callback*/, is_blob_index, blob_fetcher);
 }
 
 bool MemTableListVersion::GetFromList(
@@ -156,17 +159,18 @@ bool MemTableListVersion::GetFromList(
     std::string* value, PinnableWideColumns* columns, std::string* timestamp,
     Status* s, MergeContext* merge_context,
     SequenceNumber* max_covering_tombstone_seq, SequenceNumber* seq,
-    const ReadOptions& read_opts, ReadCallback* callback, bool* is_blob_index) {
+    const ReadOptions& read_opts, ReadCallback* callback, bool* is_blob_index,
+    const BlobFetcher* blob_fetcher) {
   *seq = kMaxSequenceNumber;
 
   for (auto& memtable : *list) {
     assert(memtable->IsFragmentedRangeTombstonesConstructed());
     SequenceNumber current_seq = kMaxSequenceNumber;
 
-    bool done =
-        memtable->Get(key, value, columns, timestamp, s, merge_context,
-                      max_covering_tombstone_seq, &current_seq, read_opts,
-                      true /* immutable_memtable */, callback, is_blob_index);
+    bool done = memtable->Get(key, value, columns, timestamp, s, merge_context,
+                              max_covering_tombstone_seq, &current_seq,
+                              read_opts, true /* immutable_memtable */,
+                              callback, is_blob_index, true, blob_fetcher);
     if (*seq == kMaxSequenceNumber) {
       // Store the most recent sequence number of any operation on this key.
       // Since we only care about the most recent change, we only need to

db/memtable_list.h

@@ -61,29 +61,33 @@ class MemTableListVersion {
            MergeContext* merge_context,
            SequenceNumber* max_covering_tombstone_seq, SequenceNumber* seq,
            const ReadOptions& read_opts, ReadCallback* callback = nullptr,
-           bool* is_blob_index = nullptr);
+           bool* is_blob_index = nullptr,
+           const BlobFetcher* blob_fetcher = nullptr);
 
   bool Get(const LookupKey& key, std::string* value,
            PinnableWideColumns* columns, std::string* timestamp, Status* s,
            MergeContext* merge_context,
            SequenceNumber* max_covering_tombstone_seq,
            const ReadOptions& read_opts, ReadCallback* callback = nullptr,
-           bool* is_blob_index = nullptr) {
+           bool* is_blob_index = nullptr,
+           const BlobFetcher* blob_fetcher = nullptr) {
     SequenceNumber seq;
     return Get(key, value, columns, timestamp, s, merge_context,
                max_covering_tombstone_seq, &seq, read_opts, callback,
-               is_blob_index);
+               is_blob_index, blob_fetcher);
   }
 
   void MultiGet(const ReadOptions& read_options, MultiGetRange* range,
-                ReadCallback* callback);
+                ReadCallback* callback,
+                const BlobFetcher* blob_fetcher = nullptr);
 
   // Returns all the merge operands corresponding to the key by searching all
   // memtables starting from the most recent one.
   bool GetMergeOperands(const LookupKey& key, Status* s,
                         MergeContext* merge_context,
                         SequenceNumber* max_covering_tombstone_seq,
-                        const ReadOptions& read_opts);
+                        const ReadOptions& read_opts,
+                        const BlobFetcher* blob_fetcher = nullptr);
 
   // Similar to Get(), but searches the Memtable history of memtables that
   // have already been flushed.  Should only be used from in-memory only
@@ -94,17 +98,19 @@ class MemTableListVersion {
                       Status* s, MergeContext* merge_context,
                       SequenceNumber* max_covering_tombstone_seq,
                       SequenceNumber* seq, const ReadOptions& read_opts,
-                      bool* is_blob_index = nullptr);
+                      bool* is_blob_index = nullptr,
+                      const BlobFetcher* blob_fetcher = nullptr);
   bool GetFromHistory(const LookupKey& key, std::string* value,
                       PinnableWideColumns* columns, std::string* timestamp,
                       Status* s, MergeContext* merge_context,
                       SequenceNumber* max_covering_tombstone_seq,
                       const ReadOptions& read_opts,
-                      bool* is_blob_index = nullptr) {
+                      bool* is_blob_index = nullptr,
+                      const BlobFetcher* blob_fetcher = nullptr) {
     SequenceNumber seq;
     return GetFromHistory(key, value, columns, timestamp, s, merge_context,
                           max_covering_tombstone_seq, &seq, read_opts,
-                          is_blob_index);
+                          is_blob_index, blob_fetcher);
   }
 
   Status AddRangeTombstoneIterators(const ReadOptions& read_opts, Arena* arena,
@@ -188,7 +194,8 @@ class MemTableListVersion {
                    SequenceNumber* max_covering_tombstone_seq,
                    SequenceNumber* seq, const ReadOptions& read_opts,
                    ReadCallback* callback = nullptr,
-                   bool* is_blob_index = nullptr);
+                   bool* is_blob_index = nullptr,
+                   const BlobFetcher* blob_fetcher = nullptr);
 
   void AddMemTable(ReadOnlyMemTable* m);
 

db/obsolete_files_test.cc

@@ -52,7 +52,8 @@ void WriteFooterlessBlobFile(const ImmutableOptions& immutable_options,
   constexpr bool use_fsync = false;
   constexpr bool do_flush = false;
   BlobLogWriter blob_log_writer(std::move(file_writer), immutable_options.clock,
-                                statistics, use_fsync, do_flush);
+                                statistics, blob_file_number, use_fsync,
+                                do_flush);
 
   constexpr bool has_ttl = false;
   constexpr ExpirationRange expiration_range;

db/version_edit.cc

@@ -30,7 +30,7 @@ PinnedTableReader& PinnedTableReader::operator=(
   TableReader* r = other.reader_.load(std::memory_order_acquire);
   // Only read handle_ when reader_ is non-null. Pin() writes handle_ before
   // reader_ (with release), so a non-null reader_ guarantees handle_ is stable.
-  // If reader_ is null, Pin() may be in progress — avoid reading handle_.
+  // If reader_ is null, Pin() may be in progress -- avoid reading handle_.
   handle_ = (r != nullptr) ? other.handle_ : nullptr;
   reader_.store(r, std::memory_order_release);
   return *this;
@@ -109,6 +109,17 @@ Status FileMetaData::UpdateBoundaries(const Slice& key, const Slice& value,
 
 void VersionEdit::Clear() { *this = VersionEdit(); }
 
+bool VersionEdit::ShouldEmitPerColumnFamilyRecoveryEdit(
+    uint64_t current_log_number) const {
+  return (HasLogNumber() && GetLogNumber() > current_log_number) ||
+         NumEntries() > 0 || HasComparatorName() || HasPrevLogNumber() ||
+         HasNextFile() || HasMaxColumnFamily() || HasMinLogNumberToKeep() ||
+         HasLastSequence() || !GetCompactCursors().empty() || HasDbId() ||
+         IsColumnFamilyManipulation() || IsInAtomicGroup() ||
+         HasFullHistoryTsLow() || HasPersistUserDefinedTimestamps() ||
+         HasSubcompactionProgress();
+}
+
 bool VersionEdit::EncodeTo(std::string* dst,
                            std::optional<size_t> ts_sz) const {
   assert(!IsNoManifestWriteDummy());
@@ -229,6 +240,13 @@ bool VersionEdit::EncodeTo(std::string* dst,
     PutLengthPrefixedSlice(dst, progress_data);
   }
 
+  if (has_last_compacted_manifest_file_size_) {
+    PutVarint32(dst, kLastCompactedManifestFileSize);
+    std::string varint_size;
+    PutVarint64(&varint_size, last_compacted_manifest_file_size_);
+    PutLengthPrefixedSlice(dst, Slice(varint_size));
+  }
+
   return true;
 }
 
@@ -879,6 +897,17 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
         break;
       }
 
+      case kLastCompactedManifestFileSize: {
+        Slice encoded;
+        if (GetLengthPrefixedSlice(&input, &encoded) &&
+            GetVarint64(&encoded, &last_compacted_manifest_file_size_)) {
+          has_last_compacted_manifest_file_size_ = true;
+        } else {
+          msg = "last compacted manifest file size";
+        }
+        break;
+      }
+
       default:
         if (tag & kTagSafeIgnoreMask) {
           // Tag from future which can be safely ignored.
@@ -1053,6 +1082,10 @@ std::string VersionEdit::DebugString(bool hex_key) const {
     r.append("\n SubcompactionProgress: ");
     r.append(subcompaction_progress_.ToString());
   }
+  if (has_last_compacted_manifest_file_size_) {
+    r.append("\n  LastCompactedManifestFileSize: ");
+    AppendNumberTo(&r, last_compacted_manifest_file_size_);
+  }
   r.append("\n}\n");
   return r;
 }
@@ -1209,6 +1242,10 @@ std::string VersionEdit::DebugJSON(int edit_num, bool hex_key) const {
     jw << "SubcompactionProgress" << subcompaction_progress_.ToString();
   }
 
+  if (has_last_compacted_manifest_file_size_) {
+    jw << "LastCompactedManifestFileSize" << last_compacted_manifest_file_size_;
+  }
+
   jw.EndObject();
 
   return jw.Get();

db/version_edit.h

@@ -74,6 +74,7 @@ enum Tag : uint32_t {
   kWalDeletion2,
   kPersistUserDefinedTimestamps,
   kSubcompactionProgress,
+  kLastCompactedManifestFileSize,
 };
 
 enum SubcompactionProgressPerLevelCustomTag : uint32_t {
@@ -977,6 +978,11 @@ class VersionEdit {
 
   bool IsColumnFamilyDrop() const { return is_column_family_drop_; }
 
+  void MarkForegroundOperation() { is_foreground_operation_ = true; }
+  bool IsForegroundOperation() const {
+    return is_foreground_operation_ || IsColumnFamilyManipulation();
+  }
+
   void MarkNoManifestWriteDummy() { is_no_manifest_write_dummy_ = true; }
   bool IsNoManifestWriteDummy() const { return is_no_manifest_write_dummy_; }
 
@@ -1016,6 +1022,21 @@ class VersionEdit {
     subcompaction_progress_.Clear();
   }
 
+  void SetLastCompactedManifestFileSize(uint64_t size) {
+    has_last_compacted_manifest_file_size_ = true;
+    last_compacted_manifest_file_size_ = size;
+  }
+  bool HasLastCompactedManifestFileSize() const {
+    return has_last_compacted_manifest_file_size_;
+  }
+  uint64_t GetLastCompactedManifestFileSize() const {
+    return last_compacted_manifest_file_size_;
+  }
+
+  // Recovery-time per-column-family edits only need to be written when they
+  // advance the CF's log number or carry some other manifest state.
+  bool ShouldEmitPerColumnFamilyRecoveryEdit(uint64_t current_log_number) const;
+
   // return true on success.
   // `ts_sz` is the size in bytes for the user-defined timestamp contained in
   // a user key. This argument is optional because it's only required for
@@ -1095,6 +1116,7 @@ class VersionEdit {
   // it also includes column family name.
   bool is_column_family_drop_ = false;
   bool is_column_family_add_ = false;
+  bool is_foreground_operation_ = false;
   std::string column_family_name_;
 
   uint32_t remaining_entries_ = 0;
@@ -1107,6 +1129,9 @@ class VersionEdit {
   bool has_subcompaction_progress_ = false;
   SubcompactionProgress subcompaction_progress_;
 
+  bool has_last_compacted_manifest_file_size_ = false;
+  uint64_t last_compacted_manifest_file_size_ = 0;
+
   // Newly created table files and blob files are eligible for deletion if they
   // are not registered as live files after the background jobs creating them
   // have finished. In case committing the VersionEdit containing such changes

db/version_edit_handler.cc

@@ -35,6 +35,7 @@ void VersionEditHandlerBase::Iterate(log::Reader& reader,
     VersionEdit edit;
     s = edit.DecodeFrom(record);
     if (s.ok()) {
+      last_valid_record_end_ = reader.LastRecordEnd();
       s = read_buffer_.AddEdit(&edit);
     }
     if (s.ok()) {
@@ -460,6 +461,7 @@ void VersionEditHandler::CheckIterationResult(const log::Reader& reader,
   if (s->ok()) {
     version_set_->manifest_file_size_ = reader.GetReadOffset();
     assert(version_set_->manifest_file_size_ > 0);
+    version_set_->manifest_last_valid_record_end_ = last_valid_record_end_;
     version_set_->next_file_number_.store(version_edit_params_.GetNextFile() +
                                           1);
     SequenceNumber last_seq = version_edit_params_.GetLastSequence();
@@ -640,6 +642,11 @@ Status VersionEditHandler::ExtractInfoFromVersionEdit(ColumnFamilyData* cfd,
              version_edit_params_.GetLastSequence() <= edit.GetLastSequence());
       version_edit_params_.SetLastSequence(edit.GetLastSequence());
     }
+    if (edit.HasLastCompactedManifestFileSize()) {
+      version_set_->last_compacted_manifest_file_size_ =
+          edit.GetLastCompactedManifestFileSize();
+      version_set_->TuneMaxManifestFileSize();
+    }
     if (!version_edit_params_.HasPrevLogNumber()) {
       version_edit_params_.SetPrevLogNumber(0);
     }

db/version_edit_handler.h

@@ -31,6 +31,8 @@ class VersionEditHandlerBase {
 
   AtomicGroupReadBuffer& GetReadBuffer() { return read_buffer_; }
 
+  uint64_t GetLastValidRecordEnd() const { return last_valid_record_end_; }
+
  protected:
   explicit VersionEditHandlerBase(const ReadOptions& read_options,
                                   uint64_t max_read_size)
@@ -52,6 +54,10 @@ class VersionEditHandlerBase {
 
   const ReadOptions& read_options_;
 
+  // File offset at the end of the last successfully completed and
+  // decoded logical record.
+  uint64_t last_valid_record_end_ = 0;
+
  private:
   AtomicGroupReadBuffer read_buffer_;
   const uint64_t max_manifest_read_size_;

db/version_set.cc

@@ -2243,8 +2243,10 @@ void Version::GetCreationTimeOfOldestFile(uint64_t* creation_time) {
   uint64_t oldest_time = std::numeric_limits<uint64_t>::max();
   for (int level = 0; level < storage_info_.num_non_empty_levels_; level++) {
     for (FileMetaData* meta : storage_info_.LevelFiles(level)) {
-      assert(meta->fd.pinned_reader.Get() != nullptr);
       uint64_t file_creation_time = meta->TryGetFileCreationTime();
+      TEST_SYNC_POINT_CALLBACK(
+          "Version::GetCreationTimeOfOldestFile::FileCreationTime",
+          &file_creation_time);
       if (file_creation_time == kUnknownFileCreationTime) {
         *creation_time = 0;
         return;
@@ -4217,10 +4219,10 @@ void VersionStorageInfo::ComputeFilesMarkedForReadTriggeredCompaction(
     return;
   }
 
-  // Skip files at the last non-empty level — there is no lower level to
+  // Skip files at the last non-empty level -- there is no lower level to
   // compact into. Exception: L0 files are allowed even when L0 is the last
   // non-empty level, because in single-level universal or FIFO compaction, L0
-  // files can be compacted together (L0 → L0).
+  // files can be compacted together (L0 -> L0).
   int last_non_empty = num_non_empty_levels_ - 1;
 
   for (int level = 0; level < num_levels(); level++) {
@@ -5618,6 +5620,7 @@ VersionSet::VersionSet(
       prev_log_number_(0),
       current_version_number_(0),
       manifest_file_size_(0),
+      manifest_last_valid_record_end_(0),
       last_compacted_manifest_file_size_(0),
       file_options_(storage_options),
       block_cache_tracer_(block_cache_tracer),
@@ -5689,7 +5692,7 @@ Status VersionSet::Close(FSDirectory* db_dir, InstrumentedMutex* mu) {
           content_manifest_name, fs_->OptimizeForManifestRead(file_options_),
           &manifest_file, nullptr);
       if (!content_io_s.ok()) {
-        // Surface I/O errors to the caller — users who call DB::Close() and
+        // Surface I/O errors to the caller -- users who call DB::Close() and
         // check the status should know about filesystem problems.
         s = content_io_s;
         ROCKS_LOG_ERROR(db_options_->info_log,
@@ -5741,7 +5744,7 @@ Status VersionSet::Close(FSDirectory* db_dir, InstrumentedMutex* mu) {
       corrupt_io_s.PermitUncheckedError();
       io_error_info.io_status.PermitUncheckedError();
       if (content_check == 0) {
-        // First check failed — rewrite and verify again
+        // First check failed -- rewrite and verify again
         ROCKS_LOG_ERROR(db_options_->info_log,
                         "MANIFEST content verification on Close failed, "
                         "filename %s, rewriting manifest\n",
@@ -5752,7 +5755,7 @@ Status VersionSet::Close(FSDirectory* db_dir, InstrumentedMutex* mu) {
         s = LogAndApply(cfd, ReadOptions(), WriteOptions(), &recovery_edit, mu,
                         db_dir);
       } else {
-        // Rewritten manifest is also corrupt — likely a recurring filesystem
+        // Rewritten manifest is also corrupt -- likely a recurring filesystem
         // issue. Surface it so DB::Close() callers can detect the problem.
         ROCKS_LOG_ERROR(db_options_->info_log,
                         "MANIFEST content verification on Close failed again "
@@ -5827,6 +5830,7 @@ void VersionSet::Reset() {
   current_version_number_ = 0;
   manifest_writers_.clear();
   manifest_file_size_ = 0;
+  manifest_last_valid_record_end_ = 0;
   last_compacted_manifest_file_size_ = 0;
   TuneMaxManifestFileSize();
   obsolete_files_.clear();
@@ -6107,9 +6111,32 @@ Status VersionSet::ProcessManifestWrites(
 
   uint64_t prev_manifest_file_size = manifest_file_size_;
   assert(pending_manifest_file_number_ == 0);
+  bool has_foreground_operation = false;
+  for (const VersionEdit* e : batch_edits) {
+    if (e->IsForegroundOperation()) {
+      has_foreground_operation = true;
+      break;
+    }
+  }
+  // For MANIFEST write batches with any foreground operation (external file
+  // ingestion/import, DeleteFilesInRange(s), and column family manipulations
+  // like CreateColumnFamily and DropColumnFamily), relax the size limit by 25%
+  // to reduce the likelihood of a user operation blocking on MANIFEST rotation.
+  // Background-only batches (flush/compaction) still rotate at the normal
+  // threshold.
+  // TODO/future: for workloads like atomic-replace ingestion-only, with zero
+  // or few flushes and compactions, it might be nice to trigger background
+  // manifest rotation if we are beyond the soft limit. But the vast majority
+  // of workloads should have plenty of background manifest ops to avoid
+  // foreground rotation.
+  uint64_t enforced_limit = tuned_max_manifest_file_size_;
+  if (has_foreground_operation) {
+    uint64_t new_limit = enforced_limit + enforced_limit / 4;
+    // don't keep in case of overflow
+    enforced_limit = std::max(enforced_limit, new_limit);
+  }
   if (!skip_manifest_write &&
-      (!descriptor_log_ ||
-       prev_manifest_file_size >= tuned_max_manifest_file_size_)) {
+      (!descriptor_log_ || prev_manifest_file_size >= enforced_limit)) {
     TEST_SYNC_POINT("VersionSet::ProcessManifestWrites:BeforeNewManifest");
     new_descriptor_log = true;
   } else {
@@ -6161,11 +6188,7 @@ Status VersionSet::ProcessManifestWrites(
       }
     }
   } else {
-    FileOptions opt_file_opts = fs_->OptimizeForManifestWrite(file_options_);
-    // DB option (in file_options_) takes precedence when not kUnknown
-    if (file_options_.temperature != Temperature::kUnknown) {
-      opt_file_opts.temperature = file_options_.temperature;
-    }
+    FileOptions opt_file_opts = GetFileOptionsForManifestWrite();
     mu->Unlock();
     TEST_SYNC_POINT("VersionSet::LogAndApply:WriteManifestStart");
     TEST_SYNC_POINT_CALLBACK("VersionSet::LogAndApply:WriteManifest", nullptr);
@@ -6200,16 +6223,9 @@ Status VersionSet::ProcessManifestWrites(
       io_s = NewWritableFile(fs_.get(), descriptor_fname, &descriptor_file,
                              opt_file_opts);
       if (io_s.ok()) {
-        descriptor_file->SetPreallocationBlockSize(manifest_preallocation_size);
-        FileTypeSet tmp_set = db_options_->checksum_handoff_file_types;
-        std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
-            std::move(descriptor_file), descriptor_fname, opt_file_opts, clock_,
-            io_tracer_, nullptr, Histograms::HISTOGRAM_ENUM_MAX /* hist_type */,
-            db_options_->listeners, nullptr,
-            tmp_set.Contains(FileType::kDescriptorFile),
-            tmp_set.Contains(FileType::kDescriptorFile)));
-        new_desc_log_ptr.reset(
-            new log::Writer(std::move(file_writer), 0, false));
+        new_desc_log_ptr =
+            CreateManifestWriter(std::move(descriptor_file), descriptor_fname,
+                                 opt_file_opts, manifest_preallocation_size);
         raw_desc_log_ptr = new_desc_log_ptr.get();
         s = WriteCurrentStateToManifest(write_options, curr_state,
                                         wal_additions, raw_desc_log_ptr, io_s);
@@ -6259,6 +6275,7 @@ Status VersionSet::ProcessManifestWrites(
         }
         ++idx;
 #endif /* !NDEBUG */
+        TEST_SYNC_POINT("VersionSet::ProcessManifestWrites:AddRecord");
         io_s = raw_desc_log_ptr->AddRecord(write_options, record);
         if (!io_s.ok()) {
           s = io_s;
@@ -6290,6 +6307,8 @@ Status VersionSet::ProcessManifestWrites(
       io_s = SetCurrentFile(
           write_options, fs_.get(), dbname_, pending_manifest_file_number_,
           file_options_.temperature, dir_contains_current_file);
+      TEST_SYNC_POINT_CALLBACK(
+          "VersionSet::ProcessManifestWrites:AfterSetCurrentFile", &io_s);
       if (!io_s.ok()) {
         s = io_s;
         // Quarantine old manifest file in case new manifest file's CURRENT
@@ -6680,6 +6699,106 @@ Status VersionSet::LogAndApplyHelper(ColumnFamilyData* cfd,
   return builder ? builder->Apply(edit) : Status::OK();
 }
 
+FileOptions VersionSet::GetFileOptionsForManifestWrite() const {
+  FileOptions opt_file_opts = fs_->OptimizeForManifestWrite(file_options_);
+  // DB-configured temperature wins over the FS's default (when not kUnknown).
+  if (file_options_.temperature != Temperature::kUnknown) {
+    opt_file_opts.temperature = file_options_.temperature;
+  }
+  return opt_file_opts;
+}
+
+std::unique_ptr<log::Writer> VersionSet::CreateManifestWriter(
+    std::unique_ptr<FSWritableFile> file, const std::string& fname,
+    const FileOptions& opts, uint64_t preallocation_size,
+    uint64_t initial_file_size) const {
+  file->SetPreallocationBlockSize(preallocation_size);
+  FileTypeSet tmp_set = db_options_->checksum_handoff_file_types;
+  auto file_writer = std::make_unique<WritableFileWriter>(
+      std::move(file), fname, opts, clock_, io_tracer_,
+      /*stats=*/nullptr, Histograms::HISTOGRAM_ENUM_MAX, db_options_->listeners,
+      /*file_checksum_gen_factory=*/nullptr,
+      tmp_set.Contains(FileType::kDescriptorFile),
+      tmp_set.Contains(FileType::kDescriptorFile), initial_file_size);
+  const size_t block_offset = initial_file_size % log::kBlockSize;
+  return std::make_unique<log::Writer>(
+      std::move(file_writer), /*log_number=*/0, /*recycle_log_files=*/false,
+      /*manual_flush=*/false, kNoCompression,
+      /*track_and_verify_wals=*/false, block_offset);
+}
+
+Status VersionSet::ReopenManifestForAppend(const std::string& manifest_path) {
+  assert(db_options_->reuse_manifest_on_open);
+  assert(manifest_last_valid_record_end_ > 0);
+
+  // Disabled under best_efforts_recovery: that mode rebuilds the
+  // MANIFEST + CURRENT from scratch via the side-effect of
+  // LogAndApplyForRecovery emitting an edit. Reusing the (possibly
+  // stale) prior MANIFEST contradicts the salvage contract.
+  if (db_options_->best_efforts_recovery) {
+    return Status::OK();
+  }
+
+  FileOptions opt_file_opts = GetFileOptionsForManifestWrite();
+
+  // Bail if the on-disk size diverges from what Recover's Reader
+  // consumed -- likely a torn tail from a prior crash; mid-block append
+  // would mis-frame the record stream.
+  uint64_t physical_size = 0;
+  IOStatus stat_s = fs_->GetFileSize(manifest_path, IOOptions(), &physical_size,
+                                     /*dbg=*/nullptr);
+  if (!stat_s.ok() || physical_size != manifest_last_valid_record_end_) {
+    ROCKS_LOG_WARN(db_options_->info_log,
+                   "reuse_manifest_on_open: physical size %" PRIu64
+                   " != last valid record end %" PRIu64
+                   " (tail corruption?); falling back to fresh MANIFEST",
+                   physical_size, manifest_last_valid_record_end_);
+    return Status::OK();
+  }
+
+  // Direct-writes path is hard to reason about for an already-populated
+  // file (alignment + tail-pad). POSIX's OptimizeForManifestWrite forces
+  // it off, but custom FileSystems may not.
+  if (opt_file_opts.use_direct_writes) {
+    ROCKS_LOG_WARN(db_options_->info_log,
+                   "reuse_manifest_on_open: direct writes enabled; "
+                   "falling back to fresh MANIFEST");
+    return Status::OK();
+  }
+
+  std::unique_ptr<FSWritableFile> manifest_file;
+  IOStatus io_s = fs_->ReopenWritableFile(manifest_path, opt_file_opts,
+                                          &manifest_file, /*dbg=*/nullptr);
+  if (!io_s.ok()) {
+    ROCKS_LOG_WARN(db_options_->info_log,
+                   "Failed to reopen MANIFEST for append: %s",
+                   io_s.ToString().c_str());
+    return Status::OK();
+  }
+
+  const uint64_t reopened_size =
+      manifest_file->GetFileSize(opt_file_opts.io_options, /*dbg=*/nullptr);
+  if (reopened_size != manifest_last_valid_record_end_) {
+    ROCKS_LOG_WARN(db_options_->info_log,
+                   "reuse_manifest_on_open: reopened handle size %" PRIu64
+                   " != last valid record end %" PRIu64
+                   "; falling back to fresh MANIFEST",
+                   reopened_size, manifest_last_valid_record_end_);
+    return Status::OK();
+  }
+
+  descriptor_log_ = CreateManifestWriter(
+      std::move(manifest_file), manifest_path, opt_file_opts,
+      manifest_preallocation_size_, reopened_size);
+
+  ROCKS_LOG_INFO(db_options_->info_log,
+                 "Reusing existing MANIFEST file: %s (valid data size: %" PRIu64
+                 ")",
+                 manifest_path.c_str(), manifest_last_valid_record_end_);
+  TEST_SYNC_POINT("VersionSet::ReopenManifestForAppend:Reopened");
+  return Status::OK();
+}
+
 Status VersionSet::Recover(
     const std::vector<ColumnFamilyDescriptor>& column_families, bool read_only,
     std::string* db_id, bool no_error_if_files_missing, bool is_retry,
@@ -6767,6 +6886,11 @@ Status VersionSet::Recover(
     }
   }
 
+  if (s.ok() && !read_only && db_options_->reuse_manifest_on_open &&
+      manifest_last_valid_record_end_ > 0) {
+    s = ReopenManifestForAppend(manifest_path);
+  }
+
   return s;
 }
 
@@ -7397,6 +7521,28 @@ Status VersionSet::WriteCurrentStateToManifest(
       }
     }
   }
+
+  // Record the approximate compacted manifest size so it's available at
+  // recovery time for TuneMaxManifestFileSize(). This record must come last
+  // in WriteCurrentStateToManifest for an accurate size estimate.
+  {
+    // Include a rough estimate of this record's own size (~15 bytes for the
+    // VersionEdit payload + log record header).
+    constexpr uint64_t kEstimatedRecordOverhead = 15;
+    VersionEdit edit;
+    edit.SetLastCompactedManifestFileSize(log->file()->GetFileSize() +
+                                          kEstimatedRecordOverhead);
+    std::string record;
+    if (!edit.EncodeTo(&record)) {
+      return Status::Corruption("Unable to Encode VersionEdit:" +
+                                edit.DebugString(true));
+    }
+    io_s = log->AddRecord(write_options, record);
+    if (!io_s.ok()) {
+      return io_s;
+    }
+  }
+
   return Status::OK();
 }
 
@@ -8064,11 +8210,12 @@ ReactiveVersionSet::ReactiveVersionSet(
     const MutableDBOptions& mutable_db_options,
     const FileOptions& _file_options, Cache* table_cache,
     WriteBufferManager* write_buffer_manager, WriteController* write_controller,
-    const std::shared_ptr<IOTracer>& io_tracer)
+    const std::shared_ptr<IOTracer>& io_tracer, const std::string& db_id,
+    const std::string& db_session_id)
     : VersionSet(dbname, imm_db_options, mutable_db_options, _file_options,
                  table_cache, write_buffer_manager, write_controller,
-                 /*block_cache_tracer=*/nullptr, io_tracer, /*db_id*/ "",
-                 /*db_session_id*/ "", /*daily_offpeak_time_utc*/ "",
+                 /*block_cache_tracer=*/nullptr, io_tracer, db_id,
+                 db_session_id, /*daily_offpeak_time_utc*/ "",
                  /*error_handler=*/nullptr, /*unchanging=*/false) {}
 
 ReactiveVersionSet::~ReactiveVersionSet() = default;

db/version_set.h

@@ -1719,6 +1719,33 @@ class VersionSet {
       const std::unordered_map<uint32_t, MutableCFState>& curr_state,
       const VersionEdit& wal_additions, log::Writer* log, IOStatus& io_s);
 
+  // Reopen the existing MANIFEST file for append at the end of Recover()
+  // when reuse_manifest_on_open is set, so the next LogAndApply appends
+  // to it instead of creating a fresh MANIFEST. Falls back to OK
+  // (descriptor_log_ stays null, next LogAndApply creates a fresh
+  // MANIFEST as before) if ReopenWritableFile fails.
+  Status ReopenManifestForAppend(const std::string& manifest_path);
+
+  // FileOptions for MANIFEST writes -- applies the FS's
+  // OptimizeForManifestWrite tuning, then re-applies the user-configured
+  // temperature so a custom FS can't override it.
+  FileOptions GetFileOptionsForManifestWrite() const;
+
+  // Create a log::Writer over the given FSWritableFile with all standard
+  // MANIFEST setup applied (preallocation block size, checksum-handoff
+  // classification, listeners, etc.). preallocation_size should be a
+  // snapshot of manifest_preallocation_size_ taken under the DB mutex
+  // (the fresh-path call site reads the field before releasing mu).
+  // When initial_file_size > 0 the writer is treated as resuming over
+  // an already-populated file: WritableFileWriter is constructed with the
+  // existing size so size accounting is correct, and the log writer's
+  // initial_block_offset aligns to the existing file's tail within the
+  // current 32 KiB block.
+  std::unique_ptr<log::Writer> CreateManifestWriter(
+      std::unique_ptr<FSWritableFile> file, const std::string& fname,
+      const FileOptions& opts, uint64_t preallocation_size,
+      uint64_t initial_file_size = 0) const;
+
   void AppendVersion(ColumnFamilyData* column_family_data, Version* v);
 
   ColumnFamilyData* CreateColumnFamily(const ColumnFamilyOptions& cf_options,
@@ -1788,6 +1815,17 @@ class VersionSet {
   // Current size of manifest file
   uint64_t manifest_file_size_;
 
+  // File offset at the end of the last successfully completed logical
+  // record during MANIFEST recovery. Unlike manifest_file_size_ (the
+  // reader's I/O high-water mark, which includes any tolerated tail
+  // garbage), this value points to the byte after the last valid record.
+  // Used by ReopenManifestForAppend to detect intra-block tail
+  // corruption that doesn't extend the physical file size.
+  // manifest_file_size_ is kept separate because it is used for
+  // rotation decisions (ProcessManifestWrites), close-time verification
+  // (Close), and backup metadata.
+  uint64_t manifest_last_valid_record_end_;
+
   // Size of the populated manifest file last time it was re-written from
   // scratch.
   uint64_t last_compacted_manifest_file_size_;
@@ -1860,7 +1898,9 @@ class ReactiveVersionSet : public VersionSet {
                      const FileOptions& _file_options, Cache* table_cache,
                      WriteBufferManager* write_buffer_manager,
                      WriteController* write_controller,
-                     const std::shared_ptr<IOTracer>& io_tracer);
+                     const std::shared_ptr<IOTracer>& io_tracer,
+                     const std::string& db_id,
+                     const std::string& db_session_id);
 
   ~ReactiveVersionSet() override;
 

db/version_set_test.cc

@@ -1203,7 +1203,7 @@ class VersionSetTestBase {
     reactive_versions_ = std::make_shared<ReactiveVersionSet>(
         dbname_, &imm_db_options_, mutable_db_options_, env_options_,
         table_cache_.get(), &write_buffer_manager_, &write_controller_,
-        nullptr);
+        /*io_tracer=*/nullptr, /*db_id=*/"", /*db_session_id=*/"");
     imm_db_options_.db_paths.emplace_back(dbname_,
                                           std::numeric_limits<uint64_t>::max());
   }
@@ -2440,7 +2440,7 @@ TEST_F(VersionSetTest, ManifestContentValidationOnCloseClean) {
   // Verify content validation actually ran
   ASSERT_TRUE(content_validation_ran);
 
-  // No corruption — counter should be zero
+  // No corruption -- counter should be zero
   ASSERT_EQ(0, stats->getTickerCount(MANIFEST_VALIDATION_FAILURE_COUNT));
 
   // Manifest path should be unchanged (no rotation)
@@ -2513,7 +2513,7 @@ TEST_F(VersionSetTest, ManifestContentValidationOnCloseCorruptRecord) {
 
 TEST_F(VersionSetTest, ManifestContentValidationOnCloseDisabled) {
   // Default (option disabled), corrupt manifest after writer close,
-  // verify no rotation occurred — corrupt manifest persists.
+  // verify no rotation occurred -- corrupt manifest persists.
   NewDB();
   auto stats = CreateDBStatistics();
   imm_db_options_.statistics = stats;
@@ -2538,7 +2538,7 @@ TEST_F(VersionSetTest, ManifestContentValidationOnCloseDisabled) {
 
   ASSERT_FALSE(content_validation_ran);
 
-  // Validation disabled — counter should be zero
+  // Validation disabled -- counter should be zero
   ASSERT_EQ(0, stats->getTickerCount(MANIFEST_VALIDATION_FAILURE_COUNT));
 
   // Manifest path should be unchanged (no rotation since validation is off)
@@ -2585,7 +2585,7 @@ TEST_F(VersionSetTest, ManifestContentValidationOnCloseSizeCheckFails) {
   SyncPoint::GetInstance()->DisableProcessing();
 
   // Size check caught the issue; content validation on rewritten manifest
-  // should pass — no content validation failure recorded
+  // should pass -- no content validation failure recorded
   ASSERT_EQ(0, stats->getTickerCount(MANIFEST_VALIDATION_FAILURE_COUNT));
 
   // Size check should have triggered rotation
@@ -2697,7 +2697,7 @@ TEST_F(VersionSetTest, ManifestContentValidationOnCloseOpenFails) {
 
   ASSERT_TRUE(close_s.IsIOError()) << close_s.ToString();
 
-  // File couldn't be opened — no content validation ran
+  // File couldn't be opened -- no content validation ran
   ASSERT_EQ(0, stats->getTickerCount(MANIFEST_VALIDATION_FAILURE_COUNT));
 }
 
@@ -4463,7 +4463,7 @@ class BestEffortsRecoverIncompleteVersionTest
         std::move(file), blob_file_path, FileOptions(), options.clock));
 
     BlobLogWriter blob_log_writer(std::move(file_writer), options.clock,
-                                  /*statistics*/ nullptr,
+                                  /*statistics*/ nullptr, blob_file_number,
                                   /*use_fsync*/ true,
                                   /*do_flush*/ false);
 
@@ -4481,8 +4481,8 @@ class BestEffortsRecoverIncompleteVersionTest
     footer.expiration_range = expiration_range;
     std::string checksum_method;
     std::string checksum_value;
-    ASSERT_OK(blob_log_writer.LegacyAppendFooterAndClose(
-        WriteOptions(), footer, &checksum_method, &checksum_value));
+    ASSERT_OK(blob_log_writer.AppendFooter(WriteOptions(), footer,
+                                           &checksum_method, &checksum_value));
   }
 
   void RecoverFromManifestWithMissingFiles(

db/wide/db_wide_basic_test.cc

@@ -30,6 +30,10 @@ class DBWideBasicTest : public DBTestBase {
     return wide_column_test_util::GetOptionsForBlobTest(GetDefaultOptions());
   }
 
+  Options GetDirectWriteOptions() {
+    return wide_column_test_util::GetDirectWriteOptions(GetDefaultOptions());
+  }
+
   // Helper: runs the EntityBlobAfterFlush test logic with the given options.
   void RunEntityBlobAfterFlush(const Options& options);
 
@@ -2159,6 +2163,57 @@ TEST_F(DBWideBasicTest, MultiGetEntityWithBlobResolution) {
   }
 }
 
+TEST_F(DBWideBasicTest,
+       MultiGetBlobBackedEntityDirectWriteMemtableBatchLookup) {
+  // Goal: force the memtable batch MultiGet optimization to read a direct-write
+  // blob-backed entity from the active memtable. One key exercises merge
+  // resolution against a blob-backed entity base, which requires Saver to carry
+  // the BlobFetcher through the batched callback path. The second key proves
+  // the non-merge default-column path still resolves correctly in the same
+  // batched request.
+  Options options = GetDirectWriteOptions();
+  options.min_blob_size = 50;
+  options.memtable_batch_lookup_optimization = true;
+  options.merge_operator = MergeOperators::CreateStringAppendOperator("|");
+
+  DestroyAndReopen(options);
+
+  const std::string merge_key = "multiget_direct_write_merge_key";
+  const std::string plain_key = "multiget_direct_write_plain_key";
+  const std::string default_value = GenerateLargeValue(100, 'd');
+  const std::string other_default_value = GenerateLargeValue(110, 'p');
+  const std::string large_value = GenerateLargeValue(120, 'l');
+  const std::string small_value = GenerateSmallValue();
+  const std::string merge_operand = "tail";
+
+  WideColumns merge_columns{{kDefaultWideColumnName, default_value},
+                            {"col_large", large_value},
+                            {"col_small", small_value}};
+  WideColumns plain_columns{{kDefaultWideColumnName, other_default_value},
+                            {"col_large", large_value},
+                            {"col_small", small_value}};
+
+  ASSERT_OK(db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(),
+                           merge_key, merge_columns));
+  ASSERT_OK(db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(),
+                           plain_key, plain_columns));
+  ASSERT_OK(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), merge_key,
+                       merge_operand));
+
+  std::array<Slice, 2> keys{{merge_key, plain_key}};
+  std::array<PinnableSlice, 2> values;
+  std::array<Status, 2> statuses;
+
+  db_->MultiGet(ReadOptions(), db_->DefaultColumnFamily(), keys.size(),
+                keys.data(), values.data(), statuses.data());
+
+  ASSERT_OK(statuses[0]);
+  ASSERT_EQ(values[0], default_value + "|" + merge_operand);
+
+  ASSERT_OK(statuses[1]);
+  ASSERT_EQ(values[1], other_default_value);
+}
+
 void DBWideBasicTest::RunEntityBlobAfterFlush(const Options& options) {
   Reopen(options);
 
@@ -2776,7 +2831,7 @@ TEST_F(DBWideBasicTest, EntityBlobBlockCacheTierGet) {
   // Reopen to clear any caches
   Reopen(options);
 
-  // Read with kBlockCacheTier — blob is not in cache
+  // Read with kBlockCacheTier -- blob is not in cache
   ReadOptions read_opts;
   read_opts.read_tier = kBlockCacheTier;
   PinnableSlice result;
@@ -2909,6 +2964,347 @@ TEST_F(DBWideBasicTest, MergeEntityWithBlobColumns) {
   }
 }
 
+TEST_F(DBWideBasicTest, GetMergeOperandsWithBlobBackedEntityDefaultColumn) {
+  // Goal: cover both GetMergeOperands code paths that read a compacted V2
+  // wide-column entity whose default column was moved to a blob file. The
+  // first read exercises the base-value path directly, then a merge operand is
+  // added so the second read exercises the merge-plus-base path.
+  Options options = GetBlobTestOptions();
+  options.min_blob_size = 50;
+  options.merge_operator = MergeOperators::CreateStringAppendOperator("|");
+
+  DestroyAndReopen(options);
+
+  const std::string key = "merge_operands_blob_entity";
+  const std::string default_value = GenerateLargeValue(100, 'd');
+  const std::string large_value = GenerateLargeValue(120, 'l');
+  const std::string small_value = GenerateSmallValue();
+  const std::string merge_operand = "suffix";
+
+  WideColumns columns{{kDefaultWideColumnName, default_value},
+                      {"col_large", large_value},
+                      {"col_small", small_value}};
+  ASSERT_OK(
+      db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(), key, columns));
+  ASSERT_OK(Flush());
+  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+  ASSERT_FALSE(GetBlobFileNumbers().empty());
+
+  {
+    // The compacted V2 entity becomes the first operand when there are no
+    // newer merge operands above it.
+    GetMergeOperandsOptions get_merge_opts;
+    get_merge_opts.expected_max_number_of_operands = 1;
+
+    std::array<PinnableSlice, 2> merge_operands;
+    int number_of_operands = 0;
+
+    ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(),
+                                    key, merge_operands.data(), &get_merge_opts,
+                                    &number_of_operands));
+    ASSERT_EQ(number_of_operands, 1);
+    ASSERT_EQ(merge_operands[0], default_value);
+  }
+
+  ASSERT_OK(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), key,
+                       merge_operand));
+
+  {
+    // After a merge is added, GetMergeOperands must still resolve the blob
+    // backed base default column while traversing the older base entry below
+    // the newer merge operand.
+    GetMergeOperandsOptions get_merge_opts;
+    get_merge_opts.expected_max_number_of_operands = 2;
+
+    std::array<PinnableSlice, 2> merge_operands;
+    int number_of_operands = 0;
+
+    ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(),
+                                    key, merge_operands.data(), &get_merge_opts,
+                                    &number_of_operands));
+    ASSERT_EQ(number_of_operands, 2);
+    ASSERT_EQ(merge_operands[0], default_value);
+    ASSERT_EQ(merge_operands[1], merge_operand);
+  }
+}
+
+TEST_F(DBWideBasicTest,
+       GetMergeOperandsWithBlobBackedEntityDefaultColumnDirectWriteMemtable) {
+  // Goal: cover the pre-flush memtable path when blob direct write stores a
+  // blob-backed V2 entity in memory. The first GetMergeOperands call exercises
+  // the no-merge base-operand path. After adding a merge, DB::Get exercises
+  // merge resolution against the same memtable entity, and a second
+  // GetMergeOperands call exercises the merge-plus-base path.
+  Options options = GetDirectWriteOptions();
+  options.min_blob_size = 50;
+  options.merge_operator = MergeOperators::CreateStringAppendOperator("|");
+
+  DestroyAndReopen(options);
+
+  const std::string key = "direct_write_merge_operands_blob_entity";
+  const std::string default_value = GenerateLargeValue(100, 'd');
+  const std::string large_value = GenerateLargeValue(120, 'l');
+  const std::string small_value = GenerateSmallValue();
+  const std::string merge_operand = "suffix";
+  const std::string expected_merged = default_value + "|" + merge_operand;
+
+  WideColumns columns{{kDefaultWideColumnName, default_value},
+                      {"col_large", large_value},
+                      {"col_small", small_value}};
+  ASSERT_OK(
+      db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(), key, columns));
+
+  {
+    GetMergeOperandsOptions get_merge_opts;
+    get_merge_opts.expected_max_number_of_operands = 1;
+
+    std::array<PinnableSlice, 2> merge_operands;
+    int number_of_operands = 0;
+
+    ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(),
+                                    key, merge_operands.data(), &get_merge_opts,
+                                    &number_of_operands));
+    ASSERT_EQ(number_of_operands, 1);
+    ASSERT_EQ(merge_operands[0], default_value);
+  }
+
+  ASSERT_OK(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), key,
+                       merge_operand));
+
+  {
+    PinnableSlice result;
+    ASSERT_OK(
+        db_->Get(ReadOptions(), db_->DefaultColumnFamily(), key, &result));
+    ASSERT_EQ(result, expected_merged);
+  }
+
+  {
+    GetMergeOperandsOptions get_merge_opts;
+    get_merge_opts.expected_max_number_of_operands = 2;
+
+    std::array<PinnableSlice, 2> merge_operands;
+    int number_of_operands = 0;
+
+    ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(),
+                                    key, merge_operands.data(), &get_merge_opts,
+                                    &number_of_operands));
+    ASSERT_EQ(number_of_operands, 2);
+    ASSERT_EQ(merge_operands[0], default_value);
+    ASSERT_EQ(merge_operands[1], merge_operand);
+  }
+}
+
+TEST_F(DBWideBasicTest,
+       GetAndGetEntityWithBlobBackedDefaultColumnDirectWriteMemtableReadOnly) {
+  // Goal: cover the read-only reopen path after writing a blob-backed
+  // direct-write entity without manually flushing it first. The test checks
+  // both `Get()`, which must resolve the default-column blob reference, and
+  // `GetEntity()`, which must eagerly resolve the entity's unresolved blob
+  // columns before returning them to the caller.
+  Options options = GetDirectWriteOptions();
+  options.min_blob_size = 50;
+
+  DestroyAndReopen(options);
+
+  const std::string key = "readonly_direct_write_memtable_entity";
+  const std::string default_value = GenerateLargeValue(100, 'd');
+  const std::string large_value = GenerateLargeValue(120, 'l');
+  const std::string small_value = GenerateSmallValue();
+  WideColumns columns{{kDefaultWideColumnName, default_value},
+                      {"col_large", large_value},
+                      {"col_small", small_value}};
+
+  ASSERT_OK(
+      db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(), key, columns));
+
+  Close();
+  options.avoid_flush_during_recovery = true;
+  ASSERT_OK(ReadOnlyReopen(options));
+
+  {
+    PinnableSlice result;
+    ASSERT_OK(
+        db_->Get(ReadOptions(), db_->DefaultColumnFamily(), key, &result));
+    ASSERT_EQ(result, default_value);
+  }
+
+  {
+    PinnableWideColumns result;
+    ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), key,
+                             &result));
+    ASSERT_EQ(result.columns(), columns);
+  }
+}
+
+TEST_F(DBWideBasicTest, ReadOnlyDirectWriteMemtableBlobBlockCacheTier) {
+  // Goal: cover the read-only memtable path under kBlockCacheTier. The test
+  // keeps a blob-backed direct-write entity in WAL-backed recovery state so
+  // resolving either Get() or GetEntity() would require blob I/O, which must
+  // surface as Incomplete rather than silently reading the blob file.
+  Options options = GetDirectWriteOptions();
+  options.min_blob_size = 50;
+
+  DestroyAndReopen(options);
+
+  const std::string key = "readonly_direct_write_memtable_block_cache_tier";
+  const std::string default_value = GenerateLargeValue(100, 'd');
+  const std::string large_value = GenerateLargeValue(120, 'l');
+  const std::string small_value = GenerateSmallValue();
+  WideColumns columns{{kDefaultWideColumnName, default_value},
+                      {"col_large", large_value},
+                      {"col_small", small_value}};
+
+  ASSERT_OK(
+      db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(), key, columns));
+
+  Close();
+  options.avoid_flush_during_recovery = true;
+  ASSERT_OK(ReadOnlyReopen(options));
+
+  ReadOptions read_opts;
+  read_opts.read_tier = kBlockCacheTier;
+
+  {
+    PinnableSlice result;
+    const Status s =
+        db_->Get(read_opts, db_->DefaultColumnFamily(), key, &result);
+    ASSERT_TRUE(s.IsIncomplete()) << s.ToString();
+    ASSERT_TRUE(result.empty());
+  }
+
+  {
+    PinnableWideColumns result;
+    const Status s =
+        db_->GetEntity(read_opts, db_->DefaultColumnFamily(), key, &result);
+    ASSERT_TRUE(s.IsIncomplete()) << s.ToString();
+    ASSERT_TRUE(result.columns().empty());
+  }
+}
+
+TEST_F(DBWideBasicTest,
+       GetMergeOperandsWithBlobBackedEntityDefaultColumnReadOnly) {
+  // Goal: exercise OpenForReadOnly on a blob-backed V2 entity base in SST with
+  // a newer merge operand in a separate SST. The read-only DB must resolve the
+  // blob-backed default column for both Get() and GetMergeOperands() without
+  // relying on mutable memtable state.
+  Options options = GetBlobTestOptions();
+  options.min_blob_size = 50;
+  options.merge_operator = MergeOperators::CreateStringAppendOperator("|");
+
+  DestroyAndReopen(options);
+
+  const std::string key = "readonly_merge_operands_blob_entity";
+  const std::string default_value = GenerateLargeValue(100, 'd');
+  const std::string large_value = GenerateLargeValue(120, 'l');
+  const std::string small_value = GenerateSmallValue();
+  const std::string merge_operand = "suffix";
+  const std::string expected_merged = default_value + "|" + merge_operand;
+
+  WideColumns columns{{kDefaultWideColumnName, default_value},
+                      {"col_large", large_value},
+                      {"col_small", small_value}};
+  ASSERT_OK(
+      db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(), key, columns));
+  ASSERT_OK(Flush());
+  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+  ASSERT_FALSE(GetBlobFileNumbers().empty());
+
+  ASSERT_OK(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), key,
+                       merge_operand));
+  ASSERT_OK(Flush());
+
+  Close();
+  ASSERT_OK(ReadOnlyReopen(options));
+
+  {
+    PinnableSlice result;
+    ASSERT_OK(
+        db_->Get(ReadOptions(), db_->DefaultColumnFamily(), key, &result));
+    ASSERT_EQ(result, expected_merged);
+  }
+
+  {
+    GetMergeOperandsOptions get_merge_opts;
+    get_merge_opts.expected_max_number_of_operands = 2;
+
+    std::array<PinnableSlice, 2> merge_operands;
+    int number_of_operands = 0;
+
+    ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(),
+                                    key, merge_operands.data(), &get_merge_opts,
+                                    &number_of_operands));
+    ASSERT_EQ(number_of_operands, 2);
+    ASSERT_EQ(merge_operands[0], default_value);
+    ASSERT_EQ(merge_operands[1], merge_operand);
+  }
+}
+
+TEST_F(DBWideBasicTest,
+       GetMergeOperandsWithBlobBackedEntityDefaultColumnBlockCacheTier) {
+  // Goal: prove the SST-backed GetMergeOperands path propagates Incomplete
+  // when resolving a blob-backed default column would require I/O. One key
+  // covers the direct base-entity path, and the other covers the merge-plus-
+  // base path with the merge operand flushed to a newer SST.
+  Options options = GetBlobTestOptions();
+  options.min_blob_size = 50;
+  options.merge_operator = MergeOperators::CreateStringAppendOperator("|");
+
+  DestroyAndReopen(options);
+
+  const std::string base_key = "merge_operands_blob_entity_cache_base";
+  const std::string merge_key = "merge_operands_blob_entity_cache_merge";
+  const std::string default_value = GenerateLargeValue(100, 'd');
+  const std::string large_value = GenerateLargeValue(120, 'l');
+  const std::string small_value = GenerateSmallValue();
+
+  WideColumns columns{{kDefaultWideColumnName, default_value},
+                      {"col_large", large_value},
+                      {"col_small", small_value}};
+  ASSERT_OK(db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(), base_key,
+                           columns));
+  ASSERT_OK(db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(),
+                           merge_key, columns));
+  ASSERT_OK(Flush());
+  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+
+  ASSERT_OK(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), merge_key,
+                       "tail"));
+  ASSERT_OK(Flush());
+
+  // Reopen to clear any cached blob readers/values so kBlockCacheTier must
+  // report Incomplete instead of succeeding through a warm cache.
+  Reopen(options);
+
+  ReadOptions read_opts;
+  read_opts.read_tier = kBlockCacheTier;
+
+  {
+    GetMergeOperandsOptions get_merge_opts;
+    get_merge_opts.expected_max_number_of_operands = 1;
+
+    std::array<PinnableSlice, 2> merge_operands;
+    int number_of_operands = 0;
+
+    const Status s = db_->GetMergeOperands(
+        read_opts, db_->DefaultColumnFamily(), base_key, merge_operands.data(),
+        &get_merge_opts, &number_of_operands);
+    ASSERT_TRUE(s.IsIncomplete()) << s.ToString();
+  }
+
+  {
+    GetMergeOperandsOptions get_merge_opts;
+    get_merge_opts.expected_max_number_of_operands = 2;
+
+    std::array<PinnableSlice, 2> merge_operands;
+    int number_of_operands = 0;
+
+    const Status s = db_->GetMergeOperands(
+        read_opts, db_->DefaultColumnFamily(), merge_key, merge_operands.data(),
+        &get_merge_opts, &number_of_operands);
+    ASSERT_TRUE(s.IsIncomplete()) << s.ToString();
+  }
+}
+
 TEST_F(DBWideBasicTest, MergeEntityWithBlobColumnsNoDefault) {
   // Test: Merge on a V2 entity without a default column. The merge result
   // should produce a valid entity with all original columns plus a default
@@ -2998,7 +3394,7 @@ TEST_F(DBWideBasicTest, MergeEntityWithBlobColumnsBlockCacheTier) {
   // Reopen to clear caches
   Reopen(options);
 
-  // Read with kBlockCacheTier — blob not in cache, merge needs blob resolution
+  // Read with kBlockCacheTier -- blob not in cache, merge needs blob resolution
   ReadOptions read_opts;
   read_opts.read_tier = kBlockCacheTier;
   PinnableSlice result;

db/wide/read_path_blob_resolver.cc

@@ -59,7 +59,7 @@ Status ReadPathBlobResolver::ResolveColumn(size_t column_index,
         blob_resolver_util::FindBlobColumn(blob_columns_, column_index);
 
     if (blob_index_ptr == nullptr) {
-      // Inline column — return the value directly
+      // Inline column -- return the value directly
       *resolved_value = (*columns_)[column_index].value();
     } else {
       // Check if already resolved

db/wide/wide_column_serialization.h

@@ -153,6 +153,8 @@ class WideColumnSerialization {
   // value() is initially the serialized BlobIndex bytes from the entity.
   // blob_columns: receives (column_index, blob_index) pairs identifying which
   // entries in `columns` are blob references and their decoded BlobIndex data.
+  // For valid input, this must agree with HasBlobColumns() about whether blob
+  // column references are present.
   static Status DeserializeV2(
       Slice& input, std::vector<WideColumn>& columns,
       std::vector<std::pair<size_t, BlobIndex>>& blob_columns);
@@ -160,6 +162,8 @@ class WideColumnSerialization {
   // Check if the serialized entity has any blob column references.
   // Sets *has_blob_columns to true if version >= 2 and at least one column
   // has blob type; false otherwise.
+  // For valid input, this must agree with DeserializeV2() about whether blob
+  // column references are present.
   // Returns Status::Corruption on decode errors.
   static Status HasBlobColumns(const Slice& input, bool& has_blob_columns);
 

db/wide/wide_column_serialization_test.cc

@@ -391,7 +391,7 @@ static BlobIndex MakeBlobIndex(uint64_t file_number, uint64_t offset,
   return bi;
 }
 
-// Helper: V2 serialize → DeserializeV2 round-trip, returning
+// Helper: V2 serialize -> DeserializeV2 round-trip, returning
 // deserialized columns and blob column info.
 static void V2SerializeAndDeserialize(
     const std::vector<std::pair<std::string, std::string>>& columns,
@@ -798,7 +798,7 @@ TEST_F(WideColumnSerializationTest, RandomizedSerializeDeserializeRoundTrip) {
       }
     }
 
-    // V2 serialize → DeserializeV2 round-trip
+    // V2 serialize -> DeserializeV2 round-trip
     std::string serialized;
     std::vector<WideColumn> deserialized;
     std::vector<std::pair<size_t, BlobIndex>> blob_out;

db_stress_tool/batched_ops_stress.cc

@@ -13,7 +13,9 @@
 namespace ROCKSDB_NAMESPACE {
 class BatchedOpsStressTest : public StressTest {
  public:
-  BatchedOpsStressTest() = default;
+  BatchedOpsStressTest(int db_index, const std::string& db_path,
+                       const std::string& ev_path, const std::string& sec_path)
+      : StressTest(db_index, db_path, ev_path, sec_path) {}
   virtual ~BatchedOpsStressTest() = default;
 
   bool IsStateTracked() const override { return false; }
@@ -722,7 +724,11 @@ class BatchedOpsStressTest : public StressTest {
   }
 };
 
-StressTest* CreateBatchedOpsStressTest() { return new BatchedOpsStressTest(); }
+StressTest* CreateBatchedOpsStressTest(int db_index, const std::string& db_path,
+                                       const std::string& ev_path,
+                                       const std::string& sec_path) {
+  return new BatchedOpsStressTest(db_index, db_path, ev_path, sec_path);
+}
 
 }  // namespace ROCKSDB_NAMESPACE
 #endif  // GFLAGS

db_stress_tool/cf_consistency_stress.cc

@@ -21,7 +21,10 @@
 namespace ROCKSDB_NAMESPACE {
 class CfConsistencyStressTest : public StressTest {
  public:
-  CfConsistencyStressTest() : batch_id_(0) {}
+  CfConsistencyStressTest(int db_index, const std::string& db_path,
+                          const std::string& ev_path,
+                          const std::string& sec_path)
+      : StressTest(db_index, db_path, ev_path, sec_path), batch_id_(0) {}
 
   ~CfConsistencyStressTest() override = default;
 
@@ -227,10 +230,10 @@ class CfConsistencyStressTest : public StressTest {
                    key, &value0);
 
       // Temporarily disable error injection for verification
-      if (fault_fs_guard) {
-        fault_fs_guard->DisableThreadLocalErrorInjection(
+      if (db_fault_injection_fs_) {
+        db_fault_injection_fs_->DisableThreadLocalErrorInjection(
             FaultInjectionIOType::kRead);
-        fault_fs_guard->DisableThreadLocalErrorInjection(
+        db_fault_injection_fs_->DisableThreadLocalErrorInjection(
             FaultInjectionIOType::kMetadataRead);
       }
 
@@ -279,10 +282,10 @@ class CfConsistencyStressTest : public StressTest {
       }
 
       //  Enable back error injection disabled for verification
-      if (fault_fs_guard) {
-        fault_fs_guard->EnableThreadLocalErrorInjection(
+      if (db_fault_injection_fs_) {
+        db_fault_injection_fs_->EnableThreadLocalErrorInjection(
             FaultInjectionIOType::kRead);
-        fault_fs_guard->EnableThreadLocalErrorInjection(
+        db_fault_injection_fs_->EnableThreadLocalErrorInjection(
             FaultInjectionIOType::kMetadataRead);
       }
       db_->ReleaseSnapshot(snapshot);
@@ -397,10 +400,10 @@ class CfConsistencyStressTest : public StressTest {
                          &cmp_result);
 
       //  Temporarily disable error injection for verification
-      if (fault_fs_guard) {
-        fault_fs_guard->DisableThreadLocalErrorInjection(
+      if (db_fault_injection_fs_) {
+        db_fault_injection_fs_->DisableThreadLocalErrorInjection(
             FaultInjectionIOType::kRead);
-        fault_fs_guard->DisableThreadLocalErrorInjection(
+        db_fault_injection_fs_->DisableThreadLocalErrorInjection(
             FaultInjectionIOType::kMetadataRead);
       }
 
@@ -559,10 +562,10 @@ class CfConsistencyStressTest : public StressTest {
       }
 
       //  Enable back error injection disabled for verification
-      if (fault_fs_guard) {
-        fault_fs_guard->EnableThreadLocalErrorInjection(
+      if (db_fault_injection_fs_) {
+        db_fault_injection_fs_->EnableThreadLocalErrorInjection(
             FaultInjectionIOType::kRead);
-        fault_fs_guard->EnableThreadLocalErrorInjection(
+        db_fault_injection_fs_->EnableThreadLocalErrorInjection(
             FaultInjectionIOType::kMetadataRead);
       }
     }
@@ -1543,8 +1546,11 @@ class CfConsistencyStressTest : public StressTest {
   std::deque<DebugEvent> recent_debug_events_;
 };
 
-StressTest* CreateCfConsistencyStressTest() {
-  return new CfConsistencyStressTest();
+StressTest* CreateCfConsistencyStressTest(int db_index,
+                                          const std::string& db_path,
+                                          const std::string& ev_path,
+                                          const std::string& sec_path) {
+  return new CfConsistencyStressTest(db_index, db_path, ev_path, sec_path);
 }
 
 }  // namespace ROCKSDB_NAMESPACE

db_stress_tool/db_stress_common.cc

@@ -13,16 +13,17 @@
 
 #include <cmath>
 
+#include "db_stress_tool/db_stress_test_base.h"
 #include "file/file_util.h"
 #include "rocksdb/secondary_cache.h"
 #include "util/file_checksum_helper.h"
 #include "util/xxhash.h"
 
-ROCKSDB_NAMESPACE::Env* db_stress_listener_env = nullptr;
-ROCKSDB_NAMESPACE::Env* db_stress_env = nullptr;
-std::shared_ptr<ROCKSDB_NAMESPACE::FaultInjectionTestFS> fault_fs_guard;
+ROCKSDB_NAMESPACE::Env* raw_env = nullptr;
 std::shared_ptr<ROCKSDB_NAMESPACE::SecondaryCache> compressed_secondary_cache;
 std::shared_ptr<ROCKSDB_NAMESPACE::Cache> block_cache;
+std::shared_ptr<ROCKSDB_NAMESPACE::WriteBufferManager> wbm;
+std::shared_ptr<ROCKSDB_NAMESPACE::RateLimiter> rate_limiter;
 enum ROCKSDB_NAMESPACE::CompressionType compression_type_e =
     ROCKSDB_NAMESPACE::kSnappyCompression;
 enum ROCKSDB_NAMESPACE::CompressionType bottommost_compression_type_e =
@@ -114,10 +115,10 @@ void PoolSizeChangeThread(void* v) {
     if (new_thread_pool_size < 1) {
       new_thread_pool_size = 1;
     }
-    db_stress_env->SetBackgroundThreads(new_thread_pool_size,
-                                        ROCKSDB_NAMESPACE::Env::Priority::LOW);
+    raw_env->SetBackgroundThreads(new_thread_pool_size,
+                                  ROCKSDB_NAMESPACE::Env::Priority::LOW);
     // Sleep up to 3 seconds
-    db_stress_env->SleepForMicroseconds(
+    raw_env->SleepForMicroseconds(
         thread->rand.Next() % FLAGS_compaction_thread_pool_adjust_interval *
             1000 +
         1);
@@ -144,7 +145,7 @@ void DbVerificationThread(void* v) {
     if (!shared->HasVerificationFailedYet()) {
       stress_test->ContinuouslyVerifyDb(thread);
     }
-    db_stress_env->SleepForMicroseconds(
+    raw_env->SleepForMicroseconds(
         thread->rand.Next() % FLAGS_continuous_verification_interval * 1000 +
         1);
   }
@@ -166,7 +167,7 @@ void CompressedCacheSetCapacityThread(void* v) {
         return;
       }
     }
-    db_stress_env->SleepForMicroseconds(FLAGS_secondary_cache_update_interval);
+    raw_env->SleepForMicroseconds(FLAGS_secondary_cache_update_interval);
     if (FLAGS_compressed_secondary_cache_size > 0) {
       Status s = compressed_secondary_cache->SetCapacity(0);
       size_t capacity;
@@ -174,7 +175,7 @@ void CompressedCacheSetCapacityThread(void* v) {
         s = compressed_secondary_cache->GetCapacity(capacity);
         assert(capacity == 0);
       }
-      db_stress_env->SleepForMicroseconds(10 * 1000 * 1000);
+      raw_env->SleepForMicroseconds(10 * 1000 * 1000);
       if (s.ok()) {
         s = compressed_secondary_cache->SetCapacity(
             FLAGS_compressed_secondary_cache_size);
@@ -201,7 +202,7 @@ void CompressedCacheSetCapacityThread(void* v) {
         block_cache->SetCapacity(capacity - adjustment);
         fprintf(stdout, "New cache capacity = %zu\n",
                 block_cache->GetCapacity());
-        db_stress_env->SleepForMicroseconds(10 * 1000 * 1000);
+        raw_env->SleepForMicroseconds(10 * 1000 * 1000);
         block_cache->SetCapacity(capacity);
       } else {
         Status s;
@@ -214,7 +215,7 @@ void CompressedCacheSetCapacityThread(void* v) {
         s = UpdateTieredCache(block_cache, /*capacity*/ -1,
                               new_comp_cache_ratio);
         if (s.ok()) {
-          db_stress_env->SleepForMicroseconds(10 * 1000 * 1000);
+          raw_env->SleepForMicroseconds(10 * 1000 * 1000);
         }
         if (s.ok()) {
           s = UpdateTieredCache(block_cache, /*capacity*/ -1,
@@ -231,36 +232,37 @@ void CompressedCacheSetCapacityThread(void* v) {
 
 #ifndef NDEBUG
 static void SetupFaultInjectionForRemoteCompaction(SharedState* shared) {
-  if (!fault_fs_guard) {
+  auto fault_fs = shared->GetStressTest()->GetDbFaultInjectionFs();
+  if (!fault_fs) {
     return;
   }
 
-  fault_fs_guard->SetThreadLocalErrorContext(
+  fault_fs->SetThreadLocalErrorContext(
       FaultInjectionIOType::kRead, shared->GetSeed(), FLAGS_read_fault_one_in,
       FLAGS_inject_error_severity == 1 /* retryable */,
       FLAGS_inject_error_severity == 2 /* has_data_loss*/);
-  fault_fs_guard->EnableThreadLocalErrorInjection(FaultInjectionIOType::kRead);
+  fault_fs->EnableThreadLocalErrorInjection(FaultInjectionIOType::kRead);
 
-  fault_fs_guard->SetThreadLocalErrorContext(
+  fault_fs->SetThreadLocalErrorContext(
       FaultInjectionIOType::kWrite, shared->GetSeed(), FLAGS_write_fault_one_in,
       FLAGS_inject_error_severity == 1 /* retryable */,
       FLAGS_inject_error_severity == 2 /* has_data_loss*/);
-  fault_fs_guard->EnableThreadLocalErrorInjection(FaultInjectionIOType::kWrite);
+  fault_fs->EnableThreadLocalErrorInjection(FaultInjectionIOType::kWrite);
 
-  fault_fs_guard->SetThreadLocalErrorContext(
+  fault_fs->SetThreadLocalErrorContext(
       FaultInjectionIOType::kMetadataRead, shared->GetSeed(),
       FLAGS_metadata_read_fault_one_in,
       FLAGS_inject_error_severity == 1 /* retryable */,
       FLAGS_inject_error_severity == 2 /* has_data_loss*/);
-  fault_fs_guard->EnableThreadLocalErrorInjection(
+  fault_fs->EnableThreadLocalErrorInjection(
       FaultInjectionIOType::kMetadataRead);
 
-  fault_fs_guard->SetThreadLocalErrorContext(
+  fault_fs->SetThreadLocalErrorContext(
       FaultInjectionIOType::kMetadataWrite, shared->GetSeed(),
       FLAGS_metadata_write_fault_one_in,
       FLAGS_inject_error_severity == 1 /* retryable */,
       FLAGS_inject_error_severity == 2 /* has_data_loss*/);
-  fault_fs_guard->EnableThreadLocalErrorInjection(
+  fault_fs->EnableThreadLocalErrorInjection(
       FaultInjectionIOType::kMetadataWrite);
 }
 #endif  // NDEBUG
@@ -268,7 +270,7 @@ static void SetupFaultInjectionForRemoteCompaction(SharedState* shared) {
 static CompactionServiceOptionsOverride CreateOverrideOptions(
     const Options& options, const CompactionServiceJobInfo& job_info) {
   CompactionServiceOptionsOverride override_options{
-      .env = db_stress_env,
+      .env = options.env,
       .file_checksum_gen_factory = options.file_checksum_gen_factory,
       .merge_operator = options.merge_operator,
       .compaction_filter = options.compaction_filter,
@@ -311,14 +313,8 @@ static CompactionServiceOptionsOverride CreateOverrideOptions(
 }
 
 static Status CleanupOutputDirectory(const std::string& output_directory) {
-#ifndef NDEBUG
-  // Temporarily disable fault injection to ensure deletion always succeeds
-  if (fault_fs_guard) {
-    fault_fs_guard->DisableAllThreadLocalErrorInjection();
-  }
-#endif  // NDEBUG
-
-  Status s = DestroyDir(db_stress_env, output_directory);
+  // Uses raw_env (no fault injection); cleanup must always succeed.
+  Status s = DestroyDir(raw_env, output_directory);
   if (!s.ok()) {
     fprintf(stderr,
             "Failed to destroy output directory %s when allow_resumption is "
@@ -327,7 +323,7 @@ static Status CleanupOutputDirectory(const std::string& output_directory) {
   }
 
   if (s.ok()) {
-    s = db_stress_env->CreateDir(output_directory);
+    s = raw_env->CreateDir(output_directory);
     if (!s.ok()) {
       fprintf(stderr,
               "Failed to recreate output directory %s when allow_resumption is "
@@ -336,13 +332,6 @@ static Status CleanupOutputDirectory(const std::string& output_directory) {
     }
   }
 
-#ifndef NDEBUG
-  // Re-enable fault injection after deletion
-  if (fault_fs_guard) {
-    fault_fs_guard->EnableAllThreadLocalErrorInjection();
-  }
-#endif  // NDEBUG
-
   return s;
 }
 
@@ -497,7 +486,7 @@ void RemoteCompactionWorkerThread(void* v) {
           shared, stress_test, rand, successful_compaction_end_to_end_micros);
     }
 
-    db_stress_env->SleepForMicroseconds(
+    raw_env->SleepForMicroseconds(
         thread->rand.Next() % FLAGS_remote_compaction_worker_interval * 1000 +
         1);
   }
@@ -679,7 +668,7 @@ bool VerifyIteratorAttributeGroups(
 }
 
 std::string GetNowNanos() {
-  uint64_t t = db_stress_env->NowNanos();
+  uint64_t t = raw_env->NowNanos();
   std::string ret;
   PutFixed64(&ret, t);
   return ret;
@@ -691,7 +680,7 @@ uint64_t GetWriteUnixTime(ThreadState* thread) {
                FLAGS_preclude_last_level_data_seconds);
   static uint64_t kFallbackTime = std::numeric_limits<uint64_t>::max();
   int64_t write_time = 0;
-  Status s = db_stress_env->GetCurrentTime(&write_time);
+  Status s = raw_env->GetCurrentTime(&write_time);
   uint32_t write_time_mode = thread->rand.Uniform(3);
   if (write_time_mode == 0 || !s.ok()) {
     return kFallbackTime;
@@ -806,6 +795,8 @@ std::shared_ptr<FileChecksumGenFactory> GetFileChecksumImpl(
   return std::make_shared<DbStressChecksumGenFactory>(internal_name);
 }
 
+// Expected values state files are always on local filesystem (Python owns
+// this dir), so use Env::Default() (PosixEnv) even when raw_env is remote.
 Status DeleteFilesInDirectory(const std::string& dirname) {
   std::vector<std::string> filenames;
   Status s = Env::Default()->GetChildren(dirname, &filenames);
@@ -880,8 +871,12 @@ Status DestroyUnverifiedSubdir(const std::string& dirname) {
 Status DbStressDestroyDb(const std::string& db_path) {
   Status s;
   Options options;
-  // NOTE: using db_stress_listener_env in order to see obsolete MANIFEST files
-  options.env = db_stress_listener_env;
+  // Use raw_env (no DbStressFSWrapper) because DbStressFSWrapper renames
+  // MANIFEST files to MANIFEST-xxx_renamed_ instead of deleting them during
+  // normal DB operation (to preserve history for debugging). Using raw_env
+  // here ensures DestroyDB actually deletes these files. DestroyDir below
+  // catches any remaining files not known to DestroyDB.
+  options.env = raw_env;
   // Remove DB files in a principled way to avoid issues
   if (FLAGS_use_blob_db) {
     s = blob_db::DestroyBlobDB(db_path, options, blob_db::BlobDBOptions());
@@ -891,9 +886,8 @@ Status DbStressDestroyDb(const std::string& db_path) {
   if (!s.ok()) {
     return s;
   }
-  // Remove everything else recursively, only reporting success if able to
-  // delete everything
-  return DestroyDir(db_stress_listener_env, db_path);
+  // Remove everything else recursively (catches MANIFEST_renamed_ files)
+  return DestroyDir(raw_env, db_path);
 }
 
 }  // namespace ROCKSDB_NAMESPACE

db_stress_tool/db_stress_common.h

@@ -49,6 +49,7 @@
 #include "rocksdb/advanced_options.h"
 #include "rocksdb/cache.h"
 #include "rocksdb/env.h"
+#include "rocksdb/rate_limiter.h"
 #include "rocksdb/slice.h"
 #include "rocksdb/slice_transform.h"
 #include "rocksdb/statistics.h"
@@ -61,6 +62,7 @@
 #include "rocksdb/utilities/transaction.h"
 #include "rocksdb/utilities/transaction_db.h"
 #include "rocksdb/write_batch.h"
+#include "rocksdb/write_buffer_manager.h"
 #include "test_util/testutil.h"
 #include "util/coding.h"
 #include "util/compression.h"
@@ -114,6 +116,7 @@ DECLARE_double(memtable_prefix_bloom_size_ratio);
 DECLARE_bool(memtable_whole_key_filtering);
 DECLARE_int32(open_files);
 DECLARE_bool(open_files_async);
+DECLARE_bool(async_wal_precreate);
 DECLARE_uint64(compressed_secondary_cache_size);
 DECLARE_int32(compressed_secondary_cache_numshardbits);
 DECLARE_int32(secondary_cache_update_interval);
@@ -190,6 +193,7 @@ DECLARE_string(db);
 DECLARE_string(secondaries_base);
 DECLARE_bool(test_secondary);
 DECLARE_string(expected_values_dir);
+DECLARE_int32(num_dbs);
 DECLARE_bool(expected_state_trace_debug);
 DECLARE_int64(expected_state_trace_debug_key);
 DECLARE_int32(expected_state_trace_debug_max_logs);
@@ -279,7 +283,9 @@ DECLARE_bool(use_full_merge_v1);
 DECLARE_int32(sync_wal_one_in);
 DECLARE_bool(avoid_unnecessary_blocking_io);
 DECLARE_bool(write_dbid_to_manifest);
+DECLARE_bool(optimize_manifest_for_recovery);
 DECLARE_bool(write_identity_file);
+DECLARE_bool(reuse_manifest_on_open);
 DECLARE_bool(avoid_flush_during_recovery);
 DECLARE_bool(enforce_write_buffer_manager_during_recovery);
 DECLARE_uint64(max_write_batch_group_size_bytes);
@@ -363,8 +369,6 @@ DECLARE_uint64(wp_commit_cache_bits);
 DECLARE_bool(adaptive_readahead);
 DECLARE_bool(async_io);
 DECLARE_string(wal_compression);
-DECLARE_bool(use_blog_format_for_blobs);
-DECLARE_string(blog_checksum);
 DECLARE_bool(verify_sst_unique_id_in_manifest);
 DECLARE_bool(fast_sst_open);
 
@@ -462,6 +466,7 @@ DECLARE_uint32(ingest_wbwi_one_in);
 DECLARE_bool(universal_reduce_file_locking);
 DECLARE_bool(use_multiscan);
 DECLARE_bool(multiscan_use_async_io);
+DECLARE_bool(read_scoped_block_buffer_provider);
 DECLARE_uint64(multiscan_max_prefetch_memory_bytes);
 
 // Compaction deletion trigger declarations for stress testing
@@ -472,18 +477,23 @@ DECLARE_int32(compaction_on_deletion_window_size);
 DECLARE_double(compaction_on_deletion_ratio);
 DECLARE_double(read_triggered_compaction_threshold);
 
+DECLARE_string(listener_uri);
+
 constexpr long KB = 1024;
 constexpr int kRandomValueMaxFactor = 3;
 constexpr int kValueMaxLen = 100;
 constexpr uint32_t kLargePrimeForCommonFactorSkew = 1872439133;
 
-// wrapped posix environment
-extern ROCKSDB_NAMESPACE::Env* db_stress_env;
-extern ROCKSDB_NAMESPACE::Env* db_stress_listener_env;
-extern std::shared_ptr<ROCKSDB_NAMESPACE::FaultInjectionTestFS> fault_fs_guard;
+// Base env from --env_uri/--fs_uri. No wrappers (no DbStressFSWrapper, no
+// fault injection). Could be PosixEnv or remote. Used for infrastructure ops
+// (threads, time, dirs, cleanup). Per-StressTest env adds fault injection +
+// DbStressFSWrapper on top for DB I/O.
+extern ROCKSDB_NAMESPACE::Env* raw_env;
 extern std::shared_ptr<ROCKSDB_NAMESPACE::SecondaryCache>
     compressed_secondary_cache;
 extern std::shared_ptr<ROCKSDB_NAMESPACE::Cache> block_cache;
+extern std::shared_ptr<ROCKSDB_NAMESPACE::WriteBufferManager> wbm;
+extern std::shared_ptr<ROCKSDB_NAMESPACE::RateLimiter> rate_limiter;
 
 extern enum ROCKSDB_NAMESPACE::CompressionType compression_type_e;
 extern enum ROCKSDB_NAMESPACE::CompressionType bottommost_compression_type_e;
@@ -828,10 +838,21 @@ AttributeGroups GenerateAttributeGroups(
     const std::vector<ColumnFamilyHandle*>& cfhs, uint32_t value_base,
     const Slice& slice);
 
-StressTest* CreateCfConsistencyStressTest();
-StressTest* CreateBatchedOpsStressTest();
-StressTest* CreateNonBatchedOpsStressTest();
-StressTest* CreateMultiOpsTxnsStressTest();
+StressTest* CreateCfConsistencyStressTest(int db_index,
+                                          const std::string& db_path,
+                                          const std::string& ev_path,
+                                          const std::string& sec_path);
+StressTest* CreateBatchedOpsStressTest(int db_index, const std::string& db_path,
+                                       const std::string& ev_path,
+                                       const std::string& sec_path);
+StressTest* CreateNonBatchedOpsStressTest(int db_index,
+                                          const std::string& db_path,
+                                          const std::string& ev_path,
+                                          const std::string& sec_path);
+StressTest* CreateMultiOpsTxnsStressTest(int db_index,
+                                         const std::string& db_path,
+                                         const std::string& ev_path,
+                                         const std::string& sec_path);
 void CheckAndSetOptionsForMultiOpsTxnStressTest();
 void InitializeHotKeyGenerator(double alpha);
 int64_t GetOneHotKeyID(double rand_seed, int64_t max_key);