Update HISTORY.md and version for 10.8.3

Summary:
There are instances where  an application might be interested in knowing the distribution in SST files for a key range in a particular level.

This implementation creates an overloaded GetColumnFamilyMetaData api where  (startKey, EndKey) can be passed along with level information to filter the necessary sst files along with the keyranges for each sst file

Pull Request resolved: https://github.com/facebook/rocksdb/pull/14009

Reviewed By: anand1976

Differential Revision: D83389707

fbshipit-source-id: 6df1dc1f9233efe9000b03cc1831b3c618cbcef3

.github/actions/build-folly/action.yml

@@ -1,7 +1,17 @@
 name: build-folly
+description: Build folly and dependencies (skipped if cache hit)
+inputs:
+  cache-hit:
+    description: Whether the folly cache was hit
+    required: true
 runs:
   using: composite
   steps:
   - name: Build folly and dependencies
+    if: ${{ inputs.cache-hit != 'true' }}
     run: make build_folly
     shell: bash
+  - name: Skip folly build (using cached version)
+    if: ${{ inputs.cache-hit == 'true' }}
+    run: echo "Folly build skipped - using cached version"
+    shell: bash

.github/actions/cache-folly/action.yml

@@ -0,0 +1,26 @@
+name: cache-folly
+description: Cache folly build to speed up CI
+outputs:
+  cache-hit:
+    description: Whether the cache was hit
+    value: ${{ steps.cache-folly-build.outputs.cache-hit }}
+runs:
+  using: composite
+  steps:
+  - name: Extract FOLLY_COMMIT_HASH from Makefile
+    id: extract-folly-hash
+    shell: bash
+    run: |
+      FOLLY_COMMIT_HASH=$(grep '^FOLLY_COMMIT_HASH' Makefile | awk '{print $3}')
+      echo "hash=$FOLLY_COMMIT_HASH" >> $GITHUB_OUTPUT
+  - name: Cache folly build
+    id: cache-folly-build
+    uses: actions/cache@v4
+    with:
+      # Cache the folly build directory
+      path: /tmp/fbcode_builder_getdeps-Z__wZrocksdbZrocksdbZthird-partyZfollyZbuildZfbcode_builder-root/installed
+      # Key is based on:
+      # - OS and architecture
+      # - The specific folly commit hash from Makefile
+      # - The container image version to account for different compiler/library versions
+      key: folly-build-${{ runner.os }}-${{ runner.arch }}-${{ steps.extract-folly-hash.outputs.hash }}-ubuntu22.1-v1

.github/actions/setup-folly/action.yml

@@ -3,5 +3,7 @@ runs:
   using: composite
   steps:
   - name: Checkout folly sources
-    run: make checkout_folly
+    run: |
+      make checkout_folly
+      echo "GETDEPS_USE_WGET=1" >> "$GITHUB_ENV"
     shell: bash

.github/actions/windows-build-steps/action.yml

@@ -4,6 +4,16 @@ runs:
   steps:
   - name: Add msbuild to PATH
     uses: microsoft/setup-msbuild@v1.3.1
+  - name: Cache ccache directory
+    id: ccache-cache
+    uses: actions/cache@v4
+    with:
+      path: C:\a\rocksdb\rocksdb\.ccache
+      key: rocksdb-build-${{ runner.os }}-${{ runner.arch }}-ccache-${{ hashFiles('CMakeLists.txt', 'cmake/**/*.cmake') }}-v1
+  - name: ccache
+    uses: hendrikmuhs/ccache-action@v1.2
+    with:
+      max-size: "10GB"
   - name: Custom steps
     env:
       THIRDPARTY_HOME: ${{ github.workspace }}/thirdparty
@@ -38,11 +48,12 @@ runs:
       $env:Path = $env:JAVA_HOME + ";" + $env:Path
       mkdir build
       cd build
-      & cmake -G "$Env:CMAKE_GENERATOR" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE="$Env:CMAKE_PORTABLE" -DSNAPPY=1 -DXPRESS=1 -DJNI=1 ..
+      & cmake -G "$Env:CMAKE_GENERATOR" -DCMAKE_BUILD_TYPE=Debug -DWIN_CI=1 -DPORTABLE="$Env:CMAKE_PORTABLE" -DSNAPPY=1 -DXPRESS=1 -DJNI=1 ..
       if(!$?) { Exit $LASTEXITCODE }
       cd ..
       echo "Building with VS version: $Env:CMAKE_GENERATOR"
-      msbuild build/rocksdb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
+      # use more parallel processes than the number of processes available, as most of the compile command would be cache hit
+      msbuild build/rocksdb.sln /m:32 /p:LinkIncremental=false -property:Configuration=Debug -property:Platform=x64
       if(!$?) { Exit $LASTEXITCODE }
       echo ========================= Test RocksDB =========================
       build_tools\run_ci_db_test.ps1 -SuiteRun arena_test,db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16
@@ -52,3 +63,7 @@ runs:
       & ctest -C Debug -j 16
       if(!$?) { Exit $LASTEXITCODE }
     shell: pwsh
+  - name: Show ccache stats
+    shell: pwsh
+    run: |
+      ccache --show-stats -v

.github/workflows/pr-jobs.yml

@@ -103,7 +103,11 @@ jobs:
     - uses: actions/checkout@v4.1.0
     - uses: "./.github/actions/pre-steps"
     - uses: "./.github/actions/setup-folly"
+    - uses: "./.github/actions/cache-folly"
+      id: cache-folly
     - uses: "./.github/actions/build-folly"
+      with:
+        cache-hit: ${{ steps.cache-folly.outputs.cache-hit }}
     - run: "(mkdir build && cd build && cmake -DUSE_FOLLY=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20)"
     - uses: "./.github/actions/post-steps"
   build-linux-make-with-folly:
@@ -117,7 +121,11 @@ jobs:
     - uses: actions/checkout@v4.1.0
     - uses: "./.github/actions/pre-steps"
     - uses: "./.github/actions/setup-folly"
+    - uses: "./.github/actions/cache-folly"
+      id: cache-folly
     - uses: "./.github/actions/build-folly"
+      with:
+        cache-hit: ${{ steps.cache-folly.outputs.cache-hit }}
     - run: USE_FOLLY=1 LIB_MODE=static V=1 make -j32 check
     - uses: "./.github/actions/post-steps"
   build-linux-make-with-folly-lite-no-test:
@@ -144,7 +152,11 @@ jobs:
     - uses: actions/checkout@v4.1.0
     - uses: "./.github/actions/pre-steps"
     - uses: "./.github/actions/setup-folly"
+    - uses: "./.github/actions/cache-folly"
+      id: cache-folly
     - uses: "./.github/actions/build-folly"
+      with:
+        cache-hit: ${{ steps.cache-folly.outputs.cache-hit }}
     - run: "(mkdir build && cd build && cmake -DUSE_COROUTINES=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20)"
     - uses: "./.github/actions/post-steps"
   build-linux-cmake-with-benchmark:
@@ -328,51 +340,57 @@ jobs:
   # ========================= MacOS build only ======================== #
   build-macos:
     if: ${{ github.repository_owner == 'facebook' }}
-    runs-on: macos-13
+    runs-on: macos-15-xlarge
     env:
       ROCKSDB_DISABLE_JEMALLOC: 1
     steps:
     - uses: actions/checkout@v4.1.0
     - uses: maxim-lobanov/setup-xcode@v1.6.0
       with:
-        xcode-version: 14.3.1
+        xcode-version: 16.4.0
     - uses: "./.github/actions/increase-max-open-files-on-macos"
     - uses: "./.github/actions/install-gflags-on-macos"
     - uses: "./.github/actions/pre-steps-macos"
     - name: Build
-      run: ulimit -S -n `ulimit -H -n` && make V=1 J=16 -j16 all
+      run: ulimit -S -n `ulimit -H -n` && make V=1 J=16 -j8 all
     - uses: "./.github/actions/post-steps"
   # ========================= MacOS with Tests ======================== #
   build-macos-cmake:
     if: ${{ github.repository_owner == 'facebook' }}
-    runs-on: macos-13
+    runs-on: macos-15-xlarge
     strategy:
       matrix:
-        run_even_tests: [true, false]
+        run_sharded_tests: [0, 1, 2, 3]
     steps:
     - uses: actions/checkout@v4.1.0
     - uses: maxim-lobanov/setup-xcode@v1.6.0
       with:
-        xcode-version: 14.3.1
+        xcode-version: 16.4.0
     - uses: "./.github/actions/increase-max-open-files-on-macos"
     - uses: "./.github/actions/install-gflags-on-macos"
     - uses: "./.github/actions/pre-steps-macos"
     - name: cmake generate project file
       run: ulimit -S -n `ulimit -H -n` && mkdir build && cd build && cmake -DWITH_GFLAGS=1 ..
     - name: Build tests
-      run: cd build && make V=1 -j16
-    - name: Run even tests
-      run: ulimit -S -n `ulimit -H -n` && cd build && ctest -j16 -I 0,,2
-      if: ${{ matrix.run_even_tests }}
-    - name: Run odd tests
-      run: ulimit -S -n `ulimit -H -n` && cd build && ctest -j16 -I 1,,2
-      if: ${{ ! matrix.run_even_tests  }}
+      run: cd build && make V=1 -j8
+    - name: Run shard 0 out of 4 test shards
+      run: ulimit -S -n `ulimit -H -n` && cd build && ctest -j8 -I 0,,4
+      if: ${{ matrix.run_sharded_tests == 0 }}
+    - name: Run shard 1 out of 4 test shards
+      run: ulimit -S -n `ulimit -H -n` && cd build && ctest -j8 -I 1,,4
+      if: ${{ matrix.run_sharded_tests == 1 }}
+    - name: Run shard 2 out of 4 test shards
+      run: ulimit -S -n `ulimit -H -n` && cd build && ctest -j8 -I 2,,4
+      if: ${{ matrix.run_sharded_tests == 2 }}
+    - name: Run shard 3 out of 4 test shards
+      run: ulimit -S -n `ulimit -H -n` && cd build && ctest -j8 -I 3,,4
+      if: ${{ matrix.run_sharded_tests == 3 }}
     - uses: "./.github/actions/post-steps"
   # ======================== Windows with Tests ======================= #
   # NOTE: some windows jobs are in "nightly" to save resources
   build-windows-vs2022:
     if: ${{ github.repository_owner == 'facebook' }}
-    runs-on: windows-2022
+    runs-on: windows-8-core
     env:
       CMAKE_GENERATOR: Visual Studio 17 2022
       CMAKE_PORTABLE: 1
@@ -444,7 +462,7 @@ jobs:
     # post-steps skipped because of compatibility issues with docker image
   build-macos-java:
     if: ${{ github.repository_owner == 'facebook' }}
-    runs-on: macos-13
+    runs-on: macos-15-xlarge
     env:
       JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home"
       ROCKSDB_DISABLE_JEMALLOC: 1
@@ -452,7 +470,7 @@ jobs:
     - uses: actions/checkout@v4.1.0
     - uses: maxim-lobanov/setup-xcode@v1.6.0
       with:
-        xcode-version: 14.3.1
+        xcode-version: 16.4.0
     - uses: "./.github/actions/increase-max-open-files-on-macos"
     - uses: "./.github/actions/install-gflags-on-macos"
     - uses: "./.github/actions/install-jdk8-on-macos"
@@ -467,14 +485,14 @@ jobs:
     - uses: "./.github/actions/post-steps"
   build-macos-java-static:
     if: ${{ github.repository_owner == 'facebook' }}
-    runs-on: macos-13
+    runs-on: macos-15-xlarge
     env:
       JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home"
     steps:
     - uses: actions/checkout@v4.1.0
     - uses: maxim-lobanov/setup-xcode@v1.6.0
       with:
-        xcode-version: 14.3.1
+        xcode-version: 16.4.0
     - uses: "./.github/actions/increase-max-open-files-on-macos"
     - uses: "./.github/actions/install-gflags-on-macos"
     - uses: "./.github/actions/install-jdk8-on-macos"
@@ -489,14 +507,14 @@ jobs:
     - uses: "./.github/actions/post-steps"
   build-macos-java-static-universal:
     if: ${{ github.repository_owner == 'facebook' }}
-    runs-on: macos-13
+    runs-on: macos-15-xlarge
     env:
       JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home"
     steps:
     - uses: actions/checkout@v4.1.0
     - uses: maxim-lobanov/setup-xcode@v1.6.0
       with:
-        xcode-version: 14.3.1
+        xcode-version: 16.4.0
     - uses: "./.github/actions/increase-max-open-files-on-macos"
     - uses: "./.github/actions/install-gflags-on-macos"
     - uses: "./.github/actions/install-jdk8-on-macos"

BUCK

@@ -425,6 +425,7 @@ rocks_cpp_library_wrapper(name="rocksdb_stress_lib", srcs=[
         "db_stress_tool/batched_ops_stress.cc",
         "db_stress_tool/cf_consistency_stress.cc",
         "db_stress_tool/db_stress_common.cc",
+        "db_stress_tool/db_stress_compaction_service.cc",
         "db_stress_tool/db_stress_compression_manager.cc",
         "db_stress_tool/db_stress_driver.cc",
         "db_stress_tool/db_stress_filters.cc",
@@ -4843,6 +4844,12 @@ cpp_unittest_wrapper(name="db_encryption_test",
             extra_compiler_flags=[])
 
 
+cpp_unittest_wrapper(name="db_etc3_test",
+            srcs=["db/db_etc3_test.cc"],
+            deps=[":rocksdb_test_lib"],
+            extra_compiler_flags=[])
+
+
 cpp_unittest_wrapper(name="db_flush_test",
             srcs=["db/db_flush_test.cc"],
             deps=[":rocksdb_test_lib"],
@@ -5193,6 +5200,12 @@ cpp_unittest_wrapper(name="import_column_family_test",
             extra_compiler_flags=[])
 
 
+cpp_unittest_wrapper(name="index_builder_test",
+            srcs=["table/block_based/index_builder_test.cc"],
+            deps=[":rocksdb_test_lib"],
+            extra_compiler_flags=[])
+
+
 cpp_unittest_wrapper(name="inlineskiplist_test",
             srcs=["memtable/inlineskiplist_test.cc"],
             deps=[":rocksdb_test_lib"],

CMakeLists.txt

@@ -203,9 +203,16 @@ if(WIN32 AND MSVC)
   endif()
 endif()
 
+option(WIN_CI "Accelerate build speed and reduce build artifect size for github CI with MSVC" OFF)
+
 if(MSVC)
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W4 /wd4127 /wd4996 /wd4100 /wd4324")
+  if(WIN_CI)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /MP /nologo /EHsc /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /W4 /wd4127 /wd4996 /wd4100 /wd4324 /wd4702")
+  else()
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W4 /wd4127 /wd4996 /wd4100 /wd4324")
+  endif()
 else()
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall -pthread")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wshadow -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers -Wno-strict-aliasing -Wno-invalid-offsetof")
@@ -313,7 +320,7 @@ if(NOT MSVC)
 endif()
 
 # Check if -latomic is required or not
-if (NOT MSVC)
+if (NOT MSVC AND NOT APPLE)
   CHECK_CXX_SOURCE_COMPILES("
 #include <atomic>
 std::atomic<uint64_t> x(0);
@@ -450,24 +457,33 @@ else()
   endif()
 endif()
 
-# Used to run CI build and tests so we can run faster
+# Used to run optimized debug build and tests so we can run faster
 option(OPTDBG "Build optimized debug build with MSVC" OFF)
 option(WITH_RUNTIME_DEBUG "build with debug version of runtime library" ON)
 if(MSVC)
-  if(OPTDBG)
+  if (WIN_CI)
     message(STATUS "Debug optimization is enabled")
     set(CMAKE_CXX_FLAGS_DEBUG "/Oxt")
+    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /DEBUG:FASTLINK")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG:FASTLINK")
   else()
-    set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1")
-
-    # Minimal Build is deprecated after MSVC 2015
-    if( MSVC_VERSION GREATER 1900 )
-      set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Gm-")
+    if(OPTDBG)
+      message(STATUS "Debug optimization is enabled")
+      set(CMAKE_CXX_FLAGS_DEBUG "/Oxt")
     else()
-      set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Gm")
-    endif()
+      set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1")
 
+      # Minimal Build is deprecated after MSVC 2015
+      if( MSVC_VERSION GREATER 1900 )
+        set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Gm-")
+      else()
+        set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Gm")
+      endif()
+    endif()
+    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /DEBUG")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG")
   endif()
+
   if(WITH_RUNTIME_DEBUG)
     set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /${RUNTIME_LIBRARY}d")
   else()
@@ -475,8 +491,6 @@ if(MSVC)
   endif()
   set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Oxt /Zp8 /Gm- /Gy /${RUNTIME_LIBRARY}")
 
-  set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /DEBUG")
-  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG")
 endif()
 
 if(CMAKE_COMPILER_IS_GNUCXX)
@@ -1345,6 +1359,7 @@ if(WITH_TESTS)
         db/db_clip_test.cc
         db/db_dynamic_level_test.cc
         db/db_encryption_test.cc
+        db/db_etc3_test.cc
         db/db_flush_test.cc
         db/db_inplace_update_test.cc
         db/db_io_failure_test.cc
@@ -1438,6 +1453,7 @@ if(WITH_TESTS)
         table/block_based/block_based_table_reader_test.cc
         table/block_based/block_test.cc
         table/block_based/data_block_hash_index_test.cc
+        table/block_based/index_builder_test.cc
         table/block_based/full_filter_block_test.cc
         table/block_based/partitioned_filter_block_test.cc
         table/cleanable_test.cc

Directory.Build.props

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project>
+    <PropertyGroup>
+        <CLToolExe>ccache_msvc_compiler.bat</CLToolExe>
+        <CLToolPath>$(MSBuildThisFileDirectory)</CLToolPath>
+        <UseMultiToolTask>true</UseMultiToolTask>
+        <EnforceProcessCountAcrossBuilds>true</EnforceProcessCountAcrossBuilds>
+    </PropertyGroup>
+</Project>

HISTORY.md

@@ -1,6 +1,68 @@
 # Rocksdb Change Log
 > NOTE: Entries for next release do not go here. Follow instructions in `unreleased_history/README.txt`
 
+## 10.8.3 (11/10/2025)
+### New Features
+* Add a GetColumnFamilyMetaData API variant in DB to get the SST files intersecting a given key range.
+
+## 10.8.2 (11/07/2025)
+### New Features
+* Added an auto-tuning feature for DB manifest file size that also (by default) improves the safety of existing configurations in case `max_manifest_file_size` is repeatedly exceeded. The new recommendation is to set `max_manifest_file_size` to something small like 1MB and tune `max_manifest_space_amp_pct` as needed to balance write amp and space amp in the manifest. Refer to comments on those options in `DBOptions` for details. Both options are (now) mutable.
+* Add a new option allow_trivial_move in CompactionOptions to allow CompactFiles to perform trivial move if possible. By default the flag of allow_trivial_move is false, so it preserve the original behavior.
+
+## 10.8.1 (10/29/2025)
+### Behavior Changes
+* PosixWritableFile now repositions the seek pointer to the new end of file after a call to Truncate.
+
+## 10.8.0 (10/21/2025)
+### New Features
+* Add kFSPrefetch to FSSupportedOps enum to allow file systems to indicate prefetch support capability, avoiding unnecessary prefetch system calls on file systems that don't support them.
+* Added experimental support `OpenAndCompactOptions::allow_resumption` for resumable compaction that persists progress during `OpenAndCompact()`, allowing interrupted compactions to resume from the last progress persitence. The default behavior is to not persist progress.
+
+### Public API Changes
+* Allow specifying output temperature in CompactionOptions
+* Added `DB::FlushWAL(const FlushWALOptions&)` as an alternative to `DB::FlushWAL(bool sync)`, where `FlushWALOptions` includes a new `rate_limiter_priority` field (default `Env::IO_TOTAL`) that allows rate limiting and priority passing of manual WAL flush's IO operations.
+* The MultiScan API contract is updated. After a multi scan range got prepared with Prepare API call, the following seeks must seek the start of each prepared scan range in order. In addition, when limit is set, upper bound must be set to the same value of limit before each seek
+
+### Behavior Changes
+* `kChangeTemperature` FIFO compaction will now honor `compaction_target_temp` to all levels regardless of `cf_options::last_level_temperature`
+* Allow UDIs with a non BytewiseComparator
+
+### Bug Fixes
+* Fix incorrect MultiScan seek error status due to bugs in handling range limit falling between adjacent SST files key range.
+* Fix a bug in Page unpinning in MultiScan
+
+### Performance Improvements
+* Fixed a performance regression in LZ4 compression that started in version 10.6.0
+
+## 10.7.0 (09/19/2025)
+### New Features
+* Add the fail_if_no_udi_on_open flag in BlockBasedTableOption to control whether a missing user defined index block in a SST is a hard error or not.
+* A new flag memtable_verify_per_key_checksum_on_seek is added to AdvancedColumnFamilyOptions. When it is enabled, it will validate key checksum along the binary search path on skiplist based memtable during seek operation.
+* Introduce option MultiScanArgs::use_async_io to enable asynchronous I/O during MultiScan, instead of waiting for I/O to be done in Prepare().
+* Add new option `MultiScanArgs::max_prefetch_size` that limits the memory usage of per file pinning of prefetched blocks.
+* Improved `sst_dump` by allowing standalone file and directory arguments without `--file=`. Also added new options and better output for `sst_dump --command=recompress`. See `sst_dump --help`
+
+### Public API Changes
+* HyperClockCache with no `estimated_entry_charge` is now production-ready and is the preferred block cache implementation vs. LRUCache. Please consider updating your code to minimize the risk of hitting performance bottlenecks or anomalies from LRUCache. See cache.h for more detail.
+* RocksDB now requires a C++20 compatible compiler (GCC >= 11, Clang >= 10, Visual Studio >= 2019), including for any code using RocksDB headers.
+* MultiScanArgs used to have a default constructor with default parameter of BytewiseComparator. Now it always requires Comparator in its constructor.
+
+### Behavior Changes
+* The default provided block cache implementation is now HyperClockCache instead of LRUCache, when `block_cache` is nullptr (default) and `no_block_cache==false` (default). We recommend explicitly creating a HyperClockCache block cache based on memory budget and sharing it across all column families and even DB instances. This change could expose previously hidden memory or resource leaks.
+
+### Bug Fixes
+* Reported numbers for compaction and flush CPU usage now include time spent by parallel compression worker threads. This now means compaction/flush CPU usage could exceed the wall clock time.
+* Fix a race condition in FIFO size-based compaction where concurrent threads could select the same non-L0 file, causing assertion failures in debug builds or "Cannot delete table file from LSM tree" errors in release builds.
+* Fix a bug in RocksDB MultiScan with UDI when one of the scan ranges is determined to be empty by the UDI, which causes incorrect results.
+
+### Performance Improvements
+* Add a new table property "rocksdb.key.smallest.seqno" which records the smallest sequence number of all keys in file. It makes ingesting DB generated files faster by
+avoiding scanning the whole file to find the smallest sequence number.
+* Add a new experimental PerKeyPointLockManager to improve efficiency under high lock contention. PointLockManager was not efficient when there is high write contention on same key, as it uses a single conditional variable per lock stripe. PerKeyPointLockManager uses per thread conditional variable supporting fifo order. Although this is an experimental feature. By default, it is disabled. A new boolean flag TransactionDBOptions::use_per_key_point_lock_mgr is added to optionally enable it. Search the flag in code for more info.
+Together, a new configuration TransactionOptions::deadlock_timeout_us is added, which allows the transaction to wait for a short period before perform deadlock detection. When the workload has low lock contention, the deadlock_timeout_us can be configured to be slightly higher than average transaction execution time, so that transaction would likely be able to take the lock before deadlock detection is performed when it is waiting for a lock. This allows transaction to reduce CPU cost on performing deadlock detection, which could be expensive in CPU time. When the workload has high lock contention, the deadlock_timeout_us can be configured to 0, so that transaction would perform deadlock detection immediately. By default the value is 0 to keep the behavior same as before.
+* Majorly improved CPU efficiency and scalability of parallel compression (`CompressionOptions::parallel_threads` > 1), though this efficiency improvement makes parallel compression currently incompatible with UserDefinedIndex and with old setting of `decouple_partitioned_filters=false`. Parallel compression is now considered a production-ready feature. Maximum performance is available with `-DROCKSDB_USE_STD_SEMAPHORES` at compile time, but this is not currently recommended because of reported bugs in implementations of `std::counting_semaphore`/`binary_semaphore`.
+
 ## 10.6.0 (08/22/2025)
 ### New Features
 * Introduce column family option `cf_allow_ingest_behind`. This option aims to replace `DBOptions::allow_ingest_behind` to enable ingest behind at the per-CF level. `DBOptions::allow_ingest_behind` is deprecated.
@@ -34,7 +96,7 @@
 * DB option skip_checking_sst_file_sizes_on_db_open is deprecated, in favor of validating file size in parallel in a thread pool, when db is opened. When DB is opened, with paranoid check enabled, a file with the wrong size would fail the DB open. With paranoid check disabled, the DB open would succeed, the column family with the corrupted file would not be read or write, while the other healthy column families could be read and write normally. When max_open_files option is not set to -1, only a subset of the files will be opened and checked. The rest of the files will be opened and checked when they are accessed.
 
 ### Behavior Changes
-* PessimisticTransaction::GetWaitingTxns now returns waiting transaction information even if the current transaction has timed out. This allows the information to be surfaced to users for debugging purposes once it is known that the timeout has occured.
+* PessimisticTransaction::GetWaitingTxns now returns waiting transaction information even if the current transaction has timed out. This allows the information to be surfaced to users for debugging purposes once it is known that the timeout has occurred.
 * A new API GetFileSize is added to FSRandomAccessFile interface class. It uses fstat vs stat on the posix implementation which is more efficient. Caller could use it to get file size faster. This function might be required in the future for FileSystem implementation outside of the RocksDB code base.
 * RocksDB now triggers eligible compactions every 12 hours when periodic compaction is configured. This solves a limitation of the compaction trigger mechanism, which would only trigger compaction after specific events like flush, compaction, or SetOptions.
 
@@ -91,7 +153,7 @@ system's prefetch) on SST file during compaction read
 * Deprecated API `DB::MaxMemCompactionLevel()`.
 * Deprecated `ReadOptions::ignore_range_deletions`.
 * Deprecated API `experimental::PromoteL0()`.
-* Added arbitrary string map for additional options to be overriden for remote compactions
+* Added arbitrary string map for additional options to be overridden for remote compactions
 * The fail_if_options_file_error option in DBOptions has been removed. The behavior now is to always return failure in any API that fails to persist the OPTIONS file.
 
 ### Behavior Changes
@@ -241,7 +303,7 @@ system's prefetch) on SST file during compaction read
 * In FIFO compaction, compactions for changing file temperature (configured by option `file_temperature_age_thresholds`) will compact one file at a time, instead of merging multiple eligible file together (#13018).
 * Support ingesting db generated files using hard link, i.e. IngestExternalFileOptions::move_files/link_files and IngestExternalFileOptions::allow_db_generated_files.
 * Add a new file ingestion option `IngestExternalFileOptions::link_files` to hard link input files and preserve original files links after ingestion.
-* DB::Close now untracks files in SstFileManager, making avaialble any space used
+* DB::Close now untracks files in SstFileManager, making available any space used
 by them. Prior to this change they would be orphaned until the DB is re-opened.
 
 ### Bug Fixes
@@ -437,7 +499,7 @@ MultiGetBenchmarks.multiGetList10 no_column_family 10000 16 100 1024 thrpt 25 76
 * Removed deprecated option `ColumnFamilyOptions::check_flush_compaction_key_order`
 * Remove the default `WritableFile::GetFileSize` and `FSWritableFile::GetFileSize` implementation that returns 0 and make it pure virtual, so that subclasses are enforced to explicitly provide an implementation.
 * Removed deprecated option `ColumnFamilyOptions::level_compaction_dynamic_file_size`
-* Removed tickers with typos "rocksdb.error.handler.bg.errro.count", "rocksdb.error.handler.bg.io.errro.count", "rocksdb.error.handler.bg.retryable.io.errro.count".
+* Removed tickers with typos "rocksdb.error.handler.bg.error.count", "rocksdb.error.handler.bg.io.error.count", "rocksdb.error.handler.bg.retryable.io.error.count".
 * Remove the force mode for `EnableFileDeletions` API because it is unsafe with no known legitimate use.
 * Removed deprecated option `ColumnFamilyOptions::ignore_max_compaction_bytes_for_input`
 * `sst_dump --command=check` now compares the number of records in a table with `num_entries` in table property, and reports corruption if there is a mismatch. API `SstFileDumper::ReadSequential()` is updated to optionally do this verification. (#12322)
@@ -464,7 +526,7 @@ MultiGetBenchmarks.multiGetList10 no_column_family 10000 16 100 1024 thrpt 25 76
 * Exposed options ttl via c api.
 
 ### Behavior Changes
-* `rocksdb.blobdb.blob.file.write.micros` expands to also measure time writing the header and footer. Therefore the COUNT may be higher and values may be smaller than before. For stacked BlobDB, it no longer measures the time of explictly flushing blob file.
+* `rocksdb.blobdb.blob.file.write.micros` expands to also measure time writing the header and footer. Therefore the COUNT may be higher and values may be smaller than before. For stacked BlobDB, it no longer measures the time of explicitly flushing blob file.
 * Files will be compacted to the next level if the data age exceeds periodic_compaction_seconds except for the last level.
 * Reduced the compaction debt ratio trigger for scheduling parallel compactions
 * For leveled compaction with default compaction pri (kMinOverlappingRatio), files marked for compaction will be prioritized over files not marked when picking a file from a level for compaction.
@@ -529,7 +591,7 @@ want to continue to use force enabling, they need to explicitly pass a `true` to
 
 ### Behavior Changes
 * During off-peak hours defined by `daily_offpeak_time_utc`, the compaction picker will select a larger number of files for periodic compaction. This selection will include files that are projected to expire by the next off-peak start time, ensuring that these files are not chosen for periodic compaction outside of off-peak hours.
-* If an error occurs when writing to a trace file after `DB::StartTrace()`, the subsequent trace writes are skipped to avoid writing to a file that has previously seen error. In this case, `DB::EndTrace()` will also return a non-ok status with info about the error occured previously in its status message.
+* If an error occurs when writing to a trace file after `DB::StartTrace()`, the subsequent trace writes are skipped to avoid writing to a file that has previously seen error. In this case, `DB::EndTrace()` will also return a non-ok status with info about the error occurred previously in its status message.
 * Deleting stale files upon recovery are delegated to SstFileManger if available so they can be rate limited.
 * Make RocksDB only call `TablePropertiesCollector::Finish()` once.
 * When `WAL_ttl_seconds > 0`, we now process archived WALs for deletion at least every `WAL_ttl_seconds / 2` seconds. Previously it could be less frequent in case of small `WAL_ttl_seconds` values when size-based expiration (`WAL_size_limit_MB > 0 `) was simultaneously enabled.
@@ -1317,7 +1379,7 @@ Note: The next release will be major release 7.0. See https://github.com/faceboo
 ### Public API change
 * Extend WriteBatch::AssignTimestamp and AssignTimestamps API so that both functions can accept an optional `checker` argument that performs additional checking on timestamp sizes.
 * Introduce a new EventListener callback that will be called upon the end of automatic error recovery.
-* Add IncreaseFullHistoryTsLow API so users can advance each column family's full_history_ts_low seperately.
+* Add IncreaseFullHistoryTsLow API so users can advance each column family's full_history_ts_low separately.
 * Add GetFullHistoryTsLow API so users can query current full_history_low value of specified column family.
 
 ### Performance Improvements

Makefile

@@ -1496,6 +1496,9 @@ db_test: $(OBJ_DIR)/db/db_test.o $(TEST_LIBRARY) $(LIBRARY)
 db_test2: $(OBJ_DIR)/db/db_test2.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
+db_etc3_test: $(OBJ_DIR)/db/db_etc3_test.o $(TEST_LIBRARY) $(LIBRARY)
+	$(AM_LINK)
+
 compression_test: $(OBJ_DIR)/util/compression_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
@@ -1739,6 +1742,9 @@ block_test: $(OBJ_DIR)/table/block_based/block_test.o $(TEST_LIBRARY) $(LIBRARY)
 data_block_hash_index_test: $(OBJ_DIR)/table/block_based/data_block_hash_index_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
+index_builder_test: $(OBJ_DIR)/table/block_based/index_builder_test.o $(TEST_LIBRARY) $(LIBRARY)
+	$(AM_LINK)
+
 inlineskiplist_test: $(OBJ_DIR)/memtable/inlineskiplist_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
@@ -2492,6 +2498,8 @@ commit_prereq:
 	false # J=$(J) build_tools/precommit_checker.py unit clang_unit release clang_release tsan asan ubsan lite unit_non_shm
 	# $(MAKE) clean && $(MAKE) jclean && $(MAKE) rocksdbjava;
 
+FOLLY_COMMIT_HASH = e95383b7c8b5b1e46cf47acf2f317d54f93c8268
+
 # For public CI runs, checkout folly in a way that can build with RocksDB.
 # This is mostly intended as a test-only simulation of Meta-internal folly
 # integration.
@@ -2503,7 +2511,7 @@ checkout_folly:
 	fi
 	@# Pin to a particular version for public CI, so that PR authors don't
 	@# need to worry about folly breaking our integration. Update periodically
-	cd third-party/folly && git reset --hard e95383b7c8b5b1e46cf47acf2f317d54f93c8268
+	cd third-party/folly && git reset --hard $(FOLLY_COMMIT_HASH)
 	@# Apparently missing include
 	perl -pi -e 's/(#include <atomic>)/$$1\n#include <cstring>/' third-party/folly/folly/lang/Exception.h
 	@# Warning-as-error on memcpy

cache/clock_cache.cc

@@ -2062,9 +2062,10 @@ AutoHyperClockTable::~AutoHyperClockTable() {
   }
   // This check can be extra expensive for a cache that is just created,
   // maybe used for a small number of entries, as in a unit test, and then
-  // destroyed. Only do this in rare modes.
+  // destroyed. Only do this in rare modes. REVISED: Don't scan the whole mmap,
+  // just a reasonable frontier past what we expect to have written.
 #ifdef MUST_FREE_HEAP_ALLOCATIONS
-  for (size_t i = used_end; i < array_.Count(); i++) {
+  for (size_t i = used_end; i < array_.Count() && i < used_end + 64U; i++) {
     assert(array_[i].head_next_with_shift.LoadRelaxed() == 0);
     assert(array_[i].chain_next_with_shift.LoadRelaxed() == 0);
     assert(array_[i].meta.LoadRelaxed() == 0);

cache/secondary_cache_adapter.cc

@@ -33,7 +33,7 @@ const char* kTieredCacheName = "TieredCache";
 // proportionally across the primary/secondary caches.
 //
 // The primary block cache is initially sized to the sum of the primary cache
-// budget + teh secondary cache budget, as follows -
+// budget + the secondary cache budget, as follows -
 //   |---------    Primary Cache Configured Capacity  -----------|
 //   |---Secondary Cache Budget----|----Primary Cache Budget-----|
 //
@@ -51,7 +51,7 @@ const char* kTieredCacheName = "TieredCache";
 // placeholder is counted against the primary cache. To compensate and count
 // a portion of it against the secondary cache, the secondary cache Deflate()
 // method is called to shrink it. Since the Deflate() causes the secondary
-// actual usage to shrink, it is refelcted here by releasing an equal amount
+// actual usage to shrink, it is reflected here by releasing an equal amount
 // from the pri_cache_res_ reservation. The Deflate() in the secondary cache
 // can be, but is not required to be, implemented using its own cache
 // reservation manager.
@@ -72,7 +72,7 @@ const char* kTieredCacheName = "TieredCache";
 // reservation is increased by an equal amount.
 //
 // Another way of implementing this would have been to simply split the user
-// reservation into primary and seconary components. However, this would
+// reservation into primary and secondary components. However, this would
 // require allocating a structure to track the associated secondary cache
 // reservation, which adds some complexity and overhead.
 //
@@ -621,7 +621,7 @@ Status CacheWithSecondaryAdapter::UpdateCacheReservationRatio(
   } else {
     // We're shrinking the ratio. Try to avoid unnecessary evictions -
     // 1. Lower the secondary cache capacity
-    // 2. Decrease pri_cache_res_ reservation to relect lower secondary
+    // 2. Decrease pri_cache_res_ reservation to reflect lower secondary
     //    cache utilization (decrease in capacity - decrease in share of cache
     //    reservations)
     // 3. Inflate the secondary cache to give it back the reduction in its

ccache_msvc_compiler.bat

@@ -0,0 +1 @@
+ccache.exe cl.exe %*

db/compact_files_test.cc

@@ -534,6 +534,232 @@ TEST_F(CompactFilesTest, GetCompactionJobInfo) {
   delete db;
 }
 
+// Helper function to generate zero-padded keys
+// e.g., MakeKey("a", 5) -> "a05", MakeKey("b", 42) -> "b42"
+static std::string MakeKey(const std::string& prefix, int index) {
+  return prefix + (index < 10 ? "0" : "") + std::to_string(index);
+}
+
+TEST_F(CompactFilesTest, TrivialMoveNonOverlappingFiles) {
+  Options options;
+  options.create_if_missing = true;
+  options.disable_auto_compactions = true;
+  options.compression = kNoCompression;
+  options.level_compaction_dynamic_level_bytes = false;
+
+  DB* db = nullptr;
+  ASSERT_OK(DestroyDB(db_name_, options));
+  Status s = DB::Open(options, db_name_, &db);
+  ASSERT_OK(s);
+  ASSERT_NE(db, nullptr);
+
+  // Create 3 non-overlapping files in L0
+  // File 1: keys [a00-a99]
+  for (int i = 0; i < 100; i++) {
+    std::string key = MakeKey("a", i);
+    ASSERT_OK(db->Put(WriteOptions(), key, "value_" + key));
+  }
+  ASSERT_OK(db->Flush(FlushOptions()));
+
+  // File 2: keys [b00-b99]
+  for (int i = 0; i < 100; i++) {
+    std::string key = MakeKey("b", i);
+    ASSERT_OK(db->Put(WriteOptions(), key, "value_" + key));
+  }
+  ASSERT_OK(db->Flush(FlushOptions()));
+
+  // File 3: keys [c00-c99]
+  for (int i = 0; i < 100; i++) {
+    std::string key = MakeKey("c", i);
+    ASSERT_OK(db->Put(WriteOptions(), key, "value_" + key));
+  }
+  ASSERT_OK(db->Flush(FlushOptions()));
+
+  // Verify files are in L0
+  ColumnFamilyMetaData meta;
+  db->GetColumnFamilyMetaData(&meta);
+  ASSERT_EQ(meta.levels[0].files.size(), 3);
+  ASSERT_EQ(meta.levels[1].files.size(), 0);
+
+  // Get L0 files
+  std::vector<std::string> l0_files;
+  for (const auto& file : meta.levels[0].files) {
+    l0_files.push_back(file.db_path + "/" + file.name);
+  }
+
+  CompactionOptions compact_option;
+  compact_option.allow_trivial_move = true;
+  // Compact all L0 files to L1 (non-overlapping in L1)
+  ASSERT_OK(db->CompactFiles(compact_option, l0_files, 1));
+
+  // Verify files are now in L1
+  db->GetColumnFamilyMetaData(&meta);
+  ASSERT_EQ(meta.levels[0].files.size(), 0);
+  ASSERT_EQ(meta.levels[1].files.size(), 3);
+
+  // Get the first file from L1 (should be the one with keys a00-a99)
+  std::string l1_file_to_move;
+  std::vector<std::string> l1_files_to_move_later;
+  uint64_t l1_file_number = 0;
+  for (const auto& file : meta.levels[1].files) {
+    if (file.smallestkey[0] == 'a') {
+      l1_file_to_move = file.db_path + "/" + file.name;
+      l1_file_number = file.file_number;
+    } else {
+      l1_files_to_move_later.push_back(file.db_path + "/" + file.name);
+    }
+  }
+  ASSERT_FALSE(l1_file_to_move.empty());
+
+  // Set up sync point to verify trivial move path is taken
+  bool trivial_move_executed = false;
+  SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::CompactFilesImpl:TrivialMove",
+      [&](void* /*arg*/) { trivial_move_executed = true; });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  // Move the file from L1 to L6 - this should be a trivial move
+  // because the file doesn't overlap with anything in L6
+  std::vector<std::string> files_to_move = {l1_file_to_move};
+  ASSERT_OK(db->CompactFiles(compact_option, files_to_move, 6));
+
+  // Verify trivial move was executed
+  ASSERT_TRUE(trivial_move_executed);
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+
+  // Verify the file is now in L6
+  db->GetColumnFamilyMetaData(&meta);
+  ASSERT_EQ(meta.levels[1].files.size(), 2);  // Two files remain in L1
+  ASSERT_EQ(meta.levels[6].files.size(), 1);  // One file in L6
+
+  // Verify it's the correct file in L6
+  bool found_file_in_l6 = false;
+  for (const auto& file : meta.levels[6].files) {
+    if (file.file_number == l1_file_number) {
+      found_file_in_l6 = true;
+      // Verify key range hasn't changed
+      ASSERT_EQ(file.smallestkey[0], 'a');
+      ASSERT_EQ(file.largestkey[0], 'a');
+      break;
+    }
+  }
+  ASSERT_TRUE(found_file_in_l6);
+
+  // Move the other 2 files from L1 to L6, with allow_trivial_move set to false.
+  // This will trigger a normal compaction, so the 2 files will be compacted
+  // into a single file in L6.
+  ASSERT_OK(db->CompactFiles(CompactionOptions(), l1_files_to_move_later, 6));
+
+  // Verify files in L6
+  db->GetColumnFamilyMetaData(&meta);
+  ASSERT_EQ(meta.levels[1].files.size(), 0);  // Zero files remain in L1
+  ASSERT_EQ(meta.levels[6].files.size(), 2);  // Two file in L6
+
+  // Verify data integrity - all keys should still be readable
+  for (int i = 0; i < 100; i++) {
+    std::string key = MakeKey("a", i);
+    std::string value;
+    ASSERT_OK(db->Get(ReadOptions(), key, &value));
+    ASSERT_EQ(value, "value_" + key);
+  }
+
+  delete db;
+}
+
+TEST_F(CompactFilesTest, TrivialMoveBlockedByOverlap) {
+  Options options;
+  options.create_if_missing = true;
+  options.disable_auto_compactions = true;
+  options.compression = kNoCompression;
+  options.level_compaction_dynamic_level_bytes = false;
+  options.num_levels = 7;
+
+  DB* db = nullptr;
+  ASSERT_OK(DestroyDB(db_name_, options));
+  Status s = DB::Open(options, db_name_, &db);
+  ASSERT_OK(s);
+  ASSERT_NE(db, nullptr);
+
+  // Create a file in L6 with keys [m00-m99] (wide range)
+  for (int i = 0; i < 100; i++) {
+    std::string key = MakeKey("m", i);
+    ASSERT_OK(db->Put(WriteOptions(), key, "value_" + key));
+  }
+  ASSERT_OK(db->Flush(FlushOptions()));
+
+  // Get L0 file
+  ColumnFamilyMetaData meta;
+  db->GetColumnFamilyMetaData(&meta);
+  std::vector<std::string> l0_files;
+  for (const auto& file : meta.levels[0].files) {
+    l0_files.push_back(file.db_path + "/" + file.name);
+  }
+
+  CompactionOptions compact_option;
+  compact_option.allow_trivial_move = true;
+
+  // Move to L6
+  ASSERT_OK(db->CompactFiles(compact_option, l0_files, 6));
+
+  // Now create a file in L1 with overlapping keys [m50-m60]
+  for (int i = 50; i <= 60; i++) {
+    std::string key = "m" + std::to_string(i);
+    ASSERT_OK(db->Put(WriteOptions(), key, "updated_value_" + key));
+  }
+  ASSERT_OK(db->Flush(FlushOptions()));
+
+  // Get the L0 file
+  db->GetColumnFamilyMetaData(&meta);
+  std::vector<std::string> l0_files_2;
+  for (const auto& file : meta.levels[0].files) {
+    l0_files_2.push_back(file.db_path + "/" + file.name);
+  }
+
+  // Move to L1
+  ASSERT_OK(db->CompactFiles(compact_option, l0_files_2, 1));
+
+  // Get the L1 file
+  db->GetColumnFamilyMetaData(&meta);
+  ASSERT_EQ(meta.levels[1].files.size(), 1);
+  std::string l1_file =
+      meta.levels[1].files[0].db_path + "/" + meta.levels[1].files[0].name;
+
+  // Set up sync point to verify full compaction path is taken
+  bool trivial_move_executed = false;
+  SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::CompactFilesImpl:TrivialMove",
+      [&](void* /*arg*/) { trivial_move_executed = true; });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  // Try to move from L1 to L6 - this should NOT be a trivial move
+  // because the file overlaps with the existing file in L6
+  ASSERT_OK(db->CompactFiles(compact_option, {l1_file}, 6));
+
+  // Verify trivial move was NOT executed (full compaction happened)
+  ASSERT_FALSE(trivial_move_executed);
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+
+  // Verify the result - should have merged data in L6
+  db->GetColumnFamilyMetaData(&meta);
+  ASSERT_EQ(meta.levels[1].files.size(), 0);  // L1 should be empty
+  // L6 should have the merged file (may be 1 file if merged, or 2 if not)
+  ASSERT_GE(meta.levels[6].files.size(), 1);
+
+  // Verify updated values are present
+  for (int i = 50; i <= 60; i++) {
+    std::string key = "m" + std::to_string(i);
+    std::string value;
+    ASSERT_OK(db->Get(ReadOptions(), key, &value));
+    ASSERT_EQ(value, "updated_value_" + key);
+  }
+
+  delete db;
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

db/compaction/compaction.cc

@@ -281,8 +281,9 @@ Compaction::Compaction(
     std::vector<CompactionInputFiles> _inputs, int _output_level,
     uint64_t _target_file_size, uint64_t _max_compaction_bytes,
     uint32_t _output_path_id, CompressionType _compression,
-    CompressionOptions _compression_opts, Temperature _output_temperature,
-    uint32_t _max_subcompactions, std::vector<FileMetaData*> _grandparents,
+    CompressionOptions _compression_opts,
+    Temperature _output_temperature_override, uint32_t _max_subcompactions,
+    std::vector<FileMetaData*> _grandparents,
     std::optional<SequenceNumber> _earliest_snapshot,
     const SnapshotChecker* _snapshot_checker,
     CompactionReason _compaction_reason, const std::string& _trim_ts,
@@ -303,7 +304,7 @@ Compaction::Compaction(
       output_path_id_(_output_path_id),
       output_compression_(_compression),
       output_compression_opts_(_compression_opts),
-      output_temperature_(_output_temperature),
+      output_temperature_override_(_output_temperature_override),
       deletion_compaction_(_compaction_reason == CompactionReason::kFIFOTtl ||
                            _compaction_reason ==
                                CompactionReason::kFIFOMaxSize),
@@ -1128,4 +1129,17 @@ void Compaction::FilterInputsForCompactionIterator() {
   }
 }
 
+Temperature Compaction::GetOutputTemperature(bool is_proximal_level) const {
+  if (output_temperature_override_ != Temperature::kUnknown) {
+    return output_temperature_override_;
+  }
+
+  if (is_last_level() && !is_proximal_level &&
+      mutable_cf_options_.last_level_temperature != Temperature::kUnknown) {
+    return mutable_cf_options_.last_level_temperature;
+  }
+
+  return mutable_cf_options_.default_write_temperature;
+}
+
 }  // namespace ROCKSDB_NAMESPACE

db/compaction/compaction.h

@@ -90,7 +90,8 @@ class Compaction {
              uint64_t target_file_size, uint64_t max_compaction_bytes,
              uint32_t output_path_id, CompressionType compression,
              CompressionOptions compression_opts,
-             Temperature output_temperature, uint32_t max_subcompactions,
+             Temperature output_temperature_override,
+             uint32_t max_subcompactions,
              std::vector<FileMetaData*> grandparents,
              std::optional<SequenceNumber> earliest_snapshot,
              const SnapshotChecker* snapshot_checker,
@@ -409,7 +410,11 @@ class Compaction {
 
   uint64_t max_compaction_bytes() const { return max_compaction_bytes_; }
 
-  Temperature output_temperature() const { return output_temperature_; }
+  // Order of precedence for temperature:
+  // 1. Override temp if not kUnknown
+  // 2. Temperature of the last level files if applicable
+  // 3. Default write temperature
+  Temperature GetOutputTemperature(bool is_proximal_level = false) const;
 
   uint32_t max_subcompactions() const { return max_subcompactions_; }
 
@@ -541,7 +546,7 @@ class Compaction {
   const uint32_t output_path_id_;
   CompressionType output_compression_;
   CompressionOptions output_compression_opts_;
-  Temperature output_temperature_;
+  Temperature output_temperature_override_;
   // If true, then the compaction can be done by simply deleting input files.
   const bool deletion_compaction_;
   // should it split the output file using the compact cursor?

db/compaction/compaction_iterator.h

@@ -270,6 +270,16 @@ class CompactionIterator {
   // true, unless `must_count_input_entries=true` was specified during iterator
   // creation (which ensures the count is always accurate).
   uint64_t NumInputEntryScanned() const { return input_.NumItered(); }
+
+  // Returns true if the current valid key was already scanned/counted during
+  // a lookahead operation in a previous iteration.
+  //
+  // REQUIRED: Valid() must be true
+  bool IsCurrentKeyAlreadyScanned() const {
+    assert(Valid());
+    return at_next_ || merge_out_iter_.Valid();
+  }
+
   Status InputStatus() const { return input_.status(); }
 
   bool IsDeleteRangeSentinelKey() const { return is_range_del_; }

db/compaction/compaction_job.cc

@@ -51,7 +51,9 @@
 #include "rocksdb/status.h"
 #include "rocksdb/table.h"
 #include "rocksdb/utilities/options_type.h"
+#include "table/format.h"
 #include "table/merging_iterator.h"
+#include "table/meta_blocks.h"
 #include "table/table_builder.h"
 #include "table/unique_id_impl.h"
 #include "test_util/sync_point.h"
@@ -253,7 +255,9 @@ void CompactionJob::ReportStartedCompaction(Compaction* compaction) {
 
 void CompactionJob::Prepare(
     std::optional<std::pair<std::optional<Slice>, std::optional<Slice>>>
-        known_single_subcompact) {
+        known_single_subcompact,
+    const CompactionProgress& compaction_progress,
+    log::Writer* compaction_progress_writer) {
   db_mutex_->AssertHeld();
   AutoThreadOperationStageUpdater stage_updater(
       ThreadStatus::STAGE_COMPACTION_PREPARE);
@@ -303,6 +307,9 @@ void CompactionJob::Prepare(
                                               /*sub_job_id*/ 0);
   }
 
+  MaybeAssignCompactionProgressAndWriter(compaction_progress,
+                                         compaction_progress_writer);
+
   // collect all seqno->time information from the input files which will be used
   // to encode seqno->time to the output files.
   SequenceNumber preserve_time_min_seqno = kMaxSequenceNumber;
@@ -401,6 +408,25 @@ void CompactionJob::Prepare(
   options_file_number_ = versions_->options_file_number();
 }
 
+void CompactionJob::MaybeAssignCompactionProgressAndWriter(
+    const CompactionProgress& compaction_progress,
+    log::Writer* compaction_progress_writer) {
+  // LIMITATION: Only supports resuming single subcompaction for now
+  if (compact_->sub_compact_states.size() != 1) {
+    return;
+  }
+
+  if (!compaction_progress.empty()) {
+    assert(compaction_progress.size() == 1);
+    SubcompactionState* sub_compact = &compact_->sub_compact_states[0];
+    const SubcompactionProgress& subcompaction_progress =
+        compaction_progress[0];
+    sub_compact->SetSubcompactionProgress(subcompaction_progress);
+  }
+
+  compaction_progress_writer_ = compaction_progress_writer;
+}
+
 uint64_t CompactionJob::GetSubcompactionsLimit() {
   return extra_num_subcompaction_threads_reserved_ +
          std::max(
@@ -786,20 +812,19 @@ Status CompactionJob::SyncOutputDirectories() {
 Status CompactionJob::VerifyOutputFiles() {
   Status status;
   std::vector<port::Thread> thread_pool;
-  std::vector<const CompactionOutputs::Output*> files_output;
-  for (const auto& state : compact_->sub_compact_states) {
-    for (const auto& output : state.GetOutputs()) {
-      files_output.emplace_back(&output);
-    }
-  }
   ColumnFamilyData* cfd = compact_->compaction->column_family_data();
-  std::atomic<size_t> next_file_idx(0);
-  auto verify_table = [&](Status& output_status) {
-    while (true) {
-      size_t file_idx = next_file_idx.fetch_add(1);
-      if (file_idx >= files_output.size()) {
-        break;
-      }
+  VerifyOutputFlags verify_output_flags =
+      compact_->compaction->mutable_cf_options().verify_output_flags;
+
+  // For backward compatibility
+  if (paranoid_file_checks_) {
+    verify_output_flags |= VerifyOutputFlags::kVerifyIteration;
+    verify_output_flags |= VerifyOutputFlags::kEnableForLocalCompaction;
+    verify_output_flags |= VerifyOutputFlags::kEnableForRemoteCompaction;
+  }
+
+  auto verify_table = [&](SubcompactionState& subcompaction_state) {
+    for (const auto& output_file : subcompaction_state.GetOutputs()) {
       // Verify that the table is usable
       // We set for_compaction to false and don't
       // OptimizeForCompactionTableRead here because this is a special case
@@ -808,13 +833,19 @@ Status CompactionJob::VerifyOutputFiles() {
       // verification as user reads since the goal is to cache it here for
       // further user reads
       ReadOptions verify_table_read_options(Env::IOActivity::kCompaction);
+      verify_table_read_options.verify_checksums = true;
+      verify_table_read_options.readahead_size =
+          file_options_for_read_.compaction_readahead_size;
+
+      std::unique_ptr<TableReader> table_reader_guard;
+      TableReader* table_reader_ptr = table_reader_guard.get();
       verify_table_read_options.rate_limiter_priority =
           GetRateLimiterPriority();
       InternalIterator* iter = cfd->table_cache()->NewIterator(
           verify_table_read_options, file_options_, cfd->internal_comparator(),
-          files_output[file_idx]->meta,
+          output_file.meta,
           /*range_del_agg=*/nullptr, compact_->compaction->mutable_cf_options(),
-          /*table_reader_ptr=*/nullptr,
+          /*table_reader_ptr=*/&table_reader_ptr,
           cfd->internal_stats()->GetFileReadHist(
               compact_->compaction->output_level()),
           TableReaderCaller::kCompactionRefill, /*arena=*/nullptr,
@@ -824,38 +855,63 @@ Status CompactionJob::VerifyOutputFiles() {
           /*largest_compaction_key=*/nullptr,
           /*allow_unprepared_value=*/false);
       auto s = iter->status();
+      if (s.ok()) {
+        // Check for remote/local compaction and verify_output_flags flags
+        const bool should_verify =
+            (subcompaction_state.compaction_job_stats.is_remote_compaction &&
+             !!(verify_output_flags &
+                VerifyOutputFlags::kEnableForRemoteCompaction)) ||
+            (!subcompaction_state.compaction_job_stats.is_remote_compaction &&
+             !!(verify_output_flags &
+                VerifyOutputFlags::kEnableForLocalCompaction));
 
-      if (s.ok() && paranoid_file_checks_) {
-        OutputValidator validator(cfd->internal_comparator(),
-                                  /*_enable_hash=*/true);
-        for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
-          s = validator.Add(iter->key(), iter->value());
-          if (!s.ok()) {
-            break;
+        if (should_verify) {
+          const bool should_verify_block_checksum =
+              !!(verify_output_flags & VerifyOutputFlags::kVerifyBlockChecksum);
+          const bool should_verify_iteration =
+              !!(verify_output_flags & VerifyOutputFlags::kVerifyIteration);
+          if (should_verify_block_checksum) {
+            assert(table_reader_ptr != nullptr);
+            // If verifying iteration as well, verify meta blocks here only to
+            // avoid redundant checks on data blocks
+            s = table_reader_ptr->VerifyChecksum(
+                verify_table_read_options, TableReaderCaller::kCompaction,
+                /*meta_blocks_only=*/should_verify_iteration);
+          }
+          if (s.ok() && should_verify_iteration) {
+            OutputValidator validator(cfd->internal_comparator(),
+                                      /*_enable_hash=*/true);
+            for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+              s = validator.Add(iter->key(), iter->value());
+              if (!s.ok()) {
+                break;
+              }
+            }
+            if (s.ok()) {
+              s = iter->status();
+            }
+            if (s.ok() && !validator.CompareValidator(output_file.validator)) {
+              s = Status::Corruption(
+                  "Key-value checksum of compaction output doesn't match what "
+                  "was computed when written");
+            }
           }
-        }
-        if (s.ok()) {
-          s = iter->status();
-        }
-        if (s.ok() &&
-            !validator.CompareValidator(files_output[file_idx]->validator)) {
-          s = Status::Corruption("Paranoid checksums do not match");
         }
       }
 
       delete iter;
 
       if (!s.ok()) {
-        output_status = s;
+        subcompaction_state.status = s;
         break;
       }
     }
   };
   for (size_t i = 1; i < compact_->sub_compact_states.size(); i++) {
     thread_pool.emplace_back(verify_table,
-                             std::ref(compact_->sub_compact_states[i].status));
+                             std::ref(compact_->sub_compact_states[i]));
   }
-  verify_table(compact_->sub_compact_states[0].status);
+  verify_table(compact_->sub_compact_states[0]);
   for (auto& thread : thread_pool) {
     thread.join();
   }
@@ -924,7 +980,8 @@ void CompactionJob::FinalizeCompactionRun(
     UpdateCompactionJobInputStatsFromInternalStats(internal_stats_,
                                                    num_input_range_del);
   }
-  UpdateCompactionJobOutputStatsFromInternalStats(internal_stats_);
+  UpdateCompactionJobOutputStatsFromInternalStats(input_status,
+                                                  internal_stats_);
   RecordCompactionIOStats();
 
   LogFlush(db_options_.info_log);
@@ -1249,8 +1306,8 @@ Status CompactionJob::SetupAndValidateCompactionFilter(
   return Status::OK();
 }
 
-void CompactionJob::InitializeReadOptions(
-    ColumnFamilyData* cfd, ReadOptions& read_options,
+void CompactionJob::InitializeReadOptionsAndBoundaries(
+    const size_t ts_sz, ReadOptions& read_options,
     SubcompactionKeyBoundaries& boundaries) {
   read_options.verify_checksums = true;
   read_options.fill_cache = false;
@@ -1264,8 +1321,6 @@ void CompactionJob::InitializeReadOptions(
 
   // Remove the timestamps from boundaries because boundaries created in
   // GenSubcompactionBoundaries doesn't strip away the timestamp.
-  const size_t ts_sz = cfd->user_comparator()->timestamp_size();
-
   if (boundaries.start.has_value()) {
     read_options.iterate_lower_bound = &(*boundaries.start);
     if (ts_sz > 0) {
@@ -1282,30 +1337,7 @@ void CompactionJob::InitializeReadOptions(
       read_options.iterate_upper_bound = &(*boundaries.end_without_ts);
     }
   }
-}
 
-InternalIterator* CompactionJob::CreateInputIterator(
-    SubcompactionState* sub_compact, ColumnFamilyData* cfd,
-    SubcompactionInternalIterators& iterators,
-    SubcompactionKeyBoundaries& boundaries, ReadOptions& read_options) {
-  // This is assigned after creation of SubcompactionState to simplify that
-  // creation across both CompactionJob and CompactionServiceCompactionJob
-  sub_compact->AssignRangeDelAggregator(
-      std::make_unique<CompactionRangeDelAggregator>(
-          &cfd->internal_comparator(), job_context_->snapshot_seqs,
-          &full_history_ts_low_, &trim_ts_));
-
-  InitializeReadOptions(cfd, read_options, boundaries);
-
-  // Although the v2 aggregator is what the level iterator(s) know about,
-  // the AddTombstones calls will be propagated down to the v1 aggregator.
-  iterators.raw_input =
-      std::unique_ptr<InternalIterator>(versions_->MakeInputIterator(
-          read_options, sub_compact->compaction, sub_compact->RangeDelAgg(),
-          file_options_for_read_, boundaries.start, boundaries.end));
-  InternalIterator* input = iterators.raw_input.get();
-
-  const size_t ts_sz = cfd->user_comparator()->timestamp_size();
   if (ts_sz > 0) {
     if (ts_sz <= strlen(boundaries.kMaxTs)) {
       boundaries.ts_slice = Slice(boundaries.kMaxTs, ts_sz);
@@ -1314,7 +1346,6 @@ InternalIterator* CompactionJob::CreateInputIterator(
       boundaries.ts_slice = Slice(boundaries.max_ts);
     }
   }
-
   if (boundaries.start.has_value()) {
     boundaries.start_ikey.SetInternalKey(*boundaries.start, kMaxSequenceNumber,
                                          kValueTypeForSeek);
@@ -1335,6 +1366,29 @@ InternalIterator* CompactionJob::CreateInputIterator(
     boundaries.end_internal_key = boundaries.end_ikey.GetInternalKey();
     boundaries.end_user_key = boundaries.end_ikey.GetUserKey();
   }
+}
+
+InternalIterator* CompactionJob::CreateInputIterator(
+    SubcompactionState* sub_compact, ColumnFamilyData* cfd,
+    SubcompactionInternalIterators& iterators,
+    SubcompactionKeyBoundaries& boundaries, ReadOptions& read_options) {
+  const size_t ts_sz = cfd->user_comparator()->timestamp_size();
+  InitializeReadOptionsAndBoundaries(ts_sz, read_options, boundaries);
+
+  // This is assigned after creation of SubcompactionState to simplify that
+  // creation across both CompactionJob and CompactionServiceCompactionJob
+  sub_compact->AssignRangeDelAggregator(
+      std::make_unique<CompactionRangeDelAggregator>(
+          &cfd->internal_comparator(), job_context_->snapshot_seqs,
+          &full_history_ts_low_, &trim_ts_));
+
+  // Although the v2 aggregator is what the level iterator(s) know about,
+  // the AddTombstones calls will be propagated down to the v1 aggregator.
+  iterators.raw_input =
+      std::unique_ptr<InternalIterator>(versions_->MakeInputIterator(
+          read_options, sub_compact->compaction, sub_compact->RangeDelAgg(),
+          file_options_for_read_, boundaries.start, boundaries.end));
+  InternalIterator* input = iterators.raw_input.get();
 
   if (boundaries.start.has_value() || boundaries.end.has_value()) {
     iterators.clip = std::make_unique<ClippingIterator>(
@@ -1404,7 +1458,8 @@ std::unique_ptr<CompactionIterator> CompactionJob::CreateCompactionIterator(
       env_, ShouldReportDetailedTime(env_, stats_), sub_compact->RangeDelAgg(),
       blob_resources.blob_file_builder.get(), db_options_.allow_data_in_errors,
       db_options_.enforce_single_del_contracts, manual_compaction_canceled_,
-      sub_compact->compaction->DoesInputReferenceBlobFiles(),
+      sub_compact->compaction
+          ->DoesInputReferenceBlobFiles() /* must_count_input_entries */,
       sub_compact->compaction, compaction_filter, shutting_down_,
       db_options_.info_log, full_history_ts_low, preserve_seqno_after_);
 }
@@ -1424,11 +1479,13 @@ CompactionJob::CreateFileHandlers(SubcompactionState* sub_compact,
 
   const CompactionFileCloseFunc close_file_func =
       [this, sub_compact, start_user_key, end_user_key](
-          CompactionOutputs& outputs, const Status& status,
-          const Slice& next_table_min_key) {
-        return this->FinishCompactionOutputFile(status, sub_compact, outputs,
-                                                next_table_min_key,
-                                                start_user_key, end_user_key);
+          const Status& status,
+          const ParsedInternalKey& prev_table_last_internal_key,
+          const Slice& next_table_min_key, const CompactionIterator* c_iter,
+          CompactionOutputs& outputs) {
+        return this->FinishCompactionOutputFile(
+            status, prev_table_last_internal_key, next_table_min_key,
+            start_user_key, end_user_key, c_iter, sub_compact, outputs);
       };
 
   return {open_file_func, close_file_func};
@@ -1442,6 +1499,9 @@ Status CompactionJob::ProcessKeyValue(
   const uint64_t kRecordStatsEvery = 1000;
   [[maybe_unused]] const std::optional<const Slice> end = sub_compact->end;
 
+  IterKey last_output_key;
+  ParsedInternalKey last_output_ikey;
+
   TEST_SYNC_POINT_CALLBACK(
       "CompactionJob::ProcessKeyValueCompaction()::Processing",
       static_cast<void*>(const_cast<Compaction*>(sub_compact->compaction)));
@@ -1491,8 +1551,9 @@ Status CompactionJob::ProcessKeyValue(
     // and `close_file_func`.
     // TODO: it would be better to have the compaction file open/close moved
     // into `CompactionOutputs` which has the output file information.
-    status = sub_compact->AddToOutput(*c_iter, use_proximal_output,
-                                      open_file_func, close_file_func);
+    status =
+        sub_compact->AddToOutput(*c_iter, use_proximal_output, open_file_func,
+                                 close_file_func, last_output_ikey);
     if (!status.ok()) {
       break;
     }
@@ -1500,6 +1561,10 @@ Status CompactionJob::ProcessKeyValue(
     TEST_SYNC_POINT_CALLBACK("CompactionJob::Run():PausingManualCompaction:2",
                              static_cast<void*>(const_cast<std::atomic<bool>*>(
                                  &manual_compaction_canceled_)));
+
+    last_output_key.SetInternalKey(c_iter->key(), &last_output_ikey);
+    last_output_ikey.sequence = ikey.sequence;
+    last_output_ikey.type = ikey.type;
     c_iter->Next();
 
 #ifndef NDEBUG
@@ -1684,6 +1749,22 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
   ReadOptions read_options;
   const WriteOptions write_options(Env::IOPriority::IO_LOW,
                                    Env::IOActivity::kCompaction);
+
+  InternalIterator* input_iter = CreateInputIterator(
+      sub_compact, cfd, iterators, boundaries, read_options);
+
+  assert(input_iter);
+
+  Status status =
+      MaybeResumeSubcompactionProgressOnInputIterator(sub_compact, input_iter);
+
+  if (status.IsNotFound()) {
+    input_iter->SeekToFirst();
+  } else if (!status.ok()) {
+    sub_compact->status = status;
+    return;
+  }
+
   MergeHelper merge(
       env_, cfd->user_comparator(), cfd->ioptions().merge_operator.get(),
       compaction_filter, db_options_.info_log.get(),
@@ -1692,11 +1773,6 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
       compact_->compaction->level(), db_options_.stats);
   BlobFileResources blob_resources;
 
-  InternalIterator* input_iter = CreateInputIterator(
-      sub_compact, cfd, iterators, boundaries, read_options);
-  assert(input_iter);
-  input_iter->SeekToFirst();
-
   auto c_iter =
       CreateCompactionIterator(sub_compact, cfd, input_iter, compaction_filter,
                                merge, blob_resources, write_options);
@@ -1711,9 +1787,8 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
   auto [open_file_func, close_file_func] =
       CreateFileHandlers(sub_compact, boundaries);
 
-  Status status =
-      ProcessKeyValue(sub_compact, cfd, c_iter.get(), open_file_func,
-                      close_file_func, prev_cpu_micros);
+  status = ProcessKeyValue(sub_compact, cfd, c_iter.get(), open_file_func,
+                           close_file_func, prev_cpu_micros);
 
   status = FinalizeProcessKeyValueStatus(cfd, input_iter, c_iter.get(), status);
 
@@ -1795,9 +1870,11 @@ void CompactionJob::RecordDroppedKeys(
 }
 
 Status CompactionJob::FinishCompactionOutputFile(
-    const Status& input_status, SubcompactionState* sub_compact,
-    CompactionOutputs& outputs, const Slice& next_table_min_key,
-    const Slice* comp_start_user_key, const Slice* comp_end_user_key) {
+    const Status& input_status,
+    const ParsedInternalKey& prev_table_last_internal_key,
+    const Slice& next_table_min_key, const Slice* comp_start_user_key,
+    const Slice* comp_end_user_key, const CompactionIterator* c_iter,
+    SubcompactionState* sub_compact, CompactionOutputs& outputs) {
   AutoThreadOperationStageUpdater stage_updater(
       ThreadStatus::STAGE_COMPACTION_SYNC_FILE);
   assert(sub_compact != nullptr);
@@ -1971,10 +2048,94 @@ Status CompactionJob::FinishCompactionOutputFile(
     }
   }
 
+  if (s.ok() && ShouldUpdateSubcompactionProgress(sub_compact, c_iter,
+                                                  prev_table_last_internal_key,
+                                                  next_table_min_key, meta)) {
+    UpdateSubcompactionProgress(c_iter, next_table_min_key, sub_compact);
+    s = PersistSubcompactionProgress(sub_compact);
+  }
   outputs.ResetBuilder();
   return s;
 }
 
+bool CompactionJob::ShouldUpdateSubcompactionProgress(
+    const SubcompactionState* sub_compact, const CompactionIterator* c_iter,
+    const ParsedInternalKey& prev_table_last_internal_key,
+    const Slice& next_table_min_internal_key, const FileMetaData* meta) const {
+  const auto* cfd = sub_compact->compaction->column_family_data();
+  // No need to update when the output will not get persisted
+  if (compaction_progress_writer_ == nullptr) {
+    return false;
+  }
+
+  // No need to update for a new empty output
+  if (meta == nullptr) {
+    return false;
+  }
+
+  // TODO(hx235): save progress even on the last output file
+  if (next_table_min_internal_key.empty()) {
+    return false;
+  }
+
+  // LIMITATION: Persisting compaction progress with timestamp
+  // is not supported since the feature of persisting timestamp of the key in
+  // SST files itself is still experimental
+  size_t ts_sz = cfd->user_comparator()->timestamp_size();
+  if (ts_sz > 0) {
+    return false;
+  }
+
+  // LIMITATION: Compaction progress persistence disabled for file boundaries
+  // contaning range deletions. Range deletions can span file boundaries, making
+  // it difficult (but possible) to ensure adjacent output tables have different
+  // user keys. See the last check for why different users keys of adjacent
+  // output tables are needed
+  const ValueType next_table_min_internal_key_type =
+      ExtractValueType(next_table_min_internal_key);
+  const ValueType prev_table_last_internal_key_type =
+      prev_table_last_internal_key.user_key.empty()
+          ? ValueType::kTypeValue
+          : prev_table_last_internal_key.type;
+
+  if (next_table_min_internal_key_type == ValueType::kTypeRangeDeletion ||
+      prev_table_last_internal_key_type == ValueType::kTypeRangeDeletion) {
+    return false;
+  }
+
+  // LIMITATION: Compaction progress persistence disabled when adjacent output
+  // tables share the same user key at boundaries. This ensures a simple Seek()
+  // of the next key when resuming can process all versions of a user key
+  const Slice next_table_min_user_key =
+      ExtractUserKey(next_table_min_internal_key);
+  const Slice prev_table_last_user_key =
+      prev_table_last_internal_key.user_key.empty()
+          ? Slice()
+          : prev_table_last_internal_key.user_key;
+
+  if (cfd->user_comparator()->EqualWithoutTimestamp(next_table_min_user_key,
+                                                    prev_table_last_user_key)) {
+    return false;
+  }
+
+  // LIMITATION: Don't save progress if the current key has already been scanned
+  // (looked ahead) in the input but not yet output. This can happen with merge
+  // operations, single deletes, and deletes at the bottommost level where
+  // CompactionIterator needs to look ahead to process multiple entries for the
+  // same user key before outputting a result. If we saved progress and resumed
+  // at this boundary, the resumed session would see and process the same input
+  // key again through Seek(), leading to incorrect double-counting in
+  // number of processed input entries and input count verification failure
+  //
+  // TODO(hx235): Offset num_processed_input_records to avoid double counting
+  // instead of disabling progress persistence.
+  if (c_iter->IsCurrentKeyAlreadyScanned()) {
+    return false;
+  }
+
+  return true;
+}
+
 Status CompactionJob::InstallCompactionResults(bool* compaction_released) {
   assert(compact_);
 
@@ -2120,15 +2281,8 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
 
   // Pass temperature of the last level files to FileSystem.
   FileOptions fo_copy = file_options_;
-  Temperature temperature = sub_compact->compaction->output_temperature();
-  Temperature last_level_temp =
-      sub_compact->compaction->mutable_cf_options().last_level_temperature;
-  // Here last_level_temperature supersedes default_write_temperature, when
-  // enabled and applicable
-  if (last_level_temp != Temperature::kUnknown &&
-      sub_compact->compaction->is_last_level() && !outputs.IsProximalLevel()) {
-    temperature = last_level_temp;
-  }
+  auto temperature =
+      sub_compact->compaction->GetOutputTemperature(outputs.IsProximalLevel());
   fo_copy.temperature = temperature;
   fo_copy.write_hint = write_hint_;
 
@@ -2404,6 +2558,7 @@ void CompactionJob::UpdateCompactionJobInputStatsFromInternalStats(
 }
 
 void CompactionJob::UpdateCompactionJobOutputStatsFromInternalStats(
+    const Status& status,
     const InternalStats::CompactionStatsFull& internal_stats) const {
   assert(job_stats_);
   job_stats_->elapsed_micros = internal_stats.output_level_stats.micros;
@@ -2434,7 +2589,7 @@ void CompactionJob::UpdateCompactionJobOutputStatsFromInternalStats(
         internal_stats.proximal_level_stats.num_output_files_blob;
   }
 
-  if (job_stats_->num_output_files > 0) {
+  if (status.ok() && job_stats_->num_output_files > 0) {
     CopyPrefix(compact_->SmallestUserKey(),
                CompactionJobStats::kMaxPrefixLength,
                &job_stats_->smallest_output_key_prefix);
@@ -2515,6 +2670,344 @@ Env::IOPriority CompactionJob::GetRateLimiterPriority() {
   return Env::IO_LOW;
 }
 
+Status CompactionJob::ReadTablePropertiesDirectly(
+    const ImmutableOptions& ioptions, const MutableCFOptions& moptions,
+    const FileMetaData* file_meta, const ReadOptions& read_options,
+    std::shared_ptr<const TableProperties>* tp) {
+  std::unique_ptr<FSRandomAccessFile> file;
+  std::string file_name = GetTableFileName(file_meta->fd.GetNumber());
+  Status s = ioptions.fs->NewRandomAccessFile(file_name, file_options_, &file,
+                                              nullptr /* dbg */);
+  if (!s.ok()) {
+    return s;
+  }
+
+  std::unique_ptr<RandomAccessFileReader> file_reader(
+      new RandomAccessFileReader(
+          std::move(file), file_name, ioptions.clock, io_tracer_,
+          ioptions.stats, Histograms::SST_READ_MICROS /* hist_type */,
+          nullptr /* file_read_hist */, ioptions.rate_limiter.get(),
+          ioptions.listeners));
+
+  std::unique_ptr<TableProperties> props;
+
+  uint64_t magic_number = kBlockBasedTableMagicNumber;
+
+  const auto* table_factory = moptions.table_factory.get();
+  if (table_factory == nullptr) {
+    return Status::Incomplete("Table factory is not set");
+  } else {
+    const auto& table_factory_name = table_factory->Name();
+    if (table_factory_name == TableFactory::kPlainTableName()) {
+      magic_number = kPlainTableMagicNumber;
+    } else if (table_factory_name == TableFactory::kCuckooTableName()) {
+      magic_number = kCuckooTableMagicNumber;
+    }
+  }
+
+  s = ReadTableProperties(file_reader.get(), file_meta->fd.GetFileSize(),
+                          magic_number, ioptions, read_options, &props);
+  if (!s.ok()) {
+    return s;
+  }
+
+  *tp = std::move(props);
+  return s;
+}
+
+Status CompactionJob::ReadOutputFilesTableProperties(
+    const autovector<FileMetaData>& output_files,
+    const ReadOptions& read_options,
+    std::vector<std::shared_ptr<const TableProperties>>&
+        output_files_table_properties,
+    bool is_proximal_level) {
+  assert(!output_files.empty());
+
+  static const char* level_type =
+      is_proximal_level ? "proximal output" : "output";
+
+  output_files_table_properties.reserve(output_files.size());
+
+  Status s;
+
+  for (const FileMetaData& metadata : output_files) {
+    std::shared_ptr<const TableProperties> tp;
+    s = ReadTablePropertiesDirectly(compact_->compaction->immutable_options(),
+                                    compact_->compaction->mutable_cf_options(),
+                                    &metadata, read_options, &tp);
+    if (!s.ok()) {
+      ROCKS_LOG_ERROR(
+          db_options_.info_log,
+          "Failed to read table properties for %s level output file #%" PRIu64
+          ": %s",
+          level_type, metadata.fd.GetNumber(), s.ToString().c_str());
+      return s;
+    }
+
+    if (tp == nullptr) {
+      ROCKS_LOG_ERROR(db_options_.info_log,
+                      "Empty table property for %s level output file #%" PRIu64
+                      "",
+                      level_type, metadata.fd.GetNumber());
+
+      s = Status::Corruption("Empty table property for " +
+                             std::string(level_type) +
+                             " level output files during resuming");
+      return s;
+    }
+    output_files_table_properties.push_back(tp);
+  }
+  return s;
+}
+
+void CompactionJob::RestoreCompactionOutputs(
+    const ColumnFamilyData* cfd,
+    const std::vector<std::shared_ptr<const TableProperties>>&
+        output_files_table_properties,
+    SubcompactionProgressPerLevel& subcompaction_progress_per_level,
+    CompactionOutputs* outputs_to_restore) {
+  assert(outputs_to_restore->GetOutputs().size() == 0);
+
+  const auto& output_files = subcompaction_progress_per_level.GetOutputFiles();
+
+  for (size_t i = 0; i < output_files.size(); i++) {
+    FileMetaData file_copy = output_files[i];
+
+    outputs_to_restore->AddOutput(std::move(file_copy),
+                                  cfd->internal_comparator(),
+                                  paranoid_file_checks_, true /* finished */);
+
+    outputs_to_restore->UpdateTableProperties(
+        *output_files_table_properties[i]);
+  }
+
+  outputs_to_restore->SetNumOutputRecords(
+      subcompaction_progress_per_level.GetNumProcessedOutputRecords());
+}
+
+// Attempt to resume compaction from a previously persisted compaction progress.
+//
+// RETURNS:
+// - Status::OK():
+// * Input iterator positioned at next unprocessed key
+// * CompactionOutputs objects fully restored for both output and proximal
+// output levels in SubcompactionState
+// * Compaction job statistics accurately reflect input and output records
+// processed for record count verification
+// * File number generation advanced to prevent conflicts with existing outputs
+// - Status::NotFound(): No valid progress to resume from
+// - Status::Corruption(): Resume key is invalid, beyond input range, or output
+// restoration failed
+// - Other non-OK status: Iterator errors or file system issues during
+// restoration
+//
+// The caller must check for Status::IsIncomplete() to distinguish between
+// "no resume needed" (proceed with `InternalIterator::SeekToFirst()`) vs
+// "resume failed" scenarios.
+Status CompactionJob::MaybeResumeSubcompactionProgressOnInputIterator(
+    SubcompactionState* sub_compact, InternalIterator* input_iter) {
+  const ReadOptions read_options(Env::IOActivity::kCompaction);
+  ColumnFamilyData* cfd = sub_compact->compaction->column_family_data();
+  SubcompactionProgress& subcompaction_progress =
+      sub_compact->GetSubcompactionProgressRef();
+
+  if (subcompaction_progress.output_level_progress
+              .GetNumProcessedOutputRecords() == 0 &&
+      subcompaction_progress.proximal_output_level_progress
+              .GetNumProcessedOutputRecords() == 0) {
+    return Status::NotFound("No subcompaction progress to resume");
+  }
+
+  ROCKS_LOG_INFO(db_options_.info_log, "[%s] [JOB %d] Resuming compaction : %s",
+                 cfd->GetName().c_str(), job_id_,
+                 subcompaction_progress.ToString().c_str());
+
+  input_iter->Seek(subcompaction_progress.next_internal_key_to_compact);
+
+  if (!input_iter->Valid()) {
+    ROCKS_LOG_ERROR(db_options_.info_log,
+                    "[%s] [JOB %d] Iterator is invalid after "
+                    "seeking to the key to resume. This indicates the key is "
+                    "incorrectly beyond the input data range.",
+                    cfd->GetName().c_str(), job_id_);
+    return Status::Corruption(
+        "The key to resume is beyond the input data range");
+  } else if (!input_iter->status().ok()) {
+    ROCKS_LOG_ERROR(db_options_.info_log,
+                    "[%s] [JOB %d] Iterator has error after seeking to "
+                    "the key to resume: %s",
+                    cfd->GetName().c_str(), job_id_,
+                    input_iter->status().ToString().c_str());
+    return Status::Corruption(
+        "Iterator has error status after seeking to the key: " +
+        input_iter->status().ToString());
+  }
+
+  sub_compact->compaction_job_stats.has_accurate_num_input_records =
+      subcompaction_progress.num_processed_input_records != 0;
+
+  sub_compact->compaction_job_stats.num_input_records =
+      subcompaction_progress.num_processed_input_records;
+
+  for (const bool& is_proximal_level : {false, true}) {
+    if (is_proximal_level &&
+        !sub_compact->compaction->SupportsPerKeyPlacement()) {
+      continue;
+    }
+
+    Status s;
+    SubcompactionProgressPerLevel& subcompaction_progress_per_level =
+        is_proximal_level
+            ? subcompaction_progress.proximal_output_level_progress
+            : subcompaction_progress.output_level_progress;
+
+    const auto& output_files =
+        subcompaction_progress_per_level.GetOutputFiles();
+
+    std::vector<std::shared_ptr<const TableProperties>>
+        output_files_table_properties;
+
+    // TODO(hx235): investigate if we can skip reading properties to save read
+    // IO
+    s = ReadOutputFilesTableProperties(output_files, read_options,
+                                       output_files_table_properties);
+    if (!s.ok()) {
+      ROCKS_LOG_ERROR(
+          db_options_.info_log,
+          "[%s] [JOB %d] Failed to read table properties for %s output level"
+          "files "
+          "during resume: %s.",
+          cfd->GetName().c_str(), job_id_, is_proximal_level ? "proximal" : "",
+          s.ToString().c_str());
+      return Status::Corruption(
+          "Not able to resume due to table property reading error " +
+          s.ToString());
+    }
+
+    RestoreCompactionOutputs(cfd, output_files_table_properties,
+                             subcompaction_progress_per_level,
+                             sub_compact->Outputs(is_proximal_level));
+
+    // Skip past all the used file numbers to avoid creating new output files
+    // after resumption that conflict with the existing output files
+    for (const auto& file_meta : output_files) {
+      uint64_t file_number = file_meta.fd.GetNumber();
+      while (versions_->NewFileNumber() <= file_number) {
+        versions_->FetchAddFileNumber(1);
+      }
+    }
+  }
+
+  return Status::OK();
+}
+
+void CompactionJob::UpdateSubcompactionProgress(
+    const CompactionIterator* c_iter, const Slice next_table_min_key,
+    SubcompactionState* sub_compact) {
+  assert(c_iter);
+  SubcompactionProgress& subcompaction_progress =
+      sub_compact->GetSubcompactionProgressRef();
+
+  IterKey next_ikey_to_compact;
+  next_ikey_to_compact.SetInternalKey(ExtractUserKey(next_table_min_key),
+                                      kMaxSequenceNumber, kValueTypeForSeek);
+  subcompaction_progress.next_internal_key_to_compact =
+      next_ikey_to_compact.GetInternalKey().ToString();
+
+  // Track total processed input records for progress reporting by combining:
+  // - Resumed count: records already processed before compaction was
+  // interrupted
+  // - Current count: records scanned in the current compaction session
+  // Only update when both tracking mechanisms provide accurate counts to ensure
+  // reliability.
+  subcompaction_progress.num_processed_input_records =
+      c_iter->HasNumInputEntryScanned() &&
+              sub_compact->compaction_job_stats.has_accurate_num_input_records
+          ? c_iter->NumInputEntryScanned() +
+                sub_compact->compaction_job_stats.num_input_records
+          : 0;
+
+  UpdateSubcompactionProgressPerLevel(
+      sub_compact, false /* is_proximal_level */, subcompaction_progress);
+
+  if (sub_compact->compaction->SupportsPerKeyPlacement()) {
+    UpdateSubcompactionProgressPerLevel(
+        sub_compact, true /* is_proximal_level */, subcompaction_progress);
+  }
+}
+
+void CompactionJob::UpdateSubcompactionProgressPerLevel(
+    SubcompactionState* sub_compact, bool is_proximal_level,
+    SubcompactionProgress& subcompaction_progress) {
+  SubcompactionProgressPerLevel& subcompaction_progress_per_level =
+      is_proximal_level ? subcompaction_progress.proximal_output_level_progress
+                        : subcompaction_progress.output_level_progress;
+
+  subcompaction_progress_per_level.SetNumProcessedOutputRecords(
+      sub_compact->OutputStats(is_proximal_level)->num_output_records);
+
+  const auto& prev_output_files =
+      subcompaction_progress_per_level.GetOutputFiles();
+
+  const auto& current_output_files =
+      sub_compact->Outputs(is_proximal_level)->GetOutputs();
+
+  for (size_t i = prev_output_files.size(); i < current_output_files.size();
+       i++) {
+    subcompaction_progress_per_level.AddToOutputFiles(
+        current_output_files[i].meta);
+  }
+}
+
+Status CompactionJob::PersistSubcompactionProgress(
+    SubcompactionState* sub_compact) {
+  SubcompactionProgress& subcompaction_progress =
+      sub_compact->GetSubcompactionProgressRef();
+
+  assert(compaction_progress_writer_);
+
+  VersionEdit edit;
+  edit.SetSubcompactionProgress(subcompaction_progress);
+
+  std::string record;
+  if (!edit.EncodeTo(&record)) {
+    ROCKS_LOG_ERROR(
+        db_options_.info_log,
+        "[%s] [JOB %d] Failed to encode subcompaction "
+        "progress",
+        compact_->compaction->column_family_data()->GetName().c_str(), job_id_);
+    return Status::Corruption("Failed to encode subcompaction progress");
+  }
+
+  WriteOptions write_options(Env::IOActivity::kCompaction);
+  Status s = compaction_progress_writer_->AddRecord(write_options, record);
+  IOOptions opts;
+  if (s.ok()) {
+    s = WritableFileWriter::PrepareIOOptions(write_options, opts);
+  }
+  if (s.ok()) {
+    s = compaction_progress_writer_->file()->Sync(opts, db_options_.use_fsync);
+  }
+
+  if (!s.ok()) {
+    ROCKS_LOG_ERROR(
+        db_options_.info_log,
+        "[%s] [JOB %d] Failed to persist subcompaction "
+        "progress: %s",
+        compact_->compaction->column_family_data()->GetName().c_str(), job_id_,
+        s.ToString().c_str());
+    return s;
+  }
+
+  subcompaction_progress.output_level_progress
+      .UpdateLastPersistedOutputFilesCount();
+
+  subcompaction_progress.proximal_output_level_progress
+      .UpdateLastPersistedOutputFilesCount();
+
+  return Status::OK();
+}
+
 Status CompactionJob::VerifyInputRecordCount(
     uint64_t num_input_range_del) const {
   size_t ts_sz = compact_->compaction->column_family_data()
@@ -2585,5 +3078,4 @@ Status CompactionJob::VerifyOutputRecordCount() const {
   }
   return Status::OK();
 }
-
 }  // namespace ROCKSDB_NAMESPACE

db/compaction/compaction_job.h

@@ -176,9 +176,20 @@ class CompactionJob {
   // and organizing seqno <-> time info. `known_single_subcompact` is non-null
   // if we already have a known single subcompaction, with optional key bounds
   // (currently for executing a remote compaction).
+  //
+  // @param compaction_progress Previously saved compaction progress
+  //   to resume from. If empty, compaction starts fresh from the
+  //   beginning.
+  //
+  // @param compaction_progress_writer Writer for persisting
+  //   subcompaction progress periodically during compaction
+  //   execution. If nullptr, progress tracking is disabled and compaction
+  //   cannot be resumed later.
   void Prepare(
       std::optional<std::pair<std::optional<Slice>, std::optional<Slice>>>
-          known_single_subcompact);
+          known_single_subcompact,
+      const CompactionProgress& compaction_progress = CompactionProgress{},
+      log::Writer* compaction_progress_writer = nullptr);
 
   // REQUIRED mutex not held
   // Launch threads for each subcompaction and wait for them to finish. After
@@ -197,6 +208,7 @@ class CompactionJob {
 
  protected:
   void UpdateCompactionJobOutputStatsFromInternalStats(
+      const Status& status,
       const InternalStats::CompactionStatsFull& internal_stats) const;
 
   void LogCompaction();
@@ -259,6 +271,10 @@ class CompactionJob {
   // consecutive groups such that each group has a similar size.
   void GenSubcompactionBoundaries();
 
+  void MaybeAssignCompactionProgressAndWriter(
+      const CompactionProgress& compaction_progress,
+      log::Writer* compaction_progress_writer);
+
   // Get the number of planned subcompactions based on max_subcompactions and
   // extra reserved resources
   uint64_t GetSubcompactionsLimit();
@@ -359,8 +375,9 @@ class CompactionJob {
       const CompactionFilter* configured_compaction_filter,
       const CompactionFilter*& compaction_filter,
       std::unique_ptr<CompactionFilter>& compaction_filter_from_factory);
-  void InitializeReadOptions(ColumnFamilyData* cfd, ReadOptions& read_options,
-                             SubcompactionKeyBoundaries& boundaries);
+  void InitializeReadOptionsAndBoundaries(
+      size_t ts_sz, ReadOptions& read_options,
+      SubcompactionKeyBoundaries& boundaries);
   InternalIterator* CreateInputIterator(
       SubcompactionState* sub_compact, ColumnFamilyData* cfd,
       SubcompactionInternalIterators& iterators,
@@ -411,12 +428,12 @@ class CompactionJob {
   // update the thread status for starting a compaction.
   void ReportStartedCompaction(Compaction* compaction);
 
-  Status FinishCompactionOutputFile(const Status& input_status,
-                                    SubcompactionState* sub_compact,
-                                    CompactionOutputs& outputs,
-                                    const Slice& next_table_min_key,
-                                    const Slice* comp_start_user_key,
-                                    const Slice* comp_end_user_key);
+  Status FinishCompactionOutputFile(
+      const Status& input_status,
+      const ParsedInternalKey& prev_table_last_internal_key,
+      const Slice& next_table_min_key, const Slice* comp_start_user_key,
+      const Slice* comp_end_user_key, const CompactionIterator* c_iter,
+      SubcompactionState* sub_compact, CompactionOutputs& outputs);
   Status InstallCompactionResults(bool* compaction_released);
   Status OpenCompactionOutputFile(SubcompactionState* sub_compact,
                                   CompactionOutputs& outputs);
@@ -493,6 +510,9 @@ class CompactionJob {
   // Setting this requires DBMutex.
   uint64_t options_file_number_ = 0;
 
+  // Writer for persisting compaction progress during compaction
+  log::Writer* compaction_progress_writer_ = nullptr;
+
   // Get table file name in where it's outputting to, which should also be in
   // `output_directory_`.
   virtual std::string GetTableFileName(uint64_t file_number);
@@ -500,6 +520,43 @@ class CompactionJob {
   // The Compaction Read and Write priorities are the same for different
   // scenarios, such as write stalled.
   Env::IOPriority GetRateLimiterPriority();
+
+  Status MaybeResumeSubcompactionProgressOnInputIterator(
+      SubcompactionState* sub_compact, InternalIterator* input_iter);
+
+  Status ReadOutputFilesTableProperties(
+      const autovector<FileMetaData>& temporary_output_file_allocation,
+      const ReadOptions& read_options,
+      std::vector<std::shared_ptr<const TableProperties>>&
+          output_files_table_properties,
+      bool is_proximal_level = false);
+
+  Status ReadTablePropertiesDirectly(
+      const ImmutableOptions& ioptions, const MutableCFOptions& moptions,
+      const FileMetaData* file_meta, const ReadOptions& read_options,
+      std::shared_ptr<const TableProperties>* tp);
+
+  void RestoreCompactionOutputs(
+      const ColumnFamilyData* cfd,
+      const std::vector<std::shared_ptr<const TableProperties>>&
+          output_files_table_properties,
+      SubcompactionProgressPerLevel& subcompaction_progress_per_level,
+      CompactionOutputs* outputs_to_restore);
+
+  bool ShouldUpdateSubcompactionProgress(
+      const SubcompactionState* sub_compact, const CompactionIterator* c_iter,
+      const ParsedInternalKey& prev_table_last_internal_key,
+      const Slice& next_table_min_internal_key, const FileMetaData* meta) const;
+
+  void UpdateSubcompactionProgress(const CompactionIterator* c_iter,
+                                   const Slice next_table_min_key,
+                                   SubcompactionState* sub_compact);
+
+  Status PersistSubcompactionProgress(SubcompactionState* sub_compact);
+
+  void UpdateSubcompactionProgressPerLevel(
+      SubcompactionState* sub_compact, bool is_proximal_level,
+      SubcompactionProgress& subcompaction_progress);
 };
 
 // CompactionServiceInput is used the pass compaction information between two
@@ -649,7 +706,9 @@ class CompactionServiceCompactionJob : private CompactionJob {
 
   // REQUIRED: mutex held
   // Like CompactionJob::Prepare()
-  void Prepare();
+  void Prepare(
+      const CompactionProgress& compaction_progress = CompactionProgress{},
+      log::Writer* compaction_progress_writer = nullptr);
 
   // Run the compaction in current thread and return the result
   Status Run();

db/compaction/compaction_job_test.cc

@@ -17,6 +17,7 @@
 #include "db/db_impl/db_impl.h"
 #include "db/error_handler.h"
 #include "db/version_set.h"
+#include "file/filename.h"
 #include "file/random_access_file_reader.h"
 #include "file/writable_file_writer.h"
 #include "options/options_helper.h"
@@ -210,12 +211,12 @@ class CompactionJobTestBase : public testing::Test {
         table_cache_(NewLRUCache(50000, 16)),
         write_buffer_manager_(db_options_.db_write_buffer_size),
         versions_(new VersionSet(
-            dbname_, &db_options_, env_options_, table_cache_.get(),
-            &write_buffer_manager_, &write_controller_,
+            dbname_, &db_options_, mutable_db_options_, env_options_,
+            table_cache_.get(), &write_buffer_manager_, &write_controller_,
             /*block_cache_tracer=*/nullptr,
             /*io_tracer=*/nullptr, /*db_id=*/"", /*db_session_id=*/"",
             /*daily_offpeak_time_utc=*/"",
-            /*error_handler=*/nullptr, /*read_only=*/false)),
+            /*error_handler=*/nullptr, /*unchanging=*/false)),
         shutting_down_(false),
         mock_table_factory_(new mock::MockTableFactory()),
         error_handler_(nullptr, db_options_, &mutex_),
@@ -545,13 +546,13 @@ class CompactionJobTestBase : public testing::Test {
     ASSERT_OK(s);
     db_options_.info_log = info_log;
 
-    versions_.reset(
-        new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(),
-                       &write_buffer_manager_, &write_controller_,
-                       /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr,
-                       test::kUnitTestDbId, /*db_session_id=*/"",
-                       /*daily_offpeak_time_utc=*/"",
-                       /*error_handler=*/nullptr, /*read_only=*/false));
+    versions_.reset(new VersionSet(
+        dbname_, &db_options_, mutable_db_options_, env_options_,
+        table_cache_.get(), &write_buffer_manager_, &write_controller_,
+        /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr,
+        test::kUnitTestDbId, /*db_session_id=*/"",
+        /*daily_offpeak_time_utc=*/"",
+        /*error_handler=*/nullptr, /*unchanging=*/false));
     compaction_job_stats_.Reset();
 
     VersionEdit new_db;
@@ -2409,6 +2410,450 @@ TEST_F(CompactionJobIOPriorityTest, GetRateLimiterPriority) {
                 Env::IO_LOW, Env::IO_LOW);
 }
 
+class ResumableCompactionJobTest : public CompactionJobTestBase {
+ public:
+  ResumableCompactionJobTest()
+      : CompactionJobTestBase(
+            test::PerThreadDBPath("allow_resumption_job_test"),
+            BytewiseComparator(), [](uint64_t /*ts*/) { return ""; },
+            /*test_io_priority=*/false, TableTypeForTest::kBlockBasedTable) {}
+
+ protected:
+  static constexpr const char* kCancelBeforeThisKey = "cancel_before_this_key";
+  std::string progress_dir_;
+  bool enable_cancel_ = false;
+  std::atomic<int> stop_count_{0};
+  std::atomic<bool> cancel_{false};
+
+  void SetUp() override {
+    CompactionJobTestBase::SetUp();
+    SyncPoint::GetInstance()->SetCallBack(
+        "CompactionOutputs::ShouldStopBefore::manual_decision",
+        [this](void* p) {
+          auto* pair = static_cast<std::pair<bool*, const Slice>*>(p);
+          *(pair->first) = true;
+
+          // Cancel after outputting a specific key
+          if (enable_cancel_) {
+            ParsedInternalKey parsed_key;
+            if (ParseInternalKey(pair->second, &parsed_key, true).ok()) {
+              if (parsed_key.user_key == kCancelBeforeThisKey) {
+                cancel_.store(true);
+              }
+            }
+          }
+        });
+    SyncPoint::GetInstance()->EnableProcessing();
+  }
+
+  void TearDown() override {
+    SyncPoint::GetInstance()->DisableProcessing();
+    SyncPoint::GetInstance()->ClearAllCallBacks();
+
+    if (env_->FileExists(progress_dir_).ok()) {
+      std::vector<std::string> files;
+      EXPECT_OK(env_->GetChildren(progress_dir_, &files));
+      for (const auto& file : files) {
+        if (file != "." && file != "..") {
+          EXPECT_OK(env_->DeleteFile(progress_dir_ + "/" + file));
+        }
+      }
+      EXPECT_OK(env_->DeleteDir(progress_dir_));
+    }
+
+    CompactionJobTestBase::TearDown();
+  }
+
+  void NewDB() {
+    if (env_->FileExists(progress_dir_).ok()) {
+      std::vector<std::string> files;
+      EXPECT_OK(env_->GetChildren(progress_dir_, &files));
+      for (const auto& file : files) {
+        if (file != "." && file != "..") {
+          EXPECT_OK(env_->DeleteFile(progress_dir_ + "/" + file));
+        }
+      }
+      EXPECT_OK(env_->DeleteDir(progress_dir_));
+    }
+
+    CompactionJobTestBase::NewDB();
+
+    progress_dir_ = test::PerThreadDBPath("compaction_progress");
+    ASSERT_OK(env_->CreateDirIfMissing(progress_dir_));
+  }
+
+  void EnableCompactionCancel() { enable_cancel_ = true; }
+
+  void DisableCompactionCancel() {
+    enable_cancel_ = false;
+    cancel_.store(false);
+  }
+
+  std::unique_ptr<log::Writer> CreateCompactionProgressWriter(
+      const std::string& compaction_progress_file) {
+    std::unique_ptr<FSWritableFile> file;
+    EXPECT_OK(fs_->NewWritableFile(compaction_progress_file, FileOptions(),
+                                   &file, nullptr));
+    auto file_writer = std::make_unique<WritableFileWriter>(
+        std::move(file), compaction_progress_file, FileOptions());
+    auto compaction_progress_writer =
+        std::make_unique<log::Writer>(std::move(file_writer), 0, false);
+    return compaction_progress_writer;
+  }
+
+  Status RunCompactionWithProgressTracking(
+      const CompactionProgress& compaction_progress,
+      log::Writer* compaction_progress_writer,
+      std::vector<SequenceNumber> snapshots = {},
+      std::shared_ptr<Statistics> stats = nullptr) {
+    mutex_.Lock();
+
+    auto cfd = versions_->GetColumnFamilySet()->GetDefault();
+    auto files = cfd->current()->storage_info()->LevelFiles(0);
+
+    db_options_.statistics = stats;
+    db_options_.stats = db_options_.statistics.get();
+
+    std::vector<CompactionInputFiles> compaction_input_files;
+    CompactionInputFiles level;
+    level.level = 0;
+    level.files = files;
+    compaction_input_files.push_back(level);
+
+    Compaction compaction(
+        cfd->current()->storage_info(), cfd->ioptions(),
+        cfd->GetLatestMutableCFOptions(), mutable_db_options_,
+        compaction_input_files, 1, mutable_cf_options_.target_file_size_base,
+        mutable_cf_options_.max_compaction_bytes, 0, kNoCompression,
+        cfd->GetLatestMutableCFOptions().compression_opts,
+        Temperature::kUnknown, 0, {}, std::nullopt, nullptr,
+        CompactionReason::kManualCompaction);
+    compaction.FinalizeInputInfo(cfd->current());
+
+    LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, db_options_.info_log.get());
+    EventLogger event_logger(db_options_.info_log.get());
+    JobContext job_context(1, false);
+    job_context.InitSnapshotContext(nullptr, nullptr, kMaxSequenceNumber,
+                                    std::move(snapshots));
+    CompactionJobStats job_stats;
+
+    CompactionJob compaction_job(
+        0, &compaction, db_options_, mutable_db_options_, env_options_,
+        versions_.get(), &shutting_down_, &log_buffer, nullptr, nullptr,
+        nullptr, stats.get(), &mutex_, &error_handler_, &job_context,
+        table_cache_, &event_logger, false, false, dbname_, &job_stats,
+        Env::Priority::USER, nullptr, cancel_, env_->GenerateUniqueId(),
+        DBImpl::GenerateDbSessionId(nullptr), "");
+
+    compaction_job.Prepare(std::nullopt, compaction_progress,
+                           compaction_progress_writer);
+    mutex_.Unlock();
+
+    compaction_job.Run().PermitUncheckedError();
+    EXPECT_OK(compaction_job.io_status());
+
+    mutex_.Lock();
+
+    bool compaction_released = false;
+    Status s = compaction_job.Install(&compaction_released);
+
+    mutex_.Unlock();
+    if (!compaction_released) {
+      compaction.ReleaseCompactionFiles(s);
+    }
+
+    return s;
+  }
+
+  SubcompactionProgress ReadAndParseProgress(
+      const std::string& compaction_progress_file) {
+    std::unique_ptr<FSSequentialFile> seq_file;
+    EXPECT_OK(fs_->NewSequentialFile(compaction_progress_file, FileOptions(),
+                                     &seq_file, nullptr));
+    auto file_reader = std::make_unique<SequentialFileReader>(
+        std::move(seq_file), compaction_progress_file, 0, nullptr);
+    log::Reader reader(nullptr, std::move(file_reader), nullptr, true, 0);
+
+    SubcompactionProgressBuilder builder;
+    std::string record;
+    Slice slice;
+
+    while (reader.ReadRecord(&slice, &record)) {
+      VersionEdit edit;
+      if (!edit.DecodeFrom(slice).ok()) {
+        continue;
+      }
+      builder.ProcessVersionEdit(edit);
+    }
+
+    EXPECT_TRUE(builder.HasAccumulatedSubcompactionProgress());
+
+    return builder.GetAccumulatedSubcompactionProgress();
+  }
+
+  // Test utility function to verify that compaction progress was correctly
+  // persisted to the progress file after compaction interruption.
+  //
+  // VERIFIES:
+  // - Progress file exists and has expected size (empty if no progress
+  // expected)
+  // - Next internal key to compact matches expected user key with proper format
+  // - Number of processed input records matches position in ordered input keys
+  // - Number of processed output records equals number of processed input
+  // records (by test design to simplify verification)
+  // - Each output file contains exactly one user key (by test design to
+  // simplify verification)
+  void VerifyCompactionProgressPersisted(
+      const std::string& compaction_progress_file,
+      const std::string& next_user_key_to_compact,
+      const std::vector<std::string>& ordered_intput_keys) {
+    ASSERT_OK(env_->FileExists(compaction_progress_file));
+
+    uint64_t file_size;
+    ASSERT_OK(env_->GetFileSize(compaction_progress_file, &file_size));
+
+    if (next_user_key_to_compact.empty()) {
+      ASSERT_EQ(file_size, 0);
+      return;
+    }
+
+    const auto& subcompaction_progress =
+        ReadAndParseProgress(compaction_progress_file);
+
+    ASSERT_FALSE(subcompaction_progress.next_internal_key_to_compact.empty());
+    ParsedInternalKey parsed_next_key;
+    ASSERT_OK(
+        ParseInternalKey(subcompaction_progress.next_internal_key_to_compact,
+                         &parsed_next_key, true /* log_err_key */));
+    ASSERT_EQ(parsed_next_key.user_key, next_user_key_to_compact);
+    ASSERT_EQ(parsed_next_key.sequence, kMaxSequenceNumber);
+    ASSERT_EQ(parsed_next_key.type, kValueTypeForSeek);
+
+    auto it = std::find(ordered_intput_keys.begin(), ordered_intput_keys.end(),
+                        next_user_key_to_compact);
+    ASSERT_TRUE(it != ordered_intput_keys.end());
+
+    auto next_key_index = std::distance(ordered_intput_keys.begin(), it);
+
+    ASSERT_EQ(subcompaction_progress.num_processed_input_records,
+              next_key_index);
+
+    ASSERT_EQ(subcompaction_progress.output_level_progress
+                  .GetNumProcessedOutputRecords(),
+              next_key_index);
+
+    ASSERT_EQ(
+        subcompaction_progress.output_level_progress.GetOutputFiles().size(),
+
+        next_key_index);
+
+    for (size_t i = 0;
+         i <
+         subcompaction_progress.output_level_progress.GetOutputFiles().size();
+         ++i) {
+      const auto& output_file =
+          subcompaction_progress.output_level_progress.GetOutputFiles()[i];
+      ASSERT_EQ(output_file.smallest.user_key().ToString(),
+                output_file.largest.user_key().ToString());
+      ASSERT_EQ(output_file.largest.user_key().ToString(),
+                ordered_intput_keys[i]);
+    }
+  }
+
+  void RunCancelAndResumeTest(
+      const std::initializer_list<mock::KVPair>& input_file_1,
+      const std::initializer_list<mock::KVPair>& input_file_2,
+      uint64_t last_sequence, const std::vector<uint64_t>& snapshots,
+      const std::string& expected_next_key_to_compact,
+      const std::vector<std::string>& expected_input_keys, bool exists_progress,
+      bool cancelled_past_mid_point = false) {
+    std::shared_ptr<Statistics> stats = ROCKSDB_NAMESPACE::CreateDBStatistics();
+
+    auto file1 = mock::MakeMockFile(input_file_1);
+    AddMockFile(file1);
+    auto file2 = mock::MakeMockFile(input_file_2);
+    AddMockFile(file2);
+    SetLastSequence(last_sequence);
+
+    // First compaction (will be cancelled)
+    std::string compaction_progress_file =
+        CompactionProgressFileName(progress_dir_, 123);
+    std::unique_ptr<log::Writer> compaction_progress_writer =
+        CreateCompactionProgressWriter(compaction_progress_file);
+
+    ASSERT_OK(stats->Reset());
+    EnableCompactionCancel();
+
+    Status status = RunCompactionWithProgressTracking(
+        CompactionProgress{}, compaction_progress_writer.get(), snapshots,
+        stats);
+
+    ASSERT_TRUE(status.IsManualCompactionPaused());
+    DisableCompactionCancel();
+
+    HistogramData cancelled_compaction_stats;
+    stats->histogramData(FILE_WRITE_COMPACTION_MICROS,
+                         &cancelled_compaction_stats);
+
+    VerifyCompactionProgressPersisted(compaction_progress_file,
+                                      expected_next_key_to_compact,
+                                      expected_input_keys);
+
+    // Resume compaction
+    CompactionProgress compaction_progress;
+    if (exists_progress) {
+      compaction_progress.push_back(
+          ReadAndParseProgress(compaction_progress_file));
+    }
+
+    std::string compaction_progress_file_2 =
+        CompactionProgressFileName(progress_dir_, 234);
+    std::unique_ptr<log::Writer> compaction_progress_writer_2 =
+        CreateCompactionProgressWriter(compaction_progress_file_2);
+
+    ASSERT_OK(stats->Reset());
+
+    status = RunCompactionWithProgressTracking(
+        compaction_progress, compaction_progress_writer_2.get(),
+        {} /* snapshots */, stats);
+
+    ASSERT_OK(status);
+
+    if (cancelled_past_mid_point) {
+      HistogramData resumed_compaction_stats;
+      stats->histogramData(FILE_WRITE_COMPACTION_MICROS,
+                           &resumed_compaction_stats);
+      ASSERT_GT(cancelled_compaction_stats.count,
+                resumed_compaction_stats.count);
+    }
+  }
+};
+
+TEST_F(ResumableCompactionJobTest, BasicProgressPersistence) {
+  NewDB();
+
+  auto file1 = mock::MakeMockFile({
+      {KeyStr("a", 1U, kTypeValue), "val1"},
+      {KeyStr("b", 2U, kTypeValue), "val2"},
+  });
+  AddMockFile(file1);
+
+  auto file2 = mock::MakeMockFile({
+      {KeyStr("c", 3U, kTypeValue), "val3"},
+      {KeyStr("d", 4U, kTypeValue), "val4"},
+  });
+  AddMockFile(file2);
+
+  SetLastSequence(4U);
+
+  std::string compaction_progress_file =
+      CompactionProgressFileName(progress_dir_, 123);
+
+  std::unique_ptr<log::Writer> compaction_progress_writer =
+      CreateCompactionProgressWriter(compaction_progress_file);
+
+  Status status = RunCompactionWithProgressTracking(
+      CompactionProgress(), compaction_progress_writer.get());
+
+  ASSERT_OK(status);
+
+  VerifyCompactionProgressPersisted(
+      compaction_progress_file, "d" /* next_user_key_to_compact */,
+      {"a", "b", "c", "d"} /* ordered_intput_keys */);
+}
+
+TEST_F(ResumableCompactionJobTest, BasicProgressResume) {
+  NewDB();
+
+  RunCancelAndResumeTest(
+      {{KeyStr("a", 1U, kTypeValue), "val1"},
+       {KeyStr("b", 2U, kTypeValue), "val2"}} /* input_file_1 */,
+      {{KeyStr("bb", 3U, kTypeValue), "val3"},
+       {KeyStr(kCancelBeforeThisKey, 4U, kTypeValue),
+        "val4"}} /* input_file_2 */,
+      4U /* last_sequence */, {} /* snapshots */,
+      kCancelBeforeThisKey /* expected_next_key_to_compact */,
+      {"a", "b", "bb", kCancelBeforeThisKey} /* expected_input_keys */,
+      true /* exists_progress */, true /* cancelled_past_mid_point*/);
+}
+
+TEST_F(ResumableCompactionJobTest, NoProgressResumeOnSameKey) {
+  NewDB();
+
+  RunCancelAndResumeTest(
+      {{KeyStr(kCancelBeforeThisKey, 1U, kTypeValue),
+        "val1"}} /* input_file_1 */,
+      {{KeyStr(kCancelBeforeThisKey, 2U, kTypeValue),
+        "val2"}} /* input_file_2 */,
+      2U /* last_sequence */, {1U} /* snapshots */,
+      "" /* expected_next_key_to_compact */,
+      {kCancelBeforeThisKey, kCancelBeforeThisKey} /* expected_input_keys */,
+      false /* exists_progress */);
+}
+
+TEST_F(ResumableCompactionJobTest, NoProgressResumeOnDeleteRange) {
+  NewDB();
+
+  RunCancelAndResumeTest(
+      {{KeyStr(kCancelBeforeThisKey, 1U, kTypeValue),
+        "val1"}} /* input_file_1 */,
+      {{KeyStr(kCancelBeforeThisKey, 2U, kTypeRangeDeletion),
+        "val2"}} /* input_file_2 */,
+      2U /* last_sequence */, {1U} /* snapshots */,
+      "" /* expected_next_key_to_compact */,
+      {kCancelBeforeThisKey, kCancelBeforeThisKey} /* expected_input_keys */,
+      false /* exists_progress */);
+}
+
+TEST_F(ResumableCompactionJobTest, NoProgressResumeOnMerge) {
+  merge_op_ = MergeOperators::CreateStringAppendOperator();
+  NewDB();
+
+  RunCancelAndResumeTest(
+      {{KeyStr("a", 1U, kTypeValue), "val1"},
+       {KeyStr("b", 2U, kTypeValue), "val2"}} /* input_file_1 */,
+      {{KeyStr("bb", 3U, kTypeValue), "val3"},
+       {KeyStr(kCancelBeforeThisKey, 4U, kTypeMerge),
+        "val4"}} /* input_file_2 */,
+      4U /* last_sequence */, {} /* snapshots */,
+      "bb" /* expected_next_key_to_compact */,
+      {"a", "b", "bb", kCancelBeforeThisKey} /* expected_input_keys */,
+      true /* exists_progress */);
+}
+
+TEST_F(ResumableCompactionJobTest, NoProgressResumeOnSingleDelete) {
+  NewDB();
+
+  RunCancelAndResumeTest(
+      {{KeyStr("a", 1U, kTypeValue), "val1"},
+       {KeyStr("b", 2U, kTypeValue), "val2"},
+       {KeyStr(kCancelBeforeThisKey, 3U, kTypeValue),
+        "val3"}} /* input_file_1 */,
+      {{KeyStr(kCancelBeforeThisKey, 4U, kTypeSingleDeletion), ""},
+       {KeyStr("d", 5U, kTypeValue), "val4"}} /* input_file_2 */,
+      5U /* last_sequence */, {3U} /* snapshots */,
+      "b" /* expected_next_key_to_compact */,
+      {"a", "b", kCancelBeforeThisKey, kCancelBeforeThisKey,
+       "d"} /* expected_input_keys */,
+      true /* exists_progress */);
+}
+
+TEST_F(ResumableCompactionJobTest, NoProgressResumeOnDeletionAtBottom) {
+  NewDB();
+
+  RunCancelAndResumeTest(
+      {{KeyStr("a", 1U, kTypeValue), "val1"},
+       {KeyStr("b", 2U, kTypeValue), "val2"},
+       {KeyStr(kCancelBeforeThisKey, 3U, kTypeValue),
+        "val3"}} /* input_file_1 */,
+      {{KeyStr(kCancelBeforeThisKey, 4U, kTypeDeletion), ""},
+       {KeyStr("d", 5U, kTypeValue), "val4"}} /* input_file_2 */,
+      5U /* last_sequence */, {3U} /* snapshots */,
+      "b" /* expected_next_key_to_compact */,
+      {"a", "b", kCancelBeforeThisKey, kCancelBeforeThisKey,
+       "d"} /* expected_input_keys */,
+      true /* exists_progress */);
+}
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

db/compaction/compaction_outputs.cc

@@ -359,7 +359,8 @@ bool CompactionOutputs::ShouldStopBefore(const CompactionIterator& c_iter) {
 Status CompactionOutputs::AddToOutput(
     const CompactionIterator& c_iter,
     const CompactionFileOpenFunc& open_file_func,
-    const CompactionFileCloseFunc& close_file_func) {
+    const CompactionFileCloseFunc& close_file_func,
+    const ParsedInternalKey& prev_table_last_internal_key) {
   Status s;
   bool is_range_del = c_iter.IsDeleteRangeSentinelKey();
   if (is_range_del && compaction_->bottommost_level()) {
@@ -370,7 +371,8 @@ Status CompactionOutputs::AddToOutput(
   }
   const Slice& key = c_iter.key();
   if (ShouldStopBefore(c_iter) && HasBuilder()) {
-    s = close_file_func(*this, c_iter.InputStatus(), key);
+    s = close_file_func(c_iter.InputStatus(), prev_table_last_internal_key, key,
+                        &c_iter, *this);
     if (!s.ok()) {
       return s;
     }

db/compaction/compaction_outputs.h

@@ -21,7 +21,8 @@ namespace ROCKSDB_NAMESPACE {
 class CompactionOutputs;
 using CompactionFileOpenFunc = std::function<Status(CompactionOutputs&)>;
 using CompactionFileCloseFunc =
-    std::function<Status(CompactionOutputs&, const Status&, const Slice&)>;
+    std::function<Status(const Status&, const ParsedInternalKey&, const Slice&,
+                         const CompactionIterator*, CompactionOutputs&)>;
 
 // Files produced by subcompaction, most of the functions are used by
 // compaction_job Open/Close compaction file functions.
@@ -58,6 +59,8 @@ class CompactionOutputs {
                           precalculated_hash, is_proximal_level_);
   }
 
+  const std::vector<Output>& GetOutputs() const { return outputs_; }
+
   // Set new table builder for the current output
   void NewBuilder(const TableBuilderOptions& tboptions);
 
@@ -195,6 +198,10 @@ class CompactionOutputs {
       std::pair<SequenceNumber, SequenceNumber> keep_seqno_range,
       const Slice& next_table_min_key, const std::string& full_history_ts_low);
 
+  void SetNumOutputRecords(uint64_t num_output_records) {
+    stats_.num_output_records = num_output_records;
+  }
+
  private:
   friend class SubcompactionState;
 
@@ -254,7 +261,8 @@ class CompactionOutputs {
   // close and open new compaction output with the functions provided.
   Status AddToOutput(const CompactionIterator& c_iter,
                      const CompactionFileOpenFunc& open_file_func,
-                     const CompactionFileCloseFunc& close_file_func);
+                     const CompactionFileCloseFunc& close_file_func,
+                     const ParsedInternalKey& prev_table_last_internal_key);
 
   // Close the current output. `open_file_func` is needed for creating new file
   // for range-dels only output file.
@@ -270,9 +278,12 @@ class CompactionOutputs {
         !range_del_agg->IsEmpty()) {
       status = open_file_func(*this);
     }
+
     if (HasBuilder()) {
+      const ParsedInternalKey empty_internal_key{};
       const Slice empty_key{};
-      Status s = close_file_func(*this, status, empty_key);
+      Status s = close_file_func(status, empty_internal_key, empty_key,
+                                 nullptr /* c_iter */, *this);
       if (!s.ok() && status.ok()) {
         status = s;
       }

db/compaction/compaction_picker.cc

@@ -375,12 +375,13 @@ Compaction* CompactionPicker::PickCompactionForCompactFiles(
     // without configurable `CompressionOptions`, which is inconsistent.
     compression_type = compact_options.compression;
   }
+
   auto c = new Compaction(
       vstorage, ioptions_, mutable_cf_options, mutable_db_options, input_files,
       output_level, compact_options.output_file_size_limit,
       mutable_cf_options.max_compaction_bytes, output_path_id, compression_type,
       GetCompressionOptions(mutable_cf_options, vstorage, output_level),
-      mutable_cf_options.default_write_temperature,
+      compact_options.output_temperature_override,
       compact_options.max_subcompactions,
       /* grandparents */ {}, earliest_snapshot, snapshot_checker,
       CompactionReason::kManualCompaction);
@@ -679,8 +680,7 @@ Compaction* CompactionPicker::PickCompactionForCompactRange(
         compact_range_options.target_path_id,
         GetCompressionType(vstorage, mutable_cf_options, output_level, 1),
         GetCompressionOptions(mutable_cf_options, vstorage, output_level),
-        mutable_cf_options.default_write_temperature,
-        compact_range_options.max_subcompactions,
+        Temperature::kUnknown, compact_range_options.max_subcompactions,
         /* grandparents */ {}, /* earliest_snapshot */ std::nullopt,
         /* snapshot_checker */ nullptr, CompactionReason::kManualCompaction,
         trim_ts, /* score */ -1,
@@ -871,8 +871,8 @@ Compaction* CompactionPicker::PickCompactionForCompactRange(
       GetCompressionType(vstorage, mutable_cf_options, output_level,
                          vstorage->base_level()),
       GetCompressionOptions(mutable_cf_options, vstorage, output_level),
-      mutable_cf_options.default_write_temperature,
-      compact_range_options.max_subcompactions, std::move(grandparents),
+      Temperature::kUnknown, compact_range_options.max_subcompactions,
+      std::move(grandparents),
       /* earliest_snapshot */ std::nullopt, /* snapshot_checker */ nullptr,
       CompactionReason::kManualCompaction, trim_ts, /* score */ -1,
       /* l0_files_might_overlap */ true,

db/compaction/compaction_picker_fifo.cc

@@ -124,8 +124,7 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction(
   Compaction* c = new Compaction(
       vstorage, ioptions_, mutable_cf_options, mutable_db_options,
       std::move(inputs), 0, 0, 0, 0, kNoCompression,
-      mutable_cf_options.compression_opts,
-      mutable_cf_options.default_write_temperature,
+      mutable_cf_options.compression_opts, Temperature::kUnknown,
       /* max_subcompactions */ 0, {}, /* earliest_snapshot */ std::nullopt,
       /* snapshot_checker */ nullptr, CompactionReason::kFIFOTtl,
       /* trim_ts */ "", vstorage->CompactionScore(0),
@@ -194,8 +193,7 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction(
             {comp_inputs}, 0, 16 * 1024 * 1024 /* output file size limit */,
             0 /* max compaction bytes, not applicable */,
             0 /* output path ID */, mutable_cf_options.compression,
-            mutable_cf_options.compression_opts,
-            mutable_cf_options.default_write_temperature,
+            mutable_cf_options.compression_opts, Temperature::kUnknown,
             0 /* max_subcompactions */, {},
             /* earliest_snapshot */ std::nullopt,
             /* snapshot_checker */ nullptr,
@@ -294,8 +292,7 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction(
       /* target_file_size */ 0,
       /* max_compaction_bytes */ 0,
       /* output_path_id */ 0, kNoCompression,
-      mutable_cf_options.compression_opts,
-      mutable_cf_options.default_write_temperature,
+      mutable_cf_options.compression_opts, Temperature::kUnknown,
       /* max_subcompactions */ 0, {}, /* earliest_snapshot */ std::nullopt,
       /* snapshot_checker */ nullptr, CompactionReason::kFIFOMaxSize,
       /* trim_ts */ "", vstorage->CompactionScore(0),

db/compaction/compaction_picker_level.cc

@@ -557,7 +557,7 @@ Compaction* LevelCompactionBuilder::GetCompaction() {
       GetCompressionType(vstorage_, mutable_cf_options_, output_level_,
                          vstorage_->base_level()),
       GetCompressionOptions(mutable_cf_options_, vstorage_, output_level_),
-      mutable_cf_options_.default_write_temperature,
+      Temperature::kUnknown,
       /* max_subcompactions */ 0, std::move(grandparents_),
       /* earliest_snapshot */ std::nullopt, /* snapshot_checker */ nullptr,
       compaction_reason_,

db/compaction/compaction_picker_test.cc

@@ -1178,7 +1178,7 @@ TEST_F(CompactionPickerTest, FIFOToCold1) {
     ASSERT_TRUE(compaction.get() != nullptr);
     ASSERT_EQ(compaction->compaction_reason(),
               CompactionReason::kChangeTemperature);
-    ASSERT_EQ(compaction->output_temperature(), Temperature::kCold);
+    ASSERT_EQ(compaction->GetOutputTemperature(), Temperature::kCold);
     ASSERT_EQ(1U, compaction->num_input_files(0));
     ASSERT_EQ(3U, compaction->input(0, 0)->fd.GetNumber());
   }
@@ -1248,7 +1248,7 @@ TEST_F(CompactionPickerTest, FIFOToColdMaxCompactionSize) {
     ASSERT_EQ(compaction->compaction_reason(),
               CompactionReason::kChangeTemperature);
     // Compaction picker picks older files first and picks one file at a time.
-    ASSERT_EQ(compaction->output_temperature(), Temperature::kCold);
+    ASSERT_EQ(compaction->GetOutputTemperature(), Temperature::kCold);
     ASSERT_EQ(1U, compaction->num_input_files(0));
     ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
   }
@@ -1316,7 +1316,7 @@ TEST_F(CompactionPickerTest, FIFOToColdWithExistingCold) {
     ASSERT_EQ(compaction->compaction_reason(),
               CompactionReason::kChangeTemperature);
     // Compaction picker picks older files first and picks one file at a time.
-    ASSERT_EQ(compaction->output_temperature(), Temperature::kCold);
+    ASSERT_EQ(compaction->GetOutputTemperature(), Temperature::kCold);
     ASSERT_EQ(1U, compaction->num_input_files(0));
     ASSERT_EQ(2U, compaction->input(0, 0)->fd.GetNumber());
   }
@@ -1383,7 +1383,7 @@ TEST_F(CompactionPickerTest, FIFOToColdWithHotBetweenCold) {
     ASSERT_TRUE(compaction.get() != nullptr);
     ASSERT_EQ(compaction->compaction_reason(),
               CompactionReason::kChangeTemperature);
-    ASSERT_EQ(compaction->output_temperature(), Temperature::kCold);
+    ASSERT_EQ(compaction->GetOutputTemperature(), Temperature::kCold);
     ASSERT_EQ(1U, compaction->num_input_files(0));
     ASSERT_EQ(2U, compaction->input(0, 0)->fd.GetNumber());
   }
@@ -1464,12 +1464,35 @@ TEST_F(CompactionPickerTest, FIFOToHotAndWarm) {
     ASSERT_EQ(compaction->compaction_reason(),
               CompactionReason::kChangeTemperature);
     // Compaction picker picks older files first and picks one file at a time.
-    ASSERT_EQ(compaction->output_temperature(), Temperature::kWarm);
+    ASSERT_EQ(compaction->GetOutputTemperature(), Temperature::kWarm);
     ASSERT_EQ(1U, compaction->num_input_files(0));
     ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
   }
 }
 
+TEST_F(CompactionPickerTest, CompactFilesOutputTemperature) {
+  NewVersionStorage(6, kCompactionStyleLevel);
+  auto file_number = 66U;
+  Add(0, file_number, "150", "200", 1000000000U);
+  UpdateVersionStorageInfo();
+
+  std::unordered_set<uint64_t> input{file_number};
+  std::vector<CompactionInputFiles> input_files;
+  ASSERT_OK(level_compaction_picker.GetCompactionInputsFromFileNumbers(
+      &input_files, &input, vstorage_.get(), CompactionOptions()));
+
+  auto compaction_options = CompactionOptions();
+  compaction_options.output_temperature_override = Temperature::kCold;
+
+  std::unique_ptr<Compaction> compaction(
+      level_compaction_picker.PickCompactionForCompactFiles(
+          compaction_options, input_files, 1, vstorage_.get(),
+          mutable_cf_options_, mutable_db_options_, /*output_path_id=*/0));
+
+  ASSERT_TRUE(compaction.get() != nullptr);
+  ASSERT_EQ(compaction->GetOutputTemperature(), Temperature::kCold);
+}
+
 TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) {
   NewVersionStorage(6, kCompactionStyleLevel);
   ioptions_.compaction_pri = kMinOverlappingRatio;

db/compaction/compaction_picker_universal.cc

@@ -1093,7 +1093,7 @@ Compaction* UniversalCompactionBuilder::PickCompactionToReduceSortedRuns(
                                            output_level, 1, enable_compression),
                         GetCompressionOptions(mutable_cf_options_, vstorage_,
                                               output_level, enable_compression),
-                        mutable_cf_options_.default_write_temperature,
+                        Temperature::kUnknown,
                         /* max_subcompactions */ 0, grandparents,
                         /* earliest_snapshot */ std::nullopt,
                         /* snapshot_checker */ nullptr, compaction_reason,
@@ -1441,7 +1441,7 @@ Compaction* UniversalCompactionBuilder::PickIncrementalForReduceSizeAmp(
                          true /* enable_compression */),
       GetCompressionOptions(mutable_cf_options_, vstorage_, output_level,
                             true /* enable_compression */),
-      mutable_cf_options_.default_write_temperature,
+      Temperature::kUnknown,
       /* max_subcompactions */ 0, /* grandparents */ {},
       /* earliest_snapshot */ std::nullopt,
       /* snapshot_checker */ nullptr,
@@ -1594,7 +1594,7 @@ Compaction* UniversalCompactionBuilder::PickDeleteTriggeredCompaction() {
       /* max_grandparent_overlap_bytes */ GetMaxOverlappingBytes(), path_id,
       GetCompressionType(vstorage_, mutable_cf_options_, output_level, 1),
       GetCompressionOptions(mutable_cf_options_, vstorage_, output_level),
-      mutable_cf_options_.default_write_temperature,
+      Temperature::kUnknown,
       /* max_subcompactions */ 0, grandparents, earliest_snapshot_,
       snapshot_checker_, CompactionReason::kFilesMarkedForCompaction,
       /* trim_ts */ "", score_,
@@ -1690,7 +1690,7 @@ Compaction* UniversalCompactionBuilder::PickCompactionWithSortedRunRange(
                          true /* enable_compression */),
       GetCompressionOptions(mutable_cf_options_, vstorage_, output_level,
                             true /* enable_compression */),
-      mutable_cf_options_.default_write_temperature,
+      Temperature::kUnknown,
       /* max_subcompactions */ 0, /* grandparents */ {},
       /* earliest_snapshot */ std::nullopt,
       /* snapshot_checker */ nullptr, compaction_reason,

db/compaction/compaction_service_job.cc

@@ -326,7 +326,9 @@ CompactionServiceCompactionJob::CompactionServiceCompactionJob(
       compaction_input_(compaction_service_input),
       compaction_result_(compaction_service_result) {}
 
-void CompactionServiceCompactionJob::Prepare() {
+void CompactionServiceCompactionJob::Prepare(
+    const CompactionProgress& compaction_progress,
+    log::Writer* compaction_progress_writer) {
   std::optional<Slice> begin;
   if (compaction_input_.has_begin) {
     begin = compaction_input_.begin;
@@ -335,7 +337,8 @@ void CompactionServiceCompactionJob::Prepare() {
   if (compaction_input_.has_end) {
     end = compaction_input_.end;
   }
-  CompactionJob::Prepare(std::make_pair(begin, end));
+  CompactionJob::Prepare(std::make_pair(begin, end), compaction_progress,
+                         compaction_progress_writer);
 }
 
 Status CompactionServiceCompactionJob::Run() {
@@ -408,7 +411,7 @@ Status CompactionServiceCompactionJob::Run() {
   // 2. Update job-level output stats with the aggregated internal_stats_
   // Please note that input stats will be updated by primary host when all
   // subcompactions are finished
-  UpdateCompactionJobOutputStatsFromInternalStats(internal_stats_);
+  UpdateCompactionJobOutputStatsFromInternalStats(status, internal_stats_);
   // and set fields that are not propagated as part of the update
   compaction_result_->stats.is_manual_compaction = c->is_manual_compaction();
   compaction_result_->stats.is_full_compaction = c->is_full_compaction();

db/compaction/compaction_service_test.cc

@@ -4,6 +4,7 @@
 //  (found in the LICENSE.Apache file in the root directory).
 
 #include "db/db_test_util.h"
+#include "file/file_util.h"
 #include "port/stack_trace.h"
 #include "rocksdb/utilities/options_util.h"
 #include "table/unique_id_impl.h"
@@ -16,17 +17,17 @@ class MyTestCompactionService : public CompactionService {
   MyTestCompactionService(
       std::string db_path, Options& options,
       std::shared_ptr<Statistics>& statistics,
-      std::vector<std::shared_ptr<EventListener>>& listeners,
+      std::vector<std::shared_ptr<EventListener>> listeners,
       std::vector<std::shared_ptr<TablePropertiesCollectorFactory>>
           table_properties_collector_factories)
       : db_path_(std::move(db_path)),
-        options_(options),
         statistics_(statistics),
+        options_(options),
         start_info_("na", "na", "na", 0, "na", 0, Env::TOTAL,
                     CompactionReason::kUnknown, false, false, false, -1, -1),
         wait_info_("na", "na", "na", 0, "na", 0, Env::TOTAL,
                    CompactionReason::kUnknown, false, false, false, -1, -1),
-        listeners_(listeners),
+        listeners_(std::move(listeners)),
         table_properties_collector_factories_(
             std::move(table_properties_collector_factories)) {}
 
@@ -72,6 +73,31 @@ class MyTestCompactionService : public CompactionService {
     if (is_override_wait_status_) {
       return override_wait_status_;
     }
+
+    CompactionServiceOptionsOverride options_override = GetOptionsOverride();
+
+    OpenAndCompactOptions options;
+    options.canceled = &canceled_;
+
+    Status s =
+        DB::OpenAndCompact(options, db_path_, GetOutputPath(scheduled_job_id),
+                           compaction_input, result, options_override);
+    {
+      InstrumentedMutexLock l(&mutex_);
+      if (is_override_wait_result_) {
+        *result = override_wait_result_;
+      }
+      result_ = *result;
+    }
+    compaction_num_.fetch_add(1);
+    if (s.ok()) {
+      return CompactionServiceJobStatus::kSuccess;
+    } else {
+      return CompactionServiceJobStatus::kFailure;
+    }
+  }
+
+  CompactionServiceOptionsOverride GetOptionsOverride() {
     CompactionServiceOptionsOverride options_override;
     options_override.env = options_.env;
     options_override.file_checksum_gen_factory =
@@ -94,26 +120,7 @@ class MyTestCompactionService : public CompactionService {
       options_override.table_properties_collector_factories =
           table_properties_collector_factories_;
     }
-
-    OpenAndCompactOptions options;
-    options.canceled = &canceled_;
-
-    Status s =
-        DB::OpenAndCompact(options, db_path_, db_path_ + "/" + scheduled_job_id,
-                           compaction_input, result, options_override);
-    {
-      InstrumentedMutexLock l(&mutex_);
-      if (is_override_wait_result_) {
-        *result = override_wait_result_;
-      }
-      result_ = *result;
-    }
-    compaction_num_.fetch_add(1);
-    if (s.ok()) {
-      return CompactionServiceJobStatus::kSuccess;
-    } else {
-      return CompactionServiceJobStatus::kFailure;
-    }
+    return options_override;
   }
 
   void CancelAwaitingJobs() override { canceled_ = true; }
@@ -160,14 +167,21 @@ class MyTestCompactionService : public CompactionService {
     return final_updated_status_.load();
   }
 
- private:
+ protected:
   InstrumentedMutex mutex_;
-  std::atomic_int compaction_num_{0};
+  const std::string db_path_;
+  std::shared_ptr<Statistics> statistics_;
   std::map<std::string, std::string> jobs_;
   std::map<std::string, CompactionServiceJobInfo> infos_;
-  const std::string db_path_;
+  std::string result_;
+
+  std::string GetOutputPath(const std::string& scheduled_job_id) {
+    return db_path_ + "/" + scheduled_job_id;
+  }
+
+ private:
+  std::atomic_int compaction_num_{0};
   Options options_;
-  std::shared_ptr<Statistics> statistics_;
   CompactionServiceJobInfo start_info_;
   CompactionServiceJobInfo wait_info_;
   bool is_override_start_status_ = false;
@@ -177,7 +191,6 @@ class MyTestCompactionService : public CompactionService {
   CompactionServiceJobStatus override_wait_status_ =
       CompactionServiceJobStatus::kFailure;
   bool is_override_wait_result_ = false;
-  std::string result_;
   std::string override_wait_result_;
   std::vector<std::shared_ptr<EventListener>> listeners_;
   std::vector<std::shared_ptr<TablePropertiesCollectorFactory>>
@@ -463,8 +476,6 @@ TEST_F(CompactionServiceTest, ManualCompaction) {
 
 TEST_F(CompactionServiceTest, StandaloneDeleteRangeTombstoneOptimization) {
   Options options = CurrentOptions();
-  options.compaction_style = CompactionStyle::kCompactionStyleUniversal;
-  ReopenWithCompactionService(&options);
 
   size_t num_files_after_filtered = 0;
   SyncPoint::GetInstance()->SetCallBack(
@@ -472,83 +483,97 @@ TEST_F(CompactionServiceTest, StandaloneDeleteRangeTombstoneOptimization) {
       [&](void* arg) {
         num_files_after_filtered = *static_cast<size_t*>(arg);
       });
-
   SyncPoint::GetInstance()->EnableProcessing();
 
-  std::vector<std::string> files;
-  {
-    // Writes first version of data in range partitioned files.
-    SstFileWriter sst_file_writer(EnvOptions(), options);
-    std::string file1 = dbname_ + "file1.sst";
-    ASSERT_OK(sst_file_writer.Open(file1));
-    ASSERT_OK(sst_file_writer.Put("a", "a1"));
-    ASSERT_OK(sst_file_writer.Put("b", "b1"));
-    ExternalSstFileInfo file1_info;
-    ASSERT_OK(sst_file_writer.Finish(&file1_info));
-    files.push_back(std::move(file1));
+  for (auto compaction_style : {CompactionStyle::kCompactionStyleLevel,
+                                CompactionStyle::kCompactionStyleUniversal}) {
+    SCOPED_TRACE("Style: " + std::to_string(compaction_style));
+    options.compaction_style = compaction_style;
+    ReopenWithCompactionService(&options);
 
-    std::string file2 = dbname_ + "file2.sst";
-    ASSERT_OK(sst_file_writer.Open(file2));
-    ASSERT_OK(sst_file_writer.Put("x", "x1"));
-    ASSERT_OK(sst_file_writer.Put("y", "y1"));
-    ExternalSstFileInfo file2_info;
-    ASSERT_OK(sst_file_writer.Finish(&file2_info));
-    files.push_back(std::move(file2));
+    num_files_after_filtered = 0;
+
+    std::vector<std::string> files;
+    {
+      // Writes first version of data in range partitioned files.
+      SstFileWriter sst_file_writer(EnvOptions(), options);
+      std::string file1 = dbname_ + "file1.sst";
+      ASSERT_OK(sst_file_writer.Open(file1));
+      ASSERT_OK(sst_file_writer.Put("a", "a1"));
+      ASSERT_OK(sst_file_writer.Put("b", "b1"));
+      ExternalSstFileInfo file1_info;
+      ASSERT_OK(sst_file_writer.Finish(&file1_info));
+      files.push_back(std::move(file1));
+
+      std::string file2 = dbname_ + "file2.sst";
+      ASSERT_OK(sst_file_writer.Open(file2));
+      ASSERT_OK(sst_file_writer.Put("x", "x1"));
+      ASSERT_OK(sst_file_writer.Put("y", "y1"));
+      ExternalSstFileInfo file2_info;
+      ASSERT_OK(sst_file_writer.Finish(&file2_info));
+      files.push_back(std::move(file2));
+    }
+
+    IngestExternalFileOptions ifo;
+    ASSERT_OK(db_->IngestExternalFile(files, ifo));
+    ASSERT_EQ(Get("a"), "a1");
+    ASSERT_EQ(Get("b"), "b1");
+    ASSERT_EQ(Get("x"), "x1");
+    ASSERT_EQ(Get("y"), "y1");
+    ASSERT_EQ(2, NumTableFilesAtLevel(6));
+
+    auto my_cs = GetCompactionService();
+    uint64_t comp_num = my_cs->GetCompactionNum();
+
+    {
+      // Atomically delete old version of data with one range delete file.
+      // And a new batch of range partitioned files with new version of data.
+      files.clear();
+      SstFileWriter sst_file_writer(EnvOptions(), options);
+      std::string file2 = dbname_ + "file2.sst";
+      ASSERT_OK(sst_file_writer.Open(file2));
+      ASSERT_OK(sst_file_writer.DeleteRange("a", "z"));
+      ExternalSstFileInfo file2_info;
+      ASSERT_OK(sst_file_writer.Finish(&file2_info));
+      files.push_back(std::move(file2));
+
+      std::string file3 = dbname_ + "file3.sst";
+      ASSERT_OK(sst_file_writer.Open(file3));
+      ASSERT_OK(sst_file_writer.Put("a", "a2"));
+      ASSERT_OK(sst_file_writer.Put("b", "b2"));
+      ExternalSstFileInfo file3_info;
+      ASSERT_OK(sst_file_writer.Finish(&file3_info));
+      files.push_back(std::move(file3));
+
+      std::string file4 = dbname_ + "file4.sst";
+      ASSERT_OK(sst_file_writer.Open(file4));
+      ASSERT_OK(sst_file_writer.Put("x", "x2"));
+      ASSERT_OK(sst_file_writer.Put("y", "y2"));
+      ExternalSstFileInfo file4_info;
+      ASSERT_OK(sst_file_writer.Finish(&file4_info));
+      files.push_back(std::move(file4));
+    }
+
+    ASSERT_OK(db_->IngestExternalFile(files, ifo));
+    ASSERT_OK(db_->WaitForCompact(WaitForCompactOptions()));
+    ASSERT_GE(my_cs->GetCompactionNum(), comp_num + 1);
+
+    CompactionServiceResult result;
+    my_cs->GetResult(&result);
+    ASSERT_OK(result.status);
+    ASSERT_TRUE(result.stats.is_manual_compaction);
+    ASSERT_TRUE(result.stats.is_remote_compaction);
+
+    if (compaction_style == kCompactionStyleUniversal) {
+      ASSERT_EQ(num_files_after_filtered, 1);
+    } else {
+      // Not filtered
+      ASSERT_EQ(num_files_after_filtered, 3);
+    }
+
+    Close();
   }
 
-  IngestExternalFileOptions ifo;
-  ASSERT_OK(db_->IngestExternalFile(files, ifo));
-  ASSERT_EQ(Get("a"), "a1");
-  ASSERT_EQ(Get("b"), "b1");
-  ASSERT_EQ(Get("x"), "x1");
-  ASSERT_EQ(Get("y"), "y1");
-  ASSERT_EQ(2, NumTableFilesAtLevel(6));
-
-  auto my_cs = GetCompactionService();
-  uint64_t comp_num = my_cs->GetCompactionNum();
-
-  {
-    // Atomically delete old version of data with one range delete file.
-    // And a new batch of range partitioned files with new version of data.
-    files.clear();
-    SstFileWriter sst_file_writer(EnvOptions(), options);
-    std::string file2 = dbname_ + "file2.sst";
-    ASSERT_OK(sst_file_writer.Open(file2));
-    ASSERT_OK(sst_file_writer.DeleteRange("a", "z"));
-    ExternalSstFileInfo file2_info;
-    ASSERT_OK(sst_file_writer.Finish(&file2_info));
-    files.push_back(std::move(file2));
-
-    std::string file3 = dbname_ + "file3.sst";
-    ASSERT_OK(sst_file_writer.Open(file3));
-    ASSERT_OK(sst_file_writer.Put("a", "a2"));
-    ASSERT_OK(sst_file_writer.Put("b", "b2"));
-    ExternalSstFileInfo file3_info;
-    ASSERT_OK(sst_file_writer.Finish(&file3_info));
-    files.push_back(std::move(file3));
-
-    std::string file4 = dbname_ + "file4.sst";
-    ASSERT_OK(sst_file_writer.Open(file4));
-    ASSERT_OK(sst_file_writer.Put("x", "x2"));
-    ASSERT_OK(sst_file_writer.Put("y", "y2"));
-    ExternalSstFileInfo file4_info;
-    ASSERT_OK(sst_file_writer.Finish(&file4_info));
-    files.push_back(std::move(file4));
-  }
-
-  ASSERT_OK(db_->IngestExternalFile(files, ifo));
-  ASSERT_OK(db_->WaitForCompact(WaitForCompactOptions()));
-  ASSERT_GE(my_cs->GetCompactionNum(), comp_num + 1);
-
-  CompactionServiceResult result;
-  my_cs->GetResult(&result);
-  ASSERT_OK(result.status);
-  ASSERT_TRUE(result.stats.is_manual_compaction);
-  ASSERT_TRUE(result.stats.is_remote_compaction);
-
-  ASSERT_EQ(num_files_after_filtered, 1);
-
-  Close();
   SyncPoint::GetInstance()->DisableProcessing();
 }
 
@@ -1022,6 +1047,7 @@ TEST_F(CompactionServiceTest, CorruptedOutputParanoidFileCheck) {
     Destroy(options);
     options.disable_auto_compactions = true;
     options.paranoid_file_checks = paranoid_file_check_enabled;
+    options.verify_output_flags = VerifyOutputFlags::kVerifyNone;
     ReopenWithCompactionService(&options);
     GenerateTestData();
 
@@ -1076,6 +1102,87 @@ TEST_F(CompactionServiceTest, CorruptedOutputParanoidFileCheck) {
   }
 }
 
+TEST_F(CompactionServiceTest, CorruptedOutputVerifyOutputFlags) {
+  for (VerifyOutputFlags verify_output_flags :
+       {VerifyOutputFlags::kVerifyNone,
+        VerifyOutputFlags::kEnableForLocalCompaction |
+            VerifyOutputFlags::kVerifyBlockChecksum,
+        VerifyOutputFlags::kEnableForRemoteCompaction |
+            VerifyOutputFlags::kVerifyBlockChecksum,
+        VerifyOutputFlags::kEnableForRemoteCompaction |
+            VerifyOutputFlags::kVerifyIteration,
+        VerifyOutputFlags::kVerifyAll}) {
+    SCOPED_TRACE(
+        "verify_output_flags=" +
+        std::to_string(static_cast<std::underlying_type_t<VerifyOutputFlags>>(
+            verify_output_flags)));
+
+    Options options = CurrentOptions();
+    Destroy(options);
+    options.disable_auto_compactions = true;
+    options.paranoid_file_checks = false;
+    options.verify_output_flags = verify_output_flags;
+    ReopenWithCompactionService(&options);
+    GenerateTestData();
+
+    auto my_cs = GetCompactionService();
+
+    std::string start_str = Key(15);
+    std::string end_str = Key(45);
+    Slice start(start_str);
+    Slice end(end_str);
+    uint64_t comp_num = my_cs->GetCompactionNum();
+
+    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+        "CompactionServiceCompactionJob::Run:0", [&](void* arg) {
+          CompactionServiceResult* compaction_result =
+              *(static_cast<CompactionServiceResult**>(arg));
+          ASSERT_TRUE(compaction_result != nullptr &&
+                      !compaction_result->output_files.empty());
+          // Corrupt files here
+          for (const auto& output_file : compaction_result->output_files) {
+            std::string file_name =
+                compaction_result->output_path + "/" + output_file.file_name;
+
+            // Corrupt very small range of bytes. This corruption is so small
+            // that this isn't caught by default light-weight check
+            ASSERT_OK(test::CorruptFile(env_, file_name, 0, 1,
+                                        false /* verifyChecksum */));
+          }
+        });
+    SyncPoint::GetInstance()->EnableProcessing();
+    const bool is_enabled_for_remote_compaction =
+        !!(verify_output_flags & VerifyOutputFlags::kEnableForRemoteCompaction);
+    const bool should_verify_block_checksum =
+        !!(verify_output_flags & VerifyOutputFlags::kVerifyBlockChecksum);
+    const bool should_verify_iteration =
+        !!(verify_output_flags & VerifyOutputFlags::kVerifyIteration);
+
+    Status s = db_->CompactRange(CompactRangeOptions(), &start, &end);
+    if (is_enabled_for_remote_compaction &&
+        (should_verify_block_checksum || should_verify_iteration)) {
+      ASSERT_NOK(s);
+      ASSERT_TRUE(s.IsCorruption());
+    } else {
+      // CompactRange() goes through if block checksum wasn't verified
+      ASSERT_OK(s);
+    }
+
+    ASSERT_GE(my_cs->GetCompactionNum(), comp_num + 1);
+
+    SyncPoint::GetInstance()->DisableProcessing();
+    SyncPoint::GetInstance()->ClearAllCallBacks();
+
+    // On the worker side, the compaction is considered success
+    // Verification is done on the primary side
+    CompactionServiceResult result;
+    my_cs->GetResult(&result);
+    ASSERT_OK(result.status);
+    ASSERT_TRUE(result.stats.is_manual_compaction);
+    ASSERT_TRUE(result.stats.is_remote_compaction);
+  }
+}
+
 TEST_F(CompactionServiceTest, TruncatedOutput) {
   Options options = CurrentOptions();
   options.disable_auto_compactions = true;
@@ -1993,6 +2100,338 @@ TEST_F(CompactionServiceTest, TablePropertiesCollector) {
   ASSERT_TRUE(has_user_property);
 }
 
+class ResumableCompactionService : public MyTestCompactionService {
+ public:
+  enum class TestScenario {
+    // Test scenario 1: Two-phase compaction with resumption
+    // - Phase 1: Cancel the compaction running with resumption enabled (saves
+    // progress)
+    // - Phase 2: Resume from saved progress and complete
+    // Validates: Resumption reduces redundant work
+    kCancelThenResume,
+
+    // Test scenario 2: Two-phase compaction without resumption
+    // - Phase 1: Cancel the compaction running with resumption enabled (saves
+    // progress)
+    // - Phase 2: Start fresh without resumption (ignores saved progress) and
+    // complete
+    // Validates: Disabling resumption causes full reprocessing
+    kCancelThenFreshStart,
+
+    // Test scenario 3: Three-phase compaction toggling resumption on/off/on
+    // - Phase 1: Cancel the compaction running with resumption enabled (saves
+    // progress)
+    // - Phase 2: Start fresh wtihout resumption (ignores saved progress) and
+    // cancel agains
+    // - Phase 3: Resume with resumption support (loads Phase 1's progress) and
+    // complete
+    // Validates: Resumption state can be toggled;
+    kMultipleCancelToggleResumption
+  };
+
+  ResumableCompactionService(const std::string& db_path, Options& options,
+                             std::shared_ptr<Statistics> statistics,
+                             TestScenario scenario)
+      : MyTestCompactionService(db_path, options, statistics,
+                                {} /* listeners */,
+                                {} /* table_properties_collector_factories */),
+        scenario_(scenario) {}
+
+  CompactionServiceJobStatus Wait(const std::string& scheduled_job_id,
+                                  std::string* result) override {
+    std::string compaction_input = ExtractCompactionInput(scheduled_job_id);
+    EXPECT_FALSE(compaction_input.empty());
+
+    OpenAndCompactOptions open_and_compaction_options;
+    auto override_options = GetOptionsOverride();
+
+    // Force creation of one key per output file for test simplicity.
+    // ASSUMPTION: This makes stats.count directly proportional to keys
+    // processed.
+    SyncPoint::GetInstance()->SetCallBack(
+        "CompactionOutputs::ShouldStopBefore::manual_decision", [](void* p) {
+          auto* pair = static_cast<std::pair<bool*, const Slice>*>(p);
+          *(pair->first) = true;
+        });
+    // Simulate cancelled compaction by overriding status at completion. So
+    // compaction processes all keys before this point to make stats.count
+    // comparison straightforward.
+    SyncPoint::GetInstance()->SetCallBack(
+        "DBImplSecondary::CompactWithoutInstallation::End", [&](void* status) {
+          auto s = static_cast<Status*>(status);
+          *s = Status::Incomplete(Status::SubCode::kManualCompactionPaused);
+        });
+    SyncPoint::GetInstance()->EnableProcessing();
+
+    // Phase 1: Run compaction with resumption enabled and cancel it
+    // - Processes all input keys
+    // - Creates output files and saves progress
+    // - Status overridden to "paused"
+    open_and_compaction_options.allow_resumption = true;
+    auto phase1_stats =
+        RunCancelledCompaction(open_and_compaction_options, scheduled_job_id,
+                               compaction_input, override_options);
+
+    HistogramData phase2_stats;
+
+    if (scenario_ == TestScenario::kMultipleCancelToggleResumption) {
+      // Phase 2: Run compaction WITHOUT resumption (fresh start) and cancel it
+      // - Delete all files left behind Phase 1 before calling OpenAndCompact()
+      // - Processes all input keys again from scratch
+      // - Creates output files but does NOT save progress
+      // - Status overridden to "paused"
+      open_and_compaction_options.allow_resumption = false;
+
+      // Clean up output folder for fresh start
+      std::string output_dir = GetOutputPath(scheduled_job_id);
+      Status cleanup_status = DestroyDir(override_options.env, output_dir);
+      EXPECT_TRUE(cleanup_status.ok());
+      EXPECT_OK(override_options.env->CreateDir(output_dir));
+
+      phase2_stats =
+          RunCancelledCompaction(open_and_compaction_options, scheduled_job_id,
+                                 compaction_input, override_options);
+
+      // Validation: Phase 2 starts from scratch, so it processes the same
+      // input keys as Phase 1.
+      // ASSUMPTION: With fixed input (10 keys) and deterministic cancellation
+      // (after processing), both phases create the same number of output files.
+      EXPECT_EQ(phase2_stats.count, phase1_stats.count);
+    }
+
+    SyncPoint::GetInstance()->ClearCallBack(
+        "DBImplSecondary::CompactWithoutInstallation::End");
+
+    // Final phase: Run compaction to completion (no cancellation)
+    if (scenario_ == TestScenario::kMultipleCancelToggleResumption) {
+      // Attempt to resume but it ends up starting fresh
+      open_and_compaction_options.allow_resumption = true;
+    } else if (scenario_ == TestScenario::kCancelThenResume) {
+      // Resume from Phase 1's saved progress
+      open_and_compaction_options.allow_resumption = true;
+    } else {  // kCancelThenFreshStart
+      // Start fresh without resumption
+      open_and_compaction_options.allow_resumption = false;
+
+      // Clean up output folder for fresh start
+      std::string output_dir = GetOutputPath(scheduled_job_id);
+      Status cleanup_status = DestroyDir(override_options.env, output_dir);
+      EXPECT_TRUE(cleanup_status.ok());
+      EXPECT_OK(override_options.env->CreateDir(output_dir));
+    }
+
+    auto final_phase_stats =
+        RunCompaction(open_and_compaction_options, scheduled_job_id,
+                      compaction_input, override_options, result);
+
+    SyncPoint::GetInstance()->DisableProcessing();
+    SyncPoint::GetInstance()->ClearAllCallBacks();
+
+    // Validate statistics based on scenario
+    if (scenario_ == TestScenario::kMultipleCancelToggleResumption) {
+      // ASSUMPTION: Phase 1 processes all keys before cancellation
+      EXPECT_GT(phase1_stats.count, 0);
+
+      // ASSUMPTION: Phase 2 runs with allow_resumption=false and an empty
+      // folder. Phase 2 then creates its own output files (but doesn't save
+      // progress). When Phase 3 starts with allow_resumption=true, it finds no
+      // progress file exists, so it cannot resume and must start from scratch,
+      // processing all input keys again.
+      // Result: Phase 3 does the same amount of work as Phase 1.
+      EXPECT_EQ(final_phase_stats.count, phase1_stats.count);
+
+    } else if (scenario_ == TestScenario::kCancelThenResume) {
+      // ASSUMPTION: Phase 1 processes all keys before cancellation
+      EXPECT_GT(phase1_stats.count, 0);
+
+      // ASSUMPTION: Phase 1 processes all keys and saves progress before
+      // cancellation. Final phase resumes from Phase 1's saved progress.
+      // Since Phase 1 completed all processing before being cancelled, the
+      // final phase should do less work than Phase 1.
+      EXPECT_LT(final_phase_stats.count, phase1_stats.count);
+
+    } else {  // kCancelThenFreshStart
+      // ASSUMPTION: Phase 1 processes all keys before cancellation
+      EXPECT_GT(phase1_stats.count, 0);
+
+      // ASSUMPTION: Final phase starts fresh without resumption, so it
+      // processes all input keys again and creates the same number of files
+      EXPECT_EQ(final_phase_stats.count, phase1_stats.count);
+    }
+
+    StoreResult(*result);
+
+    return CompactionServiceJobStatus::kSuccess;
+  }
+
+ private:
+  std::string ExtractCompactionInput(const std::string& scheduled_job_id) {
+    InstrumentedMutexLock l(&mutex_);
+
+    auto job_index = jobs_.find(scheduled_job_id);
+    if (job_index == jobs_.end()) {
+      return "";
+    }
+    std::string compaction_input = std::move(job_index->second);
+    jobs_.erase(job_index);
+
+    auto info_index = infos_.find(scheduled_job_id);
+    if (info_index == infos_.end()) {
+      return "";
+    }
+    infos_.erase(info_index);
+
+    return compaction_input;
+  }
+
+  HistogramData RunCancelledCompaction(
+      const OpenAndCompactOptions& options, const std::string& scheduled_job_id,
+      const std::string& compaction_input,
+      const CompactionServiceOptionsOverride& override_options) {
+    std::string temp_result;
+    EXPECT_OK(statistics_->Reset());
+
+    Status s =
+        DB::OpenAndCompact(options, db_path_, GetOutputPath(scheduled_job_id),
+                           compaction_input, &temp_result, override_options);
+
+    EXPECT_TRUE(s.IsManualCompactionPaused());
+
+    HistogramData stats;
+    statistics_->histogramData(FILE_WRITE_COMPACTION_MICROS, &stats);
+    return stats;
+  }
+
+  HistogramData RunCompaction(
+      const OpenAndCompactOptions& options, const std::string& scheduled_job_id,
+      const std::string& compaction_input,
+      const CompactionServiceOptionsOverride& override_options,
+      std::string* result) {
+    EXPECT_OK(statistics_->Reset());
+
+    Status s =
+        DB::OpenAndCompact(options, db_path_, GetOutputPath(scheduled_job_id),
+                           compaction_input, result, override_options);
+
+    EXPECT_TRUE(s.ok());
+
+    HistogramData stats;
+    statistics_->histogramData(FILE_WRITE_COMPACTION_MICROS, &stats);
+    return stats;
+  }
+
+  void StoreResult(const std::string& result) {
+    InstrumentedMutexLock l(&mutex_);
+    result_ = result;
+  }
+
+  TestScenario scenario_;
+};
+
+class ResumableCompactionServiceTest : public CompactionServiceTest {
+ public:
+  explicit ResumableCompactionServiceTest() : CompactionServiceTest() {}
+
+  void RunCompactionCancelTest(
+      ResumableCompactionService::TestScenario scenario) {
+    Options options = CurrentOptions();
+    options.disable_auto_compactions = true;
+    std::shared_ptr<Statistics> statistics = CreateDBStatistics();
+
+    options.file_checksum_gen_factory = GetFileChecksumGenCrc32cFactory();
+    BlockBasedTableOptions table_options;
+    table_options.verify_compression = true;
+    options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+
+    auto resume_cs = std::make_shared<ResumableCompactionService>(
+        dbname_, options, statistics, scenario);
+    options.compaction_service = resume_cs;
+
+    DestroyAndReopen(options);
+
+    GenerateTestData();
+
+    ASSERT_OK(statistics->Reset());
+
+    CompactRangeOptions cro;
+    cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
+    Status s = db_->CompactRange(cro, nullptr, nullptr);
+    ASSERT_OK(s);
+
+    VerifyTestData();
+
+    s = db_->VerifyChecksum();
+    ASSERT_OK(s);
+
+    s = db_->VerifyFileChecksums(ReadOptions());
+    ASSERT_OK(s);
+
+    CompactionServiceResult result;
+    resume_cs->GetResult(&result);
+    ASSERT_OK(result.status);
+    ASSERT_TRUE(result.stats.is_manual_compaction);
+    ASSERT_TRUE(result.stats.is_remote_compaction);
+    ASSERT_GT(result.output_files.size(), 0);
+
+    uint64_t resumed_bytes =
+        statistics->getTickerCount(REMOTE_COMPACT_RESUMED_BYTES);
+    if (scenario ==
+        ResumableCompactionService::TestScenario::kCancelThenResume) {
+      // When resuming compaction, some bytes should be resumed from previous
+      // progress
+      ASSERT_GT(resumed_bytes, 0);
+    } else if (scenario == ResumableCompactionService::TestScenario::
+                               kCancelThenFreshStart) {
+      // When starting fresh (ignoring existing progress), no bytes should be
+      // resumed
+      ASSERT_EQ(resumed_bytes, 0);
+    } else {  // kMultipleCancelToggleResumption
+      // Phase 2 ran without resumption (fresh start), so Phase 3 has no
+      // progress to resume from. It processes all keys again from scratch.
+      ASSERT_EQ(resumed_bytes, 0);
+    }
+  }
+
+  void GenerateTestData() {
+    for (int i = 0; i < kNumKeys; ++i) {
+      ASSERT_OK(Put(Key(i), "value"));
+      ASSERT_OK(Flush());
+      if (i % 2 == 0) {
+        ASSERT_OK(Delete(Key(i)));
+        ASSERT_OK(Flush());
+      }
+    }
+  }
+
+  void VerifyTestData() {
+    for (int i = 0; i < kNumKeys; ++i) {
+      if (i % 2 == 0) {
+        ASSERT_EQ("NOT_FOUND", Get((Key(i))));
+      } else {
+        ASSERT_EQ("value", Get((Key(i))));
+      }
+    }
+  }
+
+ private:
+  static constexpr int kNumKeys = 10;
+};
+
+TEST_F(ResumableCompactionServiceTest, CompactionCancelThenResume) {
+  RunCompactionCancelTest(
+      ResumableCompactionService::TestScenario::kCancelThenResume);
+}
+
+TEST_F(ResumableCompactionServiceTest, CompactionCancelThenFreshStart) {
+  RunCompactionCancelTest(
+      ResumableCompactionService::TestScenario::kCancelThenFreshStart);
+}
+
+TEST_F(ResumableCompactionServiceTest,
+       CompactionMultipleCancelToggleResumption) {
+  RunCompactionCancelTest(ResumableCompactionService::TestScenario::
+                              kMultipleCancelToggleResumption);
+}
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

db/compaction/subcompaction_state.cc

@@ -108,11 +108,13 @@ Slice SubcompactionState::LargestUserKey() const {
 Status SubcompactionState::AddToOutput(
     const CompactionIterator& iter, bool use_proximal_output,
     const CompactionFileOpenFunc& open_file_func,
-    const CompactionFileCloseFunc& close_file_func) {
+    const CompactionFileCloseFunc& close_file_func,
+    const ParsedInternalKey& prev_table_last_internal_key) {
   // update target output
   current_outputs_ =
       use_proximal_output ? &proximal_level_outputs_ : &compaction_outputs_;
-  return current_outputs_->AddToOutput(iter, open_file_func, close_file_func);
+  return current_outputs_->AddToOutput(iter, open_file_func, close_file_func,
+                                       prev_table_last_internal_key);
 }
 
 }  // namespace ROCKSDB_NAMESPACE

db/compaction/subcompaction_state.h

@@ -208,10 +208,20 @@ class SubcompactionState {
     return range_del_agg_ && !range_del_agg_->IsEmpty();
   }
 
+  void SetSubcompactionProgress(
+      const SubcompactionProgress& subcompaction_progress) {
+    subcompaction_progress_ = subcompaction_progress;
+  }
+
+  SubcompactionProgress& GetSubcompactionProgressRef() {
+    return subcompaction_progress_;
+  }
+
   // Add compaction_iterator key/value to the `Current` output group.
   Status AddToOutput(const CompactionIterator& iter, bool use_proximal_output,
                      const CompactionFileOpenFunc& open_file_func,
-                     const CompactionFileCloseFunc& close_file_func);
+                     const CompactionFileCloseFunc& close_file_func,
+                     const ParsedInternalKey& prev_table_last_internal_key);
 
   // Close all compaction output files, both output_to_proximal_level outputs
   // and normal outputs.
@@ -241,6 +251,8 @@ class SubcompactionState {
   CompactionOutputs proximal_level_outputs_;
   CompactionOutputs* current_outputs_ = &compaction_outputs_;
   std::unique_ptr<CompactionRangeDelAggregator> range_del_agg_;
+
+  SubcompactionProgress subcompaction_progress_;
 };
 
 }  // namespace ROCKSDB_NAMESPACE

db/compaction/tiered_compaction_test.cc

@@ -1764,9 +1764,10 @@ TEST_P(PrecludeLastLevelTest, SmallPrecludeTime) {
   options.env = mock_env_.get();
   options.level0_file_num_compaction_trigger = kNumTrigger;
   options.num_levels = kNumLevels;
-  // This existing test selected to also check the kIce case, which should not
-  // be interesting enough to exercise across all the test cases
-  options.last_level_temperature = Temperature::kIce;
+  // This existing test selected to also check the case of various temperatures
+  // for last_level_temperature, which should not be interesting enough to
+  // exercise across many/all test cases
+  options.last_level_temperature = RandomKnownTemperature();
   DestroyAndReopen(options);
 
   Random rnd(301);
@@ -1794,8 +1795,9 @@ TEST_P(PrecludeLastLevelTest, SmallPrecludeTime) {
   auto seqs = tp_mapping.TEST_GetInternalMapping();
   ASSERT_FALSE(seqs.empty());
   ASSERT_GE(GetSstSizeHelper(Temperature::kUnknown), 1);
-  ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0);
-  ASSERT_EQ(GetSstSizeHelper(Temperature::kIce), 0);
+  for (auto t : kKnownTemperatures) {
+    ASSERT_EQ(GetSstSizeHelper(t), 0);
+  }
 
   // Wait more than preclude_last_level time, then make sure all the data is
   // compacted to the last level even there's no write (no seqno -> time
@@ -1804,9 +1806,14 @@ TEST_P(PrecludeLastLevelTest, SmallPrecludeTime) {
 
   ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
   ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel());
-  ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0);
-  ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0);
-  ASSERT_GE(GetSstSizeHelper(Temperature::kIce), 1);
+
+  for (auto t : kKnownTemperatures) {
+    if (t == options.last_level_temperature) {
+      ASSERT_GT(GetSstSizeHelper(t), 0);
+    } else {
+      ASSERT_EQ(GetSstSizeHelper(t), 0);
+    }
+  }
 
   Close();
 }

db/db_basic_test.cc

@@ -675,30 +675,6 @@ TEST_F(DBBasicTest, Flush) {
   } while (ChangeCompactOptions());
 }
 
-TEST_F(DBBasicTest, ManifestRollOver) {
-  do {
-    Options options;
-    options.max_manifest_file_size = 10;  // 10 bytes
-    options = CurrentOptions(options);
-    CreateAndReopenWithCF({"pikachu"}, options);
-    {
-      ASSERT_OK(Put(1, "manifest_key1", std::string(1000, '1')));
-      ASSERT_OK(Put(1, "manifest_key2", std::string(1000, '2')));
-      ASSERT_OK(Put(1, "manifest_key3", std::string(1000, '3')));
-      uint64_t manifest_before_flush = dbfull()->TEST_Current_Manifest_FileNo();
-      ASSERT_OK(Flush(1));  // This should trigger LogAndApply.
-      uint64_t manifest_after_flush = dbfull()->TEST_Current_Manifest_FileNo();
-      ASSERT_GT(manifest_after_flush, manifest_before_flush);
-      ReopenWithColumnFamilies({"default", "pikachu"}, options);
-      ASSERT_GT(dbfull()->TEST_Current_Manifest_FileNo(), manifest_after_flush);
-      // check if a new manifest file got inserted or not.
-      ASSERT_EQ(std::string(1000, '1'), Get(1, "manifest_key1"));
-      ASSERT_EQ(std::string(1000, '2'), Get(1, "manifest_key2"));
-      ASSERT_EQ(std::string(1000, '3'), Get(1, "manifest_key3"));
-    }
-  } while (ChangeCompactOptions());
-}
-
 TEST_F(DBBasicTest, IdentityAcrossRestarts) {
   constexpr size_t kMinIdSize = 10;
   do {

db/db_compaction_test.cc

@@ -9915,6 +9915,20 @@ static void VerifyTemperatureFileReadStats(const Statistics& st,
     EXPECT_EQ(iostats->file_io_stats_by_temperature.warm_file_read_count, 0);
   }
 
+  if (temps.Contains(Temperature::kCool)) {
+    EXPECT_GE(st.getTickerCount(COOL_FILE_READ_BYTES), min_bytes);
+    EXPECT_GE(st.getTickerCount(COOL_FILE_READ_COUNT), min_count);
+    EXPECT_GE(iostats->file_io_stats_by_temperature.cool_file_bytes_read,
+              min_bytes);
+    EXPECT_GE(iostats->file_io_stats_by_temperature.cool_file_read_count,
+              min_count);
+  } else {
+    EXPECT_EQ(st.getTickerCount(COOL_FILE_READ_BYTES), 0);
+    EXPECT_EQ(st.getTickerCount(COOL_FILE_READ_COUNT), 0);
+    EXPECT_EQ(iostats->file_io_stats_by_temperature.cool_file_bytes_read, 0);
+    EXPECT_EQ(iostats->file_io_stats_by_temperature.cool_file_read_count, 0);
+  }
+
   if (temps.Contains(Temperature::kCold)) {
     EXPECT_GE(st.getTickerCount(COLD_FILE_READ_BYTES), min_bytes);
     EXPECT_GE(st.getTickerCount(COLD_FILE_READ_COUNT), min_count);
@@ -9945,7 +9959,7 @@ static void VerifyTemperatureFileReadStats(const Statistics& st,
 }
 
 TEST_F(DBCompactionTest, FIFOMultiTierTemperatureAging) {
-  // Test multi-tier aging: Hot -> Warm -> Cold -> Ice
+  // Test multi-tier aging: Hot -> Warm -> Cool -> Cold -> Ice
   Options options = CurrentOptions();
   options.compaction_style = kCompactionStyleFIFO;
   options.num_levels = 1;
@@ -9961,8 +9975,9 @@ TEST_F(DBCompactionTest, FIFOMultiTierTemperatureAging) {
   // Multi-tier aging: files age through multiple temperatures
   fifo_options.file_temperature_age_thresholds = {
       {Temperature::kWarm, 500},   // Hot -> Warm after 500s
-      {Temperature::kCold, 1000},  // Warm -> Cold after 1000s
-      {Temperature::kIce, 1500}    // Cold -> Ice after 1500s
+      {Temperature::kCool, 1000},  // Warm -> Cool
+      {Temperature::kCold, 1500},  // Cool -> Cold
+      {Temperature::kIce, 2000}    // Cold -> Ice
   };
   fifo_options.max_table_files_size = 100000000;
   fifo_options.allow_trivial_copy_when_change_temperature = true;
@@ -9973,8 +9988,8 @@ TEST_F(DBCompactionTest, FIFOMultiTierTemperatureAging) {
   env_->SetMockSleep();
 
   // Track all temperature file creations
-  int total_hot = 0, total_warm = 0, total_cold = 0, total_ice = 0,
-      total_unknown = 0;
+  int total_hot = 0, total_warm = 0, total_cool = 0, total_cold = 0,
+      total_ice = 0, total_unknown = 0;
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
       "NewWritableFile::FileOptions.temperature", [&](void* arg) {
         Temperature temperature = *(static_cast<Temperature*>(arg));
@@ -9985,6 +10000,9 @@ TEST_F(DBCompactionTest, FIFOMultiTierTemperatureAging) {
           case Temperature::kWarm:
             total_warm++;
             break;
+          case Temperature::kCool:
+            total_cool++;
+            break;
           case Temperature::kCold:
             total_cold++;
             break;
@@ -10016,8 +10034,11 @@ TEST_F(DBCompactionTest, FIFOMultiTierTemperatureAging) {
 
   VerifyTemperatureFileReadStats(*options.statistics, Temperature::kHot);
 
+  // Land well into each time interval
+  env_->MockSleepForSeconds(100);
+
   // Age initial files to warm
-  env_->MockSleepForSeconds(600);
+  env_->MockSleepForSeconds(500);
   ASSERT_OK(Put(Key(1), Random::GetTLSInstance()->RandomBinaryString(101)));
   ASSERT_OK(Flush());
   ASSERT_OK(dbfull()->TEST_WaitForCompact());
@@ -10031,12 +10052,26 @@ TEST_F(DBCompactionTest, FIFOMultiTierTemperatureAging) {
   // Verify Warm file statistics
   VerifyTemperatureFileReadStats(*options.statistics, Temperature::kWarm);
 
-  // Age initial files to cold
-  env_->MockSleepForSeconds(600);
+  // Age initial files to cool
+  env_->MockSleepForSeconds(500);
   ASSERT_OK(Put(Key(2), Random::GetTLSInstance()->RandomBinaryString(102)));
   ASSERT_OK(Flush());
   ASSERT_OK(dbfull()->TEST_WaitForCompact());
 
+  // Test reading from Cool temperature file (the aged file)
+  ASSERT_OK(options.statistics->Reset());
+  get_iostats_context()->Reset();
+
+  ASSERT_EQ(100U, Get(Key(0)).size());
+
+  VerifyTemperatureFileReadStats(*options.statistics, Temperature::kCool);
+
+  // Age initial files to cold
+  env_->MockSleepForSeconds(500);
+  ASSERT_OK(Put(Key(3), Random::GetTLSInstance()->RandomBinaryString(103)));
+  ASSERT_OK(Flush());
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+
   // Test reading from Cold temperature file (the aged file)
   ASSERT_OK(options.statistics->Reset());
   get_iostats_context()->Reset();
@@ -10046,8 +10081,8 @@ TEST_F(DBCompactionTest, FIFOMultiTierTemperatureAging) {
   VerifyTemperatureFileReadStats(*options.statistics, Temperature::kCold);
 
   // Age initial files to ice
-  env_->MockSleepForSeconds(600);
-  ASSERT_OK(Put(Key(3), Random::GetTLSInstance()->RandomBinaryString(103)));
+  env_->MockSleepForSeconds(500);
+  ASSERT_OK(Put(Key(4), Random::GetTLSInstance()->RandomBinaryString(104)));
   ASSERT_OK(Flush());
   ASSERT_OK(dbfull()->TEST_WaitForCompact());
 
@@ -10072,12 +10107,14 @@ TEST_F(DBCompactionTest, FIFOMultiTierTemperatureAging) {
   // Verify current files temperatures
   EXPECT_EQ(temp_counts[Temperature::kHot], 1);
   EXPECT_EQ(temp_counts[Temperature::kWarm], 1);
+  EXPECT_EQ(temp_counts[Temperature::kCool], 1);
   EXPECT_EQ(temp_counts[Temperature::kCold], 1);
   EXPECT_EQ(temp_counts[Temperature::kIce], 3);
 
   // Verify historical (and current) file temperatures
-  EXPECT_EQ(total_hot, 6);
-  EXPECT_EQ(total_warm, 5);
+  EXPECT_EQ(total_hot, 7);
+  EXPECT_EQ(total_warm, 6);
+  EXPECT_EQ(total_cool, 5);
   EXPECT_EQ(total_cold, 4);
   EXPECT_EQ(total_ice, 3);
 
@@ -10087,7 +10124,7 @@ TEST_F(DBCompactionTest, FIFOMultiTierTemperatureAging) {
   get_iostats_context()->Reset();
 
   // Read from all files to verify cumulative statistics
-  for (int i = 0; i < 4; i++) {
+  for (int i = 0; i < 5; i++) {
     ASSERT_EQ(static_cast<unsigned>(100 + i), Get(Key(i)).size());
   }
 

db/db_etc3_test.cc

@@ -0,0 +1,161 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#include "db/db_test_util.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+class DBEtc3Test : public DBTestBase {
+ public:
+  DBEtc3Test() : DBTestBase("db_etc3_test", /*env_do_fsync=*/true) {}
+};
+
+TEST_F(DBEtc3Test, ManifestRollOver) {
+  do {
+    Options options;
+    // Force new manifest on each manifest write
+    options.max_manifest_file_size = 0;
+    options.max_manifest_space_amp_pct = 0;
+    options = CurrentOptions(options);
+    CreateAndReopenWithCF({"pikachu"}, options);
+    {
+      ASSERT_OK(Put(1, "key1", std::string(1000, '1')));
+      ASSERT_OK(Put(1, "key2", std::string(1000, '2')));
+      ASSERT_OK(Put(1, "key3", std::string(1000, '3')));
+      uint64_t manifest_before_flush = dbfull()->TEST_Current_Manifest_FileNo();
+      ASSERT_OK(Flush(1));  // This should trigger LogAndApply.
+      uint64_t manifest_after_flush = dbfull()->TEST_Current_Manifest_FileNo();
+      ASSERT_GT(manifest_after_flush, manifest_before_flush);
+      // Re-open should always re-create manifest file
+      ReopenWithColumnFamilies({"default", "pikachu"}, options);
+      ASSERT_GT(dbfull()->TEST_Current_Manifest_FileNo(), manifest_after_flush);
+      ASSERT_EQ(std::string(1000, '1'), Get(1, "key1"));
+      ASSERT_EQ(std::string(1000, '2'), Get(1, "key2"));
+      ASSERT_EQ(std::string(1000, '3'), Get(1, "key3"));
+    }
+  } while (ChangeCompactOptions());
+}
+
+TEST_F(DBEtc3Test, AutoTuneManifestSize) {
+  // Ensure we have auto-tuning beyond max_manifest_file_size by default
+  ASSERT_EQ(DBOptions{}.max_manifest_space_amp_pct, 500);
+
+  Options options = CurrentOptions();
+  ASSERT_OK(db_->SetOptions({{"level0_file_num_compaction_trigger", "20"}}));
+
+  // Use large column family names to essentially control the amount of payload
+  // data needed for the manifest file. Drop manifest entries don't include the
+  // CF name so are small.
+  uint64_t prev_manifest_num = 0, cur_manifest_num = 0;
+  std::deque<ColumnFamilyHandle*> handles;
+  int counter = 5;
+  auto AddCfFn = [&]() {
+    std::string name = "cf" + std::to_string(counter++);
+    name.resize(1000, 'a');
+    ASSERT_OK(db_->CreateColumnFamily(options, name, &handles.emplace_back()));
+    prev_manifest_num = cur_manifest_num;
+    cur_manifest_num = dbfull()->TEST_Current_Manifest_FileNo();
+  };
+  auto DropCfFn = [&]() {
+    ASSERT_OK(db_->DropColumnFamily(handles.front()));
+    ASSERT_OK(db_->DestroyColumnFamilyHandle(handles.front()));
+    handles.pop_front();
+    prev_manifest_num = cur_manifest_num;
+    cur_manifest_num = dbfull()->TEST_Current_Manifest_FileNo();
+  };
+  auto TrivialManifestWriteFn = [&]() {
+    ASSERT_OK(Put("x", std::to_string(counter++)));
+    ASSERT_OK(Flush());
+    prev_manifest_num = cur_manifest_num;
+    cur_manifest_num = dbfull()->TEST_Current_Manifest_FileNo();
+  };
+
+  options.max_manifest_file_size = 1000000;
+  options.max_manifest_space_amp_pct = 0;  // no auto-tuning yet
+  DestroyAndReopen(options);
+
+  // With the generous (minimum) maximum manifest size, should not be rotated
+  AddCfFn();
+  AddCfFn();
+  AddCfFn();
+  ASSERT_EQ(prev_manifest_num, cur_manifest_num);
+
+  // Change options for small max and (still) no auto-tuning
+  ASSERT_OK(db_->SetDBOptions({{"max_manifest_file_size", "3000"}}));
+
+  // Takes effect on the next manifest write
+  TrivialManifestWriteFn();
+  ASSERT_LT(prev_manifest_num, cur_manifest_num);
+
+  // Now we have to rewrite the whole manifest on each write because the
+  // compacted size exceeds the "max" size.
+  AddCfFn();
+  ASSERT_LT(prev_manifest_num, cur_manifest_num);
+  DropCfFn();
+  ASSERT_LT(prev_manifest_num, cur_manifest_num);
+  AddCfFn();
+  ASSERT_LT(prev_manifest_num, cur_manifest_num);
+  TrivialManifestWriteFn();
+  ASSERT_LT(prev_manifest_num, cur_manifest_num);
+
+  // Enabling auto-tuning should fix this, immediately for next manifest writes.
+  // This will allow up to double-ish the size of the compacted manifest,
+  // which last should have been 4000 + some bytes.
+  ASSERT_EQ(handles.size(), 4U);
+  ASSERT_OK(db_->SetDBOptions({{"max_manifest_space_amp_pct", "105"}}));
+
+  // After 9 CF names should be enough to rotate the manifest
+  for (int i = 1; i <= 5; ++i) {
+    if ((i % 2) == 1) {
+      DropCfFn();
+    }
+    AddCfFn();
+    ASSERT_EQ(prev_manifest_num, cur_manifest_num);
+  }
+  TrivialManifestWriteFn();
+  ASSERT_LT(prev_manifest_num, cur_manifest_num);
+
+  // We now have a different last compacted manifest size, should be
+  // able to go beyond 9 CFs named in manifest this time.
+  ASSERT_EQ(handles.size(), 6U);
+
+  DropCfFn();
+  DropCfFn();
+  for (int i = 1; i <= 4; ++i) {
+    DropCfFn();
+    AddCfFn();
+    ASSERT_EQ(prev_manifest_num, cur_manifest_num);
+  }
+  // We've written 10 named CFs to the manifest. We should be able to
+  // dynamically change the auto-tuning still based on the last "compacted"
+  // manifest size of 7000 + some bytes.
+  ASSERT_OK(db_->SetDBOptions({{"max_manifest_space_amp_pct", "51"}}));
+  TrivialManifestWriteFn();
+  ASSERT_LT(prev_manifest_num, cur_manifest_num);
+  // And the "compacted" manifest size has reset again, so should be changed
+  // again sooner.
+  ASSERT_EQ(handles.size(), 4U);
+  for (int i = 1; i <= 2; ++i) {
+    AddCfFn();
+    ASSERT_EQ(prev_manifest_num, cur_manifest_num);
+  }
+  // Enough for manifest change
+  AddCfFn();
+  ASSERT_LT(prev_manifest_num, cur_manifest_num);
+
+  // Wrap up
+  while (!handles.empty()) {
+    DropCfFn();
+  }
+}
+
+}  // namespace ROCKSDB_NAMESPACE
+
+int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
+  ::testing::InitGoogleTest(&argc, argv);
+  RegisterCustomObjects(argc, argv);
+  return RUN_ALL_TESTS();
+}

db/db_flush_test.cc

@@ -3561,6 +3561,152 @@ TEST_F(DBFlushTest, VerifyOutputRecordCount) {
     ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
   }
 }
+
+class DBFlushSuperBlockTest
+    : public DBFlushTest,
+      public ::testing::WithParamInterface<std::tuple<bool, size_t, size_t>> {
+ public:
+  DBFlushSuperBlockTest() : DBFlushTest() {}
+
+  std::string formatKey(int i) {
+    int desired_length = 10;
+    char buffer[64];
+    snprintf(buffer, 64, "%0*d", desired_length, i);
+    return buffer;
+  }
+
+  void VerifyReadWithGet(int key_count) {
+    for (int i = 0; i < key_count; ++i) {
+      PinnableSlice value;
+      ASSERT_OK(Get(formatKey(i), &value));
+      ASSERT_EQ(value.ToString(), added_data[formatKey(i)]);
+    }
+  }
+
+  void VerifyReadWithIterator(int key_count) {
+    {
+      std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
+      int i = 0;
+      for (it->SeekToFirst(); it->Valid(); it->Next()) {
+        ASSERT_OK(it->status());
+        ASSERT_EQ((it->key()).ToString(), formatKey(i));
+        ASSERT_EQ((it->value()).ToString(), added_data[formatKey(i)]);
+        i++;
+      }
+      ASSERT_OK(it->status());
+      ASSERT_EQ(i, key_count);
+    }
+  }
+
+ protected:
+  Random rnd{123};
+  std::unordered_map<std::string, std::string> added_data;
+};
+
+constexpr size_t kLowSpaceOverheadRatio = 256;
+
+TEST_P(DBFlushSuperBlockTest, SuperBlock) {
+  constexpr int key_count = 12345;
+  Options options;
+  options.env = env_;
+  options.file_checksum_gen_factory = GetFileChecksumGenCrc32cFactory();
+  options.paranoid_file_checks = true;
+  options.write_buffer_size = 1024 * 1024;
+  BlockBasedTableOptions block_options;
+  block_options.block_align = get<0>(GetParam());
+  block_options.index_block_restart_interval = 3;
+  block_options.super_block_alignment_size = get<1>(GetParam());
+  block_options.super_block_alignment_space_overhead_ratio = get<2>(GetParam());
+  options.table_factory.reset(NewBlockBasedTableFactory(block_options));
+  if (block_options.block_align) {
+    // When block align is enabled, disable compression
+    options.compression = kNoCompression;
+  }
+
+  ASSERT_OK(options.table_factory->ValidateOptions(
+      DBOptions(options), ColumnFamilyOptions(options)));
+
+  Reopen(options);
+
+  int super_block_pad_count = 0;
+  int super_block_pad_exceed_limit_count = 0;
+  SyncPoint::GetInstance()->SetCallBack(
+      "BlockBasedTableBuilder::WriteMaybeCompressedBlock:"
+      "SuperBlockAlignment",
+      [&super_block_pad_count](void* /*arg*/) { super_block_pad_count++; });
+  SyncPoint::GetInstance()->SetCallBack(
+      "BlockBasedTableBuilder::WriteMaybeCompressedBlock:"
+      "SuperBlockAlignmentPaddingBytesExceedLimit",
+      [&super_block_pad_exceed_limit_count](void* /*arg*/) {
+        super_block_pad_exceed_limit_count++;
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  // Add lots of keys
+  for (int i = 0; i < key_count; ++i) {
+    added_data[formatKey(i)] = std::string(rnd.RandomString(rnd.Next() % 1000));
+    ASSERT_OK(Put(formatKey(i), added_data[formatKey(i)]));
+  }
+
+  // flush the data in memory to disk to verify with super block alignment, the
+  // data could be read back properly
+  Reopen(options);
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+
+  // When block_align is enabled, super block is always aligned, so there should
+  // be 0 padding for super block alignment
+  if (block_options.super_block_alignment_size != 0 &&
+      !block_options.block_align) {
+    ASSERT_GT(super_block_pad_count, 0);
+  } else {
+    ASSERT_EQ(super_block_pad_count, 0);
+  }
+
+  if (!block_options.block_align &&
+      block_options.super_block_alignment_size != 0 &&
+      block_options.super_block_alignment_space_overhead_ratio ==
+          kLowSpaceOverheadRatio) {
+    ASSERT_GT(super_block_pad_exceed_limit_count, 0);
+  }
+
+  // verify the values are correct
+  VerifyReadWithGet(key_count);
+  Reopen(options);
+  VerifyReadWithIterator(key_count);
+
+  // verify checksum
+  ASSERT_OK(db_->VerifyFileChecksums(ReadOptions()));
+
+  // Reopen options and flip the option of super block configuration, read still
+  // works. This verifies the forward/backward compatibility
+  if (block_options.super_block_alignment_size == 0) {
+    block_options.super_block_alignment_size = 16 * 1024;
+  } else {
+    block_options.super_block_alignment_size = 0;
+  }
+  options.table_factory.reset(NewBlockBasedTableFactory(block_options));
+
+  Reopen(options);
+
+  // verify the values are correct
+  VerifyReadWithGet(key_count);
+  Reopen(options);
+  VerifyReadWithIterator(key_count);
+
+  // verify checksum
+  ASSERT_OK(db_->VerifyFileChecksums(ReadOptions()));
+}
+
+INSTANTIATE_TEST_CASE_P(
+    SuperBlockTests, DBFlushSuperBlockTest,
+    testing::Combine(testing::Bool(), testing::Values(0, 32 * 1024, 16 * 1024),
+                     // Use very low space overhead ratio to test
+                     // the case where required padded bytes is
+                     // larger than the max allowed padding size
+                     testing::Values(4, kLowSpaceOverheadRatio)));
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

db/db_follower_test.cc

@@ -370,10 +370,10 @@ TEST_F(DBFollowerTest, RetryCatchupManifestRollover) {
 
 // This test creates 4 L0 files and compacts them. The follower, during catchup,
 // successfully instantiates 4 Versions corresponding to the 4 files (but
-// donesn't install them yet), followed by deleting those 4 and adding a new
+// doesn't install them yet), followed by deleting those 4 and adding a new
 // file from compaction. The test verifies that the 4 L0 files are deleted
 // correctly by the follower.
-// We use teh Barrier* functions to ensure that the follower first sees the 4
+// We use the Barrier* functions to ensure that the follower first sees the 4
 // L0 files and is able to link them, and then sees the compaction that
 // obsoletes those L0 files (so those L0 files are intermediates that it has
 // to explicitly delete). Suppose we don't have any barriers, its possible

db/db_impl/db_impl.cc

@@ -258,10 +258,10 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname,
       [this]() { this->TriggerPeriodicCompaction(); });
 
   versions_.reset(new VersionSet(
-      dbname_, &immutable_db_options_, file_options_, table_cache_.get(),
-      write_buffer_manager_, &write_controller_, &block_cache_tracer_,
-      io_tracer_, db_id_, db_session_id_, options.daily_offpeak_time_utc,
-      &error_handler_, read_only));
+      dbname_, &immutable_db_options_, mutable_db_options_, file_options_,
+      table_cache_.get(), write_buffer_manager_, &write_controller_,
+      &block_cache_tracer_, io_tracer_, db_id_, db_session_id_,
+      options.daily_offpeak_time_utc, &error_handler_, read_only));
   column_family_memtables_.reset(
       new ColumnFamilyMemTablesImpl(versions_->GetColumnFamilySet()));
 
@@ -1412,7 +1412,7 @@ Status DBImpl::SetDBOptions(
       file_options_for_compaction_ = FileOptions(new_db_options);
       file_options_for_compaction_ = fs_->OptimizeForCompactionTableWrite(
           file_options_for_compaction_, immutable_db_options_);
-      versions_->ChangeFileOptions(mutable_db_options_);
+      versions_->UpdatedMutableDbOptions(mutable_db_options_, &mutex_);
       // TODO(xiez): clarify why apply optimize for read to write options
       file_options_for_compaction_ = fs_->OptimizeForCompactionTableRead(
           file_options_for_compaction_, immutable_db_options_);
@@ -1480,6 +1480,12 @@ int DBImpl::FindMinimumEmptyLevelFitting(
   return minimum_level;
 }
 
+Status DBImpl::FlushWAL(const FlushWALOptions& options) {
+  WriteOptions write_options;
+  write_options.rate_limiter_priority = options.rate_limiter_priority;
+  return FlushWAL(write_options, options.sync);
+}
+
 Status DBImpl::FlushWAL(const WriteOptions& write_options, bool sync) {
   if (manual_wal_flush_) {
     IOStatus io_s;
@@ -5041,6 +5047,19 @@ void DBImpl::GetColumnFamilyMetaData(ColumnFamilyHandle* column_family,
   }
 }
 
+void DBImpl::GetColumnFamilyMetaData(
+    ColumnFamilyHandle* column_family,
+    const GetColumnFamilyMetaDataOptions& options,
+    ColumnFamilyMetaData* metadata) {
+  assert(column_family);
+  auto* cfd =
+      static_cast_with_check<ColumnFamilyHandleImpl>(column_family)->cfd();
+  {
+    InstrumentedMutexLock l(&mutex_);
+    cfd->current()->GetColumnFamilyMetaData(options, metadata);
+  }
+}
+
 void DBImpl::GetAllColumnFamilyMetaData(
     std::vector<ColumnFamilyMetaData>* metadata) {
   InstrumentedMutexLock l(&mutex_);

db/db_impl/db_impl.h

@@ -484,10 +484,13 @@ class DBImpl : public DB {
       const FlushOptions& options,
       const std::vector<ColumnFamilyHandle*>& column_families) override;
   Status FlushWAL(bool sync) override {
-    // TODO: plumb Env::IOActivity, Env::IOPriority
-    return FlushWAL(WriteOptions(), sync);
+    FlushWALOptions options;
+    options.sync = sync;
+    return FlushWAL(options);
   }
 
+  Status FlushWAL(const FlushWALOptions& options) override;
+
   virtual Status FlushWAL(const WriteOptions& write_options, bool sync);
   bool WALBufferIsEmpty();
   Status SyncWAL() override;
@@ -570,6 +573,11 @@ class DBImpl : public DB {
   void GetColumnFamilyMetaData(ColumnFamilyHandle* column_family,
                                ColumnFamilyMetaData* metadata) override;
 
+  // Get column family metadata with filtering based on key range and level
+  void GetColumnFamilyMetaData(ColumnFamilyHandle* column_family,
+                               const GetColumnFamilyMetaDataOptions& options,
+                               ColumnFamilyMetaData* metadata) override;
+
   void GetAllColumnFamilyMetaData(
       std::vector<ColumnFamilyMetaData>* metadata) override;
 
@@ -2393,6 +2401,14 @@ class DBImpl : public DB {
                           JobContext* job_context, LogBuffer* log_buffer,
                           CompactionJobInfo* compaction_job_info);
 
+  // Helper function to perform trivial move by updating manifest metadata
+  // without rewriting data files. This is called when IsTrivialMove() is true.
+  // REQUIRES: mutex held
+  // Returns: Status of the trivial move operation
+  Status PerformTrivialMove(Compaction& c, LogBuffer* log_buffer,
+                            bool& compaction_released, size_t& moved_files,
+                            size_t& moved_bytes);
+
   // REQUIRES: mutex unlocked
   void TrackOrUntrackFiles(const std::vector<std::string>& existing_data_files,
                            bool track);

db/db_impl/db_impl_compaction_flush.cc

@@ -1424,6 +1424,56 @@ Status DBImpl::CompactFiles(const CompactionOptions& compact_options,
   return s;
 }
 
+Status DBImpl::PerformTrivialMove(Compaction& c, LogBuffer* log_buffer,
+                                  bool& compaction_released,
+                                  size_t& moved_files, size_t& moved_bytes) {
+  mutex_.AssertHeld();
+
+  ROCKS_LOG_BUFFER(log_buffer, "[%s] Moving %d files to level-%d\n",
+                   c.column_family_data()->GetName().c_str(),
+                   static_cast<int>(c.num_input_files(0)), c.output_level());
+
+  // Move files to the output level by editing the manifest
+  for (unsigned int l = 0; l < c.num_input_levels(); l++) {
+    if (c.level(l) == c.output_level()) {
+      continue;
+    }
+    for (size_t i = 0; i < c.num_input_files(l); i++) {
+      FileMetaData* f = c.input(l, i);
+      c.edit()->DeleteFile(c.level(l), f->fd.GetNumber());
+      c.edit()->AddFile(c.output_level(), f->fd.GetNumber(), f->fd.GetPathId(),
+                        f->fd.GetFileSize(), f->smallest, f->largest,
+                        f->fd.smallest_seqno, f->fd.largest_seqno,
+                        f->marked_for_compaction, f->temperature,
+                        f->oldest_blob_file_number, f->oldest_ancester_time,
+                        f->file_creation_time, f->epoch_number,
+                        f->file_checksum, f->file_checksum_func_name,
+                        f->unique_id, f->compensated_range_deletion_size,
+                        f->tail_size, f->user_defined_timestamps_persisted);
+      moved_bytes += static_cast<size_t>(c.input(l, i)->fd.GetFileSize());
+      ROCKS_LOG_BUFFER(
+          log_buffer, "[%s] Moved #%" PRIu64 " to level-%d %" PRIu64 " bytes\n",
+          c.column_family_data()->GetName().c_str(), f->fd.GetNumber(),
+          c.output_level(), f->fd.GetFileSize());
+    }
+    moved_files += c.num_input_files(l);
+  }
+
+  // Install the new version
+  const ReadOptions read_options(Env::IOActivity::kCompaction);
+  const WriteOptions write_options(Env::IOActivity::kCompaction);
+  Status status = versions_->LogAndApply(
+      c.column_family_data(), read_options, write_options, c.edit(), &mutex_,
+      directories_.GetDbDir(), /*new_descriptor_log=*/false,
+      /*column_family_options=*/nullptr,
+      [&c, &compaction_released](const Status& s) {
+        c.ReleaseCompactionFiles(s);
+        compaction_released = true;
+      });
+
+  return status;
+}
+
 Status DBImpl::CompactFilesImpl(
     const CompactionOptions& compact_options, ColumnFamilyData* cfd,
     Version* version, const std::vector<std::string>& input_file_names,
@@ -1511,6 +1561,63 @@ Status DBImpl::CompactFilesImpl(
   // deletion compaction currently not allowed in CompactFiles.
   assert(!c->deletion_compaction());
 
+  // Check if this can be a trivial move (metadata-only update)
+  // Similar to the logic in DBImpl::BackgroundCompaction
+  // Note: We disable trivial move when compaction_service is present because
+  // the service expects all compactions to go through CompactionJob for
+  // tracking
+  bool is_trivial_move = compact_options.allow_trivial_move &&
+                         c->IsTrivialMove() &&
+                         immutable_db_options().compaction_service == nullptr;
+
+  if (is_trivial_move) {
+    // Perform trivial move: just update manifest without rewriting data
+    TEST_SYNC_POINT("DBImpl::CompactFilesImpl:TrivialMove");
+
+    bool compaction_released = false;
+    size_t moved_files = 0;
+    size_t moved_bytes = 0;
+    Status status = PerformTrivialMove(
+        *c.get(), log_buffer, compaction_released, moved_files, moved_bytes);
+
+    if (status.ok()) {
+      InstallSuperVersionAndScheduleWork(
+          c->column_family_data(), job_context->superversion_contexts.data());
+
+      // Populate output file names for trivial move
+      if (output_file_names != nullptr) {
+        for (const auto& newf : c->edit()->GetNewFiles()) {
+          output_file_names->push_back(TableFileName(
+              c->immutable_options().cf_paths, newf.second.fd.GetNumber(),
+              newf.second.fd.GetPathId()));
+        }
+      }
+
+      ROCKS_LOG_BUFFER(
+          log_buffer,
+          "[%s] Trivial move succeeded for %zu files, %zu bytes total\n",
+          c->column_family_data()->GetName().c_str(), moved_files, moved_bytes);
+    } else {
+      if (!compaction_released) {
+        c->ReleaseCompactionFiles(status);
+      }
+      ROCKS_LOG_BUFFER(log_buffer, "[%s] Trivial move failed: %s\n",
+                       c->column_family_data()->GetName().c_str(),
+                       status.ToString().c_str());
+      error_handler_.SetBGError(status, BackgroundErrorReason::kCompaction);
+    }
+
+    c.reset();
+    bg_compaction_scheduled_--;
+    if (bg_compaction_scheduled_ == 0) {
+      bg_cv_.SignalAll();
+    }
+    MaybeScheduleFlushOrCompaction();
+
+    return status;
+  }
+
+  // Not a trivial move, proceed with full compaction
   InitSnapshotContext(job_context);
 
   std::unique_ptr<std::list<uint64_t>::iterator> pending_outputs_inserted_elem(
@@ -1843,8 +1950,7 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
         ,
         LLONG_MAX /* max compaction bytes, not applicable */,
         0 /* output path ID, not applicable */, mutable_cf_options.compression,
-        mutable_cf_options.compression_opts,
-        mutable_cf_options.default_write_temperature,
+        mutable_cf_options.compression_opts, Temperature::kUnknown,
         0 /* max_subcompactions, not applicable */,
         {} /* grandparents, not applicable */,
         std::nullopt /* earliest_snapshot */, nullptr /* snapshot_checker */,
@@ -3861,7 +3967,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
       uint64_t out_file_creation_time = static_cast<uint64_t>(tmp_current_time);
 
       FileOptions copied_file_options = file_options_;
-      copied_file_options.temperature = c->output_temperature();
+      copied_file_options.temperature = c->GetOutputTemperature();
       std::unique_ptr<WritableFileWriter> dest_writer;
       {
         std::unique_ptr<FSWritableFile> dest_file;
@@ -3879,7 +3985,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
               "NewWritableFile %s\n"
               " out_fname=%s, temperature=%s, io_status=%s",
               c->column_family_data()->GetName().c_str(), out_fname.c_str(),
-              temperature_to_string[c->output_temperature()].c_str(),
+              temperature_to_string[c->GetOutputTemperature()].c_str(),
               io_s.ToString().c_str());
           break;
         }
@@ -3901,7 +4007,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
           c->column_family_data()->GetName().c_str(), in_fname.c_str(),
           temperature_to_string[in_file->temperature].c_str(),
           out_fname.c_str(),
-          temperature_to_string[c->output_temperature()].c_str(),
+          temperature_to_string[c->GetOutputTemperature()].c_str(),
           c->mutable_cf_options()
               .compaction_options_fifo.trivial_copy_buffer_size);
       // Add IO_LOW HINT for compaction
@@ -3941,7 +4047,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
               c->column_family_data()->GetName().c_str(), in_fname.c_str(),
               temperature_to_string[in_file->temperature].c_str(),
               out_fname.c_str(),
-              temperature_to_string[c->output_temperature()].c_str(),
+              temperature_to_string[c->GetOutputTemperature()].c_str(),
               io_s.ToString().c_str());
           break;
         }
@@ -3950,15 +4056,15 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
       io_s = copy_file_io_status;
 
       if (!io_s.ok()) {
-        ROCKS_LOG_BUFFER(log_buffer,
-                         "[%s] Failed to copy from: %s\n"
-                         " temperature=%s, to=%s, temperature=%s, io_status=%s",
-                         c->column_family_data()->GetName().c_str(),
-                         in_fname.c_str(),
-                         temperature_to_string[in_file->temperature].c_str(),
-                         out_fname.c_str(),
-                         temperature_to_string[c->output_temperature()].c_str(),
-                         io_s.ToString().c_str());
+        ROCKS_LOG_BUFFER(
+            log_buffer,
+            "[%s] Failed to copy from: %s\n"
+            " temperature=%s, to=%s, temperature=%s, io_status=%s",
+            c->column_family_data()->GetName().c_str(), in_fname.c_str(),
+            temperature_to_string[in_file->temperature].c_str(),
+            out_fname.c_str(),
+            temperature_to_string[c->GetOutputTemperature()].c_str(),
+            io_s.ToString().c_str());
         break;
       }
       ROCKS_LOG_BUFFER(log_buffer,
@@ -3968,7 +4074,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
                        in_fname.c_str(),
                        temperature_to_string[in_file->temperature].c_str(),
                        out_fname.c_str(),
-                       temperature_to_string[c->output_temperature()].c_str(),
+                       temperature_to_string[c->GetOutputTemperature()].c_str(),
                        io_s.ToString().c_str());
 
       FileMetaData out_file_metadata{
@@ -3980,7 +4086,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
           in_file->fd.smallest_seqno,
           in_file->fd.largest_seqno,
           false /* marked_for_compact */,
-          c->output_temperature() /* temperature */,
+          c->GetOutputTemperature() /* temperature */,
           in_file->oldest_blob_file_number,
           in_file->oldest_ancester_time,
           out_file_creation_time,
@@ -4049,7 +4155,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
             " temperature=%s, to temperature=%s, status=%s, io_status=%s",
             c->column_family_data()->GetName().c_str(), in_fname.c_str(),
             temperature_to_string[in_file->temperature].c_str(),
-            temperature_to_string[c->output_temperature()].c_str(),
+            temperature_to_string[c->GetOutputTemperature()].c_str(),
             status.ToString().c_str(), io_s.ToString().c_str());
       }
     }
@@ -4075,35 +4181,6 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
     NotifyOnCompactionBegin(c->column_family_data(), c.get(), status,
                             compaction_job_stats, job_context->job_id);
 
-    // Move files to next level
-    int32_t moved_files = 0;
-    int64_t moved_bytes = 0;
-    for (unsigned int l = 0; l < c->num_input_levels(); l++) {
-      if (c->level(l) == c->output_level()) {
-        continue;
-      }
-      for (size_t i = 0; i < c->num_input_files(l); i++) {
-        FileMetaData* f = c->input(l, i);
-        c->edit()->DeleteFile(c->level(l), f->fd.GetNumber());
-        c->edit()->AddFile(
-            c->output_level(), f->fd.GetNumber(), f->fd.GetPathId(),
-            f->fd.GetFileSize(), f->smallest, f->largest, f->fd.smallest_seqno,
-            f->fd.largest_seqno, f->marked_for_compaction, f->temperature,
-            f->oldest_blob_file_number, f->oldest_ancester_time,
-            f->file_creation_time, f->epoch_number, f->file_checksum,
-            f->file_checksum_func_name, f->unique_id,
-            f->compensated_range_deletion_size, f->tail_size,
-            f->user_defined_timestamps_persisted);
-
-        ROCKS_LOG_BUFFER(
-            log_buffer,
-            "[%s] Moving #%" PRIu64 " to level-%d %" PRIu64 " bytes\n",
-            c->column_family_data()->GetName().c_str(), f->fd.GetNumber(),
-            c->output_level(), f->fd.GetFileSize());
-        ++moved_files;
-        moved_bytes += f->fd.GetFileSize();
-      }
-    }
     if (c->compaction_reason() == CompactionReason::kLevelMaxLevelSize &&
         c->immutable_options().compaction_pri == kRoundRobin) {
       int start_level = c->start_level();
@@ -4114,14 +4191,12 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
             vstorage->GetNextCompactCursor(start_level, c->num_input_files(0)));
       }
     }
-    status = versions_->LogAndApply(
-        c->column_family_data(), read_options, write_options, c->edit(),
-        &mutex_, directories_.GetDbDir(),
-        /*new_descriptor_log=*/false, /*column_family_options=*/nullptr,
-        [&c, &compaction_released](const Status& s) {
-          c->ReleaseCompactionFiles(s);
-          compaction_released = true;
-        });
+
+    // Perform the trivial move
+    size_t moved_files = 0;
+    size_t moved_bytes = 0;
+    status = PerformTrivialMove(*c.get(), log_buffer, compaction_released,
+                                moved_files, moved_bytes);
     io_s = versions_->io_status();
     InstallSuperVersionAndScheduleWork(
         c->column_family_data(), job_context->superversion_contexts.data());
@@ -4136,8 +4211,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
           << "total_files_size" << moved_bytes;
     }
     ROCKS_LOG_BUFFER(
-        log_buffer,
-        "[%s] Moved #%d files to level-%d %" PRIu64 " bytes %s: %s\n",
+        log_buffer, "[%s] Moved #%d files to level-%zu %zu bytes %s: %s\n",
         c->column_family_data()->GetName().c_str(), moved_files,
         c->output_level(), moved_bytes, status.ToString().c_str(),
         c->column_family_data()->current()->storage_info()->LevelSummary(&tmp));
@@ -4428,7 +4502,7 @@ Compaction* DBImpl::CreateIntendedCompactionForwardedToBottomPriorityPool(
                      c->output_level(), c->target_output_file_size(),
                      c->max_compaction_bytes(), c->output_path_id(),
                      c->output_compression(), c->output_compression_opts(),
-                     c->output_temperature(), c->max_subcompactions(),
+                     c->GetOutputTemperature(), c->max_subcompactions(),
                      c->grandparents(), std::nullopt /* earliest_snapshot */,
                      nullptr /* snapshot_checker */, c->compaction_reason());
 

db/db_impl/db_impl_files.cc

@@ -614,6 +614,11 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
       case kOptionsFile:
         keep = (number >= optsfile_num2);
         break;
+      case kCompactionProgressFile:
+        // Keep compaction progress files - they are managed
+        // separately by DBImplSecondary for now
+        keep = true;
+        break;
       case kCurrentFile:
       case kDBLockFile:
       case kIdentityFile:

db/db_impl/db_impl_follower.cc

@@ -293,9 +293,9 @@ Status DB::OpenAsFollower(
   DBImplFollower* impl =
       new DBImplFollower(tmp_opts, std::move(new_env), dbname, src_path);
   impl->versions_.reset(new ReactiveVersionSet(
-      dbname, &impl->immutable_db_options_, impl->file_options_,
-      impl->table_cache_.get(), impl->write_buffer_manager_,
-      &impl->write_controller_, impl->io_tracer_));
+      dbname, &impl->immutable_db_options_, impl->mutable_db_options_,
+      impl->file_options_, impl->table_cache_.get(),
+      impl->write_buffer_manager_, &impl->write_controller_, impl->io_tracer_));
   impl->column_family_memtables_.reset(
       new ColumnFamilyMemTablesImpl(impl->versions_->GetColumnFamilySet()));
   impl->wal_in_db_path_ = impl->immutable_db_options_.IsWalDirSameAsDBPath();

db/db_impl/db_impl_open.cc

@@ -329,7 +329,7 @@ Status DBImpl::NewDB(std::vector<std::string>* new_filenames) {
     }
     FileTypeSet tmp_set = immutable_db_options_.checksum_handoff_file_types;
     file->SetPreallocationBlockSize(
-        immutable_db_options_.manifest_preallocation_size);
+        mutable_db_options_.manifest_preallocation_size);
     std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
         std::move(file), manifest, file_options, immutable_db_options_.clock,
         io_tracer_, nullptr /* stats */,

db/db_impl/db_impl_secondary.cc

@@ -8,7 +8,12 @@
 #include <cinttypes>
 
 #include "db/arena_wrapped_db_iter.h"
+#include "db/log_reader.h"
+#include "db/log_writer.h"
 #include "db/merge_context.h"
+#include "db/version_edit.h"
+#include "file/filename.h"
+#include "file/writable_file_writer.h"
 #include "logging/auto_roll_logger.h"
 #include "logging/logging.h"
 #include "monitoring/perf_context_imp.h"
@@ -778,9 +783,9 @@ Status DB::OpenAsSecondary(
   handles->clear();
   DBImplSecondary* impl = new DBImplSecondary(tmp_opts, dbname, secondary_path);
   impl->versions_.reset(new ReactiveVersionSet(
-      dbname, &impl->immutable_db_options_, impl->file_options_,
-      impl->table_cache_.get(), impl->write_buffer_manager_,
-      &impl->write_controller_, impl->io_tracer_));
+      dbname, &impl->immutable_db_options_, impl->mutable_db_options_,
+      impl->file_options_, impl->table_cache_.get(),
+      impl->write_buffer_manager_, &impl->write_controller_, impl->io_tracer_));
   impl->column_family_memtables_.reset(
       new ColumnFamilyMemTablesImpl(impl->versions_->GetColumnFamilySet()));
   impl->wal_in_db_path_ = impl->immutable_db_options_.IsWalDirSameAsDBPath();
@@ -823,18 +828,507 @@ Status DB::OpenAsSecondary(
   return s;
 }
 
+Status DBImplSecondary::ScanCompactionProgressFiles(
+    CompactionProgressFilesScan* scan_result) {
+  assert(scan_result != nullptr);
+  scan_result->Clear();
+
+  WriteOptions write_options(Env::IOActivity::kCompaction);
+  IOOptions opts;
+  Status s = WritableFileWriter::PrepareIOOptions(write_options, opts);
+  if (!s.ok()) {
+    return s;
+  }
+
+  std::vector<std::string> all_filenames;
+  s = fs_->GetChildren(secondary_path_, opts, &all_filenames, nullptr /* dbg*/);
+  if (!s.ok()) {
+    return s;
+  }
+
+  for (const auto& filename : all_filenames) {
+    if (filename == "." || filename == "..") {
+      continue;
+    }
+
+    uint64_t number;
+    FileType type;
+
+    if (!ParseFileName(filename, &number, &type)) {
+      continue;
+    }
+
+    // Categorize compaction progress files
+    if (type == kCompactionProgressFile) {
+      if (number > scan_result->latest_progress_timestamp) {
+        // Found a newer progress file
+        if (scan_result->HasLatestProgressFile()) {
+          // Previous "latest" becomes "old"
+          scan_result->old_progress_filenames.push_back(
+              scan_result->latest_progress_filename.value());
+        }
+        scan_result->latest_progress_timestamp = number;
+        scan_result->latest_progress_filename = filename;
+      } else {
+        // This is an older progress file
+        scan_result->old_progress_filenames.push_back(filename);
+      }
+    } else if (type == kTempFile &&
+               filename.find(kCompactionProgressFileNamePrefix) == 0) {
+      // Temporary progress files
+      scan_result->temp_progress_filenames.push_back(filename);
+    } else if (type == kTableFile) {
+      // Collect table file numbers for CleanupPhysicalCompactionOutputFiles
+      scan_result->table_file_numbers.push_back(number);
+    }
+  }
+
+  return Status::OK();
+}
+
+Status DBImplSecondary::DeleteCompactionProgressFiles(
+    const std::vector<std::string>& filenames) {
+  WriteOptions write_options(Env::IOActivity::kCompaction);
+  IOOptions opts;
+  Status s = WritableFileWriter::PrepareIOOptions(write_options, opts);
+  if (!s.ok()) {
+    return s;
+  }
+
+  for (const auto& filename : filenames) {
+    std::string file_path = secondary_path_ + "/" + filename;
+    Status delete_status = fs_->DeleteFile(file_path, opts, nullptr /* dbg */);
+    if (!delete_status.ok()) {
+      return delete_status;
+    }
+  }
+
+  return Status::OK();
+}
+
+Status DBImplSecondary::CleanupOldAndTemporaryCompactionProgressFiles(
+    bool preserve_latest, const CompactionProgressFilesScan& scan_result) {
+  std::vector<std::string> filenames_to_delete;
+
+  // Always delete old progress files
+  filenames_to_delete.insert(filenames_to_delete.end(),
+                             scan_result.old_progress_filenames.begin(),
+                             scan_result.old_progress_filenames.end());
+
+  // Always delete temp files
+  filenames_to_delete.insert(filenames_to_delete.end(),
+                             scan_result.temp_progress_filenames.begin(),
+                             scan_result.temp_progress_filenames.end());
+
+  // Conditionally delete latest file
+  if (!preserve_latest && scan_result.HasLatestProgressFile()) {
+    filenames_to_delete.push_back(scan_result.latest_progress_filename.value());
+  }
+
+  return DeleteCompactionProgressFiles(filenames_to_delete);
+}
+
+// Loads compaction progress from a file and cleans up extra output
+// files. After loading the progress, this function identifies and deletes any
+// SST files in the output folder that are NOT tracked in the
+// progress. This ensures consistency between the progress file and
+// actual output files on disk.
+Status DBImplSecondary::LoadCompactionProgressAndCleanupExtraOutputFiles(
+    const std::string& compaction_progress_file_path,
+    const CompactionProgressFilesScan& scan_result) {
+  Status s = ParseCompactionProgressFile(compaction_progress_file_path,
+                                         &compaction_progress_);
+  if (s.ok()) {
+    s = CleanupPhysicalCompactionOutputFiles(true /* preserve_tracked_files */,
+                                             scan_result);
+  }
+  return s;
+}
+
+Status DBImplSecondary::ParseCompactionProgressFile(
+    const std::string& compaction_progress_file_path,
+    CompactionProgress* compaction_progress) {
+  std::unique_ptr<FSSequentialFile> file;
+  Status s = fs_->NewSequentialFile(compaction_progress_file_path,
+                                    FileOptions(), &file, nullptr /* dbg */);
+  if (!s.ok()) {
+    return s;
+  }
+
+  std::unique_ptr<SequentialFileReader> file_reader(new SequentialFileReader(
+      std::move(file), compaction_progress_file_path,
+      immutable_db_options_.log_readahead_size, io_tracer_, {} /* listeners */,
+      immutable_db_options_.rate_limiter.get()));
+
+  Status reader_status;
+
+  struct CompactionProgressReaderReporter : public log::Reader::Reporter {
+    Status* status;
+    explicit CompactionProgressReaderReporter(Status* s) : status(s) {}
+
+    void Corruption(size_t /*bytes*/, const Status& s,
+                    uint64_t /*log_number*/) override {
+      if (status->ok()) {
+        *status = s;
+      }
+    }
+
+    void OldLogRecord(size_t /*bytes*/) override {
+      // Ignore old records
+    }
+  } progress_reporter(&reader_status);
+
+  log::Reader compaction_progress_reader(
+      immutable_db_options_.info_log, std::move(file_reader),
+      &progress_reporter, true /* checksum */, 0 /* log_num */);
+
+  // LIMITATION: Only supports resuming single subcompaction
+  SubcompactionProgressBuilder progress_builder;
+  Slice slice;
+  std::string record;
+
+  while (compaction_progress_reader.ReadRecord(&slice, &record)) {
+    if (!reader_status.ok()) {
+      return reader_status;
+    }
+
+    VersionEdit edit;
+    s = edit.DecodeFrom(slice);
+    if (!s.ok()) {
+      break;
+    }
+
+    bool res = progress_builder.ProcessVersionEdit(edit);
+    if (!res) {
+      break;
+    }
+  }
+
+  if (!s.ok()) {
+    return s;
+  }
+
+  if (progress_builder.HasAccumulatedSubcompactionProgress()) {
+    compaction_progress->clear();
+    compaction_progress->push_back(
+        progress_builder.GetAccumulatedSubcompactionProgress());
+  } else {
+    s = Status::NotFound("No compaction progress was persisted yet");
+  }
+
+  return s;
+}
+
+Status DBImplSecondary::RenameCompactionProgressFile(
+    const std::string& temp_file_path, std::string* final_file_path) {
+  uint64_t current_time = env_->NowMicros();
+  *final_file_path = CompactionProgressFileName(secondary_path_, current_time);
+
+  WriteOptions write_options(Env::IOActivity::kCompaction);
+  IOOptions opts;
+  Status s = WritableFileWriter::PrepareIOOptions(write_options, opts);
+  if (!s.ok()) {
+    return s;
+  }
+
+  s = fs_->RenameFile(temp_file_path, *final_file_path, opts,
+                      nullptr /* dbg */);
+
+  return s;
+}
+
+Status DBImplSecondary::CleanupPhysicalCompactionOutputFiles(
+    bool preserve_tracked_files,
+    const CompactionProgressFilesScan& scan_result) {
+  std::unordered_set<uint64_t> files_to_preserve;
+
+  if (preserve_tracked_files) {
+    for (const auto& subcompaction_progress : compaction_progress_) {
+      for (const auto& file_metadata :
+           subcompaction_progress.output_level_progress.GetOutputFiles()) {
+        files_to_preserve.insert(file_metadata.fd.GetNumber());
+      }
+      for (const auto& file_metadata :
+           subcompaction_progress.proximal_output_level_progress
+               .GetOutputFiles()) {
+        files_to_preserve.insert(file_metadata.fd.GetNumber());
+      }
+    }
+  }
+
+  WriteOptions write_options(Env::IOActivity::kCompaction);
+  IOOptions opts;
+  Status s = WritableFileWriter::PrepareIOOptions(write_options, opts);
+  if (!s.ok()) {
+    return s;
+  }
+
+  for (uint64_t file_number : scan_result.table_file_numbers) {
+    bool should_delete =
+        !preserve_tracked_files ||
+        (files_to_preserve.find(file_number) == files_to_preserve.end());
+
+    if (should_delete) {
+      std::string file_path = MakeTableFileName(secondary_path_, file_number);
+      Status delete_status =
+          fs_->DeleteFile(file_path, opts, nullptr /* dbg */);
+      if (!delete_status.ok()) {
+        return delete_status;
+      }
+    }
+  }
+
+  return Status::OK();
+}
+
+Status DBImplSecondary::InitializeCompactionWorkspace(
+    bool allow_resumption, std::unique_ptr<FSDirectory>* output_dir,
+    std::unique_ptr<log::Writer>* compaction_progress_writer) {
+  // Create output directory if it doest exist yet
+  Status s = CreateAndNewDirectory(fs_.get(), secondary_path_, output_dir);
+  if (!s.ok() || !allow_resumption) {
+    return s;
+  }
+
+  s = PrepareCompactionProgressState();
+
+  if (!s.ok()) {
+    return s;
+  }
+
+  s = FinalizeCompactionProgressWriter(compaction_progress_writer);
+
+  if (!s.ok()) {
+    return s;
+  }
+
+  ROCKS_LOG_INFO(immutable_db_options_.info_log,
+                 "Initialized compaction workspace with %zu subcompaction "
+                 "progress to resume",
+                 compaction_progress_.size());
+
+  return Status::OK();
+}
+
+// PrepareCompactionProgressState() manages compaction progress files and output
+// files to ensure a clean, consistent state for resuming or starting fresh
+// compaction.
+//
+// PRECONDITION:
+// - This function is ONLY called when allow_resumption = true
+// - The caller wants resumption support for this compaction attempt
+//
+// FILE SYSTEM STATE (before entering this function):
+// - 0 or more compaction progress files may exist in `secondary_path_`:
+//   * Latest progress file (from the most recent compaction attempt)
+//   * Older progress files (left by crashing during a previous
+//     InitializeCompactionWorkspace() call)
+//   * Temporary progress files (left by crashing during a previous
+//     InitializeCompactionWorkspace() call)
+// - 0 or more compaction output files may exist in `secondary_path_`
+//
+// POSTCONDITIONS (after this function):
+// - IF the latest progress file exists AND it parses successfully AND
+//   actually contains valid compaction progress:
+//   * Exactly one latest progress file remains
+//   * All older and temporary compaction progress files are deleted
+//   * All corresponding compaction output files are preserved
+//   * All extra compaction output files are deleted (files left by
+//   compaction
+//     crashing before persisting the progress)
+//   * Result: Ready to resume compaction from the saved progress
+// - OTHERWISE (no latest progress file OR it fails to parse OR it's
+// invalid):
+//   * ALL compaction progress files are deleted (latest + older +
+//   temporary)
+//   * ALL compaction output files are deleted
+//   * Result: Ready to start fresh compaction (despite allow_resumption =
+//   true, we cannot resume because there's no valid progress to resume from)
+//
+// ERROR HANDLING:
+// - ON ERROR (if any of the postconditions cannot be achieved):
+//   * Function returns error status
+//   * File system may be left in a partially modified state
+//   * Caller should manually clean up secondary_path_ before retrying
+//   * Subsequent OpenAndCompact() calls to this clean secondary_path_ will
+//     effectively start fresh compaction
+Status DBImplSecondary::PrepareCompactionProgressState() {
+  Status s;
+
+  // STEP 1: Scan directory ONCE (includes progress files + table files)
+  CompactionProgressFilesScan scan_result;
+  s = ScanCompactionProgressFiles(&scan_result);
+  if (!s.ok()) {
+    ROCKS_LOG_ERROR(immutable_db_options_.info_log,
+                    "Encountered error when scanning for compaction "
+                    "progress files: %s",
+                    s.ToString().c_str());
+    return s;
+  }
+
+  std::optional<std::string> latest_progress_file =
+      scan_result.latest_progress_filename;
+
+  // STEP 2: Determine if we should resume
+  bool should_resume = false;
+  if (latest_progress_file.has_value()) {
+    should_resume = true;
+  } else {
+    ROCKS_LOG_WARN(immutable_db_options_.info_log,
+                   "Did not find any latest compaction progress file. "
+                   "Will perform clean up to start fresh compaction");
+  }
+
+  // STEP 3: Cleanup using pre-scanned results
+  if (should_resume) {
+    // Keep latest, delete old/temp
+    s = CleanupOldAndTemporaryCompactionProgressFiles(
+        true /* preserve_latest */, scan_result);
+  } else {
+    // Delete everything including latest
+    s = CleanupOldAndTemporaryCompactionProgressFiles(
+        false /* preserve_latest */, scan_result);
+    latest_progress_file.reset();
+  }
+
+  if (!s.ok()) {
+    ROCKS_LOG_ERROR(immutable_db_options_.info_log,
+                    "Failed to clean up compaction progress file(s): %s. "
+                    "Will fail the compaction",
+                    s.ToString().c_str());
+    return s;
+  }
+
+  // STEP 4: Load progress if resuming
+  if (latest_progress_file.has_value()) {
+    uint64_t timestamp = scan_result.latest_progress_timestamp;
+
+    std::string compaction_progress_file_path =
+        CompactionProgressFileName(secondary_path_, timestamp);
+
+    s = LoadCompactionProgressAndCleanupExtraOutputFiles(
+        compaction_progress_file_path, scan_result);
+
+    if (!s.ok()) {
+      ROCKS_LOG_WARN(immutable_db_options_.info_log,
+                     "Failed to load the latest compaction "
+                     "progress from %s: %s. Will perform clean up "
+                     "to start fresh compaction",
+                     latest_progress_file.value().c_str(),
+                     s.ToString().c_str());
+      return HandleInvalidOrNoCompactionProgress(compaction_progress_file_path,
+                                                 scan_result);
+    }
+    return s;
+  } else {
+    return HandleInvalidOrNoCompactionProgress(
+        std::nullopt /* compaction_progress_file_path */, scan_result);
+  }
+}
+
+uint64_t DBImplSecondary::CalculateResumedCompactionBytes(
+    const CompactionProgress& compaction_progress) const {
+  uint64_t total_resumed_bytes = 0;
+
+  for (const auto& subcompaction_progress : compaction_progress) {
+    for (const auto& file_meta :
+         subcompaction_progress.output_level_progress.GetOutputFiles()) {
+      total_resumed_bytes += file_meta.fd.file_size;
+    }
+
+    for (const auto& file_meta :
+         subcompaction_progress.proximal_output_level_progress
+             .GetOutputFiles()) {
+      total_resumed_bytes += file_meta.fd.file_size;
+    }
+  }
+
+  return total_resumed_bytes;
+}
+
+Status DBImplSecondary::HandleInvalidOrNoCompactionProgress(
+    const std::optional<std::string>& compaction_progress_file_path,
+    const CompactionProgressFilesScan& scan_result) {
+  compaction_progress_.clear();
+
+  Status s;
+  if (compaction_progress_file_path.has_value()) {
+    WriteOptions write_options(Env::IOActivity::kCompaction);
+    IOOptions opts;
+    s = WritableFileWriter::PrepareIOOptions(write_options, opts);
+    if (s.ok()) {
+      s = fs_->DeleteFile(compaction_progress_file_path.value(), opts,
+                          nullptr /* dbg */);
+    }
+    if (!s.ok()) {
+      ROCKS_LOG_ERROR(immutable_db_options_.info_log,
+                      "Failed to remove invalid progress file: %s",
+                      s.ToString().c_str());
+      return s;
+    }
+  }
+
+  s = CleanupPhysicalCompactionOutputFiles(false /* preserve_tracked_files */,
+                                           scan_result);
+  if (!s.ok()) {
+    ROCKS_LOG_ERROR(immutable_db_options_.info_log,
+                    "Failed to cleanup existing compaction output files: %s",
+                    s.ToString().c_str());
+    return s;
+  }
+
+  return Status::OK();
+}
+
 Status DBImplSecondary::CompactWithoutInstallation(
     const OpenAndCompactOptions& options, ColumnFamilyHandle* cfh,
     const CompactionServiceInput& input, CompactionServiceResult* result) {
   if (options.canceled && options.canceled->load(std::memory_order_acquire)) {
     return Status::Incomplete(Status::SubCode::kManualCompactionPaused);
   }
+
+  std::unique_ptr<FSDirectory> output_dir;
+  std::unique_ptr<log::Writer> compaction_progress_writer;
+
   InstrumentedMutexLock l(&mutex_);
+
   auto cfd = static_cast_with_check<ColumnFamilyHandleImpl>(cfh)->cfd();
   if (!cfd) {
     return Status::InvalidArgument("Cannot find column family" +
                                    cfh->GetName());
   }
+  Status s;
+
+  // TODO(hx235): Resuming compaction is currently incompatible with
+  // paranoid_file_checks=true because OutputValidator hash verification would
+  // fail during compaction resumption. Before interruption, resuming
+  // compaction needs to persist the hash of each output file to enable
+  // validation after resumption. Alternatively and preferably, we could move
+  // the output verification to happen immediately after each output file is
+  // created. This workaround currently disables resuming compaction when
+  // paranoid_file_checks is enabled. Note that paranoid_file_checks is
+  // disabled by default.
+  bool allow_resumption =
+      options.allow_resumption &&
+      !cfd->GetLatestMutableCFOptions().paranoid_file_checks;
+
+  if (options.allow_resumption &&
+      cfd->GetLatestMutableCFOptions().paranoid_file_checks) {
+    ROCKS_LOG_WARN(immutable_db_options_.info_log,
+                   "Resume compaction configured but disabled due to "
+                   "incompatible with paranoid_file_checks=true");
+  }
+
+  mutex_.Unlock();
+
+  s = InitializeCompactionWorkspace(allow_resumption, &output_dir,
+                                    &compaction_progress_writer);
+
+  mutex_.Lock();
+
+  if (!s.ok()) {
+    return s;
+  }
 
   std::unordered_set<uint64_t> input_set;
   for (const auto& file_name : input.input_files) {
@@ -856,7 +1350,7 @@ Status DBImplSecondary::CompactWithoutInstallation(
       cfd->ioptions().level_compaction_dynamic_level_bytes);
 
   std::vector<CompactionInputFiles> input_files;
-  Status s = cfd->compaction_picker()->GetCompactionInputsFromFileNumbers(
+  s = cfd->compaction_picker()->GetCompactionInputsFromFileNumbers(
       &input_files, &input_set, vstorage, comp_options);
   if (!s.ok()) {
     ROCKS_LOG_ERROR(
@@ -879,23 +1373,22 @@ Status DBImplSecondary::CompactWithoutInstallation(
   // input instead of recreating it in the remote worker
   std::unique_ptr<Compaction> c;
   assert(cfd->compaction_picker());
+  std::optional<SequenceNumber> earliest_snapshot = std::nullopt;
+  // Standalone Range Deletion Optimization is only supported in Universal
+  // Compactions - https://github.com/facebook/rocksdb/pull/13078
+  if (cfd->GetLatestCFOptions().compaction_style ==
+      CompactionStyle::kCompactionStyleUniversal) {
+    earliest_snapshot = !job_context.snapshot_seqs.empty()
+                            ? job_context.snapshot_seqs.front()
+                            : kMaxSequenceNumber;
+  }
   c.reset(cfd->compaction_picker()->PickCompactionForCompactFiles(
       comp_options, input_files, input.output_level, vstorage,
       cfd->GetLatestMutableCFOptions(), mutable_db_options_, 0,
-      /*earliest_snapshot=*/job_context.snapshot_seqs.empty()
-          ? kMaxSequenceNumber
-          : job_context.snapshot_seqs.front(),
-      job_context.snapshot_checker));
+      earliest_snapshot, job_context.snapshot_checker));
   assert(c != nullptr);
   c->FinalizeInputInfo(version);
 
-  // Create output directory if it's not existed yet
-  std::unique_ptr<FSDirectory> output_dir;
-  s = CreateAndNewDirectory(fs_.get(), secondary_path_, &output_dir);
-  if (!s.ok()) {
-    return s;
-  }
-
   LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL,
                        immutable_db_options_.info_log.get());
 
@@ -913,13 +1406,15 @@ Status DBImplSecondary::CompactWithoutInstallation(
       options.canceled ? *options.canceled : kManualCompactionCanceledFalse_,
       input.db_id, db_session_id_, secondary_path_, input, result);
 
-  compaction_job.Prepare();
+  compaction_job.Prepare(compaction_progress_,
+                         compaction_progress_writer.get());
 
   mutex_.Unlock();
   s = compaction_job.Run();
   mutex_.Lock();
 
-  // clean up
+  // These cleanup functions handle metadata and state cleanup only and
+  // not the physical files
   compaction_job.io_status().PermitUncheckedError();
   compaction_job.CleanupCompaction();
   c->ReleaseCompactionFiles(s);
@@ -927,6 +1422,18 @@ Status DBImplSecondary::CompactWithoutInstallation(
 
   TEST_SYNC_POINT_CALLBACK("DBImplSecondary::CompactWithoutInstallation::End",
                            &s);
+
+  if (!compaction_progress_.empty() && s.ok()) {
+    uint64_t total_resumed_bytes =
+        CalculateResumedCompactionBytes(compaction_progress_);
+
+    if (total_resumed_bytes > 0 &&
+        immutable_db_options_.statistics != nullptr) {
+      RecordTick(immutable_db_options_.statistics.get(),
+                 REMOTE_COMPACT_RESUMED_BYTES, total_resumed_bytes);
+    }
+  }
+
   result->status = s;
   return s;
 }
@@ -1082,4 +1589,148 @@ Status DB::OpenAndCompact(
                         output, override_options);
 }
 
+Status DBImplSecondary::CreateCompactionProgressWriter(
+    const std::string& file_path,
+    std::unique_ptr<log::Writer>* compaction_progress_writer) {
+  std::unique_ptr<FSWritableFile> file;
+  Status s =
+      fs_->NewWritableFile(file_path, FileOptions(), &file, nullptr /* dbg */);
+  if (!s.ok()) {
+    return s;
+  }
+
+  std::unique_ptr<WritableFileWriter> file_writer(
+      new WritableFileWriter(std::move(file), file_path, FileOptions()));
+
+  compaction_progress_writer->reset(
+      new log::Writer(std::move(file_writer), 0 /* log_number */,
+                      false /* recycle_log_files */));
+
+  return Status::OK();
+}
+
+Status DBImplSecondary::PersistInitialCompactionProgress(
+    log::Writer* compaction_progress_writer,
+    const CompactionProgress& compaction_progress) {
+  assert(compaction_progress_writer);
+
+  // LIMITATION: Only supports resuming single subcompaction
+  assert(compaction_progress.size() == 1);
+  const SubcompactionProgress& subcompaction_progress = compaction_progress[0];
+
+  VersionEdit edit;
+  edit.SetSubcompactionProgress(subcompaction_progress);
+
+  std::string record;
+  if (!edit.EncodeTo(&record)) {
+    return Status::IOError("Failed to encode the initial compaction progress");
+  }
+
+  WriteOptions write_options(Env::IOActivity::kCompaction);
+  Status s = compaction_progress_writer->AddRecord(write_options, record);
+  if (!s.ok()) {
+    return s;
+  }
+  IOOptions opts;
+  s = WritableFileWriter::PrepareIOOptions(write_options, opts);
+  if (!s.ok()) {
+    return s;
+  }
+
+  s = compaction_progress_writer->file()->Sync(opts,
+                                               immutable_db_options_.use_fsync);
+
+  return s;
+}
+
+Status DBImplSecondary::HandleCompactionProgressWriterCreationFailure(
+    const std::string& temp_file_path, const std::string& final_file_path,
+    std::unique_ptr<log::Writer>* compaction_progress_writer) {
+  compaction_progress_writer->reset();
+
+  const std::vector<std::string> paths_to_delete = {final_file_path,
+                                                    temp_file_path};
+
+  Status s;
+  for (const auto& file_path : paths_to_delete) {
+    WriteOptions write_options(Env::IOActivity::kCompaction);
+    IOOptions opts;
+    s = WritableFileWriter::PrepareIOOptions(write_options, opts);
+    if (s.ok()) {
+      s = fs_->DeleteFile(file_path, opts, nullptr /* dbg */);
+    }
+
+    if (!s.ok()) {
+      ROCKS_LOG_ERROR(immutable_db_options_.info_log,
+                      "Failed to cleanup the compaction progress file "
+                      "during writer creation failure: %s",
+                      s.ToString().c_str());
+      return s;
+    }
+  }
+
+  return s;
+}
+
+Status DBImplSecondary::FinalizeCompactionProgressWriter(
+    std::unique_ptr<log::Writer>* compaction_progress_writer) {
+  uint64_t timestamp = env_->NowMicros();
+  const std::string temp_file_path =
+      TempCompactionProgressFileName(secondary_path_, timestamp);
+
+  Status s = CreateCompactionProgressWriter(temp_file_path,
+                                            compaction_progress_writer);
+  if (!s.ok()) {
+    ROCKS_LOG_WARN(immutable_db_options_.info_log,
+                   "Failed to create compaction progress writer at "
+                   "temp path %s: %s. Will perform clean up "
+                   "to start compaction without progress persistence",
+                   temp_file_path.c_str(), s.ToString().c_str());
+    return HandleCompactionProgressWriterCreationFailure(
+        temp_file_path, "" /* final_file_path */, compaction_progress_writer);
+  }
+
+  if (!compaction_progress_.empty()) {
+    s = PersistInitialCompactionProgress(compaction_progress_writer->get(),
+                                         compaction_progress_);
+    if (!s.ok()) {
+      ROCKS_LOG_WARN(immutable_db_options_.info_log,
+                     "Failed to persist the initial copmaction "
+                     "progress: %s. Will perform clean up "
+                     "to start compaction without progress persistence",
+                     s.ToString().c_str());
+      return HandleCompactionProgressWriterCreationFailure(
+          temp_file_path, "" /* final_file_path */, compaction_progress_writer);
+    }
+  }
+
+  compaction_progress_writer->reset();
+
+  std::string final_file_path;
+  s = RenameCompactionProgressFile(temp_file_path, &final_file_path);
+
+  if (!s.ok()) {
+    ROCKS_LOG_WARN(immutable_db_options_.info_log,
+                   "Failed to rename temporary compaction progress "
+                   "file from %s to %s: %s.  Will perform clean up "
+                   "to start compaction without progress persistence",
+                   temp_file_path.c_str(), final_file_path.c_str(),
+                   s.ToString().c_str());
+    return HandleCompactionProgressWriterCreationFailure(
+        temp_file_path, final_file_path, compaction_progress_writer);
+  }
+
+  s = CreateCompactionProgressWriter(final_file_path,
+                                     compaction_progress_writer);
+  if (!s.ok()) {
+    ROCKS_LOG_WARN(immutable_db_options_.info_log,
+                   "Failed to create the final compaction progress "
+                   "writer: %s. Will attempt clean to start the compaction "
+                   "without progress persistence",
+                   s.ToString().c_str());
+    return HandleCompactionProgressWriterCreationFailure(
+        "" /* temp_file_path */, final_file_path, compaction_progress_writer);
+  }
+  return Status::OK();
+}
 }  // namespace ROCKSDB_NAMESPACE

db/db_impl/db_impl_secondary.h

@@ -303,6 +303,87 @@ class DBImplSecondary : public DBImpl {
                                     const CompactionServiceInput& input,
                                     CompactionServiceResult* result);
 
+ private:
+  // Holds results of compaction progress files and output files from a single
+  // directory scan
+  struct CompactionProgressFilesScan {
+    // The latest (newest) progress file filename
+    std::optional<std::string> latest_progress_filename;
+    uint64_t latest_progress_timestamp = 0;
+
+    // Older progress file filenames (to be deleted)
+    autovector<std::string> old_progress_filenames;
+
+    // Temporary progress file filenames (to be deleted)
+    autovector<std::string> temp_progress_filenames;
+
+    // All output file numbers - for cleanup optimization
+    std::vector<uint64_t> table_file_numbers;
+
+    bool HasLatestProgressFile() const {
+      return latest_progress_filename.has_value();
+    }
+
+    void Clear() {
+      latest_progress_filename.reset();
+      latest_progress_timestamp = 0;
+      old_progress_filenames.clear();
+      temp_progress_filenames.clear();
+      table_file_numbers.clear();
+    }
+  };
+
+  Status InitializeCompactionWorkspace(
+      bool allow_resumption, std::unique_ptr<FSDirectory>* output_dir,
+      std::unique_ptr<log::Writer>* compaction_progress_writer);
+
+  Status PrepareCompactionProgressState();
+
+  Status ScanCompactionProgressFiles(CompactionProgressFilesScan* scan_result);
+
+  Status DeleteCompactionProgressFiles(
+      const std::vector<std::string>& filenames);
+
+  Status CleanupOldAndTemporaryCompactionProgressFiles(
+      bool preserve_latest, const CompactionProgressFilesScan& scan_result);
+
+  Status LoadCompactionProgressAndCleanupExtraOutputFiles(
+      const std::string& compaction_progress_file_path,
+      const CompactionProgressFilesScan& scan_result);
+
+  Status ParseCompactionProgressFile(
+      const std::string& compaction_progress_file_path,
+      CompactionProgress* compaction_progress);
+
+  Status HandleInvalidOrNoCompactionProgress(
+      const std::optional<std::string>& compaction_progress_file_path,
+      const CompactionProgressFilesScan& scan_result);
+
+  Status CleanupPhysicalCompactionOutputFiles(
+      bool preserve_tracked_files,
+      const CompactionProgressFilesScan& scan_result);
+
+  Status FinalizeCompactionProgressWriter(
+      std::unique_ptr<log::Writer>* compaction_progress_writer);
+
+  Status CreateCompactionProgressWriter(
+      const std::string& file_path,
+      std::unique_ptr<log::Writer>* compaction_progress_writer);
+
+  Status PersistInitialCompactionProgress(
+      log::Writer* compaction_progress_writer,
+      const CompactionProgress& compaction_progress);
+
+  Status RenameCompactionProgressFile(const std::string& temp_file_path,
+                                      std::string* final_file_path);
+
+  Status HandleCompactionProgressWriterCreationFailure(
+      const std::string& temp_file_path, const std::string& final_file_path,
+      std::unique_ptr<log::Writer>* compaction_progress_writer);
+
+  uint64_t CalculateResumedCompactionBytes(
+      const CompactionProgress& compaction_progress) const;
+
   // Cache log readers for each log number, used for continue WAL replay
   // after recovery
   std::map<uint64_t, std::unique_ptr<LogReaderContainer>> log_readers_;
@@ -311,6 +392,8 @@ class DBImplSecondary : public DBImpl {
   std::unordered_map<ColumnFamilyData*, uint64_t> cfd_to_current_log_;
 
   const std::string secondary_path_;
+
+  CompactionProgress compaction_progress_;
 };
 
 }  // namespace ROCKSDB_NAMESPACE

db/db_iter.cc

@@ -1565,11 +1565,115 @@ void DBIter::SetSavedKeyToSeekForPrevTarget(const Slice& target) {
   }
 }
 
+Status DBIter::ValidateScanOptions(const MultiScanArgs& multiscan_opts) const {
+  if (multiscan_opts.empty()) {
+    return Status::InvalidArgument("Empty MultiScanArgs");
+  }
+
+  const std::vector<ScanOptions>& scan_opts = multiscan_opts.GetScanRanges();
+  const bool has_limit = scan_opts.front().range.limit.has_value();
+  if (!has_limit && scan_opts.size() > 1) {
+    return Status::InvalidArgument("Scan has no upper bound");
+  }
+
+  for (size_t i = 0; i < scan_opts.size(); ++i) {
+    const auto& scan_range = scan_opts[i].range;
+    if (!scan_range.start.has_value()) {
+      return Status::InvalidArgument("Scan has no start key at index " +
+                                     std::to_string(i));
+    }
+
+    if (scan_range.limit.has_value()) {
+      if (user_comparator_.CompareWithoutTimestamp(
+              scan_range.start.value(), /*a_has_ts=*/false,
+              scan_range.limit.value(), /*b_has_ts=*/false) >= 0) {
+        return Status::InvalidArgument(
+            "Scan start key is large or equal than limit at index " +
+            std::to_string(i));
+      }
+    }
+
+    if (i > 0) {
+      if (!scan_range.limit.has_value()) {
+        // multiple scan without limit scan ranges
+        return Status::InvalidArgument("Scan has no upper bound at index " +
+                                       std::to_string(i));
+      }
+
+      const auto& last_end_key = scan_opts[i - 1].range.limit.value();
+      if (user_comparator_.CompareWithoutTimestamp(
+              scan_range.start.value(), /*a_has_ts=*/false, last_end_key,
+              /*b_has_ts=*/false) < 0) {
+        return Status::InvalidArgument("Overlapping ranges at index " +
+                                       std::to_string(i));
+      }
+    }
+  }
+  return Status::OK();
+}
+
+void DBIter::Prepare(const MultiScanArgs& scan_opts) {
+  status_ = ValidateScanOptions(scan_opts);
+  if (!status_.ok()) {
+    return;
+  }
+  std::optional<MultiScanArgs> new_scan_opts;
+  new_scan_opts.emplace(scan_opts);
+  scan_opts_.swap(new_scan_opts);
+  scan_index_ = 0;
+  if (!scan_opts.empty()) {
+    iter_.Prepare(&scan_opts_.value());
+  } else {
+    iter_.Prepare(nullptr);
+  }
+}
+
 void DBIter::Seek(const Slice& target) {
   PERF_COUNTER_ADD(iter_seek_count, 1);
   PERF_CPU_TIMER_GUARD(iter_seek_cpu_nanos, clock_);
   StopWatch sw(clock_, statistics_, DB_SEEK);
 
+  if (scan_opts_.has_value()) {
+    // Validate the seek target is as expected in the previously prepared range
+    auto const& scan_ranges = scan_opts_.value().GetScanRanges();
+    if (scan_index_ >= scan_ranges.size()) {
+      status_ = Status::InvalidArgument(
+          "Seek called after exhausting all of the scan ranges");
+      valid_ = false;
+      return;
+    }
+
+    // Validate start key of next prepare range matches the seek target
+    auto const& range = scan_ranges[scan_index_];
+    auto const& start = range.range.start;
+    assert(start.has_value());
+    if (user_comparator_.CompareWithoutTimestamp(target, *start) != 0) {
+      status_ = Status::InvalidArgument(
+          "Seek target does not match the start of the next prepared range at "
+          "index " +
+          std::to_string(scan_index_));
+      valid_ = false;
+      return;
+    }
+
+    // validate the upper bound is set to the same value of limit, if limit
+    // exists
+    auto const& limit = range.range.limit;
+    if (limit.has_value()) {
+      if (iterate_upper_bound_ == nullptr ||
+          user_comparator_.CompareWithoutTimestamp(
+              limit.value(), *iterate_upper_bound_) != 0) {
+        status_ = Status::InvalidArgument(
+            "Upper bound is not set to the same limit value of the next "
+            "prepared range at index " +
+            std::to_string(scan_index_));
+        valid_ = false;
+        return;
+      }
+    }
+    scan_index_++;
+  }
+
   if (cfh_ != nullptr) {
     // TODO: What do we do if this returns an error?
     Slice lower_bound, upper_bound;

db/db_iter.h

@@ -240,16 +240,8 @@ class DBIter final : public Iterator {
 
   bool PrepareValue() override;
 
-  void Prepare(const MultiScanArgs& scan_opts) override {
-    std::optional<MultiScanArgs> new_scan_opts;
-    new_scan_opts.emplace(scan_opts);
-    scan_opts_.swap(new_scan_opts);
-    if (!scan_opts.empty()) {
-      iter_.Prepare(&scan_opts_.value());
-    } else {
-      iter_.Prepare(nullptr);
-    }
-  }
+  void Prepare(const MultiScanArgs& scan_opts) override;
+  Status ValidateScanOptions(const MultiScanArgs& multiscan_opts) const;
 
  private:
   DBIter(Env* _env, const ReadOptions& read_options,
@@ -506,6 +498,7 @@ class DBIter final : public Iterator {
   const size_t timestamp_size_;
   std::string saved_timestamp_;
   std::optional<MultiScanArgs> scan_opts_;
+  size_t scan_index_{0};
   ReadOnlyMemTable* const active_mem_;
   SequenceNumber memtable_seqno_lb_;
   uint32_t memtable_op_scan_flush_trigger_;

db/db_iterator_test.cc

@@ -4149,6 +4149,7 @@ class DBMultiScanIteratorTest : public DBTestBase,
       : DBTestBase("db_multi_scan_iterator_test", /*env_do_fsync=*/true) {}
 };
 
+// Param 0: ReadOptions::fill_cache
 INSTANTIATE_TEST_CASE_P(DBMultiScanIteratorTest, DBMultiScanIteratorTest,
                         ::testing::Bool());
 
@@ -4192,66 +4193,6 @@ TEST_P(DBMultiScanIteratorTest, BasicTest) {
     abort();
   }
   iter.reset();
-
-  // Test the overlapping scan case
-  key_ranges[1] = "k30";
-  scan_options = MultiScanArgs(BytewiseComparator());
-  scan_options.insert(key_ranges[0], key_ranges[1]);
-  scan_options.insert(key_ranges[2], key_ranges[3]);
-
-  iter = dbfull()->NewMultiScan(ro, cfh, scan_options);
-  try {
-    int idx = 0;
-    int count = 0;
-    for (auto range : *iter) {
-      for (auto it : range) {
-        ASSERT_GE(it.first.ToString().compare(key_ranges[idx]), 0);
-        ASSERT_LT(it.first.ToString().compare(key_ranges[idx + 1]), 0);
-        count++;
-      }
-      idx += 2;
-    }
-    ASSERT_EQ(count, 52);
-  } catch (MultiScanException& ex) {
-    // Make sure exception contains the status
-    ASSERT_NOK(ex.status());
-    std::cerr << "Iterator returned status " << ex.what();
-    abort();
-  } catch (std::logic_error& ex) {
-    std::cerr << "Iterator returned logic error " << ex.what();
-    abort();
-  }
-  iter.reset();
-
-  // Test the no limit scan case
-  scan_options = MultiScanArgs(BytewiseComparator());
-  scan_options.insert(key_ranges[0]);
-  scan_options.insert(key_ranges[2]);
-  iter = dbfull()->NewMultiScan(ro, cfh, scan_options);
-  try {
-    int idx = 0;
-    int count = 0;
-    for (auto range : *iter) {
-      for (auto it : range) {
-        ASSERT_GE(it.first.ToString().compare(key_ranges[idx]), 0);
-        if (it.first.ToString().compare(key_ranges[idx + 1]) == 0) {
-          break;
-        }
-        count++;
-      }
-      idx += 2;
-    }
-    ASSERT_EQ(count, 52);
-  } catch (MultiScanException& ex) {
-    // Make sure exception contains the status
-    ASSERT_NOK(ex.status());
-    std::cerr << "Iterator returned status " << ex.what();
-    abort();
-  } catch (std::logic_error& ex) {
-    std::cerr << "Iterator returned logic error " << ex.what();
-    abort();
-  }
-  iter.reset();
 }
 
 TEST_P(DBMultiScanIteratorTest, MixedBoundsTest) {
@@ -4366,15 +4307,365 @@ TEST_P(DBMultiScanIteratorTest, RangeAcrossFiles) {
   ColumnFamilyHandle* cfh = dbfull()->DefaultColumnFamily();
   std::unique_ptr<MultiScan> iter =
       dbfull()->NewMultiScan(ro, cfh, scan_options);
-  int i = 10;
-  for (auto range : *iter) {
-    for (auto it : range) {
-      ASSERT_EQ(it.first.ToString(), Key(i));
-      ++i;
+  try {
+    int i = 10;
+    for (auto range : *iter) {
+      for (auto it : range) {
+        ASSERT_EQ(it.first.ToString(), Key(i));
+        ++i;
+      }
     }
+    ASSERT_EQ(i, 90);
+  } catch (MultiScanException& ex) {
+    // Make sure exception contains the status
+    ASSERT_NOK(ex.status());
+    std::cerr << "Iterator returned status " << ex.what();
+    abort();
+  } catch (std::logic_error& ex) {
+    std::cerr << "Iterator returned logic error " << ex.what();
+    abort();
   }
-  ASSERT_EQ(i, 90);
+  iter.reset();
 }
+
+TEST_P(DBMultiScanIteratorTest, FailureTest) {
+  auto options = CurrentOptions();
+  options.compression = kNoCompression;
+  DestroyAndReopen(options);
+
+  Random rnd(301);
+  // Create a file
+  for (int i = 0; i < 100; ++i) {
+    std::stringstream ss;
+    ss << std::setw(2) << std::setfill('0') << i;
+    ASSERT_OK(Put("k" + ss.str(), rnd.RandomString(1024)));
+  }
+  ASSERT_OK(Flush());
+
+  std::vector<std::string> key_ranges({"k04", "k06", "k12", "k14"});
+  ReadOptions ro;
+  Slice ub;
+  ro.iterate_upper_bound = &ub;
+  ro.fill_cache = GetParam();
+  MultiScanArgs scan_options(BytewiseComparator());
+  scan_options.insert(key_ranges[0], key_ranges[1]);
+  scan_options.insert(key_ranges[2], key_ranges[3]);
+  scan_options.max_prefetch_size = 4500;
+  ColumnFamilyHandle* cfh = dbfull()->DefaultColumnFamily();
+  std::unique_ptr<Iterator> iter(dbfull()->NewIterator(ro, cfh));
+  ASSERT_NE(iter, nullptr);
+  iter->Prepare(scan_options);
+  int count = 0;
+  ub = key_ranges[1];
+  iter->Seek(key_ranges[0]);
+  while (iter->status().ok() && iter->Valid()) {
+    ASSERT_GE(iter->key().compare(key_ranges[0]), 0);
+    ASSERT_LT(iter->key().compare(key_ranges[1]), 0);
+    count++;
+    iter->Next();
+  }
+  ASSERT_OK(iter->status()) << iter->status().ToString();
+  ASSERT_EQ(count, 2);
+
+  // Second seek should hit the max_prefetch_size limit
+  ub = key_ranges[3];
+  iter->Seek(key_ranges[2]);
+  ASSERT_NOK(iter->status());
+  iter.reset();
+
+  // Test the case of unexpected Seek key
+  iter.reset(dbfull()->NewIterator(ro, cfh));
+  ASSERT_NE(iter, nullptr);
+  scan_options.max_prefetch_size = 0;
+  iter->Prepare(scan_options);
+  ub = key_ranges[3];
+  iter->Seek(key_ranges[2]);
+  ASSERT_NOK(iter->status());
+  iter.reset();
+}
+
+TEST_P(DBMultiScanIteratorTest, OutOfL0FileRange) {
+  // Test that prepare does not fail scan when a scan range
+  // is outside of a L0 file's key range.
+  auto options = CurrentOptions();
+  options.compression = kNoCompression;
+  DestroyAndReopen(options);
+
+  Random rnd(301);
+  // Create a Lmax file
+  // key01 ~ key99
+  for (int i = 0; i < 100; ++i) {
+    std::stringstream ss;
+    ss << std::setw(2) << std::setfill('0') << i;
+    ASSERT_OK(Put("k" + ss.str(), rnd.RandomString(1024)));
+  }
+  ASSERT_OK(Flush());
+  CompactRangeOptions cro;
+  cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
+  ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
+
+  // Create a L0 file
+  // key00 ~ key09
+  for (int i = 0; i < 10; ++i) {
+    std::stringstream ss;
+    ss << std::setw(2) << std::setfill('0') << i;
+    ASSERT_OK(Put("k" + ss.str(), rnd.RandomString(1024)));
+  }
+  ASSERT_OK(Flush());
+  ASSERT_EQ(NumTableFilesAtLevel(0), 1);
+
+  // The second range is outside of L0 file's key range
+  std::vector<std::string> key_ranges({"k04", "k06", "k12", "k14"});
+  ReadOptions ro;
+  Slice ub;
+  ro.iterate_upper_bound = &ub;
+  ro.fill_cache = GetParam();
+  MultiScanArgs scan_options(BytewiseComparator());
+  scan_options.insert(key_ranges[0], key_ranges[1]);
+  scan_options.insert(key_ranges[2], key_ranges[3]);
+  ColumnFamilyHandle* cfh = dbfull()->DefaultColumnFamily();
+  std::unique_ptr<Iterator> iter(dbfull()->NewIterator(ro, cfh));
+  ASSERT_NE(iter, nullptr);
+  iter->Prepare(scan_options);
+  int count = 0;
+  ub = key_ranges[1];
+  iter->Seek(key_ranges[0]);
+  while (iter->status().ok() && iter->Valid()) {
+    ASSERT_GE(iter->key().compare(key_ranges[0]), 0);
+    ASSERT_LT(iter->key().compare(key_ranges[1]), 0);
+    count++;
+    iter->Next();
+  }
+  ASSERT_OK(iter->status()) << iter->status().ToString();
+  ASSERT_EQ(count, 2);
+
+  ub = key_ranges[3];
+  count = 0;
+  iter->Seek(key_ranges[2]);
+  while (iter->status().ok() && iter->Valid()) {
+    ASSERT_GE(iter->key().compare(key_ranges[2]), 0);
+    ASSERT_LT(iter->key().compare(key_ranges[3]), 0);
+    count++;
+    iter->Next();
+  }
+  ASSERT_OK(iter->status()) << iter->status().ToString();
+  ASSERT_EQ(count, 2);
+}
+
+TEST_P(DBMultiScanIteratorTest, RangeBetweenFiles) {
+  auto options = CurrentOptions();
+  options.target_file_size_base = 100 << 10;  // 20KB
+  options.compaction_style = kCompactionStyleUniversal;
+  options.num_levels = 50;
+  options.compression = kNoCompression;
+  DestroyAndReopen(options);
+
+  auto rnd = Random::GetTLSInstance();
+  // Write ~200KB data
+  for (int i = 0; i < 100; ++i) {
+    ASSERT_OK(Put(Key(i), rnd->RandomString(2 << 10)));
+  }
+  ASSERT_OK(Flush());
+
+  ASSERT_OK(db_->CompactRange({}, nullptr, nullptr));
+  ASSERT_EQ(2, NumTableFilesAtLevel(49));
+
+  // Test with a scan range that overlaps an entire file, with upper bound
+  // between 2 files
+  std::vector<LiveFileMetaData> file_meta;
+  dbfull()->GetLiveFilesMetaData(&file_meta);
+  ASSERT_EQ(file_meta.size(), 2);
+  std::vector<std::string> key_ranges(4);
+  key_ranges[0] = file_meta[0].smallestkey;
+  key_ranges[1] = file_meta[0].largestkey + "0";
+  key_ranges[2] = file_meta[1].smallestkey + "0";
+  key_ranges[3] = file_meta[1].largestkey;
+  ReadOptions ro;
+  ro.fill_cache = GetParam();
+  MultiScanArgs scan_options(BytewiseComparator());
+  scan_options.insert(key_ranges[0], key_ranges[1]);
+  scan_options.insert(key_ranges[2], key_ranges[3]);
+  ColumnFamilyHandle* cfh = dbfull()->DefaultColumnFamily();
+  std::unique_ptr<MultiScan> iter =
+      dbfull()->NewMultiScan(ro, cfh, scan_options);
+  try {
+    for (auto range : *iter) {
+      for (auto it : range) {
+        ASSERT_GE(it.first.ToString(), key_ranges[0]);
+      }
+    }
+  } catch (MultiScanException& ex) {
+    // Make sure exception contains the status
+    ASSERT_NOK(ex.status());
+    std::cerr << "Iterator returned status " << ex.what();
+    abort();
+  } catch (std::logic_error& ex) {
+    std::cerr << "Iterator returned logic error " << ex.what();
+    abort();
+  }
+  iter.reset();
+
+  // Test multiscan with a range entirely between adjacent files
+  key_ranges[0] = file_meta[0].largestkey + "0";
+  key_ranges[1] = file_meta[0].largestkey + "1";
+  key_ranges[2] = file_meta[1].smallestkey + "0";
+  key_ranges[3] = file_meta[1].largestkey;
+  (*scan_options).clear();
+  scan_options.insert(key_ranges[0], key_ranges[1]);
+  scan_options.insert(key_ranges[2], key_ranges[3]);
+  iter = dbfull()->NewMultiScan(ro, cfh, scan_options);
+  try {
+    for (auto range : *iter) {
+      for (auto it : range) {
+        ASSERT_GE(it.first.ToString(), key_ranges[0]);
+      }
+    }
+  } catch (MultiScanException& ex) {
+    // Make sure exception contains the status
+    ASSERT_NOK(ex.status());
+    std::cerr << "Iterator returned status " << ex.what();
+    abort();
+  } catch (std::logic_error& ex) {
+    std::cerr << "Iterator returned logic error " << ex.what();
+    abort();
+  }
+  iter.reset();
+}
+
+// This test case tests multiscan in the presence of fragmented range
+// tombstones in the LSM.
+TEST_P(DBMultiScanIteratorTest, FragmentedRangeTombstones) {
+  auto options = CurrentOptions();
+  // Compaction may create files 2x the target_file_size_base,
+  // so set this to 50KB so we atleast end up with 2 files of
+  // 100KB
+  options.target_file_size_base = 50 << 10;  // 50KB
+  options.compaction_style = kCompactionStyleUniversal;
+  options.num_levels = 50;
+  options.compression = kNoCompression;
+  DestroyAndReopen(options);
+
+  // Setup the LSM as follows -
+  // 1. Ingest a file with 100 keys
+  // 2. Ingest a file with one overlapping key
+  // 3. Do a Put and flush a file to L0 with one overlapping key
+  // 4. Ingest a standalone delete range file that covers the full key space
+  //    and a file with the same 100 keys with new values. This will ingest
+  //    into L0 due to the presence of an existing file in L0
+  // The final LSM will have an SST in Lmax with 100 keys, and 2 SST files
+  // in Lmax-1 with half the keys each and completely overlapping delete ranges
+  std::unordered_map<std::string, std::string> kvs;
+  auto rnd = Random::GetTLSInstance();
+  auto create_ingestion_data_file_and_update_key_value =
+      [&](const std::string& filename, int start_key, int end_key) {
+        std::unique_ptr<SstFileWriter> writer;
+        writer.reset(new SstFileWriter(EnvOptions(), options));
+        ASSERT_OK(writer->Open(filename));
+        for (int i = start_key; i < end_key; ++i) {
+          auto kiter = kvs.find(Key(i));
+          if (kiter != kvs.end()) {
+            kvs.erase(kiter);
+          }
+          auto res =
+              kvs.emplace(std::make_pair(Key(i), rnd->RandomString(2 << 10)));
+          ASSERT_OK(writer->Put(res.first->first, res.first->second));
+        }
+        ASSERT_OK(writer->Finish());
+        writer.reset();
+      };
+
+  CreateColumnFamilies({"new_cf"}, options);
+  std::string ingest_file = dbname_ + "test.sst";
+  // Write ~200KB data
+  create_ingestion_data_file_and_update_key_value(ingest_file + "_0", 0, 100);
+  create_ingestion_data_file_and_update_key_value(ingest_file + "_1", 50, 51);
+  ColumnFamilyHandle* cfh = handles_[0];
+  IngestExternalFileOptions ifo;
+  Status s = dbfull()->IngestExternalFile(
+      cfh, {ingest_file + "_0", ingest_file + "_1"}, ifo);
+  ASSERT_OK(s);
+
+  ASSERT_OK(Put(0, Key(50), rnd->RandomString(2 << 10)));
+  ASSERT_OK(Flush());
+
+  {
+    std::unique_ptr<SstFileWriter> writer;
+    writer.reset(new SstFileWriter(EnvOptions(), options));
+    ASSERT_OK(writer->Open(ingest_file + "_2"));
+    ASSERT_OK(writer->DeleteRange("a", "z"));
+    ASSERT_OK(writer->Finish());
+    writer.reset();
+  }
+  create_ingestion_data_file_and_update_key_value(ingest_file + "_3", 0, 100);
+  s = dbfull()->IngestExternalFile(
+      cfh, {ingest_file + "_2", ingest_file + "_3"}, ifo);
+  ASSERT_OK(s);
+
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+
+  // The first scan range overlaps the DB key range, while the second extends
+  // beyond but overlaps the delete range
+  std::vector<std::string> key_ranges({"key000085", "key000090", "l", "n"});
+  ReadOptions ro;
+  ro.fill_cache = GetParam();
+  MultiScanArgs scan_options(BytewiseComparator());
+  scan_options.insert(key_ranges[0], key_ranges[1]);
+  scan_options.insert(key_ranges[2], key_ranges[3]);
+  std::unique_ptr<MultiScan> iter =
+      dbfull()->NewMultiScan(ro, cfh, scan_options);
+  try {
+    int i = 0;
+    int count = 0;
+    for (auto range : *iter) {
+      for (auto it : range) {
+        ASSERT_GE(it.first.ToString(), key_ranges[i]);
+        ASSERT_LT(it.first.ToString(), key_ranges[i + 1]);
+        auto kiter = kvs.find(it.first.ToString());
+        ASSERT_NE(kiter, kvs.end());
+        ASSERT_EQ(kiter->second, it.second.ToString());
+        count++;
+      }
+      i += 2;
+    }
+    ASSERT_EQ(i, 4);
+    ASSERT_EQ(count, 5);
+  } catch (MultiScanException& ex) {
+    ASSERT_OK(ex.status());
+  }
+  iter.reset();
+
+  // The second scan range start overlaps the delete range in the first file
+  // in Lmax-1, while the end overlaps the keys in the second file
+  (*scan_options).clear();
+  key_ranges[0] = "key000010";
+  key_ranges[1] = "key000020";
+  key_ranges[2] = "key0000500";
+  key_ranges[3] = "key000060";
+  scan_options.insert(key_ranges[0], key_ranges[1]);
+  scan_options.insert(key_ranges[2], key_ranges[3]);
+  iter = dbfull()->NewMultiScan(ro, cfh, scan_options);
+  try {
+    int i = 0;
+    int count = 0;
+    for (auto range : *iter) {
+      for (auto it : range) {
+        ASSERT_GE(it.first.ToString(), key_ranges[i]);
+        ASSERT_LT(it.first.ToString(), key_ranges[i + 1]);
+        auto kiter = kvs.find(it.first.ToString());
+        ASSERT_NE(kiter, kvs.end());
+        ASSERT_EQ(kiter->second, it.second.ToString());
+        count++;
+      }
+      i += 2;
+    }
+    ASSERT_EQ(i, 4);
+    ASSERT_EQ(count, 19);
+  } catch (MultiScanException& ex) {
+    ASSERT_OK(ex.status());
+  }
+  iter.reset();
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

db/db_options_test.cc

@@ -432,12 +432,47 @@ TEST_F(DBOptionsTest, SetWalBytesPerSync) {
   ASSERT_GT(low_bytes_per_sync, counter);
 }
 
+TEST_F(DBOptionsTest, MutableManifestOptions) {
+  // These aren't end-to-end tests, but sufficient to ensure the VersionSet
+  // receives the updates with SetDBOptions
+  for (int64_t i : {0, 1, 100, 100000, 10000000}) {
+    ASSERT_OK(
+        db_->SetDBOptions({{"max_manifest_file_size", std::to_string(i)}}));
+    ASSERT_EQ(i,
+              static_cast<int64_t>(db_->GetDBOptions().max_manifest_file_size));
+    ASSERT_EQ(i,
+              static_cast<int64_t>(
+                  dbfull()->GetVersionSet()->TEST_GetMinMaxManifestFileSize()));
+    if (i > 1) {
+      ++i;
+    }
+    ASSERT_OK(
+        db_->SetDBOptions({{"max_manifest_space_amp_pct", std::to_string(i)}}));
+    ASSERT_EQ(i, static_cast<int64_t>(
+                     db_->GetDBOptions().max_manifest_space_amp_pct));
+    ASSERT_EQ(i,
+              static_cast<int64_t>(
+                  dbfull()->GetVersionSet()->TEST_GetMaxManifestSpaceAmpPct()));
+    if (i > 1) {
+      ++i;
+    }
+    ASSERT_OK(db_->SetDBOptions(
+        {{"manifest_preallocation_size", std::to_string(i)}}));
+    ASSERT_EQ(i, static_cast<int64_t>(
+                     db_->GetDBOptions().manifest_preallocation_size));
+    ASSERT_EQ(
+        i, static_cast<int64_t>(
+               dbfull()->GetVersionSet()->TEST_GetManifestPreallocationSize()));
+  }
+}
+
 TEST_F(DBOptionsTest, WritableFileMaxBufferSize) {
   Options options;
   options.create_if_missing = true;
   options.writable_file_max_buffer_size = 1024 * 1024;
   options.level0_file_num_compaction_trigger = 3;
   options.max_manifest_file_size = 1;
+  options.max_manifest_space_amp_pct = 0;
   options.env = env_;
   int buffer_size = 1024 * 1024;
   Reopen(options);

db/db_rate_limiter_test.cc

@@ -442,6 +442,107 @@ TEST_P(DBRateLimiterOnWriteWALTest, AutoWalFlush) {
   EXPECT_EQ(actual_auto_wal_flush_request,
             options_.rate_limiter->GetTotalRequests(Env::IO_USER));
 }
+
+class DBRateLimiterOnManualWALFlushTest
+    : public DBRateLimiterOnWriteTest,
+      public ::testing::WithParamInterface<Env::IOPriority> {
+ public:
+  static std::string GetTestNameSuffix(
+      ::testing::TestParamInfo<Env::IOPriority> info) {
+    std::ostringstream oss;
+    if (info.param == Env::IO_USER) {
+      oss << "RateLimitManualWALFlush";
+    } else if (info.param == Env::IO_TOTAL) {
+      oss << "NoRateLimitManualWALFlush";
+    } else if (info.param == Env::IO_HIGH) {
+      oss << "RateLimitManualWALFlushWithHighPriority";
+    } else {
+      oss << "RateLimitManualWALFlushWithLowPriority";
+    }
+    return oss.str();
+  }
+
+  explicit DBRateLimiterOnManualWALFlushTest()
+      : rate_limiter_priority_(GetParam()) {}
+
+  void Init() {
+    options_ = GetOptions();
+    // Enable manual WAL flush mode
+    options_.manual_wal_flush = true;
+    Reopen(options_);
+  }
+
+  WriteOptions GetWriteOptions() {
+    WriteOptions write_options;
+    // WAL must be enabled for manual WAL flush to work
+    write_options.disableWAL = false;
+    // In manual WAL flush mode, WAL write rate limiting should be done through
+    // FlushWAL(), not WriteOptions::rate_limiter_priority
+    write_options.rate_limiter_priority = Env::IO_TOTAL;
+    return write_options;
+  }
+
+ protected:
+  Env::IOPriority rate_limiter_priority_;
+};
+
+INSTANTIATE_TEST_CASE_P(DBRateLimiterOnManualWALFlushTest,
+                        DBRateLimiterOnManualWALFlushTest,
+                        ::testing::Values(Env::IO_TOTAL, Env::IO_USER,
+                                          Env::IO_HIGH, Env::IO_LOW),
+                        DBRateLimiterOnManualWALFlushTest::GetTestNameSuffix);
+
+TEST_P(DBRateLimiterOnManualWALFlushTest, ManualWALFlush) {
+  Init();
+
+  const bool no_rate_limit = (rate_limiter_priority_ == Env::IO_TOTAL);
+
+  ASSERT_EQ(0, options_.rate_limiter->GetTotalRequests(Env::IO_TOTAL));
+
+  for (bool sync : {false, true}) {
+    std::int64_t prev_total_request =
+        options_.rate_limiter->GetTotalRequests(Env::IO_TOTAL);
+
+    Status put_status = Put("key_" + std::to_string(sync),
+                            "value_" + std::to_string(sync), GetWriteOptions());
+
+    EXPECT_TRUE(put_status.ok());
+
+    // Since manual_wal_flush is enabled and write_options.rate_limiter_priority
+    // is IO_TOTAL, no rate limiting should have occurred for this user write
+    EXPECT_EQ(0, options_.rate_limiter->GetTotalRequests(Env::IO_TOTAL) -
+                     prev_total_request);
+
+    // Now explicitly flush the WAL with the test's rate_limiter_priority
+    prev_total_request = options_.rate_limiter->GetTotalRequests(Env::IO_TOTAL);
+    std::int64_t prev_priority_request =
+        options_.rate_limiter->GetTotalRequests(rate_limiter_priority_);
+
+    FlushWALOptions flush_options;
+    flush_options.sync = sync;
+    flush_options.rate_limiter_priority = rate_limiter_priority_;
+    Status flush_status = db_->FlushWAL(flush_options);
+
+    EXPECT_TRUE(flush_status.ok());
+
+    std::int64_t manual_wal_flush_requests_total =
+        options_.rate_limiter->GetTotalRequests(Env::IO_TOTAL) -
+        prev_total_request;
+    std::int64_t manual_wal_flush_requests_for_priority =
+        options_.rate_limiter->GetTotalRequests(rate_limiter_priority_) -
+        prev_priority_request;
+
+    if (no_rate_limit) {
+      EXPECT_EQ(0, manual_wal_flush_requests_total);
+      EXPECT_EQ(0, manual_wal_flush_requests_for_priority);
+    } else {
+      EXPECT_EQ(manual_wal_flush_requests_total,
+                manual_wal_flush_requests_for_priority);
+      EXPECT_GT(manual_wal_flush_requests_for_priority, 0);
+    }
+  }
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

db/db_test.cc

@@ -1492,6 +1492,246 @@ TEST_F(DBTest, MetaDataTest) {
   CheckLiveFilesMeta(live_file_meta, files_by_level);
 }
 
+TEST_F(DBTest, GetColumnFamilyMetaDataWithKeyRangeAndLevel) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.disable_auto_compactions = true;
+
+  int64_t temp_time = 0;
+  ASSERT_OK(options.env->GetCurrentTime(&temp_time));
+
+  DestroyAndReopen(options);
+
+  Random rnd(301);
+  int key_index = 0;
+  for (int i = 0; i < 100; ++i) {
+    // Add a single blob reference to each file
+    std::string blob_index;
+    BlobIndex::EncodeBlob(&blob_index, /* blob_file_number */ i + 1000,
+                          /* offset */ 1234, /* size */ 5678, kNoCompression);
+
+    WriteBatch batch;
+    ASSERT_OK(WriteBatchInternal::PutBlobIndex(&batch, 0, Key(key_index),
+                                               blob_index));
+    ASSERT_OK(dbfull()->Write(WriteOptions(), &batch));
+
+    ++key_index;
+
+    // Fill up the rest of the file with random values.
+    GenerateNewFile(&rnd, &key_index, /* nowait */ true);
+
+    ASSERT_OK(Flush());
+  }
+
+  std::vector<std::vector<FileMetaData>> files_by_level;
+  dbfull()->TEST_GetFilesMetaData(db_->DefaultColumnFamily(), &files_by_level);
+
+  ASSERT_OK(options.env->GetCurrentTime(&temp_time));
+
+  ColumnFamilyMetaData cf_meta;
+  // Keys in the SST files are distributed
+  // (key000000, key000100) ->File 1
+  // (key000101, key000201) -> File 2
+  // (key000202, key000302) -> File 3
+  // (key009999, key010099) -> File 100
+
+  // With keySlice (key000050, key000150) => should only pick 2 files(instead of
+  // default 100 that is in the level)
+  auto startKey = Slice("key000050");
+  auto endKey = Slice("key000150");
+  GetColumnFamilyMetaDataOptions cf_options(startKey, endKey, 0);
+  db_->GetColumnFamilyMetaData(cf_options, &cf_meta);
+  ASSERT_EQ(cf_meta.levels.size(), 1);
+  const auto& level_meta_from_cf = cf_meta.levels[0];
+  ASSERT_EQ(level_meta_from_cf.files.size(), 2);
+  ASSERT_LT(level_meta_from_cf.files[1].smallestkey,
+            std::string(startKey.data()));
+  ASSERT_GT(level_meta_from_cf.files[0].largestkey, std::string(endKey.data()));
+
+  GetColumnFamilyMetaDataOptions cf_option_default;
+  db_->GetColumnFamilyMetaData(cf_option_default, &cf_meta);
+  ASSERT_EQ(cf_meta.levels.size(), 1);
+  ASSERT_EQ(cf_meta.levels[0].files.size(), 100);
+
+  // Test with start key valid and end key unbounded
+  // This should get all files from key000150 onwards (99 files)
+  auto startKeyUnbounded = Slice("key000150");
+  GetColumnFamilyMetaDataOptions cf_options_unbounded_end(startKeyUnbounded,
+                                                          OptSlice(), 0);
+  db_->GetColumnFamilyMetaData(cf_options_unbounded_end, &cf_meta);
+  ASSERT_EQ(cf_meta.levels.size(), 1);
+  ASSERT_EQ(cf_meta.levels[0].files.size(), 99);
+
+  // Test with end key valid and start key unbounded
+  // This should get all files from beginning to key000250 ( 3 files)
+  auto endKeyUnbounded = Slice("key000250");
+  GetColumnFamilyMetaDataOptions cf_options_unbounded_start(OptSlice(),
+                                                            endKeyUnbounded, 0);
+  db_->GetColumnFamilyMetaData(cf_options_unbounded_start, &cf_meta);
+  ASSERT_EQ(cf_meta.levels.size(), 1);
+  ASSERT_EQ(cf_meta.levels[0].files.size(), 3);
+}
+
+TEST_F(DBTest, GetColumnFamilyMetaDataBottommostLevel) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.disable_auto_compactions = true;
+  options.num_levels = 7;
+
+  DestroyAndReopen(options);
+
+  Random rnd(301);
+  int key_index = 0;
+
+  for (int i = 0; i < 100; ++i) {
+    GenerateNewFile(&rnd, &key_index, /* nowait */ true);
+    ASSERT_OK(Flush());
+  }
+
+  CompactRangeOptions compact_options;
+  compact_options.bottommost_level_compaction =
+      BottommostLevelCompaction::kForce;
+  compact_options.change_level = true;
+  compact_options.target_level = 6;
+  ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr));
+
+  // Nothing on Level 0 after compaction
+  ColumnFamilyMetaData cf_meta;
+  GetColumnFamilyMetaDataOptions cf_options_0(OptSlice(), OptSlice(), 0);
+  db_->GetColumnFamilyMetaData(cf_options_0, &cf_meta);
+
+  ASSERT_EQ(cf_meta.levels.size(), 0);
+  ASSERT_EQ(cf_meta.file_count, 0);
+
+  // Data should be in Level 6
+  GetColumnFamilyMetaDataOptions cf_options(OptSlice(), OptSlice(), 6);
+  db_->GetColumnFamilyMetaData(cf_options, &cf_meta);
+
+  ASSERT_EQ(cf_meta.levels.size(), 1);
+  ASSERT_EQ(cf_meta.levels[0].level, 6);
+  ASSERT_GT(cf_meta.levels[0].files.size(), 0);
+  size_t all_files = cf_meta.levels[0].files.size();
+
+  // Keys in the SST files are distributed across level 6
+  // Test with key range - should only return files within the range
+  auto startKey = Slice("key000050");
+  auto endKey = Slice("key000150");
+  GetColumnFamilyMetaDataOptions cf_options_range(startKey, endKey, 6);
+  db_->GetColumnFamilyMetaData(cf_options_range, &cf_meta);
+
+  ASSERT_EQ(cf_meta.levels.size(), 1);
+  ASSERT_EQ(cf_meta.levels[0].level, 6);
+  ASSERT_GT(cf_meta.levels[0].files.size(), 0);
+  size_t files_in_range = cf_meta.levels[0].files.size();
+
+  // Files in range should be less than or equal to all files
+  ASSERT_LE(files_in_range, all_files);
+}
+
+TEST_F(DBTest, GetColumnFamilyMetaDataMultipleLevels) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.disable_auto_compactions = true;
+  options.num_levels = 7;
+
+  DestroyAndReopen(options);
+
+  Random rnd(301);
+  int key_index = 0;
+
+  for (int i = 0; i < 50; ++i) {
+    GenerateNewFile(&rnd, &key_index, /* nowait */ true);
+    ASSERT_OK(Flush());
+  }
+
+  CompactRangeOptions compact_options;
+  compact_options.bottommost_level_compaction =
+      BottommostLevelCompaction::kForce;
+  compact_options.change_level = true;
+  compact_options.target_level = 6;
+  ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr));
+
+  for (int i = 0; i < 30; ++i) {
+    GenerateNewFile(&rnd, &key_index, /* nowait */ true);
+    ASSERT_OK(Flush());
+  }
+
+  // First verify both levels have files without key range filter
+  ColumnFamilyMetaData cf_meta_all_no_range;
+  GetColumnFamilyMetaDataOptions cf_options_all_no_range;
+  db_->GetColumnFamilyMetaData(cf_options_all_no_range, &cf_meta_all_no_range);
+
+  bool has_level_0 = false;
+  bool has_level_6 = false;
+  for (const auto& level : cf_meta_all_no_range.levels) {
+    if (level.level == 0 && level.files.size() > 0) {
+      has_level_0 = true;
+    }
+    if (level.level == 6 && level.files.size() > 0) {
+      has_level_6 = true;
+    }
+  }
+
+  ASSERT_TRUE(has_level_0);
+  ASSERT_TRUE(has_level_6);
+
+  // Test querying bottommost level only with key range
+  // Use a range that should be in the first set of files (now in level 6)
+  auto startKey = Slice("key000050");
+  auto endKey = Slice("key000150");
+  ColumnFamilyMetaData cf_meta_bottommost;
+  GetColumnFamilyMetaDataOptions cf_options_bottommost(startKey, endKey, 6);
+  db_->GetColumnFamilyMetaData(cf_options_bottommost, &cf_meta_bottommost);
+
+  ASSERT_EQ(cf_meta_bottommost.levels.size(), 1);
+  ASSERT_EQ(cf_meta_bottommost.levels[0].level, 6);
+  ASSERT_GT(cf_meta_bottommost.levels[0].files.size(), 0);
+  size_t level_6_files_in_range = cf_meta_bottommost.levels[0].files.size();
+
+  // Test querying all levels with same key range
+  ColumnFamilyMetaData cf_meta_all;
+  GetColumnFamilyMetaDataOptions cf_options_all(startKey, endKey);
+  db_->GetColumnFamilyMetaData(cf_options_all, &cf_meta_all);
+
+  size_t level_6_files_in_range_from_all = 0;
+  for (const auto& level : cf_meta_all.levels) {
+    if (level.level == 6) {
+      level_6_files_in_range_from_all = level.files.size();
+    }
+  }
+
+  ASSERT_GT(level_6_files_in_range_from_all, 0);
+  ASSERT_EQ(level_6_files_in_range, level_6_files_in_range_from_all);
+}
+
+TEST_F(DBTest, GetColumnFamilyMetaDataEmptyDB) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.num_levels = 7;
+
+  DestroyAndReopen(options);
+
+  // Test on empty database
+  ColumnFamilyMetaData cf_meta_empty_db;
+  GetColumnFamilyMetaDataOptions cf_options_empty_db;
+  db_->GetColumnFamilyMetaData(cf_options_empty_db, &cf_meta_empty_db);
+
+  ASSERT_EQ(cf_meta_empty_db.levels.size(), 0);
+  ASSERT_EQ(cf_meta_empty_db.file_count, 0);
+  ASSERT_EQ(cf_meta_empty_db.size, 0);
+
+  // Test on empty database with key range
+  auto startKey = Slice("key000050");
+  auto endKey = Slice("key000150");
+  ColumnFamilyMetaData cf_meta_empty_range;
+  GetColumnFamilyMetaDataOptions cf_options_empty_range(startKey, endKey);
+  db_->GetColumnFamilyMetaData(cf_options_empty_range, &cf_meta_empty_range);
+
+  ASSERT_EQ(cf_meta_empty_range.levels.size(), 0);
+  ASSERT_EQ(cf_meta_empty_range.file_count, 0);
+  ASSERT_EQ(cf_meta_empty_range.size, 0);
+}
+
 TEST_F(DBTest, AllMetaDataTest) {
   Options options = CurrentOptions();
   options.create_if_missing = true;
@@ -3535,6 +3775,11 @@ class ModelDB : public DB {
   void GetColumnFamilyMetaData(ColumnFamilyHandle* /*column_family*/,
                                ColumnFamilyMetaData* /*metadata*/) override {}
 
+  void GetColumnFamilyMetaData(
+      ColumnFamilyHandle* /*column_family*/,
+      const GetColumnFamilyMetaDataOptions& /*options*/,
+      ColumnFamilyMetaData* /*metadata*/) override {}
+
   Status GetDbIdentity(std::string& /*identity*/) const override {
     return Status::OK();
   }

db/db_test2.cc

@@ -5205,6 +5205,7 @@ TEST_F(DBTest2, SwitchMemtableRaceWithNewManifest) {
   Options options = CurrentOptions();
   DestroyAndReopen(options);
   options.max_manifest_file_size = 10;
+  options.max_manifest_space_amp_pct = 0;
   options.create_if_missing = true;
   CreateAndReopenWithCF({"pikachu"}, options);
   ASSERT_EQ(2, handles_.size());
@@ -5896,6 +5897,7 @@ TEST_P(RenameCurrentTest, Flush) {
   Destroy(last_options_);
   Options options = GetDefaultOptions();
   options.max_manifest_file_size = 1;
+  options.max_manifest_space_amp_pct = 0;
   options.create_if_missing = true;
   Reopen(options);
   ASSERT_OK(Put("key", "value"));
@@ -5915,6 +5917,7 @@ TEST_P(RenameCurrentTest, Compaction) {
   Destroy(last_options_);
   Options options = GetDefaultOptions();
   options.max_manifest_file_size = 1;
+  options.max_manifest_space_amp_pct = 0;
   options.create_if_missing = true;
   Reopen(options);
   ASSERT_OK(Put("a", "a_value"));
@@ -6063,16 +6066,9 @@ TEST_F(DBTest2, VariousFileTemperatures) {
   };
 
   // We don't have enough non-unknown temps to confidently distinguish that
-  // a specific setting caused a specific outcome, in a single run. This is a
-  // reasonable work-around without blowing up test time. Only returns
-  // non-unknown temperatures.
-  auto RandomTemp = [] {
-    static std::vector<Temperature> temps = {
-        Temperature::kHot, Temperature::kWarm, Temperature::kCold,
-        Temperature::kIce};
-    return temps[Random::GetTLSInstance()->Uniform(
-        static_cast<int>(temps.size()))];
-  };
+  // a specific setting caused a specific outcome, in a single run. Using
+  // RandomKnownTemperature() is a reasonable work-around without blowing up
+  // test time.
 
   auto test_fs = std::make_shared<MyTestFS>(env_->GetFileSystem());
   std::unique_ptr<Env> env(new CompositeEnvWrapper(env_, test_fs));
@@ -6088,22 +6084,22 @@ TEST_F(DBTest2, VariousFileTemperatures) {
       options.env = env.get();
       test_fs->Reset();
       if (use_optimize) {
-        test_fs->optimize_manifest_temperature = RandomTemp();
+        test_fs->optimize_manifest_temperature = RandomKnownTemperature();
         test_fs->expected_manifest_temperature =
             test_fs->optimize_manifest_temperature;
-        test_fs->optimize_wal_temperature = RandomTemp();
+        test_fs->optimize_wal_temperature = RandomKnownTemperature();
         test_fs->expected_wal_temperature = test_fs->optimize_wal_temperature;
       }
       if (use_temp_options) {
-        options.metadata_write_temperature = RandomTemp();
+        options.metadata_write_temperature = RandomKnownTemperature();
         test_fs->expected_manifest_temperature =
             options.metadata_write_temperature;
         test_fs->expected_other_metadata_temperature =
             options.metadata_write_temperature;
-        options.wal_write_temperature = RandomTemp();
+        options.wal_write_temperature = RandomKnownTemperature();
         test_fs->expected_wal_temperature = options.wal_write_temperature;
-        options.last_level_temperature = RandomTemp();
-        options.default_write_temperature = RandomTemp();
+        options.last_level_temperature = RandomKnownTemperature();
+        options.default_write_temperature = RandomKnownTemperature();
       }
 
       DestroyAndReopen(options);

db/db_test_util.cc

@@ -454,7 +454,8 @@ Options DBTestBase::GetOptions(
       options.allow_mmap_reads = can_allow_mmap;
       break;
     case kManifestFileSize:
-      options.max_manifest_file_size = 50;  // 50 bytes
+      options.max_manifest_file_size = 50;     // 50 bytes
+      options.max_manifest_space_amp_pct = 0;  // old behavior
       break;
     case kPerfOptions:
       options.delayed_write_rate = 8 * 1024 * 1024;
@@ -1872,4 +1873,13 @@ template class TargetCacheChargeTrackingCache<
     CacheEntryRole::kBlockBasedTableReader>;
 template class TargetCacheChargeTrackingCache<CacheEntryRole::kFileMetadata>;
 
+const std::vector<Temperature> kKnownTemperatures = {
+    Temperature::kHot, Temperature::kWarm, Temperature::kCool,
+    Temperature::kCold, Temperature::kIce};
+
+Temperature RandomKnownTemperature() {
+  return kKnownTemperatures[Random::GetTLSInstance()->Uniform(
+      static_cast<int>(kKnownTemperatures.size()))];
+}
+
 }  // namespace ROCKSDB_NAMESPACE

db/db_test_util.h

@@ -1467,4 +1467,8 @@ class DBTestBase : public testing::Test {
 // unique ids.
 void VerifySstUniqueIds(const TablePropertiesCollection& props);
 
+// Excludes kUnknown
+extern const std::vector<Temperature> kKnownTemperatures;
+Temperature RandomKnownTemperature();
+
 }  // namespace ROCKSDB_NAMESPACE

db/db_wal_test.cc

@@ -1746,8 +1746,8 @@ class RecoveryTestHelper {
     WriteController write_controller;
 
     versions.reset(new VersionSet(
-        test->dbname_, &db_options, file_options, table_cache.get(),
-        &write_buffer_manager, &write_controller,
+        test->dbname_, &db_options, MutableDBOptions{options}, file_options,
+        table_cache.get(), &write_buffer_manager, &write_controller,
         /*block_cache_tracer=*/nullptr,
         /*io_tracer=*/nullptr, /*db_id=*/"", /*db_session_id=*/"",
         options.daily_offpeak_time_utc,
@@ -2277,6 +2277,7 @@ TEST_F(DBWALTest, FixSyncWalOnObseletedWalWithNewManifestCausingMissingWAL) {
   Options options = CurrentOptions();
   // Small size to force manifest creation
   options.max_manifest_file_size = 1;
+  options.max_manifest_space_amp_pct = 0;
   options.track_and_verify_wals_in_manifest = true;
   DestroyAndReopen(options);
 

db/dbformat_test.cc

@@ -333,6 +333,50 @@ TEST_F(FormatTest, ReplaceInternalKeyWithMinTimestamp) {
   ASSERT_EQ(kTypeValue, new_key.type);
 }
 
+TEST(RocksdbVersionTest, Version) {
+  // Test preprocessor macros for versioning
+  ASSERT_GT(ROCKSDB_MAJOR, 0);
+  ASSERT_GE(ROCKSDB_MINOR, 0);
+  ASSERT_GE(ROCKSDB_PATCH, 0);
+  ASSERT_LT(ROCKSDB_MAJOR, 1000);
+  ASSERT_LT(ROCKSDB_MINOR, 1000);
+  ASSERT_LT(ROCKSDB_PATCH, 1000);
+  ASSERT_EQ(ROCKSDB_MAKE_VERSION_INT(123, 456, 789), 123456789);
+  ASSERT_GT(ROCKSDB_VERSION_INT, 9999999);
+  ASSERT_LT(ROCKSDB_VERSION_INT, 99999999);
+  static_assert(ROCKSDB_VERSION_GE(9, 8, 7));
+  static_assert(
+      ROCKSDB_VERSION_GE(ROCKSDB_MAJOR, ROCKSDB_MINOR, ROCKSDB_PATCH));
+  static_assert(
+      ROCKSDB_VERSION_GE(ROCKSDB_MAJOR, ROCKSDB_MINOR, ROCKSDB_PATCH - 1));
+  static_assert(
+      ROCKSDB_VERSION_GE(ROCKSDB_MAJOR, ROCKSDB_MINOR, ROCKSDB_PATCH - 100));
+  static_assert(
+      ROCKSDB_VERSION_GE(ROCKSDB_MAJOR, ROCKSDB_MINOR - 1, ROCKSDB_PATCH + 1));
+  static_assert(ROCKSDB_VERSION_GE(ROCKSDB_MAJOR - 1, ROCKSDB_MINOR + 1,
+                                   ROCKSDB_PATCH + 1));
+  static_assert(
+      !ROCKSDB_VERSION_GE(ROCKSDB_MAJOR, ROCKSDB_MINOR, ROCKSDB_PATCH + 1));
+  static_assert(
+      !ROCKSDB_VERSION_GE(ROCKSDB_MAJOR, ROCKSDB_MINOR, ROCKSDB_PATCH + 100));
+  static_assert(
+      !ROCKSDB_VERSION_GE(ROCKSDB_MAJOR, ROCKSDB_MINOR + 1, ROCKSDB_PATCH - 1));
+  static_assert(!ROCKSDB_VERSION_GE(ROCKSDB_MAJOR + 1, ROCKSDB_MINOR - 1,
+                                    ROCKSDB_PATCH - 1));
+  // More typical usage (but with literal numbers based on relevant API
+  // features)
+#if ROCKSDB_VERSION_GE(ROCKSDB_MAJOR, ROCKSDB_MINOR, ROCKSDB_PATCH)
+  static_assert(true);
+#else
+  static_assert(false);
+#endif
+#if !ROCKSDB_VERSION_GE(ROCKSDB_MAJOR, ROCKSDB_MINOR, ROCKSDB_PATCH + 1)
+  static_assert(true);
+#else
+  static_assert(false);
+#endif
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

db/external_sst_file_basic_test.cc

@@ -1311,7 +1311,7 @@ TEST_F(ExternalSSTFileBasicTest, SyncFailure) {
     });
     if (i == 0) {
       SyncPoint::GetInstance()->SetCallBack(
-          "ExternalSstFileIngestionJob::CheckSyncReturnCode", [&](void* s) {
+          "ExternalSstFileIngestionJob::Prepare:Reopen", [&](void* s) {
             Status* status = static_cast<Status*>(s);
             if (status->IsNotSupported()) {
               no_sync = true;
@@ -1372,11 +1372,11 @@ TEST_F(ExternalSSTFileBasicTest, ReopenNotSupported) {
   options.create_if_missing = true;
   options.env = env_;
 
-  SyncPoint::GetInstance()->SetCallBack("FileSystem::SyncFile:Open",
-                                        [&](void* arg) {
-                                          Status* s = static_cast<Status*>(arg);
-                                          *s = Status::NotSupported();
-                                        });
+  SyncPoint::GetInstance()->SetCallBack(
+      "ExternalSstFileIngestionJob::Prepare:Reopen", [&](void* arg) {
+        Status* s = static_cast<Status*>(arg);
+        *s = Status::NotSupported();
+      });
   SyncPoint::GetInstance()->EnableProcessing();
 
   DestroyAndReopen(options);

db/external_sst_file_ingestion_job.cc

@@ -163,26 +163,35 @@ Status ExternalSstFileIngestionJob::Prepare(
         // It is unsafe to assume application had sync the file and file
         // directory before ingest the file. For integrity of RocksDB we need
         // to sync the file.
-        TEST_SYNC_POINT("ExternalSstFileIngestionJob::BeforeSyncIngestedFile");
-        auto s = fs_->SyncFile(path_inside_db, env_options_, IOOptions(),
-                               db_options_.use_fsync, nullptr);
-        TEST_SYNC_POINT("ExternalSstFileIngestionJob::AfterSyncIngestedFile");
-        TEST_SYNC_POINT_CALLBACK(
-            "ExternalSstFileIngestionJob::CheckSyncReturnCode", &s);
-        if (!s.ok()) {
-          if (s.IsNotSupported()) {
-            // Some file systems (especially remote/distributed) don't support
-            // SyncFile API. Ignore the NotSupported error in that case.
-            ROCKS_LOG_WARN(db_options_.info_log,
-                           "After link the file, SyncFile API is not supported "
-                           "for file %s: %s",
-                           path_inside_db.c_str(), status.ToString().c_str());
-          } else {
-            // for other errors, propagate the error
-            status = s;
-            ROCKS_LOG_WARN(db_options_.info_log,
-                           "Failed to sync ingested file %s: %s",
-                           path_inside_db.c_str(), status.ToString().c_str());
+
+        // TODO(xingbo), We should in general be moving away from production
+        // uses of ReuseWritableFile (except explicitly for WAL recycling),
+        // ReopenWritableFile, and NewRandomRWFile. We should create a
+        // FileSystem::SyncFile/FsyncFile API that by default does the
+        // re-open+sync+close combo but can (a) be reused easily, and (b) be
+        // overridden to do that more cleanly, e.g. in EncryptedEnv.
+        // https://github.com/facebook/rocksdb/issues/13741
+        std::unique_ptr<FSWritableFile> file_to_sync;
+        Status s = fs_->ReopenWritableFile(path_inside_db, env_options_,
+                                           &file_to_sync, nullptr);
+        TEST_SYNC_POINT_CALLBACK("ExternalSstFileIngestionJob::Prepare:Reopen",
+                                 &s);
+        // Some file systems (especially remote/distributed) don't support
+        // reopening a file for writing and don't require reopening and
+        // syncing the file. Ignore the NotSupported error in that case.
+        if (!s.IsNotSupported()) {
+          status = s;
+          if (status.ok()) {
+            TEST_SYNC_POINT(
+                "ExternalSstFileIngestionJob::BeforeSyncIngestedFile");
+            status = SyncIngestedFile(file_to_sync.get());
+            TEST_SYNC_POINT(
+                "ExternalSstFileIngestionJob::AfterSyncIngestedFile");
+            if (!status.ok()) {
+              ROCKS_LOG_WARN(db_options_.info_log,
+                             "Failed to sync ingested file %s: %s",
+                             path_inside_db.c_str(), status.ToString().c_str());
+            }
           }
         }
       } else if (status.IsNotSupported() &&
@@ -740,8 +749,7 @@ void ExternalSstFileIngestionJob::CreateEquivalentFileIngestingCompactions() {
                             cfd_->ioptions().compaction_style),
         LLONG_MAX /* max compaction bytes, not applicable */,
         0 /* output path ID, not applicable */, mutable_cf_options.compression,
-        mutable_cf_options.compression_opts,
-        mutable_cf_options.default_write_temperature,
+        mutable_cf_options.compression_opts, Temperature::kUnknown,
         0 /* max_subcompaction, not applicable */,
         {} /* grandparents, not applicable */,
         std::nullopt /* earliest_snapshot */, nullptr /* snapshot_checker */,

db/external_sst_file_test.cc

@@ -4094,6 +4094,17 @@ TEST_P(IngestDBGeneratedFileTest2, NonZeroSeqno) {
   ingest_opts.fail_if_not_bottommost_level = std::get<3>(GetParam());
   ingest_opts.link_files = std::get<4>(GetParam());
   Random* rnd = Random::GetTLSInstance();
+  rnd->Reset(std::random_device{}());
+  std::ostringstream ingest_opts_trace;
+  ingest_opts_trace << "ingest_opts params: " << "snapshot_consistency="
+                    << ingest_opts.snapshot_consistency << ", "
+                    << "allow_global_seqno=" << ingest_opts.allow_global_seqno
+                    << ", " << "allow_blocking_flush="
+                    << ingest_opts.allow_blocking_flush << ", "
+                    << "fail_if_not_bottommost_level="
+                    << ingest_opts.fail_if_not_bottommost_level << ", "
+                    << "link_files=" << ingest_opts.link_files;
+  SCOPED_TRACE(ingest_opts_trace.str());
 
   do {
     SCOPED_TRACE("option_config_ = " + std::to_string(option_config_));
@@ -4263,11 +4274,17 @@ TEST_P(IngestDBGeneratedFileTest2, NonZeroSeqno) {
       s = db_->IngestExternalFile(overlap_cf, sst_file_paths, ingest_opts);
 
       ASSERT_NOK(s);
-      ASSERT_TRUE(s.ToString().find("An ingested file overlaps with existing "
-                                    "data in the DB and has been "
-                                    "assigned a non-zero sequence number") !=
-                  std::string::npos)
-          << s.ToString();
+      if (ingest_opts.fail_if_not_bottommost_level) {
+        ASSERT_TRUE(s.ToString().find("Files cannot be ingested to Lmax") !=
+                    std::string::npos)
+            << s.ToString();
+      } else {
+        ASSERT_TRUE(s.ToString().find("An ingested file overlaps with existing "
+                                      "data in the DB and has been "
+                                      "assigned a non-zero sequence number") !=
+                    std::string::npos)
+            << s.ToString();
+      }
     }
 
     // Cleanup

db/flush_job_test.cc

@@ -142,13 +142,13 @@ class FlushJobTestBase : public testing::Test {
       column_families.emplace_back(cf_name, cf_options_);
     }
 
-    versions_.reset(
-        new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(),
-                       &write_buffer_manager_, &write_controller_,
-                       /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr,
-                       test::kUnitTestDbId, /*db_session_id=*/"",
-                       /*daily_offpeak_time_utc=*/"",
-                       /*error_handler=*/nullptr, /*read_only=*/false));
+    versions_.reset(new VersionSet(
+        dbname_, &db_options_, MutableDBOptions{options_}, env_options_,
+        table_cache_.get(), &write_buffer_manager_, &write_controller_,
+        /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr,
+        test::kUnitTestDbId, /*db_session_id=*/"",
+        /*daily_offpeak_time_utc=*/"",
+        /*error_handler=*/nullptr, /*read_only=*/false));
     EXPECT_OK(versions_->Recover(column_families, false));
   }
 

db/log_reader.h

@@ -59,7 +59,7 @@ class Reader {
   // live while this Reader is in use.
   //
   // If "checksum" is true, verify checksums if available.
-  // TODO(hx235): seperate WAL related parameters from general `Reader`
+  // TODO(hx235): separate WAL related parameters from general `Reader`
   // parameters
   Reader(std::shared_ptr<Logger> info_log,
          std::unique_ptr<SequentialFileReader>&& file, Reporter* reporter,
@@ -155,7 +155,7 @@ class Reader {
   // which log number this is
   uint64_t const log_number_;
 
-  // See `Optinos::track_and_verify_wals`
+  // See `Options::track_and_verify_wals`
   bool track_and_verify_wals_;
   // Below variables are used for WAL verification
   // TODO(hx235): To revise `stop_replay_for_corruption_` inside `LogReader`
@@ -208,8 +208,8 @@ class Reader {
   };
 
   // Return type, or one of the preceding special values
-  // If WAL compressioned is enabled, fragment_checksum is the checksum of the
-  // fragment computed from the orginal buffer containinng uncompressed
+  // If WAL compression is enabled, fragment_checksum is the checksum of the
+  // fragment computed from the original buffer containing uncompressed
   // fragment.
   uint8_t ReadPhysicalRecord(Slice* result, size_t* drop_size,
                              uint64_t* fragment_checksum = nullptr);

db/log_writer.h

@@ -77,7 +77,7 @@ class Writer {
   // Create a writer that will append data to "*dest".
   // "*dest" must be initially empty.
   // "*dest" must remain live while this Writer is in use.
-  // TODO(hx235): seperate WAL related parameters from general `Reader`
+  // TODO(hx235): separate WAL related parameters from general `Reader`
   // parameters
   explicit Writer(std::unique_ptr<WritableFileWriter>&& dest,
                   uint64_t log_number, bool recycle_log_files,

db/memtable_list_test.cc

@@ -112,7 +112,8 @@ class MemTableListTest : public testing::Test {
     WriteBufferManager write_buffer_manager(db_options.db_write_buffer_size);
     WriteController write_controller(10000000u);
 
-    VersionSet versions(dbname, &immutable_db_options, env_options,
+    VersionSet versions(dbname, &immutable_db_options,
+                        MutableDBOptions{db_options}, env_options,
                         table_cache.get(), &write_buffer_manager,
                         &write_controller, /*block_cache_tracer=*/nullptr,
                         /*io_tracer=*/nullptr, /*db_id=*/"",
@@ -163,7 +164,8 @@ class MemTableListTest : public testing::Test {
     WriteBufferManager write_buffer_manager(db_options.db_write_buffer_size);
     WriteController write_controller(10000000u);
 
-    VersionSet versions(dbname, &immutable_db_options, env_options,
+    VersionSet versions(dbname, &immutable_db_options,
+                        MutableDBOptions{db_options}, env_options,
                         table_cache.get(), &write_buffer_manager,
                         &write_controller, /*block_cache_tracer=*/nullptr,
                         /*io_tracer=*/nullptr, /*db_id=*/"",

db/merge_helper.h

@@ -307,7 +307,7 @@ class MergeOutputIterator {
 
   Slice key() { return Slice(*it_keys_); }
   Slice value() { return Slice(*it_values_); }
-  bool Valid() { return it_keys_ != merge_helper_->keys().rend(); }
+  bool Valid() const { return it_keys_ != merge_helper_->keys().rend(); }
 
  private:
   const MergeHelper* merge_helper_;

db/multi_scan.cc

@@ -40,6 +40,11 @@ MultiScan::MultiScan(const ReadOptions& read_options,
 }
 
 MultiScanIterator& MultiScanIterator::operator++() {
+  status_ = db_iter_->status();
+  if (!status_.ok()) {
+    throw MultiScanException(status_);
+  }
+
   if (idx_ >= scan_opts_.size()) {
     throw std::logic_error("Index out of range");
   }

db/repair.cc

@@ -120,8 +120,8 @@ class Repairer {
                                     /*io_tracer=*/nullptr, db_session_id_)),
         wb_(db_options_.db_write_buffer_size),
         wc_(db_options_.delayed_write_rate),
-        vset_(dbname_, &immutable_db_options_, file_options_,
-              raw_table_cache_.get(), &wb_, &wc_,
+        vset_(dbname_, &immutable_db_options_, MutableDBOptions{db_options_},
+              file_options_, raw_table_cache_.get(), &wb_, &wc_,
               /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr,
               /*db_id=*/"", db_session_id_, db_options.daily_offpeak_time_utc,
               /*error_handler=*/nullptr, /*read_only=*/false),

db/version_edit.cc

@@ -112,124 +112,9 @@ bool VersionEdit::EncodeTo(std::string* dst,
         f.epoch_number == kUnknownEpochNumber) {
       return false;
     }
-    PutVarint32(dst, kNewFile4);
-    PutVarint32Varint64(dst, new_files_[i].first /* level */, f.fd.GetNumber());
-    PutVarint64(dst, f.fd.GetFileSize());
-    EncodeFileBoundaries(dst, f, ts_sz.value());
-    PutVarint64Varint64(dst, f.fd.smallest_seqno, f.fd.largest_seqno);
-    // Customized fields' format:
-    // +-----------------------------+
-    // | 1st field's tag (varint32)  |
-    // +-----------------------------+
-    // | 1st field's size (varint32) |
-    // +-----------------------------+
-    // |    bytes for 1st field      |
-    // |  (based on size decoded)    |
-    // +-----------------------------+
-    // |                             |
-    // |          ......             |
-    // |                             |
-    // +-----------------------------+
-    // | last field's size (varint32)|
-    // +-----------------------------+
-    // |    bytes for last field     |
-    // |  (based on size decoded)    |
-    // +-----------------------------+
-    // | terminating tag (varint32)  |
-    // +-----------------------------+
-    //
-    // Customized encoding for fields:
-    //   tag kPathId: 1 byte as path_id
-    //   tag kNeedCompaction:
-    //        now only can take one char value 1 indicating need-compaction
-    //
-    PutVarint32(dst, NewFileCustomTag::kOldestAncesterTime);
-    std::string varint_oldest_ancester_time;
-    PutVarint64(&varint_oldest_ancester_time, f.oldest_ancester_time);
-    TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:VarintOldestAncesterTime",
-                             &varint_oldest_ancester_time);
-    PutLengthPrefixedSlice(dst, Slice(varint_oldest_ancester_time));
-
-    PutVarint32(dst, NewFileCustomTag::kFileCreationTime);
-    std::string varint_file_creation_time;
-    PutVarint64(&varint_file_creation_time, f.file_creation_time);
-    TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:VarintFileCreationTime",
-                             &varint_file_creation_time);
-    PutLengthPrefixedSlice(dst, Slice(varint_file_creation_time));
-
-    PutVarint32(dst, NewFileCustomTag::kEpochNumber);
-    std::string varint_epoch_number;
-    PutVarint64(&varint_epoch_number, f.epoch_number);
-    PutLengthPrefixedSlice(dst, Slice(varint_epoch_number));
-
-    if (f.file_checksum_func_name != kUnknownFileChecksumFuncName) {
-      PutVarint32(dst, NewFileCustomTag::kFileChecksum);
-      PutLengthPrefixedSlice(dst, Slice(f.file_checksum));
-
-      PutVarint32(dst, NewFileCustomTag::kFileChecksumFuncName);
-      PutLengthPrefixedSlice(dst, Slice(f.file_checksum_func_name));
-    }
-
-    if (f.fd.GetPathId() != 0) {
-      PutVarint32(dst, NewFileCustomTag::kPathId);
-      char p = static_cast<char>(f.fd.GetPathId());
-      PutLengthPrefixedSlice(dst, Slice(&p, 1));
-    }
-    if (f.temperature != Temperature::kUnknown) {
-      PutVarint32(dst, NewFileCustomTag::kTemperature);
-      char p = static_cast<char>(f.temperature);
-      PutLengthPrefixedSlice(dst, Slice(&p, 1));
-    }
-    if (f.marked_for_compaction) {
-      PutVarint32(dst, NewFileCustomTag::kNeedCompaction);
-      char p = static_cast<char>(1);
-      PutLengthPrefixedSlice(dst, Slice(&p, 1));
-    }
-    if (has_min_log_number_to_keep_ && !min_log_num_written) {
-      PutVarint32(dst, NewFileCustomTag::kMinLogNumberToKeepHack);
-      std::string varint_log_number;
-      PutFixed64(&varint_log_number, min_log_number_to_keep_);
-      PutLengthPrefixedSlice(dst, Slice(varint_log_number));
-      min_log_num_written = true;
-    }
-    if (f.oldest_blob_file_number != kInvalidBlobFileNumber) {
-      PutVarint32(dst, NewFileCustomTag::kOldestBlobFileNumber);
-      std::string oldest_blob_file_number;
-      PutVarint64(&oldest_blob_file_number, f.oldest_blob_file_number);
-      PutLengthPrefixedSlice(dst, Slice(oldest_blob_file_number));
-    }
-    UniqueId64x2 unique_id = f.unique_id;
-    TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:UniqueId", &unique_id);
-    if (unique_id != kNullUniqueId64x2) {
-      PutVarint32(dst, NewFileCustomTag::kUniqueId);
-      std::string unique_id_str = EncodeUniqueIdBytes(&unique_id);
-      PutLengthPrefixedSlice(dst, Slice(unique_id_str));
-    }
-    if (f.compensated_range_deletion_size) {
-      PutVarint32(dst, kCompensatedRangeDeletionSize);
-      std::string compensated_range_deletion_size;
-      PutVarint64(&compensated_range_deletion_size,
-                  f.compensated_range_deletion_size);
-      PutLengthPrefixedSlice(dst, Slice(compensated_range_deletion_size));
-    }
-    if (f.tail_size) {
-      PutVarint32(dst, NewFileCustomTag::kTailSize);
-      std::string varint_tail_size;
-      PutVarint64(&varint_tail_size, f.tail_size);
-      PutLengthPrefixedSlice(dst, Slice(varint_tail_size));
-    }
-    if (!f.user_defined_timestamps_persisted) {
-      // The default value for the flag is true, it's only explicitly persisted
-      // when it's false. We are putting 0 as the value here to signal false
-      // (i.e. UDTS not persisted).
-      PutVarint32(dst, NewFileCustomTag::kUserDefinedTimestampsPersisted);
-      char p = static_cast<char>(0);
-      PutLengthPrefixedSlice(dst, Slice(&p, 1));
-    }
-    TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:NewFile4:CustomizeFields",
-                             dst);
-
-    PutVarint32(dst, NewFileCustomTag::kTerminate);
+    EncodeToNewFile4(f, new_files_[i].first, ts_sz.value(),
+                     has_min_log_number_to_keep_, min_log_number_to_keep_,
+                     min_log_num_written, dst);
   }
 
   for (const auto& blob_file_addition : blob_file_additions_) {
@@ -288,9 +173,142 @@ bool VersionEdit::EncodeTo(std::string* dst,
     char p = static_cast<char>(persist_user_defined_timestamps_);
     PutLengthPrefixedSlice(dst, Slice(&p, 1));
   }
+
+  if (HasSubcompactionProgress()) {
+    PutVarint32(dst, kSubcompactionProgress);
+    std::string progress_data;
+    subcompaction_progress_.EncodeTo(&progress_data);
+    PutLengthPrefixedSlice(dst, progress_data);
+  }
+
   return true;
 }
 
+void VersionEdit::EncodeToNewFile4(const FileMetaData& f, int level,
+                                   size_t ts_sz,
+                                   bool has_min_log_number_to_keep,
+                                   uint64_t min_log_number_to_keep,
+                                   bool& min_log_num_written,
+                                   std::string* dst) {
+  PutVarint32(dst, kNewFile4);
+  PutVarint32Varint64(dst, level, f.fd.GetNumber());
+  PutVarint64(dst, f.fd.GetFileSize());
+  EncodeFileBoundaries(dst, f, ts_sz);
+  PutVarint64Varint64(dst, f.fd.smallest_seqno, f.fd.largest_seqno);
+  // Customized fields' format:
+  // +-----------------------------+
+  // | 1st field's tag (varint32)  |
+  // +-----------------------------+
+  // | 1st field's size (varint32) |
+  // +-----------------------------+
+  // |    bytes for 1st field      |
+  // |  (based on size decoded)    |
+  // +-----------------------------+
+  // |                             |
+  // |          ......             |
+  // |                             |
+  // +-----------------------------+
+  // | last field's size (varint32)|
+  // +-----------------------------+
+  // |    bytes for last field     |
+  // |  (based on size decoded)    |
+  // +-----------------------------+
+  // | terminating tag (varint32)  |
+  // +-----------------------------+
+  //
+  // Customized encoding for fields:
+  //   tag kPathId: 1 byte as path_id
+  //   tag kNeedCompaction:
+  //        now only can take one char value 1 indicating need-compaction
+  //
+  PutVarint32(dst, NewFileCustomTag::kOldestAncesterTime);
+  std::string varint_oldest_ancester_time;
+  PutVarint64(&varint_oldest_ancester_time, f.oldest_ancester_time);
+  TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:VarintOldestAncesterTime",
+                           &varint_oldest_ancester_time);
+  PutLengthPrefixedSlice(dst, Slice(varint_oldest_ancester_time));
+
+  PutVarint32(dst, NewFileCustomTag::kFileCreationTime);
+  std::string varint_file_creation_time;
+  PutVarint64(&varint_file_creation_time, f.file_creation_time);
+  TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:VarintFileCreationTime",
+                           &varint_file_creation_time);
+  PutLengthPrefixedSlice(dst, Slice(varint_file_creation_time));
+
+  PutVarint32(dst, NewFileCustomTag::kEpochNumber);
+  std::string varint_epoch_number;
+  PutVarint64(&varint_epoch_number, f.epoch_number);
+  PutLengthPrefixedSlice(dst, Slice(varint_epoch_number));
+
+  if (f.file_checksum_func_name != kUnknownFileChecksumFuncName) {
+    PutVarint32(dst, NewFileCustomTag::kFileChecksum);
+    PutLengthPrefixedSlice(dst, Slice(f.file_checksum));
+
+    PutVarint32(dst, NewFileCustomTag::kFileChecksumFuncName);
+    PutLengthPrefixedSlice(dst, Slice(f.file_checksum_func_name));
+  }
+
+  if (f.fd.GetPathId() != 0) {
+    PutVarint32(dst, NewFileCustomTag::kPathId);
+    char p = static_cast<char>(f.fd.GetPathId());
+    PutLengthPrefixedSlice(dst, Slice(&p, 1));
+  }
+  if (f.temperature != Temperature::kUnknown) {
+    PutVarint32(dst, NewFileCustomTag::kTemperature);
+    char p = static_cast<char>(f.temperature);
+    PutLengthPrefixedSlice(dst, Slice(&p, 1));
+  }
+  if (f.marked_for_compaction) {
+    PutVarint32(dst, NewFileCustomTag::kNeedCompaction);
+    char p = static_cast<char>(1);
+    PutLengthPrefixedSlice(dst, Slice(&p, 1));
+  }
+  if (has_min_log_number_to_keep && !min_log_num_written) {
+    PutVarint32(dst, NewFileCustomTag::kMinLogNumberToKeepHack);
+    std::string varint_log_number;
+    PutFixed64(&varint_log_number, min_log_number_to_keep);
+    PutLengthPrefixedSlice(dst, Slice(varint_log_number));
+    min_log_num_written = true;
+  }
+  if (f.oldest_blob_file_number != kInvalidBlobFileNumber) {
+    PutVarint32(dst, NewFileCustomTag::kOldestBlobFileNumber);
+    std::string oldest_blob_file_number;
+    PutVarint64(&oldest_blob_file_number, f.oldest_blob_file_number);
+    PutLengthPrefixedSlice(dst, Slice(oldest_blob_file_number));
+  }
+  UniqueId64x2 unique_id = f.unique_id;
+  TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:UniqueId", &unique_id);
+  if (unique_id != kNullUniqueId64x2) {
+    PutVarint32(dst, NewFileCustomTag::kUniqueId);
+    std::string unique_id_str = EncodeUniqueIdBytes(&unique_id);
+    PutLengthPrefixedSlice(dst, Slice(unique_id_str));
+  }
+  if (f.compensated_range_deletion_size) {
+    PutVarint32(dst, NewFileCustomTag::kCompensatedRangeDeletionSize);
+    std::string compensated_range_deletion_size;
+    PutVarint64(&compensated_range_deletion_size,
+                f.compensated_range_deletion_size);
+    PutLengthPrefixedSlice(dst, Slice(compensated_range_deletion_size));
+  }
+  if (f.tail_size) {
+    PutVarint32(dst, NewFileCustomTag::kTailSize);
+    std::string varint_tail_size;
+    PutVarint64(&varint_tail_size, f.tail_size);
+    PutLengthPrefixedSlice(dst, Slice(varint_tail_size));
+  }
+  if (!f.user_defined_timestamps_persisted) {
+    // The default value for the flag is true, it's only explicitly persisted
+    // when it's false. We are putting 0 as the value here to signal false
+    // (i.e. UDTS not persisted).
+    PutVarint32(dst, NewFileCustomTag::kUserDefinedTimestampsPersisted);
+    char p = static_cast<char>(0);
+    PutLengthPrefixedSlice(dst, Slice(&p, 1));
+  }
+  TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:NewFile4:CustomizeFields",
+                           dst);
+
+  PutVarint32(dst, NewFileCustomTag::kTerminate);
+}
 static bool GetInternalKey(Slice* input, InternalKey* dst) {
   Slice str;
   if (GetLengthPrefixedSlice(input, &str)) {
@@ -301,12 +319,12 @@ static bool GetInternalKey(Slice* input, InternalKey* dst) {
   }
 }
 
-bool VersionEdit::GetLevel(Slice* input, int* level, const char** /*msg*/) {
+bool VersionEdit::GetLevel(Slice* input, int* level, int& max_level) {
   uint32_t v = 0;
   if (GetVarint32(input, &v)) {
     *level = v;
-    if (max_level_ < *level) {
-      max_level_ = *level;
+    if (max_level < *level) {
+      max_level = *level;
     }
     return true;
   } else {
@@ -314,16 +332,18 @@ bool VersionEdit::GetLevel(Slice* input, int* level, const char** /*msg*/) {
   }
 }
 
-const char* VersionEdit::DecodeNewFile4From(Slice* input) {
-  const char* msg = nullptr;
+const char* VersionEdit::DecodeNewFile4From(Slice* input, int& max_level,
+                                            uint64_t& min_log_number_to_keep,
+                                            bool& has_min_log_number_to_keep,
+                                            NewFiles& new_files,
+                                            FileMetaData& f) {
   int level = 0;
-  FileMetaData f;
   uint64_t number = 0;
   uint32_t path_id = 0;
   uint64_t file_size = 0;
   SequenceNumber smallest_seqno = 0;
   SequenceNumber largest_seqno = kMaxSequenceNumber;
-  if (GetLevel(input, &level, &msg) && GetVarint64(input, &number) &&
+  if (GetLevel(input, &level, max_level) && GetVarint64(input, &number) &&
       GetVarint64(input, &file_size) && GetInternalKey(input, &f.smallest) &&
       GetInternalKey(input, &f.largest) &&
       GetVarint64(input, &smallest_seqno) &&
@@ -381,10 +401,10 @@ const char* VersionEdit::DecodeNewFile4From(Slice* input) {
         case kMinLogNumberToKeepHack:
           // This is a hack to encode kMinLogNumberToKeep in a
           // forward-compatible fashion.
-          if (!GetFixed64(&field, &min_log_number_to_keep_)) {
+          if (!GetFixed64(&field, &min_log_number_to_keep)) {
             return "deleted log number malformatted";
           }
-          has_min_log_number_to_keep_ = true;
+          has_min_log_number_to_keep = true;
           break;
         case kOldestBlobFileNumber:
           if (!GetVarint64(&field, &f.oldest_blob_file_number)) {
@@ -436,13 +456,12 @@ const char* VersionEdit::DecodeNewFile4From(Slice* input) {
   }
   f.fd =
       FileDescriptor(number, path_id, file_size, smallest_seqno, largest_seqno);
-  new_files_.push_back(std::make_pair(level, f));
+  new_files.emplace_back(level, f);
   return nullptr;
 }
 
 void VersionEdit::EncodeFileBoundaries(std::string* dst,
-                                       const FileMetaData& meta,
-                                       size_t ts_sz) const {
+                                       const FileMetaData& meta, size_t ts_sz) {
   if (ts_sz == 0 || meta.user_defined_timestamps_persisted) {
     PutLengthPrefixedSlice(dst, meta.smallest.Encode());
     PutLengthPrefixedSlice(dst, meta.largest.Encode());
@@ -545,7 +564,8 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
         break;
 
       case kCompactCursor:
-        if (GetLevel(&input, &level, &msg) && GetInternalKey(&input, &key)) {
+        if (GetLevel(&input, &level, max_level_) &&
+            GetInternalKey(&input, &key)) {
           // Here we re-use the output format of compact pointer in LevelDB
           // to persist compact_cursors_
           compact_cursors_.push_back(std::make_pair(level, key));
@@ -558,7 +578,8 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
 
       case kDeletedFile: {
         uint64_t number = 0;
-        if (GetLevel(&input, &level, &msg) && GetVarint64(&input, &number)) {
+        if (GetLevel(&input, &level, max_level_) &&
+            GetVarint64(&input, &number)) {
           deleted_files_.insert(std::make_pair(level, number));
         } else {
           if (!msg) {
@@ -571,8 +592,8 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
       case kNewFile: {
         uint64_t number = 0;
         uint64_t file_size = 0;
-        if (GetLevel(&input, &level, &msg) && GetVarint64(&input, &number) &&
-            GetVarint64(&input, &file_size) &&
+        if (GetLevel(&input, &level, max_level_) &&
+            GetVarint64(&input, &number) && GetVarint64(&input, &file_size) &&
             GetInternalKey(&input, &f.smallest) &&
             GetInternalKey(&input, &f.largest)) {
           f.fd = FileDescriptor(number, 0, file_size);
@@ -589,8 +610,8 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
         uint64_t file_size = 0;
         SequenceNumber smallest_seqno = 0;
         SequenceNumber largest_seqno = kMaxSequenceNumber;
-        if (GetLevel(&input, &level, &msg) && GetVarint64(&input, &number) &&
-            GetVarint64(&input, &file_size) &&
+        if (GetLevel(&input, &level, max_level_) &&
+            GetVarint64(&input, &number) && GetVarint64(&input, &file_size) &&
             GetInternalKey(&input, &f.smallest) &&
             GetInternalKey(&input, &f.largest) &&
             GetVarint64(&input, &smallest_seqno) &&
@@ -612,8 +633,9 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
         uint64_t file_size = 0;
         SequenceNumber smallest_seqno = 0;
         SequenceNumber largest_seqno = kMaxSequenceNumber;
-        if (GetLevel(&input, &level, &msg) && GetVarint64(&input, &number) &&
-            GetVarint32(&input, &path_id) && GetVarint64(&input, &file_size) &&
+        if (GetLevel(&input, &level, max_level_) &&
+            GetVarint64(&input, &number) && GetVarint32(&input, &path_id) &&
+            GetVarint64(&input, &file_size) &&
             GetInternalKey(&input, &f.smallest) &&
             GetInternalKey(&input, &f.largest) &&
             GetVarint64(&input, &smallest_seqno) &&
@@ -630,7 +652,10 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
       }
 
       case kNewFile4: {
-        msg = DecodeNewFile4From(&input);
+        FileMetaData ignored_file;
+        msg = DecodeNewFile4From(&input, max_level_, min_log_number_to_keep_,
+                                 has_min_log_number_to_keep_, new_files_,
+                                 ignored_file);
         break;
       }
 
@@ -767,6 +792,23 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
         }
         break;
 
+      case kSubcompactionProgress: {
+        Slice encoded;
+        if (!GetLengthPrefixedSlice(&input, &encoded)) {
+          msg = "SubcompactionProgress not prefixed by length";
+          break;
+        }
+
+        SubcompactionProgress progress;
+        Status s = progress.DecodeFrom(&encoded);
+        if (!s.ok()) {
+          return s;
+        }
+
+        SetSubcompactionProgress(progress);
+        break;
+      }
+
       default:
         if (tag & kTagSafeIgnoreMask) {
           // Tag from future which can be safely ignored.
@@ -933,6 +975,10 @@ std::string VersionEdit::DebugString(bool hex_key) const {
     r.append("\n FullHistoryTsLow: ");
     r.append(Slice(full_history_ts_low_).ToString(hex_key));
   }
+  if (HasSubcompactionProgress()) {
+    r.append("\n SubcompactionProgress: ");
+    r.append(subcompaction_progress_.ToString());
+  }
   r.append("\n}\n");
   return r;
 }
@@ -1082,9 +1128,301 @@ std::string VersionEdit::DebugJSON(int edit_num, bool hex_key) const {
     jw << "FullHistoryTsLow" << Slice(full_history_ts_low_).ToString(hex_key);
   }
 
+  if (HasSubcompactionProgress()) {
+    jw << "SubcompactionProgress" << subcompaction_progress_.ToString();
+  }
+
   jw.EndObject();
 
   return jw.Get();
 }
 
+void SubcompactionProgressPerLevel::EncodeTo(std::string* dst) const {
+  if (num_processed_output_records_ > 0) {
+    PutVarint32(
+        dst,
+        SubcompactionProgressPerLevelCustomTag::kNumProcessedOutputRecords);
+    std::string varint_records;
+    PutVarint64(&varint_records, num_processed_output_records_);
+    PutLengthPrefixedSlice(dst, varint_records);
+  }
+
+  if (!output_files_.empty()) {
+    PutVarint32(dst, SubcompactionProgressPerLevelCustomTag::kOutputFilesDelta);
+    std::string files_data;
+    EncodeOutputFiles(&files_data);
+    PutLengthPrefixedSlice(dst, files_data);
+  }
+
+  PutVarint32(dst, SubcompactionProgressPerLevelCustomTag::
+                       kSubcompactionProgressPerLevelTerminate);
+}
+
+Status SubcompactionProgressPerLevel::DecodeFrom(Slice* input) {
+  Clear();
+
+  while (true) {
+    uint32_t tag = 0;
+    if (!GetVarint32(input, &tag)) {
+      return Status::Corruption("SubcompactionProgressPerLevel", "tag error");
+    }
+
+    if (tag == SubcompactionProgressPerLevelCustomTag::
+                   kSubcompactionProgressPerLevelTerminate) {
+      break;
+    }
+
+    Slice field;
+    if (!GetLengthPrefixedSlice(input, &field)) {
+      return Status::Corruption("SubcompactionProgressPerLevel",
+                                "field length prefixed slice error");
+    }
+
+    switch (tag) {
+      case SubcompactionProgressPerLevelCustomTag::kNumProcessedOutputRecords: {
+        if (!GetVarint64(&field, &num_processed_output_records_)) {
+          return Status::Corruption("SubcompactionProgressPerLevel",
+                                    "invalid num_processed_output_records_");
+        }
+        break;
+      }
+
+      case SubcompactionProgressPerLevelCustomTag::kOutputFilesDelta: {
+        Status s = DecodeOutputFiles(&field, output_files_);
+        if (!s.ok()) {
+          return s;
+        }
+        break;
+      }
+
+      default:
+        // Forward compatibility: Handle unknown tags
+        if ((tag & SubcompactionProgressPerLevelCustomTag::
+                       kSubcompactionProgressPerLevelCustomTagSafeIgnoreMask) !=
+            0) {
+          break;
+        } else {
+          return Status::NotSupported("SubcompactionProgress",
+                                      "unsupported critical custom field");
+        }
+    }
+  }
+
+  return Status::OK();
+}
+
+void SubcompactionProgressPerLevel::EncodeOutputFiles(std::string* dst) const {
+  size_t new_files_count =
+      output_files_.size() > last_persisted_output_files_count_
+          ? output_files_.size() - last_persisted_output_files_count_
+          : 0;
+
+  assert(new_files_count > 0);
+
+  PutVarint32(dst, static_cast<uint32_t>(new_files_count));
+
+  for (size_t i = last_persisted_output_files_count_; i < output_files_.size();
+       ++i) {
+    std::string file_dst;
+    bool ignored_min_log_written = false;
+
+    VersionEdit::EncodeToNewFile4(
+        output_files_[i], -1 /* level */, 0 /* ts_sz */,
+        false /* has_min_log_number_to_keep */, 0 /* min_log_number_to_keep */,
+        ignored_min_log_written, &file_dst);
+
+    PutLengthPrefixedSlice(dst, file_dst);
+  }
+}
+
+Status SubcompactionProgressPerLevel::DecodeOutputFiles(
+    Slice* input, autovector<FileMetaData>& output_files) {
+  uint32_t new_file_count = 0;
+  if (!GetVarint32(input, &new_file_count)) {
+    return Status::Corruption("SubcompactionProgressPerLevel",
+                              "new output file count");
+  }
+
+  assert(output_files.size() == 0);
+
+  output_files.reserve(new_file_count);
+
+  for (uint32_t i = 0; i < new_file_count; ++i) {
+    Slice file_input;
+    if (!GetLengthPrefixedSlice(input, &file_input)) {
+      return Status::Corruption("SubcompactionProgressPerLevel",
+                                "output file metadata");
+    }
+
+    uint32_t tag = 0;
+    if (!GetVarint32(&file_input, &tag) || tag != kNewFile4) {
+      return Status::Corruption("SubcompactionProgressPerLevel",
+                                "expected kNewFile4 tag");
+    }
+
+    int ignored_max_level = -1;
+    uint64_t ignored_min_log_number_to_keep = 0;
+    bool ignored_has_min_log_number_to_keep = false;
+    VersionEdit::NewFiles ignored_new_files;
+    FileMetaData file;
+
+    const char* err = VersionEdit::DecodeNewFile4From(
+        &file_input, ignored_max_level, ignored_min_log_number_to_keep,
+        ignored_has_min_log_number_to_keep, ignored_new_files, file);
+
+    if (err != nullptr) {
+      return Status::Corruption("SubcompactionProgressPerLevel", err);
+    }
+
+    output_files.push_back(std::move(file));
+  }
+
+  return Status::OK();
+}
+
+void SubcompactionProgress::EncodeTo(std::string* dst) const {
+  if (!next_internal_key_to_compact.empty()) {
+    PutVarint32(dst, SubcompactionProgressCustomTag::kNextInternalKeyToCompact);
+    PutLengthPrefixedSlice(dst, next_internal_key_to_compact);
+  }
+
+  PutVarint32(dst, SubcompactionProgressCustomTag::kNumProcessedInputRecords);
+  std::string varint_records;
+  PutVarint64(&varint_records, num_processed_input_records);
+  PutLengthPrefixedSlice(dst, varint_records);
+
+  if (output_level_progress.GetOutputFiles().size() >
+      output_level_progress.GetLastPersistedOutputFilesCount()) {
+    PutVarint32(dst, SubcompactionProgressCustomTag::kOutputLevelProgress);
+    std::string level_progress_data;
+    output_level_progress.EncodeTo(&level_progress_data);
+    PutLengthPrefixedSlice(dst, level_progress_data);
+  }
+
+  if (proximal_output_level_progress.GetOutputFiles().size() >
+      proximal_output_level_progress.GetLastPersistedOutputFilesCount()) {
+    PutVarint32(dst,
+                SubcompactionProgressCustomTag::kProximalOutputLevelProgress);
+    std::string level_progress_data;
+    proximal_output_level_progress.EncodeTo(&level_progress_data);
+    PutLengthPrefixedSlice(dst, level_progress_data);
+  }
+  PutVarint32(dst,
+              SubcompactionProgressCustomTag::kSubcompactionProgressTerminate);
+}
+
+Status SubcompactionProgress::DecodeFrom(Slice* input) {
+  Clear();
+
+  while (true) {
+    uint32_t custom_tag = 0;
+    if (!GetVarint32(input, &custom_tag)) {
+      return Status::Corruption("SubcompactionProgress",
+                                "custom field tag error");
+    }
+
+    if (custom_tag ==
+        SubcompactionProgressCustomTag::kSubcompactionProgressTerminate) {
+      break;
+    }
+
+    Slice field;
+    if (!GetLengthPrefixedSlice(input, &field)) {
+      return Status::Corruption("SubcompactionProgress",
+                                "custom field length prefixed slice error");
+    }
+
+    switch (custom_tag) {
+      case SubcompactionProgressCustomTag::kNextInternalKeyToCompact:
+        next_internal_key_to_compact = field.ToString();
+        break;
+
+      case SubcompactionProgressCustomTag::kNumProcessedInputRecords:
+        if (!GetVarint64(&field, &num_processed_input_records)) {
+          return Status::Corruption("SubcompactionProgress",
+                                    "invalid num_processed_input_records");
+        }
+        break;
+
+      case SubcompactionProgressCustomTag::kOutputLevelProgress: {
+        Status s = output_level_progress.DecodeFrom(&field);
+        if (!s.ok()) {
+          return s;
+        }
+        break;
+      }
+
+      case SubcompactionProgressCustomTag::kProximalOutputLevelProgress: {
+        Status s = proximal_output_level_progress.DecodeFrom(&field);
+        if (!s.ok()) {
+          return s;
+        }
+        break;
+      }
+
+      default:
+        if ((custom_tag & SubcompactionProgressCustomTag::
+                              kSubcompactionProgressCustomTagSafeIgnoreMask) !=
+            0) {
+          break;
+        } else {
+          return Status::NotSupported("SubcompactionProgress",
+                                      "unsupported critical custom field");
+        }
+    }
+  }
+
+  return Status::OK();
+}
+
+bool SubcompactionProgressBuilder::ProcessVersionEdit(const VersionEdit& edit) {
+  if (!edit.HasSubcompactionProgress()) {
+    return false;
+  }
+
+  const SubcompactionProgress& progress = edit.GetSubcompactionProgress();
+
+  MergeDeltaProgress(progress);
+
+  has_subcompaction_progress_ = true;
+
+  return true;
+}
+
+void SubcompactionProgressBuilder::MergeDeltaProgress(
+    const SubcompactionProgress& delta_progress) {
+  accumulated_subcompaction_progress_.next_internal_key_to_compact =
+      delta_progress.next_internal_key_to_compact;
+
+  accumulated_subcompaction_progress_.num_processed_input_records =
+      delta_progress.num_processed_input_records;
+
+  MaybeMergeDeltaProgressPerLevel(
+      accumulated_subcompaction_progress_.output_level_progress,
+      delta_progress.output_level_progress);
+
+  MaybeMergeDeltaProgressPerLevel(
+      accumulated_subcompaction_progress_.proximal_output_level_progress,
+      delta_progress.proximal_output_level_progress);
+}
+
+void SubcompactionProgressBuilder::MaybeMergeDeltaProgressPerLevel(
+    SubcompactionProgressPerLevel& accumulated_level_progress,
+    const SubcompactionProgressPerLevel& delta_level_progress) {
+  const auto& delta_files = delta_level_progress.GetOutputFiles();
+  if (delta_files.empty()) {
+    return;
+  }
+  for (const FileMetaData& file : delta_files) {
+    accumulated_level_progress.AddToOutputFiles(file);  // Stored as copy
+  }
+
+  accumulated_level_progress.SetNumProcessedOutputRecords(
+      delta_level_progress.GetNumProcessedOutputRecords());
+}
+
+void SubcompactionProgressBuilder::Clear() {
+  accumulated_subcompaction_progress_.Clear();
+  has_subcompaction_progress_ = false;
+}
 }  // namespace ROCKSDB_NAMESPACE

db/version_edit.h

@@ -72,6 +72,23 @@ enum Tag : uint32_t {
   kWalAddition2,
   kWalDeletion2,
   kPersistUserDefinedTimestamps,
+  kSubcompactionProgress,
+};
+
+enum SubcompactionProgressPerLevelCustomTag : uint32_t {
+  kSubcompactionProgressPerLevelTerminate = 1,  // End of fields marker
+  kOutputFilesDelta = 2,
+  kNumProcessedOutputRecords = 3,
+  kSubcompactionProgressPerLevelCustomTagSafeIgnoreMask = 1 << 16,
+};
+
+enum SubcompactionProgressCustomTag : uint32_t {
+  kSubcompactionProgressTerminate = 1,  // End of fields marker
+  kNextInternalKeyToCompact = 2,
+  kNumProcessedInputRecords = 3,
+  kOutputLevelProgress = 4,
+  kProximalOutputLevelProgress = 5,
+  kSubcompactionProgressCustomTagSafeIgnoreMask = 1 << 16,
 };
 
 enum NewFileCustomTag : uint32_t {
@@ -440,12 +457,191 @@ struct LevelFilesBrief {
   }
 };
 
+struct SubcompactionProgressPerLevel {
+  uint64_t GetNumProcessedOutputRecords() const {
+    return num_processed_output_records_;
+  }
+
+  void SetNumProcessedOutputRecords(uint64_t num) {
+    num_processed_output_records_ = num;
+  }
+
+  const autovector<FileMetaData>& GetOutputFiles() const {
+    return output_files_;
+  }
+
+  void AddToOutputFiles(const FileMetaData& file) {
+    output_files_.push_back(file);
+  }
+
+  size_t GetLastPersistedOutputFilesCount() const {
+    return last_persisted_output_files_count_;
+  }
+
+  void UpdateLastPersistedOutputFilesCount() {
+    last_persisted_output_files_count_ = output_files_.size();
+  }
+
+  void EncodeTo(std::string* dst) const;
+
+  Status DecodeFrom(Slice* input);
+
+  void Clear() {
+    num_processed_output_records_ = 0;
+    output_files_.clear();
+    last_persisted_output_files_count_ = 0;
+  }
+
+  std::string ToString() const {
+    std::ostringstream oss;
+    oss << "SubcompactionProgressPerLevel{";
+    oss << " num_processed_output_records=" << num_processed_output_records_;
+    oss << ", output_files_count=" << output_files_.size();
+    oss << ", last_persisted_output_files_count="
+        << last_persisted_output_files_count_;
+    oss << " }";
+    return oss.str();
+  }
+
+  void TEST_ClearOutputFiles() { output_files_.clear(); }
+
+ private:
+  uint64_t num_processed_output_records_ = 0;
+
+  autovector<FileMetaData> output_files_ = {};
+
+  // Number of files already persisted to help calculate the new output files to
+  // persist in the future. This is to prevent having to persist all the output
+  // files metadata so far every time of a "snapshot" of a progress is persisted
+  // which can lead to O(1+2+...+n) = O(n^2) file metadata being persisted. The
+  // current approach of persisting only the delta should always persist
+  // exactly the number (n) of output files in total.
+  size_t last_persisted_output_files_count_ = 0;
+
+  void EncodeOutputFiles(std::string* dst) const;
+
+  Status DecodeOutputFiles(Slice* input,
+                           autovector<FileMetaData>& temp_storage);
+};
+
+struct SubcompactionProgress {
+  std::string next_internal_key_to_compact;
+
+  uint64_t num_processed_input_records = 0;
+
+  SubcompactionProgressPerLevel output_level_progress;
+
+  SubcompactionProgressPerLevel proximal_output_level_progress;
+
+  SubcompactionProgress() = default;
+
+  void Clear() {
+    next_internal_key_to_compact.clear();
+    num_processed_input_records = 0;
+    output_level_progress.Clear();
+    proximal_output_level_progress.Clear();
+  }
+
+  void EncodeTo(std::string* dst) const;
+
+  Status DecodeFrom(Slice* input);
+
+  std::string ToString() const {
+    std::ostringstream oss;
+    oss << "SubcompactionProgress{";
+    oss << " next_internal_key_to_compact=";
+    if (next_internal_key_to_compact.empty()) {
+      oss << "";
+    } else {
+      ParsedInternalKey parsed_key;
+      Slice key_slice(next_internal_key_to_compact);
+      if (ParseInternalKey(key_slice, &parsed_key, false /* log_err_key */)
+              .ok()) {
+        oss << "user_key=\"" << parsed_key.user_key.ToString(false /* hex */)
+            << "\" (hex:" << parsed_key.user_key.ToString(true /* hex */)
+            << ")";
+        oss << ", seq=";
+        if (parsed_key.sequence == kMaxSequenceNumber) {
+          oss << "kMaxSequenceNumber";
+        } else {
+          oss << parsed_key.sequence;
+        }
+        oss << ", type=" << static_cast<int>(parsed_key.type);
+      } else {
+        oss << "raw=" << key_slice.ToString(true /* hex */);
+      }
+    }
+    oss << ", num_processed_input_records=" << num_processed_input_records;
+    oss << ", output_level_progress=" << output_level_progress.ToString();
+    oss << ", proximal_output_level_progress="
+        << proximal_output_level_progress.ToString();
+    oss << " }";
+    return oss.str();
+  }
+};
+
+class VersionEdit;
+
+// Builder class to reconstruct complete subcompaction progress object
+// from multiple decoded VersionEdits containing delta output files information
+// of the same subcompaction. See
+// `SubcompactionProgressPerLevel::last_persisted_output_files_count_`'s comment
+//
+// WARNING: This class currently assumes all input VersionEdits contain progress
+// information for the SAME subcompaction. It does not validate
+// progress data from different subcompactions so mixing progress from
+// multiple subcompactions can result in corrupted state silently. The caller is
+// responsible for ensuring all VersionEdits processed by a single instance
+// of this builder correspond to the same subcompaction.
+class SubcompactionProgressBuilder {
+ public:
+  SubcompactionProgressBuilder() = default;
+
+  bool ProcessVersionEdit(const VersionEdit& edit);
+
+  const SubcompactionProgress& GetAccumulatedSubcompactionProgress() const {
+    return accumulated_subcompaction_progress_;
+  }
+
+  bool HasAccumulatedSubcompactionProgress() const {
+    return has_subcompaction_progress_;
+  }
+
+  void Clear();
+
+ private:
+  void MergeDeltaProgress(const SubcompactionProgress& delta_progress);
+
+  void MaybeMergeDeltaProgressPerLevel(
+      SubcompactionProgressPerLevel& accumulated_level_progress,
+      const SubcompactionProgressPerLevel& delta_level_progress);
+
+  SubcompactionProgress accumulated_subcompaction_progress_;
+  bool has_subcompaction_progress_ = false;
+};
+
+// Type alias for backward compatibility - vector of subcompaction progress
+using CompactionProgress = std::vector<SubcompactionProgress>;
+
 // The state of a DB at any given time is referred to as a Version.
 // Any modification to the Version is considered a Version Edit. A Version is
 // constructed by joining a sequence of Version Edits. Version Edits are written
 // to the MANIFEST file.
 class VersionEdit {
  public:
+  // Retrieve the table files added as well as their associated levels.
+  using NewFiles = std::vector<std::pair<int, FileMetaData>>;
+
+  static void EncodeToNewFile4(const FileMetaData& f, int level, size_t ts_sz,
+                               bool has_min_log_number_to_keep,
+                               uint64_t min_log_number_to_keep,
+                               bool& min_log_num_written, std::string* dst);
+
+  static const char* DecodeNewFile4From(Slice* input, int& max_level,
+                                        uint64_t& min_log_number_to_keep,
+                                        bool& has_min_log_number_to_keep,
+                                        NewFiles& new_files, FileMetaData& f);
+
   void Clear();
 
   void SetDBId(const std::string& db_id) {
@@ -564,8 +760,6 @@ class VersionEdit {
     }
   }
 
-  // Retrieve the table files added as well as their associated levels.
-  using NewFiles = std::vector<std::pair<int, FileMetaData>>;
   const NewFiles& GetNewFiles() const { return new_files_; }
 
   NewFiles& GetMutableNewFiles() { return new_files_; }
@@ -735,6 +929,22 @@ class VersionEdit {
     full_history_ts_low_ = std::move(full_history_ts_low);
   }
 
+  void SetSubcompactionProgress(const SubcompactionProgress& progress) {
+    has_subcompaction_progress_ = true;
+    subcompaction_progress_ = progress;
+  }
+
+  bool HasSubcompactionProgress() const { return has_subcompaction_progress_; }
+
+  const SubcompactionProgress& GetSubcompactionProgress() const {
+    return subcompaction_progress_;
+  }
+
+  void ClearSubcompactionProgress() {
+    has_subcompaction_progress_ = false;
+    subcompaction_progress_.Clear();
+  }
+
   // return true on success.
   // `ts_sz` is the size in bytes for the user-defined timestamp contained in
   // a user key. This argument is optional because it's only required for
@@ -757,15 +967,22 @@ class VersionEdit {
   std::string DebugJSON(int edit_num, bool hex_key = false) const;
 
  private:
-  bool GetLevel(Slice* input, int* level, const char** msg);
-
-  const char* DecodeNewFile4From(Slice* input);
-
+  // Decode level information from serialized VersionEdit data and and track the
+  // maximum level seen.
+  //
+  // Parameters:
+  //   input: Pointer to serialized data slice
+  //   level: Output parameter for the decoded level value
+  //   max_level: get updated if the decoded level is higher than passed in
+  //   value
+  //
+  // Returns: true on successful decode, false on parse error
+  static bool GetLevel(Slice* input, int* level, int& max_level);
   // Encode file boundaries `FileMetaData.smallest` and `FileMetaData.largest`.
   // User-defined timestamps in the user key will be stripped if they shouldn't
   // be persisted.
-  void EncodeFileBoundaries(std::string* dst, const FileMetaData& meta,
-                            size_t ts_sz) const;
+  static void EncodeFileBoundaries(std::string* dst, const FileMetaData& meta,
+                                   size_t ts_sz);
 
   int max_level_ = 0;
   std::string db_id_;
@@ -816,6 +1033,9 @@ class VersionEdit {
   std::string full_history_ts_low_;
   bool persist_user_defined_timestamps_ = true;
 
+  bool has_subcompaction_progress_ = false;
+  SubcompactionProgress subcompaction_progress_;
+
   // Newly created table files and blob files are eligible for deletion if they
   // are not registered as live files after the background jobs creating them
   // have finished. In case committing the VersionEdit containing such changes

db/version_edit_test.cc

@@ -794,6 +794,339 @@ TEST(FileMetaDataTest, UpdateBoundariesBlobIndex) {
   }
 }
 
+class SubcompactionProgressTest : public VersionEditTest {
+ protected:
+  static constexpr uint64_t kTestFileSize = 1024;
+  static constexpr SequenceNumber kTestSmallestSeq = 50;
+  static constexpr SequenceNumber kTestLargestSeq = 150;
+  static constexpr uint64_t kTestOldestAncesterTime = 12345;
+  static constexpr uint64_t kTestFileCreationTime = 67890;
+  static constexpr uint64_t kTestEpochNumber = 10;
+  static const std::string kTestChecksumFuncName;
+
+  FileMetaData CreateTestFile(uint64_t file_number, const std::string& prefix) {
+    FileMetaData file;
+    file.fd = FileDescriptor(file_number, 0, kTestFileSize, kTestSmallestSeq,
+                             kTestLargestSeq);
+    file.smallest = InternalKey(prefix + "a", kTestSmallestSeq, kTypeValue);
+    file.largest = InternalKey(prefix + "z", kTestLargestSeq, kTypeValue);
+    file.oldest_ancester_time = kTestOldestAncesterTime;
+    file.file_creation_time = kTestFileCreationTime;
+    file.epoch_number = kTestEpochNumber;
+    file.file_checksum = "checksum_" + std::to_string(file_number);
+    file.file_checksum_func_name = kTestChecksumFuncName;
+    file.marked_for_compaction = false;
+    file.temperature = Temperature::kUnknown;
+    return file;
+  }
+
+  // Store external file metadata objects for testing
+  // These simulate files owned by CompactionOutputs
+  std::vector<FileMetaData> compaction_output_files_;
+  std::vector<FileMetaData> proximal_level_compaction_output_files_;
+
+  SubcompactionProgress CreateSubcompactionProgress(
+      const std::string& next_key, uint64_t num_processed_input_records,
+      uint64_t num_processed_output_records,
+      uint64_t num_processed_proximal_level_output_records,
+      const std::vector<uint64_t>& output_file_numbers = {},
+      const std::vector<uint64_t>& proximal_file_numbers = {},
+      const std::string& file_prefix = "file_") {
+    SubcompactionProgress progress;
+    progress.next_internal_key_to_compact = next_key;
+    progress.num_processed_input_records = num_processed_input_records;
+    progress.output_level_progress.SetNumProcessedOutputRecords(
+        num_processed_output_records);
+    progress.proximal_output_level_progress.SetNumProcessedOutputRecords(
+        num_processed_proximal_level_output_records);
+
+    for (uint64_t file_num : output_file_numbers) {
+      FileMetaData file = CreateTestFile(file_num, file_prefix + "output_");
+      progress.output_level_progress.AddToOutputFiles(file);
+    }
+
+    for (uint64_t file_num : proximal_file_numbers) {
+      FileMetaData file = CreateTestFile(file_num, file_prefix + "proximal_");
+      progress.proximal_output_level_progress.AddToOutputFiles(file);
+    }
+
+    return progress;
+  }
+
+  std::pair<const VersionEdit, const SubcompactionProgress>
+  EncodeDecodeProgress(const SubcompactionProgress& progress) {
+    VersionEdit edit;
+    edit.SetSubcompactionProgress(progress);
+
+    std::string encoded;
+    EXPECT_TRUE(edit.EncodeTo(&encoded, 0 /* ts_sz */));
+
+    VersionEdit decoded_edit;
+    EXPECT_OK(decoded_edit.DecodeFrom(encoded));
+    EXPECT_TRUE(decoded_edit.HasSubcompactionProgress());
+
+    SubcompactionProgress decoded_progress =
+        decoded_edit.GetSubcompactionProgress();
+
+    return {std::move(decoded_edit), std::move(decoded_progress)};
+  }
+
+  void VerifyFileMetaDataEquality(const FileMetaData& expected,
+                                  const FileMetaData& actual) {
+    // Verify the major fields only
+    ASSERT_EQ(actual.fd.GetNumber(), expected.fd.GetNumber());
+    ASSERT_EQ(actual.fd.GetFileSize(), expected.fd.GetFileSize());
+    ASSERT_EQ(actual.smallest.Encode(), expected.smallest.Encode());
+    ASSERT_EQ(actual.largest.Encode(), expected.largest.Encode());
+    ASSERT_EQ(actual.oldest_ancester_time, expected.oldest_ancester_time);
+    ASSERT_EQ(actual.file_creation_time, expected.file_creation_time);
+    ASSERT_EQ(actual.epoch_number, expected.epoch_number);
+    ASSERT_EQ(actual.file_checksum, expected.file_checksum);
+    ASSERT_EQ(actual.file_checksum_func_name, expected.file_checksum_func_name);
+    ASSERT_EQ(actual.marked_for_compaction, expected.marked_for_compaction);
+    ASSERT_EQ(actual.temperature, expected.temperature);
+  }
+
+  void VerifyProgressEquality(const SubcompactionProgress& expected,
+                              const SubcompactionProgress& actual) {
+    ASSERT_EQ(actual.next_internal_key_to_compact,
+              expected.next_internal_key_to_compact);
+
+    ASSERT_EQ(actual.num_processed_input_records,
+              expected.num_processed_input_records);
+
+    for (const bool is_proximal_level : {false, true}) {
+      const SubcompactionProgressPerLevel&
+          actual_subcompaction_progress_by_level =
+              is_proximal_level ? actual.proximal_output_level_progress
+                                : actual.output_level_progress;
+
+      const SubcompactionProgressPerLevel&
+          expected_subcompaction_progress_by_level =
+              is_proximal_level ? expected.proximal_output_level_progress
+                                : expected.output_level_progress;
+
+      ASSERT_EQ(
+          actual_subcompaction_progress_by_level.GetNumProcessedOutputRecords(),
+          expected_subcompaction_progress_by_level
+              .GetNumProcessedOutputRecords());
+
+      ASSERT_EQ(
+          actual_subcompaction_progress_by_level.GetOutputFiles().size(),
+          expected_subcompaction_progress_by_level.GetOutputFiles().size());
+
+      for (size_t i = 0;
+           i < expected_subcompaction_progress_by_level.GetOutputFiles().size();
+           ++i) {
+        VerifyFileMetaDataEquality(
+            expected_subcompaction_progress_by_level.GetOutputFiles()[i],
+            actual_subcompaction_progress_by_level.GetOutputFiles()[i]);
+      }
+    }
+  }
+};
+
+const std::string SubcompactionProgressTest::kTestChecksumFuncName = "crc32c";
+
+TEST_F(SubcompactionProgressTest, BasicEncodeDecode) {
+  // Create progress with files for both levels
+  SubcompactionProgress progress = CreateSubcompactionProgress(
+      "key_100",  // next_internal_key_to_compact
+      500,        // num_processed_input_records
+      400,        // num_processed_output_records
+      100,        // num_processed_proximal_level_output_records
+      {1},        // output_file_numbers
+      {2},        // proximal_file_numbers
+      "test_"     // file_prefix
+  );
+
+  auto [ignored, decoded_progress] = EncodeDecodeProgress(progress);
+
+  VerifyProgressEquality(progress, decoded_progress);
+}
+
+TEST_F(SubcompactionProgressTest, OutputFilesDeltaEncodeDecode) {
+  // Test Delta Encoding/Decoding
+  SubcompactionProgress initial_progress = CreateSubcompactionProgress(
+      "key_100",  // next_internal_key_to_compact
+      100,        // num_processed_input_records
+      40,         // num_processed_output_records
+      60,         // num_processed_proximal_level_output_records
+      {1},        // output_file_numbers
+      {2},        // proximal_file_numbers
+      "initial_"  // file_prefix
+  );
+
+  auto [initial_decoded_edit, ignored_1] =
+      EncodeDecodeProgress(initial_progress);
+  initial_progress.output_level_progress.UpdateLastPersistedOutputFilesCount();
+  initial_progress.proximal_output_level_progress
+      .UpdateLastPersistedOutputFilesCount();
+
+  // Add one new output file to output and proximal level
+  SubcompactionProgress updated_progress = initial_progress;
+  updated_progress.next_internal_key_to_compact = "key_300";
+  updated_progress.num_processed_input_records = 1000;
+  updated_progress.output_level_progress.SetNumProcessedOutputRecords(400);
+  updated_progress.proximal_output_level_progress.SetNumProcessedOutputRecords(
+      600);
+
+  FileMetaData new_file = CreateTestFile(3, "new_");
+  updated_progress.output_level_progress.AddToOutputFiles(new_file);
+
+  FileMetaData new_file_proximal = CreateTestFile(4, "new_");
+  updated_progress.proximal_output_level_progress.AddToOutputFiles(
+      new_file_proximal);
+
+  auto [delta_decoded_edit, delta_decoded_progress] =
+      EncodeDecodeProgress(updated_progress);
+
+  ASSERT_EQ(delta_decoded_progress.next_internal_key_to_compact,
+            updated_progress.next_internal_key_to_compact);
+
+  ASSERT_EQ(delta_decoded_progress.num_processed_input_records,
+            updated_progress.num_processed_input_records);
+
+  for (const bool& is_proximal_level : {false, true}) {
+    const SubcompactionProgressPerLevel& delta_progress_per_level =
+        is_proximal_level
+            ? delta_decoded_progress.proximal_output_level_progress
+            : delta_decoded_progress.output_level_progress;
+
+    const SubcompactionProgressPerLevel& updated_progress_per_level =
+        is_proximal_level ? updated_progress.proximal_output_level_progress
+                          : updated_progress.output_level_progress;
+
+    ASSERT_EQ(delta_progress_per_level.GetNumProcessedOutputRecords(),
+              updated_progress_per_level.GetNumProcessedOutputRecords());
+
+    // Only the newly added file since last persistence should be present
+    ASSERT_EQ(delta_progress_per_level.GetOutputFiles().size(), 1);
+
+    ASSERT_EQ(delta_progress_per_level.GetOutputFiles()[0].fd.GetNumber(),
+              is_proximal_level ? new_file_proximal.fd.GetNumber()
+                                : new_file.fd.GetNumber());
+  }
+
+  // Test SubcompactionProgressBuilder
+  SubcompactionProgressBuilder builder;
+  ASSERT_FALSE(builder.HasAccumulatedSubcompactionProgress());
+
+  ASSERT_TRUE(builder.ProcessVersionEdit(initial_decoded_edit));
+  ASSERT_TRUE(builder.HasAccumulatedSubcompactionProgress());
+  ASSERT_TRUE(builder.ProcessVersionEdit(delta_decoded_edit));
+
+  const auto& accumulated_progress =
+      builder.GetAccumulatedSubcompactionProgress();
+
+  ASSERT_EQ(accumulated_progress.next_internal_key_to_compact,
+            updated_progress.next_internal_key_to_compact);
+
+  ASSERT_EQ(accumulated_progress.num_processed_input_records,
+            updated_progress.num_processed_input_records);
+
+  for (const bool& is_proximal_level : {false, true}) {
+    const SubcompactionProgressPerLevel& accumulated_progress_per_level =
+        is_proximal_level ? accumulated_progress.proximal_output_level_progress
+                          : accumulated_progress.output_level_progress;
+
+    const SubcompactionProgressPerLevel& updated_progress_per_level =
+        is_proximal_level ? updated_progress.proximal_output_level_progress
+                          : updated_progress.output_level_progress;
+
+    ASSERT_EQ(accumulated_progress_per_level.GetNumProcessedOutputRecords(),
+              updated_progress_per_level.GetNumProcessedOutputRecords());
+
+    ASSERT_EQ(accumulated_progress_per_level.GetOutputFiles().size(),
+              updated_progress_per_level.GetOutputFiles().size());
+
+    std::set<uint64_t> accumulated_file_numbers;
+    for (const auto& file : accumulated_progress_per_level.GetOutputFiles()) {
+      accumulated_file_numbers.insert(file.fd.GetNumber());
+    }
+
+    std::set<uint64_t> expected_file_numbers;
+    for (const auto& file : updated_progress_per_level.GetOutputFiles()) {
+      expected_file_numbers.insert(file.fd.GetNumber());
+    }
+
+    ASSERT_EQ(accumulated_file_numbers, expected_file_numbers);
+  }
+
+  // ===== PART 3: Test Builder Reset =====
+  builder.Clear();
+  ASSERT_FALSE(builder.HasAccumulatedSubcompactionProgress());
+}
+
+TEST_F(SubcompactionProgressTest, UnknownTags) {
+  SubcompactionProgress progress;
+  std::string encoded;
+
+  // 1. Test unknown ignorable tag
+  progress.next_internal_key_to_compact = "test_key";
+  progress.num_processed_input_records = 100;
+
+  PutVarint32(&encoded,
+              SubcompactionProgressCustomTag::kNextInternalKeyToCompact);
+  PutLengthPrefixedSlice(&encoded, progress.next_internal_key_to_compact);
+
+  PutVarint32(&encoded,
+              SubcompactionProgressCustomTag::kNumProcessedInputRecords);
+  std::string varint_records;
+  PutVarint64(&varint_records, progress.num_processed_input_records);
+  PutLengthPrefixedSlice(&encoded, varint_records);
+
+  // Manually encode with unknown ignorable tag (has
+  // SubcompactionProgressCustomTag::kSubcompactionProgressCustomTagSafeIgnoreMask
+  // bit set)
+  uint32_t unknown_ignorable_tag =
+      SubcompactionProgressCustomTag::
+          kSubcompactionProgressCustomTagSafeIgnoreMask +
+      1;
+  PutVarint32(&encoded, unknown_ignorable_tag);
+  PutLengthPrefixedSlice(&encoded, "future_data");
+
+  PutVarint32(&encoded,
+              SubcompactionProgressCustomTag::kSubcompactionProgressTerminate);
+
+  // Test decoding - should succeed and ignore unknown tag
+  Slice input(encoded);
+  SubcompactionProgress decoded_progress;
+  Status s = decoded_progress.DecodeFrom(&input);
+  ASSERT_OK(s);
+
+  // Verify known fields are preserved
+  ASSERT_EQ(decoded_progress.next_internal_key_to_compact,
+            progress.next_internal_key_to_compact);
+  ASSERT_EQ(decoded_progress.num_processed_input_records,
+            progress.num_processed_input_records);
+
+  // 2. Test unknown non-ignorable tag
+  encoded.clear();
+  PutVarint32(&encoded,
+              SubcompactionProgressCustomTag::kNextInternalKeyToCompact);
+  PutLengthPrefixedSlice(&encoded, "test_key");
+
+  // Manually encode with unknown non-ignorable tag (do not have
+  // SubcompactionProgressCustomTag::kSubcompactionProgressCustomTagSafeIgnoreMask
+  // bit set)
+  uint32_t unknown_critical_tag =
+      SubcompactionProgressCustomTag::
+          kSubcompactionProgressCustomTagSafeIgnoreMask -
+      1;
+  PutVarint32(&encoded, unknown_critical_tag);
+  PutLengthPrefixedSlice(&encoded, "critical_future_data");
+  PutVarint32(&encoded,
+              SubcompactionProgressCustomTag::kSubcompactionProgressTerminate);
+
+  // Test decoding - should fail on critical unknown tag
+  Slice critical_input(encoded);
+  SubcompactionProgress critical_progress;
+  Status critical_status = critical_progress.DecodeFrom(&critical_input);
+  ASSERT_NOK(critical_status);
+  ASSERT_TRUE(critical_status.IsNotSupported());
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

db/version_set.cc

@@ -1181,15 +1181,25 @@ class LevelIterator final : public InternalIterator {
       // 3. [  S  ] ...... [  E  ]
       for (auto i = fstart; i <= fend; i++) {
         if (i < flevel_->num_files) {
-          auto args = GetMultiScanArgForFile(i);
+          // FindFile only compares against the largest_key, so we need this
+          // additional check to ensure the scan range overlaps the file
+          if (icomparator_.InternalKeyComparator::Compare(
+                  iend.Encode(), flevel_->files[i].smallest_key) < 0) {
+            continue;
+          }
+          auto const metadata = flevel_->files[i].file_metadata;
+          if (metadata->num_entries == metadata->num_range_deletions) {
+            // Skip range deletion only files.
+            continue;
+          }
+          auto& args = GetMultiScanArgForFile(i);
           args.insert(start.value(), end.value(), opt.property_bag);
         }
       }
     }
-    // Propagate io colaescing threshold
+    // Propagate multiscan configs
     for (auto& file_to_arg : *file_to_scan_opts_) {
-      file_to_arg.second.io_coalesce_threshold = so->io_coalesce_threshold;
-      file_to_arg.second.use_async_io = so->use_async_io;
+      file_to_arg.second.CopyConfigFrom(*so);
     }
   }
 
@@ -1265,6 +1275,10 @@ class LevelIterator final : public InternalIterator {
     }
   }
 
+#ifndef NDEBUG
+  bool OverlapRange(const ScanOptions& opts);
+#endif
+
   TableCache* table_cache_;
   const ReadOptions& read_options_;
   const FileOptions& file_options_;
@@ -1362,6 +1376,14 @@ void LevelIterator::Seek(const Slice& target) {
   }
 
   if (file_iter_.iter() != nullptr) {
+    if (scan_opts_) {
+      // At this point, we only know that the seek target is < largest_key
+      // in the file. We need to check whether there is actual overlap.
+      const FdWithKeyRange& cur_file = flevel_->files[file_index_];
+      if (KeyReachedUpperBound(cur_file.smallest_key)) {
+        return;
+      }
+    }
     file_iter_.Seek(target);
     // Status::TryAgain indicates asynchronous request for retrieval of data
     // blocks has been submitted. So it should return at this point and Seek
@@ -1639,6 +1661,19 @@ void LevelIterator::SkipEmptyFileBackward() {
   }
 }
 
+#ifndef NDEBUG
+bool LevelIterator::OverlapRange(const ScanOptions& opts) {
+  return (user_comparator_.CompareWithoutTimestamp(
+              opts.range.start.value(), /*a_has_ts=*/false,
+              ExtractUserKey(flevel_->files[file_index_].largest_key),
+              /*b_has_ts=*/true) <= 0 &&
+          user_comparator_.CompareWithoutTimestamp(
+              opts.range.limit.value(), /*a_has_ts=*/false,
+              ExtractUserKey(flevel_->files[file_index_].smallest_key),
+              /*b_has_ts=*/true) > 0);
+}
+#endif
+
 void LevelIterator::SetFileIterator(InternalIterator* iter) {
   if (pinned_iters_mgr_ && iter) {
     iter->SetPinnedItersMgr(pinned_iters_mgr_);
@@ -1648,9 +1683,9 @@ void LevelIterator::SetFileIterator(InternalIterator* iter) {
   if (iter && scan_opts_) {
     if (FileHasMultiScanArg(file_index_)) {
       const MultiScanArgs& new_opts = GetMultiScanArgForFile(file_index_);
+      assert(OverlapRange(*new_opts.GetScanRanges().begin()) &&
+             OverlapRange(*new_opts.GetScanRanges().rbegin()));
       file_iter_.Prepare(&new_opts);
-    } else {
-      file_iter_.Prepare(scan_opts_);
     }
   }
 
@@ -1989,6 +2024,79 @@ void Version::GetColumnFamilyMetaData(ColumnFamilyMetaData* cf_meta) {
   }
 }
 
+void Version::GetColumnFamilyMetaData(
+    const GetColumnFamilyMetaDataOptions& options,
+    ColumnFamilyMetaData* cf_meta) {
+  assert(cf_meta);
+  assert(cfd_);
+
+  cf_meta->name = cfd_->GetName();
+  cf_meta->size = 0;
+  cf_meta->file_count = 0;
+  cf_meta->levels.clear();
+  cf_meta->blob_file_size = 0;
+  cf_meta->blob_file_count = 0;
+  cf_meta->blob_files.clear();
+
+  const auto& ioptions = cfd_->ioptions();
+  auto* vstorage = storage_info();
+
+  int first_level = (options.level >= 0) ? options.level : 0;
+  int last_level =
+      (options.level >= 0) ? options.level + 1 : cfd_->NumberLevels();
+
+  InternalKey ikey_start, ikey_end;
+  const InternalKey* begin = nullptr;
+  const InternalKey* end = nullptr;
+
+  if (options.range.start.has_value()) {
+    ikey_start = InternalKey(options.range.start.value(), kMaxSequenceNumber,
+                             kValueTypeForSeek);
+    begin = &ikey_start;
+  }
+
+  if (options.range.limit.has_value()) {
+    ikey_end = InternalKey(options.range.limit.value(), kMaxSequenceNumber,
+                           kValueTypeForSeek);
+    end = &ikey_end;
+  }
+
+  for (int l = first_level; l < last_level; ++l) {
+    uint64_t level_size = 0;
+    std::vector<SstFileMetaData> files;
+    std::vector<FileMetaData*> overlapping_files;
+    vstorage->GetOverlappingInputs(l, begin, end, &overlapping_files);
+
+    for (const auto& file : overlapping_files) {
+      uint32_t path_id = file->fd.GetPathId();
+      const auto& file_path = (path_id < ioptions.cf_paths.size())
+                                  ? ioptions.cf_paths[path_id].path
+                                  : ioptions.cf_paths.back().path;
+      const uint64_t file_number = file->fd.GetNumber();
+      files.emplace_back(
+          MakeTableFileName("", file_number), file_number, file_path,
+          file->fd.GetFileSize(), file->fd.smallest_seqno,
+          file->fd.largest_seqno, file->smallest.user_key().ToString(),
+          file->largest.user_key().ToString(),
+          file->stats.num_reads_sampled.load(std::memory_order_relaxed),
+          file->being_compacted, file->temperature,
+          file->oldest_blob_file_number, file->TryGetOldestAncesterTime(),
+          file->TryGetFileCreationTime(), file->epoch_number,
+          file->file_checksum, file->file_checksum_func_name);
+      files.back().num_entries = file->num_entries;
+      files.back().num_deletions = file->num_deletions;
+      files.back().smallest = file->smallest.Encode().ToString();
+      files.back().largest = file->largest.Encode().ToString();
+      level_size += file->fd.GetFileSize();
+      cf_meta->file_count++;
+    }
+    if (!files.empty()) {
+      cf_meta->levels.emplace_back(l, level_size, std::move(files));
+      cf_meta->size += level_size;
+    }
+  }
+}
+
 uint64_t Version::GetSstFilesSize() {
   uint64_t sst_files_size = 0;
   for (int level = 0; level < storage_info_.num_levels_; level++) {
@@ -5272,6 +5380,7 @@ void AtomicGroupReadBuffer::Clear() {
 
 VersionSet::VersionSet(
     const std::string& dbname, const ImmutableDBOptions* _db_options,
+    const MutableDBOptions& mutable_db_options,
     const FileOptions& storage_options, Cache* table_cache,
     WriteBufferManager* write_buffer_manager, WriteController* write_controller,
     BlockCacheTracer* const block_cache_tracer,
@@ -5300,6 +5409,7 @@ VersionSet::VersionSet(
       prev_log_number_(0),
       current_version_number_(0),
       manifest_file_size_(0),
+      last_compacted_manifest_file_size_(0),
       file_options_(storage_options),
       block_cache_tracer_(block_cache_tracer),
       io_tracer_(io_tracer),
@@ -5307,7 +5417,9 @@ VersionSet::VersionSet(
       offpeak_time_option_(OffpeakTimeOption(daily_offpeak_time_utc)),
       error_handler_(error_handler),
       unchanging_(unchanging),
-      closed_(false) {}
+      closed_(false) {
+  UpdatedMutableDbOptions(mutable_db_options, /*mu=*/nullptr);
+}
 
 Status VersionSet::Close(FSDirectory* db_dir, InstrumentedMutex* mu) {
   Status s;
@@ -5403,11 +5515,35 @@ void VersionSet::Reset() {
   current_version_number_ = 0;
   manifest_writers_.clear();
   manifest_file_size_ = 0;
+  last_compacted_manifest_file_size_ = 0;
+  TuneMaxManifestFileSize();
   obsolete_files_.clear();
   obsolete_manifests_.clear();
   wals_.Reset();
 }
 
+void VersionSet::UpdatedMutableDbOptions(
+    const MutableDBOptions& updated_options, InstrumentedMutex* mu) {
+  // Must be holding mutex if not called during initialization
+  if (manifest_file_size_ > 0) {
+    mu->AssertHeld();
+  }
+  file_options_.writable_file_max_buffer_size =
+      updated_options.writable_file_max_buffer_size;
+  min_max_manifest_file_size_ = updated_options.max_manifest_file_size;
+  max_manifest_space_amp_pct_ = static_cast<unsigned>(
+      std::max(updated_options.max_manifest_space_amp_pct, 0));
+  manifest_preallocation_size_ = updated_options.manifest_preallocation_size;
+  TuneMaxManifestFileSize();
+}
+
+void VersionSet::TuneMaxManifestFileSize() {
+  tuned_max_manifest_file_size_ =
+      std::max(min_max_manifest_file_size_,
+               last_compacted_manifest_file_size_ *
+                   (100U + max_manifest_space_amp_pct_) / 100U);
+}
+
 void VersionSet::AppendVersion(ColumnFamilyData* column_family_data,
                                Version* v) {
   // compute new compaction score
@@ -5651,10 +5787,11 @@ Status VersionSet::ProcessManifestWrites(
   }
 #endif  // NDEBUG
 
+  uint64_t prev_manifest_file_size = manifest_file_size_;
   assert(pending_manifest_file_number_ == 0);
   if (!skip_manifest_write &&
       (!descriptor_log_ ||
-       manifest_file_size_ > db_options_->max_manifest_file_size)) {
+       prev_manifest_file_size >= tuned_max_manifest_file_size_)) {
     TEST_SYNC_POINT("VersionSet::ProcessManifestWrites:BeforeNewManifest");
     new_descriptor_log = true;
   } else {
@@ -5694,6 +5831,8 @@ Status VersionSet::ProcessManifestWrites(
   IOStatus manifest_io_status;
   manifest_io_status.PermitUncheckedError();
   std::unique_ptr<log::Writer> new_desc_log_ptr;
+  // Save before releasing mu
+  uint64_t manifest_preallocation_size = manifest_preallocation_size_;
   if (skip_manifest_write) {
     if (s.ok()) {
       constexpr bool update_stats = true;
@@ -5737,16 +5876,13 @@ Status VersionSet::ProcessManifestWrites(
       // This is fine because everything inside of this block is serialized --
       // only one thread can be here at the same time
       // create new manifest file
-      ROCKS_LOG_INFO(db_options_->info_log, "Creating manifest %" PRIu64 "\n",
-                     pending_manifest_file_number_);
       std::string descriptor_fname =
           DescriptorFileName(dbname_, pending_manifest_file_number_);
       std::unique_ptr<FSWritableFile> descriptor_file;
       io_s = NewWritableFile(fs_.get(), descriptor_fname, &descriptor_file,
                              opt_file_opts);
       if (io_s.ok()) {
-        descriptor_file->SetPreallocationBlockSize(
-            db_options_->manifest_preallocation_size);
+        descriptor_file->SetPreallocationBlockSize(manifest_preallocation_size);
         FileTypeSet tmp_set = db_options_->checksum_handoff_file_types;
         std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
             std::move(descriptor_file), descriptor_fname, opt_file_opts, clock_,
@@ -5847,6 +5983,13 @@ Status VersionSet::ProcessManifestWrites(
     if (s.ok()) {
       // find offset in manifest file where this version is stored.
       new_manifest_file_size = raw_desc_log_ptr->file()->GetFileSize();
+      if (new_descriptor_log) {
+        ROCKS_LOG_INFO(db_options_->info_log,
+                       "Created manifest %" PRIu64
+                       ", compacted+appended from %" PRIu64 " to %" PRIu64 "\n",
+                       pending_manifest_file_number_, prev_manifest_file_size,
+                       new_manifest_file_size);
+      }
     }
 
     if (first_writer.edit_list.front()->IsColumnFamilyDrop()) {
@@ -5895,6 +6038,8 @@ Status VersionSet::ProcessManifestWrites(
     descriptor_log_ = std::move(new_desc_log_ptr);
     obsolete_manifests_.emplace_back(
         DescriptorFileName("", manifest_file_number_));
+    last_compacted_manifest_file_size_ = new_manifest_file_size;
+    TuneMaxManifestFileSize();
   }
 
   // Install the new versions
@@ -6528,14 +6673,16 @@ Status VersionSet::ReduceNumberOfLevels(const std::string& dbname,
   const ReadOptions read_options;
   const WriteOptions write_options;
 
-  ImmutableDBOptions db_options(*options);
+  ImmutableDBOptions imm_db_options(*options);
+  MutableDBOptions mutable_db_options(*options);
   ColumnFamilyOptions cf_options(*options);
   std::shared_ptr<Cache> tc(NewLRUCache(options->max_open_files - 10,
                                         options->table_cache_numshardbits));
   WriteController wc(options->delayed_write_rate);
   WriteBufferManager wb(options->db_write_buffer_size);
-  VersionSet versions(dbname, &db_options, file_options, tc.get(), &wb, &wc,
-                      nullptr /*BlockCacheTracer*/, nullptr /*IOTracer*/,
+  VersionSet versions(dbname, &imm_db_options, mutable_db_options, file_options,
+                      tc.get(), &wb, &wc, nullptr /*BlockCacheTracer*/,
+                      nullptr /*IOTracer*/,
                       /*db_id*/ "",
                       /*db_session_id*/ "", options->daily_offpeak_time_utc,
                       /*error_handler_*/ nullptr, /*unchanging=*/false);
@@ -7587,12 +7734,13 @@ Status VersionSet::VerifyFileMetadata(const ReadOptions& read_options,
 }
 
 ReactiveVersionSet::ReactiveVersionSet(
-    const std::string& dbname, const ImmutableDBOptions* _db_options,
+    const std::string& dbname, const ImmutableDBOptions* imm_db_options,
+    const MutableDBOptions& mutable_db_options,
     const FileOptions& _file_options, Cache* table_cache,
     WriteBufferManager* write_buffer_manager, WriteController* write_controller,
     const std::shared_ptr<IOTracer>& io_tracer)
-    : VersionSet(dbname, _db_options, _file_options, table_cache,
-                 write_buffer_manager, write_controller,
+    : VersionSet(dbname, imm_db_options, mutable_db_options, _file_options,
+                 table_cache, write_buffer_manager, write_controller,
                  /*block_cache_tracer=*/nullptr, io_tracer, /*db_id*/ "",
                  /*db_session_id*/ "", /*daily_offpeak_time_utc*/ "",
                  /*error_handler=*/nullptr, /*unchanging=*/false) {}

db/version_set.h

@@ -1044,6 +1044,10 @@ class Version {
 
   void GetColumnFamilyMetaData(ColumnFamilyMetaData* cf_meta);
 
+  // Get column family metadata with optional filtering by key range and level.
+  void GetColumnFamilyMetaData(const GetColumnFamilyMetaDataOptions& options,
+                               ColumnFamilyMetaData* cf_meta);
+
   void GetSstFilesBoundaryKeys(Slice* smallest_user_key,
                                Slice* largest_user_key);
 
@@ -1186,7 +1190,9 @@ class AtomicGroupReadBuffer {
 // but false for secondary instance or writable DB).
 class VersionSet {
  public:
-  VersionSet(const std::string& dbname, const ImmutableDBOptions* db_options,
+  VersionSet(const std::string& dbname,
+             const ImmutableDBOptions* imm_db_options,
+             const MutableDBOptions& mutable_db_options,
              const FileOptions& file_options, Cache* table_cache,
              WriteBufferManager* write_buffer_manager,
              WriteController* write_controller,
@@ -1203,6 +1209,13 @@ class VersionSet {
 
   virtual Status Close(FSDirectory* db_dir, InstrumentedMutex* mu);
 
+  // Requires: already holding DB mutex `mu`, to ensure
+  // * Safely read values from `updated_options`
+  // * Safely update fields on `this` (must be read elsewhere while holding mu)
+  // except `mu` can be nullptr during initialization
+  void UpdatedMutableDbOptions(const MutableDBOptions& updated_options,
+                               InstrumentedMutex* mu);
+
   Status LogAndApplyToDefaultColumnFamily(
       const ReadOptions& read_options, const WriteOptions& write_options,
       VersionEdit* edit, InstrumentedMutex* mu,
@@ -1548,10 +1561,6 @@ class VersionSet {
   }
 
   const FileOptions& file_options() { return file_options_; }
-  void ChangeFileOptions(const MutableDBOptions& new_options) {
-    file_options_.writable_file_max_buffer_size =
-        new_options.writable_file_max_buffer_size;
-  }
 
   // TODO - Consider updating together when file options change in SetDBOptions
   const OffpeakTimeOption& offpeak_time_option() {
@@ -1590,6 +1599,16 @@ class VersionSet {
 
   bool& TEST_unchanging() { return const_cast<bool&>(unchanging_); }
 
+  uint64_t TEST_GetMinMaxManifestFileSize() {
+    return min_max_manifest_file_size_;
+  }
+  unsigned TEST_GetMaxManifestSpaceAmpPct() {
+    return max_manifest_space_amp_pct_;
+  }
+  size_t TEST_GetManifestPreallocationSize() {
+    return manifest_preallocation_size_;
+  }
+
  protected:
   struct ManifestWriter;
 
@@ -1610,6 +1629,7 @@ class VersionSet {
     }
   };
 
+  // Revert back to a post-construction state (keep same options/settings)
   void Reset();
 
   // Returns approximated offset of a key in a file for a given version.
@@ -1648,6 +1668,11 @@ class VersionSet {
                             ColumnFamilyData* cfd, const std::string& fpath,
                             int level, const FileMetaData& meta);
 
+  // Auto-tune next max size for the current manifest file based on its initial
+  // "compacted" size and other parameters saved in this VersionSet. Must be
+  // holding DB mutex if outside of DB startup.
+  void TuneMaxManifestFileSize();
+
   // Protected by DB mutex.
   WalSet wals_;
 
@@ -1699,6 +1724,20 @@ class VersionSet {
   // Current size of manifest file
   uint64_t manifest_file_size_;
 
+  // Size of the populated manifest file last time it was re-written from
+  // scratch.
+  uint64_t last_compacted_manifest_file_size_;
+
+  // Auto-tuned max allowed size for the current manifest file
+  uint64_t tuned_max_manifest_file_size_;
+
+  // Saved copy of max_manifest_file_size in (Mutable)DBOptions
+  uint64_t min_max_manifest_file_size_;
+  // Saved, sanitized copy from (Mutable)DBOptions
+  unsigned max_manifest_space_amp_pct_;
+  // Saved copy from (Mutable)DBOptions
+  size_t manifest_preallocation_size_;
+
   // Obsolete files, or during DB shutdown any files not referenced by what's
   // left of the in-memory LSM state.
   std::vector<ObsoleteFileInfo> obsolete_files_;
@@ -1751,6 +1790,7 @@ class ReactiveVersionSet : public VersionSet {
  public:
   ReactiveVersionSet(const std::string& dbname,
                      const ImmutableDBOptions* _db_options,
+                     const MutableDBOptions& mutable_db_options,
                      const FileOptions& _file_options, Cache* table_cache,
                      WriteBufferManager* write_buffer_manager,
                      WriteController* write_controller,

db/version_set_test.cc

@@ -1159,12 +1159,12 @@ class VersionSetTestBase {
       : env_(nullptr),
         dbname_(test::PerThreadDBPath(name)),
         options_(),
-        db_options_(options_),
+        imm_db_options_(options_),
         cf_options_(options_),
-        immutable_options_(db_options_, cf_options_),
+        immutable_options_(imm_db_options_, cf_options_),
         mutable_cf_options_(cf_options_),
         table_cache_(NewLRUCache(50000, 16)),
-        write_buffer_manager_(db_options_.db_write_buffer_size),
+        write_buffer_manager_(imm_db_options_.db_write_buffer_size),
         shutting_down_(false),
         table_factory_(std::make_shared<mock::MockTableFactory>()) {
     EXPECT_OK(test::CreateEnvFromSystem(ConfigOptions(), &env_, &env_guard_));
@@ -1178,8 +1178,8 @@ class VersionSetTestBase {
     EXPECT_OK(fs_->CreateDirIfMissing(dbname_, IOOptions(), nullptr));
 
     options_.env = env_;
-    db_options_.env = env_;
-    db_options_.fs = fs_;
+    imm_db_options_.env = env_;
+    imm_db_options_.fs = fs_;
     immutable_options_.env = env_;
     immutable_options_.fs = fs_;
     immutable_options_.clock = env_->GetSystemClock().get();
@@ -1188,16 +1188,17 @@ class VersionSetTestBase {
     mutable_cf_options_.table_factory = table_factory_;
 
     versions_.reset(new VersionSet(
-        dbname_, &db_options_, env_options_, table_cache_.get(),
-        &write_buffer_manager_, &write_controller_,
+        dbname_, &imm_db_options_, mutable_db_options_, env_options_,
+        table_cache_.get(), &write_buffer_manager_, &write_controller_,
         /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr,
         /*db_id=*/"", /*db_session_id=*/"", /*daily_offpeak_time_utc=*/"",
         /*error_handler=*/nullptr, /*read_only=*/false));
     reactive_versions_ = std::make_shared<ReactiveVersionSet>(
-        dbname_, &db_options_, env_options_, table_cache_.get(),
-        &write_buffer_manager_, &write_controller_, nullptr);
-    db_options_.db_paths.emplace_back(dbname_,
-                                      std::numeric_limits<uint64_t>::max());
+        dbname_, &imm_db_options_, mutable_db_options_, env_options_,
+        table_cache_.get(), &write_buffer_manager_, &write_controller_,
+        nullptr);
+    imm_db_options_.db_paths.emplace_back(dbname_,
+                                          std::numeric_limits<uint64_t>::max());
   }
 
   virtual ~VersionSetTestBase() {
@@ -1220,7 +1221,7 @@ class VersionSetTestBase {
     ASSERT_OK(
         SetIdentityFile(WriteOptions(), env_, dbname_, Temperature::kUnknown));
     VersionEdit new_db;
-    if (db_options_.write_dbid_to_manifest) {
+    if (imm_db_options_.write_dbid_to_manifest) {
       DBOptions tmp_db_options;
       tmp_db_options.env = env_;
       std::unique_ptr<DBImpl> impl(new DBImpl(tmp_db_options, dbname_));
@@ -1381,8 +1382,8 @@ class VersionSetTestBase {
 
   void ReopenDB() {
     versions_.reset(new VersionSet(
-        dbname_, &db_options_, env_options_, table_cache_.get(),
-        &write_buffer_manager_, &write_controller_,
+        dbname_, &imm_db_options_, mutable_db_options_, env_options_,
+        table_cache_.get(), &write_buffer_manager_, &write_controller_,
         /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr,
         /*db_id=*/"", /*db_session_id=*/"", /*daily_offpeak_time_utc=*/"",
         /*error_handler=*/nullptr, /*read_only=*/false));
@@ -1471,7 +1472,8 @@ class VersionSetTestBase {
   const std::string dbname_;
   EnvOptions env_options_;
   Options options_;
-  ImmutableDBOptions db_options_;
+  ImmutableDBOptions imm_db_options_;
+  MutableDBOptions mutable_db_options_;
   ColumnFamilyOptions cf_options_;
   ImmutableOptions immutable_options_;
   MutableCFOptions mutable_cf_options_;
@@ -1902,8 +1904,8 @@ TEST_F(VersionSetTest, WalAddition) {
   // Recover a new VersionSet.
   {
     std::unique_ptr<VersionSet> new_versions(new VersionSet(
-        dbname_, &db_options_, env_options_, table_cache_.get(),
-        &write_buffer_manager_, &write_controller_,
+        dbname_, &imm_db_options_, mutable_db_options_, env_options_,
+        table_cache_.get(), &write_buffer_manager_, &write_controller_,
         /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr,
         /*db_id=*/"", /*db_session_id=*/"", /*daily_offpeak_time_utc=*/"",
         /*error_handler=*/nullptr, /*unchanging=*/false));
@@ -1970,8 +1972,8 @@ TEST_F(VersionSetTest, WalCloseWithoutSync) {
   // Recover a new VersionSet.
   {
     std::unique_ptr<VersionSet> new_versions(new VersionSet(
-        dbname_, &db_options_, env_options_, table_cache_.get(),
-        &write_buffer_manager_, &write_controller_,
+        dbname_, &imm_db_options_, mutable_db_options_, env_options_,
+        table_cache_.get(), &write_buffer_manager_, &write_controller_,
         /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr,
         /*db_id=*/"", /*db_session_id=*/"", /*daily_offpeak_time_utc=*/"",
         /*error_handler=*/nullptr, /*unchanging=*/false));
@@ -2024,8 +2026,8 @@ TEST_F(VersionSetTest, WalDeletion) {
   // Recover a new VersionSet, only the non-closed WAL should show up.
   {
     std::unique_ptr<VersionSet> new_versions(new VersionSet(
-        dbname_, &db_options_, env_options_, table_cache_.get(),
-        &write_buffer_manager_, &write_controller_,
+        dbname_, &imm_db_options_, mutable_db_options_, env_options_,
+        table_cache_.get(), &write_buffer_manager_, &write_controller_,
         /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr,
         /*db_id=*/"", /*db_session_id=*/"", /*daily_offpeak_time_utc=*/"",
         /*error_handler=*/nullptr, /*unchanging=*/false));
@@ -2063,8 +2065,8 @@ TEST_F(VersionSetTest, WalDeletion) {
   // Recover from the new MANIFEST, only the non-closed WAL should show up.
   {
     std::unique_ptr<VersionSet> new_versions(new VersionSet(
-        dbname_, &db_options_, env_options_, table_cache_.get(),
-        &write_buffer_manager_, &write_controller_,
+        dbname_, &imm_db_options_, mutable_db_options_, env_options_,
+        table_cache_.get(), &write_buffer_manager_, &write_controller_,
         /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr,
         /*db_id=*/"", /*db_session_id=*/"", /*daily_offpeak_time_utc=*/"",
         /*error_handler=*/nullptr, /*unchanging=*/false));
@@ -2184,8 +2186,8 @@ TEST_F(VersionSetTest, DeleteWalsBeforeNonExistingWalNumber) {
   // Recover a new VersionSet, WAL0 is deleted, WAL1 is not.
   {
     std::unique_ptr<VersionSet> new_versions(new VersionSet(
-        dbname_, &db_options_, env_options_, table_cache_.get(),
-        &write_buffer_manager_, &write_controller_,
+        dbname_, &imm_db_options_, mutable_db_options_, env_options_,
+        table_cache_.get(), &write_buffer_manager_, &write_controller_,
         /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr,
         /*db_id=*/"", /*db_session_id=*/"", /*daily_offpeak_time_utc=*/"",
         /*error_handler=*/nullptr, /*unchanging=*/false));
@@ -2221,8 +2223,8 @@ TEST_F(VersionSetTest, DeleteAllWals) {
   // Recover a new VersionSet, all WALs are deleted.
   {
     std::unique_ptr<VersionSet> new_versions(new VersionSet(
-        dbname_, &db_options_, env_options_, table_cache_.get(),
-        &write_buffer_manager_, &write_controller_,
+        dbname_, &imm_db_options_, mutable_db_options_, env_options_,
+        table_cache_.get(), &write_buffer_manager_, &write_controller_,
         /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr,
         /*db_id=*/"", /*db_session_id=*/"", /*daily_offpeak_time_utc=*/"",
         /*error_handler=*/nullptr, /*unchanging=*/false));
@@ -2264,8 +2266,8 @@ TEST_F(VersionSetTest, AtomicGroupWithWalEdits) {
   // kept.
   {
     std::unique_ptr<VersionSet> new_versions(new VersionSet(
-        dbname_, &db_options_, env_options_, table_cache_.get(),
-        &write_buffer_manager_, &write_controller_,
+        dbname_, &imm_db_options_, mutable_db_options_, env_options_,
+        table_cache_.get(), &write_buffer_manager_, &write_controller_,
         /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr,
         /*db_id=*/"", /*db_session_id=*/"", /*daily_offpeak_time_utc=*/"",
         /*error_handler=*/nullptr, /*unchanging=*/false));
@@ -2444,8 +2446,8 @@ class VersionSetWithTimestampTest : public VersionSetTest {
 
   void VerifyFullHistoryTsLow(uint64_t expected_ts_low) {
     std::unique_ptr<VersionSet> vset(new VersionSet(
-        dbname_, &db_options_, env_options_, table_cache_.get(),
-        &write_buffer_manager_, &write_controller_,
+        dbname_, &imm_db_options_, mutable_db_options_, env_options_,
+        table_cache_.get(), &write_buffer_manager_, &write_controller_,
         /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr,
         /*db_id=*/"", /*db_session_id=*/"", /*daily_offpeak_time_utc=*/"",
         /*error_handler=*/nullptr, /*unchanging=*/false));
@@ -3500,7 +3502,7 @@ class VersionSetTestEmptyDb
                        std::unique_ptr<log::Writer>* log_writer) override {
     assert(nullptr != log_writer);
     VersionEdit new_db;
-    if (db_options_.write_dbid_to_manifest) {
+    if (imm_db_options_.write_dbid_to_manifest) {
       ASSERT_OK(SetIdentityFile(WriteOptions(), env_, dbname_,
                                 Temperature::kUnknown));
       DBOptions tmp_db_options;
@@ -3532,7 +3534,7 @@ class VersionSetTestEmptyDb
 const std::string VersionSetTestEmptyDb::kUnknownColumnFamilyName = "unknown";
 
 TEST_P(VersionSetTestEmptyDb, OpenFromIncompleteManifest0) {
-  db_options_.write_dbid_to_manifest = std::get<0>(GetParam());
+  imm_db_options_.write_dbid_to_manifest = std::get<0>(GetParam());
   PrepareManifest(nullptr, nullptr, &log_writer_);
   log_writer_.reset();
   CreateCurrentFile();
@@ -3564,7 +3566,7 @@ TEST_P(VersionSetTestEmptyDb, OpenFromIncompleteManifest0) {
 }
 
 TEST_P(VersionSetTestEmptyDb, OpenFromIncompleteManifest1) {
-  db_options_.write_dbid_to_manifest = std::get<0>(GetParam());
+  imm_db_options_.write_dbid_to_manifest = std::get<0>(GetParam());
   PrepareManifest(nullptr, nullptr, &log_writer_);
   // Only a subset of column families in the MANIFEST.
   VersionEdit new_cf1;
@@ -3605,7 +3607,7 @@ TEST_P(VersionSetTestEmptyDb, OpenFromIncompleteManifest1) {
 }
 
 TEST_P(VersionSetTestEmptyDb, OpenFromInCompleteManifest2) {
-  db_options_.write_dbid_to_manifest = std::get<0>(GetParam());
+  imm_db_options_.write_dbid_to_manifest = std::get<0>(GetParam());
   PrepareManifest(nullptr, nullptr, &log_writer_);
   // Write all column families but no log_number, next_file_number and
   // last_sequence.
@@ -3651,7 +3653,7 @@ TEST_P(VersionSetTestEmptyDb, OpenFromInCompleteManifest2) {
 }
 
 TEST_P(VersionSetTestEmptyDb, OpenManifestWithUnknownCF) {
-  db_options_.write_dbid_to_manifest = std::get<0>(GetParam());
+  imm_db_options_.write_dbid_to_manifest = std::get<0>(GetParam());
   PrepareManifest(nullptr, nullptr, &log_writer_);
   // Write all column families but no log_number, next_file_number and
   // last_sequence.
@@ -3708,7 +3710,7 @@ TEST_P(VersionSetTestEmptyDb, OpenManifestWithUnknownCF) {
 }
 
 TEST_P(VersionSetTestEmptyDb, OpenCompleteManifest) {
-  db_options_.write_dbid_to_manifest = std::get<0>(GetParam());
+  imm_db_options_.write_dbid_to_manifest = std::get<0>(GetParam());
   PrepareManifest(nullptr, nullptr, &log_writer_);
   // Write all column families but no log_number, next_file_number and
   // last_sequence.
@@ -3828,7 +3830,7 @@ class VersionSetTestMissingFiles : public VersionSetTestBase,
     ASSERT_OK(s);
     log_writer->reset(new log::Writer(std::move(file_writer), 0, false));
     VersionEdit new_db;
-    if (db_options_.write_dbid_to_manifest) {
+    if (imm_db_options_.write_dbid_to_manifest) {
       DBOptions tmp_db_options;
       tmp_db_options.env = env_;
       std::unique_ptr<DBImpl> impl(new DBImpl(tmp_db_options, dbname_));
@@ -4088,7 +4090,7 @@ TEST_F(VersionSetTestMissingFiles, NoFileMissing) {
 }
 
 TEST_F(VersionSetTestMissingFiles, MinLogNumberToKeep2PC) {
-  db_options_.allow_2pc = true;
+  imm_db_options_.allow_2pc = true;
   NewDB();
 
   SstInfo sst(100, kDefaultColumnFamilyName, "a", 0 /* level */,

db/version_util.h

@@ -1,4 +1,4 @@
-//  Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
 //  This source code is licensed under both the GPLv2 (found in the
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
@@ -23,7 +23,8 @@ class OfflineManifestWriter {
         immutable_db_options_(WithDbPath(options, db_path)),
         tc_(NewLRUCache(1 << 20 /* capacity */,
                         options.table_cache_numshardbits)),
-        versions_(db_path, &immutable_db_options_, sopt_, tc_.get(), &wb_, &wc_,
+        versions_(db_path, &immutable_db_options_, MutableDBOptions{options},
+                  sopt_, tc_.get(), &wb_, &wc_,
                   /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr,
                   /*db_id=*/"", /*db_session_id=*/"",
                   options.daily_offpeak_time_utc,

db/wal_manager_test.cc

@@ -50,8 +50,8 @@ class WalManagerTest : public testing::Test {
     db_options_.clock = env_->GetSystemClock().get();
 
     versions_.reset(new VersionSet(
-        dbname_, &db_options_, env_options_, table_cache_.get(),
-        &write_buffer_manager_, &write_controller_,
+        dbname_, &db_options_, MutableDBOptions{}, env_options_,
+        table_cache_.get(), &write_buffer_manager_, &write_controller_,
         /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr,
         /*db_id=*/"", /*db_session_id=*/"", /*daily_offpeak_time_utc=*/"",
         /*error_handler=*/nullptr, /*read_only=*/false));

db/write_thread.h

@@ -179,7 +179,7 @@ class WriteThread {
            PostMemTableCallback* _post_memtable_callback = nullptr,
            bool _ingest_wbwi = false)
         : batch(_batch),
-          // TODO: store a copy of WriteOptions instead of its seperated data
+          // TODO: store a copy of WriteOptions instead of its separated data
           // members
           sync(write_options.sync),
           no_slowdown(write_options.no_slowdown),

db_stress_tool/CMakeLists.txt

@@ -2,6 +2,7 @@ add_executable(db_stress${ARTIFACT_SUFFIX}
   batched_ops_stress.cc
   cf_consistency_stress.cc
   db_stress.cc
+  db_stress_compaction_service.cc
   db_stress_compression_manager.cc
   db_stress_common.cc
   db_stress_driver.cc

db_stress_tool/db_stress_common.cc

@@ -13,6 +13,7 @@
 
 #include <cmath>
 
+#include "file/file_util.h"
 #include "rocksdb/secondary_cache.h"
 #include "util/file_checksum_helper.h"
 #include "util/xxhash.h"
@@ -228,14 +229,250 @@ void CompressedCacheSetCapacityThread(void* v) {
   }
 }
 
+#ifndef NDEBUG
+static void SetupFaultInjectionForRemoteCompaction(SharedState* shared) {
+  if (!fault_fs_guard) {
+    return;
+  }
+
+  fault_fs_guard->SetThreadLocalErrorContext(
+      FaultInjectionIOType::kRead, shared->GetSeed(), FLAGS_read_fault_one_in,
+      FLAGS_inject_error_severity == 1 /* retryable */,
+      FLAGS_inject_error_severity == 2 /* has_data_loss*/);
+  fault_fs_guard->EnableThreadLocalErrorInjection(FaultInjectionIOType::kRead);
+
+  fault_fs_guard->SetThreadLocalErrorContext(
+      FaultInjectionIOType::kWrite, shared->GetSeed(), FLAGS_write_fault_one_in,
+      FLAGS_inject_error_severity == 1 /* retryable */,
+      FLAGS_inject_error_severity == 2 /* has_data_loss*/);
+  fault_fs_guard->EnableThreadLocalErrorInjection(FaultInjectionIOType::kWrite);
+
+  fault_fs_guard->SetThreadLocalErrorContext(
+      FaultInjectionIOType::kMetadataRead, shared->GetSeed(),
+      FLAGS_metadata_read_fault_one_in,
+      FLAGS_inject_error_severity == 1 /* retryable */,
+      FLAGS_inject_error_severity == 2 /* has_data_loss*/);
+  fault_fs_guard->EnableThreadLocalErrorInjection(
+      FaultInjectionIOType::kMetadataRead);
+
+  fault_fs_guard->SetThreadLocalErrorContext(
+      FaultInjectionIOType::kMetadataWrite, shared->GetSeed(),
+      FLAGS_metadata_write_fault_one_in,
+      FLAGS_inject_error_severity == 1 /* retryable */,
+      FLAGS_inject_error_severity == 2 /* has_data_loss*/);
+  fault_fs_guard->EnableThreadLocalErrorInjection(
+      FaultInjectionIOType::kMetadataWrite);
+}
+#endif  // NDEBUG
+
+static CompactionServiceOptionsOverride CreateOverrideOptions(
+    const Options& options, const CompactionServiceJobInfo& job_info) {
+  CompactionServiceOptionsOverride override_options{
+      .env = db_stress_env,
+      .file_checksum_gen_factory = options.file_checksum_gen_factory,
+      .merge_operator = options.merge_operator,
+      .compaction_filter = options.compaction_filter,
+      .compaction_filter_factory = options.compaction_filter_factory,
+      .prefix_extractor = options.prefix_extractor,
+      .sst_partitioner_factory = options.sst_partitioner_factory,
+      .listeners = options.listeners,
+      .statistics = options.statistics,
+      .table_properties_collector_factories =
+          options.table_properties_collector_factories};
+
+  // TODO(jaykorean) - create a new compaction filter / merge operator and
+  // others for remote compactions
+  //
+  // Create a new Table Factory
+  ConfigOptions config_options;
+  config_options.ignore_unknown_options = false;
+  config_options.ignore_unsupported_options = false;
+
+  Status s = TableFactory::CreateFromString(config_options,
+                                            options.table_factory->Name(),
+                                            &override_options.table_factory);
+
+  if (s.ok()) {
+    std::string options_str;
+    s = options.table_factory->GetOptionString(config_options, &options_str);
+    if (s.ok()) {
+      s = override_options.table_factory->ConfigureFromString(config_options,
+                                                              options_str);
+    }
+  }
+
+  if (!s.ok()) {
+    fprintf(stdout,
+            "Failed to set up TableFactory for remote compaction - (%s): %s\n",
+            job_info.db_name.c_str(), s.ToString().c_str());
+  }
+
+  return override_options;
+}
+
+static Status CleanupOutputDirectory(const std::string& output_directory) {
+#ifndef NDEBUG
+  // Temporarily disable fault injection to ensure deletion always succeeds
+  if (fault_fs_guard) {
+    fault_fs_guard->DisableAllThreadLocalErrorInjection();
+  }
+#endif  // NDEBUG
+
+  Status s = DestroyDir(db_stress_env, output_directory);
+  if (!s.ok()) {
+    fprintf(stderr,
+            "Failed to destroy output directory %s when allow_resumption is "
+            "false: %s\n",
+            output_directory.c_str(), s.ToString().c_str());
+  }
+
+  if (s.ok()) {
+    s = db_stress_env->CreateDir(output_directory);
+    if (!s.ok()) {
+      fprintf(stderr,
+              "Failed to recreate output directory %s when allow_resumption is "
+              "false: %s\n",
+              output_directory.c_str(), s.ToString().c_str());
+    }
+  }
+
+#ifndef NDEBUG
+  // Re-enable fault injection after deletion
+  if (fault_fs_guard) {
+    fault_fs_guard->EnableAllThreadLocalErrorInjection();
+  }
+#endif  // NDEBUG
+
+  return s;
+}
+
+// Set up cancellation mechanism for testing resumable remote compactions.
+// Spawns a detached thread to trigger cancellation after a delay (50ms
+// initially, or 2/3 of the previous successful compaction time for adaptive
+// timing). First-time jobs are always canceled; retries have a 10% chance
+// to test consecutive cancellation scenarios.
+static std::shared_ptr<std::atomic<bool>> SetupCancellation(
+    OpenAndCompactOptions& open_compact_options, bool was_canceled,
+    Random& rand, uint64_t successful_compaction_end_to_end_micros) {
+  auto canceled = std::make_shared<std::atomic<bool>>(false);
+  open_compact_options.canceled = canceled.get();
+
+  bool should_cancel = !was_canceled || rand.OneIn(10);
+
+  if (should_cancel) {
+    std::thread interruption_thread(
+        [canceled, successful_compaction_end_to_end_micros]() {
+          uint64_t sleep_micros =
+              successful_compaction_end_to_end_micros == 0
+                  ? 50000
+                  : successful_compaction_end_to_end_micros * 2 / 3;
+          std::this_thread::sleep_for(std::chrono::microseconds(sleep_micros));
+          canceled->store(true);
+        });
+    interruption_thread.detach();
+  }
+
+  return canceled;
+}
+
+// Process the result of OpenAndCompact operation
+static void ProcessCompactionResult(
+    const Status& s, const std::string& job_id,
+    const CompactionServiceJobInfo& job_info,
+    const std::string& serialized_input, const std::string& output_directory,
+    const std::string& serialized_output, SharedState* shared,
+    uint64_t& successful_compaction_end_to_end_micros, uint64_t start_micros,
+    Env* env) {
+  if (s.IsManualCompactionPaused() && FLAGS_allow_resumption_one_in > 0) {
+    // Re-enqueue for retry
+    shared->EnqueueRemoteCompaction(job_id, job_info, serialized_input,
+                                    output_directory, true /* was_cancelled */);
+    return;
+  }
+
+  if (!s.ok()) {
+    if (!StressTest::IsErrorInjectedAndRetryable(s)) {
+      // Print in stdout instead of stderr to avoid stress test failure,
+      // because OpenAndCompact() failure doesn't necessarily mean
+      // primary db instance failure.
+      fprintf(stdout, "Failed to run OpenAndCompact(%s): %s\n",
+              job_info.db_name.c_str(), s.ToString().c_str());
+    }
+  } else {
+    // Track successful completion time
+    successful_compaction_end_to_end_micros = env->NowMicros() - start_micros;
+  }
+
+  // Add the output regardless of status, so that primary DB doesn't rely
+  // on the timeout to finish waiting. The actual failure from the
+  // deserialization can fail the compaction properly
+  shared->AddRemoteCompactionResult(job_id, s, serialized_output);
+}
+
+static void ProcessRemoteCompactionJob(
+    const std::string& job_id, const CompactionServiceJobInfo& job_info,
+    const std::string& serialized_input, const std::string& output_directory,
+    bool was_canceled, SharedState* shared, StressTest* stress_test,
+    Random& rand, uint64_t& successful_compaction_end_to_end_micros) {
+  auto options = stress_test->GetOptions(job_info.cf_id);
+  assert(options.env != nullptr);
+
+  auto override_options = CreateOverrideOptions(options, job_info);
+
+  OpenAndCompactOptions open_compact_options;
+  if (FLAGS_allow_resumption_one_in > 0) {
+    open_compact_options.allow_resumption =
+        rand.OneIn(FLAGS_allow_resumption_one_in);
+  } else {
+    open_compact_options.allow_resumption = false;
+  }
+
+  if (!open_compact_options.allow_resumption) {
+    CleanupOutputDirectory(output_directory);
+  }
+
+  std::shared_ptr<std::atomic<bool>> canceled = nullptr;
+  if (FLAGS_allow_resumption_one_in > 0) {
+    canceled = SetupCancellation(open_compact_options, was_canceled, rand,
+                                 successful_compaction_end_to_end_micros);
+  }
+
+  std::string serialized_output;
+  uint64_t start_micros = options.env->NowMicros();
+
+  Status s = DB::OpenAndCompact(open_compact_options, job_info.db_name,
+                                output_directory, serialized_input,
+                                &serialized_output, override_options);
+
+  ProcessCompactionResult(s, job_id, job_info, serialized_input,
+                          output_directory, serialized_output, shared,
+                          successful_compaction_end_to_end_micros, start_micros,
+                          options.env);
+}
+
 void RemoteCompactionWorkerThread(void* v) {
   assert(FLAGS_remote_compaction_worker_threads > 0);
   assert(FLAGS_remote_compaction_worker_interval > 0);
+
   auto* thread = static_cast<ThreadState*>(v);
   SharedState* shared = thread->shared;
   StressTest* stress_test = shared->GetStressTest();
   assert(stress_test != nullptr);
+
+#ifndef NDEBUG
+  SetupFaultInjectionForRemoteCompaction(shared);
+#endif  // NDEBUG
+
+  // Tracks the duration (in microseconds) of the most recent successfully
+  // completed compaction from start to finish. This value is used in
+  // SetupCancellation() to adaptively set up cancellation point for a
+  // compaction
+  uint64_t successful_compaction_end_to_end_micros = 0;
+  Random rand(static_cast<uint32_t>(FLAGS_seed));
+
+  // Main worker loop
   while (true) {
+    // Check if we should stop
     {
       MutexLock l(shared->GetMutex());
       if (shared->ShouldStopBgThread()) {
@@ -246,42 +483,20 @@ void RemoteCompactionWorkerThread(void* v) {
         return;
       }
     }
+
     std::string job_id;
     CompactionServiceJobInfo job_info;
     std::string serialized_input;
-    if (shared->DequeueRemoteCompaction(&job_id, &job_info,
-                                        &serialized_input)) {
-      auto options = stress_test->GetOptions(job_info.cf_id);
-      CompactionServiceOptionsOverride override_options{
-          .file_checksum_gen_factory = options.file_checksum_gen_factory,
-          .merge_operator = options.merge_operator,
-          .compaction_filter = options.compaction_filter,
-          .compaction_filter_factory = options.compaction_filter_factory,
-          .prefix_extractor = options.prefix_extractor,
-          .table_factory = options.table_factory,
-          .sst_partitioner_factory = options.sst_partitioner_factory,
-          .listeners = {},
-          .statistics = options.statistics,
-          .table_properties_collector_factories =
-              options.table_properties_collector_factories};
-      std::string tmp_output_dir = job_info.db_name + "/" + "tmp_output_" +
-                                   db_stress_env->GenerateUniqueId();
-      std::string serialized_output;
-      Status s = DB::OpenAndCompact(OpenAndCompactOptions{}, job_info.db_name,
-                                    tmp_output_dir, serialized_input,
-                                    &serialized_output, override_options);
-      if (!s.ok()) {
-        // Print in stdout instead of stderr to avoid stress test failure,
-        // because OpenAndCompact() failure doesn't necessarily mean
-        // primary db instance failure.
-        fprintf(stdout, "Failed to run OpenAndCompact(%s): %s\n",
-                job_info.db_name.c_str(), s.ToString().c_str());
-      }
-      // Add the output regardless of status, so that primary DB doesn't rely on
-      // the timeout to finish waiting. The actual failure from the
-      // deserialization can fail the compaction properly
-      shared->AddRemoteCompactionResult(job_id, serialized_output);
+    std::string output_directory;
+    bool was_canceled;
+
+    if (shared->DequeueRemoteCompaction(&job_id, &job_info, &serialized_input,
+                                        &output_directory, &was_canceled)) {
+      ProcessRemoteCompactionJob(
+          job_id, job_info, serialized_input, output_directory, was_canceled,
+          shared, stress_test, rand, successful_compaction_end_to_end_micros);
     }
+
     db_stress_env->SleepForMicroseconds(
         thread->rand.Next() % FLAGS_remote_compaction_worker_interval * 1000 +
         1);

db_stress_tool/db_stress_common.h

@@ -248,6 +248,7 @@ DECLARE_string(fs_uri);
 DECLARE_uint64(ops_per_thread);
 DECLARE_uint64(log2_keys_per_lock);
 DECLARE_uint64(max_manifest_file_size);
+DECLARE_int32(max_manifest_space_amp_pct);
 DECLARE_bool(in_place_update);
 DECLARE_string(memtablerep);
 DECLARE_int32(prefix_size);
@@ -410,6 +411,8 @@ DECLARE_uint64(max_sequential_skip_in_iterations);
 DECLARE_bool(enable_sst_partitioner_factory);
 DECLARE_bool(enable_do_not_compress_roles);
 DECLARE_bool(block_align);
+DECLARE_uint64(super_block_alignment_size);
+DECLARE_uint64(super_block_alignment_space_overhead_ratio);
 DECLARE_uint32(lowest_used_cache_tier);
 DECLARE_bool(enable_custom_split_merge);
 DECLARE_uint32(adm_policy);
@@ -427,6 +430,7 @@ DECLARE_bool(track_and_verify_wals);
 DECLARE_int32(remote_compaction_worker_threads);
 DECLARE_int32(remote_compaction_worker_interval);
 DECLARE_bool(remote_compaction_failure_fall_back_to_local);
+DECLARE_int32(allow_resumption_one_in);
 DECLARE_bool(auto_refresh_iterator_with_snapshot);
 DECLARE_uint32(memtable_op_scan_flush_trigger);
 DECLARE_uint32(memtable_avg_op_scan_flush_trigger);

db_stress_tool/db_stress_compaction_service.cc

@@ -0,0 +1,61 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#ifdef GFLAGS
+
+#include "db_stress_tool/db_stress_compaction_service.h"
+
+#include <string>
+
+#include "db_stress_tool/db_stress_test_base.h"
+#include "rocksdb/env.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+CompactionServiceJobStatus DbStressCompactionService::Wait(
+    const std::string& scheduled_job_id, std::string* result) {
+  while (true) {
+    if (aborted_.load()) {
+      return CompactionServiceJobStatus::kAborted;
+    }
+    const auto& maybeResultStatus =
+        shared_->GetRemoteCompactionResult(scheduled_job_id, result);
+    if (maybeResultStatus.has_value()) {
+      auto s = maybeResultStatus.value();
+      if (s.ok()) {
+        assert(result);
+        assert(!result->empty());
+        return CompactionServiceJobStatus::kSuccess;
+      } else {
+        // Remote Compaction failed
+        if (failure_should_fall_back_to_local_) {
+          return CompactionServiceJobStatus::kUseLocal;
+        }
+        if (StressTest::IsErrorInjectedAndRetryable(s)) {
+          return CompactionServiceJobStatus::kUseLocal;
+        }
+        if (result && result->empty()) {
+          // If result is empty, set the compaction status in the result so
+          // that it can be bubbled up to main thread
+          CompactionServiceResult compaction_result;
+          compaction_result.status = s;
+          if (compaction_result.Write(result).ok()) {
+            assert(result);
+            assert(!result->empty());
+          }
+        }
+        return CompactionServiceJobStatus::kFailure;
+      }
+    } else {
+      // Remote Compaction is still running
+      Env::Default()->SleepForMicroseconds(kWaitIntervalInMicros);
+    }
+  }
+  return CompactionServiceJobStatus::kFailure;
+}
+
+}  // namespace ROCKSDB_NAMESPACE
+
+#endif  // GFLAGS

db_stress_tool/db_stress_compaction_service.h

@@ -3,11 +3,13 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
 
+#ifdef GFLAGS
 #pragma once
 
+#include "db/compaction/compaction_job.h"
 #include "db_stress_shared_state.h"
-#include "db_stress_tool/db_stress_common.h"
 #include "rocksdb/options.h"
+#include "utilities/fault_injection_fs.h"
 
 namespace ROCKSDB_NAMESPACE {
 
@@ -25,47 +27,33 @@ class DbStressCompactionService : public CompactionService {
   const char* Name() const override { return kClassName(); }
 
   static constexpr uint64_t kWaitIntervalInMicros = 10 * 1000;  // 10ms
-  static constexpr uint64_t kWaitTimeoutInMicros =
-      30 * 1000 * 1000;  // 30 seconds
+
+  static constexpr const char* kTempOutputDirectoryPrefix = "tmp_output_";
 
   CompactionServiceScheduleResponse Schedule(
       const CompactionServiceJobInfo& info,
       const std::string& compaction_service_input) override {
     std::string job_id = info.db_id + "_" + info.db_session_id + "_" +
                          std::to_string(info.job_id);
+
     if (aborted_.load()) {
       return CompactionServiceScheduleResponse(
           job_id, CompactionServiceJobStatus::kUseLocal);
     }
-    shared_->EnqueueRemoteCompaction(job_id, info, compaction_service_input);
+    std::string output_directory = info.db_name + "/" +
+                                   kTempOutputDirectoryPrefix +
+                                   Env::Default()->GenerateUniqueId();
+
+    shared_->EnqueueRemoteCompaction(
+        job_id, info, compaction_service_input, output_directory,
+        false /* was_cancelled */);  // Not canceled initially
     CompactionServiceScheduleResponse response(
         job_id, CompactionServiceJobStatus::kSuccess);
     return response;
   }
 
   CompactionServiceJobStatus Wait(const std::string& scheduled_job_id,
-                                  std::string* result) override {
-    auto start = Env::Default()->NowMicros();
-    while (Env::Default()->NowMicros() - start < kWaitTimeoutInMicros) {
-      if (aborted_.load()) {
-        return CompactionServiceJobStatus::kUseLocal;
-      }
-      if (shared_->GetRemoteCompactionResult(scheduled_job_id, result).ok()) {
-        if (result && result->empty()) {
-          // Race: Remote worker aborted before client sets aborted_ = true
-          return CompactionServiceJobStatus::kUseLocal;
-        }
-        return CompactionServiceJobStatus::kSuccess;
-      }
-      Env::Default()->SleepForMicroseconds(kWaitIntervalInMicros);
-    }
-    if (failure_should_fall_back_to_local_) {
-      fprintf(stdout,
-              "Remote Compaction failed - fall back to local compaction!\n");
-      return CompactionServiceJobStatus::kUseLocal;
-    }
-    return CompactionServiceJobStatus::kFailure;
-  }
+                                  std::string* result) override;
 
   void OnInstallation(const std::string& scheduled_job_id,
                       CompactionServiceJobStatus /*status*/) override {
@@ -73,7 +61,7 @@ class DbStressCompactionService : public CompactionService {
     std::string serialized;
     CompactionServiceResult result;
     if (shared_->GetRemoteCompactionResult(scheduled_job_id, &serialized)
-            .ok()) {
+            .has_value()) {
       if (CompactionServiceResult::Read(serialized, &result).ok()) {
         std::vector<std::string> filenames;
         Status s = Env::Default()->GetChildren(result.output_path, &filenames);
@@ -100,5 +88,6 @@ class DbStressCompactionService : public CompactionService {
   std::atomic_bool aborted_{false};
   bool failure_should_fall_back_to_local_;
 };
-
 }  // namespace ROCKSDB_NAMESPACE
+
+#endif  // GFLAGS

db_stress_tool/db_stress_gflags.cc

@@ -861,6 +861,11 @@ DEFINE_bool(remote_compaction_failure_fall_back_to_local, true,
             "If true, remote compaction failures will be ignored and "
             "compactions will fall back to local and retried");
 
+DEFINE_int32(allow_resumption_one_in, 0,
+             "If non-zero, enable resumable compaction with 1/N probability "
+             "for each OpenAndCompact call.Requires "
+             "remote_compaction_worker_threads > 0");
+
 DEFINE_uint32(ingest_wbwi_one_in, 0,
               "If set, will call"
               "IngestWriteBatchWithIndex() instead of regular write operations "
@@ -973,7 +978,11 @@ DEFINE_uint64(log2_keys_per_lock, 2, "Log2 of number of keys per lock");
 static const bool FLAGS_log2_keys_per_lock_dummy __attribute__((__unused__)) =
     RegisterFlagValidator(&FLAGS_log2_keys_per_lock, &ValidateUint32Range);
 
-DEFINE_uint64(max_manifest_file_size, 16384, "Maximum size of a MANIFEST file");
+DEFINE_uint64(max_manifest_file_size, 16384,
+              "Maximum size of a MANIFEST file (without auto-tuning)");
+
+DEFINE_int32(max_manifest_space_amp_pct, 500,
+             "Max manifest space amp percentage for auto-tuning");
 
 DEFINE_bool(in_place_update, false, "On true, does inplace update in memtable");
 
@@ -1432,6 +1441,17 @@ DEFINE_bool(block_align,
             ROCKSDB_NAMESPACE::BlockBasedTableOptions().block_align,
             "BlockBasedTableOptions.block_align");
 
+DEFINE_uint64(
+    super_block_alignment_size,
+    ROCKSDB_NAMESPACE::BlockBasedTableOptions().super_block_alignment_size,
+    "BlockBasedTableOptions.super_block_alignment_size");
+
+DEFINE_uint64(
+    super_block_alignment_space_overhead_ratio,
+    ROCKSDB_NAMESPACE::BlockBasedTableOptions()
+        .super_block_alignment_space_overhead_ratio,
+    "BlockBasedTableOptions.super_block_alignment_space_overhead_ratio");
+
 DEFINE_uint32(
     lowest_used_cache_tier,
     static_cast<uint32_t>(ROCKSDB_NAMESPACE::Options().lowest_used_cache_tier),

db_stress_tool/db_stress_listener.h

@@ -9,6 +9,7 @@
 #include <mutex>
 #include <unordered_set>
 
+#include "db_stress_tool/db_stress_compaction_service.h"
 #include "db_stress_tool/db_stress_shared_state.h"
 #include "file/filename.h"
 #include "file/writable_file_writer.h"
@@ -21,7 +22,6 @@
 #include "util/gflags_compat.h"
 #include "util/random.h"
 #include "utilities/fault_injection_fs.h"
-
 DECLARE_int32(compact_files_one_in);
 
 extern std::shared_ptr<ROCKSDB_NAMESPACE::FaultInjectionTestFS> fault_fs_guard;
@@ -310,6 +310,11 @@ class DbStressListener : public EventListener {
         }
       }
     }
+    // We can't do exact matching since remote workers use dynamic temp paths
+    if (file_dir.find(DbStressCompactionService::kTempOutputDirectoryPrefix) !=
+        std::string::npos) {
+      return;
+    }
     assert(false);
 #else
     (void)file_dir;

db_stress_tool/db_stress_shared_state.h

@@ -51,6 +51,24 @@ DECLARE_bool(enable_compaction_filter);
 namespace ROCKSDB_NAMESPACE {
 class StressTest;
 
+struct RemoteCompactionQueueItem {
+  std::string job_id;
+  CompactionServiceJobInfo job_info;
+  std::string serialized_input;
+  std::string output_directory;
+  bool canceled;
+
+  RemoteCompactionQueueItem(const std::string& id,
+                            const CompactionServiceJobInfo& info,
+                            const std::string& input,
+                            const std::string& output_dir, bool was_canceled)
+      : job_id(id),
+        job_info(info),
+        serialized_input(input),
+        output_directory(output_dir),
+        canceled(was_canceled) {}
+};
+
 // State shared by all concurrent executions of the same benchmark.
 class SharedState {
  public:
@@ -278,23 +296,31 @@ class SharedState {
 
   void EnqueueRemoteCompaction(const std::string& job_id,
                                const CompactionServiceJobInfo& job_info,
-                               const std::string& serialized_input) {
+                               const std::string& serialized_input,
+                               const std::string& output_directory,
+                               bool canceled) {
     MutexLock l(&remote_compaction_queue_mu_);
-    remote_compaction_queue_.emplace(job_id, job_info, serialized_input);
+    remote_compaction_queue_.emplace(job_id, job_info, serialized_input,
+                                     output_directory, canceled);
   }
 
   bool DequeueRemoteCompaction(std::string* job_id,
                                CompactionServiceJobInfo* job_info,
-                               std::string* serialized_input) {
+                               std::string* serialized_input,
+                               std::string* output_directory, bool* canceled) {
     assert(job_id);
     assert(job_info);
     assert(serialized_input);
+    assert(output_directory);
+    assert(canceled);
     MutexLock l(&remote_compaction_queue_mu_);
     if (!remote_compaction_queue_.empty()) {
-      const auto [id, info, input] = remote_compaction_queue_.front();
-      *job_id = id;
-      *job_info = info;
-      *serialized_input = input;
+      const RemoteCompactionQueueItem& item = remote_compaction_queue_.front();
+      *job_id = item.job_id;
+      *job_info = item.job_info;
+      *serialized_input = item.serialized_input;
+      *output_directory = item.output_directory;
+      *canceled = item.canceled;
       remote_compaction_queue_.pop();
       return true;
     }
@@ -302,20 +328,23 @@ class SharedState {
   }
 
   void AddRemoteCompactionResult(const std::string& job_id,
+                                 const Status& status,
                                  const std::string& result) {
     MutexLock l(&remote_compaction_result_map_mu_);
-    remote_compaction_result_map_.emplace(job_id, result);
+    remote_compaction_result_map_.emplace(
+        job_id, std::pair<Status, std::string>{status, result});
   }
 
-  Status GetRemoteCompactionResult(const std::string& job_id,
-                                   std::string* result) {
+  std::optional<Status> GetRemoteCompactionResult(const std::string& job_id,
+                                                  std::string* result) {
     MutexLock l(&remote_compaction_result_map_mu_);
     if (remote_compaction_result_map_.find(job_id) !=
         remote_compaction_result_map_.end()) {
-      *result = remote_compaction_result_map_.at(job_id);
-      return Status::OK();
+      const auto& pair = remote_compaction_result_map_.at(job_id);
+      *result = pair.second;
+      return pair.first;
     }
-    return Status::NotFound();
+    return std::nullopt;
   }
 
   void RemoveRemoteCompactionResult(const std::string& job_id) {
@@ -477,15 +506,14 @@ class SharedState {
   std::atomic<bool> verification_failure_;
   std::atomic<bool> should_stop_test_;
 
-  // Queue for the remote compaction. Tuple of job id, job info and serialized
-  // compaction_service_input
+  // Queue for the remote compaction.
   port::Mutex remote_compaction_queue_mu_;
-  std::queue<std::tuple<std::string, CompactionServiceJobInfo, std::string>>
-      remote_compaction_queue_;
+  std::queue<RemoteCompactionQueueItem> remote_compaction_queue_;
   // Result Map for the remote compaciton. Key is the scheduled_job_id and value
   // is serialized compaction_service_result
   port::Mutex remote_compaction_result_map_mu_;
-  std::unordered_map<std::string, std::string> remote_compaction_result_map_;
+  std::unordered_map<std::string, std::pair<Status, std::string>>
+      remote_compaction_result_map_;
 
   // Keys that should not be overwritten
   const std::unordered_set<int64_t> no_overwrite_ids_;

db_stress_tool/db_stress_test_base.cc

@@ -428,8 +428,9 @@ bool StressTest::BuildOptionsTable() {
     options_tbl.emplace(
         "file_temperature_age_thresholds",
         std::vector<std::string>{
-            "{{temperature=kWarm;age=10}:{temperature=kCold;age=50}:{"
-            "temperature=kIce;age=250}}",
+            "{{temperature=kWarm;age=10}:{temperature=kCool;age=30}:{"
+            "temperature=kCold;age=100}:{"
+            "temperature=kIce;age=300}}",
             "{{temperature=kWarm;age=30}:{temperature=kCold;age=300}}",
             "{{temperature=kCold;age=100}}", "{}"});
     options_tbl.emplace(
@@ -1693,7 +1694,7 @@ Status StressTest::TestMultiScan(ThreadState* thread,
   std::vector<std::string> start_key_strs;
   std::vector<std::string> end_key_strs;
   // TODO support reverse BytewiseComparator in the stress test
-  MultiScanArgs scan_opts(BytewiseComparator());
+  MultiScanArgs scan_opts(options_.comparator);
   scan_opts.use_async_io = FLAGS_multiscan_use_async_io;
   start_key_strs.reserve(num_scans);
   end_key_strs.reserve(num_scans);
@@ -3674,8 +3675,21 @@ void StressTest::Open(SharedState* shared, bool reopen) {
               "Compaction\n");
       exit(1);
     }
-    options_.compaction_service = std::make_shared<DbStressCompactionService>(
+    // Each DB open/reopen gets a fresh compaction service instance with a clean
+    // aborted_ state
+    auto compaction_service = std::make_shared<DbStressCompactionService>(
         shared, FLAGS_remote_compaction_failure_fall_back_to_local);
+
+    options_.compaction_service = compaction_service;
+  }
+
+  if (FLAGS_allow_resumption_one_in > 0) {
+    if (FLAGS_remote_compaction_worker_threads == 0) {
+      fprintf(stderr,
+              "allow_resumption or randomize_allow_resumption requires "
+              "remote_compaction_worker_threads > 0\n");
+      exit(1);
+    }
   }
 
   if ((options_.enable_blob_files || options_.enable_blob_garbage_collection ||
@@ -4318,6 +4332,10 @@ void InitializeOptionsFromFlags(
       static_cast<BlockBasedTableOptions::IndexShorteningMode>(
           FLAGS_index_shortening);
   block_based_options.block_align = FLAGS_block_align;
+  block_based_options.super_block_alignment_size =
+      fLU64::FLAGS_super_block_alignment_size;
+  block_based_options.super_block_alignment_space_overhead_ratio =
+      fLU64::FLAGS_super_block_alignment_space_overhead_ratio;
   options.table_factory.reset(NewBlockBasedTableFactory(block_based_options));
   options.db_write_buffer_size = FLAGS_db_write_buffer_size;
   options.write_buffer_size = FLAGS_write_buffer_size;
@@ -4402,6 +4420,7 @@ void InitializeOptionsFromFlags(
     options.compression_opts.checksum = true;
   }
   options.max_manifest_file_size = FLAGS_max_manifest_file_size;
+  options.max_manifest_space_amp_pct = FLAGS_max_manifest_space_amp_pct;
   options.max_subcompactions = static_cast<uint32_t>(FLAGS_subcompactions);
   options.allow_concurrent_memtable_write =
       FLAGS_allow_concurrent_memtable_write;

db_stress_tool/db_stress_test_base.h

@@ -14,6 +14,7 @@
 #include "db_stress_tool/db_stress_common.h"
 #include "db_stress_tool/db_stress_shared_state.h"
 #include "rocksdb/experimental.h"
+#include "utilities/fault_injection_fs.h"
 
 namespace ROCKSDB_NAMESPACE {
 class SystemClock;
@@ -25,6 +26,13 @@ using experimental::SstQueryFilterConfigsManager;
 
 class StressTest {
  public:
+  static bool IsErrorInjectedAndRetryable(const Status& error_s) {
+    assert(!error_s.ok());
+    return error_s.getState() &&
+           FaultInjectionTestFS::IsInjectedError(error_s) &&
+           !status_to_io_status(Status(error_s)).GetDataLoss();
+  }
+
   StressTest();
 
   virtual ~StressTest() {}
@@ -350,13 +358,6 @@ class StressTest {
     return Status::NotSupported("TestCustomOperations() must be overridden");
   }
 
-  bool IsErrorInjectedAndRetryable(const Status& error_s) const {
-    assert(!error_s.ok());
-    return error_s.getState() &&
-           FaultInjectionTestFS::IsInjectedError(error_s) &&
-           !status_to_io_status(Status(error_s)).GetDataLoss();
-  }
-
   void ProcessStatus(SharedState* shared, std::string msg, const Status& s,
                      bool ignore_injected_error = true) const;
 

db_stress_tool/expected_value.h

@@ -253,20 +253,20 @@ class PendingExpectedValue {
 class ExpectedValueHelper {
  public:
   // Return whether the key associated with `pre_read_expected_value` and
-  // `post_read_expected_value` is expected not to exist from begining till the
+  // `post_read_expected_value` is expected not to exist from beginning till the
   // end of the read
   //
   // The negation of `MustHaveNotExisted()` is "may have not existed".
-  // To assert some key must have existsed, please use `MustHaveExisted()`
+  // To assert some key must have existed, please use `MustHaveExisted()`
   static bool MustHaveNotExisted(ExpectedValue pre_read_expected_value,
                                  ExpectedValue post_read_expected_value);
 
   // Return whether the key associated with `pre_read_expected_value` and
-  // `post_read_expected_value` is expected to exist from begining till the end
+  // `post_read_expected_value` is expected to exist from beginning till the end
   // of the read.
   //
   // The negation of `MustHaveExisted()` is "may have existed".
-  // To assert some key must have not existsed, please use
+  // To assert some key must have not existed, please use
   // `MustHaveNotExisted()`
   static bool MustHaveExisted(ExpectedValue pre_read_expected_value,
                               ExpectedValue post_read_expected_value);

docs/_posts/2025-09-24-unified-memory-tracking.markdown

@@ -0,0 +1,59 @@
+---
+title: Unified Memory Tracking
+layout: post
+author: hx235
+category: blog
+---
+
+## Context / Problem
+Modern RocksDB deployments often run in environments with strict memory constraints—cloud VMs, containers, or hosts with hundreds of DB instances. Unpredictable memory usage can lead to out-of-memory (OOM) errors, degraded performance, or even service outages.
+Historically, while the block cache was the main source of memory usage, other components—such as memtables, table readers, file metadata, and temporary buffers—could consume significant memory outside the block cache’s control. This made it difficult for users to set a single memory limit and guarantee resource usage stays within expectations.
+
+## Goal
+The goal of recent memory tracking work in RocksDB is to enable users to cap the total memory usage of RocksDB instances under a single, configurable limit—the block cache capacity. This is achieved by:
+- **Tracking and charging** all major memory consumers (memtables, table readers, file metadata, compression buffers, filter construction) to the block cache.
+- **Evicting** data blocks or other memory when the total tracked usage exceeds the configured limit.
+- **Providing a fixed memory footprint** for RocksDB, making it easier to run in resource-constrained environments and avoid OOMs.
+
+## Memtable Memory Charging
+A major source of memory usage in RocksDB is the memtable. To ensure memtable memory is tracked and capped under a single limit, RocksDB provides the WriteBufferManager (WBM). When WBM is configured with a block cache, memtable memory usage is charged to the block cache. This helps prevent OOM errors and simplifies resource management.
+
+```cpp
+std::shared_ptr<Cache> cache = HyperClockCacheOptions(capacity).MakeSharedCache();;
+DBOptions db_options;
+db_options.write_buffer_manager = std::make_shared<WriteBufferManager>(.., cache);
+```
+
+## Other Memory Charging
+Beyond memtables, RocksDB allows users to control memory charging for other internal roles using the cache_usage_options API. This provides fine-grained control over how memory is tracked for components like table readers, file metadata, compression dictionary buffers (`CompressionOptions::max_dict_buffer_bytes:`) and filter construction.
+
+```cpp
+struct CacheEntryRoleOptions {
+  enum class Decision {
+    kEnabled,
+    kDisabled,
+    kFallback,
+  };
+  Decision charged = Decision::kFallback;
+};
+struct CacheUsageOptions {
+  CacheEntryRoleOptions options;
+  std::map<CacheEntryRole, CacheEntryRoleOptions> options_overrides;
+};
+
+...
+BlockBasedTableOptions table_options;
+table_options.cache_usage_options.options.charged = CacheEntryRoleOptions::Decision::kFallback;
+table_options.cache_usage_options.options_overrides[CacheEntryRole::kTableBuilder] = {
+  .charged = CacheEntryRoleOptions::Decision::kEnabled,
+};
+```
+
+Default (`Decision::kFallback`) behavior for each memory type:
+- `CacheEntryRole::kCompressionDictionaryBuildingBuffer`: `kEnabled`
+- `CacheEntryRole::kFilterConstruction`: `kDisabled`
+- `CacheEntryRole::kBlockBasedTableReader`: `kDisabled`
+- `CacheEntryRole::kFileMetadata`: `kDisabled`
+
+## Monitoring and Observability
+RocksDB provides built-in statistics to help users monitor memory usage and cache behavior. The `DB::Properties::kBlockCacheEntryStats` exposes detailed statistics about block cache entries, including breakdowns by each `CacheEntryRole`. These statistics are essential for understanding memory consumption and tuning cache configuration.

docs/_posts/2025-09-25-io-tagging.markdown

@@ -0,0 +1,74 @@
+---
+title: IO Activity Tagging
+layout: post
+author: hx235
+category: blog
+---
+
+## Context
+
+RocksDB performs a variety of IO operations—user reads, background compactions, flushes, database opens, and verification tasks. Treating all these operations the same makes it difficult for file system implementers to optimize performance, prioritize latency-sensitive IOs, and diagnose bottlenecks. To solve that, RocksDB internally tags every IO operation with its activity type using the `IOActivity` enum. This automatic tagging provides precise context for each IO, enabling file systems to make smarter, context-aware decisions for scheduling, caching, and resource management.
+
+## How Internal IO Tagging Works
+RocksDB automatically assigns an `IOActivity` tag to each IO operation. This tag is propagated through the storage stack and included in the IO options passed to the file system.
+
+```cpp
+enum class IOActivity : uint8_t {
+    kFlush = 0,                        // IO for flush operations (background write)
+    kCompaction = 1,                   // IO for compaction (background read/write)
+    kDBOpen = 2,                       // IO during database open (read/write)
+    kGet = 3,                          // User Get() read
+    kMultiGet = 4,                     // User MultiGet() read
+    kDBIterator = 5,                   // User iterator read
+    kVerifyDBChecksum = 6,             // Verification: DB checksum
+    kVerifyFileChecksums = 7,          // Verification: file checksums
+    kGetEntity = 8,                    // Entity Get (e.g., wide-column)
+    kMultiGetEntity = 9,               // Entity MultiGet
+    kGetFileChecksumsFromCurrentManifest = 10, // Manifest checksum reads
+    // 0x80–0xFE: Reserved for custom/internal use
+    kUnknown = 0xFF                    // Unknown/unspecified activity
+};
+```
+
+## Access IO Tag in File System
+Custom file systems can access the IOActivity tag via the IO options structure provided by RocksDB. This allows them to optimize behavior based on the specific IO activity.
+
+```cpp
+Status CustomFileSystem::Append(uint64_t offset, const Slice& data, const IOOptions& io_opts, ...) {
+    switch (io_opts.io_activity) {
+        case Env::IOActivity::kGet:
+            // Prioritize or cache user reads
+            break;
+        case Env::IOActivity::kCompaction:
+            // Throttle or deprioritize background compaction IO
+            break;
+        case Env::IOActivity::kDBOpen:
+            // Track or optimize DB open IO
+            break;
+        // ... handle other activities ...
+        default:
+            // Default handling
+            break;
+    }
+}
+```
+## IO Activity Statistics in RocksDB
+RocksDB provides detailed histograms for IO activities, allowing you to analyze both the aggregate time spent (in microseconds) and the count of IOs for each activity type.
+```cpp
+// Read Histograms
+FILE_READ_FLUSH_MICROS
+FILE_READ_COMPACTION_MICROS
+FILE_READ_DB_OPEN_MICROS
+FILE_READ_GET_MICROS
+FILE_READ_MULTIGET_MICROS
+FILE_READ_DB_ITERATOR_MICROS
+FILE_READ_VERIFY_DB_CHECKSUM_MICROS
+FILE_READ_VERIFY_FILE_CHECKSUMS_MICROS
+
+// Write Histograms
+FILE_WRITE_FLUSH_MICROS
+FILE_WRITE_COMPACTION_MICROS
+FILE_WRITE_DB_OPEN_MICROS
+```
+
+Thanks to Maciej Szeszko and Andrew Chang from the RocksDB team for their contributions in expanding and maintaining the IOActivity enum.

docs/_posts/2025-10-08-parallel-compression-revamp.markdown

@@ -0,0 +1,89 @@
+---
+title: "Parallel Compression Revamp: Dramatically Reduced CPU Overhead"
+layout: post
+author: peterd
+category: blog
+---
+
+The upcoming RocksDB 10.7 release includes a major revamp of parallel compression that **dramatically reduces the feature's CPU overhead by up to 65%** while maintaining or improving throughput for compression-heavy workloads. We expect this to broaden the set of workloads that could benefit from parallel compression, especially for **bulk SST generation and remote compaction use cases** that are less sensitive to CPU responsiveness.
+
+## Background
+
+Parallel compression in RocksDB (`CompressionOptions::parallel_threads > 1`) allows multiple threads to compress different blocks simultaneously during SST file generation, which can significantly improve compaction throughput for workloads where compression is a bottleneck. However, the original implementation had substantial CPU overhead that often outweighed the benefits, limiting its practical adoption.
+
+## What's New: A Complete Reimplementation
+
+The parallel compression framework has been completely rewritten from the ground up in [pull request #13910](https://github.com/facebook/rocksdb/pull/13910) to address the core inefficiencies:
+
+### Ring Buffer Architecture
+Instead of separate compression and write queues with complex thread coordination, the new implementation uses a ring buffer of blocks-in-progress that enables efficient work distribution across threads. This bounds working memory while enabling high throughput with minimal cross-thread synchronization.
+
+![Ring Buffer Architecture](/static/images/parallel-compression/ring-buffer-architecture.svg)
+
+### Work-Stealing Design
+Previously, the calling thread could only generate uncompressed blocks, dedicated compression threads could only compress, and a writer thread could only write the SST file to storage. Now, all threads can participate in compression work in a quasi-work-stealing manner, dramatically reducing the need for threads to block waiting for work. While only one thread (the calling thread or "emit thread") can generate uncompressed SST blocks in the new implementation, feeding compression work to other threads and itself, all other threads are compatible with writing compressed blocks to storage.
+
+### Auto-Scaling Thread Management
+The ring buffer enables another key feature: auto-scaling of active threads based on ring buffer utilization. The framework intelligently wakes up idle worker threads only when there's sufficient work to justify the overhead, achieving near-maximum throughput while minimizing CPU waste from unnecessary thread wake-ups.
+
+### Lock-Free Synchronization
+The entire framework is now lock-free (and wait-free as long as compatible work units are available for each thread), based primarily on atomic operations. To cleanly pack and leverage many data fields into a single atomic value, I've developed a new `BitFields` utility API. This is proving useful for cleaning up the HyperClockCache implementation as well, and will be the topic of a later blog post.
+
+Semaphores are used for lock-free management of idle threads (assuming a lock-free semaphore implementation, which is likely the case with `ROCKSDB_USE_STD_SEMAPHORES` but that is untrustworthy; see below).
+
+## Performance Improvements
+
+The results speak for themselves. Here's a comparison using `db_bench` fillseq benchmarks with various compression configurations:
+
+### ZSTD Compression (Default Level)
+Note:
+* "throughput" = how quickly a given CPU-bound flush or compaction can complete
+* "CPU increase" = total CPU usage in amount of time that each core was used
+* "PT" = parallel_threads setting.
+
+**Before:**
+- PT=3: ~38% throughput increase for ~73% CPU increase
+- PT=6: No throughput increase for ~70% CPU increase
+
+**After:**
+- PT=3: ~58% throughput increase for ~25% CPU increase
+- PT=6: ~58% throughput increase for ~28% CPU increase
+
+### High Compression Scenarios
+For ZSTD compression level 8, the improvements are even more dramatic:
+
+**Before:**
+- PT=4: 2.6x throughput increase for 139% CPU increase
+- PT=8: 3.6x throughput increase for 135% CPU increase
+
+**After:**
+- PT=4: 2.8x throughput increase for 114% CPU increase
+- PT=8: 3.7x throughput increase for 116% CPU increase
+
+## Compression Algorithm Optimizations
+
+Alongside the parallel compression revamp, some optimizations have gone into the underlying compression implementations/integrations. Most notably, **LZ4HC received dramatic performance improvements** through better reuse of internal data structures between compression calls (detailed in [pull request #13805](https://github.com/facebook/rocksdb/pull/13805)). A small regression in LZ4 performance from that change was fixed in [pull request #14017](https://github.com/facebook/rocksdb/pull/14017).
+
+While **ZSTD remains the gold standard** for medium-to-high compression ratios in RocksDB, these LZ4HC optimizations make it an increasingly attractive option for read-heavy workloads where LZ4's faster decompression can provide overall performance benefits.
+
+## Production Ready
+
+With these efficiency improvements, parallel compression is now considered **production-ready**. The feature has been thoroughly tested in both unit tests and stress testing, including validation on high-load scenarios with hundreds of concurrent compression jobs and thousands of threads.
+
+Some notes on current limitations:
+- Parallel compression is currently incompatible with `UserDefinedIndex` and with the deprecated `decouple_partitioned_filters=false` setting
+- Maximum performance is available with `-DROCKSDB_USE_STD_SEMAPHORES` at compile time, though this is not currently recommended due to reported bugs in some implementations of C++20 semaphores
+
+## Configuration Recommendations
+
+The dramatically reduced CPU overhead means parallel compression is now viable for a broader range of workloads, particularly those using higher compression levels or compression-heavy scenarios like time-series data. However, simply enabling parallel compression could result in more *spiky* CPU loads for hosts serving live DB data. **Parallel compression might be most useful for bulk SST file generation and/or remote compaction workloads** because they are less sensitive to CPU responsiveness. In these scenarios there is little danger in setting `parallel_threads=8` even with the possibility of over-subscribing CPU cores, though the potentially safer "sweet spot" is typically around `parallel_threads=3`, depending on compression level, etc.
+
+## Limitations and Future
+
+Although this offers a great improvement in the implementation of an existing option, we recognize that this setup is suboptimal in a number of ways:
+* There is no work sharing / thread pooling for these SST compression/writer threads among compactions in the same process, so not well able to fit the workload to available CPU cores and not able to use other SST file compression work to avoid a worker thread going to sleep.
+* We are not (yet) using a framework that would allow micro-work sharing with things other than SST generation on a set of threads. That would be a good direction for effective sharing of CPU resources without spikes in usage, but might incur intolerable CPU overhead in managing work. With this "hand optimized" and specialized framework, we can at least evaluate such future endeavors against a perhaps ideal framework in terms of parallelizing with minimal overhead.
+
+## Try It Out
+
+Parallel compression revamp will be available in RocksDB 10.7. As always, we recommend testing in your specific environment to determine the optimal configuration for your workload.

docs/static/images/parallel-compression/ring-buffer-architecture.svg

@@ -0,0 +1,136 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 800 500" style="background-color: #fafafa;">
+  <defs>
+    <marker id="arrowhead" markerWidth="10" markerHeight="7"
+     refX="10" refY="3.5" orient="auto">
+      <polygon points="0 0, 10 3.5, 0 7" fill="#333" />
+    </marker>
+    <marker id="red-arrowhead" markerWidth="10" markerHeight="7"
+     refX="10" refY="3.5" orient="auto">
+      <polygon points="0 0, 10 3.5, 0 7" fill="#d32f2f" />
+    </marker>
+    <marker id="blue-arrowhead" markerWidth="10" markerHeight="7"
+     refX="10" refY="3.5" orient="auto">
+      <polygon points="0 0, 10 3.5, 0 7" fill="#1976d2" />
+    </marker>
+    <marker id="green-arrowhead" markerWidth="10" markerHeight="7"
+     refX="10" refY="3.5" orient="auto">
+      <polygon points="0 0, 10 3.5, 0 7" fill="#388e3c" />
+    </marker>
+    <filter id="shadow" x="-20%" y="-20%" width="140%" height="140%">
+      <feDropShadow dx="2" dy="2" stdDeviation="2" flood-color="#00000020"/>
+    </filter>
+  </defs>
+
+  <!-- Title -->
+  <text x="400" y="30" font-family="Arial, sans-serif" font-size="18" font-weight="bold" text-anchor="middle" fill="#333">
+    Ring Buffer Architecture (8 slots shown) for Parallel Compression
+  </text>
+
+  <!-- Ring Buffer Array -->
+  <g transform="translate(100,150)">
+    <!-- Array slots laid out horizontally -->
+
+    <!-- Slot 0 -->
+    <rect x="0" y="0" width="70" height="50" fill="#f5f5f5" stroke="#666" stroke-width="1" rx="5" filter="url(#shadow)"/>
+    <text x="35" y="20" font-family="Arial, sans-serif" font-size="10" font-weight="bold" text-anchor="middle" fill="#333">Slot 0</text>
+    <text x="35" y="35" font-family="Arial, sans-serif" font-size="9" text-anchor="middle" fill="#666">Empty</text>
+
+    <!-- Slot 1 - NextToWrite=1, being written by Worker Thread 2 -->
+    <rect x="75" y="0" width="70" height="50" fill="#e8f5e8" stroke="#388e3c" stroke-width="3" rx="5" filter="url(#shadow)"/>
+    <text x="110" y="20" font-family="Arial, sans-serif" font-size="10" font-weight="bold" text-anchor="middle" fill="#333">Slot 1</text>
+    <text x="110" y="35" font-family="Arial, sans-serif" font-size="9" text-anchor="middle" fill="#388e3c">Writing...</text>
+
+    <!-- Slot 2 - Compressed -->
+    <rect x="150" y="0" width="70" height="50" fill="#fff3e0" stroke="#f57c00" stroke-width="2" rx="5" filter="url(#shadow)"/>
+    <text x="185" y="20" font-family="Arial, sans-serif" font-size="10" font-weight="bold" text-anchor="middle" fill="#333">Slot 2</text>
+    <text x="185" y="35" font-family="Arial, sans-serif" font-size="9" text-anchor="middle" fill="#666">Compressed</text>
+
+    <!-- Slot 3 - Being compressed by Worker Thread 1 -->
+    <rect x="225" y="0" width="70" height="50" fill="#e3f2fd" stroke="#1976d2" stroke-width="3" rx="5" filter="url(#shadow)"/>
+    <text x="260" y="20" font-family="Arial, sans-serif" font-size="10" font-weight="bold" text-anchor="middle" fill="#333">Slot 3</text>
+    <text x="260" y="35" font-family="Arial, sans-serif" font-size="9" text-anchor="middle" fill="#1976d2">Compressing...</text>
+
+    <!-- Slot 4 - NextToCompress=4, uncompressed -->
+    <rect x="300" y="0" width="70" height="50" fill="#ffebee" stroke="#d32f2f" stroke-width="2" rx="5" filter="url(#shadow)"/>
+    <text x="335" y="20" font-family="Arial, sans-serif" font-size="10" font-weight="bold" text-anchor="middle" fill="#333">Slot 4</text>
+    <text x="335" y="35" font-family="Arial, sans-serif" font-size="9" text-anchor="middle" fill="#666">Uncompressed</text>
+
+    <!-- Slot 5 - NextToEmit=5, adding block -->
+    <rect x="375" y="0" width="70" height="50" fill="#e1f5fe" stroke="#4fc3f7" stroke-width="3" rx="5" filter="url(#shadow)"/>
+    <text x="410" y="20" font-family="Arial, sans-serif" font-size="10" font-weight="bold" text-anchor="middle" fill="#333">Slot 5</text>
+    <text x="410" y="35" font-family="Arial, sans-serif" font-size="9" text-anchor="middle" fill="#4fc3f7">Adding block...</text>
+
+    <!-- Slot 6 -->
+    <rect x="450" y="0" width="70" height="50" fill="#f5f5f5" stroke="#666" stroke-width="1" rx="5" filter="url(#shadow)"/>
+    <text x="485" y="20" font-family="Arial, sans-serif" font-size="10" font-weight="bold" text-anchor="middle" fill="#333">Slot 6</text>
+    <text x="485" y="35" font-family="Arial, sans-serif" font-size="9" text-anchor="middle" fill="#666">Empty</text>
+
+    <!-- Slot 7 -->
+    <rect x="525" y="0" width="70" height="50" fill="#f5f5f5" stroke="#666" stroke-width="1" rx="5" filter="url(#shadow)"/>
+    <text x="560" y="20" font-family="Arial, sans-serif" font-size="10" font-weight="bold" text-anchor="middle" fill="#333">Slot 7</text>
+    <text x="560" y="35" font-family="Arial, sans-serif" font-size="9" text-anchor="middle" fill="#666">Empty</text>
+  </g>
+
+  <!-- Arrows pointing to array positions - staggered vertically -->
+  <!-- NextToWrite=1 arrow (red) -->
+  <line x1="210" y1="100" x2="210" y2="145" stroke="#d32f2f" stroke-width="3" marker-end="url(#red-arrowhead)"/>
+  <text x="210" y="90" font-family="Arial, sans-serif" font-size="12" font-weight="bold" text-anchor="middle" fill="#d32f2f">NextToWrite=1</text>
+
+  <!-- NextToCompress=4 arrow (blue) -->
+  <line x1="435" y1="110" x2="435" y2="145" stroke="#1976d2" stroke-width="3" marker-end="url(#blue-arrowhead)"/>
+  <text x="435" y="100" font-family="Arial, sans-serif" font-size="12" font-weight="bold" text-anchor="middle" fill="#1976d2">NextToCompress=4</text>
+
+  <!-- NextToEmit=5 arrow (light blue) -->
+  <line x1="510" y1="120" x2="510" y2="145" stroke="#4fc3f7" stroke-width="3" marker-end="url(#arrowhead)"/>
+  <text x="510" y="110" font-family="Arial, sans-serif" font-size="12" font-weight="bold" text-anchor="middle" fill="#4fc3f7">NextToEmit=5</text>
+
+  <!-- Worker Thread 2 (wider box) -->
+  <g transform="translate(30,250)">
+    <rect x="0" y="0" width="160" height="70" fill="#e8f5e8" stroke="#388e3c" stroke-width="2" rx="8" filter="url(#shadow)"/>
+    <text x="80" y="25" font-family="Arial, sans-serif" font-size="14" font-weight="bold" text-anchor="middle" fill="#333">Worker Thread 2</text>
+    <text x="80" y="40" font-family="Arial, sans-serif" font-size="11" text-anchor="middle" fill="#666">Currently writing</text>
+    <text x="80" y="55" font-family="Arial, sans-serif" font-size="11" text-anchor="middle" fill="#666">Can also compress</text>
+  </g>
+
+  <!-- Worker Thread 1 (wider box) -->
+  <g transform="translate(240,280)">
+    <rect x="0" y="0" width="160" height="70" fill="#e3f2fd" stroke="#1976d2" stroke-width="2" rx="8" filter="url(#shadow)"/>
+    <text x="80" y="25" font-family="Arial, sans-serif" font-size="14" font-weight="bold" text-anchor="middle" fill="#333">Worker Thread 1</text>
+    <text x="80" y="40" font-family="Arial, sans-serif" font-size="11" text-anchor="middle" fill="#666">Currently compressing</text>
+    <text x="80" y="55" font-family="Arial, sans-serif" font-size="11" text-anchor="middle" fill="#666">Can also write</text>
+  </g>
+
+  <!-- Emit Thread (wider box) -->
+  <g transform="translate(460,250)">
+    <rect x="0" y="0" width="180" height="70" fill="#e1f5fe" stroke="#4fc3f7" stroke-width="2" rx="8" filter="url(#shadow)"/>
+    <text x="90" y="25" font-family="Arial, sans-serif" font-size="14" font-weight="bold" text-anchor="middle" fill="#333">Emit Thread</text>
+    <text x="90" y="40" font-family="Arial, sans-serif" font-size="11" text-anchor="middle" fill="#666">Generates uncompressed blocks</text>
+    <text x="90" y="55" font-family="Arial, sans-serif" font-size="11" text-anchor="middle" fill="#666">Can help with compression</text>
+  </g>
+
+  <!-- Arrows from threads to array (no labels) -->
+  <!-- Worker Thread 2 to Slot 1 -->
+  <line x1="190" y1="285" x2="210" y2="210" stroke="#388e3c" stroke-width="2" marker-end="url(#green-arrowhead)"/>
+
+  <!-- Worker Thread 1 to Slot 3 -->
+  <line x1="320" y1="280" x2="360" y2="210" stroke="#1976d2" stroke-width="2" marker-end="url(#blue-arrowhead)"/>
+
+  <!-- Emit Thread to Slot 5 -->
+  <line x1="550" y1="250" x2="510" y2="210" stroke="#4fc3f7" stroke-width="2" marker-end="url(#arrowhead)"/>
+
+  <!-- SST File Output (centered below Worker Thread 2) -->
+  <g transform="translate(50,380)">
+    <rect x="0" y="0" width="120" height="40" fill="#e8f5e8" stroke="#388e3c" stroke-width="2" rx="5" filter="url(#shadow)"/>
+    <text x="60" y="25" font-family="Arial, sans-serif" font-size="12" font-weight="bold" text-anchor="middle" fill="#333">SST File</text>
+  </g>
+
+  <!-- Arrow from Worker Thread 2 to SST file -->
+  <line x1="110" y1="320" x2="110" y2="380" stroke="#388e3c" stroke-width="2" marker-end="url(#green-arrowhead)"/>
+
+  <!-- Invariant (moved to the right) -->
+  <g transform="translate(400,430)">
+    <text x="0" y="0" font-family="Arial, sans-serif" font-size="12" font-weight="bold" fill="#333">Invariant:</text>
+    <text x="0" y="20" font-family="Arial, sans-serif" font-size="11" fill="#666">NextToWrite ≤ NextToCompress ≤ NextToEmit (modulo ring buffer size)</text>
+  </g>
+</svg>

env/composite_env_wrapper.h

@@ -142,12 +142,6 @@ class CompositeEnv : public Env {
     return file_system_->LinkFile(s, t, io_opts, &dbg);
   }
 
-  Status SyncFile(const std::string& fname, const EnvOptions& env_options,
-                  bool use_fsync) override {
-    return file_system_->SyncFile(fname, env_options, IOOptions(), use_fsync,
-                                  nullptr);
-  }
-
   Status NumFileLinks(const std::string& fname, uint64_t* count) override {
     IOOptions io_opts;
     IODebugContext dbg;

env/env.cc

@@ -529,13 +529,6 @@ class LegacyFileSystemWrapper : public FileSystem {
     return status_to_io_status(target_->LinkFile(s, t));
   }
 
-  IOStatus SyncFile(const std::string& fname, const FileOptions& file_options,
-                    const IOOptions& /*io_options*/, bool use_fsync,
-                    IODebugContext* /*dbg*/) override {
-    return status_to_io_status(
-        target_->SyncFile(fname, file_options, use_fsync));
-  }
-
   IOStatus NumFileLinks(const std::string& fname, const IOOptions& /*options*/,
                         uint64_t* count, IODebugContext* /*dbg*/) override {
     return status_to_io_status(target_->NumFileLinks(fname, count));
@@ -878,43 +871,6 @@ std::string Env::GenerateUniqueId() {
   return result;
 }
 
-// This API Env::SyncFile is used for testing for 2 reasons:
-//
-// 1. The default implementation of SyncFile API is essentially a wrapper of
-// other FileSystem APIs. FaultInjectionTestEnv uses this default
-// implementation to call other FileSystem APIs defined at
-// FaultInjectionTestEnv class to inject failurses. See
-// FaultInjectionTestEnv::SyncFile for more details
-//
-// 2. Some of old tests are using LegacyFileSystemWrapper.
-// LegacyFileSystemWrapper forwards the API call to EnvWrapper, which forwards
-// to CompositeEnv, and then forwards to the actual FileSystem implemention.
-// Without this API in Env, LegacyFileSystemWrapper will not be able to
-// forward the API call to EnvWrapper, causing the default FileSystem API to
-// be called.
-//
-// Due to the above reason, adding a new API in FileSystem, would very likely
-// require the same API to be added to Env.
-//
-// TODO xingbo. Getting rid of FileSystem functions from Env.
-// We need to simplify the relationship between Env and FileSystem. At least
-// for internal test, we should stop using Env and switch to FileSystem, if
-// possible. Related github issue #9274
-Status Env::SyncFile(const std::string& fname, const EnvOptions& env_options,
-                     bool use_fsync) {
-  std::unique_ptr<WritableFile> file_to_sync;
-  auto status = ReopenWritableFile(fname, &file_to_sync, env_options);
-  TEST_SYNC_POINT_CALLBACK("FileSystem::SyncFile:Open", &status);
-  if (status.ok()) {
-    if (use_fsync) {
-      status = file_to_sync->Fsync();
-    } else {
-      status = file_to_sync->Sync();
-    }
-  }
-  return status;
-}
-
 SequentialFile::~SequentialFile() = default;
 
 RandomAccessFile::~RandomAccessFile() = default;

env/env_encryption.cc

@@ -664,8 +664,6 @@ class EncryptedFileSystemImpl : public EncryptedFileSystem {
                               const FileOptions& options,
                               std::unique_ptr<FSWritableFile>* result,
                               IODebugContext* dbg) override {
-    // TODO xingbo Add unit test for the new implementation of
-    // EncryptedFileSysmteImpl::ReopenWritableFile.
     result->reset();
     if (options.use_mmap_reads || options.use_mmap_writes) {
       return IOStatus::InvalidArgument();
@@ -816,15 +814,6 @@ class EncryptedFileSystemImpl : public EncryptedFileSystem {
     return status;
   }
 
-  IOStatus SyncFile(const std::string& fname, const FileOptions& file_options,
-                    const IOOptions& io_options, bool use_fsync,
-                    IODebugContext* dbg) override {
-    // Use the underlying file system to sync the file, as we don't need to
-    // read/write the file.
-    return FileSystemWrapper::SyncFile(fname, file_options, io_options,
-                                       use_fsync, dbg);
-  }
-
  private:
   std::shared_ptr<EncryptionProvider> provider_;
 };

env/file_system.cc

@@ -107,23 +107,6 @@ IOStatus FileSystem::ReuseWritableFile(const std::string& fname,
   return NewWritableFile(fname, opts, result, dbg);
 }
 
-IOStatus FileSystem::SyncFile(const std::string& fname,
-                              const FileOptions& file_options,
-                              const IOOptions& io_options, bool use_fsync,
-                              IODebugContext* dbg) {
-  std::unique_ptr<FSWritableFile> file_to_sync;
-  auto status = ReopenWritableFile(fname, file_options, &file_to_sync, dbg);
-  TEST_SYNC_POINT_CALLBACK("FileSystem::SyncFile:Open", &status);
-  if (status.ok()) {
-    if (use_fsync) {
-      status = file_to_sync->Fsync(io_options, dbg);
-    } else {
-      status = file_to_sync->Sync(io_options, dbg);
-    }
-  }
-  return status;
-}
-
 IOStatus FileSystem::NewLogger(const std::string& fname,
                                const IOOptions& io_opts,
                                std::shared_ptr<Logger>* result,

env/fs_posix.cc

@@ -1272,6 +1272,7 @@ class PosixFileSystem : public FileSystem {
       supported_ops |= (1 << FSSupportedOps::kAsyncIO);
     }
 #endif
+    supported_ops |= (1 << FSSupportedOps::kFSPrefetch);
   }
 
 #if defined(ROCKSDB_IOURING_PRESENT)

env/io_posix.cc

@@ -1406,6 +1406,7 @@ IOStatus PosixWritableFile::Truncate(uint64_t size, const IOOptions& /*opts*/,
                 filename_, errno);
   } else {
     filesize_ = size;
+    lseek(fd_, filesize_, SEEK_SET);
   }
   return s;
 }

env/io_posix_test.cc

@@ -4,6 +4,7 @@
 // (found in the LICENSE.Apache file in the root directory).
 
 #include "test_util/testharness.h"
+#include "util/random.h"
 
 #ifdef ROCKSDB_LIB_IO_POSIX
 #include "env/io_posix.h"
@@ -131,6 +132,48 @@ TEST_F(LogicalBlockSizeCacheTest, Ref) {
 }
 #endif
 
+class PosixWritableFileTest : public testing::Test {};
+
+TEST_F(PosixWritableFileTest, SeekAfterTruncate) {
+  std::shared_ptr<FileSystem> fs = FileSystem::Default();
+  std::string path =
+      test::PerThreadDBPath("PosixWritableFileTest_SeekAfterTruncate");
+  Random rnd(300);
+  std::unique_ptr<FSWritableFile> wfile;
+
+  ASSERT_OK(fs->NewWritableFile(path, FileOptions(), &wfile, nullptr));
+  ASSERT_OK(wfile->Append(rnd.RandomString(16384), IOOptions(), nullptr));
+  ASSERT_OK(wfile->Truncate(4096, IOOptions(), nullptr));
+  ASSERT_OK(wfile->Append(rnd.RandomString(4096), IOOptions(), nullptr));
+  ASSERT_OK(wfile->Close(IOOptions(), nullptr));
+  wfile.reset();
+
+  uint64_t size = 0;
+  ASSERT_OK(fs->GetFileSize(path, IOOptions(), &size, nullptr));
+  ASSERT_EQ(size, 8192);
+  ASSERT_OK(fs->DeleteFile(path, IOOptions(), nullptr));
+}
+
+TEST_F(PosixWritableFileTest, SeekAfterExtend) {
+  std::shared_ptr<FileSystem> fs = FileSystem::Default();
+  std::string path =
+      test::PerThreadDBPath("PosixWritableFileTest_SeekAfterTruncate");
+  Random rnd(300);
+  std::unique_ptr<FSWritableFile> wfile;
+
+  ASSERT_OK(fs->NewWritableFile(path, FileOptions(), &wfile, nullptr));
+  ASSERT_OK(wfile->Append(rnd.RandomString(4096), IOOptions(), nullptr));
+  ASSERT_OK(wfile->Truncate(8192, IOOptions(), nullptr));
+  ASSERT_OK(wfile->Append(rnd.RandomString(8192), IOOptions(), nullptr));
+  ASSERT_OK(wfile->Close(IOOptions(), nullptr));
+  wfile.reset();
+
+  uint64_t size = 0;
+  ASSERT_OK(fs->GetFileSize(path, IOOptions(), &size, nullptr));
+  ASSERT_EQ(size, 16384);
+  ASSERT_OK(fs->DeleteFile(path, IOOptions(), nullptr));
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 #endif
 

env/mock_env.cc

@@ -957,14 +957,6 @@ IOStatus MockFileSystem::LinkFile(const std::string& src,
   return IOStatus::OK();
 }
 
-IOStatus MockFileSystem::SyncFile(const std::string& /*fname*/,
-                                  const FileOptions& /*file_options*/,
-                                  const IOOptions& /*io_options*/,
-                                  bool /*use_fsync*/, IODebugContext* /*dbg*/) {
-  // Noop
-  return IOStatus::OK();
-}
-
 IOStatus MockFileSystem::NewLogger(const std::string& fname,
                                    const IOOptions& io_opts,
                                    std::shared_ptr<Logger>* result,

env/mock_env.h

@@ -86,10 +86,6 @@ class MockFileSystem : public FileSystem {
   IOStatus LinkFile(const std::string& /*src*/, const std::string& /*target*/,
                     const IOOptions& /*options*/,
                     IODebugContext* /*dbg*/) override;
-  IOStatus SyncFile(const std::string& /*fname*/,
-                    const FileOptions& /*file_options*/,
-                    const IOOptions& /*io_options*/, bool /*use_fsync*/,
-                    IODebugContext* /*dbg*/) override;
   IOStatus LockFile(const std::string& fname, const IOOptions& options,
                     FileLock** lock, IODebugContext* dbg) override;
   IOStatus UnlockFile(FileLock* lock, const IOOptions& options,

file/filename.cc

@@ -25,6 +25,7 @@ namespace ROCKSDB_NAMESPACE {
 
 const std::string kCurrentFileName = "CURRENT";
 const std::string kOptionsFileNamePrefix = "OPTIONS-";
+const std::string kCompactionProgressFileNamePrefix = "COMPACTION_PROGRESS-";
 const std::string kTempFileNameSuffix = "dbtmp";
 
 static const std::string kRocksDbTFileExt = "sst";
@@ -242,6 +243,25 @@ std::string TempOptionsFileName(const std::string& dbname, uint64_t file_num) {
   return dbname + "/" + buffer;
 }
 
+std::string CompactionProgressFileName(const std::string& dbname,
+                                       uint64_t timestamp) {
+  char buffer[256];
+  snprintf(buffer, sizeof(buffer), "%s%llu",
+           kCompactionProgressFileNamePrefix.c_str(),
+           static_cast<unsigned long long>(timestamp));
+  return dbname + "/" + buffer;
+}
+
+std::string TempCompactionProgressFileName(const std::string& dbname,
+                                           uint64_t timestamp) {
+  char buffer[256];
+  snprintf(buffer, sizeof(buffer), "%s%llu.%s",
+           kCompactionProgressFileNamePrefix.c_str(),
+           static_cast<unsigned long long>(timestamp),
+           kTempFileNameSuffix.c_str());
+  return dbname + "/" + buffer;
+}
+
 std::string MetaDatabaseName(const std::string& dbname, uint64_t number) {
   char buf[100];
   snprintf(buf, sizeof(buf), "/METADB-%llu",
@@ -264,6 +284,8 @@ std::string IdentityFileName(const std::string& dbname) {
 //    dbname/METADB-[0-9]+
 //    dbname/OPTIONS-[0-9]+
 //    dbname/OPTIONS-[0-9]+.dbtmp
+//    dbname/COMPACTION_PROGRESS-[timestamp]
+//    dbname/COMPACTION_PROGRESS-[timestamp].dbtmp
 //    Disregards / at the beginning
 bool ParseFileName(const std::string& fname, uint64_t* number, FileType* type,
                    WalFileType* log_type) {
@@ -339,6 +361,24 @@ bool ParseFileName(const std::string& fname, uint64_t* number,
     }
     *number = ts_suffix;
     *type = is_temp_file ? kTempFile : kOptionsFile;
+  } else if (rest.starts_with(kCompactionProgressFileNamePrefix)) {
+    uint64_t timestamp;
+    bool is_temp_file = false;
+    rest.remove_prefix(kCompactionProgressFileNamePrefix.size());
+    const std::string kTempFileNameSuffixWithDot =
+        std::string(".") + kTempFileNameSuffix;
+    if (rest.ends_with(kTempFileNameSuffixWithDot)) {
+      rest.remove_suffix(kTempFileNameSuffixWithDot.size());
+      is_temp_file = true;
+    }
+    if (!ConsumeDecimalNumber(&rest, &timestamp)) {
+      return false;
+    }
+    if (!rest.empty()) {
+      return false;
+    }
+    *number = timestamp;
+    *type = is_temp_file ? kTempFile : kCompactionProgressFile;
   } else {
     // Avoid strtoull() to keep filename format independent of the
     // current locale