Update HISTORY and version for 10.7.6

Follow up on MultiScan change in #14040 (#14055 )
Summary: * Address feedback from https://github.com/facebook/rocksdb/issues/14040 * Add additional test for MultiScan * Fix a bug when del range and data are in same file for multi-scan * Rewrite the cases need to be handled in SeekMultiScan Pull Request resolved: https://github.com/facebook/rocksdb/pull/14055 Test Plan: Unit test Reviewed By: cbi42, anand1976 Differential Revision: D84851788 Pulled By: xingbowang fbshipit-source-id: 0f69632733afb99685f6341badbf239681010c38
2026-11-06 19:12:36 -05:00 · 2025-10-27 16:42:03 -07:00 · 2025-10-27 16:19:57 -07:00 · 2025-10-20 11:17:17 -07:00 · 2025-10-20 11:13:02 -07:00 · 2025-10-14 15:36:55 -07:00
191 changed files with 13279 additions and 3479 deletions
@@ -3,5 +3,7 @@ runs:
  using: composite
  steps:
  - name: Checkout folly sources
-    run: make checkout_folly
+    run: |
+      make checkout_folly
+      echo "GETDEPS_USE_WGET=1" >> "$GITHUB_ENV"
    shell: bash
@@ -10,7 +10,7 @@ jobs:
    runs-on:
      labels: 16-core-ubuntu
    container:
-      image: zjay437/rocksdb:0.6
+      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
      options: --shm-size=16gb
    steps:
    - uses: actions/checkout@v4.1.0
@@ -32,7 +32,7 @@ jobs:
    runs-on:
      labels: 16-core-ubuntu
    container:
-      image: zjay437/rocksdb:0.6
+      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
      options: --shm-size=16gb
    env:
      TEST_TMPDIR: "/tmp/rocksdb_test_tmp"
@@ -41,16 +41,16 @@ jobs:
    - uses: "./.github/actions/pre-steps"
    - run: make V=1 -j32 check
    - uses: "./.github/actions/post-steps"
-  build-linux-clang-13-asan-ubsan-with-folly:
+  build-linux-clang-18-asan-ubsan-with-folly:
    if: ${{ github.repository_owner == 'facebook' }}
    runs-on:
      labels: 16-core-ubuntu
    container:
-      image: zjay437/rocksdb:0.6
+      image: ghcr.io/facebook/rocksdb_ubuntu:24.0
      options: --shm-size=16gb
    env:
-      CC: clang-13
-      CXX: clang++-13
+      CC: clang-18
+      CXX: clang++-18
    steps:
    - uses: actions/checkout@v4.1.0
    - uses: "./.github/actions/pre-steps"
@@ -63,7 +63,7 @@ jobs:
    runs-on:
      labels: 16-core-ubuntu
    container:
-      image: zjay437/rocksdb:0.6
+      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
      options: --shm-size=16gb
    steps:
    - uses: actions/checkout@v4.1.0
@@ -94,7 +94,7 @@ jobs:
    runs-on:
      labels: 4-core-ubuntu
    container:
-      image: zjay437/rocksdb:0.6
+      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
      options: --shm-size=16gb
    steps:
    - uses: actions/checkout@v4.1.0
@@ -107,41 +107,26 @@ jobs:
    runs-on:
      labels: 4-core-ubuntu
    container:
-      image: zjay437/rocksdb:0.6
+      image: ghcr.io/facebook/rocksdb_ubuntu:24.0
      options: --shm-size=16gb
    steps:
    - uses: actions/checkout@v4.1.0
    - uses: "./.github/actions/pre-steps"
    - name: Build rocksdb lib
-      run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j4 static_lib
+      run: CC=clang-18 CXX=clang++-18 USE_CLANG=1 make -j4 static_lib
    - name: Build fuzzers
      run: cd fuzz && make sst_file_writer_fuzzer db_fuzzer db_map_fuzzer
    - uses: "./.github/actions/post-steps"
-  build-linux-gcc-11-no_test_run:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on:
-      labels: 16-core-ubuntu
-    container:
-      image: zjay437/rocksdb:0.6
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - run: LIB_MODE=static CC=gcc-11 CXX=g++-11 V=1 make -j32 all microbench
-    - uses: "./.github/actions/post-steps"
  build-linux-cmake-with-folly-lite-no-test:
    if: ${{ github.repository_owner == 'facebook' }}
    runs-on:
      labels: 16-core-ubuntu
    container:
-      image: zjay437/rocksdb:0.6
+      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
      options: --shm-size=16gb
-    env:
-      CC: gcc-10
-      CXX: g++-10
    steps:
    - uses: actions/checkout@v4.1.0
    - uses: "./.github/actions/pre-steps"
    - uses: "./.github/actions/setup-folly"
-    - run: "(mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 .. && make V=1 -j20)"
+    - run: "(mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 -DCMAKE_CXX_FLAGS=-DGLOG_USE_GLOG_EXPORT .. && make V=1 -j20)"
    - uses: "./.github/actions/post-steps"
@@ -66,7 +66,7 @@ jobs:
    runs-on:
      labels: 16-core-ubuntu
    container:
-      image: zjay437/rocksdb:0.6
+      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
      options: --shm-size=16gb
    steps:
    - uses: actions/checkout@v4.1.0
@@ -78,7 +78,7 @@ jobs:
    runs-on:
      labels: 4-core-ubuntu
    container:
-      image: zjay437/rocksdb:0.6
+      image: ghcr.io/facebook/rocksdb_ubuntu:24.0
      options: --shm-size=16gb
    steps:
    - uses: actions/checkout@v4.1.0
@@ -97,11 +97,8 @@ jobs:
    runs-on:
      labels: 16-core-ubuntu
    container:
-      image: zjay437/rocksdb:0.6
+      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
      options: --shm-size=16gb
-    env:
-      CC: gcc-10
-      CXX: g++-10
    steps:
    - uses: actions/checkout@v4.1.0
    - uses: "./.github/actions/pre-steps"
@@ -114,11 +111,8 @@ jobs:
    runs-on:
      labels: 16-core-ubuntu
    container:
-      image: zjay437/rocksdb:0.6
+      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
      options: --shm-size=16gb
-    env:
-      CC: gcc-10
-      CXX: g++-10
    steps:
    - uses: actions/checkout@v4.1.0
    - uses: "./.github/actions/pre-steps"
@@ -131,27 +125,21 @@ jobs:
    runs-on:
      labels: 16-core-ubuntu
    container:
-      image: zjay437/rocksdb:0.6
+      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
      options: --shm-size=16gb
-    env:
-      CC: gcc-10
-      CXX: g++-10
    steps:
    - uses: actions/checkout@v4.1.0
    - uses: "./.github/actions/pre-steps"
    - uses: "./.github/actions/setup-folly"
-    - run: USE_FOLLY_LITE=1 V=1 make -j32 all
+    - run: USE_FOLLY_LITE=1 EXTRA_CXXFLAGS=-DGLOG_USE_GLOG_EXPORT V=1 make -j32 all
    - uses: "./.github/actions/post-steps"
  build-linux-cmake-with-folly-coroutines:
    if: ${{ github.repository_owner == 'facebook' }}
    runs-on:
      labels: 16-core-ubuntu
    container:
-      image: zjay437/rocksdb:0.6
+      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
      options: --shm-size=16gb
-    env:
-      CC: gcc-10
-      CXX: g++-10
    steps:
    - uses: actions/checkout@v4.1.0
    - uses: "./.github/actions/pre-steps"
@@ -164,7 +152,7 @@ jobs:
    runs-on:
      labels: 16-core-ubuntu
    container:
-      image: zjay437/rocksdb:0.6
+      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
      options: --shm-size=16gb
    steps:
    - uses: actions/checkout@v4.1.0
@@ -176,7 +164,7 @@ jobs:
    runs-on:
      labels: 16-core-ubuntu
    container:
-      image: zjay437/rocksdb:0.6
+      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
      options: --shm-size=16gb
    steps:
    - uses: actions/checkout@v4.1.0
@@ -190,7 +178,7 @@ jobs:
    runs-on:
      labels: 16-core-ubuntu
    container:
-      image: zjay437/rocksdb:0.6
+      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
      options: --shm-size=16gb
    steps:
    - uses: actions/checkout@v4.1.0
@@ -211,68 +199,61 @@ jobs:
    - run: ls librocksdb.a
    - run: if ./trace_analyzer --version; then false; else true; fi
    - uses: "./.github/actions/post-steps"
-  build-linux-clang-no_test_run:
+  build-linux-clang-13-no_test_run:
    if: ${{ github.repository_owner == 'facebook' }}
    runs-on:
      labels: 8-core-ubuntu
    container:
-      image: zjay437/rocksdb:0.6
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j16 all
-    - uses: "./.github/actions/post-steps"
-  build-linux-clang-13-no_test_run:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on:
-      labels: 16-core-ubuntu
-    container:
-      image: zjay437/rocksdb:0.6
+      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
      options: --shm-size=16gb
    steps:
    - uses: actions/checkout@v4.1.0
    - uses: "./.github/actions/pre-steps"
-    - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j32 all microbench
+    # FIXME: get back to "all microbench" targets
+    - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 EXTRA_CXXFLAGS=-stdlib=libc++ EXTRA_LDFLAGS=-stdlib=libc++ make -j32 shared_lib
    - run: make clean
-    - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 DEBUG_LEVEL=0 make -j32 release
+    # FIXME: get back to "release" target
+    - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 EXTRA_CXXFLAGS=-stdlib=libc++ EXTRA_LDFLAGS=-stdlib=libc++ DEBUG_LEVEL=0 make -j32 shared_lib
    - uses: "./.github/actions/post-steps"
-  build-linux-gcc-8-no_test_run:
+  build-linux-clang-18-no_test_run:
    if: ${{ github.repository_owner == 'facebook' }}
    runs-on:
      labels: 16-core-ubuntu
    container:
-      image: zjay437/rocksdb:0.6
+      image: ghcr.io/facebook/rocksdb_ubuntu:24.0
      options: --shm-size=16gb
    steps:
    - uses: actions/checkout@v4.1.0
    - uses: "./.github/actions/pre-steps"
-    - run: CC=gcc-8 CXX=g++-8 V=1 make -j32 all
+    - run: CC=clang-18 CXX=clang++-18 USE_CLANG=1 make -j32 all microbench
+    - run: make clean
+    - run: CC=clang-18 CXX=clang++-18 USE_CLANG=1 DEBUG_LEVEL=0 make -j32 release
    - uses: "./.github/actions/post-steps"
-  build-linux-gcc-10-cxx20-no_test_run:
+  build-linux-gcc-14-no_test_run:
    if: ${{ github.repository_owner == 'facebook' }}
    runs-on:
      labels: 16-core-ubuntu
    container:
-      image: zjay437/rocksdb:0.6
+      image: ghcr.io/facebook/rocksdb_ubuntu:24.0
      options: --shm-size=16gb
    steps:
    - uses: actions/checkout@v4.1.0
    - uses: "./.github/actions/pre-steps"
-    - run: CC=gcc-10 CXX=g++-10 V=1 ROCKSDB_CXX_STANDARD=c++20 make -j32 all
+    - run: CC=gcc-14 CXX=g++-14 V=1 make -j32 all microbench
    - uses: "./.github/actions/post-steps"

  # ======================== Linux Other Checks ======================= #
-  build-linux-clang10-clang-analyze:
+  build-linux-clang18-clang-analyze:
    if: ${{ github.repository_owner == 'facebook' }}
    runs-on:
      labels: 16-core-ubuntu
    container:
-      image: zjay437/rocksdb:0.6
+      image: ghcr.io/facebook/rocksdb_ubuntu:24.0
      options: --shm-size=16gb
    steps:
    - uses: actions/checkout@v4.1.0
    - uses: "./.github/actions/pre-steps"
-    - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze
+    - run: CC=clang-18 CXX=clang++-18 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-18" CLANG_SCAN_BUILD=scan-build-18 USE_CLANG=1 make V=1 -j32 analyze
    - uses: "./.github/actions/post-steps"
    - name: compress test report
      run: tar -cvzf scan_build_report.tar.gz scan_build_report
@@ -300,7 +281,7 @@ jobs:
    runs-on:
      labels: 4-core-ubuntu
    container:
-      image: zjay437/rocksdb:0.6
+      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
      options: --shm-size=16gb
    steps:
    - uses: actions/checkout@v4.1.0
@@ -308,41 +289,41 @@ jobs:
    - run: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=960 --max_key=2500000' blackbox_crash_test_with_atomic_flush
    - uses: "./.github/actions/post-steps"
  # ======================= Linux with Sanitizers ===================== #
-  build-linux-clang10-asan-ubsan:
+  build-linux-clang18-asan-ubsan:
    if: ${{ github.repository_owner == 'facebook' }}
    runs-on:
      labels: 32-core-ubuntu
    container:
-      image: zjay437/rocksdb:0.6
+      image: ghcr.io/facebook/rocksdb_ubuntu:24.0
      options: --shm-size=16gb
    steps:
    - uses: actions/checkout@v4.1.0
    - uses: "./.github/actions/pre-steps"
-    - run: COMPILE_WITH_ASAN=1 COMPILE_WITH_UBSAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j40 check
+    - run: COMPILE_WITH_ASAN=1 COMPILE_WITH_UBSAN=1 CC=clang-18 CXX=clang++-18 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j40 check
    - uses: "./.github/actions/post-steps"
-  build-linux-clang13-mini-tsan:
+  build-linux-clang18-mini-tsan:
    if: ${{ github.repository_owner == 'facebook' }}
    runs-on:
      labels: 32-core-ubuntu
    container:
-      image: zjay437/rocksdb:0.6
+      image: ghcr.io/facebook/rocksdb_ubuntu:24.0
      options: --shm-size=16gb
    steps:
    - uses: actions/checkout@v4.1.0
    - uses: "./.github/actions/pre-steps"
-    - run: COMPILE_WITH_TSAN=1 CC=clang-13 CXX=clang++-13 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check
+    - run: COMPILE_WITH_TSAN=1 CC=clang-18 CXX=clang++-18 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check
    - uses: "./.github/actions/post-steps"
  build-linux-static_lib-alt_namespace-status_checked:
    if: ${{ github.repository_owner == 'facebook' }}
    runs-on:
      labels: 16-core-ubuntu
    container:
-      image: zjay437/rocksdb:0.6
+      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
      options: --shm-size=16gb
    steps:
    - uses: actions/checkout@v4.1.0
    - uses: "./.github/actions/pre-steps"
-    - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=static OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j24 check
+    - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=static OPT="-DROCKSDB_USE_STD_SEMAPHORES -DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j24 check
    - uses: "./.github/actions/post-steps"
  # ========================= MacOS build only ======================== #
  build-macos:
@@ -404,11 +385,13 @@ jobs:
    runs-on:
      labels: 4-core-ubuntu
    container:
-      image: evolvedbinary/rocksjava:centos7_x64-be
+      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
      options: --shm-size=16gb
    steps:
    # The docker image is intentionally based on an OS that has an older GLIBC version.
    # That GLIBC is incompatibile with GitHub's actions/checkout. Thus we implement a manual checkout step.
+    # NOTE: replaced evolvedbinary/rocksjava:centos7_x64-be with ghcr.io/facebook/rocksdb_ubuntu:22.1
+    # until a more appropriate docker image with C++20 support is made.
    - name: Checkout
      env:
        GH_TOKEN: ${{ github.token }}
@@ -425,18 +408,21 @@ jobs:
        which java && java -version
        which javac && javac -version
    - name: Test RocksDBJava
-      run: scl enable devtoolset-7 'make V=1 J=8 -j8 jtest'
-    # NOTE: post-steps skipped because of compatibility issues with docker image
+    # NOTE: replaced scl enable devtoolset-7 'make V=1 J=8 -j8 jtest'
+      run: make V=1 J=8 -j8 jtest
+    # post-steps skipped because of compatibility issues with docker image
  build-linux-java-static:
    if: ${{ github.repository_owner == 'facebook' }}
    runs-on:
      labels: 4-core-ubuntu
    container:
-      image: evolvedbinary/rocksjava:centos7_x64-be
+      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
      options: --shm-size=16gb
    steps:
    # The docker image is intentionally based on an OS that has an older GLIBC version.
    # That GLIBC is incompatibile with GitHub's actions/checkout. Thus we implement a manual checkout step.
+    # NOTE: replaced evolvedbinary/rocksjava:centos7_x64-be with ghcr.io/facebook/rocksdb_ubuntu:22.1
+    # until a more appropriate docker image with C++20 support is made.
    - name: Checkout
      env:
        GH_TOKEN: ${{ github.token }}
@@ -453,8 +439,9 @@ jobs:
        which java && java -version
        which javac && javac -version
    - name: Build RocksDBJava Static Library
-      run: scl enable devtoolset-7 'make V=1 J=8 -j8 rocksdbjavastatic'
-    # NOTE: post-steps skipped because of compatibility issues with docker image
+    # NOTE: replaced scl enable devtoolset-7 'make V=1 J=8 -j8 rocksdbjavastatic'
+      run: make V=1 J=8 -j8 rocksdbjavastatic
+    # post-steps skipped because of compatibility issues with docker image
  build-macos-java:
    if: ${{ github.repository_owner == 'facebook' }}
    runs-on: macos-13
@@ -419,6 +419,8 @@ cpp_library_wrapper(name="rocksdb_tools_lib", srcs=[

 cpp_library_wrapper(name="rocksdb_cache_bench_tools_lib", srcs=["cache/cache_bench_tool.cc"], deps=[":rocksdb_lib"], headers=[], link_whole=False, extra_test_libs=False)

+cpp_library_wrapper(name="rocksdb_point_lock_bench_tools_lib", srcs=["utilities/transactions/lock/point/point_lock_bench_tool.cc"], deps=[":rocksdb_lib"], headers=[], link_whole=False, extra_test_libs=False)
+
 rocks_cpp_library_wrapper(name="rocksdb_stress_lib", srcs=[
        "db_stress_tool/batched_ops_stress.cc",
        "db_stress_tool/cf_consistency_stress.cc",
@@ -429,7 +431,6 @@ rocks_cpp_library_wrapper(name="rocksdb_stress_lib", srcs=[
        "db_stress_tool/db_stress_gflags.cc",
        "db_stress_tool/db_stress_listener.cc",
        "db_stress_tool/db_stress_shared_state.cc",
-        "db_stress_tool/db_stress_stat.cc",
        "db_stress_tool/db_stress_test_base.cc",
        "db_stress_tool/db_stress_tool.cc",
        "db_stress_tool/db_stress_wide_merge_operator.cc",
@@ -451,6 +452,8 @@ cpp_binary_wrapper(name="db_bench", srcs=["tools/db_bench.cc"], deps=[":rocksdb_

 cpp_binary_wrapper(name="cache_bench", srcs=["cache/cache_bench.cc"], deps=[":rocksdb_cache_bench_tools_lib"], extra_preprocessor_flags=[], extra_bench_libs=False)

+cpp_binary_wrapper(name="point_lock_bench", srcs=["utilities/transactions/lock/point/point_lock_bench.cc"], deps=[":rocksdb_point_lock_bench_tools_lib"], extra_preprocessor_flags=[], extra_bench_libs=False)
+
 cpp_binary_wrapper(name="ribbon_bench", srcs=["microbench/ribbon_bench.cc"], deps=[], extra_preprocessor_flags=[], extra_bench_libs=True)

 cpp_binary_wrapper(name="db_basic_bench", srcs=["microbench/db_basic_bench.cc"], deps=[], extra_preprocessor_flags=[], extra_bench_libs=True)
@@ -5382,6 +5385,12 @@ cpp_unittest_wrapper(name="plain_table_db_test",
            extra_compiler_flags=[])


+cpp_unittest_wrapper(name="point_lock_manager_stress_test",
+            srcs=["utilities/transactions/lock/point/point_lock_manager_stress_test.cc"],
+            deps=[":rocksdb_test_lib"],
+            extra_compiler_flags=[])
+
+
 cpp_unittest_wrapper(name="point_lock_manager_test",
            srcs=["utilities/transactions/lock/point/point_lock_manager_test.cc"],
            deps=[":rocksdb_test_lib"],
@@ -27,7 +27,7 @@
 #
 # Linux:
 #
-# 1. Install a recent toolchain if you're on a older distro. C++17 required (GCC >= 7, Clang >= 5)
+# 1. Install a recent toolchain if you're on a older distro. C++20 required (GCC >= 11, Clang >= 10)
 # 2. mkdir build; cd build
 # 3. cmake ..
 # 4. make -j
@@ -100,7 +100,7 @@ endif()
 option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" ON)

 if( NOT DEFINED CMAKE_CXX_STANDARD )
-  set(CMAKE_CXX_STANDARD 17)
+  set(CMAKE_CXX_STANDARD 20)
 endif()

 include(CMakeDependentOption)
@@ -314,7 +314,6 @@ endif()

 # Check if -latomic is required or not
 if (NOT MSVC)
-  set(CMAKE_REQUIRED_FLAGS "--std=c++17")
  CHECK_CXX_SOURCE_COMPILES("
 #include <atomic>
 std::atomic<uint64_t> x(0);
@@ -1504,6 +1503,7 @@ if(WITH_TESTS)
        utilities/transactions/optimistic_transaction_test.cc
        utilities/transactions/transaction_test.cc
        utilities/transactions/lock/point/point_lock_manager_test.cc
+        utilities/transactions/lock/point/point_lock_manager_stress_test.cc
        utilities/transactions/write_committed_transaction_ts_test.cc
        utilities/transactions/write_prepared_transaction_test.cc
        utilities/transactions/write_unprepared_transaction_test.cc
@@ -1614,6 +1614,12 @@ if(WITH_BENCHMARK_TOOLS)
    utilities/persistent_cache/hash_table_bench.cc)
  target_link_libraries(hash_table_bench${ARTIFACT_SUFFIX}
    ${ROCKSDB_LIB} ${GFLAGS_LIB} ${FOLLY_LIBS})
+
+  add_executable(point_lock_bench${ARTIFACT_SUFFIX}
+    utilities/transactions/lock/point/point_lock_bench.cc
+    utilities/transactions/lock/point/point_lock_bench_tool.cc)
+  target_link_libraries(point_lock_bench${ARTIFACT_SUFFIX}
+    ${ROCKSDB_LIB} ${GFLAGS_LIB} ${FOLLY_LIBS})
 endif()

 option(WITH_TRACE_TOOLS "build with trace tools" ON)
@@ -1,6 +1,85 @@
 # Rocksdb Change Log
 > NOTE: Entries for next release do not go here. Follow instructions in `unreleased_history/README.txt`

+## 10.7.6 (10/23/2025)
+* Fix a bug when del range and data are in same file for multi-scan
+
+## 10.7.5 (10/20/2025)
+### Bug Fixes
+* Fix a bug in Page unpinning in MultiScan
+
+## 10.7.4 (10/14/2025)
+### Public API Changes
+* The MultiScan API contract is updated. After a multi scan range got prepared with Prepare API call, the following seeks must seek the start of each prepared scan range in order. In addition, when limit is set, upper bound must be set to the same value of limit before each seek
+
+## 10.7.3 (10/06/2025)
+### Bug Fixes
+* Fix a few bugs in MultiScan
+
+## 10.7.2 (09/30/2025)
+### Bug Fixes
+* Fix incorrect MultiScan seek error status due to bugs in handling range limit falling between adjacent SST files key range.
+
+### Performance Improvements
+* Fixed a performance regression in LZ4 compression that started in version 10.6.0
+
+## 10.7.0 (09/24/2025)
+### New Features
+* Add the fail_if_no_udi_on_open flag in BlockBasedTableOption to control whether a missing user defined index block in a SST is a hard error or not.
+* A new flag memtable_veirfy_per_key_checksum_on_seek is added to AdvancedColumnFamilyOptions. When it is enabled, it will validate key checksum along the binary search path on skiplist based memtable during seek operation.
+* Introduce option MultiScanArgs::use_async_io to enable asynchronous I/O during MultiScan, instead of waiting for I/O to be done in Prepare().
+* Add new option `MultiScanArgs::max_prefetch_size` that limits the memory usage of per file pinning of prefetched blocks.
+* Improved `sst_dump` by allowing standalone file and directory arguments without `--file=`. Also added new options and better output for `sst_dump --command=recompress`. See `sst_dump --help`
+
+### Public API Changes
+* HyperClockCache with no `estimated_entry_charge` is now production-ready and is the preferred block cache implementation vs. LRUCache. Please consider updating your code to minimize the risk of hitting performance bottlenecks or anomalies from LRUCache. See cache.h for more detail.
+* RocksDB now requires a C++20 compatible compiler (GCC >= 11, Clang >= 10, Visual Studio >= 2019), including for any code using RocksDB headers.
+* MultiScanArgs used to have a default constructor with default parameter of BytewiseComparator. Now it always requires Comparator in its constructor.
+
+### Behavior Changes
+* The default provided block cache implementation is now HyperClockCache instead of LRUCache, when `block_cache` is nullptr (default) and `no_block_cache==false` (default). We recommend explicitly creating a HyperClockCache block cache based on memory budget and sharing it across all column families and even DB instances. This change could expose previously hidden memory or resource leaks.
+* Allow UDIs with a non BytewiseComparator
+
+### Bug Fixes
+* Reported numbers for compaction and flush CPU usage now include time spent by parallel compression worker threads. This now means compaction/flush CPU usage could exceed the wall clock time.
+* Fix a race condition in FIFO size-based compaction where concurrent threads could select the same non-L0 file, causing assertion failures in debug builds or "Cannot delete table file from LSM tree" errors in release builds.
+* Fix a bug in RocksDB MultiScan with UDI when one of the scan ranges is determined to be empty by the UDI, which causes incorrect results.
+
+### Performance Improvements
+* Add a new table property "rocksdb.key.smallest.seqno" which records the smallest sequence number of all keys in file. It makes ingesting DB generated files faster by
+avoiding scanning the whole file to find the smallest sequence number.
+* Add a new experimental PerKeyPointLockManager to improve efficiency under high lock contention. PointLockManager was not efficient when there is high write contention on same key, as it uses a single conditional variable per lock stripe. PerKeyPointLockManager uses per thread conditional variable supporting fifo order. Although this is an experimental feature. By default, it is disabled. A new boolean flag TransactionDBOptions::use_per_key_point_lock_mgr is added to optionally enable it. Search the flag in code for more info.
+Together, a new configuration TransactionOptions::deadlock_timeout_us is added, which allows the transaction to wait for a short period before perform deadlock detection. When the workload has low lock contention, the deadlock_timeout_us can be configured to be slightly higher than average transaction execution time, so that transaction would likely be able to take the lock before deadlock detection is performed when it is waiting for a lock. This allows transaction to reduce CPU cost on performing deadlock detection, which could be expensive in CPU time. When the workload has high lock contention, the deadlock_timeout_us can be configured to 0, so that transaction would perform deadlock detection immediately. By default the value is 0 to keep the behavior same as before.
+* Majorly improved CPU efficiency and scalability of parallel compression (`CompressionOptions::parallel_threads` > 1), though this efficiency improvement makes parallel compression currently incompatible with UserDefinedIndex and with old setting of `decouple_partitioned_filters=false`. Parallel compression is now considered a production-ready feature. Maximum performance is available with `-DROCKSDB_USE_STD_SEMAPHORES` at compile time, but this is not currently recommended because of reported bugs in implementations of `std::counting_semaphore`/`binary_semaphore`.
+
+## 10.6.0 (08/22/2025)
+### New Features
+* Introduce column family option `cf_allow_ingest_behind`. This option aims to replace `DBOptions::allow_ingest_behind` to enable ingest behind at the per-CF level. `DBOptions::allow_ingest_behind` is deprecated.
+* Introduce `MultiScanArgs::io_coalesce_threshold` to allow a configurable IO coalescing threshold.
+
+### Public API Changes
+* `IngestExternalFileOptions::allow_db_generated_files` now allows files ingestion of any DB generated SST file, instead of only the ones with all keys having sequence number 0.
+* `decouple_partitioned_filters = true` is now the default in BlockBasedTableOptions.
+* GetTtl() API is now available in TTL DB
+* Minimum supported version of LZ4 library is now 1.7.0 (r129 from 2015)
+* Some changes to experimental Compressor and CompressionManager APIs
+* A new Filesystem::SyncFile function is added for syncing a file that was already written, such as on file ingestion. The default implementation matches previous RocksDB behavior: re-open the file for read-write, sync it, and close it. We recommend overriding for FileSystems that do not require syncing for crash recovery or do not handle (well) re-opening for writes.
+
+### Behavior Changes
+* When `allow_ingest_behind` is enabled, compaction will no longer drop tombstones based on the absence of underlying data. Tombstones will be preserved to apply to ingested files.
+
+### Bug Fixes
+* Files in dropped column family won't be returned to the caller upon successful, offline MANIFEST iteration in `GetFileChecksumsFromCurrentManifest`.
+* Fix a bug in MultiScan that causes it to fall back to a normal scan when dictionary compression is enabled.
+* Fix a crash in iterator Prepare() when fill_cache=false
+* Fix a bug in MultiScan where incorrect results can be returned when a Scan's range is across multiple files.
+* Fixed a bug in remote compaction that may mistakenly delete live SST file(s) during the cleanup phase when no keys survive the compaction (all expired)
+* Allow a user defined index to be configured from a string.
+* Make the User Defined Index interface consistently use the user key format, fixing the previous mixed usage of internal and user key.
+
+### Performance Improvements
+* Small improvement to CPU efficiency of compression using built-in algorithms, and a dramatic efficiency improvement for LZ4HC, based on reusing data structures between invocations.
+
 ## 10.5.0 (07/18/2025)
 ### Public API Changes
 * DB option skip_checking_sst_file_sizes_on_db_open is deprecated, in favor of validating file size in parallel in a thread pool, when db is opened. When DB is opened, with paranoid check enabled, a file with the wrong size would fail the DB open. With paranoid check disabled, the DB open would succeed, the column family with the corrupted file would not be read or write, while the other healthy column families could be read and write normally. When max_open_files option is not set to -1, only a subset of the files will be opened and checked. The rest of the files will be opened and checked when they are accessed.
@@ -6,7 +6,7 @@ than release mode.

 RocksDB's library should be able to compile without any dependency installed,
 although we recommend installing some compression libraries (see below).
-We do depend on newer gcc/clang with C++17 support (GCC >= 7, Clang >= 5).
+We do depend on newer gcc/clang with C++20 support (GCC >= 11, Clang >= 10).

 There are few options when compiling RocksDB:

@@ -60,7 +60,7 @@ most processors made since roughly 2013.
 ## Supported platforms

 * **Linux - Ubuntu**
-    * Upgrade your gcc to version at least 7 to get C++17 support.
+    * Upgrade your gcc to version at least 11 to get C++20 support.
    * Install gflags. First, try: `sudo apt-get install libgflags-dev`
      If this doesn't work and you're using Ubuntu, here's a nice tutorial:
      (http://askubuntu.com/questions/312173/installing-gflags-12-04)
@@ -72,7 +72,7 @@ most processors made since roughly 2013.
    * Install zstandard: `sudo apt-get install libzstd-dev`.

 * **Linux - CentOS / RHEL**
-    * Upgrade your gcc to version at least 7 to get C++17 support
+    * Upgrade your gcc to version at least 11 to get C++20 support
    * Install gflags:

              git clone https://github.com/gflags/gflags.git
@@ -122,7 +122,7 @@ most processors made since roughly 2013.
              make && sudo make install

 * **OS X**:
-    * Install latest C++ compiler that supports C++ 17:
+    * Install latest C++ compiler that supports C++20:
        * Update XCode:  run `xcode-select --install` (or install it from XCode App's settting).
        * Install via [homebrew](http://brew.sh/).
            * If you're first time developer in MacOS, you still need to run: `xcode-select --install` in your command line.
@@ -213,7 +213,7 @@ most processors made since roughly 2013.
             export PATH=/opt/freeware/bin:$PATH

 * **Solaris Sparc**
-    * Install GCC 7 and higher.
+    * Install GCC 11 and higher.
    * Use these environment variables:

             export CC=gcc
@@ -148,10 +148,8 @@ ifeq ($(USE_COROUTINES), 1)
 	USE_FOLLY = 1
 	# glog/logging.h requires HAVE_CXX11_ATOMIC
 	OPT += -DUSE_COROUTINES -DHAVE_CXX11_ATOMIC
-	ROCKSDB_CXX_STANDARD = c++2a
 	USE_RTTI = 1
 ifneq ($(USE_CLANG), 1)
-	ROCKSDB_CXX_STANDARD = c++20
 	PLATFORM_CXXFLAGS += -fcoroutines
 endif
 endif
@@ -638,13 +636,14 @@ endif
 TEST_OBJECTS = $(patsubst %.cc, $(OBJ_DIR)/%.o, $(TEST_LIB_SOURCES) $(MOCK_LIB_SOURCES)) $(GTEST)
 BENCH_OBJECTS = $(patsubst %.cc, $(OBJ_DIR)/%.o, $(BENCH_LIB_SOURCES))
 CACHE_BENCH_OBJECTS = $(patsubst %.cc, $(OBJ_DIR)/%.o, $(CACHE_BENCH_LIB_SOURCES))
+POINT_LOCK_BENCH_OBJECTS = $(patsubst %.cc, $(OBJ_DIR)/%.o, $(POINT_LOCK_BENCH_LIB_SOURCES))
 TOOL_OBJECTS = $(patsubst %.cc, $(OBJ_DIR)/%.o, $(TOOL_LIB_SOURCES))
 ANALYZE_OBJECTS = $(patsubst %.cc, $(OBJ_DIR)/%.o, $(ANALYZER_LIB_SOURCES))
 STRESS_OBJECTS =  $(patsubst %.cc, $(OBJ_DIR)/%.o, $(STRESS_LIB_SOURCES))

 # Exclude build_version.cc -- a generated source file -- from all sources.  Not needed for dependencies
 ALL_SOURCES  = $(filter-out util/build_version.cc, $(LIB_SOURCES)) $(TEST_LIB_SOURCES) $(MOCK_LIB_SOURCES) $(GTEST_DIR)/gtest/gtest-all.cc
-ALL_SOURCES += $(TOOL_LIB_SOURCES) $(BENCH_LIB_SOURCES) $(CACHE_BENCH_LIB_SOURCES) $(ANALYZER_LIB_SOURCES) $(STRESS_LIB_SOURCES)
+ALL_SOURCES += $(TOOL_LIB_SOURCES) $(BENCH_LIB_SOURCES) $(CACHE_BENCH_LIB_SOURCES) $(POINT_LOCK_BENCH_LIB_SOURCES) $(ANALYZER_LIB_SOURCES) $(STRESS_LIB_SOURCES)
 ALL_SOURCES += $(TEST_MAIN_SOURCES) $(TOOL_MAIN_SOURCES) $(BENCH_MAIN_SOURCES)
 ALL_SOURCES += $(ROCKSDB_PLUGIN_SOURCES) $(ROCKSDB_PLUGIN_TESTS)

@@ -683,7 +682,7 @@ am__v_CCH_1 =
 # user build settings
 %.h.pub: %.h # .h.pub not actually created, so re-checked on each invocation
 	$(AM_V_CCH) cd include/ && echo '#include "$(patsubst include/%,%,$<)"' | \
-	  $(CXX) -I. -DROCKSDB_NAMESPACE=42 -x c++ -c - -o /dev/null
+	  $(CXX) -std=$(or $(ROCKSDB_CXX_STANDARD),c++20) -I. -DROCKSDB_NAMESPACE=42 -x c++ -c - -o /dev/null

 check-headers: $(HEADER_OK_FILES)

@@ -1345,6 +1344,9 @@ block_cache_trace_analyzer: $(OBJ_DIR)/tools/block_cache_analyzer/block_cache_tr
 cache_bench: $(OBJ_DIR)/cache/cache_bench.o $(CACHE_BENCH_OBJECTS) $(LIBRARY)
 	$(AM_LINK)

+point_lock_bench: $(OBJ_DIR)/utilities/transactions/lock/point/point_lock_bench.o $(POINT_LOCK_BENCH_OBJECTS) $(LIBRARY)
+	$(AM_LINK)
+
 persistent_cache_bench: $(OBJ_DIR)/utilities/persistent_cache/persistent_cache_bench.o $(LIBRARY)
 	$(AM_LINK)

@@ -1425,13 +1427,13 @@ agg_merge_test: $(OBJ_DIR)/utilities/agg_merge/agg_merge_test.o $(TEST_LIBRARY)
 stringappend_test: $(OBJ_DIR)/utilities/merge_operators/string_append/stringappend_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)

-cassandra_format_test: $(OBJ_DIR)/utilities/cassandra/cassandra_format_test.o $(OBJ_DIR)/utilities/cassandra/test_utils.o $(TEST_LIBRARY) $(LIBRARY)
+cassandra_format_test: $(OBJ_DIR)/utilities/cassandra/cassandra_format_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)

-cassandra_functional_test: $(OBJ_DIR)/utilities/cassandra/cassandra_functional_test.o $(OBJ_DIR)/utilities/cassandra/test_utils.o $(TEST_LIBRARY) $(LIBRARY)
+cassandra_functional_test: $(OBJ_DIR)/utilities/cassandra/cassandra_functional_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)

-cassandra_row_merge_test: $(OBJ_DIR)/utilities/cassandra/cassandra_row_merge_test.o $(OBJ_DIR)/utilities/cassandra/test_utils.o $(TEST_LIBRARY) $(LIBRARY)
+cassandra_row_merge_test: $(OBJ_DIR)/utilities/cassandra/cassandra_row_merge_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)

 cassandra_serialize_test: $(OBJ_DIR)/utilities/cassandra/cassandra_serialize_test.o $(TEST_LIBRARY) $(LIBRARY)
@@ -1881,6 +1883,9 @@ heap_test: $(OBJ_DIR)/util/heap_test.o $(TEST_LIBRARY) $(LIBRARY)
 point_lock_manager_test: utilities/transactions/lock/point/point_lock_manager_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)

+point_lock_manager_stress_test: utilities/transactions/lock/point/point_lock_manager_stress_test.o $(TEST_LIBRARY) $(LIBRARY)
+	$(AM_LINK)
+
 transaction_test: $(OBJ_DIR)/utilities/transactions/transaction_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)

@@ -2251,7 +2256,7 @@ libsnappy.a: snappy-$(SNAPPY_VER).tar.gz
 	-rm -rf snappy-$(SNAPPY_VER)
 	tar xvzf snappy-$(SNAPPY_VER).tar.gz
 	mkdir snappy-$(SNAPPY_VER)/build
-	cd snappy-$(SNAPPY_VER)/build && CFLAGS='$(ARCHFLAG) ${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' CXXFLAGS='$(ARCHFLAG) ${JAVA_STATIC_DEPS_CXXFLAGS} ${EXTRA_CXXFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DSNAPPY_BUILD_BENCHMARKS=OFF -DSNAPPY_BUILD_TESTS=OFF --compile-no-warning-as-error ${PLATFORM_CMAKE_FLAGS} .. && $(MAKE) ${SNAPPY_MAKE_TARGET}
+	cd snappy-$(SNAPPY_VER)/build && CFLAGS='$(ARCHFLAG) ${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' CXXFLAGS='$(ARCHFLAG) ${JAVA_STATIC_DEPS_CXXFLAGS} ${EXTRA_CXXFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DSNAPPY_BUILD_BENCHMARKS=OFF -DSNAPPY_BUILD_TESTS=OFF ${PLATFORM_CMAKE_FLAGS} .. && $(MAKE) ${SNAPPY_MAKE_TARGET}
 	cp snappy-$(SNAPPY_VER)/build/libsnappy.a .

 lz4-$(LZ4_VER).tar.gz:
@@ -206,6 +206,12 @@ def generate_buck(repo_path, deps_map):
        src_mk.get("CACHE_BENCH_LIB_SOURCES", []),
        [":rocksdb_lib"],
    )
+    # rocksdb_point_lock_bench_tools_lib
+    BUCK.add_library(
+        "rocksdb_point_lock_bench_tools_lib",
+        src_mk.get("POINT_LOCK_BENCH_LIB_SOURCES", []),
+        [":rocksdb_lib"],
+    )
    # rocksdb_stress_lib
    BUCK.add_rocksdb_library(
        "rocksdb_stress_lib",
@@ -229,6 +235,12 @@ def generate_buck(repo_path, deps_map):
    BUCK.add_binary(
        "cache_bench", ["cache/cache_bench.cc"], [":rocksdb_cache_bench_tools_lib"]
    )
+    # point_lock_bench binary
+    BUCK.add_binary(
+        "point_lock_bench",
+        ["utilities/transactions/lock/point/point_lock_bench.cc"],
+        [":rocksdb_point_lock_bench_tools_lib"]
+    )
    # bench binaries
    for src in src_mk.get("MICROBENCH_SOURCES", []):
        name = src.rsplit("/", 1)[1].split(".")[0] if "/" in src else src.split(".")[0]
@@ -45,11 +45,11 @@ if test -z "$OUTPUT"; then
  exit 1
 fi

-# we depend on C++17, but should be compatible with newer standards
+# we depend on C++20, but should be compatible with newer standards
 if [ "$ROCKSDB_CXX_STANDARD" ]; then
  PLATFORM_CXXFLAGS="-std=$ROCKSDB_CXX_STANDARD"
 else
-  PLATFORM_CXXFLAGS="-std=c++17"
+  PLATFORM_CXXFLAGS="-std=c++20"
 fi

 # we currently depend on POSIX platform
@@ -0,0 +1,88 @@
+# INSTRUCTIONS:
+# I was not able to build docker images on an isolated devserver because of
+# issues with proxy internet access. Use a public cloud or other Linux system.
+# (I used a Debian system after installing docker features, adding my user to
+# the docker and docker-registry groups, and logging out and back in to pick
+# those up.)
+#
+# Follow https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry#authenticating-with-a-personal-access-token-classic
+# to login with your GitHub credentials, as in
+#
+# $ docker login ghcr.io -u pdillinger
+#
+# and paste the limited-purpose GitHub token into the terminal.
+#
+# Then in the build_tools/ubuntu22_image directory, (bump minor version for
+# random docker file updates, major version tracks Ubuntu release)
+#
+# $ docker build -t ghcr.io/facebook/rocksdb_ubuntu:22.0
+# $ docker push ghcr.io/facebook/rocksdb_ubuntu:22.0
+#
+# Might need to change visibility to public through
+# https://github.com/orgs/facebook/packages/container/rocksdb_ubuntu/settings
+# or similar.
+
+# from official ubuntu 22.04
+FROM ubuntu:22.04
+# update system
+RUN apt-get update
+RUN apt-get upgrade -y
+# install basic tools
+RUN apt-get install -y vim wget curl
+# install tzdata noninteractive
+RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata
+# install git and default compilers
+RUN apt-get install -y git gcc g++ clang clang-tools
+# install basic package
+RUN apt-get install -y lsb-release software-properties-common gnupg
+# install gflags, tbb
+RUN apt-get install -y libgflags-dev libtbb-dev
+# install compression libs
+RUN apt-get install -y libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev
+# install cmake
+RUN apt-get install -y cmake
+RUN apt-get install -y libssl-dev
+# install clang-13
+WORKDIR /root
+RUN wget https://apt.llvm.org/llvm.sh
+RUN chmod +x llvm.sh
+RUN ./llvm.sh 13 all
+# There are incompatibilities between clang with -std=c++20 and libstdc++
+# provided by gcc, so we have to compile with clang-13 using -stdlib=libc++
+# and only one version of libc++ can be installed on the system at one time.
+# So to avoid confusion we remove unusable clang-14 also.
+RUN apt-get install libc++-13-dev libc++abi-13-dev
+RUN apt-get purge -y clang-14 && apt-get autoremove -y
+
+# install gcc-10 and more, default is 11
+RUN apt-get install -y gcc-10 g++-10
+RUN add-apt-repository -y ppa:ubuntu-toolchain-r/test
+RUN apt-get install -y gcc-13 g++-13
+# install apt-get install -y valgrind
+RUN apt-get install -y valgrind
+# install folly depencencies
+# Missing compatible libunwind: RUN apt-get install -y libgoogle-glog-dev
+# So instead install from source. This currently requires compiling with
+# -DGLOG_USE_GLOG_EXPORT
+RUN wget https://github.com/google/glog/archive/refs/tags/v0.7.1.tar.gz && tar xzf v0.7.1.tar.gz && cd glog-0.7.1/ && cmake -S . -B build -G "Unix Makefiles" && cmake --build build && cmake --build build --target install && cd .. && rm -rf v0.7.1.tar.gz glog-0.7.1
+# install openjdk 8
+RUN apt-get install -y openjdk-8-jdk
+ENV JAVA_HOME /usr/lib/jvm/java-1.8.0-openjdk-amd64
+# install mingw
+RUN apt-get install -y mingw-w64
+
+# install gtest-parallel package
+RUN git clone --single-branch --branch master --depth 1 https://github.com/google/gtest-parallel.git ~/gtest-parallel
+ENV PATH $PATH:/root/gtest-parallel
+
+# install libprotobuf for fuzzers test
+RUN apt-get install -y ninja-build binutils liblzma-dev libz-dev pkg-config autoconf libtool
+RUN git clone --branch v1.0 https://github.com/google/libprotobuf-mutator.git ~/libprotobuf-mutator && cd ~/libprotobuf-mutator && git checkout ffd86a32874e5c08a143019aad1aaf0907294c9f && mkdir build && cd build && cmake .. -GNinja -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang++-13 -DCMAKE_BUILD_TYPE=Release -DLIB_PROTO_MUTATOR_DOWNLOAD_PROTOBUF=ON && ninja && ninja install
+ENV PKG_CONFIG_PATH /usr/local/OFF/:/root/libprotobuf-mutator/build/external.protobuf/lib/pkgconfig/
+ENV PROTOC_BIN /root/libprotobuf-mutator/build/external.protobuf/bin/protoc
+
+# install the latest google benchmark
+RUN git clone --depth 1 --branch v1.7.0 https://github.com/google/benchmark.git ~/benchmark && cd ~/benchmark && mkdir build && cd build && cmake .. -GNinja -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_GTEST_TESTS=0 && ninja && ninja install && cd ~ && rm -rf /root/benchmark
+
+# clean up
+RUN rm -rf /var/lib/apt/lists/*
@@ -0,0 +1,72 @@
+# INSTRUCTIONS:
+# I was not able to build docker images on an isolated devserver because of
+# issues with proxy internet access. Use a public cloud or other Linux system.
+# (I used a Debian system after installing docker features, adding my user to
+# the docker and docker-registry groups, and logging out and back in to pick
+# those up.)
+#
+# Follow https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry#authenticating-with-a-personal-access-token-classic
+# to login with your GitHub credentials, as in
+#
+# $ docker login ghcr.io -u pdillinger
+#
+# and paste the limited-purpose GitHub token into the terminal.
+#
+# Then in the build_tools/ubuntu24_image directory, (bump minor version for
+# random docker file updates, major version tracks Ubuntu release)
+#
+# $ docker build -t ghcr.io/facebook/rocksdb_ubuntu:24.0
+# $ docker push ghcr.io/facebook/rocksdb_ubuntu:24.0
+#
+# Might need to change visibility to public through
+# https://github.com/orgs/facebook/packages/container/rocksdb_ubuntu/settings
+# or similar.
+
+# from official ubuntu 24.04
+FROM ubuntu:24.04
+# update system
+RUN apt-get update
+RUN apt-get upgrade -y
+# install basic tools
+RUN apt-get install -y vim wget curl
+# install tzdata noninteractive
+RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata
+# install git and default compilers
+RUN apt-get install -y git gcc g++ clang clang-tools
+# install basic package
+RUN apt-get install -y lsb-release software-properties-common gnupg
+# install gflags, tbb
+RUN apt-get install -y libgflags-dev libtbb-dev
+# install compression libs
+RUN apt-get install -y libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev
+# install cmake
+RUN apt-get install -y cmake
+RUN apt-get install -y libssl-dev
+
+# install gcc-12 and more, default is 13
+RUN apt-get install -y gcc-12 g++-12 gcc-14 g++-14
+# install apt-get install -y valgrind
+RUN apt-get install -y valgrind
+# install folly depencencies
+RUN apt-get install -y libgoogle-glog-dev
+# install openjdk 8
+RUN apt-get install -y openjdk-8-jdk
+ENV JAVA_HOME /usr/lib/jvm/java-1.8.0-openjdk-amd64
+# install mingw
+RUN apt-get install -y mingw-w64
+
+# install gtest-parallel package
+RUN git clone --single-branch --branch master --depth 1 https://github.com/google/gtest-parallel.git ~/gtest-parallel
+ENV PATH $PATH:/root/gtest-parallel
+
+# install libprotobuf for fuzzers test
+RUN apt-get install -y ninja-build binutils liblzma-dev libz-dev pkg-config autoconf libtool
+RUN git clone --branch v1.0 https://github.com/google/libprotobuf-mutator.git ~/libprotobuf-mutator && cd ~/libprotobuf-mutator && git checkout ffd86a32874e5c08a143019aad1aaf0907294c9f && mkdir build && cd build && cmake .. -GNinja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=Release -DLIB_PROTO_MUTATOR_DOWNLOAD_PROTOBUF=ON && ninja && ninja install
+ENV PKG_CONFIG_PATH /usr/local/OFF/:/root/libprotobuf-mutator/build/external.protobuf/lib/pkgconfig/
+ENV PROTOC_BIN /root/libprotobuf-mutator/build/external.protobuf/bin/protoc
+
+# install the latest google benchmark
+RUN git clone --depth 1 --branch v1.7.0 https://github.com/google/benchmark.git ~/benchmark && cd ~/benchmark && mkdir build && cd build && cmake .. -GNinja -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_GTEST_TESTS=0 && ninja && ninja install && cd ~ && rm -rf /root/benchmark
+
+# clean up
+RUN rm -rf /var/lib/apt/lists/*
@@ -184,6 +184,11 @@ DEFINE_bool(sck_randomize, false,
 DEFINE_bool(sck_footer_unique_id, false,
            "(-stress_cache_key) Simulate using proposed footer unique id");
 // ## END stress_cache_key sub-tool options ##
+// ## BEGIN stress_cache_instances sub-tool options ##
+DEFINE_uint32(stress_cache_instances, 0,
+              "If > 0, run cache instance stress test instead");
+// Uses cache_size and cache_type, maybe more
+// ## END stress_cache_instance sub-tool options ##

 namespace ROCKSDB_NAMESPACE {

@@ -387,7 +392,12 @@ class CacheBench {
      fprintf(stderr, "Percentages must add to 100.\n");
      exit(1);
    }
+    cache_ = MakeCache();
+  }

+  ~CacheBench() = default;
+
+  static std::shared_ptr<Cache> MakeCache() {
    std::shared_ptr<MemoryAllocator> allocator;
    if (FLAGS_use_jemalloc_no_dump_allocator) {
      JemallocAllocatorOptions opts;
@@ -406,12 +416,12 @@ class CacheBench {
      opts.hash_seed = BitwiseAnd(FLAGS_seed, INT32_MAX);
      opts.memory_allocator = allocator;
      opts.eviction_effort_cap = FLAGS_eviction_effort_cap;
-      if (FLAGS_cache_type == "fixed_hyper_clock_cache" ||
-          FLAGS_cache_type == "hyper_clock_cache") {
+      if (FLAGS_cache_type == "fixed_hyper_clock_cache") {
        opts.estimated_entry_charge = FLAGS_value_bytes_estimate > 0
                                          ? FLAGS_value_bytes_estimate
                                          : FLAGS_value_bytes;
-      } else if (FLAGS_cache_type == "auto_hyper_clock_cache") {
+      } else if (FLAGS_cache_type == "auto_hyper_clock_cache" ||
+                 FLAGS_cache_type == "hyper_clock_cache") {
        if (FLAGS_value_bytes_estimate > 0) {
          opts.min_avg_entry_charge = FLAGS_value_bytes_estimate;
        }
@@ -420,7 +430,7 @@ class CacheBench {
        exit(1);
      }
      ConfigureSecondaryCache(opts);
-      cache_ = opts.MakeSharedCache();
+      return opts.MakeSharedCache();
    } else if (FLAGS_cache_type == "lru_cache") {
      LRUCacheOptions opts(FLAGS_cache_size, FLAGS_num_shard_bits,
                           false /* strict_capacity_limit */,
@@ -428,15 +438,13 @@ class CacheBench {
      opts.hash_seed = BitwiseAnd(FLAGS_seed, INT32_MAX);
      opts.memory_allocator = allocator;
      ConfigureSecondaryCache(opts);
-      cache_ = NewLRUCache(opts);
+      return NewLRUCache(opts);
    } else {
      fprintf(stderr, "Cache type not supported.\n");
      exit(1);
    }
  }

-  ~CacheBench() = default;
-
  void PopulateCache() {
    Random64 rnd(FLAGS_seed);
    KeyGen keygen;
@@ -490,7 +498,7 @@ class CacheBench {

    PrintEnv();
    SharedState shared(this);
-    std::vector<std::unique_ptr<ThreadState> > threads(FLAGS_threads);
+    std::vector<std::unique_ptr<ThreadState>> threads(FLAGS_threads);
    for (uint32_t i = 0; i < FLAGS_threads; i++) {
      threads[i].reset(new ThreadState(i, &shared));
      std::thread(ThreadBody, threads[i].get()).detach();
@@ -1152,6 +1160,59 @@ class StressCacheKey {
  double multiplier_ = 0.0;
 };

+// cache_bench -stress_cache_instances is a partially independent embedded tool
+// for evaluating the time and space required to create and destroy many cache
+// instances, as this is considered important for a default cache implementation
+// which could see many throw-away instances in handling of Options, or created
+// in large numbers for many very small DBs with many CFs. Prefix command line
+// with /usr/bin/time to see max RSS memory.
+class StressCacheInstances {
+ public:
+  void Run() {
+    const int kNumIterations = 10;
+    const auto clock = SystemClock::Default().get();
+    caches_.reserve(FLAGS_stress_cache_instances);
+
+    uint64_t total_create_time_us = 0;
+    uint64_t total_destroy_time_us = 0;
+
+    for (int iter = 0; iter < kNumIterations; ++iter) {
+      // Create many cache instances
+      uint64_t start_create = clock->NowMicros();
+      for (uint32_t i = 0; i < FLAGS_stress_cache_instances; ++i) {
+        caches_.emplace_back(CacheBench::MakeCache());
+      }
+      uint64_t end_create = clock->NowMicros();
+      uint64_t create_time = end_create - start_create;
+      total_create_time_us += create_time;
+
+      // Destroy them
+      uint64_t start_destroy = clock->NowMicros();
+      caches_.clear();
+      uint64_t end_destroy = clock->NowMicros();
+      uint64_t destroy_time = end_destroy - start_destroy;
+      total_destroy_time_us += destroy_time;
+
+      printf(
+          "Iteration %d: Created %u caches in %.3f ms, destroyed in %.3f ms\n",
+          iter + 1, FLAGS_stress_cache_instances, create_time / 1000.0,
+          destroy_time / 1000.0);
+    }
+
+    printf("Average creation time: %.3f ms (%.1f us per cache)\n",
+           static_cast<double>(total_create_time_us) / kNumIterations / 1000.0,
+           static_cast<double>(total_create_time_us) / kNumIterations /
+               FLAGS_stress_cache_instances);
+    printf("Average destruction time: %.3f ms (%.1f us per cache)\n",
+           static_cast<double>(total_destroy_time_us) / kNumIterations / 1000.0,
+           static_cast<double>(total_destroy_time_us) / kNumIterations /
+               FLAGS_stress_cache_instances);
+  }
+
+ private:
+  std::vector<std::shared_ptr<Cache>> caches_;
+};
+
 int cache_bench_tool(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ParseCommandLineFlags(&argc, &argv, true);
@@ -1162,6 +1223,11 @@ int cache_bench_tool(int argc, char** argv) {
    return 0;
  }

+  if (FLAGS_stress_cache_instances > 0) {
+    StressCacheInstances().Run();
+    return 0;
+  }
+
  if (FLAGS_threads <= 0) {
    fprintf(stderr, "threads number <= 0\n");
    exit(1);
@@ -10,12 +10,12 @@
 #include "cache/clock_cache.h"

 #include <algorithm>
-#include <atomic>
 #include <bitset>
 #include <cassert>
 #include <cinttypes>
 #include <cstddef>
 #include <cstdint>
+#include <cstdio>
 #include <exception>
 #include <functional>
 #include <numeric>
@@ -26,10 +26,9 @@
 #include "cache/cache_key.h"
 #include "cache/secondary_cache_adapter.h"
 #include "logging/logging.h"
-#include "monitoring/perf_context_imp.h"
-#include "monitoring/statistics_impl.h"
-#include "port/lang.h"
+#include "port/likely.h"
 #include "rocksdb/env.h"
+#include "util/autovector.h"
 #include "util/hash.h"
 #include "util/math.h"
 #include "util/random.h"
@@ -361,16 +360,9 @@ void ConstApplyToEntriesRange(const Func& func, const HandleImpl* begin,
  }
 }

-constexpr uint32_t kStrictCapacityLimitBit = 1u << 31;
-
-uint32_t SanitizeEncodeEecAndScl(int eviction_effort_cap,
-                                 bool strict_capacit_limit) {
+uint32_t SanitizeEvictionEffortCap(int eviction_effort_cap) {
  eviction_effort_cap = std::max(int{1}, eviction_effort_cap);
-  eviction_effort_cap =
-      std::min(static_cast<int>(~kStrictCapacityLimitBit), eviction_effort_cap);
-  uint32_t eec_and_scl = static_cast<uint32_t>(eviction_effort_cap);
-  eec_and_scl |= strict_capacit_limit ? kStrictCapacityLimitBit : 0;
-  return eec_and_scl;
+  return static_cast<uint32_t>(eviction_effort_cap);
 }

 }  // namespace
@@ -381,6 +373,22 @@ void ClockHandleBasicData::FreeData(MemoryAllocator* allocator) const {
  }
 }

+BaseClockTable::BaseClockTable(size_t capacity, bool strict_capacity_limit,
+                               int eviction_effort_cap,
+                               CacheMetadataChargePolicy metadata_charge_policy,
+                               MemoryAllocator* allocator,
+                               const Cache::EvictionCallback* eviction_callback,
+                               const uint32_t* hash_seed)
+    : capacity_(capacity),
+      eec_and_scl_(EecAndScl{}
+                       .With<EvictionEffortCap>(
+                           SanitizeEvictionEffortCap(eviction_effort_cap))
+                       .With<StrictCapacityLimit>(strict_capacity_limit)),
+      metadata_charge_policy_(metadata_charge_policy),
+      allocator_(allocator),
+      eviction_callback_(*eviction_callback),
+      hash_seed_(*hash_seed) {}
+
 template <class HandleImpl>
 HandleImpl* BaseClockTable::StandaloneInsert(
    const ClockHandleBasicData& proto) {
@@ -402,8 +410,7 @@ HandleImpl* BaseClockTable::StandaloneInsert(

 template <class Table>
 typename Table::HandleImpl* BaseClockTable::CreateStandalone(
-    ClockHandleBasicData& proto, size_t capacity, uint32_t eec_and_scl,
-    bool allow_uncharged) {
+    ClockHandleBasicData& proto, bool allow_uncharged) {
  Table& derived = static_cast<Table&>(*this);
  typename Table::InsertState state;
  derived.StartInsert(state);
@@ -412,10 +419,10 @@ typename Table::HandleImpl* BaseClockTable::CreateStandalone(
  // NOTE: we can use eec_and_scl as eviction_effort_cap below because
  // strict_capacity_limit=true is supposed to disable the limit on eviction
  // effort, and a large value effectively does that.
-  if (eec_and_scl & kStrictCapacityLimitBit) {
+  if (eec_and_scl_.LoadRelaxed().Get<StrictCapacityLimit>()) {
    Status s = ChargeUsageMaybeEvictStrict<Table>(
-        total_charge, capacity,
-        /*need_evict_for_occupancy=*/false, eec_and_scl, state);
+        total_charge,
+        /*need_evict_for_occupancy=*/false, state);
    if (!s.ok()) {
      if (allow_uncharged) {
        proto.total_charge = 0;
@@ -426,8 +433,8 @@ typename Table::HandleImpl* BaseClockTable::CreateStandalone(
  } else {
    // Case strict_capacity_limit == false
    bool success = ChargeUsageMaybeEvictNonStrict<Table>(
-        total_charge, capacity,
-        /*need_evict_for_occupancy=*/false, eec_and_scl, state);
+        total_charge,
+        /*need_evict_for_occupancy=*/false, state);
    if (!success) {
      // Force the issue
      usage_.FetchAddRelaxed(total_charge);
@@ -439,8 +446,9 @@ typename Table::HandleImpl* BaseClockTable::CreateStandalone(

 template <class Table>
 Status BaseClockTable::ChargeUsageMaybeEvictStrict(
-    size_t total_charge, size_t capacity, bool need_evict_for_occupancy,
-    uint32_t eviction_effort_cap, typename Table::InsertState& state) {
+    size_t total_charge, bool need_evict_for_occupancy,
+    typename Table::InsertState& state) {
+  const size_t capacity = capacity_.LoadRelaxed();
  if (total_charge > capacity) {
    return Status::MemoryLimit(
        "Cache entry too large for a single cache shard: " +
@@ -465,8 +473,7 @@ Status BaseClockTable::ChargeUsageMaybeEvictStrict(
  }
  if (request_evict_charge > 0) {
    EvictionData data;
-    static_cast<Table*>(this)->Evict(request_evict_charge, state, &data,
-                                     eviction_effort_cap);
+    static_cast<Table*>(this)->Evict(request_evict_charge, state, &data);
    occupancy_.FetchSub(data.freed_count);
    if (LIKELY(data.freed_charge > need_evict_charge)) {
      assert(data.freed_count > 0);
@@ -495,8 +502,8 @@ Status BaseClockTable::ChargeUsageMaybeEvictStrict(

 template <class Table>
 inline bool BaseClockTable::ChargeUsageMaybeEvictNonStrict(
-    size_t total_charge, size_t capacity, bool need_evict_for_occupancy,
-    uint32_t eviction_effort_cap, typename Table::InsertState& state) {
+    size_t total_charge, bool need_evict_for_occupancy,
+    typename Table::InsertState& state) {
  // For simplicity, we consider that either the cache can accept the insert
  // with no evictions, or we must evict enough to make (at least) enough
  // space. It could lead to unnecessary failures or excessive evictions in
@@ -506,7 +513,8 @@ inline bool BaseClockTable::ChargeUsageMaybeEvictNonStrict(
  // charge. Thus, we should evict some extra if it's not a signifcant
  // portion of the shard capacity. This can have the side benefit of
  // involving fewer threads in eviction.
-  size_t old_usage = usage_.LoadRelaxed();
+  const size_t old_usage = usage_.LoadRelaxed();
+  const size_t capacity = capacity_.LoadRelaxed();
  size_t need_evict_charge;
  // NOTE: if total_charge > old_usage, there isn't yet enough to evict
  // `total_charge` amount. Even if we only try to evict `old_usage` amount,
@@ -532,8 +540,7 @@ inline bool BaseClockTable::ChargeUsageMaybeEvictNonStrict(
  }
  EvictionData data;
  if (need_evict_charge > 0) {
-    static_cast<Table*>(this)->Evict(need_evict_charge, state, &data,
-                                     eviction_effort_cap);
+    static_cast<Table*>(this)->Evict(need_evict_charge, state, &data);
    // Deal with potential occupancy deficit
    if (UNLIKELY(need_evict_for_occupancy) && data.freed_count == 0) {
      assert(data.freed_charge == 0);
@@ -569,8 +576,10 @@ void BaseClockTable::TrackAndReleaseEvictedEntry(ClockHandle* h) {
  MarkEmpty(*h);
 }

-bool IsEvictionEffortExceeded(const BaseClockTable::EvictionData& data,
-                              uint32_t eviction_effort_cap) {
+bool BaseClockTable::IsEvictionEffortExceeded(
+    const BaseClockTable::EvictionData& data) const {
+  auto eviction_effort_cap =
+      eec_and_scl_.LoadRelaxed().GetEffectiveEvictionEffortCap();
  // Basically checks whether the ratio of useful effort to wasted effort is
  // too low, with a start-up allowance for wasted effort before any useful
  // effort.
@@ -581,8 +590,7 @@ bool IsEvictionEffortExceeded(const BaseClockTable::EvictionData& data,
 template <class Table>
 Status BaseClockTable::Insert(const ClockHandleBasicData& proto,
                              typename Table::HandleImpl** handle,
-                              Cache::Priority priority, size_t capacity,
-                              uint32_t eec_and_scl) {
+                              Cache::Priority priority) {
  using HandleImpl = typename Table::HandleImpl;
  Table& derived = static_cast<Table&>(*this);

@@ -603,9 +611,9 @@ Status BaseClockTable::Insert(const ClockHandleBasicData& proto,
  // NOTE: we can use eec_and_scl as eviction_effort_cap below because
  // strict_capacity_limit=true is supposed to disable the limit on eviction
  // effort, and a large value effectively does that.
-  if (eec_and_scl & kStrictCapacityLimitBit) {
+  if (eec_and_scl_.LoadRelaxed().Get<StrictCapacityLimit>()) {
    Status s = ChargeUsageMaybeEvictStrict<Table>(
-        total_charge, capacity, need_evict_for_occupancy, eec_and_scl, state);
+        total_charge, need_evict_for_occupancy, state);
    if (!s.ok()) {
      // Revert occupancy
      occupancy_.FetchSubRelaxed(1);
@@ -614,7 +622,7 @@ Status BaseClockTable::Insert(const ClockHandleBasicData& proto,
  } else {
    // Case strict_capacity_limit == false
    bool success = ChargeUsageMaybeEvictNonStrict<Table>(
-        total_charge, capacity, need_evict_for_occupancy, eec_and_scl, state);
+        total_charge, need_evict_for_occupancy, state);
    if (!success) {
      // Revert occupancy
      occupancy_.FetchSubRelaxed(1);
@@ -718,11 +726,13 @@ void BaseClockTable::TEST_ReleaseNMinus1(ClockHandle* h, size_t n) {
 #endif

 FixedHyperClockTable::FixedHyperClockTable(
-    size_t capacity, CacheMetadataChargePolicy metadata_charge_policy,
+    size_t capacity, bool strict_capacity_limit,
+    CacheMetadataChargePolicy metadata_charge_policy,
    MemoryAllocator* allocator,
    const Cache::EvictionCallback* eviction_callback, const uint32_t* hash_seed,
    const Opts& opts)
-    : BaseClockTable(metadata_charge_policy, allocator, eviction_callback,
+    : BaseClockTable(capacity, strict_capacity_limit, opts.eviction_effort_cap,
+                     metadata_charge_policy, allocator, eviction_callback,
                     hash_seed),
      length_bits_(CalcHashBits(capacity, opts.estimated_value_size,
                                metadata_charge_policy)),
@@ -1113,8 +1123,7 @@ inline void FixedHyperClockTable::ReclaimEntryUsage(size_t total_charge) {
 }

 inline void FixedHyperClockTable::Evict(size_t requested_charge, InsertState&,
-                                        EvictionData* data,
-                                        uint32_t eviction_effort_cap) {
+                                        EvictionData* data) {
  // precondition
  assert(requested_charge > 0);

@@ -1149,7 +1158,7 @@ inline void FixedHyperClockTable::Evict(size_t requested_charge, InsertState&,
    if (old_clock_pointer >= max_clock_pointer) {
      return;
    }
-    if (IsEvictionEffortExceeded(*data, eviction_effort_cap)) {
+    if (IsEvictionEffortExceeded(*data)) {
      eviction_effort_exceeded_count_.FetchAddRelaxed(1);
      return;
    }
@@ -1167,14 +1176,11 @@ ClockCacheShard<Table>::ClockCacheShard(
    const Cache::EvictionCallback* eviction_callback, const uint32_t* hash_seed,
    const typename Table::Opts& opts)
    : CacheShardBase(metadata_charge_policy),
-      table_(capacity, metadata_charge_policy, allocator, eviction_callback,
-             hash_seed, opts),
-      capacity_(capacity),
-      eec_and_scl_(SanitizeEncodeEecAndScl(opts.eviction_effort_cap,
-                                           strict_capacity_limit)) {
+      table_(capacity, strict_capacity_limit, metadata_charge_policy, allocator,
+             eviction_callback, hash_seed, opts) {
  // Initial charge metadata should not exceed capacity
-  assert(table_.GetUsage() <= capacity_.LoadRelaxed() ||
-         capacity_.LoadRelaxed() < sizeof(HandleImpl));
+  assert(table_.GetUsage() <= table_.GetCapacity() ||
+         table_.GetCapacity() < sizeof(HandleImpl));
 }

 template <class Table>
@@ -1240,18 +1246,14 @@ int FixedHyperClockTable::CalcHashBits(

 template <class Table>
 void ClockCacheShard<Table>::SetCapacity(size_t capacity) {
-  capacity_.StoreRelaxed(capacity);
+  table_.SetCapacity(capacity);
  // next Insert will take care of any necessary evictions
 }

 template <class Table>
 void ClockCacheShard<Table>::SetStrictCapacityLimit(
    bool strict_capacity_limit) {
-  if (strict_capacity_limit) {
-    eec_and_scl_.FetchOrRelaxed(kStrictCapacityLimitBit);
-  } else {
-    eec_and_scl_.FetchAndRelaxed(~kStrictCapacityLimitBit);
-  }
+  table_.SetStrictCapacityLimit(strict_capacity_limit);
  // next Insert will take care of any necessary evictions
 }

@@ -1271,9 +1273,7 @@ Status ClockCacheShard<Table>::Insert(const Slice& key,
  proto.value = value;
  proto.helper = helper;
  proto.total_charge = charge;
-  return table_.template Insert<Table>(proto, handle, priority,
-                                       capacity_.LoadRelaxed(),
-                                       eec_and_scl_.LoadRelaxed());
+  return table_.template Insert<Table>(proto, handle, priority);
 }

 template <class Table>
@@ -1288,9 +1288,7 @@ typename Table::HandleImpl* ClockCacheShard<Table>::CreateStandalone(
  proto.value = obj;
  proto.helper = helper;
  proto.total_charge = charge;
-  return table_.template CreateStandalone<Table>(proto, capacity_.LoadRelaxed(),
-                                                 eec_and_scl_.LoadRelaxed(),
-                                                 allow_uncharged);
+  return table_.template CreateStandalone<Table>(proto, allow_uncharged);
 }

 template <class Table>
@@ -1359,7 +1357,7 @@ size_t ClockCacheShard<Table>::GetStandaloneUsage() const {

 template <class Table>
 size_t ClockCacheShard<Table>::GetCapacity() const {
-  return capacity_.LoadRelaxed();
+  return table_.GetCapacity();
 }

 template <class Table>
@@ -1727,10 +1725,13 @@ inline uint64_t UsedLengthToLengthInfo(size_t used_length) {
  return length_info;
 }

+// Avoid potential initialization order race with port::kPageSize
+constexpr size_t kPresumedPageSize = 4096;
+
 inline size_t GetStartingLength(size_t capacity) {
-  if (capacity > port::kPageSize) {
+  if (capacity > kPresumedPageSize) {
    // Start with one memory page
-    return port::kPageSize / sizeof(AutoHyperClockTable::HandleImpl);
+    return kPresumedPageSize / sizeof(AutoHyperClockTable::HandleImpl);
  } else {
    // Mostly to make unit tests happy
    return 4;
@@ -1969,11 +1970,13 @@ class AutoHyperClockTable::ChainRewriteLock {
 };

 AutoHyperClockTable::AutoHyperClockTable(
-    size_t capacity, CacheMetadataChargePolicy metadata_charge_policy,
+    size_t capacity, bool strict_capacity_limit,
+    CacheMetadataChargePolicy metadata_charge_policy,
    MemoryAllocator* allocator,
    const Cache::EvictionCallback* eviction_callback, const uint32_t* hash_seed,
    const Opts& opts)
-    : BaseClockTable(metadata_charge_policy, allocator, eviction_callback,
+    : BaseClockTable(capacity, strict_capacity_limit, opts.eviction_effort_cap,
+                     metadata_charge_policy, allocator, eviction_callback,
                     hash_seed),
      array_(MemMapping::AllocateLazyZeroed(
          sizeof(HandleImpl) * CalcMaxUsableLength(capacity,
@@ -1985,6 +1988,11 @@ AutoHyperClockTable::AutoHyperClockTable(
      grow_frontier_(GetTableSize()),
      clock_pointer_mask_(
          BottomNBits(UINT64_MAX, LengthInfoToMinShift(length_info_.Load()))) {
+  if (array_.Get() == nullptr) {
+    fprintf(stderr,
+            "Anonymous mmap for RocksDB HyperClockCache failed. Aborting.\n");
+    std::terminate();
+  }
  if (metadata_charge_policy ==
      CacheMetadataChargePolicy::kFullChargeCacheMetadata) {
    // NOTE: ignoring page boundaries for simplicity
@@ -2052,15 +2060,21 @@ AutoHyperClockTable::~AutoHyperClockTable() {
             HandleImpl::kUnusedMarker) {
    used_end++;
  }
-#ifndef NDEBUG
-  for (size_t i = used_end; i < array_.Count(); i++) {
+  // This check can be extra expensive for a cache that is just created,
+  // maybe used for a small number of entries, as in a unit test, and then
+  // destroyed. Only do this in rare modes. REVISED: Don't scan the whole mmap,
+  // just a reasonable frontier past what we expect to have written.
+#ifdef MUST_FREE_HEAP_ALLOCATIONS
+  for (size_t i = used_end; i < array_.Count() && i < used_end + 64U; i++) {
    assert(array_[i].head_next_with_shift.LoadRelaxed() == 0);
    assert(array_[i].chain_next_with_shift.LoadRelaxed() == 0);
    assert(array_[i].meta.LoadRelaxed() == 0);
  }
+#endif          // MUST_FREE_HEAP_ALLOCATIONS
+#ifndef NDEBUG  // Extra invariant checking
  std::vector<bool> was_populated(used_end);
  std::vector<bool> was_pointed_to(used_end);
-#endif
+#endif  // !NDEBUG
  for (size_t i = 0; i < used_end; i++) {
    HandleImpl& h = array_[i];
    switch (h.meta.LoadRelaxed() >> ClockHandle::kStateShift) {
@@ -2083,7 +2097,7 @@ AutoHyperClockTable::~AutoHyperClockTable() {
          assert(!was_pointed_to[next]);
          was_pointed_to[next] = true;
        }
-#endif
+#endif  // !NDEBUG
        break;
      // otherwise
      default:
@@ -2097,7 +2111,7 @@ AutoHyperClockTable::~AutoHyperClockTable() {
      assert(!was_pointed_to[next]);
      was_pointed_to[next] = true;
    }
-#endif
+#endif  // !NDEBUG
  }
 #ifndef NDEBUG  // Extra invariant checking
  // This check is not perfect, but should detect most reasonable cases
@@ -2110,7 +2124,7 @@ AutoHyperClockTable::~AutoHyperClockTable() {
      assert(!was_pointed_to[i]);
    }
  }
-#endif
+#endif  // !NDEBUG

  // Metadata charging only follows the published table size
  assert(usage_.LoadRelaxed() == 0 ||
@@ -3468,8 +3482,7 @@ void AutoHyperClockTable::EraseUnRefEntries() {
 }

 void AutoHyperClockTable::Evict(size_t requested_charge, InsertState& state,
-                                EvictionData* data,
-                                uint32_t eviction_effort_cap) {
+                                EvictionData* data) {
  // precondition
  assert(requested_charge > 0);

@@ -3561,7 +3574,7 @@ void AutoHyperClockTable::Evict(size_t requested_charge, InsertState& state,
      return;
    }

-    if (IsEvictionEffortExceeded(*data, eviction_effort_cap)) {
+    if (IsEvictionEffortExceeded(*data)) {
      eviction_effort_exceeded_count_.FetchAddRelaxed(1);
      return;
    }
@@ -3579,7 +3592,7 @@ size_t AutoHyperClockTable::CalcMaxUsableLength(
  size_t num_slots =
      static_cast<size_t>(capacity / min_avg_slot_charge + 0.999999);

-  const size_t slots_per_page = port::kPageSize / sizeof(HandleImpl);
+  const size_t slots_per_page = kPresumedPageSize / sizeof(HandleImpl);

  // Round up to page size
  return ((num_slots + slots_per_page - 1) / slots_per_page) * slots_per_page;
@@ -9,8 +9,6 @@

 #pragma once

-#include <array>
-#include <atomic>
 #include <climits>
 #include <cstddef>
 #include <cstdint>
@@ -19,14 +17,10 @@

 #include "cache/cache_key.h"
 #include "cache/sharded_cache.h"
-#include "port/lang.h"
-#include "port/malloc.h"
 #include "port/mmap.h"
-#include "port/port.h"
 #include "rocksdb/cache.h"
-#include "rocksdb/secondary_cache.h"
 #include "util/atomic.h"
-#include "util/autovector.h"
+#include "util/bit_fields.h"
 #include "util/math.h"

 namespace ROCKSDB_NAMESPACE {
@@ -383,25 +377,20 @@ class BaseClockTable {
    int eviction_effort_cap;
  };

-  BaseClockTable(CacheMetadataChargePolicy metadata_charge_policy,
+  BaseClockTable(size_t capacity, bool strict_capacity_limit,
+                 int eviction_effort_cap,
+                 CacheMetadataChargePolicy metadata_charge_policy,
                 MemoryAllocator* allocator,
                 const Cache::EvictionCallback* eviction_callback,
-                 const uint32_t* hash_seed)
-      : metadata_charge_policy_(metadata_charge_policy),
-        allocator_(allocator),
-        eviction_callback_(*eviction_callback),
-        hash_seed_(*hash_seed) {}
+                 const uint32_t* hash_seed);

  template <class Table>
  typename Table::HandleImpl* CreateStandalone(ClockHandleBasicData& proto,
-                                               size_t capacity,
-                                               uint32_t eec_and_scl,
                                               bool allow_uncharged);

  template <class Table>
  Status Insert(const ClockHandleBasicData& proto,
-                typename Table::HandleImpl** handle, Cache::Priority priority,
-                size_t capacity, uint32_t eec_and_scl);
+                typename Table::HandleImpl** handle, Cache::Priority priority);

  void Ref(ClockHandle& handle);

@@ -411,6 +400,18 @@ class BaseClockTable {

  size_t GetStandaloneUsage() const { return standalone_usage_.LoadRelaxed(); }

+  size_t GetCapacity() const { return capacity_.LoadRelaxed(); }
+
+  void SetCapacity(size_t capacity) { capacity_.StoreRelaxed(capacity); }
+
+  void SetStrictCapacityLimit(bool strict_capacity_limit) {
+    if (strict_capacity_limit) {
+      eec_and_scl_.ApplyRelaxed(StrictCapacityLimit::SetTransform());
+    } else {
+      eec_and_scl_.ApplyRelaxed(StrictCapacityLimit::ClearTransform());
+    }
+  }
+
  uint32_t GetHashSeed() const { return hash_seed_; }

  uint64_t GetYieldCount() const { return yield_count_.LoadRelaxed(); }
@@ -427,6 +428,7 @@ class BaseClockTable {

  void TrackAndReleaseEvictedEntry(ClockHandle* h);

+  bool IsEvictionEffortExceeded(const BaseClockTable::EvictionData& data) const;
 #ifndef NDEBUG
  // Acquire N references
  void TEST_RefN(ClockHandle& handle, size_t n);
@@ -448,9 +450,8 @@ class BaseClockTable {
  // required, and the operation should fail if not possible.
  // NOTE: Otherwise, occupancy_ is not managed in this function
  template <class Table>
-  Status ChargeUsageMaybeEvictStrict(size_t total_charge, size_t capacity,
+  Status ChargeUsageMaybeEvictStrict(size_t total_charge,
                                     bool need_evict_for_occupancy,
-                                     uint32_t eviction_effort_cap,
                                     typename Table::InsertState& state);

  // Helper for updating `usage_` for new entry with given `total_charge`
@@ -462,9 +463,8 @@ class BaseClockTable {
  // true, indicating success.
  // NOTE: occupancy_ is not managed in this function
  template <class Table>
-  bool ChargeUsageMaybeEvictNonStrict(size_t total_charge, size_t capacity,
+  bool ChargeUsageMaybeEvictNonStrict(size_t total_charge,
                                      bool need_evict_for_occupancy,
-                                      uint32_t eviction_effort_cap,
                                      typename Table::InsertState& state);

 protected:  // data
@@ -497,6 +497,25 @@ class BaseClockTable {
  // Part of usage by standalone entries (not in table)
  AcqRelAtomic<size_t> standalone_usage_{};

+  // Maximum total charge of all elements stored in the table.
+  // (Relaxed: eventual consistency/update is OK)
+  RelaxedAtomic<size_t> capacity_;
+
+  // Encodes eviction_effort_cap (bottom 31 bits) and strict_capacity_limit
+  // (top bit). See HyperClockCacheOptions::eviction_effort_cap etc.
+  struct EecAndScl : public BitFields<uint32_t, EecAndScl> {
+    uint32_t GetEffectiveEvictionEffortCap() const {
+      // Because setting strict_capacity_limit is supposed to imply infinite
+      // cap on eviction effort, we can let the bit for strict_capacity_limit
+      // in the upper-most bit position to used as part of the effective cap.
+      return underlying;
+    }
+  };
+  using EvictionEffortCap = UnsignedBitField<EecAndScl, 31, NoPrevBitField>;
+  using StrictCapacityLimit = BoolBitField<EecAndScl, EvictionEffortCap>;
+  // (Relaxed: eventual consistency/update is OK)
+  RelaxedBitFieldsAtomic<EecAndScl> eec_and_scl_;
+
  ALIGN_AS(CACHE_LINE_SIZE)
  const CacheMetadataChargePolicy metadata_charge_policy_;

@@ -551,7 +570,7 @@ class FixedHyperClockTable : public BaseClockTable {
    size_t estimated_value_size;
  };

-  FixedHyperClockTable(size_t capacity,
+  FixedHyperClockTable(size_t capacity, bool strict_capacity_limit,
                       CacheMetadataChargePolicy metadata_charge_policy,
                       MemoryAllocator* allocator,
                       const Cache::EvictionCallback* eviction_callback,
@@ -573,8 +592,7 @@ class FixedHyperClockTable : public BaseClockTable {
  // Runs the clock eviction algorithm trying to reclaim at least
  // requested_charge. Returns how much is evicted, which could be less
  // if it appears impossible to evict the requested amount without blocking.
-  void Evict(size_t requested_charge, InsertState& state, EvictionData* data,
-             uint32_t eviction_effort_cap);
+  void Evict(size_t requested_charge, InsertState& state, EvictionData* data);

  HandleImpl* Lookup(const UniqueId64x2& hashed_key);

@@ -841,7 +859,7 @@ class AutoHyperClockTable : public BaseClockTable {
    size_t min_avg_value_size;
  };

-  AutoHyperClockTable(size_t capacity,
+  AutoHyperClockTable(size_t capacity, bool strict_capacity_limit,
                      CacheMetadataChargePolicy metadata_charge_policy,
                      MemoryAllocator* allocator,
                      const Cache::EvictionCallback* eviction_callback,
@@ -868,8 +886,7 @@ class AutoHyperClockTable : public BaseClockTable {
  // Runs the clock eviction algorithm trying to reclaim at least
  // requested_charge. Returns how much is evicted, which could be less
  // if it appears impossible to evict the requested amount without blocking.
-  void Evict(size_t requested_charge, InsertState& state, EvictionData* data,
-             uint32_t eviction_effort_cap);
+  void Evict(size_t requested_charge, InsertState& state, EvictionData* data);

  HandleImpl* Lookup(const UniqueId64x2& hashed_key);

@@ -1102,15 +1119,6 @@ class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShardBase {

 private:  // data
  Table table_;
-
-  // Maximum total charge of all elements stored in the table.
-  // (Relaxed: eventual consistency/update is OK)
-  RelaxedAtomic<size_t> capacity_;
-
-  // Encodes eviction_effort_cap (bottom 31 bits) and strict_capacity_limit
-  // (top bit). See HyperClockCacheOptions::eviction_effort_cap etc.
-  // (Relaxed: eventual consistency/update is OK)
-  RelaxedAtomic<uint32_t> eec_and_scl_;
 };  // class ClockCacheShard

 template <class Table>
@@ -1405,9 +1405,9 @@ TEST_P(BasicSecondaryCacheTest, SaveFailTest) {
  TestItem* item1 = new TestItem(str1.data(), str1.length());
  ASSERT_OK(cache->Insert(k1.AsSlice(), item1, GetHelperFail(), str1.length()));
  std::string str2 = rnd.RandomString(1020);
+  ASSERT_EQ(secondary_cache->num_inserts(), 0u);
  TestItem* item2 = new TestItem(str2.data(), str2.length());
  // k1 should be demoted to NVM
-  ASSERT_EQ(secondary_cache->num_inserts(), 0u);
  ASSERT_OK(cache->Insert(k2.AsSlice(), item2, GetHelperFail(), str2.length()));
  ASSERT_EQ(secondary_cache->num_inserts(), 1u);

@@ -584,7 +584,7 @@ Status CacheWithSecondaryAdapter::UpdateCacheReservationRatio(
  size_t pri_capacity = target_->GetCapacity();
  size_t sec_capacity =
      static_cast<size_t>(pri_capacity * compressed_secondary_ratio);
-  size_t old_sec_capacity;
+  size_t old_sec_capacity = 0;
  Status s = secondary_cache_->GetCapacity(old_sec_capacity);
  if (!s.ok()) {
    return s;
@@ -342,6 +342,9 @@ Status BuildTable(
    if (s.ok() && !empty) {
      if (flush_stats) {
        flush_stats->bytes_written_pre_comp = builder->PreCompressionSize();
+        // Add worker CPU micros here. Caller needs to add CPU micros from
+        // calling thread.
+        flush_stats->cpu_micros += builder->GetWorkerCPUMicros();
      }
      uint64_t file_size = builder->FileSize();
      meta->fd.file_size = file_size;
@@ -82,6 +82,7 @@ using ROCKSDB_NAMESPACE::DbPath;
 using ROCKSDB_NAMESPACE::Env;
 using ROCKSDB_NAMESPACE::EnvOptions;
 using ROCKSDB_NAMESPACE::EventListener;
+using ROCKSDB_NAMESPACE::ExportImportFilesMetaData;
 using ROCKSDB_NAMESPACE::ExternalFileIngestionInfo;
 using ROCKSDB_NAMESPACE::FileLock;
 using ROCKSDB_NAMESPACE::FilterPolicy;
@@ -89,6 +90,7 @@ using ROCKSDB_NAMESPACE::FlushJobInfo;
 using ROCKSDB_NAMESPACE::FlushOptions;
 using ROCKSDB_NAMESPACE::HistogramData;
 using ROCKSDB_NAMESPACE::HyperClockCacheOptions;
+using ROCKSDB_NAMESPACE::ImportColumnFamilyOptions;
 using ROCKSDB_NAMESPACE::InfoLogLevel;
 using ROCKSDB_NAMESPACE::IngestExternalFileOptions;
 using ROCKSDB_NAMESPACE::Iterator;
@@ -245,6 +247,9 @@ struct rocksdb_write_buffer_manager_t {
 struct rocksdb_sst_file_manager_t {
  std::shared_ptr<SstFileManager> rep;
 };
+struct rocksdb_livefile_t {
+  LiveFileMetaData rep;
+};
 struct rocksdb_livefiles_t {
  std::vector<LiveFileMetaData> rep;
 };
@@ -255,6 +260,12 @@ struct rocksdb_column_family_handle_t {
 struct rocksdb_column_family_metadata_t {
  ColumnFamilyMetaData rep;
 };
+struct rocksdb_export_import_files_metadata_t {
+  ExportImportFilesMetaData* rep;
+};
+struct rocksdb_import_column_family_options_t {
+  ImportColumnFamilyOptions rep;
+};
 struct rocksdb_level_metadata_t {
  const LevelMetaData* rep;
 };
@@ -947,6 +958,22 @@ void rocksdb_checkpoint_create(rocksdb_checkpoint_t* checkpoint,
                        std::string(checkpoint_dir), log_size_for_flush));
 }

+rocksdb_export_import_files_metadata_t* rocksdb_checkpoint_export_column_family(
+    rocksdb_checkpoint_t* checkpoint,
+    rocksdb_column_family_handle_t* column_family, const char* export_dir,
+    char** errptr) {
+  ExportImportFilesMetaData* metadata = nullptr;
+  if (SaveError(errptr,
+                checkpoint->rep->ExportColumnFamily(
+                    column_family->rep, std::string(export_dir), &metadata))) {
+    return nullptr;
+  }
+  rocksdb_export_import_files_metadata_t* result =
+      new rocksdb_export_import_files_metadata_t;
+  result->rep = metadata;
+  return result;
+}
+
 void rocksdb_checkpoint_object_destroy(rocksdb_checkpoint_t* checkpoint) {
  delete checkpoint->rep;
  delete checkpoint;
@@ -1190,6 +1217,26 @@ rocksdb_column_family_handle_t** rocksdb_create_column_families(
  return c_handles;
 }

+rocksdb_column_family_handle_t* rocksdb_create_column_family_with_import(
+    rocksdb_t* db, rocksdb_options_t* column_family_options,
+    const char* column_family_name,
+    rocksdb_import_column_family_options_t* import_options,
+    rocksdb_export_import_files_metadata_t* export_import_files_metadata,
+    char** errptr) {
+  rocksdb_column_family_handle_t* handle = new rocksdb_column_family_handle_t;
+  handle->rep = nullptr;
+  if (SaveError(errptr,
+                db->rep->CreateColumnFamilyWithImport(
+                    ColumnFamilyOptions(column_family_options->rep),
+                    std::string(column_family_name), import_options->rep,
+                    *(export_import_files_metadata->rep), &(handle->rep)))) {
+    delete handle;
+    return nullptr;
+  }
+  handle->immortal = false;
+  return handle;
+}
+
 void rocksdb_create_column_families_destroy(
    rocksdb_column_family_handle_t** list) {
  free(list);
@@ -6209,6 +6256,10 @@ void rocksdb_options_set_min_level_to_compress(rocksdb_options_t* opt,
  }
 }

+rocksdb_livefiles_t* rocksdb_livefiles_create() {
+  return new rocksdb_livefiles_t;
+}
+
 int rocksdb_livefiles_count(const rocksdb_livefiles_t* lf) {
  return static_cast<int>(lf->rep.size());
 }
@@ -6222,6 +6273,16 @@ const char* rocksdb_livefiles_name(const rocksdb_livefiles_t* lf, int index) {
  return lf->rep[index].name.c_str();
 }

+const char* rocksdb_livefiles_directory(const rocksdb_livefiles_t* lf,
+                                        int index) {
+  if (lf->rep[index].directory.empty()) {
+    // db_path is deprecated but still returned by some code paths
+    return lf->rep[index].db_path.c_str();
+  } else {
+    return lf->rep[index].directory.c_str();
+  }
+}
+
 int rocksdb_livefiles_level(const rocksdb_livefiles_t* lf, int index) {
  return lf->rep[index].level;
 }
@@ -6242,6 +6303,16 @@ const char* rocksdb_livefiles_largestkey(const rocksdb_livefiles_t* lf,
  return lf->rep[index].largestkey.data();
 }

+uint64_t rocksdb_livefiles_smallest_seqno(const rocksdb_livefiles_t* lf,
+                                          int index) {
+  return lf->rep[index].smallest_seqno;
+}
+
+uint64_t rocksdb_livefiles_largest_seqno(const rocksdb_livefiles_t* lf,
+                                         int index) {
+  return lf->rep[index].largest_seqno;
+}
+
 uint64_t rocksdb_livefiles_entries(const rocksdb_livefiles_t* lf, int index) {
  return lf->rep[index].num_entries;
 }
@@ -6252,6 +6323,71 @@ uint64_t rocksdb_livefiles_deletions(const rocksdb_livefiles_t* lf, int index) {

 void rocksdb_livefiles_destroy(const rocksdb_livefiles_t* lf) { delete lf; }

+rocksdb_livefile_t* rocksdb_livefile_create() { return new rocksdb_livefile_t; }
+
+void rocksdb_livefile_set_column_family_name(rocksdb_livefile_t* lf,
+                                             const char* column_family_name) {
+  lf->rep.column_family_name = std::string(column_family_name);
+}
+
+void rocksdb_livefile_set_level(rocksdb_livefile_t* lf, int level) {
+  lf->rep.level = level;
+}
+
+void rocksdb_livefile_set_name(rocksdb_livefile_t* lf, const char* name) {
+  lf->rep.name = std::string(name);
+}
+
+void rocksdb_livefile_set_directory(rocksdb_livefile_t* lf,
+                                    const char* directory) {
+  lf->rep.directory = std::string(directory);
+  lf->rep.db_path = std::string(directory);  // deprecated but still needed
+}
+
+void rocksdb_livefile_set_size(rocksdb_livefile_t* lf, size_t size) {
+  lf->rep.size = size;
+}
+
+void rocksdb_livefile_set_smallest_key(rocksdb_livefile_t* lf,
+                                       const char* smallest_key,
+                                       size_t smallest_key_len) {
+  lf->rep.smallestkey = std::string(smallest_key, smallest_key_len);
+}
+
+void rocksdb_livefile_set_largest_key(rocksdb_livefile_t* lf,
+                                      const char* largest_key,
+                                      size_t largest_key_len) {
+  lf->rep.largestkey = std::string(largest_key, largest_key_len);
+}
+
+void rocksdb_livefile_set_smallest_seqno(rocksdb_livefile_t* lf,
+                                         uint64_t smallest_seqno) {
+  lf->rep.smallest_seqno = smallest_seqno;
+}
+
+void rocksdb_livefile_set_largest_seqno(rocksdb_livefile_t* lf,
+                                        uint64_t largest_seqno) {
+  lf->rep.largest_seqno = largest_seqno;
+}
+
+void rocksdb_livefile_set_num_entries(rocksdb_livefile_t* lf,
+                                      uint64_t num_entries) {
+  lf->rep.num_entries = num_entries;
+}
+
+void rocksdb_livefile_set_num_deletions(rocksdb_livefile_t* lf,
+                                        uint64_t num_deletions) {
+  lf->rep.num_deletions = num_deletions;
+}
+
+void rocksdb_livefile_destroy(rocksdb_livefile_t* lf) { delete lf; }
+
+void rocksdb_livefiles_add(rocksdb_livefiles_t* lf,
+                           rocksdb_livefile_t* livefile) {
+  lf->rep.push_back(std::move(livefile->rep));
+  delete livefile;
+}
+
 void rocksdb_get_options_from_string(const rocksdb_options_t* base_options,
                                     const char* opts_str,
                                     rocksdb_options_t* new_options,
@@ -6402,6 +6538,58 @@ char* rocksdb_sst_file_metadata_get_largestkey(
  return CopyString(file_meta->rep->largestkey);
 }

+rocksdb_import_column_family_options_t*
+rocksdb_import_column_family_options_create() {
+  return new rocksdb_import_column_family_options_t;
+}
+
+void rocksdb_import_column_family_options_set_move_files(
+    rocksdb_import_column_family_options_t* opt, unsigned char v) {
+  opt->rep.move_files = v;
+}
+
+void rocksdb_import_column_family_options_destroy(
+    rocksdb_import_column_family_options_t* metadata) {
+  delete metadata;
+}
+
+rocksdb_export_import_files_metadata_t*
+rocksdb_export_import_files_metadata_create() {
+  auto metadata = new rocksdb_export_import_files_metadata_t;
+  metadata->rep = new ExportImportFilesMetaData;
+  return metadata;
+}
+
+char* rocksdb_export_import_files_metadata_get_db_comparator_name(
+    rocksdb_export_import_files_metadata_t* metadata) {
+  return strdup(metadata->rep->db_comparator_name.c_str());
+}
+
+void rocksdb_export_import_files_metadata_set_db_comparator_name(
+    rocksdb_export_import_files_metadata_t* metadata, const char* name) {
+  metadata->rep->db_comparator_name = std::string(name);
+}
+
+rocksdb_livefiles_t* rocksdb_export_import_files_metadata_get_files(
+    rocksdb_export_import_files_metadata_t* export_import_metadata) {
+  auto files = new rocksdb_livefiles_t;
+  files->rep = std::vector(export_import_metadata->rep->files);
+  return files;
+}
+
+void rocksdb_export_import_files_metadata_set_files(
+    rocksdb_export_import_files_metadata_t* metadata,
+    rocksdb_livefiles_t* files) {
+  metadata->rep->files = std::move(files->rep);
+  delete files;
+}
+
+void rocksdb_export_import_files_metadata_destroy(
+    rocksdb_export_import_files_metadata_t* metadata) {
+  delete metadata->rep;
+  delete metadata;
+}
+
 /* Transactions */

 rocksdb_transactiondb_options_t* rocksdb_transactiondb_options_create() {
@@ -6433,6 +6621,11 @@ void rocksdb_transactiondb_options_set_default_lock_timeout(
  opt->rep.default_lock_timeout = default_lock_timeout;
 }

+void rocksdb_transactiondb_options_set_use_per_key_point_lock_mgr(
+    rocksdb_transactiondb_options_t* opt, int use_per_key_point_lock_mgr) {
+  opt->rep.use_per_key_point_lock_mgr = use_per_key_point_lock_mgr;
+}
+
 rocksdb_transaction_options_t* rocksdb_transaction_options_create() {
  return new rocksdb_transaction_options_t;
 }
@@ -1036,6 +1036,78 @@ int main(int argc, char** argv) {
    rocksdb_options_set_error_if_exists(options, 1);
  }

+  StartPhase("checkpoint_export_column_family");
+  {
+    static char cf_export_path[200];
+    static char db_import_path[200];
+    snprintf(cf_export_path, sizeof(cf_export_path),
+             "%s/rocksdb_c_test-%d-cf_export", GetTempDir(), ((int)geteuid()));
+    snprintf(db_import_path, sizeof(db_import_path),
+             "%s/rocksdb_c_test-%d-db_import", GetTempDir(), ((int)geteuid()));
+
+    rocksdb_options_t* db_options = rocksdb_options_create();
+    rocksdb_column_family_handle_t* cf_export =
+        rocksdb_create_column_family(db, db_options, "cf_export", &err);
+    CheckNoError(err);
+
+    rocksdb_put_cf(db, woptions, cf_export, "k1", 2, "v1", 2, &err);
+    CheckNoError(err);
+    rocksdb_put_cf(db, woptions, cf_export, "k2", 2, "v2", 2, &err);
+    CheckNoError(err);
+
+    rocksdb_checkpoint_t* checkpoint =
+        rocksdb_checkpoint_object_create(db, &err);
+    CheckNoError(err);
+
+    rocksdb_export_import_files_metadata_t* export_metadata =
+        rocksdb_checkpoint_export_column_family(checkpoint, cf_export,
+                                                cf_export_path, &err);
+    CheckNoError(err);
+    const char* comparator_name =
+        rocksdb_export_import_files_metadata_get_db_comparator_name(
+            export_metadata);
+    CheckEqual("leveldb.BytewiseComparator", comparator_name, 26);
+    rocksdb_free((void*)comparator_name);
+    rocksdb_checkpoint_object_destroy(checkpoint);
+    checkpoint = NULL;
+    rocksdb_drop_column_family(db, cf_export, &err);
+    CheckNoError(err);
+    rocksdb_column_family_handle_destroy(cf_export);
+    rocksdb_options_set_create_if_missing(db_options, 1);
+    rocksdb_options_set_error_if_exists(db_options, 1);
+    rocksdb_t* db_import = rocksdb_open(db_options, db_import_path, &err);
+    CheckNoError(err);
+    rocksdb_import_column_family_options_t* import_options =
+        rocksdb_import_column_family_options_create();
+    rocksdb_column_family_handle_t* cf_import =
+        rocksdb_create_column_family_with_import(db_import, db_options,
+                                                 "cf_import", import_options,
+                                                 export_metadata, &err);
+    CheckNoError(err);
+    rocksdb_import_column_family_options_destroy(import_options);
+    rocksdb_export_import_files_metadata_destroy(export_metadata);
+    size_t val_len;
+    char* val =
+        rocksdb_get_cf(db_import, roptions, cf_import, "k1", 2, &val_len, &err);
+    CheckNoError(err);
+    CheckEqual("v1", val, val_len);
+    free(val);
+
+    val =
+        rocksdb_get_cf(db_import, roptions, cf_import, "k2", 2, &val_len, &err);
+    CheckNoError(err);
+    CheckEqual("v2", val, val_len);
+    free(val);
+
+    rocksdb_column_family_handle_destroy(cf_import);
+    cf_import = NULL;
+    rocksdb_close(db_import);
+    rocksdb_destroy_db(db_options, db_import_path, &err);
+    CheckNoError(err);
+    rocksdb_options_destroy(db_options);
+    db_options = NULL;
+  }
+
  StartPhase("compactall");
  rocksdb_compact_range(db, NULL, 0, NULL, 0);
  CheckGet(db, roptions, "foo", "hello");
@@ -1120,17 +1120,15 @@ void CompactionIterator::NextFromInput() {
    }
  }

-  if (!Valid() && IsShuttingDown()) {
-    status_ = Status::ShutdownInProgress();
-  }
-
-  if (IsPausingManualCompaction()) {
-    status_ = Status::Incomplete(Status::SubCode::kManualCompactionPaused);
-  }
-
-  // Propagate corruption status from memtable itereator
-  if (!input_.Valid() && input_.status().IsCorruption()) {
-    status_ = input_.status();
+  if (status_.ok()) {
+    if (!Valid() && IsShuttingDown()) {
+      status_ = Status::ShutdownInProgress();
+    } else if (IsPausingManualCompaction()) {
+      status_ = Status::Incomplete(Status::SubCode::kManualCompactionPaused);
+    } else if (!input_.Valid() && input_.status().IsCorruption()) {
+      // Propagate corruption status from memtable iterator
+      status_ = input_.status();
+    }
  }
 }

@@ -183,10 +183,20 @@ class CompactionIterator {
    const Compaction* compaction_;
  };

-  // @param must_count_input_entries  if true, `NumInputEntryScanned()` will
-  // return the number of input keys scanned. If false, `NumInputEntryScanned()`
-  // will return this number if no Seek was called on `input`. User should call
-  // `HasNumInputEntryScanned()` first in this case.
+  // @param must_count_input_entries Controls input entry counting accuracy vs
+  // performance:
+  //   - If true: `NumInputEntryScanned()` always returns the exact count of
+  //   input keys
+  //     scanned. The iterator will use sequential `Next()` calls instead of
+  //     `Seek()` to maintain count accuracy as `Seek()` will not count the
+  //     skipped input entries, which is slower but guarantees correctness.
+  //   - If false: `NumInputEntryScanned()` returns the count only if no
+  //   `Seek()` operations
+  //     were performed on the input iterator. When compaction filters request
+  //     skipping ranges of keys or other optimizations trigger seek operations,
+  //     the count becomes unreliable. Always call `HasNumInputEntryScanned()`
+  //     first to verify if the count is accurate before using
+  //     `NumInputEntryScanned()`.
  CompactionIterator(
      InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
      SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
@@ -255,6 +265,10 @@ class CompactionIterator {
  }
  const CompactionIterationStats& iter_stats() const { return iter_stats_; }
  bool HasNumInputEntryScanned() const { return input_.HasNumItered(); }
+
+  // This method should only be used when `HasNumInputEntryScanned()` returns
+  // true, unless `must_count_input_entries=true` was specified during iterator
+  // creation (which ensures the count is always accurate).
  uint64_t NumInputEntryScanned() const { return input_.NumItered(); }
  Status InputStatus() const { return input_.status(); }

@@ -170,7 +170,11 @@ CompactionJob::CompactionJob(
      blob_output_directory_(blob_output_directory),
      db_mutex_(db_mutex),
      db_error_handler_(db_error_handler),
-      earliest_snapshot_(job_context->GetEarliestSnapshotSequence()),
+      // job_context cannot be nullptr, but we will assert later in the body of
+      // the constructor.
+      earliest_snapshot_(job_context
+                             ? job_context->GetEarliestSnapshotSequence()
+                             : kMaxSequenceNumber),
      job_context_(job_context),
      table_cache_(std::move(table_cache)),
      event_logger_(event_logger),
@@ -185,6 +189,7 @@ CompactionJob::CompactionJob(
      bg_bottom_compaction_scheduled_(bg_bottom_compaction_scheduled) {
  assert(job_stats_ != nullptr);
  assert(log_buffer_ != nullptr);
+  assert(job_context);
  assert(job_context->snapshot_context_initialized);

  const auto* cfd = compact_->compaction->column_family_data();
@@ -877,7 +882,7 @@ void CompactionJob::SetOutputTableProperties() {
  }
 }

-void CompactionJob::AggregateSubcompactionStats() {
+void CompactionJob::AggregateSubcompactionOutputAndJobStats() {
  // Before the compaction starts, is_remote_compaction was set to true if
  // compaction_service is set. We now know whether each sub_compaction was
  // done remotely or not. Reset is_remote_compaction back to false and allow
@@ -891,7 +896,8 @@ void CompactionJob::AggregateSubcompactionStats() {
 Status CompactionJob::VerifyCompactionRecordCounts(
    bool stats_built_from_input_table_prop, uint64_t num_input_range_del) {
  Status status;
-  if (stats_built_from_input_table_prop && job_stats_->has_num_input_records) {
+  if (stats_built_from_input_table_prop &&
+      job_stats_->has_accurate_num_input_records) {
    status = VerifyInputRecordCount(num_input_range_del);
    if (!status.ok()) {
      return status;
@@ -915,9 +921,10 @@ void CompactionJob::FinalizeCompactionRun(
    const Status& input_status, bool stats_built_from_input_table_prop,
    uint64_t num_input_range_del) {
  if (stats_built_from_input_table_prop) {
-    UpdateCompactionJobInputStats(internal_stats_, num_input_range_del);
+    UpdateCompactionJobInputStatsFromInternalStats(internal_stats_,
+                                                   num_input_range_del);
  }
-  UpdateCompactionJobOutputStats(internal_stats_);
+  UpdateCompactionJobOutputStatsFromInternalStats(internal_stats_);
  RecordCompactionIOStats();

  LogFlush(db_options_.info_log);
@@ -952,11 +959,11 @@ Status CompactionJob::Run() {
    SetOutputTableProperties();
  }

-  AggregateSubcompactionStats();
+  AggregateSubcompactionOutputAndJobStats();

  uint64_t num_input_range_del = 0;
  bool stats_built_from_input_table_prop =
-      BuildStatsFromInputFiles(&num_input_range_del);
+      UpdateInternalStatsFromInputFiles(&num_input_range_del);

  if (status.ok()) {
    status = VerifyCompactionRecordCounts(stats_built_from_input_table_prop,
@@ -1189,58 +1196,62 @@ void CompactionJob::NotifyOnSubcompactionCompleted(
  }
 }

-void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
-  assert(sub_compact);
-  assert(sub_compact->compaction);
+bool CompactionJob::ShouldUseLocalCompaction(SubcompactionState* sub_compact) {
  if (db_options_.compaction_service) {
    CompactionServiceJobStatus comp_status =
        ProcessKeyValueCompactionWithCompactionService(sub_compact);
    if (comp_status != CompactionServiceJobStatus::kUseLocal) {
-      return;
+      return false;
    }
    // fallback to local compaction
    assert(comp_status == CompactionServiceJobStatus::kUseLocal);
    sub_compact->compaction_job_stats.is_remote_compaction = false;
  }
+  return true;
+}

-  uint64_t prev_cpu_micros = db_options_.clock->CPUMicros();
+CompactionJob::CompactionIOStatsSnapshot CompactionJob::InitializeIOStats() {
+  CompactionIOStatsSnapshot io_stats;

-  ColumnFamilyData* cfd = sub_compact->compaction->column_family_data();
+  if (measure_io_stats_) {
+    io_stats.prev_perf_level = GetPerfLevel();
+    SetPerfLevel(PerfLevel::kEnableTimeAndCPUTimeExceptForMutex);
+    io_stats.prev_write_nanos = IOSTATS(write_nanos);
+    io_stats.prev_fsync_nanos = IOSTATS(fsync_nanos);
+    io_stats.prev_range_sync_nanos = IOSTATS(range_sync_nanos);
+    io_stats.prev_prepare_write_nanos = IOSTATS(prepare_write_nanos);
+    io_stats.prev_cpu_write_nanos = IOSTATS(cpu_write_nanos);
+    io_stats.prev_cpu_read_nanos = IOSTATS(cpu_read_nanos);
+  }
+
+  return io_stats;
+}
+
+Status CompactionJob::SetupAndValidateCompactionFilter(
+    SubcompactionState* sub_compact,
+    const CompactionFilter* configured_compaction_filter,
+    const CompactionFilter*& compaction_filter,
+    std::unique_ptr<CompactionFilter>& compaction_filter_from_factory) {
+  compaction_filter = configured_compaction_filter;

-  // Create compaction filter and fail the compaction if
-  // IgnoreSnapshots() = false because it is not supported anymore
-  const CompactionFilter* compaction_filter = cfd->ioptions().compaction_filter;
-  std::unique_ptr<CompactionFilter> compaction_filter_from_factory = nullptr;
  if (compaction_filter == nullptr) {
    compaction_filter_from_factory =
        sub_compact->compaction->CreateCompactionFilter();
    compaction_filter = compaction_filter_from_factory.get();
  }
+
  if (compaction_filter != nullptr && !compaction_filter->IgnoreSnapshots()) {
-    sub_compact->status = Status::NotSupported(
+    return Status::NotSupported(
        "CompactionFilter::IgnoreSnapshots() = false is not supported "
        "anymore.");
-    return;
  }

-  NotifyOnSubcompactionBegin(sub_compact);
+  return Status::OK();
+}

-  // This is assigned after creation of SubcompactionState to simplify that
-  // creation across both CompactionJob and CompactionServiceCompactionJob
-  sub_compact->AssignRangeDelAggregator(
-      std::make_unique<CompactionRangeDelAggregator>(
-          &cfd->internal_comparator(), job_context_->snapshot_seqs,
-          &full_history_ts_low_, &trim_ts_));
-
-  // TODO: since we already use C++17, should use
-  // std::optional<const Slice> instead.
-  const std::optional<Slice> start = sub_compact->start;
-  const std::optional<Slice> end = sub_compact->end;
-
-  std::optional<Slice> start_without_ts;
-  std::optional<Slice> end_without_ts;
-
-  ReadOptions read_options;
+void CompactionJob::InitializeReadOptions(
+    ColumnFamilyData* cfd, ReadOptions& read_options,
+    SubcompactionKeyBoundaries& boundaries) {
  read_options.verify_checksums = true;
  read_options.fill_cache = false;
  read_options.rate_limiter_priority = GetRateLimiterPriority();
@@ -1251,222 +1262,207 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
  // (b) CompactionFilter::Decision::kRemoveAndSkipUntil.
  read_options.total_order_seek = true;

-  const WriteOptions write_options(Env::IOPriority::IO_LOW,
-                                   Env::IOActivity::kCompaction);
-
  // Remove the timestamps from boundaries because boundaries created in
  // GenSubcompactionBoundaries doesn't strip away the timestamp.
-  size_t ts_sz = cfd->user_comparator()->timestamp_size();
-  if (start.has_value()) {
-    read_options.iterate_lower_bound = &(*start);
+  const size_t ts_sz = cfd->user_comparator()->timestamp_size();
+
+  if (boundaries.start.has_value()) {
+    read_options.iterate_lower_bound = &(*boundaries.start);
    if (ts_sz > 0) {
-      start_without_ts = StripTimestampFromUserKey(*start, ts_sz);
-      read_options.iterate_lower_bound = &(*start_without_ts);
+      boundaries.start_without_ts =
+          StripTimestampFromUserKey(*boundaries.start, ts_sz);
+      read_options.iterate_lower_bound = &(*boundaries.start_without_ts);
    }
  }
-  if (end.has_value()) {
-    read_options.iterate_upper_bound = &(*end);
+  if (boundaries.end.has_value()) {
+    read_options.iterate_upper_bound = &(*boundaries.end);
    if (ts_sz > 0) {
-      end_without_ts = StripTimestampFromUserKey(*end, ts_sz);
-      read_options.iterate_upper_bound = &(*end_without_ts);
+      boundaries.end_without_ts =
+          StripTimestampFromUserKey(*boundaries.end, ts_sz);
+      read_options.iterate_upper_bound = &(*boundaries.end_without_ts);
    }
  }
+}
+
+InternalIterator* CompactionJob::CreateInputIterator(
+    SubcompactionState* sub_compact, ColumnFamilyData* cfd,
+    SubcompactionInternalIterators& iterators,
+    SubcompactionKeyBoundaries& boundaries, ReadOptions& read_options) {
+  // This is assigned after creation of SubcompactionState to simplify that
+  // creation across both CompactionJob and CompactionServiceCompactionJob
+  sub_compact->AssignRangeDelAggregator(
+      std::make_unique<CompactionRangeDelAggregator>(
+          &cfd->internal_comparator(), job_context_->snapshot_seqs,
+          &full_history_ts_low_, &trim_ts_));
+
+  InitializeReadOptions(cfd, read_options, boundaries);

  // Although the v2 aggregator is what the level iterator(s) know about,
  // the AddTombstones calls will be propagated down to the v1 aggregator.
-  std::unique_ptr<InternalIterator> raw_input(versions_->MakeInputIterator(
-      read_options, sub_compact->compaction, sub_compact->RangeDelAgg(),
-      file_options_for_read_, start, end));
-  InternalIterator* input = raw_input.get();
+  iterators.raw_input =
+      std::unique_ptr<InternalIterator>(versions_->MakeInputIterator(
+          read_options, sub_compact->compaction, sub_compact->RangeDelAgg(),
+          file_options_for_read_, boundaries.start, boundaries.end));
+  InternalIterator* input = iterators.raw_input.get();

-  IterKey start_ikey;
-  IterKey end_ikey;
-  Slice start_slice;
-  Slice end_slice;
-  Slice start_user_key{};
-  Slice end_user_key{};
-
-  static constexpr char kMaxTs[] =
-      "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff";
-  Slice ts_slice;
-  std::string max_ts;
+  const size_t ts_sz = cfd->user_comparator()->timestamp_size();
  if (ts_sz > 0) {
-    if (ts_sz <= strlen(kMaxTs)) {
-      ts_slice = Slice(kMaxTs, ts_sz);
+    if (ts_sz <= strlen(boundaries.kMaxTs)) {
+      boundaries.ts_slice = Slice(boundaries.kMaxTs, ts_sz);
    } else {
-      max_ts = std::string(ts_sz, '\xff');
-      ts_slice = Slice(max_ts);
+      boundaries.max_ts = std::string(ts_sz, '\xff');
+      boundaries.ts_slice = Slice(boundaries.max_ts);
    }
  }

-  if (start.has_value()) {
-    start_ikey.SetInternalKey(*start, kMaxSequenceNumber, kValueTypeForSeek);
+  if (boundaries.start.has_value()) {
+    boundaries.start_ikey.SetInternalKey(*boundaries.start, kMaxSequenceNumber,
+                                         kValueTypeForSeek);
    if (ts_sz > 0) {
-      start_ikey.UpdateInternalKey(kMaxSequenceNumber, kValueTypeForSeek,
-                                   &ts_slice);
+      boundaries.start_ikey.UpdateInternalKey(
+          kMaxSequenceNumber, kValueTypeForSeek, &boundaries.ts_slice);
    }
-    start_slice = start_ikey.GetInternalKey();
-    start_user_key = start_ikey.GetUserKey();
+    boundaries.start_internal_key = boundaries.start_ikey.GetInternalKey();
+    boundaries.start_user_key = boundaries.start_ikey.GetUserKey();
  }
-  if (end.has_value()) {
-    end_ikey.SetInternalKey(*end, kMaxSequenceNumber, kValueTypeForSeek);
+  if (boundaries.end.has_value()) {
+    boundaries.end_ikey.SetInternalKey(*boundaries.end, kMaxSequenceNumber,
+                                       kValueTypeForSeek);
    if (ts_sz > 0) {
-      end_ikey.UpdateInternalKey(kMaxSequenceNumber, kValueTypeForSeek,
-                                 &ts_slice);
+      boundaries.end_ikey.UpdateInternalKey(
+          kMaxSequenceNumber, kValueTypeForSeek, &boundaries.ts_slice);
    }
-    end_slice = end_ikey.GetInternalKey();
-    end_user_key = end_ikey.GetUserKey();
+    boundaries.end_internal_key = boundaries.end_ikey.GetInternalKey();
+    boundaries.end_user_key = boundaries.end_ikey.GetUserKey();
  }

-  std::unique_ptr<InternalIterator> clip;
-  if (start.has_value() || end.has_value()) {
-    clip = std::make_unique<ClippingIterator>(
-        raw_input.get(), start.has_value() ? &start_slice : nullptr,
-        end.has_value() ? &end_slice : nullptr, &cfd->internal_comparator());
-    input = clip.get();
+  if (boundaries.start.has_value() || boundaries.end.has_value()) {
+    iterators.clip = std::make_unique<ClippingIterator>(
+        iterators.raw_input.get(),
+        boundaries.start.has_value() ? &boundaries.start_internal_key : nullptr,
+        boundaries.end.has_value() ? &boundaries.end_internal_key : nullptr,
+        &cfd->internal_comparator());
+    input = iterators.clip.get();
  }

-  std::unique_ptr<InternalIterator> blob_counter;
-
  if (sub_compact->compaction->DoesInputReferenceBlobFiles()) {
    BlobGarbageMeter* meter = sub_compact->Current().CreateBlobGarbageMeter();
-    blob_counter = std::make_unique<BlobCountingIterator>(input, meter);
-    input = blob_counter.get();
+    iterators.blob_counter =
+        std::make_unique<BlobCountingIterator>(input, meter);
+    input = iterators.blob_counter.get();
  }

-  std::unique_ptr<InternalIterator> trim_history_iter;
  if (ts_sz > 0 && !trim_ts_.empty()) {
-    trim_history_iter = std::make_unique<HistoryTrimmingIterator>(
+    iterators.trim_history_iter = std::make_unique<HistoryTrimmingIterator>(
        input, cfd->user_comparator(), trim_ts_);
-    input = trim_history_iter.get();
+    input = iterators.trim_history_iter.get();
  }

-  input->SeekToFirst();
-
-  AutoThreadOperationStageUpdater stage_updater(
-      ThreadStatus::STAGE_COMPACTION_PROCESS_KV);
-
-  // I/O measurement variables
-  PerfLevel prev_perf_level = PerfLevel::kEnableTime;
-  const uint64_t kRecordStatsEvery = 1000;
-  uint64_t prev_write_nanos = 0;
-  uint64_t prev_fsync_nanos = 0;
-  uint64_t prev_range_sync_nanos = 0;
-  uint64_t prev_prepare_write_nanos = 0;
-  uint64_t prev_cpu_write_nanos = 0;
-  uint64_t prev_cpu_read_nanos = 0;
-  if (measure_io_stats_) {
-    prev_perf_level = GetPerfLevel();
-    SetPerfLevel(PerfLevel::kEnableTimeAndCPUTimeExceptForMutex);
-    prev_write_nanos = IOSTATS(write_nanos);
-    prev_fsync_nanos = IOSTATS(fsync_nanos);
-    prev_range_sync_nanos = IOSTATS(range_sync_nanos);
-    prev_prepare_write_nanos = IOSTATS(prepare_write_nanos);
-    prev_cpu_write_nanos = IOSTATS(cpu_write_nanos);
-    prev_cpu_read_nanos = IOSTATS(cpu_read_nanos);
-  }
-
-  MergeHelper merge(
-      env_, cfd->user_comparator(), cfd->ioptions().merge_operator.get(),
-      compaction_filter, db_options_.info_log.get(),
-      false /* internal key corruption is expected */,
-      job_context_->GetLatestSnapshotSequence(), job_context_->snapshot_checker,
-      compact_->compaction->level(), db_options_.stats);
+  return input;
+}

+void CompactionJob::CreateBlobFileBuilder(SubcompactionState* sub_compact,
+                                          ColumnFamilyData* cfd,
+                                          BlobFileResources& blob_resources,
+                                          const WriteOptions& write_options) {
  const auto& mutable_cf_options =
      sub_compact->compaction->mutable_cf_options();

-  std::vector<std::string> blob_file_paths;
-
  // TODO: BlobDB to support output_to_proximal_level compaction, which needs
  //  2 builders, so may need to move to `CompactionOutputs`
-  std::unique_ptr<BlobFileBuilder> blob_file_builder(
-      (mutable_cf_options.enable_blob_files &&
-       sub_compact->compaction->output_level() >=
-           mutable_cf_options.blob_file_starting_level)
-          ? new BlobFileBuilder(
-                versions_, fs_.get(),
-                &sub_compact->compaction->immutable_options(),
-                &mutable_cf_options, &file_options_, &write_options, db_id_,
-                db_session_id_, job_id_, cfd->GetID(), cfd->GetName(),
-                write_hint_, io_tracer_, blob_callback_,
-                BlobFileCreationReason::kCompaction, &blob_file_paths,
-                sub_compact->Current().GetBlobFileAdditionsPtr())
-          : nullptr);
+  if (mutable_cf_options.enable_blob_files &&
+      sub_compact->compaction->output_level() >=
+          mutable_cf_options.blob_file_starting_level) {
+    blob_resources.blob_file_builder = std::make_unique<BlobFileBuilder>(
+        versions_, fs_.get(), &sub_compact->compaction->immutable_options(),
+        &mutable_cf_options, &file_options_, &write_options, db_id_,
+        db_session_id_, job_id_, cfd->GetID(), cfd->GetName(), write_hint_,
+        io_tracer_, blob_callback_, BlobFileCreationReason::kCompaction,
+        &blob_resources.blob_file_paths,
+        sub_compact->Current().GetBlobFileAdditionsPtr());
+  } else {
+    blob_resources.blob_file_builder = nullptr;
+  }
+}

-  TEST_SYNC_POINT("CompactionJob::Run():Inprogress");
-  TEST_SYNC_POINT_CALLBACK("CompactionJob::Run():PausingManualCompaction:1",
-                           static_cast<void*>(const_cast<std::atomic<bool>*>(
-                               &manual_compaction_canceled_)));
+std::unique_ptr<CompactionIterator> CompactionJob::CreateCompactionIterator(
+    SubcompactionState* sub_compact, ColumnFamilyData* cfd,
+    InternalIterator* input, const CompactionFilter* compaction_filter,
+    MergeHelper& merge, BlobFileResources& blob_resources,
+    const WriteOptions& write_options) {
+  CreateBlobFileBuilder(sub_compact, cfd, blob_resources, write_options);

  const std::string* const full_history_ts_low =
      full_history_ts_low_.empty() ? nullptr : &full_history_ts_low_;
-  const SequenceNumber job_snapshot_seq =
-      job_context_ ? job_context_->GetJobSnapshotSequence()
-                   : kMaxSequenceNumber;
+  assert(job_context_);

-  auto c_iter = std::make_unique<CompactionIterator>(
+  return std::make_unique<CompactionIterator>(
      input, cfd->user_comparator(), &merge, versions_->LastSequence(),
      &(job_context_->snapshot_seqs), earliest_snapshot_,
-      job_context_->earliest_write_conflict_snapshot, job_snapshot_seq,
-      job_context_->snapshot_checker, env_,
-      ShouldReportDetailedTime(env_, stats_), sub_compact->RangeDelAgg(),
-      blob_file_builder.get(), db_options_.allow_data_in_errors,
+      job_context_->earliest_write_conflict_snapshot,
+      job_context_->GetJobSnapshotSequence(), job_context_->snapshot_checker,
+      env_, ShouldReportDetailedTime(env_, stats_), sub_compact->RangeDelAgg(),
+      blob_resources.blob_file_builder.get(), db_options_.allow_data_in_errors,
      db_options_.enforce_single_del_contracts, manual_compaction_canceled_,
-      sub_compact->compaction
-          ->DoesInputReferenceBlobFiles() /* must_count_input_entries */,
+      sub_compact->compaction->DoesInputReferenceBlobFiles(),
      sub_compact->compaction, compaction_filter, shutting_down_,
      db_options_.info_log, full_history_ts_low, preserve_seqno_after_);
-  c_iter->SeekToFirst();
+}

-  const auto& c_iter_stats = c_iter->iter_stats();
-
-  // define the open and close functions for the compaction files, which will be
-  // used open/close output files when needed.
+std::pair<CompactionFileOpenFunc, CompactionFileCloseFunc>
+CompactionJob::CreateFileHandlers(SubcompactionState* sub_compact,
+                                  SubcompactionKeyBoundaries& boundaries) {
  const CompactionFileOpenFunc open_file_func =
      [this, sub_compact](CompactionOutputs& outputs) {
        return this->OpenCompactionOutputFile(sub_compact, outputs);
      };

+  const Slice* start_user_key =
+      sub_compact->start.has_value() ? &boundaries.start_user_key : nullptr;
+  const Slice* end_user_key =
+      sub_compact->end.has_value() ? &boundaries.end_user_key : nullptr;
+
  const CompactionFileCloseFunc close_file_func =
      [this, sub_compact, start_user_key, end_user_key](
          CompactionOutputs& outputs, const Status& status,
          const Slice& next_table_min_key) {
-        return this->FinishCompactionOutputFile(
-            status, sub_compact, outputs, next_table_min_key,
-            sub_compact->start.has_value() ? &start_user_key : nullptr,
-            sub_compact->end.has_value() ? &end_user_key : nullptr);
+        return this->FinishCompactionOutputFile(status, sub_compact, outputs,
+                                                next_table_min_key,
+                                                start_user_key, end_user_key);
      };

+  return {open_file_func, close_file_func};
+}
+
+Status CompactionJob::ProcessKeyValue(
+    SubcompactionState* sub_compact, ColumnFamilyData* cfd,
+    CompactionIterator* c_iter, const CompactionFileOpenFunc& open_file_func,
+    const CompactionFileCloseFunc& close_file_func, uint64_t& prev_cpu_micros) {
  Status status;
+  const uint64_t kRecordStatsEvery = 1000;
+  [[maybe_unused]] const std::optional<const Slice> end = sub_compact->end;
+
  TEST_SYNC_POINT_CALLBACK(
      "CompactionJob::ProcessKeyValueCompaction()::Processing",
      static_cast<void*>(const_cast<Compaction*>(sub_compact->compaction)));
-  uint64_t last_cpu_micros = prev_cpu_micros;
-  while (status.ok() && !cfd->IsDropped() && c_iter->Valid()) {
-    // Invariant: c_iter.status() is guaranteed to be OK if c_iter->Valid()
-    // returns true.
+
+  while (status.ok() && !cfd->IsDropped() && c_iter->Valid() &&
+         c_iter->status().ok()) {
    assert(!end.has_value() ||
           cfd->user_comparator()->Compare(c_iter->user_key(), *end) < 0);

-    if (c_iter_stats.num_input_records % kRecordStatsEvery ==
+    if (c_iter->iter_stats().num_input_records % kRecordStatsEvery ==
        kRecordStatsEvery - 1) {
-      RecordDroppedKeys(c_iter_stats, &sub_compact->compaction_job_stats);
-      c_iter->ResetRecordCounts();
-      RecordCompactionIOStats();
-
-      uint64_t cur_cpu_micros = db_options_.clock->CPUMicros();
-      assert(cur_cpu_micros >= last_cpu_micros);
-      RecordTick(stats_, COMPACTION_CPU_TOTAL_TIME,
-                 cur_cpu_micros - last_cpu_micros);
-      last_cpu_micros = cur_cpu_micros;
+      UpdateSubcompactionJobStatsIncrementally(
+          c_iter, &sub_compact->compaction_job_stats,
+          db_options_.clock->CPUMicros(), prev_cpu_micros);
    }

    const auto& ikey = c_iter->ikey();
    bool use_proximal_output = ikey.sequence > proximal_after_seqno_;
+
 #ifndef NDEBUG
    if (sub_compact->compaction->SupportsPerKeyPlacement()) {
-      // Could be overridden by unittest
      PerKeyPlacementContext context(sub_compact->compaction->output_level(),
                                     ikey.user_key, c_iter->value(),
                                     ikey.sequence, use_proximal_output);
@@ -1505,9 +1501,6 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
                             static_cast<void*>(const_cast<std::atomic<bool>*>(
                                 &manual_compaction_canceled_)));
    c_iter->Next();
-    if (c_iter->status().IsManualCompactionPaused()) {
-      break;
-    }

 #ifndef NDEBUG
    bool stop = false;
@@ -1519,13 +1512,33 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
 #endif  // NDEBUG
  }

-  // This number may not be accurate when CompactionIterator was created
-  // with `must_count_input_entries=false`.
+  return status;
+}
+
+void CompactionJob::UpdateSubcompactionJobStatsIncrementally(
+    CompactionIterator* c_iter, CompactionJobStats* compaction_job_stats,
+    uint64_t cur_cpu_micros, uint64_t& prev_cpu_micros) {
+  RecordDroppedKeys(c_iter->iter_stats(), compaction_job_stats);
+  c_iter->ResetRecordCounts();
+  RecordCompactionIOStats();
+
+  assert(cur_cpu_micros >= prev_cpu_micros);
+  RecordTick(stats_, COMPACTION_CPU_TOTAL_TIME,
+             cur_cpu_micros - prev_cpu_micros);
+  prev_cpu_micros = cur_cpu_micros;
+}
+
+void CompactionJob::FinalizeSubcompactionJobStats(
+    SubcompactionState* sub_compact, CompactionIterator* c_iter,
+    uint64_t start_cpu_micros, uint64_t prev_cpu_micros,
+    const CompactionIOStatsSnapshot& io_stats) {
+  const CompactionIterationStats& c_iter_stats = c_iter->iter_stats();
+
  assert(!sub_compact->compaction->DoesInputReferenceBlobFiles() ||
         c_iter->HasNumInputEntryScanned());
-  sub_compact->compaction_job_stats.has_num_input_records =
+  sub_compact->compaction_job_stats.has_accurate_num_input_records &=
      c_iter->HasNumInputEntryScanned();
-  sub_compact->compaction_job_stats.num_input_records =
+  sub_compact->compaction_job_stats.num_input_records +=
      c_iter->NumInputEntryScanned();
  sub_compact->compaction_job_stats.num_blobs_read =
      c_iter_stats.num_blobs_read;
@@ -1556,84 +1569,188 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
               c_iter_stats.total_blob_bytes_relocated);
  }

-  RecordDroppedKeys(c_iter_stats, &sub_compact->compaction_job_stats);
-  RecordCompactionIOStats();
+  uint64_t cur_cpu_micros = db_options_.clock->CPUMicros();

+  // Record final compaction statistics including dropped keys, I/O stats,
+  // and CPU time delta from the last periodic measurement
+  UpdateSubcompactionJobStatsIncrementally(c_iter,
+                                           &sub_compact->compaction_job_stats,
+                                           cur_cpu_micros, prev_cpu_micros);
+
+  // Finalize timing and I/O statistics
+  sub_compact->compaction_job_stats.cpu_micros =
+      cur_cpu_micros - start_cpu_micros + sub_compact->GetWorkerCPUMicros();
+
+  if (measure_io_stats_) {
+    sub_compact->compaction_job_stats.file_write_nanos +=
+        IOSTATS(write_nanos) - io_stats.prev_write_nanos;
+    sub_compact->compaction_job_stats.file_fsync_nanos +=
+        IOSTATS(fsync_nanos) - io_stats.prev_fsync_nanos;
+    sub_compact->compaction_job_stats.file_range_sync_nanos +=
+        IOSTATS(range_sync_nanos) - io_stats.prev_range_sync_nanos;
+    sub_compact->compaction_job_stats.file_prepare_write_nanos +=
+        IOSTATS(prepare_write_nanos) - io_stats.prev_prepare_write_nanos;
+    sub_compact->compaction_job_stats.cpu_micros -=
+        (IOSTATS(cpu_write_nanos) - io_stats.prev_cpu_write_nanos +
+         IOSTATS(cpu_read_nanos) - io_stats.prev_cpu_read_nanos) /
+        1000;
+    if (io_stats.prev_perf_level !=
+        PerfLevel::kEnableTimeAndCPUTimeExceptForMutex) {
+      SetPerfLevel(io_stats.prev_perf_level);
+    }
+  }
+}
+
+Status CompactionJob::FinalizeProcessKeyValueStatus(
+    ColumnFamilyData* cfd, InternalIterator* input_iter,
+    CompactionIterator* c_iter, Status status) {
  if (status.ok() && cfd->IsDropped()) {
    status =
        Status::ColumnFamilyDropped("Column family dropped during compaction");
  }
-  if ((status.ok() || status.IsColumnFamilyDropped()) &&
-      shutting_down_->load(std::memory_order_relaxed)) {
+  if (status.ok() && shutting_down_->load(std::memory_order_relaxed)) {
    status = Status::ShutdownInProgress("Database shutdown");
  }
-  if ((status.ok() || status.IsColumnFamilyDropped()) &&
+  if (status.ok() &&
      (manual_compaction_canceled_.load(std::memory_order_relaxed))) {
    status = Status::Incomplete(Status::SubCode::kManualCompactionPaused);
  }
  if (status.ok()) {
-    status = input->status();
+    status = input_iter->status();
  }
  if (status.ok()) {
    status = c_iter->status();
  }

+  return status;
+}
+
+Status CompactionJob::CleanupCompactionFiles(
+    SubcompactionState* sub_compact, Status status,
+    const CompactionFileOpenFunc& open_file_func,
+    const CompactionFileCloseFunc& close_file_func) {
  // Call FinishCompactionOutputFile() even if status is not ok: it needs to
  // close the output files. Open file function is also passed, in case there's
  // only range-dels, no file was opened, to save the range-dels, it need to
  // create a new output file.
-  status = sub_compact->CloseCompactionFiles(status, open_file_func,
-                                             close_file_func);
+  return sub_compact->CloseCompactionFiles(status, open_file_func,
+                                           close_file_func);
+}

+Status CompactionJob::FinalizeBlobFiles(SubcompactionState* sub_compact,
+                                        BlobFileBuilder* blob_file_builder,
+                                        Status status) {
  if (blob_file_builder) {
    if (status.ok()) {
      status = blob_file_builder->Finish();
    } else {
      blob_file_builder->Abandon(status);
    }
-    blob_file_builder.reset();
    sub_compact->Current().UpdateBlobStats();
  }

-  uint64_t cur_cpu_micros = db_options_.clock->CPUMicros();
-  sub_compact->compaction_job_stats.cpu_micros =
-      cur_cpu_micros - prev_cpu_micros;
-  RecordTick(stats_, COMPACTION_CPU_TOTAL_TIME,
-             cur_cpu_micros - last_cpu_micros);
+  return status;
+}

-  if (measure_io_stats_) {
-    sub_compact->compaction_job_stats.file_write_nanos +=
-        IOSTATS(write_nanos) - prev_write_nanos;
-    sub_compact->compaction_job_stats.file_fsync_nanos +=
-        IOSTATS(fsync_nanos) - prev_fsync_nanos;
-    sub_compact->compaction_job_stats.file_range_sync_nanos +=
-        IOSTATS(range_sync_nanos) - prev_range_sync_nanos;
-    sub_compact->compaction_job_stats.file_prepare_write_nanos +=
-        IOSTATS(prepare_write_nanos) - prev_prepare_write_nanos;
-    sub_compact->compaction_job_stats.cpu_micros -=
-        (IOSTATS(cpu_write_nanos) - prev_cpu_write_nanos +
-         IOSTATS(cpu_read_nanos) - prev_cpu_read_nanos) /
-        1000;
-    if (prev_perf_level != PerfLevel::kEnableTimeAndCPUTimeExceptForMutex) {
-      SetPerfLevel(prev_perf_level);
-    }
+void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
+  assert(sub_compact);
+  assert(sub_compact->compaction);
+
+  if (!ShouldUseLocalCompaction(sub_compact)) {
+    return;
  }
+
+  AutoThreadOperationStageUpdater stage_updater(
+      ThreadStatus::STAGE_COMPACTION_PROCESS_KV);
+
+  const uint64_t start_cpu_micros = db_options_.clock->CPUMicros();
+  uint64_t prev_cpu_micros = start_cpu_micros;
+  const CompactionIOStatsSnapshot io_stats = InitializeIOStats();
+  ColumnFamilyData* cfd = sub_compact->compaction->column_family_data();
+  const CompactionFilter* compaction_filter;
+  std::unique_ptr<CompactionFilter> compaction_filter_from_factory = nullptr;
+  Status filter_status = SetupAndValidateCompactionFilter(
+      sub_compact, cfd->ioptions().compaction_filter, compaction_filter,
+      compaction_filter_from_factory);
+  if (!filter_status.ok()) {
+    sub_compact->status = filter_status;
+    return;
+  }
+
+  NotifyOnSubcompactionBegin(sub_compact);
+
+  SubcompactionKeyBoundaries boundaries(sub_compact->start, sub_compact->end);
+  SubcompactionInternalIterators iterators;
+  ReadOptions read_options;
+  const WriteOptions write_options(Env::IOPriority::IO_LOW,
+                                   Env::IOActivity::kCompaction);
+  MergeHelper merge(
+      env_, cfd->user_comparator(), cfd->ioptions().merge_operator.get(),
+      compaction_filter, db_options_.info_log.get(),
+      false /* internal key corruption is expected */,
+      job_context_->GetLatestSnapshotSequence(), job_context_->snapshot_checker,
+      compact_->compaction->level(), db_options_.stats);
+  BlobFileResources blob_resources;
+
+  InternalIterator* input_iter = CreateInputIterator(
+      sub_compact, cfd, iterators, boundaries, read_options);
+  assert(input_iter);
+  input_iter->SeekToFirst();
+
+  auto c_iter =
+      CreateCompactionIterator(sub_compact, cfd, input_iter, compaction_filter,
+                               merge, blob_resources, write_options);
+  assert(c_iter);
+  c_iter->SeekToFirst();
+
+  TEST_SYNC_POINT("CompactionJob::Run():Inprogress");
+  TEST_SYNC_POINT_CALLBACK("CompactionJob::Run():PausingManualCompaction:1",
+                           static_cast<void*>(const_cast<std::atomic<bool>*>(
+                               &manual_compaction_canceled_)));
+
+  auto [open_file_func, close_file_func] =
+      CreateFileHandlers(sub_compact, boundaries);
+
+  Status status =
+      ProcessKeyValue(sub_compact, cfd, c_iter.get(), open_file_func,
+                      close_file_func, prev_cpu_micros);
+
+  status = FinalizeProcessKeyValueStatus(cfd, input_iter, c_iter.get(), status);
+
+  FinalizeSubcompaction(sub_compact, status, open_file_func, close_file_func,
+                        blob_resources.blob_file_builder.get(), c_iter.get(),
+                        input_iter, start_cpu_micros, prev_cpu_micros,
+                        io_stats);
+
+  NotifyOnSubcompactionCompleted(sub_compact);
+}
+
+void CompactionJob::FinalizeSubcompaction(
+    SubcompactionState* sub_compact, Status status,
+    const CompactionFileOpenFunc& open_file_func,
+    const CompactionFileCloseFunc& close_file_func,
+    BlobFileBuilder* blob_file_builder, CompactionIterator* c_iter,
+    [[maybe_unused]] InternalIterator* input_iter, uint64_t start_cpu_micros,
+    uint64_t prev_cpu_micros, const CompactionIOStatsSnapshot& io_stats) {
+  status = CleanupCompactionFiles(sub_compact, status, open_file_func,
+                                  close_file_func);
+  status = FinalizeBlobFiles(sub_compact, blob_file_builder, status);
+
+  FinalizeSubcompactionJobStats(sub_compact, c_iter, start_cpu_micros,
+                                prev_cpu_micros, io_stats);
+
 #ifdef ROCKSDB_ASSERT_STATUS_CHECKED
  if (!status.ok()) {
    if (c_iter) {
      c_iter->status().PermitUncheckedError();
    }
-    if (input) {
-      input->status().PermitUncheckedError();
+    if (input_iter) {
+      input_iter->status().PermitUncheckedError();
    }
  }
 #endif  // ROCKSDB_ASSERT_STATUS_CHECKED

-  blob_counter.reset();
-  clip.reset();
-  raw_input.reset();
  sub_compact->status = status;
-  NotifyOnSubcompactionCompleted(sub_compact);
 }

 uint64_t CompactionJob::GetCompactionId(SubcompactionState* sub_compact) const {
@@ -2150,7 +2267,8 @@ void CopyPrefix(const Slice& src, size_t prefix_length, std::string* dst) {
 }
 }  // namespace

-bool CompactionJob::BuildStatsFromInputFiles(uint64_t* num_input_range_del) {
+bool CompactionJob::UpdateInternalStatsFromInputFiles(
+    uint64_t* num_input_range_del) {
  assert(compact_);

  Compaction* compaction = compact_->compaction;
@@ -2232,7 +2350,7 @@ bool CompactionJob::BuildStatsFromInputFiles(uint64_t* num_input_range_del) {
  return !has_error;
 }

-void CompactionJob::UpdateCompactionJobInputStats(
+void CompactionJob::UpdateCompactionJobInputStatsFromInternalStats(
    const InternalStats::CompactionStatsFull& internal_stats,
    uint64_t num_input_range_del) const {
  assert(job_stats_);
@@ -2285,7 +2403,7 @@ void CompactionJob::UpdateCompactionJobInputStats(
  }
 }

-void CompactionJob::UpdateCompactionJobOutputStats(
+void CompactionJob::UpdateCompactionJobOutputStatsFromInternalStats(
    const InternalStats::CompactionStatsFull& internal_stats) const {
  assert(job_stats_);
  job_stats_->elapsed_micros = internal_stats.output_level_stats.micros;
@@ -196,7 +196,7 @@ class CompactionJob {
  IOStatus io_status() const { return io_status_; }

 protected:
-  void UpdateCompactionJobOutputStats(
+  void UpdateCompactionJobOutputStatsFromInternalStats(
      const InternalStats::CompactionStatsFull& internal_stats) const;

  void LogCompaction();
@@ -242,9 +242,10 @@ class CompactionJob {
  // num_input_range_del are calculated successfully.
  //
  // This should be called only once for compactions (not per subcompaction)
-  bool BuildStatsFromInputFiles(uint64_t* num_input_range_del = nullptr);
+  bool UpdateInternalStatsFromInputFiles(
+      uint64_t* num_input_range_del = nullptr);

-  void UpdateCompactionJobInputStats(
+  void UpdateCompactionJobInputStatsFromInternalStats(
      const InternalStats::CompactionStatsFull& internal_stats,
      uint64_t num_input_range_del) const;

@@ -287,7 +288,10 @@ class CompactionJob {
  Status SyncOutputDirectories();
  Status VerifyOutputFiles();
  void SetOutputTableProperties();
-  void AggregateSubcompactionStats();
+  // Aggregates subcompaction output stats to internal stat, and aggregates
+  // subcompaction's compaction job stats to the whole entire surrounding
+  // compaction job stats.
+  void AggregateSubcompactionOutputAndJobStats();
  Status VerifyCompactionRecordCounts(bool stats_built_from_input_table_prop,
                                      uint64_t num_input_range_del);
  void FinalizeCompactionRun(const Status& status,
@@ -297,6 +301,113 @@ class CompactionJob {
  CompactionServiceJobStatus ProcessKeyValueCompactionWithCompactionService(
      SubcompactionState* sub_compact);

+  struct CompactionIOStatsSnapshot {
+    PerfLevel prev_perf_level = PerfLevel::kEnableTime;
+    uint64_t prev_write_nanos = 0;
+    uint64_t prev_fsync_nanos = 0;
+    uint64_t prev_range_sync_nanos = 0;
+    uint64_t prev_prepare_write_nanos = 0;
+    uint64_t prev_cpu_write_nanos = 0;
+    uint64_t prev_cpu_read_nanos = 0;
+  };
+
+  struct SubcompactionKeyBoundaries {
+    const std::optional<const Slice> start;
+    const std::optional<const Slice> end;
+
+    // Boundaries without timestamps for read options
+    std::optional<Slice> start_without_ts;
+    std::optional<Slice> end_without_ts;
+
+    // Timestamp management
+    static constexpr char kMaxTs[] =
+        "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff";
+    std::string max_ts;
+    Slice ts_slice;
+
+    // Internal key boundaries
+    IterKey start_ikey;
+    IterKey end_ikey;
+    Slice start_internal_key;
+    Slice end_internal_key;
+
+    // User key boundaries
+    Slice start_user_key;
+    Slice end_user_key;
+
+    SubcompactionKeyBoundaries(std::optional<const Slice> start_boundary,
+                               std::optional<const Slice> end_boundary)
+        : start(start_boundary), end(end_boundary) {}
+  };
+
+  struct SubcompactionInternalIterators {
+    std::unique_ptr<InternalIterator> raw_input;
+    std::unique_ptr<InternalIterator> clip;
+    std::unique_ptr<InternalIterator> blob_counter;
+    std::unique_ptr<InternalIterator> trim_history_iter;
+  };
+
+  struct BlobFileResources {
+    std::vector<std::string> blob_file_paths;
+    std::unique_ptr<BlobFileBuilder> blob_file_builder;
+  };
+
+  bool ShouldUseLocalCompaction(SubcompactionState* sub_compact);
+  CompactionIOStatsSnapshot InitializeIOStats();
+  Status SetupAndValidateCompactionFilter(
+      SubcompactionState* sub_compact,
+      const CompactionFilter* configured_compaction_filter,
+      const CompactionFilter*& compaction_filter,
+      std::unique_ptr<CompactionFilter>& compaction_filter_from_factory);
+  void InitializeReadOptions(ColumnFamilyData* cfd, ReadOptions& read_options,
+                             SubcompactionKeyBoundaries& boundaries);
+  InternalIterator* CreateInputIterator(
+      SubcompactionState* sub_compact, ColumnFamilyData* cfd,
+      SubcompactionInternalIterators& iterators,
+      SubcompactionKeyBoundaries& boundaries, ReadOptions& read_options);
+  void CreateBlobFileBuilder(SubcompactionState* sub_compact,
+                             ColumnFamilyData* cfd,
+                             BlobFileResources& blob_resources,
+                             const WriteOptions& write_options);
+  std::unique_ptr<CompactionIterator> CreateCompactionIterator(
+      SubcompactionState* sub_compact, ColumnFamilyData* cfd,
+      InternalIterator* input_iter, const CompactionFilter* compaction_filter,
+      MergeHelper& merge, BlobFileResources& blob_resources,
+      const WriteOptions& write_options);
+  std::pair<CompactionFileOpenFunc, CompactionFileCloseFunc> CreateFileHandlers(
+      SubcompactionState* sub_compact, SubcompactionKeyBoundaries& boundaries);
+  Status ProcessKeyValue(SubcompactionState* sub_compact, ColumnFamilyData* cfd,
+                         CompactionIterator* c_iter,
+                         const CompactionFileOpenFunc& open_file_func,
+                         const CompactionFileCloseFunc& close_file_func,
+                         uint64_t& prev_cpu_micros);
+  void UpdateSubcompactionJobStatsIncrementally(
+      CompactionIterator* c_iter, CompactionJobStats* compaction_job_stats,
+      uint64_t cur_cpu_micros, uint64_t& prev_cpu_micros);
+  void FinalizeSubcompactionJobStats(SubcompactionState* sub_compact,
+                                     CompactionIterator* c_iter,
+                                     uint64_t start_cpu_micros,
+                                     uint64_t prev_cpu_micros,
+                                     const CompactionIOStatsSnapshot& io_stats);
+  Status FinalizeProcessKeyValueStatus(ColumnFamilyData* cfd,
+                                       InternalIterator* input_iter,
+                                       CompactionIterator* c_iter,
+                                       Status status);
+  Status CleanupCompactionFiles(SubcompactionState* sub_compact, Status status,
+                                const CompactionFileOpenFunc& open_file_func,
+                                const CompactionFileCloseFunc& close_file_func);
+  Status FinalizeBlobFiles(SubcompactionState* sub_compact,
+                           BlobFileBuilder* blob_file_builder, Status status);
+  void FinalizeSubcompaction(SubcompactionState* sub_compact, Status status,
+                             const CompactionFileOpenFunc& open_file_func,
+                             const CompactionFileCloseFunc& close_file_func,
+                             BlobFileBuilder* blob_file_builder,
+                             CompactionIterator* c_iter,
+                             InternalIterator* input_iter,
+                             uint64_t start_cpu_micros,
+                             uint64_t prev_cpu_micros,
+                             const CompactionIOStatsSnapshot& io_stats);
+
  // update the thread status for starting a compaction.
  void ReportStartedCompaction(Compaction* compaction);

@@ -56,6 +56,7 @@ Status CompactionOutputs::Finish(
  stats_.bytes_written += current_bytes;
  stats_.bytes_written_pre_comp += builder_->PreCompressionSize();
  stats_.num_output_files = static_cast<int>(outputs_.size());
+  worker_cpu_micros_ += builder_->GetWorkerCPUMicros();

  return s;
 }
@@ -168,6 +168,10 @@ class CompactionOutputs {

  uint64_t NumEntries() const { return builder_->NumEntries(); }

+  uint64_t GetWorkerCPUMicros() const {
+    return worker_cpu_micros_ + (builder_ ? builder_->GetWorkerCPUMicros() : 0);
+  }
+
  void ResetBuilder() {
    builder_.reset();
    current_output_file_size_ = 0;
@@ -296,6 +300,9 @@ class CompactionOutputs {
  uint64_t current_output_file_size_ = 0;
  SequenceNumber smallest_preferred_seqno_ = kMaxSequenceNumber;

+  // Sum of all the GetWorkerCPUMicros() for all the closed builders so far.
+  uint64_t worker_cpu_micros_ = 0;
+
  // all the compaction outputs so far
  std::vector<Output> outputs_;

@@ -258,6 +258,9 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction(
    // better serves a major type of FIFO use cases where smaller keys are
    // associated with older data.
    for (const auto& f : last_level_files) {
+      if (f->being_compacted) {
+        continue;
+      }
      total_size -= f->fd.file_size;
      inputs[0].files.push_back(f);
      char tmp_fsize[16];
@@ -387,12 +390,14 @@ Compaction* FIFOCompactionPicker::PickTemperatureChangeCompaction(
      assert(compaction_target_temp == Temperature::kLastTemperature);
      compaction_target_temp = cur_target_temp;
      inputs[0].files.push_back(cur_file);
-      ROCKS_LOG_BUFFER(
-          log_buffer,
-          "[%s] FIFO compaction: picking file %" PRIu64
-          " with estimated newest key time %" PRIu64 " for temperature %s.",
-          cf_name.c_str(), cur_file->fd.GetNumber(), est_newest_key_time,
-          temperature_to_string[cur_target_temp].c_str());
+      ROCKS_LOG_BUFFER(log_buffer,
+                       "[%s] FIFO compaction: picking file %" PRIu64
+                       " with estimated newest key time %" PRIu64
+                       " and temperature %s for temperature %s.",
+                       cf_name.c_str(), cur_file->fd.GetNumber(),
+                       est_newest_key_time,
+                       temperature_to_string[cur_file->temperature].c_str(),
+                       temperature_to_string[cur_target_temp].c_str());
      break;
    }
  }
@@ -408,7 +408,7 @@ Status CompactionServiceCompactionJob::Run() {
  // 2. Update job-level output stats with the aggregated internal_stats_
  // Please note that input stats will be updated by primary host when all
  // subcompactions are finished
-  UpdateCompactionJobOutputStats(internal_stats_);
+  UpdateCompactionJobOutputStatsFromInternalStats(internal_stats_);
  // and set fields that are not propagated as part of the update
  compaction_result_->stats.is_manual_compaction = c->is_manual_compaction();
  compaction_result_->stats.is_full_compaction = c->is_full_compaction();
@@ -191,6 +191,14 @@ class SubcompactionState {
    return &compaction_outputs_.stats_;
  }

+  uint64_t GetWorkerCPUMicros() const {
+    uint64_t rv = compaction_outputs_.GetWorkerCPUMicros();
+    if (compaction->SupportsPerKeyPlacement()) {
+      rv += proximal_level_outputs_.GetWorkerCPUMicros();
+    }
+    return rv;
+  }
+
  CompactionRangeDelAggregator* RangeDelAgg() const {
    return range_del_agg_.get();
  }
@@ -1764,7 +1764,10 @@ TEST_P(PrecludeLastLevelTest, SmallPrecludeTime) {
  options.env = mock_env_.get();
  options.level0_file_num_compaction_trigger = kNumTrigger;
  options.num_levels = kNumLevels;
-  options.last_level_temperature = Temperature::kCold;
+  // This existing test selected to also check the case of various temperatures
+  // for last_level_temperature, which should not be interesting enough to
+  // exercise across many/all test cases
+  options.last_level_temperature = RandomKnownTemperature();
  DestroyAndReopen(options);

  Random rnd(301);
@@ -1791,6 +1794,10 @@ TEST_P(PrecludeLastLevelTest, SmallPrecludeTime) {
  ASSERT_FALSE(tp_mapping.Empty());
  auto seqs = tp_mapping.TEST_GetInternalMapping();
  ASSERT_FALSE(seqs.empty());
+  ASSERT_GE(GetSstSizeHelper(Temperature::kUnknown), 1);
+  for (auto t : kKnownTemperatures) {
+    ASSERT_EQ(GetSstSizeHelper(t), 0);
+  }

  // Wait more than preclude_last_level time, then make sure all the data is
  // compacted to the last level even there's no write (no seqno -> time
@@ -1799,8 +1806,14 @@ TEST_P(PrecludeLastLevelTest, SmallPrecludeTime) {

  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
  ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel());
-  ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0);
-  ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
+
+  for (auto t : kKnownTemperatures) {
+    if (t == options.last_level_temperature) {
+      ASSERT_GT(GetSstSizeHelper(t), 0);
+    } else {
+      ASSERT_EQ(GetSstSizeHelper(t), 0);
+    }
+  }

  Close();
 }
@@ -5087,6 +5087,7 @@ TEST_F(DBBasicTest, DisallowMemtableWrite) {
  Options options_disallow = options_allow;
  options_disallow.disallow_memtable_writes = true;
  options_disallow.paranoid_memory_checks = true;
+  options_disallow.memtable_veirfy_per_key_checksum_on_seek = true;

  DestroyAndReopen(options_allow);
  // CFs allowing and disallowing memtable write
@@ -19,6 +19,7 @@
 #include "rocksdb/advanced_options.h"
 #include "rocksdb/concurrent_task_limiter.h"
 #include "rocksdb/experimental.h"
+#include "rocksdb/iostats_context.h"
 #include "rocksdb/sst_file_writer.h"
 #include "test_util/mock_time_env.h"
 #include "test_util/sync_point.h"
@@ -7128,6 +7129,70 @@ TEST_F(DBCompactionTest, PartialManualCompaction) {
  ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr));
 }

+TEST_F(DBCompactionTest, ConcurrentFIFOPickingSameFileBug) {
+  Options opts = CurrentOptions();
+  opts.compaction_style = CompactionStyle::kCompactionStyleLevel;
+  opts.num_levels = 3;
+  opts.disable_auto_compactions = true;
+  opts.max_background_jobs = 3;
+
+  DestroyAndReopen(opts);
+
+  ASSERT_OK(Put("k1", "v1"));
+  ASSERT_OK(Flush());
+
+  // Create a non-L0 SST file for multi-level FIFO size-based compaction later
+  MoveFilesToLevel(2);
+
+  Options opts_new(opts);
+  opts_new.compaction_style = CompactionStyle::kCompactionStyleFIFO;
+  opts_new.max_open_files = -1;
+  // Set a low threshold to trigger multi-level size-based compaction
+  opts_new.compaction_options_fifo.max_table_files_size = 1;
+
+  Reopen(opts_new);
+
+  const CompactRangeOptions cro;
+  const Slice begin_key("k1");
+  const Slice end_key("k2");
+
+  std::unique_ptr<port::Thread> concurrent_compaction;
+
+  bool within_first_compaction = true;
+  SyncPoint::GetInstance()->SetCallBack(
+      "VersionSet::LogAndApply:WriteManifestStart", [&](void* /*arg*/) {
+        if (!within_first_compaction) {
+          return;
+        }
+        within_first_compaction = false;
+
+        // To allow the second/concurrent compaction to still see the non-L0
+        // SST file and coerce the bug of picking that file
+        SyncPoint::GetInstance()->LoadDependency({
+            {"DBImpl::BackgroundCompaction:BeforeCompaction",
+             "VersionSet::LogAndApply:WriteManifest"},
+        });
+
+        concurrent_compaction.reset(new port::Thread([&]() {
+          // Before the fix, the second CompactRange() will either fail the
+          // assertion of double file picking `being_compacted !=
+          // inputs_[i][j]->being_compacted` in debug mode or cause LSM shape
+          // corruption "Cannot delete table file XXX from level 2 since it is
+          // not in the LSM tree" in release mode
+          Status s = db_->CompactRange(cro, &begin_key, &end_key);
+          ASSERT_OK(s);
+        }));
+      });
+
+  SyncPoint::GetInstance()->EnableProcessing();
+  Status s = db_->CompactRange(cro, &begin_key, &end_key);
+  SyncPoint::GetInstance()->DisableProcessing();
+
+  ASSERT_OK(s);
+
+  concurrent_compaction->join();
+}
+
 TEST_F(DBCompactionTest, ManualCompactionFailsInReadOnlyMode) {
  // Regression test for bug where manual compaction hangs forever when the DB
  // is in read-only mode. Verify it now at least returns, despite failing.
@@ -9727,6 +9792,7 @@ TEST_F(DBCompactionTest, FIFOChangeTemperature) {
      int total_cold = 0;
      int total_warm = 0;
      int total_hot = 0;
+      int total_ice = 0;
      int total_unknown = 0;
      ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
          "NewWritableFile::FileOptions.temperature", [&](void* arg) {
@@ -9737,6 +9803,8 @@ TEST_F(DBCompactionTest, FIFOChangeTemperature) {
              total_warm++;
            } else if (temperature == Temperature::kHot) {
              total_hot++;
+            } else if (temperature == Temperature::kIce) {
+              total_ice++;
            } else {
              assert(temperature == Temperature::kUnknown);
              total_unknown++;
@@ -9810,6 +9878,261 @@ TEST_F(DBCompactionTest, FIFOChangeTemperature) {
  }
 }

+using TemperatureSet = SmallEnumSet<Temperature, Temperature::kLastTemperature>;
+static void VerifyTemperatureFileReadStats(const Statistics& st,
+                                           TemperatureSet temps) {
+  SCOPED_TRACE("Temp set size = " + std::to_string(temps.count()));
+  constexpr uint64_t min_bytes = 100;
+  constexpr uint64_t min_count = 1;
+
+  IOStatsContext* iostats = get_iostats_context();
+  if (temps.Contains(Temperature::kHot)) {
+    EXPECT_GE(st.getTickerCount(HOT_FILE_READ_BYTES), min_bytes);
+    EXPECT_GE(st.getTickerCount(HOT_FILE_READ_COUNT), min_count);
+    EXPECT_GE(iostats->file_io_stats_by_temperature.hot_file_bytes_read,
+              min_bytes);
+    EXPECT_GE(iostats->file_io_stats_by_temperature.hot_file_read_count,
+              min_count);
+
+  } else {
+    EXPECT_EQ(st.getTickerCount(HOT_FILE_READ_BYTES), 0);
+    EXPECT_EQ(st.getTickerCount(HOT_FILE_READ_COUNT), 0);
+    EXPECT_EQ(iostats->file_io_stats_by_temperature.hot_file_bytes_read, 0);
+    EXPECT_EQ(iostats->file_io_stats_by_temperature.hot_file_read_count, 0);
+  }
+
+  if (temps.Contains(Temperature::kWarm)) {
+    EXPECT_GE(st.getTickerCount(WARM_FILE_READ_BYTES), min_bytes);
+    EXPECT_GE(st.getTickerCount(WARM_FILE_READ_COUNT), min_count);
+    EXPECT_GE(iostats->file_io_stats_by_temperature.warm_file_bytes_read,
+              min_bytes);
+    EXPECT_GE(iostats->file_io_stats_by_temperature.warm_file_read_count,
+              min_count);
+  } else {
+    EXPECT_EQ(st.getTickerCount(WARM_FILE_READ_BYTES), 0);
+    EXPECT_EQ(st.getTickerCount(WARM_FILE_READ_COUNT), 0);
+    EXPECT_EQ(iostats->file_io_stats_by_temperature.warm_file_bytes_read, 0);
+    EXPECT_EQ(iostats->file_io_stats_by_temperature.warm_file_read_count, 0);
+  }
+
+  if (temps.Contains(Temperature::kCool)) {
+    EXPECT_GE(st.getTickerCount(COOL_FILE_READ_BYTES), min_bytes);
+    EXPECT_GE(st.getTickerCount(COOL_FILE_READ_COUNT), min_count);
+    EXPECT_GE(iostats->file_io_stats_by_temperature.cool_file_bytes_read,
+              min_bytes);
+    EXPECT_GE(iostats->file_io_stats_by_temperature.cool_file_read_count,
+              min_count);
+  } else {
+    EXPECT_EQ(st.getTickerCount(COOL_FILE_READ_BYTES), 0);
+    EXPECT_EQ(st.getTickerCount(COOL_FILE_READ_COUNT), 0);
+    EXPECT_EQ(iostats->file_io_stats_by_temperature.cool_file_bytes_read, 0);
+    EXPECT_EQ(iostats->file_io_stats_by_temperature.cool_file_read_count, 0);
+  }
+
+  if (temps.Contains(Temperature::kCold)) {
+    EXPECT_GE(st.getTickerCount(COLD_FILE_READ_BYTES), min_bytes);
+    EXPECT_GE(st.getTickerCount(COLD_FILE_READ_COUNT), min_count);
+    EXPECT_GE(iostats->file_io_stats_by_temperature.cold_file_bytes_read,
+              min_bytes);
+    EXPECT_GE(iostats->file_io_stats_by_temperature.cold_file_read_count,
+              min_count);
+  } else {
+    EXPECT_EQ(st.getTickerCount(COLD_FILE_READ_BYTES), 0);
+    EXPECT_EQ(st.getTickerCount(COLD_FILE_READ_COUNT), 0);
+    EXPECT_EQ(iostats->file_io_stats_by_temperature.cold_file_bytes_read, 0);
+    EXPECT_EQ(iostats->file_io_stats_by_temperature.cold_file_read_count, 0);
+  }
+
+  if (temps.Contains(Temperature::kIce)) {
+    EXPECT_GE(st.getTickerCount(ICE_FILE_READ_BYTES), min_bytes);
+    EXPECT_GE(st.getTickerCount(ICE_FILE_READ_COUNT), min_count);
+    EXPECT_GE(iostats->file_io_stats_by_temperature.ice_file_bytes_read,
+              min_bytes);
+    EXPECT_GE(iostats->file_io_stats_by_temperature.ice_file_read_count,
+              min_count);
+  } else {
+    EXPECT_EQ(st.getTickerCount(ICE_FILE_READ_BYTES), 0);
+    EXPECT_EQ(st.getTickerCount(ICE_FILE_READ_COUNT), 0);
+    EXPECT_EQ(iostats->file_io_stats_by_temperature.ice_file_bytes_read, 0);
+    EXPECT_EQ(iostats->file_io_stats_by_temperature.ice_file_read_count, 0);
+  }
+}
+
+TEST_F(DBCompactionTest, FIFOMultiTierTemperatureAging) {
+  // Test multi-tier aging: Hot -> Warm -> Cool -> Cold -> Ice
+  Options options = CurrentOptions();
+  options.compaction_style = kCompactionStyleFIFO;
+  options.num_levels = 1;
+  options.max_open_files = -1;
+  options.level0_file_num_compaction_trigger = 2;
+  options.create_if_missing = true;
+  options.statistics = CreateDBStatistics();
+  BlockBasedTableOptions bbto;
+  bbto.no_block_cache = true;  // Simplify statistics
+  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+
+  CompactionOptionsFIFO fifo_options;
+  // Multi-tier aging: files age through multiple temperatures
+  fifo_options.file_temperature_age_thresholds = {
+      {Temperature::kWarm, 500},   // Hot -> Warm after 500s
+      {Temperature::kCool, 1000},  // Warm -> Cool
+      {Temperature::kCold, 1500},  // Cool -> Cold
+      {Temperature::kIce, 2000}    // Cold -> Ice
+  };
+  fifo_options.max_table_files_size = 100000000;
+  fifo_options.allow_trivial_copy_when_change_temperature = true;
+  options.compaction_options_fifo = fifo_options;
+  options.default_write_temperature = Temperature::kHot;
+
+  Reopen(options);
+  env_->SetMockSleep();
+
+  // Track all temperature file creations
+  int total_hot = 0, total_warm = 0, total_cool = 0, total_cold = 0,
+      total_ice = 0, total_unknown = 0;
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "NewWritableFile::FileOptions.temperature", [&](void* arg) {
+        Temperature temperature = *(static_cast<Temperature*>(arg));
+        switch (temperature) {
+          case Temperature::kHot:
+            total_hot++;
+            break;
+          case Temperature::kWarm:
+            total_warm++;
+            break;
+          case Temperature::kCool:
+            total_cool++;
+            break;
+          case Temperature::kCold:
+            total_cold++;
+            break;
+          case Temperature::kIce:
+            total_ice++;
+            break;
+          case Temperature::kUnknown:
+            total_unknown++;
+            break;
+          default:
+            break;
+        }
+      });
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  // Create initial three files (will start as Hot), enough to ensure key
+  // range filtering will be applied in FilePicker::GetNextFile() with one
+  // more file
+  for (int i = 0; i < 3; ++i) {
+    ASSERT_OK(Put(Key(0), Random::GetTLSInstance()->RandomBinaryString(100)));
+    ASSERT_OK(Flush());
+  }
+
+  // Test reading from Hot temperature file
+  ASSERT_OK(options.statistics->Reset());
+  get_iostats_context()->Reset();
+
+  ASSERT_EQ(100U, Get(Key(0)).size());
+
+  VerifyTemperatureFileReadStats(*options.statistics, Temperature::kHot);
+
+  // Land well into each time interval
+  env_->MockSleepForSeconds(100);
+
+  // Age initial files to warm
+  env_->MockSleepForSeconds(500);
+  ASSERT_OK(Put(Key(1), Random::GetTLSInstance()->RandomBinaryString(101)));
+  ASSERT_OK(Flush());
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+
+  // Test reading from Warm temperature file (the aged file)
+  ASSERT_OK(options.statistics->Reset());
+  get_iostats_context()->Reset();
+
+  ASSERT_EQ(100U, Get(Key(0)).size());
+
+  // Verify Warm file statistics
+  VerifyTemperatureFileReadStats(*options.statistics, Temperature::kWarm);
+
+  // Age initial files to cool
+  env_->MockSleepForSeconds(500);
+  ASSERT_OK(Put(Key(2), Random::GetTLSInstance()->RandomBinaryString(102)));
+  ASSERT_OK(Flush());
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+
+  // Test reading from Cool temperature file (the aged file)
+  ASSERT_OK(options.statistics->Reset());
+  get_iostats_context()->Reset();
+
+  ASSERT_EQ(100U, Get(Key(0)).size());
+
+  VerifyTemperatureFileReadStats(*options.statistics, Temperature::kCool);
+
+  // Age initial files to cold
+  env_->MockSleepForSeconds(500);
+  ASSERT_OK(Put(Key(3), Random::GetTLSInstance()->RandomBinaryString(103)));
+  ASSERT_OK(Flush());
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+
+  // Test reading from Cold temperature file (the aged file)
+  ASSERT_OK(options.statistics->Reset());
+  get_iostats_context()->Reset();
+
+  ASSERT_EQ(100U, Get(Key(0)).size());
+
+  VerifyTemperatureFileReadStats(*options.statistics, Temperature::kCold);
+
+  // Age initial files to ice
+  env_->MockSleepForSeconds(500);
+  ASSERT_OK(Put(Key(4), Random::GetTLSInstance()->RandomBinaryString(104)));
+  ASSERT_OK(Flush());
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+
+  // Test reading from Ice temperature file (the aged file)
+  ASSERT_OK(options.statistics->Reset());
+  get_iostats_context()->Reset();
+
+  ASSERT_EQ(100U, Get(Key(0)).size());
+
+  VerifyTemperatureFileReadStats(*options.statistics, Temperature::kIce);
+
+  // Verify temperature progression in metadata
+  ColumnFamilyMetaData metadata;
+  db_->GetColumnFamilyMetaData(&metadata);
+
+  // Should have files at different temperatures
+  std::map<Temperature, int> temp_counts;
+  for (const auto& file : metadata.levels[0].files) {
+    temp_counts[file.temperature]++;
+  }
+
+  // Verify current files temperatures
+  EXPECT_EQ(temp_counts[Temperature::kHot], 1);
+  EXPECT_EQ(temp_counts[Temperature::kWarm], 1);
+  EXPECT_EQ(temp_counts[Temperature::kCool], 1);
+  EXPECT_EQ(temp_counts[Temperature::kCold], 1);
+  EXPECT_EQ(temp_counts[Temperature::kIce], 3);
+
+  // Verify historical (and current) file temperatures
+  EXPECT_EQ(total_hot, 7);
+  EXPECT_EQ(total_warm, 6);
+  EXPECT_EQ(total_cool, 5);
+  EXPECT_EQ(total_cold, 4);
+  EXPECT_EQ(total_ice, 3);
+
+  // Final comprehensive test: read from all temperature files
+  Reopen(options);
+  ASSERT_OK(options.statistics->Reset());
+  get_iostats_context()->Reset();
+
+  // Read from all files to verify cumulative statistics
+  for (int i = 0; i < 5; i++) {
+    ASSERT_EQ(static_cast<unsigned>(100 + i), Get(Key(i)).size());
+  }
+
+  VerifyTemperatureFileReadStats(*options.statistics, TemperatureSet::All());
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+}
+
 TEST_F(DBCompactionTest, DisableMultiManualCompaction) {
  const int kNumL0Files = 10;

@@ -1069,7 +1069,7 @@ void DBImpl::DumpStats() {
  {
    InstrumentedMutexLock l(&mutex_);
    for (auto cfd : versions_->GetRefedColumnFamilySet()) {
-      if (!cfd->initialized()) {
+      if (!cfd->initialized() || cfd->IsDropped()) {
        continue;
      }

@@ -1245,13 +1245,11 @@ Status DBImpl::SetOptions(
          WriteOptionsFile(write_options, true /*db_mutex_already_held*/);
      bg_cv_.SignalAll();

-#if __cplusplus >= 202002L
      assert(new_options_copy == cfd->GetLatestMutableCFOptions());
      assert(cfd->GetLatestMutableCFOptions() ==
             cfd->GetCurrentMutableCFOptions());
      assert(cfd->GetCurrentMutableCFOptions() ==
             cfd->current()->GetMutableCFOptions());
-#endif
    }
  }
  sv_context.Clean();
@@ -1565,11 +1565,115 @@ void DBIter::SetSavedKeyToSeekForPrevTarget(const Slice& target) {
  }
 }

+Status DBIter::ValidateScanOptions(const MultiScanArgs& multiscan_opts) const {
+  if (multiscan_opts.empty()) {
+    return Status::InvalidArgument("Empty MultiScanArgs");
+  }
+
+  const std::vector<ScanOptions>& scan_opts = multiscan_opts.GetScanRanges();
+  const bool has_limit = scan_opts.front().range.limit.has_value();
+  if (!has_limit && scan_opts.size() > 1) {
+    return Status::InvalidArgument("Scan has no upper bound");
+  }
+
+  for (size_t i = 0; i < scan_opts.size(); ++i) {
+    const auto& scan_range = scan_opts[i].range;
+    if (!scan_range.start.has_value()) {
+      return Status::InvalidArgument("Scan has no start key at index " +
+                                     std::to_string(i));
+    }
+
+    if (scan_range.limit.has_value()) {
+      if (user_comparator_.CompareWithoutTimestamp(
+              scan_range.start.value(), /*a_has_ts=*/false,
+              scan_range.limit.value(), /*b_has_ts=*/false) >= 0) {
+        return Status::InvalidArgument(
+            "Scan start key is large or equal than limit at index " +
+            std::to_string(i));
+      }
+    }
+
+    if (i > 0) {
+      if (!scan_range.limit.has_value()) {
+        // multiple scan without limit scan ranges
+        return Status::InvalidArgument("Scan has no upper bound at index " +
+                                       std::to_string(i));
+      }
+
+      const auto& last_end_key = scan_opts[i - 1].range.limit.value();
+      if (user_comparator_.CompareWithoutTimestamp(
+              scan_range.start.value(), /*a_has_ts=*/false, last_end_key,
+              /*b_has_ts=*/false) < 0) {
+        return Status::InvalidArgument("Overlapping ranges at index " +
+                                       std::to_string(i));
+      }
+    }
+  }
+  return Status::OK();
+}
+
+void DBIter::Prepare(const MultiScanArgs& scan_opts) {
+  status_ = ValidateScanOptions(scan_opts);
+  if (!status_.ok()) {
+    return;
+  }
+  std::optional<MultiScanArgs> new_scan_opts;
+  new_scan_opts.emplace(scan_opts);
+  scan_opts_.swap(new_scan_opts);
+  scan_index_ = 0;
+  if (!scan_opts.empty()) {
+    iter_.Prepare(&scan_opts_.value());
+  } else {
+    iter_.Prepare(nullptr);
+  }
+}
+
 void DBIter::Seek(const Slice& target) {
  PERF_COUNTER_ADD(iter_seek_count, 1);
  PERF_CPU_TIMER_GUARD(iter_seek_cpu_nanos, clock_);
  StopWatch sw(clock_, statistics_, DB_SEEK);

+  if (scan_opts_.has_value()) {
+    // Validate the seek target is as expected in the previously prepared range
+    auto const& scan_ranges = scan_opts_.value().GetScanRanges();
+    if (scan_index_ >= scan_ranges.size()) {
+      status_ = Status::InvalidArgument(
+          "Seek called after exhausting all of the scan ranges");
+      valid_ = false;
+      return;
+    }
+
+    // Validate start key of next prepare range matches the seek target
+    auto const& range = scan_ranges[scan_index_];
+    auto const& start = range.range.start;
+    assert(start.has_value());
+    if (user_comparator_.CompareWithoutTimestamp(target, *start) != 0) {
+      status_ = Status::InvalidArgument(
+          "Seek target does not match the start of the next prepared range at "
+          "index " +
+          std::to_string(scan_index_));
+      valid_ = false;
+      return;
+    }
+
+    // validate the upper bound is set to the same value of limit, if limit
+    // exists
+    auto const& limit = range.range.limit;
+    if (limit.has_value()) {
+      if (iterate_upper_bound_ == nullptr ||
+          user_comparator_.CompareWithoutTimestamp(
+              limit.value(), *iterate_upper_bound_) != 0) {
+        status_ = Status::InvalidArgument(
+            "Upper bound is not set to the same limit value of the next "
+            "prepared range at index " +
+            std::to_string(scan_index_));
+        valid_ = false;
+        return;
+      }
+    }
+    scan_index_++;
+  }
+
  if (cfh_ != nullptr) {
    // TODO: What do we do if this returns an error?
    Slice lower_bound, upper_bound;
@@ -240,16 +240,8 @@ class DBIter final : public Iterator {

  bool PrepareValue() override;

-  void Prepare(const MultiScanArgs& scan_opts) override {
-    std::optional<MultiScanArgs> new_scan_opts;
-    new_scan_opts.emplace(scan_opts);
-    scan_opts_.swap(new_scan_opts);
-    if (!scan_opts.empty()) {
-      iter_.Prepare(&scan_opts_.value());
-    } else {
-      iter_.Prepare(nullptr);
-    }
-  }
+  void Prepare(const MultiScanArgs& scan_opts) override;
+  Status ValidateScanOptions(const MultiScanArgs& multiscan_opts) const;

 private:
  DBIter(Env* _env, const ReadOptions& read_options,
@@ -506,6 +498,7 @@ class DBIter final : public Iterator {
  const size_t timestamp_size_;
  std::string saved_timestamp_;
  std::optional<MultiScanArgs> scan_opts_;
+  size_t scan_index_{0};
  ReadOnlyMemTable* const active_mem_;
  SequenceNumber memtable_seqno_lb_;
  uint32_t memtable_op_scan_flush_trigger_;
@@ -4149,6 +4149,7 @@ class DBMultiScanIteratorTest : public DBTestBase,
      : DBTestBase("db_multi_scan_iterator_test", /*env_do_fsync=*/true) {}
 };

+// Param 0: ReadOptions::fill_cache
 INSTANTIATE_TEST_CASE_P(DBMultiScanIteratorTest, DBMultiScanIteratorTest,
                        ::testing::Bool());

@@ -4192,66 +4193,6 @@ TEST_P(DBMultiScanIteratorTest, BasicTest) {
    abort();
  }
  iter.reset();
-
-  // Test the overlapping scan case
-  key_ranges[1] = "k30";
-  scan_options = MultiScanArgs(BytewiseComparator());
-  scan_options.insert(key_ranges[0], key_ranges[1]);
-  scan_options.insert(key_ranges[2], key_ranges[3]);
-
-  iter = dbfull()->NewMultiScan(ro, cfh, scan_options);
-  try {
-    int idx = 0;
-    int count = 0;
-    for (auto range : *iter) {
-      for (auto it : range) {
-        ASSERT_GE(it.first.ToString().compare(key_ranges[idx]), 0);
-        ASSERT_LT(it.first.ToString().compare(key_ranges[idx + 1]), 0);
-        count++;
-      }
-      idx += 2;
-    }
-    ASSERT_EQ(count, 52);
-  } catch (MultiScanException& ex) {
-    // Make sure exception contains the status
-    ASSERT_NOK(ex.status());
-    std::cerr << "Iterator returned status " << ex.what();
-    abort();
-  } catch (std::logic_error& ex) {
-    std::cerr << "Iterator returned logic error " << ex.what();
-    abort();
-  }
-  iter.reset();
-
-  // Test the no limit scan case
-  scan_options = MultiScanArgs(BytewiseComparator());
-  scan_options.insert(key_ranges[0]);
-  scan_options.insert(key_ranges[2]);
-  iter = dbfull()->NewMultiScan(ro, cfh, scan_options);
-  try {
-    int idx = 0;
-    int count = 0;
-    for (auto range : *iter) {
-      for (auto it : range) {
-        ASSERT_GE(it.first.ToString().compare(key_ranges[idx]), 0);
-        if (it.first.ToString().compare(key_ranges[idx + 1]) == 0) {
-          break;
-        }
-        count++;
-      }
-      idx += 2;
-    }
-    ASSERT_EQ(count, 52);
-  } catch (MultiScanException& ex) {
-    // Make sure exception contains the status
-    ASSERT_NOK(ex.status());
-    std::cerr << "Iterator returned status " << ex.what();
-    abort();
-  } catch (std::logic_error& ex) {
-    std::cerr << "Iterator returned logic error " << ex.what();
-    abort();
-  }
-  iter.reset();
 }

 TEST_P(DBMultiScanIteratorTest, MixedBoundsTest) {
@@ -4366,15 +4307,297 @@ TEST_P(DBMultiScanIteratorTest, RangeAcrossFiles) {
  ColumnFamilyHandle* cfh = dbfull()->DefaultColumnFamily();
  std::unique_ptr<MultiScan> iter =
      dbfull()->NewMultiScan(ro, cfh, scan_options);
-  int i = 10;
-  for (auto range : *iter) {
-    for (auto it : range) {
-      ASSERT_EQ(it.first.ToString(), Key(i));
-      ++i;
+  try {
+    int i = 10;
+    for (auto range : *iter) {
+      for (auto it : range) {
+        ASSERT_EQ(it.first.ToString(), Key(i));
+        ++i;
+      }
    }
+    ASSERT_EQ(i, 90);
+  } catch (MultiScanException& ex) {
+    // Make sure exception contains the status
+    ASSERT_NOK(ex.status());
+    std::cerr << "Iterator returned status " << ex.what();
+    abort();
+  } catch (std::logic_error& ex) {
+    std::cerr << "Iterator returned logic error " << ex.what();
+    abort();
  }
-  ASSERT_EQ(i, 90);
+  iter.reset();
 }
+
+TEST_P(DBMultiScanIteratorTest, FailureTest) {
+  auto options = CurrentOptions();
+  options.compression = kNoCompression;
+  DestroyAndReopen(options);
+
+  Random rnd(301);
+  // Create a file
+  for (int i = 0; i < 100; ++i) {
+    std::stringstream ss;
+    ss << std::setw(2) << std::setfill('0') << i;
+    ASSERT_OK(Put("k" + ss.str(), rnd.RandomString(1024)));
+  }
+  ASSERT_OK(Flush());
+
+  std::vector<std::string> key_ranges({"k04", "k06", "k12", "k14"});
+  ReadOptions ro;
+  Slice ub;
+  ro.iterate_upper_bound = &ub;
+  ro.fill_cache = GetParam();
+  MultiScanArgs scan_options(BytewiseComparator());
+  scan_options.insert(key_ranges[0], key_ranges[1]);
+  scan_options.insert(key_ranges[2], key_ranges[3]);
+  scan_options.max_prefetch_size = 4500;
+  ColumnFamilyHandle* cfh = dbfull()->DefaultColumnFamily();
+  std::unique_ptr<Iterator> iter(dbfull()->NewIterator(ro, cfh));
+  ASSERT_NE(iter, nullptr);
+  iter->Prepare(scan_options);
+  int count = 0;
+  ub = key_ranges[1];
+  iter->Seek(key_ranges[0]);
+  while (iter->status().ok() && iter->Valid()) {
+    ASSERT_GE(iter->key().compare(key_ranges[0]), 0);
+    ASSERT_LT(iter->key().compare(key_ranges[1]), 0);
+    count++;
+    iter->Next();
+  }
+  ASSERT_OK(iter->status()) << iter->status().ToString();
+  ASSERT_EQ(count, 2);
+
+  // Second seek should hit the max_prefetch_size limit
+  ub = key_ranges[3];
+  iter->Seek(key_ranges[2]);
+  ASSERT_NOK(iter->status());
+  iter.reset();
+
+  // Test the case of unexpected Seek key
+  iter.reset(dbfull()->NewIterator(ro, cfh));
+  ASSERT_NE(iter, nullptr);
+  scan_options.max_prefetch_size = 0;
+  iter->Prepare(scan_options);
+  ub = key_ranges[3];
+  iter->Seek(key_ranges[2]);
+  ASSERT_NOK(iter->status());
+  iter.reset();
+}
+
+TEST_P(DBMultiScanIteratorTest, RangeBetweenFiles) {
+  auto options = CurrentOptions();
+  options.target_file_size_base = 100 << 10;  // 20KB
+  options.compaction_style = kCompactionStyleUniversal;
+  options.num_levels = 50;
+  options.compression = kNoCompression;
+  DestroyAndReopen(options);
+
+  auto rnd = Random::GetTLSInstance();
+  // Write ~200KB data
+  for (int i = 0; i < 100; ++i) {
+    ASSERT_OK(Put(Key(i), rnd->RandomString(2 << 10)));
+  }
+  ASSERT_OK(Flush());
+
+  ASSERT_OK(db_->CompactRange({}, nullptr, nullptr));
+  ASSERT_EQ(2, NumTableFilesAtLevel(49));
+
+  // Test with a scan range that overlaps an entire file, with upper bound
+  // between 2 files
+  std::vector<LiveFileMetaData> file_meta;
+  dbfull()->GetLiveFilesMetaData(&file_meta);
+  ASSERT_EQ(file_meta.size(), 2);
+  std::vector<std::string> key_ranges(4);
+  key_ranges[0] = file_meta[0].smallestkey;
+  key_ranges[1] = file_meta[0].largestkey + "0";
+  key_ranges[2] = file_meta[1].smallestkey + "0";
+  key_ranges[3] = file_meta[1].largestkey;
+  ReadOptions ro;
+  ro.fill_cache = GetParam();
+  MultiScanArgs scan_options(BytewiseComparator());
+  scan_options.insert(key_ranges[0], key_ranges[1]);
+  scan_options.insert(key_ranges[2], key_ranges[3]);
+  ColumnFamilyHandle* cfh = dbfull()->DefaultColumnFamily();
+  std::unique_ptr<MultiScan> iter =
+      dbfull()->NewMultiScan(ro, cfh, scan_options);
+  try {
+    for (auto range : *iter) {
+      for (auto it : range) {
+        ASSERT_GE(it.first.ToString(), key_ranges[0]);
+      }
+    }
+  } catch (MultiScanException& ex) {
+    // Make sure exception contains the status
+    ASSERT_NOK(ex.status());
+    std::cerr << "Iterator returned status " << ex.what();
+    abort();
+  } catch (std::logic_error& ex) {
+    std::cerr << "Iterator returned logic error " << ex.what();
+    abort();
+  }
+  iter.reset();
+
+  // Test multiscan with a range entirely between adjacent files
+  key_ranges[0] = file_meta[0].largestkey + "0";
+  key_ranges[1] = file_meta[0].largestkey + "1";
+  key_ranges[2] = file_meta[1].smallestkey + "0";
+  key_ranges[3] = file_meta[1].largestkey;
+  (*scan_options).clear();
+  scan_options.insert(key_ranges[0], key_ranges[1]);
+  scan_options.insert(key_ranges[2], key_ranges[3]);
+  iter = dbfull()->NewMultiScan(ro, cfh, scan_options);
+  try {
+    for (auto range : *iter) {
+      for (auto it : range) {
+        ASSERT_GE(it.first.ToString(), key_ranges[0]);
+      }
+    }
+  } catch (MultiScanException& ex) {
+    // Make sure exception contains the status
+    ASSERT_NOK(ex.status());
+    std::cerr << "Iterator returned status " << ex.what();
+    abort();
+  } catch (std::logic_error& ex) {
+    std::cerr << "Iterator returned logic error " << ex.what();
+    abort();
+  }
+  iter.reset();
+}
+
+// This test case tests multiscan in the presence of fragmented range
+// tombstones in the LSM.
+TEST_P(DBMultiScanIteratorTest, FragmentedRangeTombstones) {
+  auto options = CurrentOptions();
+  // Compaction may create files 2x the target_file_size_base,
+  // so set this to 50KB so we atleast end up with 2 files of
+  // 100KB
+  options.target_file_size_base = 50 << 10;  // 50KB
+  options.compaction_style = kCompactionStyleUniversal;
+  options.num_levels = 50;
+  options.compression = kNoCompression;
+  DestroyAndReopen(options);
+
+  // Setup the LSM as follows -
+  // 1. Ingest a file with 100 keys
+  // 2. Ingest a file with one overlapping key
+  // 3. Do a Put and flush a file to L0 with one overlapping key
+  // 4. Ingest a standalone delete range file that covers the full key space
+  //    and a file with the same 100 keys with new values. This will ingest
+  //    into L0 due to the presence of an existing file in L0
+  // The final LSM will have an SST in Lmax with 100 keys, and 2 SST files
+  // in Lmax-1 with half the keys each and completely overlapping delete ranges
+  std::unordered_map<std::string, std::string> kvs;
+  auto rnd = Random::GetTLSInstance();
+  auto create_ingestion_data_file_and_update_key_value =
+      [&](const std::string& filename, int start_key, int end_key) {
+        std::unique_ptr<SstFileWriter> writer;
+        writer.reset(new SstFileWriter(EnvOptions(), options));
+        ASSERT_OK(writer->Open(filename));
+        for (int i = start_key; i < end_key; ++i) {
+          auto kiter = kvs.find(Key(i));
+          if (kiter != kvs.end()) {
+            kvs.erase(kiter);
+          }
+          auto res =
+              kvs.emplace(std::make_pair(Key(i), rnd->RandomString(2 << 10)));
+          ASSERT_OK(writer->Put(res.first->first, res.first->second));
+        }
+        ASSERT_OK(writer->Finish());
+        writer.reset();
+      };
+
+  CreateColumnFamilies({"new_cf"}, options);
+  std::string ingest_file = dbname_ + "test.sst";
+  // Write ~200KB data
+  create_ingestion_data_file_and_update_key_value(ingest_file + "_0", 0, 100);
+  create_ingestion_data_file_and_update_key_value(ingest_file + "_1", 50, 51);
+  ColumnFamilyHandle* cfh = handles_[0];
+  IngestExternalFileOptions ifo;
+  Status s = dbfull()->IngestExternalFile(
+      cfh, {ingest_file + "_0", ingest_file + "_1"}, ifo);
+  ASSERT_OK(s);
+
+  ASSERT_OK(Put(0, Key(50), rnd->RandomString(2 << 10)));
+  ASSERT_OK(Flush());
+
+  {
+    std::unique_ptr<SstFileWriter> writer;
+    writer.reset(new SstFileWriter(EnvOptions(), options));
+    ASSERT_OK(writer->Open(ingest_file + "_2"));
+    ASSERT_OK(writer->DeleteRange("a", "z"));
+    ASSERT_OK(writer->Finish());
+    writer.reset();
+  }
+  create_ingestion_data_file_and_update_key_value(ingest_file + "_3", 0, 100);
+  s = dbfull()->IngestExternalFile(
+      cfh, {ingest_file + "_2", ingest_file + "_3"}, ifo);
+  ASSERT_OK(s);
+
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+
+  // The first scan range overlaps the DB key range, while the second extends
+  // beyond but overlaps the delete range
+  std::vector<std::string> key_ranges({"key000085", "key000090", "l", "n"});
+  ReadOptions ro;
+  ro.fill_cache = GetParam();
+  MultiScanArgs scan_options(BytewiseComparator());
+  scan_options.insert(key_ranges[0], key_ranges[1]);
+  scan_options.insert(key_ranges[2], key_ranges[3]);
+  std::unique_ptr<MultiScan> iter =
+      dbfull()->NewMultiScan(ro, cfh, scan_options);
+  try {
+    int i = 0;
+    int count = 0;
+    for (auto range : *iter) {
+      for (auto it : range) {
+        ASSERT_GE(it.first.ToString(), key_ranges[i]);
+        ASSERT_LT(it.first.ToString(), key_ranges[i + 1]);
+        auto kiter = kvs.find(it.first.ToString());
+        ASSERT_NE(kiter, kvs.end());
+        ASSERT_EQ(kiter->second, it.second.ToString());
+        count++;
+      }
+      i += 2;
+    }
+    ASSERT_EQ(i, 4);
+    ASSERT_EQ(count, 5);
+  } catch (MultiScanException& ex) {
+    ASSERT_OK(ex.status());
+  }
+  iter.reset();
+
+  // The second scan range start overlaps the delete range in the first file
+  // in Lmax-1, while the end overlaps the keys in the second file
+  (*scan_options).clear();
+  key_ranges[0] = "key000010";
+  key_ranges[1] = "key000020";
+  key_ranges[2] = "key0000500";
+  key_ranges[3] = "key000060";
+  scan_options.insert(key_ranges[0], key_ranges[1]);
+  scan_options.insert(key_ranges[2], key_ranges[3]);
+  iter = dbfull()->NewMultiScan(ro, cfh, scan_options);
+  try {
+    int i = 0;
+    int count = 0;
+    for (auto range : *iter) {
+      for (auto it : range) {
+        ASSERT_GE(it.first.ToString(), key_ranges[i]);
+        ASSERT_LT(it.first.ToString(), key_ranges[i + 1]);
+        auto kiter = kvs.find(it.first.ToString());
+        ASSERT_NE(kiter, kvs.end());
+        ASSERT_EQ(kiter->second, it.second.ToString());
+        count++;
+      }
+      i += 2;
+    }
+    ASSERT_EQ(i, 4);
+    ASSERT_EQ(count, 19);
+  } catch (MultiScanException& ex) {
+    ASSERT_OK(ex.status());
+  }
+  iter.reset();
+}
+
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -339,6 +339,135 @@ TEST_F(DBMemTableTest, ColumnFamilyId) {
  }
 }

+class DBMemTableTestForSeek : public DBMemTableTest,
+                              virtual public ::testing::WithParamInterface<
+                                  std::tuple<bool, bool, bool>> {};
+
+TEST_P(DBMemTableTestForSeek, IntegrityChecks) {
+  // Validate key corruption could be detected during seek.
+  // We insert many keys into skiplist. Then we corrupt the each key one at a
+  // time. With memtable_veirfy_per_key_checksum_on_seek enabled, when the
+  // corrupted key is searched, the checksum of every key visited during the
+  // seek is validated. It will report data corruption. Otherwise seek returns
+  // not found.
+  auto allow_data_in_error = std::get<0>(GetParam());
+  Options options = CurrentOptions();
+  options.allow_data_in_errors = allow_data_in_error;
+  options.paranoid_memory_checks = std::get<1>(GetParam());
+  options.memtable_veirfy_per_key_checksum_on_seek = std::get<2>(GetParam());
+  options.memtable_protection_bytes_per_key = 8;
+  DestroyAndReopen(options);
+
+  // capture the data pointer of all of the keys
+  std::vector<char*> raw_data_pointer;
+
+  // Insert enough keys, so memtable would create multiple levels.
+  auto key_count = 100;
+  for (int i = 0; i < key_count; i++) {
+    // The last digit of the key will be corrupted from value 0 to value 5
+    ASSERT_OK(Put(Key(i * 10), "val0"));
+  }
+
+  ReadOptions rops;
+
+  // Iterate all the keys to get key pointers
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->SetCallBack("InlineSkipList::Iterator::Next::key",
+                                        [&raw_data_pointer](void* key) {
+                                          auto p = static_cast<char*>(key);
+                                          raw_data_pointer.push_back(p);
+                                        });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  {
+    std::unique_ptr<Iterator> iter{db_->NewIterator(rops)};
+    iter->Seek(Key(0));
+    while (iter->Valid()) {
+      ASSERT_OK(iter->status());
+      iter->Next();
+    }
+    // check status after valid returned false.
+    auto status = iter->status();
+    ASSERT_TRUE(status.ok());
+  }
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+
+  ASSERT_EQ(raw_data_pointer.size(), key_count);
+
+  bool enable_key_validation_on_seek =
+      options.memtable_veirfy_per_key_checksum_on_seek;
+
+  // For each key, corrupt it, validate corruption is detected correctly, then
+  // revert it.
+  for (int i = 0; i < key_count; i++) {
+    std::string key_to_corrupt = Key(i * 10);
+    raw_data_pointer[i][key_to_corrupt.size()] = '5';
+
+    auto corrupted_key = key_to_corrupt;
+    corrupted_key.data()[key_to_corrupt.size() - 1] = '5';
+    auto corrupted_key_slice =
+        Slice(corrupted_key.data(), corrupted_key.length());
+    auto corrupted_key_hex = corrupted_key_slice.ToString(/*hex=*/true);
+
+    {
+      // Test Get API
+      std::string val;
+      auto status = db_->Get(rops, key_to_corrupt, &val);
+      if (enable_key_validation_on_seek) {
+        ASSERT_TRUE(status.IsCorruption()) << key_to_corrupt;
+        ASSERT_EQ(
+            status.ToString().find(corrupted_key_hex) != std::string::npos,
+            allow_data_in_error)
+            << status.ToString() << "\n"
+            << corrupted_key_hex;
+      } else {
+        ASSERT_TRUE(status.IsNotFound());
+      }
+    }
+
+    {
+      // Test MultiGet API
+      std::vector<std::string> vals;
+      std::vector<Status> statuses = db_->MultiGet(
+          rops, {db_->DefaultColumnFamily()}, {key_to_corrupt}, &vals, nullptr);
+      if (enable_key_validation_on_seek) {
+        ASSERT_TRUE(statuses[0].IsCorruption());
+        ASSERT_EQ(
+            statuses[0].ToString().find(corrupted_key_hex) != std::string::npos,
+            allow_data_in_error);
+      } else {
+        ASSERT_TRUE(statuses[0].IsNotFound());
+      }
+    }
+
+    {
+      // Test Iterator Seek API
+      std::unique_ptr<Iterator> iter{db_->NewIterator(rops)};
+      ASSERT_OK(iter->status());
+      iter->Seek(key_to_corrupt);
+      auto status = iter->status();
+      if (enable_key_validation_on_seek) {
+        ASSERT_TRUE(status.IsCorruption());
+        ASSERT_EQ(
+            status.ToString().find(corrupted_key_hex) != std::string::npos,
+            allow_data_in_error);
+      } else {
+        ASSERT_FALSE(iter->Valid());
+        ASSERT_FALSE(status.ok());
+      }
+    }
+
+    // revert the key corruption.
+    raw_data_pointer[i][key_to_corrupt.size()] = '0';
+  }
+}
+
+INSTANTIATE_TEST_CASE_P(DBMemTableTestForSeek, DBMemTableTestForSeek,
+                        ::testing::Combine(::testing::Bool(), ::testing::Bool(),
+                                           ::testing::Bool()));
+
 TEST_F(DBMemTableTest, IntegrityChecks) {
  // We insert keys key000000, key000001 and key000002 into skiplist at fixed
  // height 1 (smallest height). Then we corrupt the second key to aey000001 to
@@ -787,6 +787,46 @@ TEST_P(DBTablePropertiesTest, RatioBasedDeletionTriggeredCompactionMarking) {
  }
 }

+TEST_F(DBTablePropertiesTest, KeyLargestSmallestSeqno) {
+  ASSERT_OK(db_->Put(WriteOptions(), "key1", "value1"));
+  ASSERT_OK(db_->Put(WriteOptions(), "key2", "value2"));
+  ASSERT_OK(db_->Put(WriteOptions(), "key3", "value3"));
+  ASSERT_OK(db_->Flush(FlushOptions()));
+
+  {
+    TablePropertiesCollection props;
+    ASSERT_OK(db_->GetPropertiesOfAllTables(&props));
+    ASSERT_EQ(1U, props.size());
+
+    auto table_props = props.begin()->second;
+
+    ASSERT_TRUE(table_props->HasKeyLargestSeqno());
+    ASSERT_TRUE(table_props->HasKeySmallestSeqno());
+
+    ASSERT_EQ(table_props->key_largest_seqno,
+              table_props->key_smallest_seqno + 2);
+    ASSERT_GT(table_props->key_largest_seqno, 0U);
+    ASSERT_GT(table_props->key_smallest_seqno, 0U);
+  }
+
+  // Becomes zero after compaction
+  {
+    CompactRangeOptions cro;
+    cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
+    ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
+    TablePropertiesCollection props;
+    ASSERT_OK(db_->GetPropertiesOfAllTables(&props));
+    ASSERT_EQ(1U, props.size());
+
+    auto table_props = props.begin()->second;
+    ASSERT_TRUE(table_props->HasKeyLargestSeqno());
+    ASSERT_TRUE(table_props->HasKeySmallestSeqno());
+
+    ASSERT_EQ(table_props->key_largest_seqno, table_props->key_smallest_seqno);
+    ASSERT_EQ(table_props->key_largest_seqno, 0U);
+  }
+}
+
 INSTANTIATE_TEST_CASE_P(DBTablePropertiesTest, DBTablePropertiesTest,
                        ::testing::Values("kCompactionStyleLevel",
                                          "kCompactionStyleUniversal"));
@@ -6063,15 +6063,9 @@ TEST_F(DBTest2, VariousFileTemperatures) {
  };

  // We don't have enough non-unknown temps to confidently distinguish that
-  // a specific setting caused a specific outcome, in a single run. This is a
-  // reasonable work-around without blowing up test time. Only returns
-  // non-unknown temperatures.
-  auto RandomTemp = [] {
-    static std::vector<Temperature> temps = {
-        Temperature::kHot, Temperature::kWarm, Temperature::kCold};
-    return temps[Random::GetTLSInstance()->Uniform(
-        static_cast<int>(temps.size()))];
-  };
+  // a specific setting caused a specific outcome, in a single run. Using
+  // RandomKnownTemperature() is a reasonable work-around without blowing up
+  // test time.

  auto test_fs = std::make_shared<MyTestFS>(env_->GetFileSystem());
  std::unique_ptr<Env> env(new CompositeEnvWrapper(env_, test_fs));
@@ -6087,22 +6081,22 @@ TEST_F(DBTest2, VariousFileTemperatures) {
      options.env = env.get();
      test_fs->Reset();
      if (use_optimize) {
-        test_fs->optimize_manifest_temperature = RandomTemp();
+        test_fs->optimize_manifest_temperature = RandomKnownTemperature();
        test_fs->expected_manifest_temperature =
            test_fs->optimize_manifest_temperature;
-        test_fs->optimize_wal_temperature = RandomTemp();
+        test_fs->optimize_wal_temperature = RandomKnownTemperature();
        test_fs->expected_wal_temperature = test_fs->optimize_wal_temperature;
      }
      if (use_temp_options) {
-        options.metadata_write_temperature = RandomTemp();
+        options.metadata_write_temperature = RandomKnownTemperature();
        test_fs->expected_manifest_temperature =
            options.metadata_write_temperature;
        test_fs->expected_other_metadata_temperature =
            options.metadata_write_temperature;
-        options.wal_write_temperature = RandomTemp();
+        options.wal_write_temperature = RandomKnownTemperature();
        test_fs->expected_wal_temperature = options.wal_write_temperature;
-        options.last_level_temperature = RandomTemp();
-        options.default_write_temperature = RandomTemp();
+        options.last_level_temperature = RandomKnownTemperature();
+        options.default_write_temperature = RandomKnownTemperature();
      }

      DestroyAndReopen(options);
@@ -71,9 +71,9 @@ DBTestBase::DBTestBase(const std::string path, bool env_do_fsync)
  if (getenv("MEM_ENV")) {
    mem_env_ = MockEnv::Create(base_env, base_env->GetSystemClock());
  }
-  if (getenv("ENCRYPTED_ENV")) {
+  if (auto ee = getenv("ENCRYPTED_ENV")) {
    std::shared_ptr<EncryptionProvider> provider;
-    std::string provider_id = getenv("ENCRYPTED_ENV");
+    std::string provider_id = ee;
    if (provider_id.find('=') == std::string::npos &&
        !EndsWith(provider_id, "://test")) {
      provider_id = provider_id + "://test";
@@ -1872,4 +1872,13 @@ template class TargetCacheChargeTrackingCache<
    CacheEntryRole::kBlockBasedTableReader>;
 template class TargetCacheChargeTrackingCache<CacheEntryRole::kFileMetadata>;

+const std::vector<Temperature> kKnownTemperatures = {
+    Temperature::kHot, Temperature::kWarm, Temperature::kCool,
+    Temperature::kCold, Temperature::kIce};
+
+Temperature RandomKnownTemperature() {
+  return kKnownTemperatures[Random::GetTLSInstance()->Uniform(
+      static_cast<int>(kKnownTemperatures.size()))];
+}
+
 }  // namespace ROCKSDB_NAMESPACE
@@ -1438,20 +1438,23 @@ class DBTestBase : public testing::Test {
    std::replace(tp_string.begin(), tp_string.end(), ';', ' ');
    std::replace(tp_string.begin(), tp_string.end(), '=', ' ');
    ResetTableProperties(tp);
-    sscanf(tp_string.c_str(),
-           "# data blocks %" SCNu64 " # entries %" SCNu64
-           " # deletions %" SCNu64 " # merge operands %" SCNu64
-           " # range deletions %" SCNu64 " raw key size %" SCNu64
-           " raw average key size %lf "
-           " raw value size %" SCNu64
-           " raw average value size %lf "
-           " data block size %" SCNu64 " index block size (user-key? %" SCNu64
-           ", delta-value? %" SCNu64 ") %" SCNu64 " filter block size %" SCNu64,
-           &tp->num_data_blocks, &tp->num_entries, &tp->num_deletions,
-           &tp->num_merge_operands, &tp->num_range_deletions, &tp->raw_key_size,
-           &dummy_double, &tp->raw_value_size, &dummy_double, &tp->data_size,
-           &tp->index_key_is_user_key, &tp->index_value_is_delta_encoded,
-           &tp->index_size, &tp->filter_size);
+    int count = sscanf(
+        tp_string.c_str(),
+        "# data blocks %" SCNu64 " # entries %" SCNu64 " # deletions %" SCNu64
+        " # merge operands %" SCNu64 " # range deletions %" SCNu64
+        " raw key size %" SCNu64
+        " raw average key size %lf "
+        " raw value size %" SCNu64
+        " raw average value size %lf "
+        " data block size %" SCNu64 " data uncompressed size %" SCNu64
+        " index block size (user-key? %" SCNu64 ", delta-value? %" SCNu64
+        ") %" SCNu64 " filter block size %" SCNu64,
+        &tp->num_data_blocks, &tp->num_entries, &tp->num_deletions,
+        &tp->num_merge_operands, &tp->num_range_deletions, &tp->raw_key_size,
+        &dummy_double, &tp->raw_value_size, &dummy_double, &tp->data_size,
+        &tp->uncompressed_data_size, &tp->index_key_is_user_key,
+        &tp->index_value_is_delta_encoded, &tp->index_size, &tp->filter_size);
+    ASSERT_EQ(count, 15);
  }

 private:  // Prone to error on direct use
@@ -1464,4 +1467,8 @@ class DBTestBase : public testing::Test {
 // unique ids.
 void VerifySstUniqueIds(const TablePropertiesCollection& props);

+// Excludes kUnknown
+extern const std::vector<Temperature> kKnownTemperatures;
+Temperature RandomKnownTemperature();
+
 }  // namespace ROCKSDB_NAMESPACE
@@ -129,6 +129,7 @@ void EventHelpers::LogAndNotifyTableFileCreationFinished(
              << "user_defined_timestamps_persisted"
              << table_properties.user_defined_timestamps_persisted
              << "key_largest_seqno" << table_properties.key_largest_seqno
+              << "key_smallest_seqno" << table_properties.key_smallest_seqno
              << "merge_operator" << table_properties.merge_operator_name
              << "prefix_extractor_name"
              << table_properties.prefix_extractor_name << "property_collectors"
@@ -1311,7 +1311,7 @@ TEST_F(ExternalSSTFileBasicTest, SyncFailure) {
    });
    if (i == 0) {
      SyncPoint::GetInstance()->SetCallBack(
-          "ExternalSstFileIngestionJob::CheckSyncReturnCode", [&](void* s) {
+          "ExternalSstFileIngestionJob::Prepare:Reopen", [&](void* s) {
            Status* status = static_cast<Status*>(s);
            if (status->IsNotSupported()) {
              no_sync = true;
@@ -1372,11 +1372,11 @@ TEST_F(ExternalSSTFileBasicTest, ReopenNotSupported) {
  options.create_if_missing = true;
  options.env = env_;

-  SyncPoint::GetInstance()->SetCallBack("FileSystem::SyncFile:Open",
-                                        [&](void* arg) {
-                                          Status* s = static_cast<Status*>(arg);
-                                          *s = Status::NotSupported();
-                                        });
+  SyncPoint::GetInstance()->SetCallBack(
+      "ExternalSstFileIngestionJob::Prepare:Reopen", [&](void* arg) {
+        Status* s = static_cast<Status*>(arg);
+        *s = Status::NotSupported();
+      });
  SyncPoint::GetInstance()->EnableProcessing();

  DestroyAndReopen(options);
@@ -42,6 +42,9 @@ Status ExternalSstFileIngestionJob::Prepare(
    status =
        GetIngestedFileInfo(file_path, next_file_number++, &file_to_ingest, sv);
    if (!status.ok()) {
+      ROCKS_LOG_WARN(db_options_.info_log,
+                     "Failed to get ingested file info: %s: %s",
+                     file_path.c_str(), status.ToString().c_str());
      return status;
    }

@@ -160,26 +163,35 @@ Status ExternalSstFileIngestionJob::Prepare(
        // It is unsafe to assume application had sync the file and file
        // directory before ingest the file. For integrity of RocksDB we need
        // to sync the file.
-        TEST_SYNC_POINT("ExternalSstFileIngestionJob::BeforeSyncIngestedFile");
-        auto s = fs_->SyncFile(path_inside_db, env_options_, IOOptions(),
-                               db_options_.use_fsync, nullptr);
-        TEST_SYNC_POINT("ExternalSstFileIngestionJob::AfterSyncIngestedFile");
-        TEST_SYNC_POINT_CALLBACK(
-            "ExternalSstFileIngestionJob::CheckSyncReturnCode", &s);
-        if (!s.ok()) {
-          if (s.IsNotSupported()) {
-            // Some file systems (especially remote/distributed) don't support
-            // SyncFile API. Ignore the NotSupported error in that case.
-            ROCKS_LOG_WARN(db_options_.info_log,
-                           "After link the file, SyncFile API is not supported "
-                           "for file %s: %s",
-                           path_inside_db.c_str(), status.ToString().c_str());
-          } else {
-            // for other errors, propagate the error
-            status = s;
-            ROCKS_LOG_WARN(db_options_.info_log,
-                           "Failed to sync ingested file %s: %s",
-                           path_inside_db.c_str(), status.ToString().c_str());
+
+        // TODO(xingbo), We should in general be moving away from production
+        // uses of ReuseWritableFile (except explicitly for WAL recycling),
+        // ReopenWritableFile, and NewRandomRWFile. We should create a
+        // FileSystem::SyncFile/FsyncFile API that by default does the
+        // re-open+sync+close combo but can (a) be reused easily, and (b) be
+        // overridden to do that more cleanly, e.g. in EncryptedEnv.
+        // https://github.com/facebook/rocksdb/issues/13741
+        std::unique_ptr<FSWritableFile> file_to_sync;
+        Status s = fs_->ReopenWritableFile(path_inside_db, env_options_,
+                                           &file_to_sync, nullptr);
+        TEST_SYNC_POINT_CALLBACK("ExternalSstFileIngestionJob::Prepare:Reopen",
+                                 &s);
+        // Some file systems (especially remote/distributed) don't support
+        // reopening a file for writing and don't require reopening and
+        // syncing the file. Ignore the NotSupported error in that case.
+        if (!s.IsNotSupported()) {
+          status = s;
+          if (status.ok()) {
+            TEST_SYNC_POINT(
+                "ExternalSstFileIngestionJob::BeforeSyncIngestedFile");
+            status = SyncIngestedFile(file_to_sync.get());
+            TEST_SYNC_POINT(
+                "ExternalSstFileIngestionJob::AfterSyncIngestedFile");
+            if (!status.ok()) {
+              ROCKS_LOG_WARN(db_options_.info_log,
+                             "Failed to sync ingested file %s: %s",
+                             path_inside_db.c_str(), status.ToString().c_str());
+            }
          }
        }
      } else if (status.IsNotSupported() &&
@@ -189,6 +201,10 @@ Status ExternalSstFileIngestionJob::Prepare(
        ROCKS_LOG_INFO(db_options_.info_log,
                       "Tried to link file %s but it's not supported : %s",
                       path_outside_db.c_str(), status.ToString().c_str());
+      } else {
+        ROCKS_LOG_WARN(db_options_.info_log, "Failed to link file %s to %s: %s",
+                       path_outside_db.c_str(), path_inside_db.c_str(),
+                       status.ToString().c_str());
      }
    } else {
      f.copy_file = true;
@@ -213,6 +229,12 @@ Status ExternalSstFileIngestionJob::Prepare(
                        io_tracer_);
      // The destination of the copy will be ingested
      f.file_temperature = dst_temp;
+
+      if (!status.ok()) {
+        ROCKS_LOG_WARN(db_options_.info_log, "Failed to copy file %s to %s: %s",
+                       path_outside_db.c_str(), path_inside_db.c_str(),
+                       status.ToString().c_str());
+      }
    } else {
      // Note: we currently assume that linking files does not cross
      // temperatures, so no need to change f.file_temperature
@@ -438,6 +460,11 @@ Status ExternalSstFileIngestionJob::NeedsFlush(bool* flush_needed,
    }
    status = cfd_->RangesOverlapWithMemtables(
        ranges, super_version, db_options_.allow_data_in_errors, flush_needed);
+    if (!status.ok()) {
+      ROCKS_LOG_WARN(db_options_.info_log,
+                     "Failed to check ranges overlap with memtables: %s",
+                     status.ToString().c_str());
+    }
  }
  if (status.ok() && *flush_needed) {
    if (!ingestion_options_.allow_blocking_flush) {
@@ -472,6 +499,9 @@ Status ExternalSstFileIngestionJob::Run() {
  bool need_flush = false;
  status = NeedsFlush(&need_flush, super_version);
  if (!status.ok()) {
+    ROCKS_LOG_WARN(db_options_.info_log,
+                   "Failed to check if flush is needed: %s",
+                   status.ToString().c_str());
    return status;
  }
  if (need_flush) {
@@ -543,6 +573,9 @@ Status ExternalSstFileIngestionJob::Run() {
                                     &last_seqno, &batch_uppermost_level,
                                     prev_batch_uppermost_level);
    if (!status.ok()) {
+      ROCKS_LOG_WARN(db_options_.info_log,
+                     "Failed to assign levels for one batch: %s",
+                     status.ToString().c_str());
      return status;
    }

@@ -585,6 +618,8 @@ Status ExternalSstFileIngestionJob::AssignLevelsForOneBatch(
                                &largest_parsed, false /* log_err_key */);
    }
    if (!status.ok()) {
+      ROCKS_LOG_WARN(db_options_.info_log, "Failed to parse internal key: %s",
+                     status.ToString().c_str());
      return status;
    }

@@ -607,6 +642,10 @@ Status ExternalSstFileIngestionJob::AssignLevelsForOneBatch(

    status = AssignGlobalSeqnoForIngestedFile(file, assigned_seqno);
    if (!status.ok()) {
+      ROCKS_LOG_WARN(
+          db_options_.info_log,
+          "Failed to assign global sequence number for ingested file: %s",
+          status.ToString().c_str());
      return status;
    }
    TEST_SYNC_POINT_CALLBACK("ExternalSstFileIngestionJob::Run",
@@ -619,6 +658,9 @@ Status ExternalSstFileIngestionJob::AssignLevelsForOneBatch(

    status = GenerateChecksumForIngestedFile(file);
    if (!status.ok()) {
+      ROCKS_LOG_WARN(db_options_.info_log,
+                     "Failed to generate checksum for ingested file: %s",
+                     status.ToString().c_str());
      return status;
    }

@@ -844,6 +886,10 @@ Status ExternalSstFileIngestionJob::ResetTableReader(
  Status status =
      fs_->NewRandomAccessFile(external_file, fo, &sst_file, nullptr);
  if (!status.ok()) {
+    ROCKS_LOG_WARN(
+        db_options_.info_log,
+        "Failed to create random access file for external file %s: %s",
+        external_file.c_str(), status.ToString().c_str());
    return status;
  }
  Temperature updated_temp = sst_file->GetTemperature();
@@ -966,6 +1012,10 @@ Status ExternalSstFileIngestionJob::SanityCheckTableProperties(
    // user_defined_timestamps_persisted flag for the file.
    file_to_ingest->user_defined_timestamps_persisted = false;
  } else if (!s.ok()) {
+    ROCKS_LOG_WARN(
+        db_options_.info_log,
+        "ValidateUserDefinedTimestampsOptions failed for external file %s: %s",
+        external_file.c_str(), s.ToString().c_str());
    return s;
  }

@@ -990,6 +1040,9 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo(
  Status status = fs_->GetFileSize(external_file, IOOptions(),
                                   &file_to_ingest->file_size, nullptr);
  if (!status.ok()) {
+    ROCKS_LOG_WARN(db_options_.info_log,
+                   "Failed to get file size for external file %s: %s",
+                   external_file.c_str(), status.ToString().c_str());
    return status;
  }

@@ -1006,12 +1059,19 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo(
                            /*user_defined_timestamps_persisted=*/true, sv,
                            file_to_ingest, &table_reader);
  if (!status.ok()) {
+    ROCKS_LOG_WARN(db_options_.info_log,
+                   "Failed to reset table reader for external file %s: %s",
+                   external_file.c_str(), status.ToString().c_str());
    return status;
  }

  status = SanityCheckTableProperties(external_file, new_file_number, sv,
                                      file_to_ingest, &table_reader);
  if (!status.ok()) {
+    ROCKS_LOG_WARN(
+        db_options_.info_log,
+        "Failed to sanity check table properties for external file %s: %s",
+        external_file.c_str(), status.ToString().c_str());
    return status;
  }

@@ -1025,6 +1085,10 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo(
        table_reader.get(), sv, file_to_ingest, allow_data_in_errors);

    if (!seqno_status.ok()) {
+      ROCKS_LOG_WARN(
+          db_options_.info_log,
+          "Failed to get sequence number boundary for external file %s: %s",
+          external_file.c_str(), seqno_status.ToString().c_str());
      return seqno_status;
    }
    assert(file_to_ingest->smallest_seqno <= file_to_ingest->largest_seqno);
@@ -1052,6 +1116,9 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo(
    status = table_reader->VerifyChecksum(
        ro, TableReaderCaller::kExternalSSTIngestion);
    if (!status.ok()) {
+      ROCKS_LOG_WARN(db_options_.info_log,
+                     "Failed to verify checksum for table reader: %s",
+                     status.ToString().c_str());
      return status;
    }
  }
@@ -1243,6 +1310,9 @@ Status ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile(
          ro, env_options_, file_to_ingest->start_ukey,
          file_to_ingest->limit_ukey, lvl, &overlap_with_level);
      if (!status.ok()) {
+        ROCKS_LOG_WARN(db_options_.info_log,
+                       "Failed to check overlap with level iterator: %s",
+                       status.ToString().c_str());
        return status;
      }
      if (overlap_with_level) {
@@ -1355,6 +1425,14 @@ Status ExternalSstFileIngestionJob::AssignGlobalSeqnoForIngestedFile(
      PutFixed64(&seqno_val, seqno);
      status = fsptr->Write(file_to_ingest->global_seqno_offset, seqno_val,
                            IOOptions(), nullptr);
+      if (!status.ok()) {
+        ROCKS_LOG_WARN(db_options_.info_log,
+                       "Failed to write global seqno to %s: %s",
+                       file_to_ingest->internal_file_path.c_str(),
+                       status.ToString().c_str());
+        return status;
+      }
+
      if (status.ok()) {
        TEST_SYNC_POINT("ExternalSstFileIngestionJob::BeforeSyncGlobalSeqno");
        status = SyncIngestedFile(fsptr.get());
@@ -1371,6 +1449,11 @@ Status ExternalSstFileIngestionJob::AssignGlobalSeqnoForIngestedFile(
        return status;
      }
    } else if (!status.IsNotSupported()) {
+      ROCKS_LOG_WARN(
+          db_options_.info_log,
+          "Failed to open ingested file %s for random read/write: %s",
+          file_to_ingest->internal_file_path.c_str(),
+          status.ToString().c_str());
      return status;
    }
  }
@@ -1403,6 +1486,9 @@ IOStatus ExternalSstFileIngestionJob::GenerateChecksumForIngestedFile(
      db_options_.allow_mmap_reads, io_tracer_, db_options_.rate_limiter.get(),
      ro, db_options_.stats, db_options_.clock);
  if (!io_s.ok()) {
+    ROCKS_LOG_WARN(
+        db_options_.info_log, "Failed to generate checksum for %s: %s",
+        file_to_ingest->internal_file_path.c_str(), io_s.ToString().c_str());
    return io_s;
  }
  file_to_ingest->file_checksum = std::move(file_checksum);
@@ -1445,21 +1531,26 @@ Status ExternalSstFileIngestionJob::SyncIngestedFile(TWritableFile* file) {
 Status ExternalSstFileIngestionJob::GetSeqnoBoundaryForFile(
    TableReader* table_reader, SuperVersion* sv,
    IngestedFileInfo* file_to_ingest, bool allow_data_in_errors) {
-  const bool has_largest_seqno =
-      table_reader->GetTableProperties()->HasKeyLargestSeqno();
-  SequenceNumber largest_seqno =
-      table_reader->GetTableProperties()->key_largest_seqno;
-  if (has_largest_seqno && largest_seqno == 0) {
-    file_to_ingest->largest_seqno = 0;
-    file_to_ingest->smallest_seqno = 0;
-    return Status::OK();
+  const auto tp = table_reader->GetTableProperties();
+  const bool has_largest_seqno = tp->HasKeyLargestSeqno();
+  SequenceNumber largest_seqno = tp->key_largest_seqno;
+  if (has_largest_seqno) {
+    file_to_ingest->largest_seqno = largest_seqno;
+    if (largest_seqno == 0) {
+      file_to_ingest->smallest_seqno = 0;
+      return Status::OK();
+    }
+    if (tp->HasKeySmallestSeqno()) {
+      file_to_ingest->smallest_seqno = tp->key_smallest_seqno;
+      return Status::OK();
+    }
  }
-  // The following file scan is only executed when ingesting files with
-  // non-zero seqno.
-  // TODO: record smallest_seqno in table properties to avoid the
-  // file scan here.
-  SequenceNumber smallest_seqno = kMaxSequenceNumber;

+  // For older SST files they may not be recorded in table properties, so
+  // we scan the file to find out.
+  TEST_SYNC_POINT(
+      "ExternalSstFileIngestionJob::GetSeqnoBoundaryForFile:FileScan");
+  SequenceNumber smallest_seqno = kMaxSequenceNumber;
  SequenceNumber largest_seqno_from_iter = 0;
  ReadOptions ro;
  ro.fill_cache = ingestion_options_.fill_cache;
@@ -4099,6 +4099,7 @@ TEST_P(IngestDBGeneratedFileTest2, NonZeroSeqno) {
    SCOPED_TRACE("option_config_ = " + std::to_string(option_config_));

    Options options = CurrentOptions();
+    options.statistics = CreateDBStatistics();
    options.allow_concurrent_memtable_write =
        false;  // Required for VectorRepFactory
    CreateAndReopenWithCF({"non_overlap", "overlap"}, options);
@@ -4135,7 +4136,7 @@ TEST_P(IngestDBGeneratedFileTest2, NonZeroSeqno) {
    // optional L5: files in key range [70, 98]
    // L6: files in key range [1, 79]
    temp_cf_opts.target_file_size_base =
-        4 << 10;  // Small files to create multiple SSTs
+        20 << 10;  // Small files to create multiple SSTs
    temp_cf_opts.num_levels = 7;
    temp_cf_opts.disable_auto_compactions = true;  // Manually set up LSM
    temp_cf_opts.env = options.env;
@@ -4155,7 +4156,7 @@ TEST_P(IngestDBGeneratedFileTest2, NonZeroSeqno) {
    const Snapshot* snapshot = from_db->GetSnapshot();

    for (int k = 1; k < 99; ++k) {
-      expected_values[k] = rnd->RandomString(500);
+      expected_values[k] = rnd->RandomString(2000);
      ASSERT_OK(from_db->Put(wo, temp_cfh, Key(k), expected_values[k]));
    }
    ASSERT_OK(from_db->Flush({}, temp_cfh));
@@ -4233,9 +4234,21 @@ TEST_P(IngestDBGeneratedFileTest2, NonZeroSeqno) {
          "assigned a non-zero sequence number"));
      db_->ReleaseSnapshot(snapshot);
    }
+
+    std::atomic<int> file_scan_count{0};
+    SyncPoint::GetInstance()->SetCallBack(
+        "ExternalSstFileIngestionJob::GetSeqnoBoundaryForFile:FileScan",
+        [&](void* /*arg*/) { file_scan_count++; });
+    SyncPoint::GetInstance()->EnableProcessing();
+
    ASSERT_OK(
        db_->IngestExternalFile(non_overlap_cf, sst_file_paths, ingest_opts));

+    SyncPoint::GetInstance()->DisableProcessing();
+    SyncPoint::GetInstance()->ClearAllCallBacks();
+
+    EXPECT_EQ(file_scan_count, 0);
+
    // Validate ingested data.
    ReadOptions ro;
    std::string val;
@@ -1104,13 +1104,13 @@ Status FlushJob::WriteLevel0Table() {
  const uint64_t micros = clock_->NowMicros() - start_micros;
  const uint64_t cpu_micros = clock_->CPUMicros() - start_cpu_micros;
  flush_stats.micros = micros;
-  flush_stats.cpu_micros = cpu_micros;
+  flush_stats.cpu_micros += cpu_micros;

  ROCKS_LOG_INFO(db_options_.info_log,
                 "[%s] [JOB %d] Flush lasted %" PRIu64
                 " microseconds, and %" PRIu64 " cpu microseconds.\n",
                 cfd_->GetName().c_str(), job_context_->job_id, micros,
-                 cpu_micros);
+                 flush_stats.cpu_micros);

  if (has_output) {
    flush_stats.bytes_written = meta_.fd.GetFileSize();
@@ -70,7 +70,9 @@ ImmutableMemTableOptions::ImmutableMemTableOptions(
      protection_bytes_per_key(
          mutable_cf_options.memtable_protection_bytes_per_key),
      allow_data_in_errors(ioptions.allow_data_in_errors),
-      paranoid_memory_checks(mutable_cf_options.paranoid_memory_checks) {}
+      paranoid_memory_checks(mutable_cf_options.paranoid_memory_checks),
+      memtable_veirfy_per_key_checksum_on_seek(
+          mutable_cf_options.memtable_veirfy_per_key_checksum_on_seek) {}

 MemTable::MemTable(const InternalKeyComparator& cmp,
                   const ImmutableOptions& ioptions,
@@ -115,7 +117,13 @@ MemTable::MemTable(const InternalKeyComparator& cmp,
      oldest_key_time_(std::numeric_limits<uint64_t>::max()),
      approximate_memory_usage_(0),
      memtable_max_range_deletions_(
-          mutable_cf_options.memtable_max_range_deletions) {
+          mutable_cf_options.memtable_max_range_deletions),
+      key_validation_callback_(
+          (moptions_.protection_bytes_per_key != 0 &&
+           moptions_.memtable_veirfy_per_key_checksum_on_seek)
+              ? std::bind(&MemTable::ValidateKey, this, std::placeholders::_1,
+                          std::placeholders::_2)
+              : std::function<Status(const char*, bool)>(nullptr)) {
  UpdateFlushState();
  // something went wrong if we need to flush before inserting anything
  assert(!ShouldScheduleFlush());
@@ -394,7 +402,11 @@ class MemTableIterator : public InternalIterator {
            !mem.GetImmutableMemTableOptions()->inplace_update_support),
        arena_mode_(arena != nullptr),
        paranoid_memory_checks_(mem.moptions_.paranoid_memory_checks),
-        allow_data_in_error(mem.moptions_.allow_data_in_errors) {
+        validate_on_seek_(
+            mem.moptions_.paranoid_memory_checks ||
+            mem.moptions_.memtable_veirfy_per_key_checksum_on_seek),
+        allow_data_in_error_(mem.moptions_.allow_data_in_errors),
+        key_validation_callback_(mem.key_validation_callback_) {
    if (kind == kRangeDelEntries) {
      iter_ = mem.range_del_table_->GetIterator(arena);
    } else if (prefix_extractor_ != nullptr &&
@@ -463,8 +475,10 @@ class MemTableIterator : public InternalIterator {
        }
      }
    }
-    if (paranoid_memory_checks_) {
-      status_ = iter_->SeekAndValidate(k, nullptr, allow_data_in_error);
+    if (validate_on_seek_) {
+      status_ = iter_->SeekAndValidate(k, nullptr, allow_data_in_error_,
+                                       paranoid_memory_checks_,
+                                       key_validation_callback_);
    } else {
      iter_->Seek(k, nullptr);
    }
@@ -488,8 +502,10 @@ class MemTableIterator : public InternalIterator {
        }
      }
    }
-    if (paranoid_memory_checks_) {
-      status_ = iter_->SeekAndValidate(k, nullptr, allow_data_in_error);
+    if (validate_on_seek_) {
+      status_ = iter_->SeekAndValidate(k, nullptr, allow_data_in_error_,
+                                       paranoid_memory_checks_,
+                                       key_validation_callback_);
    } else {
      iter_->Seek(k, nullptr);
    }
@@ -518,7 +534,7 @@ class MemTableIterator : public InternalIterator {
    PERF_COUNTER_ADD(next_on_memtable_count, 1);
    assert(Valid());
    if (paranoid_memory_checks_) {
-      status_ = iter_->NextAndValidate(allow_data_in_error);
+      status_ = iter_->NextAndValidate(allow_data_in_error_);
    } else {
      iter_->Next();
      TEST_SYNC_POINT_CALLBACK("MemTableIterator::Next:0", iter_);
@@ -540,7 +556,7 @@ class MemTableIterator : public InternalIterator {
    PERF_COUNTER_ADD(prev_on_memtable_count, 1);
    assert(Valid());
    if (paranoid_memory_checks_) {
-      status_ = iter_->PrevAndValidate(allow_data_in_error);
+      status_ = iter_->PrevAndValidate(allow_data_in_error_);
    } else {
      iter_->Prev();
    }
@@ -599,7 +615,9 @@ class MemTableIterator : public InternalIterator {
  bool value_pinned_;
  bool arena_mode_;
  const bool paranoid_memory_checks_;
-  const bool allow_data_in_error;
+  const bool validate_on_seek_;
+  const bool allow_data_in_error_;
+  const std::function<Status(const char*, bool)> key_validation_callback_;

  void VerifyEntryChecksum() {
    if (protection_bytes_per_key_ > 0 && Valid()) {
@@ -1493,11 +1511,13 @@ void MemTable::GetFromTable(const LookupKey& key,
  saver.allow_data_in_errors = moptions_.allow_data_in_errors;
  saver.protection_bytes_per_key = moptions_.protection_bytes_per_key;

-  if (!moptions_.paranoid_memory_checks) {
+  if (!moptions_.paranoid_memory_checks &&
+      !moptions_.memtable_veirfy_per_key_checksum_on_seek) {
    table_->Get(key, &saver, SaveValue);
  } else {
-    Status check_s = table_->GetAndValidate(key, &saver, SaveValue,
-                                            moptions_.allow_data_in_errors);
+    Status check_s = table_->GetAndValidate(
+        key, &saver, SaveValue, moptions_.allow_data_in_errors,
+        moptions_.paranoid_memory_checks, key_validation_callback_);
    if (check_s.IsCorruption()) {
      *(saver.status) = check_s;
      // Should stop searching the LSM.
@@ -1508,6 +1528,11 @@ void MemTable::GetFromTable(const LookupKey& key,
  *seq = saver.seq;
 }

+Status MemTable::ValidateKey(const char* key, bool allow_data_in_errors) {
+  return VerifyEntryChecksum(key, moptions_.protection_bytes_per_key,
+                             allow_data_in_errors);
+}
+
 void MemTable::MultiGet(const ReadOptions& read_options, MultiGetRange* range,
                        ReadCallback* callback, bool immutable_memtable) {
  // The sequence number is updated synchronously in version_set.h
@@ -64,6 +64,7 @@ struct ImmutableMemTableOptions {
  uint32_t protection_bytes_per_key;
  bool allow_data_in_errors;
  bool paranoid_memory_checks;
+  bool memtable_veirfy_per_key_checksum_on_seek;
 };

 // Batched counters to updated when inserting keys in one write batch.
@@ -826,6 +827,9 @@ class MemTable final : public ReadOnlyMemTable {
                                    uint32_t protection_bytes_per_key,
                                    bool allow_data_in_errors = false);

+  // Validate the checksum of the key/value pair.
+  Status ValidateKey(const char* key, bool allow_data_in_errors);
+
 private:
  enum FlushStateEnum { FLUSH_NOT_REQUESTED, FLUSH_REQUESTED, FLUSH_SCHEDULED };

@@ -956,6 +960,8 @@ class MemTable final : public ReadOnlyMemTable {
                           SequenceNumber s, char* checksum_ptr);

  void MaybeUpdateNewestUDT(const Slice& user_key);
+
+  const std::function<Status(const char*, bool)> key_validation_callback_;
 };

 const char* EncodeKey(std::string* scratch, const Slice& target);
@@ -32,6 +32,7 @@ bool MergeOperator::FullMergeV3(const MergeOperationInputV3& merge_in,
                                MergeOperationOutputV3* merge_out) const {
  assert(merge_out);

+  Slice value_of_default;  // avoid warning about in_v2 pointing at this
  MergeOperationInput in_v2(merge_in.key, nullptr, merge_in.operand_list,
                            merge_in.logger);

@@ -66,7 +67,6 @@ bool MergeOperator::FullMergeV3(const MergeOperationInputV3& merge_in,
            const bool has_default_column =
                WideColumnsHelper::HasDefaultColumn(existing_columns);

-            Slice value_of_default;
            if (has_default_column) {
              value_of_default = existing_columns.front().value();
            }
@@ -40,6 +40,11 @@ MultiScan::MultiScan(const ReadOptions& read_options,
 }

 MultiScanIterator& MultiScanIterator::operator++() {
+  status_ = db_iter_->status();
+  if (!status_.ok()) {
+    throw MultiScanException(status_);
+  }
+
  if (idx_ >= scan_opts_.size()) {
    throw std::logic_error("Index out of range");
  }
@@ -396,7 +396,7 @@ const char* VersionEdit::DecodeNewFile4From(Slice* input) {
            return "temperature field wrong size";
          } else {
            Temperature casted_field = static_cast<Temperature>(field[0]);
-            if (casted_field <= Temperature::kCold) {
+            if (casted_field < Temperature::kLastTemperature) {
              f.temperature = casted_field;
            }
          }
@@ -1101,6 +1101,27 @@ class LevelIterator final : public InternalIterator {
    read_seq_ = read_seq;
  }

+  inline bool FileHasMultiScanArg(size_t file_index) {
+    if (file_to_scan_opts_.get()) {
+      auto it = file_to_scan_opts_->find(file_index);
+      if (it != file_to_scan_opts_->end()) {
+        return !it->second.empty();
+      }
+    }
+    return false;
+  }
+
+  MultiScanArgs& GetMultiScanArgForFile(size_t file_index) {
+    auto multi_scan_args_it = file_to_scan_opts_->find(file_index);
+    if (multi_scan_args_it == file_to_scan_opts_->end()) {
+      auto ret = file_to_scan_opts_->emplace(
+          file_index, MultiScanArgs(user_comparator_.user_comparator()));
+      multi_scan_args_it = ret.first;
+      assert(ret.second);
+    }
+    return multi_scan_args_it->second;
+  }
+
  void Prepare(const MultiScanArgs* so) override {
    // We assume here that scan_opts is sorted such that
    // scan_opts[0].range.start < scan_opts[1].range.start, and non overlapping
@@ -1109,6 +1130,9 @@ class LevelIterator final : public InternalIterator {
    }
    scan_opts_ = so;

+    // Verify comparator is consistent
+    assert(so->GetComparator() == user_comparator_.user_comparator());
+
    file_to_scan_opts_ = std::make_unique<ScanOptionsMap>();
    for (size_t k = 0; k < scan_opts_->size(); k++) {
      const ScanOptions& opt = scan_opts_->GetScanRanges().at(k);
@@ -1124,8 +1148,26 @@ class LevelIterator final : public InternalIterator {
        continue;
      }

-      InternalKey istart(start.value(), kMaxSequenceNumber, kValueTypeForSeek);
-      InternalKey iend(end.value(), 0, kValueTypeForSeekForPrev);
+      const size_t timestamp_size =
+          user_comparator_.user_comparator()->timestamp_size();
+      InternalKey istart, iend;
+      if (timestamp_size == 0) {
+        istart =
+            InternalKey(start.value(), kMaxSequenceNumber, kValueTypeForSeek);
+        // end key is exclusive for multiscan
+        iend = InternalKey(end.value(), kMaxSequenceNumber, kValueTypeForSeek);
+      } else {
+        std::string start_key_with_ts, end_key_with_ts;
+        AppendKeyWithMaxTimestamp(&start_key_with_ts, start.value(),
+                                  timestamp_size);
+        AppendKeyWithMaxTimestamp(&end_key_with_ts, end.value(),
+                                  timestamp_size);
+        istart = InternalKey(start_key_with_ts, kMaxSequenceNumber,
+                             kValueTypeForSeek);
+        // end key is exclusive for multiscan
+        iend =
+            InternalKey(end_key_with_ts, kMaxSequenceNumber, kValueTypeForSeek);
+      }

      // TODO: This needs to be optimized, right now we iterate twice, which
      // we dont need to. We can do this in N rather than 2N.
@@ -1139,14 +1181,25 @@ class LevelIterator final : public InternalIterator {
      // 3. [  S  ] ...... [  E  ]
      for (auto i = fstart; i <= fend; i++) {
        if (i < flevel_->num_files) {
-          (*file_to_scan_opts_)[i].insert(start.value(), end.value(),
-                                          opt.property_bag);
+          // FindFile only compares against the largest_key, so we need this
+          // additional check to ensure the scan range overlaps the file
+          if (icomparator_.InternalKeyComparator::Compare(
+                  iend.Encode(), flevel_->files[i].smallest_key) < 0) {
+            continue;
+          }
+          auto const metadata = flevel_->files[i].file_metadata;
+          if (metadata->FileIsStandAloneRangeTombstone()) {
+            // Skip stand alone range deletion files.
+            continue;
+          }
+          auto& args = GetMultiScanArgForFile(i);
+          args.insert(start.value(), end.value(), opt.property_bag);
        }
      }
    }
-    // Propagate io colaescing threshold
+    // Propagate multiscan configs
    for (auto& file_to_arg : *file_to_scan_opts_) {
-      file_to_arg.second.io_coalesce_threshold = so->io_coalesce_threshold;
+      file_to_arg.second.CopyConfigFrom(*so);
    }
  }

@@ -1222,6 +1275,10 @@ class LevelIterator final : public InternalIterator {
    }
  }

+#ifndef NDEBUG
+  bool OverlapRange(const ScanOptions& opts);
+#endif
+
  TableCache* table_cache_;
  const ReadOptions& read_options_;
  const FileOptions& file_options_;
@@ -1319,6 +1376,14 @@ void LevelIterator::Seek(const Slice& target) {
  }

  if (file_iter_.iter() != nullptr) {
+    if (scan_opts_) {
+      // At this point, we only know that the seek target is < largest_key
+      // in the file. We need to check whether there is actual overlap.
+      const FdWithKeyRange& cur_file = flevel_->files[file_index_];
+      if (KeyReachedUpperBound(cur_file.smallest_key)) {
+        return;
+      }
+    }
    file_iter_.Seek(target);
    // Status::TryAgain indicates asynchronous request for retrieval of data
    // blocks has been submitted. So it should return at this point and Seek
@@ -1543,9 +1608,10 @@ bool LevelIterator::SkipEmptyFileForward() {
    if (file_iter_.iter() != nullptr) {
      // If we are doing prepared scan opts then we should seek to the values
      // specified by the scan opts
-      if (scan_opts_ && (*file_to_scan_opts_)[file_index_].size()) {
+
+      if (scan_opts_ && FileHasMultiScanArg(file_index_)) {
        const ScanOptions& opts =
-            file_to_scan_opts_->at(file_index_).GetScanRanges().front();
+            GetMultiScanArgForFile(file_index_).GetScanRanges().front();
        if (opts.range.start.has_value()) {
          InternalKey target(*opts.range.start.AsPtr(), kMaxSequenceNumber,
                             kValueTypeForSeek);
@@ -1595,6 +1661,19 @@ void LevelIterator::SkipEmptyFileBackward() {
  }
 }

+#ifndef NDEBUG
+bool LevelIterator::OverlapRange(const ScanOptions& opts) {
+  return (user_comparator_.CompareWithoutTimestamp(
+              opts.range.start.value(), /*a_has_ts=*/false,
+              ExtractUserKey(flevel_->files[file_index_].largest_key),
+              /*b_has_ts=*/true) <= 0 &&
+          user_comparator_.CompareWithoutTimestamp(
+              opts.range.limit.value(), /*a_has_ts=*/false,
+              ExtractUserKey(flevel_->files[file_index_].smallest_key),
+              /*b_has_ts=*/true) > 0);
+}
+#endif
+
 void LevelIterator::SetFileIterator(InternalIterator* iter) {
  if (pinned_iters_mgr_ && iter) {
    iter->SetPinnedItersMgr(pinned_iters_mgr_);
@@ -1602,12 +1681,11 @@ void LevelIterator::SetFileIterator(InternalIterator* iter) {

  InternalIterator* old_iter = file_iter_.Set(iter);
  if (iter && scan_opts_) {
-    if (file_to_scan_opts_.get() &&
-        file_to_scan_opts_->find(file_index_) != file_to_scan_opts_->end()) {
-      const MultiScanArgs& new_opts = file_to_scan_opts_->at(file_index_);
+    if (FileHasMultiScanArg(file_index_)) {
+      const MultiScanArgs& new_opts = GetMultiScanArgForFile(file_index_);
+      assert(OverlapRange(*new_opts.GetScanRanges().begin()) &&
+             OverlapRange(*new_opts.GetScanRanges().rbegin()));
      file_iter_.Prepare(&new_opts);
-    } else {
-      file_iter_.Prepare(scan_opts_);
    }
  }

@@ -9,7 +9,6 @@ add_executable(db_stress${ARTIFACT_SUFFIX}
  db_stress_gflags.cc
  db_stress_listener.cc
  db_stress_shared_state.cc
-  db_stress_stat.cc
  db_stress_test_base.cc
  db_stress_wide_merge_operator.cc
  db_stress_tool.cc
@@ -275,6 +275,7 @@ DECLARE_string(last_level_temperature);
 DECLARE_string(default_write_temperature);
 DECLARE_string(default_temperature);
 DECLARE_bool(paranoid_memory_checks);
+DECLARE_bool(memtable_veirfy_per_key_checksum_on_seek);

 // Options for transaction dbs.
 // Use TransactionDB (a.k.a. Pessimistic Transaction DB)
@@ -284,6 +285,7 @@ DECLARE_bool(use_txn);
 // Options for TransactionDB (a.k.a. Pessimistic Transaction DB)
 DECLARE_uint64(txn_write_policy);
 DECLARE_bool(unordered_write);
+DECLARE_bool(use_per_key_point_lock_mgr);

 // Options for OptimisticTransactionDB
 DECLARE_bool(use_optimistic_txn);
@@ -424,12 +426,21 @@ DECLARE_uint32(commit_bypass_memtable_one_in);
 DECLARE_bool(track_and_verify_wals);
 DECLARE_int32(remote_compaction_worker_threads);
 DECLARE_int32(remote_compaction_worker_interval);
+DECLARE_bool(remote_compaction_failure_fall_back_to_local);
 DECLARE_bool(auto_refresh_iterator_with_snapshot);
 DECLARE_uint32(memtable_op_scan_flush_trigger);
 DECLARE_uint32(memtable_avg_op_scan_flush_trigger);
 DECLARE_uint32(ingest_wbwi_one_in);
 DECLARE_bool(universal_reduce_file_locking);
 DECLARE_bool(use_multiscan);
+DECLARE_bool(multiscan_use_async_io);
+
+// Compaction deletion trigger declarations for stress testing
+DECLARE_bool(enable_compaction_on_deletion_trigger);
+DECLARE_uint64(compaction_on_deletion_min_file_size);
+DECLARE_int32(compaction_on_deletion_trigger_count);
+DECLARE_int32(compaction_on_deletion_window_size);
+DECLARE_double(compaction_on_deletion_ratio);

 constexpr long KB = 1024;
 constexpr int kRandomValueMaxFactor = 3;
@@ -14,8 +14,11 @@ namespace ROCKSDB_NAMESPACE {
 // Service to simulate Remote Compaction in Stress Test
 class DbStressCompactionService : public CompactionService {
 public:
-  explicit DbStressCompactionService(SharedState* shared)
-      : shared_(shared), aborted_(false) {}
+  explicit DbStressCompactionService(SharedState* shared,
+                                     bool failure_should_fall_back_to_local)
+      : shared_(shared),
+        aborted_(false),
+        failure_should_fall_back_to_local_(failure_should_fall_back_to_local) {}

  static const char* kClassName() { return "DbStressCompactionService"; }

@@ -56,6 +59,11 @@ class DbStressCompactionService : public CompactionService {
      }
      Env::Default()->SleepForMicroseconds(kWaitIntervalInMicros);
    }
+    if (failure_should_fall_back_to_local_) {
+      fprintf(stdout,
+              "Remote Compaction failed - fall back to local compaction!\n");
+      return CompactionServiceJobStatus::kUseLocal;
+    }
    return CompactionServiceJobStatus::kFailure;
  }

@@ -90,6 +98,7 @@ class DbStressCompactionService : public CompactionService {
 private:
  SharedState* shared_;
  std::atomic_bool aborted_{false};
+  bool failure_should_fall_back_to_local_;
 };

 }  // namespace ROCKSDB_NAMESPACE
@@ -724,6 +724,10 @@ DEFINE_uint64(txn_write_policy, 0,
              "TxnDBWritePolicy::WRITE_COMMITTED. Note that this should not be "
              "changed across crashes.");

+DEFINE_bool(use_per_key_point_lock_mgr, true,
+            "Use PointLockManager(false) or PerKeyPointLockManager(true) in "
+            "TransactionDB.");
+
 DEFINE_bool(use_optimistic_txn, false, "Use OptimisticTransactionDB.");
 DEFINE_uint64(occ_validation_policy, 1,
              "Optimistic Concurrency Control Validation Policy for "
@@ -853,6 +857,10 @@ DEFINE_int32(remote_compaction_worker_interval, 10,
             "Remote Compaction Worker Thread dequeue tasks every N "
             "milliseconds. (Default: 10ms)");

+DEFINE_bool(remote_compaction_failure_fall_back_to_local, true,
+            "If true, remote compaction failures will be ignored and "
+            "compactions will fall back to local and retried");
+
 DEFINE_uint32(ingest_wbwi_one_in, 0,
              "If set, will call"
              "IngestWriteBatchWithIndex() instead of regular write operations "
@@ -1476,10 +1484,37 @@ DEFINE_bool(paranoid_memory_checks,
            ROCKSDB_NAMESPACE::Options().paranoid_memory_checks,
            "Sets CF option paranoid_memory_checks.");

+DEFINE_bool(
+    memtable_veirfy_per_key_checksum_on_seek,
+    ROCKSDB_NAMESPACE::Options().memtable_veirfy_per_key_checksum_on_seek,
+    "Sets CF option memtable_veirfy_per_key_checksum_on_seek.");
+
 DEFINE_uint32(commit_bypass_memtable_one_in, 0,
              "If greater than zero, transaction option will set "
              "commit_bypass_memtable to per every N transactions on average.");

+// Compaction on deletion trigger flags
+DEFINE_bool(enable_compaction_on_deletion_trigger, false,
+            "Enable CompactOnDeletionCollectorFactory for stress testing "
+            "deletion-triggered compaction scenarios.");
+
+DEFINE_uint64(compaction_on_deletion_min_file_size, 32 * 1024,
+              "Minimum file size (in bytes) for deletion-triggered compaction. "
+              "Files smaller than this will not trigger compaction even if "
+              "deletion ratio is exceeded. Default: 32KB");
+
+DEFINE_int32(compaction_on_deletion_trigger_count, 50,
+             "Number of deletions that triggers compaction when deletion "
+             "ratio is exceeded. Default: 50");
+
+DEFINE_int32(compaction_on_deletion_window_size, 100,
+             "Size of the sliding window for tracking deletions. "
+             "Default: 100");
+
+DEFINE_double(compaction_on_deletion_ratio, 0.5,
+              "Deletion ratio threshold for triggering compaction. "
+              "Default: 0.5 (50%)");
+
 DEFINE_bool(
    auto_refresh_iterator_with_snapshot,
    ROCKSDB_NAMESPACE::ReadOptions().auto_refresh_iterator_with_snapshot,
@@ -1505,4 +1540,7 @@ DEFINE_bool(
 DEFINE_bool(use_multiscan, false,
            "If set, use the batched MultiScan API for scans.");

+DEFINE_bool(multiscan_use_async_io, false,
+            "If set, enable async_io for MultiScan operations.");
+
 #endif  // GFLAGS
@@ -1,17 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#ifdef GFLAGS
-
-#include "db_stress_tool/db_stress_stat.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-std::shared_ptr<ROCKSDB_NAMESPACE::Statistics> dbstats;
-std::shared_ptr<ROCKSDB_NAMESPACE::Statistics> dbstats_secondaries;
-
-}  // namespace ROCKSDB_NAMESPACE
-
-#endif  // GFLAGS
@@ -22,10 +22,6 @@ DECLARE_bool(progress_reports);

 namespace ROCKSDB_NAMESPACE {

-// Database statistics
-extern std::shared_ptr<ROCKSDB_NAMESPACE::Statistics> dbstats;
-extern std::shared_ptr<ROCKSDB_NAMESPACE::Statistics> dbstats_secondaries;
-
 class Stats {
 private:
  uint64_t start_;
@@ -30,6 +30,7 @@
 #include "rocksdb/filter_policy.h"
 #include "rocksdb/secondary_cache.h"
 #include "rocksdb/sst_file_manager.h"
+#include "rocksdb/table_properties.h"
 #include "rocksdb/types.h"
 #include "rocksdb/utilities/object_registry.h"
 #include "rocksdb/utilities/write_batch_with_index.h"
@@ -427,6 +428,9 @@ bool StressTest::BuildOptionsTable() {
    options_tbl.emplace(
        "file_temperature_age_thresholds",
        std::vector<std::string>{
+            "{{temperature=kWarm;age=10}:{temperature=kCool;age=30}:{"
+            "temperature=kCold;age=100}:{"
+            "temperature=kIce;age=300}}",
            "{{temperature=kWarm;age=30}:{temperature=kCold;age=300}}",
            "{{temperature=kCold;age=100}}", "{}"});
    options_tbl.emplace(
@@ -653,12 +657,20 @@ std::string StressTest::DebugString(const Slice& value,
 }

 void StressTest::PrintStatistics() {
-  if (dbstats) {
-    fprintf(stdout, "STATISTICS:\n%s\n", dbstats->ToString().c_str());
+  // Print statistics from the DB instance instead of global dbstats
+  if (db_) {
+    auto stats = db_->GetOptions().statistics;
+    if (stats) {
+      fprintf(stdout, "STATISTICS:\n%s\n", stats->ToString().c_str());
+    }
  }
-  if (dbstats_secondaries) {
-    fprintf(stdout, "Secondary instances STATISTICS:\n%s\n",
-            dbstats_secondaries->ToString().c_str());
+  // Print statistics from secondary DB instance if it exists
+  if (secondary_db_) {
+    auto stats = secondary_db_->GetOptions().statistics;
+    if (stats) {
+      fprintf(stdout, "Secondary instance STATISTICS:\n%s\n",
+              stats->ToString().c_str());
+    }
  }
 }

@@ -1666,7 +1678,11 @@ Status StressTest::TestMultiScan(ThreadState* thread,
  assert(!rand_column_families.empty());
  assert(!rand_keys.empty());

+  ThreadStatus::OperationType cur_op_type =
+      ThreadStatusUtil::GetThreadOperation();
+  ThreadStatusUtil::SetThreadOperation(ThreadStatus::OperationType::OP_UNKNOWN);
  ManagedSnapshot snapshot_guard(db_);
+  ThreadStatusUtil::SetThreadOperation(cur_op_type);

  ReadOptions ro = read_opts;
  ro.snapshot = snapshot_guard.snapshot();
@@ -1677,7 +1693,9 @@ Status StressTest::TestMultiScan(ThreadState* thread,

  std::vector<std::string> start_key_strs;
  std::vector<std::string> end_key_strs;
-  MultiScanArgs scan_opts;
+  // TODO support reverse BytewiseComparator in the stress test
+  MultiScanArgs scan_opts(options_.comparator);
+  scan_opts.use_async_io = FLAGS_multiscan_use_async_io;
  start_key_strs.reserve(num_scans);
  end_key_strs.reserve(num_scans);

@@ -1803,11 +1821,14 @@ Status StressTest::TestMultiScan(ThreadState* thread,
                     key, op_logs, verify_func, &diverged);

      if (diverged) {
-        const std::vector<ScanOptions>& scanoptions = scan_opts.GetScanRanges();
-        for (const auto& t : scanoptions) {
-          fprintf(stdout, "Multiscan options: %s to %s \n",
-                  t.range.start.value().ToString(true).c_str(),
-                  t.range.limit.value().ToString(true).c_str());
+        if (thread->shared->HasVerificationFailedYet()) {
+          const std::vector<ScanOptions>& scanoptions =
+              scan_opts.GetScanRanges();
+          for (const auto& t : scanoptions) {
+            fprintf(stdout, "Multiscan options: %s to %s \n",
+                    t.range.start.value().ToString(true).c_str(),
+                    t.range.limit.value().ToString(true).c_str());
+          }
        }
        break;
      }
@@ -3503,8 +3524,9 @@ void StressTest::PrintEnv() const {
  fprintf(stdout, "Verification only         : %s\n",
          FLAGS_verification_only ? "true" : "false");

-  const char* memtablerep = "";
+  const char* memtablerep;
  switch (FLAGS_rep_factory) {
+    default:
    case kSkipList:
      memtablerep = "skip_list";
      break;
@@ -3653,8 +3675,8 @@ void StressTest::Open(SharedState* shared, bool reopen) {
              "Compaction\n");
      exit(1);
    }
-    options_.compaction_service =
-        std::make_shared<DbStressCompactionService>(shared);
+    options_.compaction_service = std::make_shared<DbStressCompactionService>(
+        shared, FLAGS_remote_compaction_failure_fall_back_to_local);
  }

  if ((options_.enable_blob_files || options_.enable_blob_garbage_collection ||
@@ -3930,6 +3952,8 @@ void StressTest::Open(SharedState* shared, bool reopen) {
            static_cast<size_t>(FLAGS_wp_snapshot_cache_bits);
        txn_db_options.wp_commit_cache_bits =
            static_cast<size_t>(FLAGS_wp_commit_cache_bits);
+        txn_db_options.use_per_key_point_lock_mgr =
+            FLAGS_use_per_key_point_lock_mgr;
        PrepareTxnDbOptions(shared, txn_db_options);
        s = TransactionDB::Open(options_, txn_db_options, FLAGS_db,
                                cf_descriptors, &column_families_, &txn_db_);
@@ -3954,6 +3978,13 @@ void StressTest::Open(SharedState* shared, bool reopen) {
    assert(s.ok());
    assert(column_families_.size() ==
           static_cast<size_t>(FLAGS_column_families));
+    // Clear statistics reference from options_ to intentionally shorten the
+    // statistics object lifetime to be same as the db object (which is the
+    // common case in practice) and detect if RocksDB access the statistics
+    // beyond its lifetime.
+    if (FLAGS_statistics) {
+      options_.statistics.reset();
+    }

    // Secondary instance does not support write-prepared/write-unprepared
    // transactions, thus just disable secondary instance if we use
@@ -4328,7 +4359,9 @@ void InitializeOptionsFromFlags(
    }
  }
  options.max_open_files = FLAGS_open_files;
-  options.statistics = dbstats;
+  if (FLAGS_statistics) {
+    options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
+  }
  options.env = db_stress_env;
  options.use_fsync = FLAGS_use_fsync;
  options.compaction_readahead_size = FLAGS_compaction_readahead_size;
@@ -4410,6 +4443,8 @@ void InitializeOptionsFromFlags(
      FLAGS_memtable_protection_bytes_per_key;
  options.block_protection_bytes_per_key = FLAGS_block_protection_bytes_per_key;
  options.paranoid_memory_checks = FLAGS_paranoid_memory_checks;
+  options.memtable_veirfy_per_key_checksum_on_seek =
+      FLAGS_memtable_veirfy_per_key_checksum_on_seek;

  // Integrated BlobDB
  options.enable_blob_files = FLAGS_enable_blob_files;
@@ -4578,8 +4613,8 @@ void InitializeOptionsGeneral(
  options.create_missing_column_families = true;
  options.create_if_missing = true;

-  if (!options.statistics) {
-    options.statistics = dbstats;
+  if (FLAGS_statistics) {
+    options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
  }

  if (options.env == Options().env) {
@@ -4652,6 +4687,16 @@ void InitializeOptionsGeneral(
  if (sqfc_factory && !sqfc_factory->GetConfigs().IsEmptyNotFound()) {
    options.table_properties_collector_factories.emplace_back(sqfc_factory);
  }
+
+  // Add CompactOnDeletionCollectorFactory if enabled
+  if (FLAGS_enable_compaction_on_deletion_trigger) {
+    options.table_properties_collector_factories.emplace_back(
+        ROCKSDB_NAMESPACE::NewCompactOnDeletionCollectorFactory(
+            FLAGS_compaction_on_deletion_window_size,
+            FLAGS_compaction_on_deletion_trigger_count,
+            FLAGS_compaction_on_deletion_ratio,
+            FLAGS_compaction_on_deletion_min_file_size));
+  }
 }

 }  // namespace ROCKSDB_NAMESPACE
@@ -53,12 +53,6 @@ int db_stress_tool(int argc, char** argv) {
    SetupSyncPointsToMockDirectIO();
  }
 #endif
-  if (FLAGS_statistics) {
-    dbstats = ROCKSDB_NAMESPACE::CreateDBStatistics();
-    if (FLAGS_test_secondary) {
-      dbstats_secondaries = ROCKSDB_NAMESPACE::CreateDBStatistics();
-    }
-  }
  compression_type_e = StringToCompressionType(FLAGS_compression_type.c_str());
  bottommost_compression_type_e =
      StringToCompressionType(FLAGS_bottommost_compression_type.c_str());
@@ -3156,13 +3156,15 @@ class NonBatchedOpsStressTest : public StressTest {

      Status s;

+      ExpectedValue new_expected_value;
+
      switch (op) {
        case Op::PutOrPutEntity:
        case Op::Merge: {
          ExpectedValue put_value;
          put_value.SyncPut(static_cast<uint32_t>(thread->rand.Uniform(
              static_cast<int>(ExpectedValue::GetValueBaseMask()))));
-          ryw_expected_values[k] = put_value;
+          new_expected_value = put_value;

          const uint32_t value_base = put_value.GetValueBase();

@@ -3186,7 +3188,7 @@ class NonBatchedOpsStressTest : public StressTest {
        case Op::Delete: {
          ExpectedValue delete_value;
          delete_value.SyncDelete();
-          ryw_expected_values[k] = delete_value;
+          new_expected_value = delete_value;

          s = txn->Delete(cfh, k);
          break;
@@ -3195,6 +3197,20 @@ class NonBatchedOpsStressTest : public StressTest {
          assert(false);
      }

+      // It is possible that multiple thread concurrently try to write to the
+      // same key, which could cause lock timeout or deadlock in the
+      // transactiondb layer, before transaction is rolled back.
+      // E.g.
+      // Timestamp 1: Transaction A: lock key M for write
+      // Timestamp 2: Transaction B: lock key N for write
+      // Timestamp 3: Transaction B: try to lock key M for write -> wait
+      // Timestamp 4: Transaction A: try to lock key N for write -> deadlock
+      if (s.IsTimedOut() || s.IsDeadlock()) {
+        return;
+      }
+
+      ryw_expected_values[k] = new_expected_value;
+
      if (!s.ok()) {
        fprintf(stderr,
                "Transaction write error in read-your-own-write test: %s\n",
@@ -142,12 +142,6 @@ class CompositeEnv : public Env {
    return file_system_->LinkFile(s, t, io_opts, &dbg);
  }

-  Status SyncFile(const std::string& fname, const EnvOptions& env_options,
-                  bool use_fsync) override {
-    return file_system_->SyncFile(fname, env_options, IOOptions(), use_fsync,
-                                  nullptr);
-  }
-
  Status NumFileLinks(const std::string& fname, uint64_t* count) override {
    IOOptions io_opts;
    IODebugContext dbg;
@@ -9,6 +9,7 @@

 #include "rocksdb/env.h"

+#include <sstream>
 #include <thread>

 #include "env/composite_env_wrapper.h"
@@ -528,13 +529,6 @@ class LegacyFileSystemWrapper : public FileSystem {
    return status_to_io_status(target_->LinkFile(s, t));
  }

-  IOStatus SyncFile(const std::string& fname, const FileOptions& file_options,
-                    const IOOptions& /*io_options*/, bool use_fsync,
-                    IODebugContext* /*dbg*/) override {
-    return status_to_io_status(
-        target_->SyncFile(fname, file_options, use_fsync));
-  }
-
  IOStatus NumFileLinks(const std::string& fname, const IOOptions& /*options*/,
                        uint64_t* count, IODebugContext* /*dbg*/) override {
    return status_to_io_status(target_->NumFileLinks(fname, count));
@@ -769,6 +763,17 @@ std::string Env::IOActivityToString(IOActivity activity) {
      return "GetFileChecksumsFromCurrentManifest";
    case Env::IOActivity::kUnknown:
      return "Unknown";
+    default:
+      int activityIndex = static_cast<int>(activity);
+      if (activityIndex >=
+              static_cast<int>(Env::IOActivity::kFirstCustomIOActivity) &&
+          activityIndex <=
+              static_cast<int>(Env::IOActivity::kLastCustomIOActivity)) {
+        std::stringstream ss;
+        ss << std::hex << std::uppercase << activityIndex;
+        return "CustomIOActivity" + ss.str();
+      }
+      return "Invalid";
  };
  assert(false);
  return "Invalid";
@@ -866,43 +871,6 @@ std::string Env::GenerateUniqueId() {
  return result;
 }

-// This API Env::SyncFile is used for testing for 2 reasons:
-//
-// 1. The default implementation of SyncFile API is essentially a wrapper of
-// other FileSystem APIs. FaultInjectionTestEnv uses this default
-// implementation to call other FileSystem APIs defined at
-// FaultInjectionTestEnv class to inject failurses. See
-// FaultInjectionTestEnv::SyncFile for more details
-//
-// 2. Some of old tests are using LegacyFileSystemWrapper.
-// LegacyFileSystemWrapper forwards the API call to EnvWrapper, which forwards
-// to CompositeEnv, and then forwards to the actual FileSystem implemention.
-// Without this API in Env, LegacyFileSystemWrapper will not be able to
-// forward the API call to EnvWrapper, causing the default FileSystem API to
-// be called.
-//
-// Due to the above reason, adding a new API in FileSystem, would very likely
-// require the same API to be added to Env.
-//
-// TODO xingbo. Getting rid of FileSystem functions from Env.
-// We need to simplify the relationship between Env and FileSystem. At least
-// for internal test, we should stop using Env and switch to FileSystem, if
-// possible. Related github issue #9274
-Status Env::SyncFile(const std::string& fname, const EnvOptions& env_options,
-                     bool use_fsync) {
-  std::unique_ptr<WritableFile> file_to_sync;
-  auto status = ReopenWritableFile(fname, &file_to_sync, env_options);
-  TEST_SYNC_POINT_CALLBACK("FileSystem::SyncFile:Open", &status);
-  if (status.ok()) {
-    if (use_fsync) {
-      status = file_to_sync->Fsync();
-    } else {
-      status = file_to_sync->Sync();
-    }
-  }
-  return status;
-}
-
 SequentialFile::~SequentialFile() = default;

 RandomAccessFile::~RandomAccessFile() = default;
@@ -664,8 +664,6 @@ class EncryptedFileSystemImpl : public EncryptedFileSystem {
                              const FileOptions& options,
                              std::unique_ptr<FSWritableFile>* result,
                              IODebugContext* dbg) override {
-    // TODO xingbo Add unit test for the new implementation of
-    // EncryptedFileSysmteImpl::ReopenWritableFile.
    result->reset();
    if (options.use_mmap_reads || options.use_mmap_writes) {
      return IOStatus::InvalidArgument();
@@ -816,15 +814,6 @@ class EncryptedFileSystemImpl : public EncryptedFileSystem {
    return status;
  }

-  IOStatus SyncFile(const std::string& fname, const FileOptions& file_options,
-                    const IOOptions& io_options, bool use_fsync,
-                    IODebugContext* dbg) override {
-    // Use the underlying file system to sync the file, as we don't need to
-    // read/write the file.
-    return FileSystemWrapper::SyncFile(fname, file_options, io_options,
-                                       use_fsync, dbg);
-  }
-
 private:
  std::shared_ptr<EncryptionProvider> provider_;
 };
@@ -3692,6 +3692,24 @@ TEST_F(TestGetFileSize, GetFileSize) {
  ASSERT_EQ(fileSizeFromFsRandomAccessFileAPI, expectedFileSize);
 }

+class TestIOActivity : public testing::Test {
+ public:
+  TestIOActivity() {}
+};
+
+TEST_F(TestIOActivity, IOActivityToString) {
+  ASSERT_EQ(Env::IOActivityToString(Env::IOActivity::kMultiGet), "MultiGet");
+
+  ASSERT_EQ(Env::IOActivityToString(Env::IOActivity::kCustomIOActivity80),
+            "CustomIOActivity80");
+  ASSERT_EQ(Env::IOActivityToString(Env::IOActivity::kCustomIOActivityA9),
+            "CustomIOActivityA9");
+  ASSERT_EQ(Env::IOActivityToString(Env::IOActivity::kCustomIOActivityFE),
+            "CustomIOActivityFE");
+
+  ASSERT_EQ(Env::IOActivityToString(Env::IOActivity::kUnknown), "Unknown");
+}
+
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -107,23 +107,6 @@ IOStatus FileSystem::ReuseWritableFile(const std::string& fname,
  return NewWritableFile(fname, opts, result, dbg);
 }

-IOStatus FileSystem::SyncFile(const std::string& fname,
-                              const FileOptions& file_options,
-                              const IOOptions& io_options, bool use_fsync,
-                              IODebugContext* dbg) {
-  std::unique_ptr<FSWritableFile> file_to_sync;
-  auto status = ReopenWritableFile(fname, file_options, &file_to_sync, dbg);
-  TEST_SYNC_POINT_CALLBACK("FileSystem::SyncFile:Open", &status);
-  if (status.ok()) {
-    if (use_fsync) {
-      status = file_to_sync->Fsync(io_options, dbg);
-    } else {
-      status = file_to_sync->Sync(io_options, dbg);
-    }
-  }
-  return status;
-}
-
 IOStatus FileSystem::NewLogger(const std::string& fname,
                               const IOOptions& io_opts,
                               std::shared_ptr<Logger>* result,
@@ -355,9 +355,11 @@ IOStatus FSRandomAccessFileTracingWrapper::ReadAsync(
  IOStatus s = target()->ReadAsync(req, opts, read_async_callback,
                                   read_async_cb_info, io_handle, del_fn, dbg);

+#ifndef __clang_analyzer__
  if (!s.ok()) {
    delete read_async_cb_info;
  }
+#endif  // __clang_analyzer__
  return s;
 }

@@ -1153,7 +1153,7 @@ class PosixFileSystem : public FileSystem {
    return IOStatus::OK();
 #else
    (void)io_handles;
-    return IOStatus::NotSupported("Poll");
+    return IOStatus::NotSupported("Poll not implemented");
 #endif
  }

@@ -935,7 +935,8 @@ IOStatus PosixRandomAccessFile::ReadAsync(

  // Init failed, platform doesn't support io_uring.
  if (iu == nullptr) {
-    return IOStatus::NotSupported("ReadAsync");
+    fprintf(stderr, "failed to init io_uring\n");
+    return IOStatus::NotSupported("ReadAsync: failed to init io_uring");
  }

  // Allocate io_handle.
@@ -978,7 +979,8 @@ IOStatus PosixRandomAccessFile::ReadAsync(
  (void)cb_arg;
  (void)io_handle;
  (void)del_fn;
-  return IOStatus::NotSupported("ReadAsync");
+  return IOStatus::NotSupported(
+      "ReadAsync: ROCKSDB_IOURING_PRESENT is not set");
 #endif
 }

@@ -957,14 +957,6 @@ IOStatus MockFileSystem::LinkFile(const std::string& src,
  return IOStatus::OK();
 }

-IOStatus MockFileSystem::SyncFile(const std::string& /*fname*/,
-                                  const FileOptions& /*file_options*/,
-                                  const IOOptions& /*io_options*/,
-                                  bool /*use_fsync*/, IODebugContext* /*dbg*/) {
-  // Noop
-  return IOStatus::OK();
-}
-
 IOStatus MockFileSystem::NewLogger(const std::string& fname,
                                   const IOOptions& io_opts,
                                   std::shared_ptr<Logger>* result,
@@ -86,10 +86,6 @@ class MockFileSystem : public FileSystem {
  IOStatus LinkFile(const std::string& /*src*/, const std::string& /*target*/,
                    const IOOptions& /*options*/,
                    IODebugContext* /*dbg*/) override;
-  IOStatus SyncFile(const std::string& /*fname*/,
-                    const FileOptions& /*file_options*/,
-                    const IOOptions& /*io_options*/, bool /*use_fsync*/,
-                    IODebugContext* /*dbg*/) override;
  IOStatus LockFile(const std::string& fname, const IOOptions& options,
                    FileLock** lock, IODebugContext* dbg) override;
  IOStatus UnlockFile(FileLock* lock, const IOOptions& options,
@@ -19,16 +19,16 @@ CFLAGS += -Wstrict-prototypes
 all: simple_example column_families_example compact_files_example c_simple_example optimistic_transaction_example transaction_example compaction_filter_example options_file_example rocksdb_backup_restore_example

 simple_example: librocksdb simple_example.cc
-	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++17 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)
+	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++20 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)

 column_families_example: librocksdb column_families_example.cc
-	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++17 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)
+	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++20 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)

 compaction_filter_example: librocksdb compaction_filter_example.cc
-	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++17 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)
+	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++20 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)

 compact_files_example: librocksdb compact_files_example.cc
-	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++17 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)
+	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++20 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)

 .c.o:
 	$(CC) $(CFLAGS) -c $< -o $@ -I../include
@@ -37,19 +37,19 @@ c_simple_example: librocksdb c_simple_example.o
 	$(CXX) $@.o -o$@ ../librocksdb.a $(PLATFORM_LDFLAGS) $(EXEC_LDFLAGS)

 optimistic_transaction_example: librocksdb optimistic_transaction_example.cc
-	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++17 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)
+	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++20 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)

 transaction_example: librocksdb transaction_example.cc
-	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++17 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)
+	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++20 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)

 options_file_example: librocksdb options_file_example.cc
-	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++17 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)
+	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++20 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)

 multi_processes_example: librocksdb multi_processes_example.cc
-	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++17 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)
+	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++20 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)

 rocksdb_backup_restore_example: librocksdb rocksdb_backup_restore_example.cc
-	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++17 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)
+	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++20 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)

 clean:
 	rm -rf ./simple_example ./column_families_example ./compact_files_example ./compaction_filter_example ./c_simple_example c_simple_example.o ./optimistic_transaction_example ./transaction_example ./options_file_example ./multi_processes_example ./rocksdb_backup_restore_example
@@ -80,12 +80,24 @@ inline void RecordIOStats(Statistics* stats, Temperature file_temperature,
        RecordTick(stats, WARM_FILE_READ_BYTES, size);
        RecordTick(stats, WARM_FILE_READ_COUNT, 1);
        break;
+      case Temperature::kCool:
+        IOSTATS_ADD(file_io_stats_by_temperature.cool_file_bytes_read, size);
+        IOSTATS_ADD(file_io_stats_by_temperature.cool_file_read_count, 1);
+        RecordTick(stats, COOL_FILE_READ_BYTES, size);
+        RecordTick(stats, COOL_FILE_READ_COUNT, 1);
+        break;
      case Temperature::kCold:
        IOSTATS_ADD(file_io_stats_by_temperature.cold_file_bytes_read, size);
        IOSTATS_ADD(file_io_stats_by_temperature.cold_file_read_count, 1);
        RecordTick(stats, COLD_FILE_READ_BYTES, size);
        RecordTick(stats, COLD_FILE_READ_COUNT, 1);
        break;
+      case Temperature::kIce:
+        IOSTATS_ADD(file_io_stats_by_temperature.ice_file_bytes_read, size);
+        IOSTATS_ADD(file_io_stats_by_temperature.ice_file_read_count, 1);
+        RecordTick(stats, ICE_FILE_READ_BYTES, size);
+        RecordTick(stats, ICE_FILE_READ_COUNT, 1);
+        break;
      default:
        break;
    }
@@ -491,6 +503,13 @@ IOStatus RandomAccessFileReader::PrepareIOOptions(const ReadOptions& ro,
  }
 }

+// Notes for when direct_io is enabled:
+// Unless req.offset, req.len, req.scratch are all already aligned,
+// RandomAccessFileReader will creats aligned requests and aligned buffer for
+// the request. User should only provide either req.scratch or aligned_buf. If
+// only req.scratch is provided, result will be copied from allocated aligned
+// buffer to req.scratch. If only alignd_buf is provided, it will be set to
+// the ailgned buf allocated by RandomAccessFileReader and saves a copy.
 IOStatus RandomAccessFileReader::ReadAsync(
    FSReadRequest& req, const IOOptions& opts,
    std::function<void(FSReadRequest&, void*)> cb, void* cb_arg,
@@ -64,9 +64,7 @@ enum CompactionPri : char {
 struct FileTemperatureAge {
  Temperature temperature = Temperature::kUnknown;
  uint64_t age = 0;
-#if __cplusplus >= 202002L
  bool operator==(const FileTemperatureAge& rhs) const = default;
-#endif
 };

 struct CompactionOptionsFIFO {
@@ -135,9 +133,7 @@ struct CompactionOptionsFIFO {
      : max_table_files_size(_max_table_files_size),
        allow_compaction(_allow_compaction) {}

-#if __cplusplus >= 202002L
  bool operator==(const CompactionOptionsFIFO& rhs) const = default;
-#endif
 };

 // The control option of how the cache tiers will be used. Currently rocksdb
@@ -1105,12 +1101,22 @@ struct AdvancedColumnFamilyOptions {
  uint32_t bottommost_file_compaction_delay = 0;

  // Enables additional integrity checks during reads/scans.
-  // Specifically, for skiplist-based memtables, we verify that keys visited
-  // are in order. This is helpful to detect corrupted memtable keys during
-  // reads. Enabling this feature incurs a performance overhead due to an
-  // additional key comparison during memtable lookup.
+  // Specifically, for skiplist-based memtables, key ordering validation could
+  // be enabled optionally. This is helpful to detect corrupted memtable keys
+  // during reads. Enabling this feature incurs a performance overhead due to
+  // additional comparison during memtable lookup.
  bool paranoid_memory_checks = false;

+  // Enables additional integrity checks during seek.
+  // Specifically, for skiplist-based memtables, key checksum validation could
+  // be enabled during seek optionally. This is helpful to detect corrupted
+  // memtable keys during reads. Enabling this feature incurs a performance
+  // overhead due to additional key checksum validation during memtable seek
+  // operation.
+  // This option depends on memtable_protection_bytes_per_key to be non zero.
+  // If memtable_protection_bytes_per_key is zero, no validation is performed.
+  bool memtable_veirfy_per_key_checksum_on_seek = false;
+
  // When an iterator scans this number of invisible entries (tombstones or
  // hidden puts) from the active memtable during a single iterator operation,
  // we will attempt to flush the memtable. Currently only forward scans are
@@ -113,10 +113,15 @@ typedef struct rocksdb_writebatch_wi_t rocksdb_writebatch_wi_t;
 typedef struct rocksdb_writeoptions_t rocksdb_writeoptions_t;
 typedef struct rocksdb_universal_compaction_options_t
    rocksdb_universal_compaction_options_t;
+typedef struct rocksdb_livefile_t rocksdb_livefile_t;
 typedef struct rocksdb_livefiles_t rocksdb_livefiles_t;
 typedef struct rocksdb_column_family_handle_t rocksdb_column_family_handle_t;
 typedef struct rocksdb_column_family_metadata_t
    rocksdb_column_family_metadata_t;
+typedef struct rocksdb_import_column_family_options_t
+    rocksdb_import_column_family_options_t;
+typedef struct rocksdb_export_import_files_metadata_t
+    rocksdb_export_import_files_metadata_t;
 typedef struct rocksdb_level_metadata_t rocksdb_level_metadata_t;
 typedef struct rocksdb_sst_file_metadata_t rocksdb_sst_file_metadata_t;
 typedef struct rocksdb_envoptions_t rocksdb_envoptions_t;
@@ -377,6 +382,12 @@ extern ROCKSDB_LIBRARY_API void rocksdb_checkpoint_create(
    rocksdb_checkpoint_t* checkpoint, const char* checkpoint_dir,
    uint64_t log_size_for_flush, char** errptr);

+extern ROCKSDB_LIBRARY_API rocksdb_export_import_files_metadata_t*
+rocksdb_checkpoint_export_column_family(
+    rocksdb_checkpoint_t* checkpoint,
+    rocksdb_column_family_handle_t* column_family, const char* export_dir,
+    char** errptr);
+
 extern ROCKSDB_LIBRARY_API void rocksdb_checkpoint_object_destroy(
    rocksdb_checkpoint_t* checkpoint);

@@ -437,6 +448,13 @@ rocksdb_create_column_families(rocksdb_t* db,
 extern ROCKSDB_LIBRARY_API void rocksdb_create_column_families_destroy(
    rocksdb_column_family_handle_t** list);

+extern ROCKSDB_LIBRARY_API rocksdb_column_family_handle_t*
+rocksdb_create_column_family_with_import(
+    rocksdb_t* db, rocksdb_options_t* column_family_options,
+    const char* column_family_name,
+    rocksdb_import_column_family_options_t* import_options,
+    rocksdb_export_import_files_metadata_t* metadata, char** errptr);
+
 extern ROCKSDB_LIBRARY_API rocksdb_column_family_handle_t*
 rocksdb_create_column_family_with_ttl(
    rocksdb_t* db, const rocksdb_options_t* column_family_options,
@@ -2681,12 +2699,16 @@ rocksdb_fifo_compaction_options_get_max_table_files_size(
 extern ROCKSDB_LIBRARY_API void rocksdb_fifo_compaction_options_destroy(
    rocksdb_fifo_compaction_options_t* fifo_opts);

+extern ROCKSDB_LIBRARY_API rocksdb_livefiles_t* rocksdb_livefiles_create(void);
+
 extern ROCKSDB_LIBRARY_API int rocksdb_livefiles_count(
    const rocksdb_livefiles_t*);
 extern ROCKSDB_LIBRARY_API const char* rocksdb_livefiles_column_family_name(
    const rocksdb_livefiles_t*, int index);
 extern ROCKSDB_LIBRARY_API const char* rocksdb_livefiles_name(
    const rocksdb_livefiles_t*, int index);
+extern ROCKSDB_LIBRARY_API const char* rocksdb_livefiles_directory(
+    const rocksdb_livefiles_t*, int index);
 extern ROCKSDB_LIBRARY_API int rocksdb_livefiles_level(
    const rocksdb_livefiles_t*, int index);
 extern ROCKSDB_LIBRARY_API size_t
@@ -2696,12 +2718,44 @@ extern ROCKSDB_LIBRARY_API const char* rocksdb_livefiles_smallestkey(
 extern ROCKSDB_LIBRARY_API const char* rocksdb_livefiles_largestkey(
    const rocksdb_livefiles_t*, int index, size_t* size);
 extern ROCKSDB_LIBRARY_API uint64_t
+rocksdb_livefiles_smallest_seqno(const rocksdb_livefiles_t*, int index);
+extern ROCKSDB_LIBRARY_API uint64_t
+rocksdb_livefiles_largest_seqno(const rocksdb_livefiles_t*, int index);
+extern ROCKSDB_LIBRARY_API uint64_t
 rocksdb_livefiles_entries(const rocksdb_livefiles_t*, int index);
 extern ROCKSDB_LIBRARY_API uint64_t
 rocksdb_livefiles_deletions(const rocksdb_livefiles_t*, int index);
 extern ROCKSDB_LIBRARY_API void rocksdb_livefiles_destroy(
    const rocksdb_livefiles_t*);

+extern ROCKSDB_LIBRARY_API rocksdb_livefile_t* rocksdb_livefile_create(void);
+extern ROCKSDB_LIBRARY_API void rocksdb_livefile_set_column_family_name(
+    rocksdb_livefile_t*, const char*);
+extern ROCKSDB_LIBRARY_API void rocksdb_livefile_set_level(rocksdb_livefile_t*,
+                                                           int);
+extern ROCKSDB_LIBRARY_API void rocksdb_livefile_set_name(rocksdb_livefile_t*,
+                                                          const char*);
+extern ROCKSDB_LIBRARY_API void rocksdb_livefile_set_directory(
+    rocksdb_livefile_t*, const char*);
+extern ROCKSDB_LIBRARY_API void rocksdb_livefile_set_size(rocksdb_livefile_t*,
+                                                          size_t);
+extern ROCKSDB_LIBRARY_API void rocksdb_livefile_set_smallest_key(
+    rocksdb_livefile_t*, const char*, size_t);
+extern ROCKSDB_LIBRARY_API void rocksdb_livefile_set_largest_key(
+    rocksdb_livefile_t*, const char*, size_t);
+extern ROCKSDB_LIBRARY_API void rocksdb_livefile_set_smallest_seqno(
+    rocksdb_livefile_t*, uint64_t);
+extern ROCKSDB_LIBRARY_API void rocksdb_livefile_set_largest_seqno(
+    rocksdb_livefile_t*, uint64_t);
+extern ROCKSDB_LIBRARY_API void rocksdb_livefile_set_num_entries(
+    rocksdb_livefile_t*, uint64_t);
+extern ROCKSDB_LIBRARY_API void rocksdb_livefile_set_num_deletions(
+    rocksdb_livefile_t*, uint64_t);
+extern ROCKSDB_LIBRARY_API void rocksdb_livefile_destroy(rocksdb_livefile_t*);
+
+extern ROCKSDB_LIBRARY_API void rocksdb_livefiles_add(rocksdb_livefiles_t*,
+                                                      rocksdb_livefile_t*);
+
 /* Utility Helpers */

 extern ROCKSDB_LIBRARY_API void rocksdb_get_options_from_string(
@@ -2722,6 +2776,37 @@ extern ROCKSDB_LIBRARY_API void rocksdb_delete_file_in_range_cf(
 extern ROCKSDB_LIBRARY_API rocksdb_column_family_metadata_t*
 rocksdb_get_column_family_metadata(rocksdb_t* db);

+extern ROCKSDB_LIBRARY_API rocksdb_import_column_family_options_t*
+rocksdb_import_column_family_options_create(void);
+
+extern ROCKSDB_LIBRARY_API void
+rocksdb_import_column_family_options_set_move_files(
+    rocksdb_import_column_family_options_t*, unsigned char);
+
+extern ROCKSDB_LIBRARY_API void rocksdb_import_column_family_options_destroy(
+    rocksdb_import_column_family_options_t*);
+
+extern ROCKSDB_LIBRARY_API rocksdb_export_import_files_metadata_t*
+rocksdb_export_import_files_metadata_create(void);
+
+extern ROCKSDB_LIBRARY_API char*
+rocksdb_export_import_files_metadata_get_db_comparator_name(
+    rocksdb_export_import_files_metadata_t*);
+
+extern ROCKSDB_LIBRARY_API void
+rocksdb_export_import_files_metadata_set_db_comparator_name(
+    rocksdb_export_import_files_metadata_t*, const char*);
+
+extern ROCKSDB_LIBRARY_API rocksdb_livefiles_t*
+rocksdb_export_import_files_metadata_get_files(
+    rocksdb_export_import_files_metadata_t*);
+
+extern ROCKSDB_LIBRARY_API void rocksdb_export_import_files_metadata_set_files(
+    rocksdb_export_import_files_metadata_t*, rocksdb_livefiles_t*);
+
+extern ROCKSDB_LIBRARY_API void rocksdb_export_import_files_metadata_destroy(
+    rocksdb_export_import_files_metadata_t*);
+
 /**
 * Returns the rocksdb_column_family_metadata_t of the specified
 * column family.
@@ -210,7 +210,15 @@ struct ShardedCacheOptions {
 // shard has its own LRU list for evictions. Each shard also has a mutex for
 // exclusive access during operations; even read operations need exclusive
 // access in order to update the LRU list. Mutex contention is usually low
-// with enough shards.
+// with enough shards. However,
+// * For a single hot block, there will be mutex contention even for reads
+// regardless of the number of shards.
+// * LRUCaches in the size of MBs instead of GBs can have shards small enough
+// that there is a random probability of some modest number of large blocks
+// (especially non-partitioned filters) thrashing a single cache shard.
+//
+// HYPERCLOCKCACHE IS NOW GENERALLY RECOMMENDED OVER LRUCACHE. See
+// HyperClockCacheOptions below.
 struct LRUCacheOptions : public ShardedCacheOptions {
  // Ratio of cache reserved for high-priority and low-priority entries,
  // respectively. (See Cache::Priority below more information on the levels.)
@@ -371,64 +379,50 @@ inline std::shared_ptr<SecondaryCache> NewCompressedSecondaryCache(
  return opts.MakeSharedSecondaryCache();
 }

-// HyperClockCache - A lock-free Cache alternative for RocksDB block cache
-// that offers much improved CPU efficiency vs. LRUCache under high parallel
-// load or high contention, with some caveats:
+// HyperClockCache (also known as HCC) - A lock-free Cache alternative for
+// RocksDB block cache that offers much improved CPU efficiency vs. LRUCache
+// under high parallel load or high contention. Additionally, HCC only uses
+// sharding for a modest performance boost, so can use much larger cache shards
+// than LRUCache, dramatically reducing the risk of thrashing in configurations
+// or work loads with some large blocks.
+//
+// HYPERCLOCKCACHE IS NOW GENERALLY RECOMMENDED OVER LRUCACHE
+//
+// Some caveats:
 // * Not a general Cache implementation: can only be used for
 // BlockBasedTableOptions::block_cache, which RocksDB uses in a way that is
 // compatible with HyperClockCache.
-// * Requires an extra tuning parameter: see estimated_entry_charge below.
-// Similarly, substantially changing the capacity with SetCapacity could
-// harm efficiency. -> EXPERIMENTAL: the tuning parameter can be set to 0
-// to find the appropriate balance automatically.
 // * Cache priorities are less aggressively enforced, which could cause
 // cache dilution from long range scans (unless they use fill_cache=false).
+// * In some configurations, depends on anonymous mmap support, available in
+// Linux, Windows and more.
+// * May have slightly lower (or slightly higher) cache hit rate vs. LRUCache,
+// because of the bounded counting-CLOCK eviction algorithm.
 //
 // See internal cache/clock_cache.h for full description.
 struct HyperClockCacheOptions : public ShardedCacheOptions {
-  // The estimated average `charge` associated with cache entries.
+  // OPTIONAL: The estimated average `charge` associated with cache entries.
  //
-  // EXPERIMENTAL: the field can be set to 0 to size the table dynamically
-  // and automatically. See also min_avg_entry_charge. This feature requires
-  // platform support for lazy anonymous memory mappings (incl Linux, Windows).
-  // Performance is very similar to choosing the best configuration parameter.
+  // When not provided (== 0, recommended and default), an HCC variant with a
+  // dynamically-growing table and generally good performance is used. This
+  // variant depends on anonymous mmaps so might not be available on all
+  // platforms.
  //
-  // PRODUCTION-TESTED: This is a critical configuration parameter for good
-  // performance, because having a table size that is fixed at creation time
-  // greatly reduces the required synchronization between threads.
-  // * If the estimate is substantially too low (e.g. less than half the true
-  // average) then metadata space overhead with be substantially higher (e.g.
-  // 200 bytes per entry rather than 100). With kFullChargeCacheMetadata, this
-  // can slightly reduce cache hit rates, and slightly reduce access times due
-  // to the larger working memory size.
-  // * If the estimate is substantially too high (e.g. 25% higher than the true
-  // average) then there might not be sufficient slots in the hash table for
-  // both efficient operation and capacity utilization (hit rate). The hyper
-  // cache will evict entries to prevent load factors that could dramatically
-  // affect lookup times, instead letting the hit rate suffer by not utilizing
-  // the full capacity.
-  //
-  // A reasonable choice is the larger of block_size and metadata_block_size.
-  // When WriteBufferManager (and similar) charge memory usage to the block
-  // cache, this can lead to the same effect as estimate being too low, which
-  // is better than the opposite. Therefore, the general recommendation is to
-  // assume that other memory charged to block cache could be negligible, and
-  // ignore it in making the estimate.
-  //
-  // The best parameter choice based on a cache in use is given by
-  // GetUsage() / GetOccupancyCount(), ignoring metadata overheads such as
-  // with kDontChargeCacheMetadata. More precisely with
-  // kFullChargeCacheMetadata is (GetUsage() - 64 * GetTableAddressCount()) /
-  // GetOccupancyCount(). However, when the average value size might vary
-  // (e.g. balance between metadata and data blocks in cache), it is better
-  // to estimate toward the lower side than the higher side.
+  // If the average "charge" (uncompressed block size) of block cache entries
+  // is reasonably predicted and provided here, the most efficient variant of
+  // HCC is used. Performance is degraded if the prediction is inaccurate.
+  // Prediction could be difficult or impossible with cache-charging features
+  // such as WriteBufferManager. The best parameter choice based on a cache
+  // in use is roughly given by GetUsage() / GetOccupancyCount(), though it is
+  // better to estimate toward the lower side than the higher side when the
+  // ratio might vary.
  size_t estimated_entry_charge;

-  // EXPERIMENTAL: When estimated_entry_charge == 0, this parameter establishes
-  // a promised lower bound on the average charge of all entries in the table,
-  // which is roughly the average uncompressed SST block size of block cache
-  // entries, typically > 4KB. The default should generally suffice with almost
-  // no cost. (This option is ignored for estimated_entry_charge > 0.)
+  // When estimated_entry_charge == 0, this parameter establishes a promised
+  // lower bound on the average charge of all entries in the table, which is
+  // roughly the average uncompressed SST block size of block cache entries,
+  // typically > 4KB. The default should generally suffice with almost no cost.
+  // (This option is ignored for estimated_entry_charge > 0.)
  //
  // More detail: The table for indexing cache entries will grow automatically
  // as needed, but a hard upper bound on that size is needed at creation time.
@@ -478,8 +472,8 @@ struct HyperClockCacheOptions : public ShardedCacheOptions {
  // keep operations very fast.
  int eviction_effort_cap = 30;

-  HyperClockCacheOptions(
-      size_t _capacity, size_t _estimated_entry_charge,
+  explicit HyperClockCacheOptions(
+      size_t _capacity, size_t _estimated_entry_charge = 0,
      int _num_shard_bits = -1, bool _strict_capacity_limit = false,
      std::shared_ptr<MemoryAllocator> _memory_allocator = nullptr,
      CacheMetadataChargePolicy _metadata_charge_policy =
@@ -24,9 +24,10 @@ struct CompactionJobStats {
  // the elapsed CPU time of this compaction in microseconds.
  uint64_t cpu_micros = 0;

-  // Used internally indicating whether a subcompaction's
-  // `num_input_records` is accurate.
-  bool has_num_input_records = false;
+  // True if `num_input_records` is accurate across all subcompactions.
+  // See CompactionIterator::must_count_input_entries for some implementation
+  // details why `num_input_records` may not be accurate.
+  bool has_accurate_num_input_records = true;
  // the number of compaction input records.
  uint64_t num_input_records = 0;
  // the number of blobs read from blob files
@@ -226,11 +226,15 @@ struct CompressionOptions {
  // The training data will be used to generate a dictionary of max_dict_bytes.
  uint32_t zstd_max_train_bytes = 0;

-  // Number of threads for parallel compression.
-  // Parallel compression is enabled only if threads > 1.
-  // THE FEATURE IS STILL EXPERIMENTAL
+  // Number of threads for parallel compression for each running flush or
+  // compaction job. Parallel compression is enabled only if threads > 1. Not
+  // recommended for lightweight compression algorithms such as Snappy, LZ4, and
+  // obviously kNoCompression because there is unlikely to be a throughput gain.
  //
-  // This option is valid only when BlockBasedTable is used.
+  // This option is valid only when BlockBasedTable is used and is disabled
+  // (sanitized to 1) with any of these:
+  // * User-defined index (UserDefinedIndexFactory)
+  // * partition_filters == true && decouple_partitioned_filters == false
  //
  // When parallel compression is enabled, SST size file sizes might be
  // more inflated compared to the target size, because more data of unknown
@@ -309,9 +313,7 @@ struct CompressionOptions {
    max_compressed_bytes_per_kb = static_cast<int>(1024.0 / min_ratio + 0.5);
  }

-#if __cplusplus >= 202002L
  bool operator==(const CompressionOptions& rhs) const = default;
-#endif
 };

 // See advanced_compression.h
@@ -1118,11 +1118,11 @@ class DB {
  //    // Check ex.what()
  //  }
  virtual std::unique_ptr<MultiScan> NewMultiScan(
-      const ReadOptions& /*options*/, ColumnFamilyHandle* /*column_family*/,
+      const ReadOptions& /*options*/, ColumnFamilyHandle* column_family,
      const MultiScanArgs& /*scan_opts*/) {
    std::unique_ptr<Iterator> iter(NewErrorIterator(Status::NotSupported()));
-    std::unique_ptr<MultiScan> ms_iter =
-        std::make_unique<MultiScan>(std::move(iter));
+    std::unique_ptr<MultiScan> ms_iter = std::make_unique<MultiScan>(
+        column_family->GetComparator(), std::move(iter));
    return ms_iter;
  }

@@ -2247,12 +2247,9 @@ inline Status DB::GetApproximateSizes(ColumnFamilyHandle* column_family,
                                      uint64_t* sizes,
                                      SizeApproximationFlags include_flags) {
  SizeApproximationOptions options;
-  options.include_memtables =
-      ((include_flags & SizeApproximationFlags::INCLUDE_MEMTABLES) !=
-       SizeApproximationFlags::NONE);
-  options.include_files =
-      ((include_flags & SizeApproximationFlags::INCLUDE_FILES) !=
-       SizeApproximationFlags::NONE);
+  using enum SizeApproximationFlags;  // Require C++20 support
+  options.include_memtables = ((include_flags & INCLUDE_MEMTABLES) != NONE);
+  options.include_files = ((include_flags & INCLUDE_FILES) != NONE);
  return GetApproximateSizes(options, column_family, ranges, n, sizes);
 }

@@ -385,13 +385,6 @@ class Env : public Customizable {
    return Status::NotSupported("LinkFile is not supported for this Env");
  }

-  // Sync the file content to file system.
-  // This API is only used for testing.
-  // See FileSystem::SyncFile comment for details
-  virtual Status SyncFile(const std::string& /*fname*/,
-                          const EnvOptions& /*env_options*/,
-                          bool /*use_fsync*/);
-
  virtual Status NumFileLinks(const std::string& /*fname*/,
                              uint64_t* /*count*/) {
    return Status::NotSupported(
@@ -463,7 +456,141 @@ class Env : public Customizable {
    kGetEntity = 8,
    kMultiGetEntity = 9,
    kGetFileChecksumsFromCurrentManifest = 10,
-    kUnknown,  // Keep last for easy array of non-unknowns
+    // Enums after this, up to 0x7F, are reserved for future use for the public
+    // RocksDB API (i.e. they should be "non-custom" IO activities). Make sure
+    // to also update IOActivityToString when adding new values.
+
+    kCustomIOActivity80 = 0x80,
+    kFirstCustomIOActivity = kCustomIOActivity80,
+    kCustomIOActivity81 = 0x81,
+    kCustomIOActivity82 = 0x82,
+    kCustomIOActivity83 = 0x83,
+    kCustomIOActivity84 = 0x84,
+    kCustomIOActivity85 = 0x85,
+    kCustomIOActivity86 = 0x86,
+    kCustomIOActivity87 = 0x87,
+    kCustomIOActivity88 = 0x88,
+    kCustomIOActivity89 = 0x89,
+    kCustomIOActivity8A = 0x8A,
+    kCustomIOActivity8B = 0x8B,
+    kCustomIOActivity8C = 0x8C,
+    kCustomIOActivity8D = 0x8D,
+    kCustomIOActivity8E = 0x8E,
+    kCustomIOActivity8F = 0x8F,
+    kCustomIOActivity90 = 0x90,
+    kCustomIOActivity91 = 0x91,
+    kCustomIOActivity92 = 0x92,
+    kCustomIOActivity93 = 0x93,
+    kCustomIOActivity94 = 0x94,
+    kCustomIOActivity95 = 0x95,
+    kCustomIOActivity96 = 0x96,
+    kCustomIOActivity97 = 0x97,
+    kCustomIOActivity98 = 0x98,
+    kCustomIOActivity99 = 0x99,
+    kCustomIOActivity9A = 0x9A,
+    kCustomIOActivity9B = 0x9B,
+    kCustomIOActivity9C = 0x9C,
+    kCustomIOActivity9D = 0x9D,
+    kCustomIOActivity9E = 0x9E,
+    kCustomIOActivity9F = 0x9F,
+    kCustomIOActivityA0 = 0xA0,
+    kCustomIOActivityA1 = 0xA1,
+    kCustomIOActivityA2 = 0xA2,
+    kCustomIOActivityA3 = 0xA3,
+    kCustomIOActivityA4 = 0xA4,
+    kCustomIOActivityA5 = 0xA5,
+    kCustomIOActivityA6 = 0xA6,
+    kCustomIOActivityA7 = 0xA7,
+    kCustomIOActivityA8 = 0xA8,
+    kCustomIOActivityA9 = 0xA9,
+    kCustomIOActivityAA = 0xAA,
+    kCustomIOActivityAB = 0xAB,
+    kCustomIOActivityAC = 0xAC,
+    kCustomIOActivityAD = 0xAD,
+    kCustomIOActivityAE = 0xAE,
+    kCustomIOActivityAF = 0xAF,
+    kCustomIOActivityB0 = 0xB0,
+    kCustomIOActivityB1 = 0xB1,
+    kCustomIOActivityB2 = 0xB2,
+    kCustomIOActivityB3 = 0xB3,
+    kCustomIOActivityB4 = 0xB4,
+    kCustomIOActivityB5 = 0xB5,
+    kCustomIOActivityB6 = 0xB6,
+    kCustomIOActivityB7 = 0xB7,
+    kCustomIOActivityB8 = 0xB8,
+    kCustomIOActivityB9 = 0xB9,
+    kCustomIOActivityBA = 0xBA,
+    kCustomIOActivityBB = 0xBB,
+    kCustomIOActivityBC = 0xBC,
+    kCustomIOActivityBD = 0xBD,
+    kCustomIOActivityBE = 0xBE,
+    kCustomIOActivityBF = 0xBF,
+    kCustomIOActivityC0 = 0xC0,
+    kCustomIOActivityC1 = 0xC1,
+    kCustomIOActivityC2 = 0xC2,
+    kCustomIOActivityC3 = 0xC3,
+    kCustomIOActivityC4 = 0xC4,
+    kCustomIOActivityC5 = 0xC5,
+    kCustomIOActivityC6 = 0xC6,
+    kCustomIOActivityC7 = 0xC7,
+    kCustomIOActivityC8 = 0xC8,
+    kCustomIOActivityC9 = 0xC9,
+    kCustomIOActivityCA = 0xCA,
+    kCustomIOActivityCB = 0xCB,
+    kCustomIOActivityCC = 0xCC,
+    kCustomIOActivityCD = 0xCD,
+    kCustomIOActivityCE = 0xCE,
+    kCustomIOActivityCF = 0xCF,
+    kCustomIOActivityD0 = 0xD0,
+    kCustomIOActivityD1 = 0xD1,
+    kCustomIOActivityD2 = 0xD2,
+    kCustomIOActivityD3 = 0xD3,
+    kCustomIOActivityD4 = 0xD4,
+    kCustomIOActivityD5 = 0xD5,
+    kCustomIOActivityD6 = 0xD6,
+    kCustomIOActivityD7 = 0xD7,
+    kCustomIOActivityD8 = 0xD8,
+    kCustomIOActivityD9 = 0xD9,
+    kCustomIOActivityDA = 0xDA,
+    kCustomIOActivityDB = 0xDB,
+    kCustomIOActivityDC = 0xDC,
+    kCustomIOActivityDD = 0xDD,
+    kCustomIOActivityDE = 0xDE,
+    kCustomIOActivityDF = 0xDF,
+    kCustomIOActivityE0 = 0xE0,
+    kCustomIOActivityE1 = 0xE1,
+    kCustomIOActivityE2 = 0xE2,
+    kCustomIOActivityE3 = 0xE3,
+    kCustomIOActivityE4 = 0xE4,
+    kCustomIOActivityE5 = 0xE5,
+    kCustomIOActivityE6 = 0xE6,
+    kCustomIOActivityE7 = 0xE7,
+    kCustomIOActivityE8 = 0xE8,
+    kCustomIOActivityE9 = 0xE9,
+    kCustomIOActivityEA = 0xEA,
+    kCustomIOActivityEB = 0xEB,
+    kCustomIOActivityEC = 0xEC,
+    kCustomIOActivityED = 0xED,
+    kCustomIOActivityEE = 0xEE,
+    kCustomIOActivityEF = 0xEF,
+    kCustomIOActivityF0 = 0xF0,
+    kCustomIOActivityF1 = 0xF1,
+    kCustomIOActivityF2 = 0xF2,
+    kCustomIOActivityF3 = 0xF3,
+    kCustomIOActivityF4 = 0xF4,
+    kCustomIOActivityF5 = 0xF5,
+    kCustomIOActivityF6 = 0xF6,
+    kCustomIOActivityF7 = 0xF7,
+    kCustomIOActivityF8 = 0xF8,
+    kCustomIOActivityF9 = 0xF9,
+    kCustomIOActivityFA = 0xFA,
+    kCustomIOActivityFB = 0xFB,
+    kCustomIOActivityFC = 0xFC,
+    kCustomIOActivityFD = 0xFD,
+    kCustomIOActivityFE = 0xFE,
+    kLastCustomIOActivity = kCustomIOActivityFE,
+
+    kUnknown = 0xFF,  // Keep last as unknown
  };

  static std::string IOActivityToString(IOActivity activity);
@@ -1550,11 +1677,6 @@ class EnvWrapper : public Env {
    return target_.env->LinkFile(s, t);
  }

-  Status SyncFile(const std::string& fname, const EnvOptions& env_options,
-                  bool use_fsync) override {
-    return target_.env->SyncFile(fname, env_options, use_fsync);
-  }
-
  Status NumFileLinks(const std::string& fname, uint64_t* count) override {
    return target_.env->NumFileLinks(fname, count);
  }
@@ -24,6 +24,7 @@
 #include <functional>
 #include <limits>
 #include <memory>
+#include <shared_mutex>
 #include <sstream>
 #include <string>
 #include <unordered_map>
@@ -254,6 +255,10 @@ struct IODebugContext {
  // Arbitrary structure containing cost information about the IO request
  std::any cost_info;

+  // FileSystem implementations can use this mutex to synchronize concurrent
+  // reads/writes as needed (e.g. to update the counters or cost_info field)
+  std::shared_mutex mutex;
+
  IODebugContext() {}

  // Copy constructor
@@ -606,18 +611,6 @@ class FileSystem : public Customizable {
        "LinkFile is not supported for this FileSystem");
  }

-  // Sync the file content to file system.
-  // The default implementation would open, sync and close the file.
-  // This function could be overridden with no-op, if the file system
-  // automatically sync the data when file is closed.
-  // This is used when a user-provided file, probably unsynced, is pulled into a
-  // context where power-outage-proof persistence is required (e.g.
-  // IngestExternalFile without copy).
-  virtual IOStatus SyncFile(const std::string& fname,
-                            const FileOptions& file_options,
-                            const IOOptions& io_options, bool use_fsync,
-                            IODebugContext* dbg);
-
  virtual IOStatus NumFileLinks(const std::string& /*fname*/,
                                const IOOptions& /*options*/,
                                uint64_t* /*count*/, IODebugContext* /*dbg*/) {
@@ -1604,12 +1597,6 @@ class FileSystemWrapper : public FileSystem {
    return target_->LinkFile(s, t, options, dbg);
  }

-  IOStatus SyncFile(const std::string& fname, const FileOptions& file_options,
-                    const IOOptions& io_options, bool use_fsync,
-                    IODebugContext* dbg) override {
-    return target_->SyncFile(fname, file_options, io_options, use_fsync, dbg);
-  }
-
  IOStatus NumFileLinks(const std::string& fname, const IOOptions& options,
                        uint64_t* count, IODebugContext* dbg) override {
    return target_->NumFileLinks(fname, options, count, dbg);
@@ -32,22 +32,34 @@ struct FileIOByTemperature {
  uint64_t hot_file_bytes_read;
  // the number of bytes read to Temperature::kWarm file
  uint64_t warm_file_bytes_read;
+  // the number of bytes read to Temperature::kCool file
+  uint64_t cool_file_bytes_read;
  // the number of bytes read to Temperature::kCold file
  uint64_t cold_file_bytes_read;
+  // the number of bytes read to Temperature::kIce file
+  uint64_t ice_file_bytes_read;
  // total number of reads to Temperature::kHot file
  uint64_t hot_file_read_count;
  // total number of reads to Temperature::kWarm file
  uint64_t warm_file_read_count;
+  // total number of reads to Temperature::kCool file
+  uint64_t cool_file_read_count;
  // total number of reads to Temperature::kCold file
  uint64_t cold_file_read_count;
+  // total number of reads to Temperature::kIce file
+  uint64_t ice_file_read_count;
  // reset all the statistics to 0.
  void Reset() {
    hot_file_bytes_read = 0;
    warm_file_bytes_read = 0;
+    cool_file_bytes_read = 0;
    cold_file_bytes_read = 0;
+    ice_file_bytes_read = 0;
    hot_file_read_count = 0;
    warm_file_read_count = 0;
+    cool_file_read_count = 0;
    cold_file_read_count = 0;
+    ice_file_read_count = 0;
  }
 };

@@ -32,10 +32,18 @@ struct LDBOptions {

 class LDBTool {
 public:
+  // DEPRECATED because this function does not return, which can result in
+  // memory leaks being reported because of the default Options() etc. not being
+  // destroyed.
  void Run(
      int argc, char** argv, Options db_options = Options(),
      const LDBOptions& ldb_options = LDBOptions(),
      const std::vector<ColumnFamilyDescriptor>* column_families = nullptr);
+
+  int RunAndReturn(
+      int argc, char** argv, const Options& db_options = Options(),
+      const LDBOptions& ldb_options = LDBOptions(),
+      const std::vector<ColumnFamilyDescriptor>* column_families = nullptr);
 };

 }  // namespace ROCKSDB_NAMESPACE
@@ -38,6 +38,7 @@
 #include <stdint.h>
 #include <stdlib.h>

+#include <functional>
 #include <memory>
 #include <stdexcept>
 #include <unordered_set>
@@ -201,11 +202,12 @@ class MemTableRep {
                   bool (*callback_func)(void* arg, const char* entry));

  // Same as Get() but performs data integrity validation.
-  virtual Status GetAndValidate(const LookupKey& /* k */,
-                                void* /* callback_args */,
-                                bool (* /* callback_func */)(void* arg,
-                                                             const char* entry),
-                                bool /*allow_data_in_error*/) {
+  virtual Status GetAndValidate(
+      const LookupKey& /* k */, void* /* callback_args */,
+      bool (* /* callback_func */)(void* arg, const char* entry),
+      bool /* allow_data_in_error */, bool /* detect_key_out_of_order */,
+      const std::function<Status(const char*, bool)>&
+      /* key_validation_callback */) {
    return Status::NotSupported("GetAndValidate() not implemented.");
  }

@@ -276,9 +278,11 @@ class MemTableRep {
    // Seek and perform integrity validations on the skip list.
    // Iterator becomes invalid and Corruption is returned if a
    // corruption is found.
-    virtual Status SeekAndValidate(const Slice& /* internal_key */,
-                                   const char* /* memtable_key */,
-                                   bool /* allow_data_in_errors */) {
+    virtual Status SeekAndValidate(
+        const Slice& /* internal_key */, const char* /* memtable_key */,
+        bool /* allow_data_in_errors */, bool /* detect_key_out_of_order */,
+        const std::function<Status(const char*, bool)>&
+        /* key_validation_callback */) {
      return Status::NotSupported("SeekAndValidate() not implemented.");
    }

@@ -97,7 +97,13 @@ class Scan {

    ScanIterator() : db_iter_(nullptr), valid_(false) {}

-    ~ScanIterator() { assert(status_.ok()); }
+    ~ScanIterator() {
+      if (!status_.ok()) {
+        fprintf(stderr, "ScanIterator status: %s\n",
+                status_.ToString().c_str());
+        assert(false);
+      }
+    }

    ScanIterator& operator++() {
      if (!valid_) {
@@ -155,8 +161,9 @@ class MultiScan {
  MultiScan(const ReadOptions& read_options, const MultiScanArgs& scan_opts,
            DB* db, ColumnFamilyHandle* cfh);

-  explicit MultiScan(std::unique_ptr<Iterator>&& db_iter)
-      : db_iter_(std::move(db_iter)) {}
+  explicit MultiScan(const Comparator* comp,
+                     std::unique_ptr<Iterator>&& db_iter)
+      : scan_opts_(comp), db_iter_(std::move(db_iter)) {}

  class MultiScanIterator {
   public:
@@ -184,6 +191,10 @@ class MultiScan {
      if (scan_opts_.empty()) {
        throw std::logic_error("Zero scans in multi-scan");
      }
+      status_ = db_iter_->status();
+      if (!status_.ok()) {
+        throw MultiScanException(status_);
+      }
      db_iter_->Seek(*scan_opts_[idx_].range.start);
      status_ = db_iter_->status();
      if (!status_.ok()) {
@@ -623,6 +623,7 @@ struct DBOptions {
  // checking for corruption, including
  // * paranoid_file_checks
  // * paranoid_memory_checks
+  // * memtable_veirfy_per_key_checksum_on_seek
  // * DB::VerifyChecksum()
  //
  // Default: true
@@ -1782,17 +1783,20 @@ struct ScanOptions {
 class MultiScanArgs {
 public:
  // Constructor that takes a comparator
-  explicit MultiScanArgs(const Comparator* comparator = BytewiseComparator())
-      : comp_(comparator) {}
+  explicit MultiScanArgs(const Comparator* comparator) : comp_(comparator) {}

  // Copy Constructor
  MultiScanArgs(const MultiScanArgs& other) {
    comp_ = other.comp_;
    original_ranges_ = other.original_ranges_;
    io_coalesce_threshold = other.io_coalesce_threshold;
+    max_prefetch_size = other.max_prefetch_size;
+    use_async_io = other.use_async_io;
  }
  MultiScanArgs(MultiScanArgs&& other) noexcept
      : io_coalesce_threshold(other.io_coalesce_threshold),
+        max_prefetch_size(other.max_prefetch_size),
+        use_async_io(other.use_async_io),
        comp_(other.comp_),
        original_ranges_(std::move(other.original_ranges_)) {}

@@ -1800,6 +1804,8 @@ class MultiScanArgs {
    comp_ = other.comp_;
    original_ranges_ = other.original_ranges_;
    io_coalesce_threshold = other.io_coalesce_threshold;
+    max_prefetch_size = other.max_prefetch_size;
+    use_async_io = other.use_async_io;
    return *this;
  }

@@ -1808,6 +1814,8 @@ class MultiScanArgs {
      comp_ = other.comp_;
      original_ranges_ = std::move(other.original_ranges_);
      io_coalesce_threshold = other.io_coalesce_threshold;
+      max_prefetch_size = other.max_prefetch_size;
+      use_async_io = other.use_async_io;
    }
    return *this;
  }
@@ -1840,15 +1848,42 @@ class MultiScanArgs {
  operator std::vector<ScanOptions>*() { return &original_ranges_; }

  operator const std::vector<ScanOptions>*() const { return &original_ranges_; }
-  // Destructor
+
  ~MultiScanArgs() {}

  const std::vector<ScanOptions>& GetScanRanges() const {
    return original_ranges_;
  }

+  const Comparator* GetComparator() const { return comp_; }
+
+  // Copies the configurations (excluding actual scan ranges) from another
+  // MultiScanArgs.
+  void CopyConfigFrom(const MultiScanArgs& other) {
+    io_coalesce_threshold = other.io_coalesce_threshold;
+    max_prefetch_size = other.max_prefetch_size;
+    use_async_io = other.use_async_io;
+  }
+
  uint64_t io_coalesce_threshold = 16 << 10;  // 16KB by default

+  // Maximum size (in bytes) for the data blocks loaded by a MultiScan.
+  // This limits the amount of I/O and memory usage by pinned data blocks.
+  //
+  // When set to 0 (the default), there is no limit. When the limit is reached,
+  // the iterator will start returning Status::PrefetchLimitReached().
+  //
+  // Note that prefetching happens only once in Prepare(), which is different
+  // from ReadOptions::readahead_size, which applies any time the iterator does
+  // I/O.
+  // Note that this limit is per file and applies to compressed block size.
+  uint64_t max_prefetch_size = 0;
+
+  // Enable async I/O for multi-scan operations
+  // When true, BlockBasedTableIterator will use ReadAsync() for reading blocks
+  // When false, it will use synchronous MultiRead().
+  bool use_async_io = false;
+
 private:
  // The comparator used for ordering ranges
  const Comparator* comp_;
@@ -0,0 +1,14 @@
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
+//
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include "rocksdb/rocksdb_namespace.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+int point_lock_bench_tool(int argc, char** argv);
+}  // namespace ROCKSDB_NAMESPACE
@@ -24,7 +24,7 @@
 #include <cstdio>
 #include <cstring>
 #include <string>
-#include <string_view>  // RocksDB now requires C++17 support
+#include <string_view>

 #include "rocksdb/cleanable.h"

@@ -443,10 +443,14 @@ enum Tickers : uint32_t {
  // Tiered storage related statistics
  HOT_FILE_READ_BYTES,
  WARM_FILE_READ_BYTES,
+  COOL_FILE_READ_BYTES,
  COLD_FILE_READ_BYTES,
+  ICE_FILE_READ_BYTES,
  HOT_FILE_READ_COUNT,
  WARM_FILE_READ_COUNT,
+  COOL_FILE_READ_COUNT,
  COLD_FILE_READ_COUNT,
+  ICE_FILE_READ_COUNT,

  // Last level and non-last level read statistics
  LAST_LEVEL_READ_BYTES,
@@ -542,6 +546,9 @@ enum Tickers : uint32_t {
  // TransactionOptions::large_txn_commit_optimize_threshold.
  NUMBER_WBWI_INGEST,

+  // Failure to load the UDI during SST table open
+  SST_USER_DEFINED_INDEX_LOAD_FAIL_COUNT,
+
  TICKER_ENUM_MAX
 };

@@ -115,6 +115,8 @@ class Status {
    kIOFenced = 14,
    kMergeOperatorFailed = 15,
    kMergeOperandThresholdExceeded = 16,
+    kPrefetchLimitReached = 17,
+    kNotExpectedCodePath = 18,
    kMaxSubCode
  };

@@ -318,12 +320,19 @@ class Status {

  static Status LockLimit() { return Status(kAborted, kLockLimit); }

+  static Status PrefetchLimitReached() {
+    return Status(kIncomplete, kPrefetchLimitReached);
+  }
+
  // Returns true iff the status indicates success.
  bool ok() const {
    MarkChecked();
    return code() == kOk;
  }

+  // Assert the status is OK in debug mode
+  void AssertOK() const { assert(ok()); }
+
  // Returns true iff the status indicates success *with* something
  // overwritten
  bool IsOkOverwritten() const {
@@ -486,6 +495,13 @@ class Status {
    return (code() == kIOError) && (subcode() == kIOFenced);
  }

+  // Returns true iff the status indicates prefetch limit reached during
+  // MultiScan.
+  bool IsPrefetchLimitReached() const {
+    MarkChecked();
+    return (code() == kIncomplete) && (subcode() == kPrefetchLimitReached);
+  }
+
  // Return a string representation of this status suitable for printing.
  // Returns the string "OK" for success.
  std::string ToString() const;
@@ -442,6 +442,10 @@ struct BlockBasedTableOptions {
  //
  // decouple_partitioned_filters = true is the new default. This option is now
  // DEPRECATED and might be ignored and/or removed in a future release.
+  //
+  // NOTE: decouple_partitioned_filters = false with partition_filters = true
+  // disables parallel compression (CompressionOptions::parallel_threads
+  // sanitized to 1).
  bool decouple_partitioned_filters = true;

  // Option to generate Bloom/Ribbon filters that minimize memory
@@ -500,8 +504,17 @@ struct BlockBasedTableOptions {
  // If non-nullptr, use the specified factory to build user-defined index.
  // This allows users to define their own index format and build the index
  // during table building.
+  //
+  // NOTE: UserDefinedIndexFactory currently disables parallel compression
+  // (CompressionOptions::parallel_threads sanitized to 1).
  std::shared_ptr<UserDefinedIndexFactory> user_defined_index_factory = nullptr;

+  // EXPERIMENTAL
+  //
+  // Return an error Status if a user_defined_index_factory is configured,
+  // but there's no corresponding UDI block in the SST file being opened.
+  bool fail_if_no_udi_on_open = false;
+
  // If true, place whole keys in the filter (not just prefixes).
  // This must generally be true for gets to be efficient.
  bool whole_key_filtering = true;
@@ -76,6 +76,7 @@ struct TablePropertiesNames {
  static const std::string kTailStartOffset;
  static const std::string kUserDefinedTimestampsPersisted;
  static const std::string kKeyLargestSeqno;
+  static const std::string kKeySmallestSeqno;
 };

 // `TablePropertiesCollector` provides the mechanism for users to collect
@@ -220,6 +221,8 @@ struct TableProperties {
  uint64_t orig_file_number = 0;
  // the total size of all data blocks.
  uint64_t data_size = 0;
+  // the total uncompressed size of all data blocks (since RocksDB 10.7)
+  uint64_t uncompressed_data_size = 0;
  // the size of index block.
  uint64_t index_size = 0;
  // Total number of index partitions if kTwoLevelIndexSearch is used
@@ -309,6 +312,14 @@ struct TableProperties {

  bool HasKeyLargestSeqno() const { return key_largest_seqno != UINT64_MAX; }

+  // The smallest sequence number of keys in this file.
+  // UINT64_MAX means unknown.
+  // Only written to properties block if known (should be known unless the
+  // table is empty).
+  uint64_t key_smallest_seqno = UINT64_MAX;
+
+  bool HasKeySmallestSeqno() const { return key_smallest_seqno != UINT64_MAX; }
+
  // DB identity
  // db_id is an identifier generated the first time the DB is created
  // If DB identity is unset or unassigned, `db_id` will be an empty string.
@@ -118,7 +118,11 @@ enum class Temperature : uint8_t {
  kUnknown = 0,
  kHot = 0x04,
  kWarm = 0x08,
+  kCool = 0x0A,
  kCold = 0x0C,
+  kIce = 0x10,
+  // XXX: this is mis-named. It is instead an invalid temperature beyond the
+  // rest
  kLastTemperature,
 };

@@ -144,9 +144,7 @@ class CompactionOptionsUniversal {
        incremental(false),
        reduce_file_locking(false) {}

-#if __cplusplus >= 202002L
  bool operator==(const CompactionOptionsUniversal& rhs) const = default;
-#endif
 };

 }  // namespace ROCKSDB_NAMESPACE
@@ -30,8 +30,7 @@ inline const std::string kUserDefinedIndexPrefix =
 //
 // This is currently supported only for a restricted set of use cases. The
 // CF must be ingest only, and only files containing Puts generated by
-// SstFileWriter are supported. The user_comparator used for the CF must
-// be BytewiseComparator.
+// SstFileWriter are supported.

 // The interface for building user-defined index.
 class UserDefinedIndexBuilder {
@@ -145,6 +144,11 @@ class UserDefinedIndexReader {
  virtual size_t ApproximateMemoryUsage() const = 0;
 };

+// Options for user defined index
+struct UserDefinedIndexOption {
+  const Comparator* comparator = BytewiseComparator();
+};
+
 // Factory for creating user-defined index builders.
 class UserDefinedIndexFactory : public Customizable {
 public:
@@ -163,6 +167,21 @@ class UserDefinedIndexFactory : public Customizable {
  // block
  virtual std::unique_ptr<UserDefinedIndexReader> NewReader(
      Slice& index_block) const = 0;
+
+  // New API for allowing customized comparator
+  virtual Status NewBuilder(
+      const UserDefinedIndexOption& /*option*/,
+      std::unique_ptr<UserDefinedIndexBuilder>& builder) const {
+    builder.reset(NewBuilder());
+    return Status::OK();
+  };
+
+  virtual Status NewReader(
+      const UserDefinedIndexOption& /*option*/, Slice& index_block,
+      std::unique_ptr<UserDefinedIndexReader>& reader) const {
+    reader = NewReader(index_block);
+    return Status::OK();
+  };
 };

 }  // namespace ROCKSDB_NAMESPACE
@@ -90,7 +90,7 @@ class CacheDumper {
 public:
  virtual ~CacheDumper() = default;
  // Only dump the blocks in the block cache that belong to the DBs in this list
-  virtual Status SetDumpFilter(std::vector<DB*> db_list) {
+  virtual Status SetDumpFilter(const std::vector<DB*>& db_list) {
    (void)db_list;
    return Status::NotSupported("SetDumpFilter is not supported");
  }
@@ -653,7 +653,12 @@ class Transaction {
  // Change the value of TransactionOptions.lock_timeout (in milliseconds) for
  // this transaction.
  // Has no effect on OptimisticTransactions.
-  virtual void SetLockTimeout(int64_t timeout) = 0;
+  virtual void SetLockTimeout(int64_t timeout_ms) = 0;
+
+  // Change the value of deadlock_timeout (in milliseconds) for this
+  // transaction.
+  // Has no effect on OptimisticTransactions.
+  virtual void SetDeadlockTimeout(int64_t timeout_ms) = 0;

  // Return the WriteOptions that will be used during Commit()
  virtual WriteOptions* GetWriteOptions() = 0;
@@ -217,6 +217,11 @@ struct TransactionDBOptions {
  // Other value means the user provides a custom lock manager.
  std::shared_ptr<LockManagerHandle> lock_mgr_handle;

+  // EXPERIMENTAL
+  //
+  // Flag to enable/disable the per key point lock manager.
+  bool use_per_key_point_lock_mgr = false;
+
  // If true, the TransactionDB implementation might skip concurrency control
  // unless it is overridden by TransactionOptions or
  // TransactionDBWriteOptimizations. This can be used in conjunction with
@@ -319,6 +324,22 @@ struct TransactionOptions {
  // If negative, TransactionDBOptions::transaction_lock_timeout will be used.
  int64_t lock_timeout = -1;

+  // Timeout in microseconds before perform dead lock detection.
+  // If 0, deadlock detection will be performed immediately.
+  //
+  // To optimize performance, this parameter could be tuned.
+  //
+  // When deadlock happens very frequently, deadlock timeout should be set to 0,
+  // so deadlock will be detected immediately.
+  //
+  // When deadlock happen very rarely, this timeout could be turned to be
+  // slightly longer than the typical transaction execution time, so that
+  // transaction will be waked up to take the lock before this timeout, which
+  // will allow the transaction to save the CPU time on deadlock detection.
+  //
+  // Deadlock timeout is always smaller than lock_timeout.
+  int64_t deadlock_timeout_us = 500;
+
  // Expiration duration in milliseconds.  If non-negative, transactions that
  // last longer than this many milliseconds will fail to commit.  If not set,
  // a forgotten transaction that is never committed, rolled back, or deleted
@@ -12,8 +12,8 @@
 // NOTE: in 'main' development branch, this should be the *next*
 // minor or major version number planned for release.
 #define ROCKSDB_MAJOR 10
-#define ROCKSDB_MINOR 6
-#define ROCKSDB_PATCH 0
+#define ROCKSDB_MINOR 7
+#define ROCKSDB_PATCH 6

 // Do not use these. We made the mistake of declaring macros starting with
 // double underscore. Now we have to live with our choice. We'll deprecate these
@@ -5275,6 +5275,18 @@ class TickerTypeJni {
        return -0x57;
      case ROCKSDB_NAMESPACE::Tickers::FIFO_CHANGE_TEMPERATURE_COMPACTIONS:
        return -0x58;
+      case ROCKSDB_NAMESPACE::Tickers::ICE_FILE_READ_BYTES:
+        return -0x59;
+      case ROCKSDB_NAMESPACE::Tickers::ICE_FILE_READ_COUNT:
+        return -0x5A;
+      case ROCKSDB_NAMESPACE::Tickers::COOL_FILE_READ_BYTES:
+        return -0x5B;
+      case ROCKSDB_NAMESPACE::Tickers::COOL_FILE_READ_COUNT:
+        return -0x5C;
+      case ROCKSDB_NAMESPACE::Tickers::NUMBER_WBWI_INGEST:
+        return -0x5D;
+      case ROCKSDB_NAMESPACE::Tickers::SST_USER_DEFINED_INDEX_LOAD_FAIL_COUNT:
+        return -0x5E;
      case ROCKSDB_NAMESPACE::Tickers::TICKER_ENUM_MAX:
        // -0x54 is the max value at this time. Since these values are exposed
        // directly to Java clients, we'll keep the value the same till the next
@@ -5739,6 +5751,19 @@ class TickerTypeJni {
            FILE_READ_CORRUPTION_RETRY_SUCCESS_COUNT;
      case -0x58:
        return ROCKSDB_NAMESPACE::Tickers::FIFO_CHANGE_TEMPERATURE_COMPACTIONS;
+      case -0x59:
+        return ROCKSDB_NAMESPACE::Tickers::ICE_FILE_READ_BYTES;
+      case -0x5A:
+        return ROCKSDB_NAMESPACE::Tickers::ICE_FILE_READ_COUNT;
+      case -0x5B:
+        return ROCKSDB_NAMESPACE::Tickers::COOL_FILE_READ_BYTES;
+      case -0x5C:
+        return ROCKSDB_NAMESPACE::Tickers::COOL_FILE_READ_COUNT;
+      case -0x5D:
+        return ROCKSDB_NAMESPACE::Tickers::NUMBER_WBWI_INGEST;
+      case -0x5E:
+        return ROCKSDB_NAMESPACE::Tickers::
+            SST_USER_DEFINED_INDEX_LOAD_FAIL_COUNT;
      case -0x54:
        // -0x54 is the max value at this time. Since these values are exposed
        // directly to Java clients, we'll keep the value the same till the next