Bump patch version to fix a mistake in release tagging

Update version to 9.11.1
Bugfix: Ensure statuses are initialized with OK() in SSTFileReader::MultiGet (#13411 )
2026-11-07 03:22:34 -05:00 · 2025-03-29 14:42:01 -07:00 · 2025-02-19 20:03:01 -08:00 · 2025-02-19 20:01:41 -08:00 · 2025-02-19 20:01:22 -08:00 · 2025-02-19 20:01:13 -08:00
374 changed files with 6876 additions and 4208 deletions
@@ -1,905 +0,0 @@
-version: 2.1
-
-orbs:
-  win: circleci/windows@5.0.0
-
-commands:
-  install-cmake-on-macos:
-    steps:
-      - run:
-          name: Install cmake on macos
-          command: |
-            HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake
-
-  install-jdk8-on-macos:
-    steps:
-      - run:
-          name: Install JDK 8 on macos
-          command: |
-            HOMEBREW_NO_AUTO_UPDATE=1 brew tap bell-sw/liberica
-            HOMEBREW_NO_AUTO_UPDATE=1 brew install --cask liberica-jdk8
-
-  increase-max-open-files-on-macos:
-    steps:
-      - run:
-          name: Increase max open files
-          command: |
-            sudo sysctl -w kern.maxfiles=1048576
-            sudo sysctl -w kern.maxfilesperproc=1048576
-            sudo launchctl limit maxfiles 1048576
-
-  pre-steps:
-    steps:
-      - checkout
-      - run:
-          name: Setup Environment Variables
-          command: |
-            echo "export GTEST_THROW_ON_FAILURE=0" >> $BASH_ENV
-            echo "export GTEST_OUTPUT=\"xml:/tmp/test-results/\"" >> $BASH_ENV
-            echo "export SKIP_FORMAT_BUCK_CHECKS=1" >> $BASH_ENV
-            echo "export GTEST_COLOR=1" >> $BASH_ENV
-            echo "export CTEST_OUTPUT_ON_FAILURE=1" >> $BASH_ENV
-            echo "export CTEST_TEST_TIMEOUT=300" >> $BASH_ENV
-            echo "export ZLIB_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zlib" >> $BASH_ENV
-            echo "export BZIP2_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/bzip2" >> $BASH_ENV
-            echo "export SNAPPY_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/snappy" >> $BASH_ENV
-            echo "export LZ4_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/lz4" >> $BASH_ENV
-            echo "export ZSTD_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zstd" >> $BASH_ENV
-
-  windows-build-steps:
-    steps:
-      - checkout
-      - run:
-          name: "Install thirdparty dependencies"
-          command: |
-            echo "Installing CMake..."
-            choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' -y
-            choco install liberica8jdk -y
-            mkdir $Env:THIRDPARTY_HOME
-            cd $Env:THIRDPARTY_HOME
-            echo "Building Snappy dependency..."
-            curl https://github.com/google/snappy/archive/refs/tags/1.1.8.zip -O snappy-1.1.8.zip
-            unzip -q snappy-1.1.8.zip
-            cd snappy-1.1.8
-            mkdir build
-            cd build
-            & $Env:CMAKE_BIN -G "$Env:CMAKE_GENERATOR" ..
-            msbuild.exe Snappy.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
-      - run:
-          name: "Build RocksDB"
-          command: |
-            $env:Path = $env:JAVA_HOME + ";" + $env:Path
-            mkdir build
-            cd build
-            & $Env:CMAKE_BIN -G "$Env:CMAKE_GENERATOR" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE="$Env:CMAKE_PORTABLE" -DSNAPPY=1 -DJNI=1 ..
-            cd ..
-            echo "Building with VS version: $Env:CMAKE_GENERATOR"
-            msbuild.exe build/rocksdb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
-      - run:
-          name: "Test RocksDB"
-          shell: powershell.exe
-          command: |
-            build_tools\run_ci_db_test.ps1 -SuiteRun arena_test,db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16
-      - run:
-          name: "Test RocksJava"
-          command: |
-            cd build\java
-            & $Env:CTEST_BIN -C Debug -j 16
-  pre-steps-macos:
-      steps:
-        - pre-steps
-
-  post-steps:
-    steps:
-      - store_test_results: # store test result if there's any
-          path: /tmp/test-results
-      - store_artifacts: # store LOG for debugging if there's any
-          path: LOG
-      - run: # on fail, compress Test Logs for diagnosing the issue
-          name: Compress Test Logs
-          command: tar -cvzf t.tar.gz t
-          when: on_fail
-      - store_artifacts: # on fail, store Test Logs for diagnosing the issue
-          path: t.tar.gz
-          destination: test_logs
-          when: on_fail
-      - run: # store core dumps if there's any
-          command: |
-            mkdir -p /tmp/core_dumps
-            cp core.* /tmp/core_dumps
-          when: on_fail
-      - store_artifacts:
-          path: /tmp/core_dumps
-          when: on_fail
-
-  post-pmd-steps:
-    steps:
-      - store_artifacts:
-          path: /home/circleci/project/java/target/pmd.xml
-          when: on_fail
-      - store_artifacts:
-          path: /home/circleci/project/java/target/site
-          when: on_fail
-
-  upgrade-cmake:
-    steps:
-      - run:
-          name: Upgrade cmake
-          command: |
-            sudo apt remove --purge cmake
-            sudo snap install cmake --classic
-
-  install-gflags:
-    steps:
-      - run:
-          name: Install gflags
-          command: |
-            sudo apt-get update -y && sudo apt-get install -y libgflags-dev
-
-  install-gflags-on-macos:
-    steps:
-      - run:
-          name: Install gflags on macos
-          command: |
-            HOMEBREW_NO_AUTO_UPDATE=1 brew install gflags
-
-  install-maven:
-    steps:
-      - run:
-          name: Install Maven
-          command: |
-            wget --no-check-certificate https://dlcdn.apache.org/maven/maven-3/3.9.6/binaries/apache-maven-3.9.6-bin.tar.gz
-            tar zxf apache-maven-3.9.6-bin.tar.gz
-            echo "export M2_HOME=$(pwd)/apache-maven-3.9.6" >> $BASH_ENV
-            echo 'export PATH=$M2_HOME/bin:$PATH' >> $BASH_ENV
-
-  setup-folly:
-    steps:
-      - run:
-          name: Checkout folly sources
-          command: |
-            make checkout_folly
-
-  build-folly:
-    steps:
-      - run:
-          name: Build folly and dependencies
-          command: |
-            make build_folly
-
-  build-for-benchmarks:
-    steps:
-      - pre-steps
-      - run:
-          name: "Linux build for benchmarks"
-          command: #sized for the resource-class rocksdb-benchmark-sys1
-            make V=1 J=8 -j8 release
-
-  perform-benchmarks:
-    steps:
-      - run:
-          name: "Test low-variance benchmarks"
-          command: ./tools/benchmark_ci.py --db_dir /tmp/rocksdb-benchmark-datadir --output_dir /tmp/benchmark-results --num_keys 20000000
-          environment:
-            LD_LIBRARY_PATH: /usr/local/lib
-            # How long to run parts of the test(s)
-            DURATION_RO: 300
-            DURATION_RW: 500
-            # Keep threads within physical capacity of server (much lower than default)
-            NUM_THREADS: 1
-            MAX_BACKGROUND_JOBS: 4
-            # Don't run a couple of "optional" initial tests
-            CI_TESTS_ONLY: "true"
-            # Reduce configured size of levels to ensure more levels in the leveled compaction LSM tree
-            WRITE_BUFFER_SIZE_MB: 16
-            TARGET_FILE_SIZE_BASE_MB: 16
-            MAX_BYTES_FOR_LEVEL_BASE_MB: 64
-            # The benchmark host has 32GB memory
-            # The following values are tailored to work with that
-            # Note, tests may not exercise the targeted issues if the memory is increased on new test hosts.
-            COMPRESSION_TYPE: "none"
-            CACHE_INDEX_AND_FILTER_BLOCKS: 1
-            MIN_LEVEL_TO_COMPRESS: 3
-            CACHE_SIZE_MB: 10240
-            MB_WRITE_PER_SEC: 2
-
-  post-benchmarks:
-    steps:
-      - store_artifacts: # store the benchmark output
-          path: /tmp/benchmark-results
-          destination: test_logs
-      - run:
-          name: Send benchmark report to visualisation
-          command: |
-            set +e
-            set +o pipefail
-            ./build_tools/benchmark_log_tool.py --tsvfile /tmp/benchmark-results/report.tsv --esdocument https://search-rocksdb-bench-k2izhptfeap2hjfxteolsgsynm.us-west-2.es.amazonaws.com/bench_test3_rix/_doc
-            true
-
-executors:
-  linux-docker:
-    docker:
-      # The image configuration is build_tools/ubuntu20_image/Dockerfile
-      # To update and build the image:
-      #  $ cd build_tools/ubuntu20_image
-      #  $ docker build -t zjay437/rocksdb:0.5 .
-      #  $ docker push zjay437/rocksdb:0.5
-      # `zjay437` is the account name for zjay@meta.com which readwrite token is shared internally. To login:
-      #  $ docker login --username zjay437
-      # Or please feel free to change it to your docker hub account for hosting the image, meta employee should already have the account and able to login with SSO.
-      # To avoid impacting the existing CI runs, please bump the version every time creating a new image
-      # to run the CI image environment locally:
-      #  $ docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -it zjay437/rocksdb:0.5 bash
-      # option `--cap-add=SYS_PTRACE --security-opt seccomp=unconfined` is used to enable gdb to attach an existing process
-      - image: zjay437/rocksdb:0.6
-  linux-java-docker:
-    docker:
-      # This is the Docker Image used for building RocksJava releases, see: https://github.com/evolvedbinary/docker-rocksjava
-      - image: evolvedbinary/rocksjava:centos6_x64-be
-
-jobs:
-  build-macos:
-    macos:
-      xcode: 14.3.1
-    resource_class: macos.m1.medium.gen1
-    environment:
-      ROCKSDB_DISABLE_JEMALLOC: 1 # jemalloc cause env_test hang, disable it for now
-    steps:
-      - increase-max-open-files-on-macos
-      - install-gflags-on-macos
-      - pre-steps-macos
-      - run: ulimit -S -n `ulimit -H -n` && make V=1 J=16 -j16 all
-      - post-steps
-
-  build-macos-cmake:
-    macos:
-      xcode: 14.3.1
-    resource_class: macos.m1.medium.gen1
-    parameters:
-      run_even_tests:
-        description: run even or odd tests, used to split tests to 2 groups
-        type: boolean
-        default: true
-    steps:
-      - increase-max-open-files-on-macos
-      - install-cmake-on-macos
-      - install-gflags-on-macos
-      - pre-steps-macos
-      - run:
-          name: "cmake generate project file"
-          command: ulimit -S -n `ulimit -H -n` && mkdir build && cd build && cmake -DWITH_GFLAGS=1 ..
-      - run:
-          name: "Build tests"
-          command: cd build && make V=1 -j16
-      - when:
-          condition: << parameters.run_even_tests >>
-          steps:
-            - run:
-                name: "Run even tests"
-                command: ulimit -S -n `ulimit -H -n` && cd build && ctest -j16 -I 0,,2
-      - when:
-          condition:
-            not: << parameters.run_even_tests >>
-          steps:
-            - run:
-                name: "Run odd tests"
-                command: ulimit -S -n `ulimit -H -n` && cd build && ctest -j16 -I 1,,2
-      - post-steps
-
-  build-linux:
-    executor: linux-docker
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - run: make V=1 J=32 -j32 check
-      - post-steps
-
-  build-linux-encrypted_env-no_compression:
-    executor: linux-docker
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - run: ENCRYPTED_ENV=1 ROCKSDB_DISABLE_SNAPPY=1 ROCKSDB_DISABLE_ZLIB=1 ROCKSDB_DISABLE_BZIP=1 ROCKSDB_DISABLE_LZ4=1 ROCKSDB_DISABLE_ZSTD=1 make V=1 J=32 -j32 check
-      - run: |
-          ./sst_dump --help | grep -E -q 'Supported compression types: kNoCompression$' # Verify no compiled in compression
-      - post-steps
-
-  build-linux-static_lib-alt_namespace-status_checked:
-    executor: linux-docker
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=static OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j24 check
-      - post-steps
-
-  build-linux-release:
-    executor: linux-docker
-    resource_class: 2xlarge
-    steps:
-      - checkout # check out the code in the project directory
-      - run: make V=1 -j32 LIB_MODE=shared release
-      - run: ls librocksdb.so # ensure shared lib built
-      - run: ./db_stress --version # ensure with gflags
-      - run: make clean
-      - run: make V=1 -j32 release
-      - run: ls librocksdb.a # ensure static lib built
-      - run: ./db_stress --version # ensure with gflags
-      - run: make clean
-      - run: apt-get remove -y libgflags-dev
-      - run: make V=1 -j32 LIB_MODE=shared release
-      - run: ls librocksdb.so # ensure shared lib built
-      - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
-      - run: make clean
-      - run: make V=1 -j32 release
-      - run: ls librocksdb.a # ensure static lib built
-      - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
-      - post-steps
-
-  build-linux-release-rtti:
-    executor: linux-docker
-    resource_class: xlarge
-    steps:
-      - checkout # check out the code in the project directory
-      - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench
-      - run: ./db_stress --version # ensure with gflags
-      - run: make clean
-      - run: apt-get remove -y libgflags-dev
-      - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench
-      - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
-
-  build-linux-clang-no_test_run:
-    executor: linux-docker
-    resource_class: xlarge
-    steps:
-      - checkout # check out the code in the project directory
-      - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j16 all
-      - post-steps
-
-  build-linux-clang10-asan:
-    executor: linux-docker
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - run: COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check # aligned new doesn't work for reason we haven't figured out
-      - post-steps
-
-  build-linux-clang10-mini-tsan:
-    executor: linux-docker
-    resource_class: 2xlarge+
-    steps:
-      - pre-steps
-      - run: COMPILE_WITH_TSAN=1 CC=clang-13 CXX=clang++-13 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check
-      - post-steps
-
-  build-linux-clang10-ubsan:
-    executor: linux-docker
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check # aligned new doesn't work for reason we haven't figured out
-      - post-steps
-
-  build-linux-valgrind:
-    executor: linux-docker
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - run: PORTABLE=1 make V=1 -j32 valgrind_test
-      - post-steps
-
-  build-linux-clang10-clang-analyze:
-    executor: linux-docker
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path.
-      - post-steps
-      - run:
-          name: "compress test report"
-          command: tar -cvzf scan_build_report.tar.gz scan_build_report
-          when: on_fail
-      - store_artifacts:
-          path: scan_build_report.tar.gz
-          destination: scan_build_report
-          when: on_fail
-
-  build-linux-runner:
-    machine: true
-    resource_class: facebook/rocksdb-benchmark-sys1
-    steps:
-      - pre-steps
-      - run:
-          name: "Checked Linux build (Runner)"
-          command: make V=1 J=8 -j8 check
-          environment:
-            LD_LIBRARY_PATH: /usr/local/lib
-      - post-steps
-
-  build-linux-cmake-with-folly:
-    executor: linux-docker
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - setup-folly
-      - build-folly
-      - run: (mkdir build && cd build && cmake -DUSE_FOLLY=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20)
-      - post-steps
-
-  build-linux-cmake-with-folly-lite-no-test:
-    executor: linux-docker
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - setup-folly
-      - run: (mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 .. && make V=1 -j20)
-      - post-steps
-
-  build-linux-cmake-with-benchmark:
-    executor: linux-docker
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - run: mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j20 && ctest -j20
-      - post-steps
-
-  build-linux-unity-and-headers:
-    docker: # executor type
-      - image: gcc:latest
-    environment:
-      EXTRA_CXXFLAGS: -mno-avx512f # Warnings-as-error in avx512fintrin.h, would be used on newer hardware
-    resource_class: large
-    steps:
-      - checkout # check out the code in the project directory
-      - run: apt-get update -y && apt-get install -y libgflags-dev
-      - run:
-          name: "Unity build"
-          command: make V=1 -j8 unity_test
-          no_output_timeout: 20m
-      - run: make V=1 -j8 -k check-headers # could be moved to a different build
-      - post-steps
-
-  build-linux-gcc-7-with-folly:
-    executor: linux-docker
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - setup-folly
-      - build-folly
-      - run: USE_FOLLY=1 LIB_MODE=static CC=gcc-7 CXX=g++-7 V=1 make -j32 check # TODO: LIB_MODE only to work around unresolved linker failures
-      - post-steps
-
-  build-linux-gcc-7-with-folly-lite-no-test:
-    executor: linux-docker
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - setup-folly
-      - run: USE_FOLLY_LITE=1 CC=gcc-7 CXX=g++-7 V=1 make -j32 all
-      - post-steps
-
-  build-linux-gcc-8-no_test_run:
-    executor: linux-docker
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - run: CC=gcc-8 CXX=g++-8 V=1 make -j32 all
-      - post-steps
-
-  build-linux-cmake-with-folly-coroutines:
-    executor: linux-docker
-    resource_class: 2xlarge
-    environment:
-      CC: gcc-10
-      CXX: g++-10
-    steps:
-      - pre-steps
-      - setup-folly
-      - build-folly
-      - run: (mkdir build && cd build && cmake -DUSE_COROUTINES=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20)
-      - post-steps
-
-  build-linux-gcc-10-cxx20-no_test_run:
-    executor: linux-docker
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - run: CC=gcc-10 CXX=g++-10 V=1 ROCKSDB_CXX_STANDARD=c++20 make -j32 all
-      - post-steps
-
-  build-linux-gcc-11-no_test_run:
-    executor: linux-docker
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - run: LIB_MODE=static CC=gcc-11 CXX=g++-11 V=1 make -j32 all microbench # TODO: LIB_MODE only to work around unresolved linker failures
-      - post-steps
-
-  build-linux-clang-13-no_test_run:
-    executor: linux-docker
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j32 all microbench
-      - post-steps
-
-  # Ensure ASAN+UBSAN with folly, and full testsuite with clang 13
-  build-linux-clang-13-asan-ubsan-with-folly:
-    executor: linux-docker
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - setup-folly
-      - build-folly
-      - run: CC=clang-13 CXX=clang++-13 LIB_MODE=static USE_CLANG=1 USE_FOLLY=1 COMPILE_WITH_UBSAN=1 COMPILE_WITH_ASAN=1 make -j32 check # TODO: LIB_MODE only to work around unresolved linker failures
-      - post-steps
-
-  # This job is only to make sure the microbench tests are able to run, the benchmark result is not meaningful as the CI host is changing.
-  build-linux-run-microbench:
-    executor: linux-docker
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - run: DEBUG_LEVEL=0 make -j32 run_microbench
-      - post-steps
-
-  build-linux-mini-crashtest:
-    executor: linux-docker
-    resource_class: large
-    steps:
-      - pre-steps
-      - run: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=960 --max_key=2500000' blackbox_crash_test_with_atomic_flush
-      - post-steps
-
-  build-linux-crashtest-tiered-storage-bb:
-    executor: linux-docker
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - run:
-          name: "run crashtest"
-          command: ulimit -S -n `ulimit -H -n` && make V=1 -j32 CRASH_TEST_EXT_ARGS='--duration=10800' blackbox_crash_test_with_tiered_storage
-          no_output_timeout: 100m
-      - post-steps
-
-  build-linux-crashtest-tiered-storage-wb:
-    executor: linux-docker
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - run:
-          name: "run crashtest"
-          command: ulimit -S -n `ulimit -H -n` && make V=1 -j32 CRASH_TEST_EXT_ARGS='--duration=10800' whitebox_crash_test_with_tiered_storage
-          no_output_timeout: 100m
-      - post-steps
-
-  build-windows-vs2022-avx2:
-    executor:
-      name: win/server-2022
-      size: 2xlarge
-    environment:
-      THIRDPARTY_HOME: C:/Users/circleci/thirdparty
-      CMAKE_HOME: C:/Program Files/CMake
-      CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe
-      CTEST_BIN: C:/Program Files/CMake/bin/ctest.exe
-      JAVA_HOME: C:/Program Files/BellSoft/LibericaJDK-8
-      SNAPPY_HOME: C:/Users/circleci/thirdparty/snappy-1.1.8
-      SNAPPY_INCLUDE: C:/Users/circleci/thirdparty/snappy-1.1.8;C:/Users/circleci/thirdparty/snappy-1.1.8/build
-      SNAPPY_LIB_DEBUG: C:/Users/circleci/thirdparty/snappy-1.1.8/build/Debug/snappy.lib
-      CMAKE_GENERATOR: Visual Studio 17 2022
-      CMAKE_PORTABLE: AVX2
-    steps:
-      - windows-build-steps
-
-  build-windows-vs2022:
-    executor:
-      name: win/server-2022
-      size: 2xlarge
-    environment:
-      THIRDPARTY_HOME: C:/Users/circleci/thirdparty
-      CMAKE_HOME: C:/Program Files/CMake
-      CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe
-      CTEST_BIN: C:/Program Files/CMake/bin/ctest.exe
-      JAVA_HOME: C:/Program Files/BellSoft/LibericaJDK-8
-      SNAPPY_HOME: C:/Users/circleci/thirdparty/snappy-1.1.8
-      SNAPPY_INCLUDE: C:/Users/circleci/thirdparty/snappy-1.1.8;C:/Users/circleci/thirdparty/snappy-1.1.8/build
-      SNAPPY_LIB_DEBUG: C:/Users/circleci/thirdparty/snappy-1.1.8/build/Debug/snappy.lib
-      CMAKE_GENERATOR: Visual Studio 17 2022
-      CMAKE_PORTABLE: 1
-    steps:
-      - windows-build-steps
-
-  build-windows-vs2019:
-    executor:
-      name: win/server-2019
-      size: 2xlarge
-    environment:
-      THIRDPARTY_HOME: C:/Users/circleci/thirdparty
-      CMAKE_HOME: C:/Program Files/CMake
-      CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe
-      CTEST_BIN: C:/Program Files/CMake/bin/ctest.exe
-      JAVA_HOME: C:/Program Files/BellSoft/LibericaJDK-8
-      SNAPPY_HOME: C:/Users/circleci/thirdparty/snappy-1.1.8
-      SNAPPY_INCLUDE: C:/Users/circleci/thirdparty/snappy-1.1.8;C:/Users/circleci/thirdparty/snappy-1.1.8/build
-      SNAPPY_LIB_DEBUG: C:/Users/circleci/thirdparty/snappy-1.1.8/build/Debug/snappy.lib
-      CMAKE_GENERATOR: Visual Studio 16 2019
-      CMAKE_PORTABLE: 1
-    steps:
-      - windows-build-steps
-
-  build-linux-java:
-    executor: linux-java-docker
-    resource_class: large
-    steps:
-      - pre-steps
-      - run:
-          name: "Set Java Environment"
-          command: |
-            echo "JAVA_HOME=${JAVA_HOME}"
-            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
-            which java && java -version
-            which javac && javac -version
-      - run:
-          name: "Test RocksDBJava"
-          command: scl enable devtoolset-7 'make V=1 J=8 -j8 jtest'
-      - post-steps
-
-  build-linux-java-pmd:
-    executor: linux-java-docker
-    resource_class: large
-    steps:
-      - pre-steps
-      - run:
-          name: "Set Java Environment"
-          command: |
-            echo "JAVA_HOME=${JAVA_HOME}"
-            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
-            which java && java -version
-            which javac && javac -version
-      - install-maven
-      - run:
-          name: "PMD RocksDBJava"
-          command: scl enable devtoolset-7 'make V=1 J=8 -j8 jpmd'
-      - post-pmd-steps
-
-  build-linux-java-static:
-    executor: linux-java-docker
-    resource_class: large
-    steps:
-      - pre-steps
-      - run:
-          name: "Set Java Environment"
-          command: |
-            echo "JAVA_HOME=${JAVA_HOME}"
-            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
-            which java && java -version
-            which javac && javac -version
-      - run:
-          name: "Build RocksDBJava Static Library"
-          command: scl enable devtoolset-7 'make V=1 J=8 -j8 rocksdbjavastatic'
-      - post-steps
-
-  build-macos-java:
-    macos:
-      xcode: 14.3.1
-    resource_class: macos.m1.medium.gen1
-    environment:
-      JAVA_HOME: /Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home
-      ROCKSDB_DISABLE_JEMALLOC: 1 # jemalloc causes java 8 crash
-    steps:
-      - increase-max-open-files-on-macos
-      - install-gflags-on-macos
-      - install-jdk8-on-macos
-      - pre-steps-macos
-      - run:
-          name: "Set Java Environment"
-          command: |
-            echo "JAVA_HOME=${JAVA_HOME}"
-            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
-            which java && java -version
-            which javac && javac -version
-      - run:
-          name: "Test RocksDBJava"
-          command: make V=1 J=16 -j16 jtest
-          no_output_timeout: 20m
-      - post-steps
-
-  build-macos-java-static:
-    macos:
-      xcode: 14.3.1
-    resource_class: macos.m1.medium.gen1
-    environment:
-      JAVA_HOME: /Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home
-    steps:
-      - increase-max-open-files-on-macos
-      - install-gflags-on-macos
-      - install-cmake-on-macos
-      - install-jdk8-on-macos
-      - pre-steps-macos
-      - run:
-          name: "Set Java Environment"
-          command: |
-            echo "JAVA_HOME=${JAVA_HOME}"
-            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
-            which java && java -version
-            which javac && javac -version
-      - run:
-          name: "Build RocksDBJava x86 and ARM Static Libraries"
-          command: make V=1 J=16 -j16 rocksdbjavastaticosx
-          no_output_timeout: 20m
-      - post-steps
-
-  build-macos-java-static-universal:
-    macos:
-      xcode: 14.3.1
-    resource_class: macos.m1.medium.gen1
-    environment:
-      JAVA_HOME: /Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home
-    steps:
-      - increase-max-open-files-on-macos
-      - install-gflags-on-macos
-      - install-cmake-on-macos
-      - install-jdk8-on-macos
-      - pre-steps-macos
-      - run:
-          name: "Set Java Environment"
-          command: |
-            echo "JAVA_HOME=${JAVA_HOME}"
-            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
-            which java && java -version
-            which javac && javac -version
-      - run:
-          name: "Build RocksDBJava Universal Binary Static Library"
-          command: make V=1 J=16 -j16 rocksdbjavastaticosx_ub
-          no_output_timeout: 20m
-      - post-steps
-
-  build-examples:
-    executor: linux-docker
-    resource_class: large
-    steps:
-      - pre-steps
-      - run:
-          name: "Build examples"
-          command: |
-            make V=1 -j4 static_lib && cd examples && make V=1 -j4
-      - post-steps
-
-  build-cmake-mingw:
-    executor: linux-docker
-    resource_class: large
-    steps:
-      - pre-steps
-      - run: update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix
-      - run:
-          name: "Build cmake-mingw"
-          command: |
-            export PATH=$JAVA_HOME/bin:$PATH
-            echo "JAVA_HOME=${JAVA_HOME}"
-            which java && java -version
-            which javac && javac -version
-            mkdir build && cd build && cmake -DJNI=1 -DWITH_GFLAGS=OFF .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni
-      - post-steps
-
-  build-linux-non-shm:
-    executor: linux-docker
-    resource_class: 2xlarge
-    environment:
-      TEST_TMPDIR: /tmp/rocksdb_test_tmp
-    steps:
-      - pre-steps
-      - run: make V=1 -j32 check
-      - post-steps
-
-  build-linux-arm-test-full:
-    machine:
-      image: ubuntu-2004:202111-02
-    resource_class: arm.large
-    steps:
-      - pre-steps
-      - install-gflags
-      - run: make V=1 J=4 -j4 check
-      - post-steps
-
-  build-linux-arm:
-    machine:
-      image: ubuntu-2004:202111-02
-    resource_class: arm.large
-    steps:
-      - pre-steps
-      - install-gflags
-      - run: ROCKSDBTESTS_PLATFORM_DEPENDENT=only make V=1 J=4 -j4 all_but_some_tests check_some
-      - post-steps
-
-  build-linux-arm-cmake-no_test_run:
-    machine:
-      image: ubuntu-2004:202111-02
-    resource_class: arm.large
-    environment:
-      JAVA_HOME: /usr/lib/jvm/java-8-openjdk-arm64
-    steps:
-      - pre-steps
-      - install-gflags
-      - run:
-          name: "Set Java Environment"
-          command: |
-            echo "JAVA_HOME=${JAVA_HOME}"
-            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
-            which java && java -version
-            which javac && javac -version
-      - run:
-          name: "Build with cmake"
-          command: |
-            mkdir build
-            cd build
-            cmake -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=0 -DWITH_GFLAGS=1 -DWITH_BENCHMARK_TOOLS=0 -DWITH_TOOLS=0 -DWITH_CORE_TOOLS=1 ..
-            make -j4
-      - run:
-          name: "Build Java with cmake"
-          command: |
-            rm -rf build
-            mkdir build
-            cd build
-            cmake -DJNI=1 -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=1 ..
-            make -j4 rocksdb rocksdbjni
-      - post-steps
-
-  build-format-compatible:
-    executor: linux-docker
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - run:
-          name: "test"
-          command: |
-            export TEST_TMPDIR=/dev/shm/rocksdb
-            rm -rf /dev/shm/rocksdb
-            mkdir /dev/shm/rocksdb
-            tools/check_format_compatible.sh
-      - post-steps
-
-  build-fuzzers:
-    executor: linux-docker
-    resource_class: large
-    steps:
-      - pre-steps
-      - run:
-          name: "Build rocksdb lib"
-          command: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j4 static_lib
-      - run:
-          name: "Build fuzzers"
-          command: cd fuzz && make sst_file_writer_fuzzer db_fuzzer db_map_fuzzer
-      - post-steps
-
-  benchmark-linux: #use a private Circle CI runner (resource_class) to run the job
-    machine: true
-    resource_class: facebook/rocksdb-benchmark-sys1
-    steps:
-      - build-for-benchmarks
-      - perform-benchmarks
-      - post-benchmarks
-
-workflows: # Only jobs that haven't been successfully migrated to GitHub Actions
-  version: 2
-  jobs-linux-arm:
-    jobs:
-      - build-linux-arm
-  benchmark-linux:
-    triggers:
-      - schedule:
-          cron: "0 * * * *"
-          filters:
-            branches:
-              only:
-                - main
-    jobs:
-      - benchmark-linux
-  nightly:
-    triggers:
-      - schedule:
-          cron: "0 9 * * *"
-          filters:
-            branches:
-              only:
-                - main
-    jobs:
-      - build-linux-arm-test-full
@@ -1,6 +0,0 @@
-# Supress UBSAN warnings related to stl_tree.h, e.g.
-# UndefinedBehaviorSanitizer: undefined-behavior /usr/bin/../lib/gcc/x86_64-linux-gnu/5.4.0/../../../../include/c++/5.4.0/bits/stl_tree.h:1505:43 in 
-# /usr/bin/../lib/gcc/x86_64-linux-gnu/5.4.0/../../../../include/c++/5.4.0/bits/stl_tree.h:1505:43:
-# runtime error: upcast of address 0x000001fa8820 with insufficient space for an object of type
-# 'std::_Rb_tree_node<std::pair<const std::__cxx11::basic_string<char>, rocksdb::(anonymous namespace)::LockHoldingInfo> >'
-src:*bits/stl_tree.h
@@ -31,7 +31,7 @@ jobs:
  # ======================== Fast Initial Checks ====================== #
  check-format-and-targets:
    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
    steps:
    - uses: actions/checkout@v4.1.0
      with:
@@ -390,9 +390,9 @@ jobs:
    steps:
    - uses: actions/checkout@v4.1.0
    - uses: "./.github/actions/pre-steps"
-    - run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check
+    - run: COMPILE_WITH_UBSAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check
    - uses: "./.github/actions/post-steps"
-  build-linux-clang10-mini-tsan:
+  build-linux-clang13-mini-tsan:
    if: ${{ github.repository_owner == 'facebook' }}
    runs-on:
      labels: 32-core-ubuntu
@@ -91,6 +91,7 @@ buckifier/__pycache__
 .arcconfig

 compile_commands.json
+meta_gen_cpp_compile_commands.json
 clang-format-diff.py
 .py3/

@@ -214,6 +214,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
        "table/cuckoo/cuckoo_table_builder.cc",
        "table/cuckoo/cuckoo_table_factory.cc",
        "table/cuckoo/cuckoo_table_reader.cc",
+        "table/external_table_reader.cc",
        "table/format.cc",
        "table/get_context.cc",
        "table/iterator.cc",
@@ -835,6 +835,7 @@ set(SOURCES
        table/cuckoo/cuckoo_table_builder.cc
        table/cuckoo/cuckoo_table_factory.cc
        table/cuckoo/cuckoo_table_reader.cc
+        table/external_table_reader.cc
        table/format.cc
        table/get_context.cc
        table/iterator.cc
@@ -1,6 +1,48 @@
 # Rocksdb Change Log
 > NOTE: Entries for next release do not go here. Follow instructions in `unreleased_history/README.txt`

+## 9.11.1 (02/19/2025)
+### New Features
+* Added the ability to plug-in a custom table reader implementation. See include/rocksdb/external_table_reader.h for more details.
+
+## 9.11.0 (01/17/2025)
+### New Features
+* Introduce CancelAwaitingJobs() in CompactionService interface which will allow users to implement cancellation of running remote compactions from the primary instance
+* Experimental feature: RocksDB now supports defining secondary indices, which are automatically maintained by the storage engine. Secondary indices provide a new customization point: applications can provide their own by implementing the new `SecondaryIndex` interface. See the `SecondaryIndex` API comments for more details. Note: this feature is currently only available in conjunction with write-committed pessimistic transactions, and `Merge` is not yet supported.
+* Provide a new option `track_and_verify_wals` to track and verify various information about WAL during WAL recovery. This is intended to be a better replacement to `track_and_verify_wals_in_manifest`.
+
+### Public API Changes
+* Add `io_buffer_size` to BackupEngineOptions to enable optimal configuration of IO size
+* Clean up all the references to `random_access_max_buffer_size`, related rules and all the clients wrappers. This option has been officially deprecated in 5.4.0.
+* Add `file_ingestion_nanos` and `file_ingestion_blocking_live_writes_nanos` in PerfContext to observe file ingestions
+* Offer new DB::Open and variants that use `std::unique_ptr<DB>*` output parameters and deprecate the old versions that use `DB**` output parameters.
+* The DB::DeleteFile API is officially deprecated.
+
+### Behavior Changes
+* For leveled compaction, manual compaction (CompactRange()) will be more strict about keeping compaction size under `max_compaction_bytes`. This prevents overly large compactions in some cases (#13306).
+* Experimental tiering options `preclude_last_level_data_seconds` and `preserve_internal_time_seconds` are now mutable with `SetOptions()`. Some changes to handling of these features along with long-lived snapshots and range deletes made this possible.
+
+### Bug Fixes
+* Fix a longstanding major bug with SetOptions() in which setting changes can be quietly reverted.
+
+## 9.10.0 (12/12/2024)
+### New Features
+* Introduce `TransactionOptions::commit_bypass_memtable` to enable transaction commit to bypass memtable insertions. This can be beneficial for transactions with many operations, as it reduces commit time that is mostly spent on memtable insertion.
+
+### Public API Changes
+* Deprecated Remote Compaction APIs (StartV2, WaitForCompleteV2) are completely removed from the codebase
+
+### Behavior Changes
+* DB::KeyMayExist() now follows its function comment, which means `value` parameter can be null, and it will be set only if `value_found` is passed in.
+
+### Bug Fixes
+* Fix the issue where compaction incorrectly drops a key when there is a snapshot with a sequence number of zero.
+* Honor ConfigOptions.ignore_unknown_options in ParseStruct()
+
+### Performance Improvements
+* Enable reuse of file system allocated buffer for synchronous prefetching.
+* In buffered IO mode, try to align writes on power of 2 if checksum handoff is not enabled for the file type being written.
+
 ## 9.9.0 (11/18/2024)
 ### New Features
 * Multi-Column-Family-Iterator (CoalescingIterator/AttributeGroupIterator) is no longer marked as experimental
@@ -2489,7 +2489,7 @@ checkout_folly:
 	fi
 	@# Pin to a particular version for public CI, so that PR authors don't
 	@# need to worry about folly breaking our integration. Update periodically
-	cd third-party/folly && git reset --hard 33f5b67fcaeb8705b04fd1b850873a180dc89aaa
+	cd third-party/folly && git reset --hard 62baa6ba07ff0a23ee4f2ea2f5207e4c88464deb
 	@# NOTE: this hack is required for clang in some cases
 	perl -pi -e 's/int rv = syscall/int rv = (int)syscall/' third-party/folly/folly/detail/Futex.cpp
 	@# NOTE: this hack is required for gcc in some cases
@@ -118,7 +118,6 @@ Status SecondaryCache::CreateFromString(
      sec_cache = NewCompressedSecondaryCache(sec_cache_opts);
    }

-
    if (status.ok()) {
      result->swap(sec_cache);
    }
@@ -191,8 +191,7 @@ namespace {
 class SharedState {
 public:
  explicit SharedState(CacheBench* cache_bench)
-      : cv_(&mu_),
-        cache_bench_(cache_bench) {}
+      : cv_(&mu_), cache_bench_(cache_bench) {}

  ~SharedState() = default;

@@ -845,7 +845,6 @@ TEST_P(CompressedSecondaryCacheTestWithCompressionParam,
  BasicTestHelper(sec_cache, sec_cache_is_compressed_);
 }

-
 TEST_P(CompressedSecondaryCacheTestWithCompressionParam, FailsTest) {
  FailsTest(sec_cache_is_compressed_);
 }
@@ -93,9 +93,8 @@ void LRUHandleTable::Resize() {

  uint32_t old_length = uint32_t{1} << length_bits_;
  int new_length_bits = length_bits_ + 1;
-  std::unique_ptr<LRUHandle* []> new_list {
-    new LRUHandle* [size_t{1} << new_length_bits] {}
-  };
+  std::unique_ptr<LRUHandle*[]> new_list{
+      new LRUHandle* [size_t{1} << new_length_bits] {}};
  [[maybe_unused]] uint32_t count = 0;
  for (uint32_t i = 0; i < old_length; i++) {
    LRUHandle* h = list_[i];
@@ -546,7 +546,7 @@ TYPED_TEST(ClockCacheTest, Limits) {
    // verify usage tracking on detached entries.)
    {
      size_t n = kCapacity * 5 + 1;
-      std::unique_ptr<HandleImpl* []> ha { new HandleImpl* [n] {} };
+      std::unique_ptr<HandleImpl*[]> ha{new HandleImpl* [n] {}};
      Status s;
      for (size_t i = 0; i < n && s.ok(); ++i) {
        hkey[1] = i;
@@ -713,7 +713,7 @@ TYPED_TEST(ClockCacheTest, ClockEvictionEffortCapTest) {
      // evictable entries are present beyond the cache capacity, despite
      // being evictable.
      constexpr size_t kCount = kCapacity - 1;
-      std::unique_ptr<HandleImpl* []> ha { new HandleImpl* [kCount] {} };
+      std::unique_ptr<HandleImpl*[]> ha{new HandleImpl* [kCount] {}};
      for (size_t i = 0; i < 2 * kCount; ++i) {
        UniqueId64x2 hkey = this->CheapHash(i);
        ASSERT_OK(shard.Insert(
@@ -7,6 +7,4 @@

 #include "cache/cache_entry_roles.h"

-namespace ROCKSDB_NAMESPACE {
-
-}  // namespace ROCKSDB_NAMESPACE
+namespace ROCKSDB_NAMESPACE {}  // namespace ROCKSDB_NAMESPACE
@@ -951,12 +951,20 @@ TEST_F(DBTieredSecondaryCacheTest, FSBufferTest) {
                           IODebugContext* dbg) override {
          for (size_t i = 0; i < num_reqs; ++i) {
            FSReadRequest& req = reqs[i];
-            FSAllocationPtr buffer(new char[req.len], [](void* ptr) {
-              delete[] static_cast<char*>(ptr);
-            });
-            req.fs_scratch = std::move(buffer);
+            // See https://github.com/facebook/rocksdb/pull/13195 for why we
+            // want to set up our test implementation for FSAllocationPtr this
+            // way.
+            char* internalData = new char[req.len];
            req.status = Read(req.offset, req.len, options, &req.result,
-                              static_cast<char*>(req.fs_scratch.get()), dbg);
+                              internalData, dbg);
+
+            Slice* internalSlice = new Slice(internalData, req.len);
+            FSAllocationPtr internalPtr(internalSlice, [](void* ptr) {
+              delete[] static_cast<const char*>(
+                  static_cast<Slice*>(ptr)->data_);
+              delete static_cast<Slice*>(ptr);
+            });
+            req.fs_scratch = std::move(internalPtr);
          }
          return IOStatus::OK();
        }
@@ -97,7 +97,7 @@ Status ArenaWrappedDBIter::Refresh(const Snapshot* snapshot) {
    if (read_callback_) {
      read_callback_->Refresh(read_seq);
    }
-    Init(env, read_options_, *(cfd->ioptions()), sv->mutable_cf_options,
+    Init(env, read_options_, cfd->ioptions(), sv->mutable_cf_options,
         sv->current, read_seq,
         sv->mutable_cf_options.max_sequential_skip_in_iterations,
         sv->version_number, read_callback_, cfh_, expose_blob_index_,
@@ -82,8 +82,8 @@ Status BlobSource::PutBlobIntoCache(
  assert(cached_blob->IsEmpty());

  TypedHandle* cache_handle = nullptr;
-  const Status s = InsertEntryIntoCache(cache_key, blob->get(),
-                                        &cache_handle, Cache::Priority::BOTTOM);
+  const Status s = InsertEntryIntoCache(cache_key, blob->get(), &cache_handle,
+                                        Cache::Priority::BOTTOM);
  if (s.ok()) {
    blob->release();

@@ -231,7 +231,7 @@ struct rocksdb_livefiles_t {
 };
 struct rocksdb_column_family_handle_t {
  ColumnFamilyHandle* rep;
-  bool immortal;  /* only true for default cf */
+  bool immortal; /* only true for default cf */
 };
 struct rocksdb_column_family_metadata_t {
  ColumnFamilyMetaData rep;
@@ -1858,8 +1858,8 @@ extern ROCKSDB_LIBRARY_API void rocksdb_approximate_sizes_cf_with_flags(
  delete[] ranges;
 }

-void rocksdb_delete_file(rocksdb_t* db, const char* name) {
-  db->rep->DeleteFile(name);
+void DEPRECATED_rocksdb_delete_file(rocksdb_t* db, const char* name) {
+  db->rep->DEPRECATED_DeleteFile(name);
 }

 const rocksdb_livefiles_t* rocksdb_livefiles(rocksdb_t* db) {
@@ -5000,6 +5000,35 @@ int rocksdb_compactoptions_get_target_level(rocksdb_compactoptions_t* opt) {
  return opt->rep.target_level;
 }

+void rocksdb_compactoptions_set_target_path_id(rocksdb_compactoptions_t* opt,
+                                               int n) {
+  opt->rep.target_path_id = n;
+}
+
+int rocksdb_compactoptions_get_target_path_id(rocksdb_compactoptions_t* opt) {
+  return opt->rep.target_path_id;
+}
+
+void rocksdb_compactoptions_set_allow_write_stall(rocksdb_compactoptions_t* opt,
+                                                  unsigned char v) {
+  opt->rep.allow_write_stall = v;
+}
+
+unsigned char rocksdb_compactoptions_get_allow_write_stall(
+    rocksdb_compactoptions_t* opt) {
+  return opt->rep.allow_write_stall;
+}
+
+void rocksdb_compactoptions_set_max_subcompactions(
+    rocksdb_compactoptions_t* opt, int n) {
+  opt->rep.max_subcompactions = n;
+}
+
+int rocksdb_compactoptions_get_max_subcompactions(
+    rocksdb_compactoptions_t* opt) {
+  return opt->rep.max_subcompactions;
+}
+
 void rocksdb_compactoptions_set_full_history_ts_low(
    rocksdb_compactoptions_t* opt, char* ts, size_t tslen) {
  if (ts == nullptr) {
@@ -2938,6 +2938,15 @@ int main(int argc, char** argv) {
    rocksdb_compactoptions_set_target_level(co, 1);
    CheckCondition(1 == rocksdb_compactoptions_get_target_level(co));

+    rocksdb_compactoptions_set_target_path_id(co, 1);
+    CheckCondition(1 == rocksdb_compactoptions_get_target_path_id(co));
+
+    rocksdb_compactoptions_set_allow_write_stall(co, 1);
+    CheckCondition(1 == rocksdb_compactoptions_get_allow_write_stall(co));
+
+    rocksdb_compactoptions_set_max_subcompactions(co, 1);
+    CheckCondition(1 == rocksdb_compactoptions_get_max_subcompactions(co));
+
    rocksdb_compactoptions_destroy(co);
  }

@@ -53,7 +53,7 @@ ColumnFamilyHandleImpl::ColumnFamilyHandleImpl(

 ColumnFamilyHandleImpl::~ColumnFamilyHandleImpl() {
  if (cfd_ != nullptr) {
-    for (auto& listener : cfd_->ioptions()->listeners) {
+    for (auto& listener : cfd_->ioptions().listeners) {
      listener->OnColumnFamilyHandleDeletionStarted(this);
    }
    // Job id == 0 means that this is not our background process, but rather
@@ -593,7 +593,7 @@ ColumnFamilyData::ColumnFamilyData(
                                      block_cache_tracer, io_tracer,
                                      db_session_id));
    blob_file_cache_.reset(
-        new BlobFileCache(_table_cache, ioptions(), soptions(), id_,
+        new BlobFileCache(_table_cache, &ioptions(), soptions(), id_,
                          internal_stats_->GetBlobFileReadHist(), io_tracer));
    blob_source_.reset(new BlobSource(ioptions_, mutable_cf_options_, db_id,
                                      db_session_id, blob_file_cache_.get()));
@@ -968,7 +968,7 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
    auto write_stall_condition_and_cause = GetWriteStallConditionAndCause(
        imm()->NumNotFlushed(), vstorage->l0_delay_trigger_count(),
        vstorage->estimated_compaction_needed_bytes(), mutable_cf_options,
-        *ioptions());
+        ioptions());
    write_stall_condition = write_stall_condition_and_cause.first;
    auto write_stall_cause = write_stall_condition_and_cause.second;

@@ -1160,12 +1160,13 @@ MemTable* ColumnFamilyData::ConstructNewMemtable(
                      write_buffer_manager_, earliest_seq, id_);
 }

-void ColumnFamilyData::CreateNewMemtable(
-    const MutableCFOptions& mutable_cf_options, SequenceNumber earliest_seq) {
+void ColumnFamilyData::CreateNewMemtable(SequenceNumber earliest_seq) {
  if (mem_ != nullptr) {
    delete mem_->Unref();
  }
-  SetMemtable(ConstructNewMemtable(mutable_cf_options, earliest_seq));
+  // NOTE: db mutex must be locked for SetMemtable, so safe for
+  // GetLatestMutableCFOptions
+  SetMemtable(ConstructNewMemtable(GetLatestMutableCFOptions(), earliest_seq));
  mem_->Ref();
 }

@@ -1384,7 +1385,7 @@ void ColumnFamilyData::InstallSuperVersion(
        new_superversion->write_stall_condition) {
      sv_context->PushWriteStallNotification(
          old_superversion->write_stall_condition,
-          new_superversion->write_stall_condition, GetName(), ioptions());
+          new_superversion->write_stall_condition, GetName(), &ioptions());
    }
    if (old_superversion->Unref()) {
      old_superversion->Cleanup();
@@ -1844,10 +1845,7 @@ const ImmutableOptions& GetImmutableOptions(ColumnFamilyHandle* column_family) {
  const ColumnFamilyData* const cfd = handle->cfd();
  assert(cfd);

-  const ImmutableOptions* ioptions = cfd->ioptions();
-  assert(ioptions);
-
-  return *ioptions;
+  return cfd->ioptions();
 }

 }  // namespace ROCKSDB_NAMESPACE
@@ -221,7 +221,8 @@ struct SuperVersion {
  // enable UDT feature, this is an empty string.
  std::string full_history_ts_low;

-  // A shared copy of the DB's seqno to time mapping.
+  // An immutable snapshot of the DB's seqno to time mapping, usually shared
+  // between SuperVersions.
  std::shared_ptr<const SeqnoToTimeMapping> seqno_to_time_mapping{nullptr};

  // should be called outside the mutex
@@ -342,17 +343,17 @@ class ColumnFamilyData {

  // thread-safe
  const FileOptions* soptions() const;
-  const ImmutableOptions* ioptions() const { return &ioptions_; }
+  const ImmutableOptions& ioptions() const { return ioptions_; }
  // REQUIRES: DB mutex held
  // This returns the MutableCFOptions used by current SuperVersion
  // You should use this API to reference MutableCFOptions most of the time.
-  const MutableCFOptions* GetCurrentMutableCFOptions() const {
-    return &(super_version_->mutable_cf_options);
+  const MutableCFOptions& GetCurrentMutableCFOptions() const {
+    return super_version_->mutable_cf_options;
  }
  // REQUIRES: DB mutex held
  // This returns the latest MutableCFOptions, which may be not in effect yet.
-  const MutableCFOptions* GetLatestMutableCFOptions() const {
-    return &mutable_cf_options_;
+  const MutableCFOptions& GetLatestMutableCFOptions() const {
+    return mutable_cf_options_;
  }

  // REQUIRES: DB mutex held
@@ -400,10 +401,11 @@ class ColumnFamilyData {
  uint64_t OldestLogToKeep();

  // See Memtable constructor for explanation of earliest_seq param.
+  // `mutable_cf_options` might need to be a saved copy if calling this without
+  // holding the DB mutex.
  MemTable* ConstructNewMemtable(const MutableCFOptions& mutable_cf_options,
                                 SequenceNumber earliest_seq);
-  void CreateNewMemtable(const MutableCFOptions& mutable_cf_options,
-                         SequenceNumber earliest_seq);
+  void CreateNewMemtable(SequenceNumber earliest_seq);

  TableCache* table_cache() const { return table_cache_.get(); }
  BlobFileCache* blob_file_cache() const { return blob_file_cache_.get(); }
@@ -3,7 +3,6 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).

-
 #include <mutex>
 #include <string>
 #include <thread>
@@ -332,11 +332,11 @@ Compaction::Compaction(
              : (_blob_garbage_collection_policy ==
                         BlobGarbageCollectionPolicy::kDisable
                     ? false
-                     : mutable_cf_options()->enable_blob_garbage_collection)),
+                     : mutable_cf_options().enable_blob_garbage_collection)),
      blob_garbage_collection_age_cutoff_(
          _blob_garbage_collection_age_cutoff < 0 ||
                  _blob_garbage_collection_age_cutoff > 1
-              ? mutable_cf_options()->blob_garbage_collection_age_cutoff
+              ? mutable_cf_options().blob_garbage_collection_age_cutoff
              : _blob_garbage_collection_age_cutoff),
      penultimate_level_(
          // For simplicity, we don't support the concept of "penultimate level"
@@ -410,6 +410,7 @@ Compaction::Compaction(

 void Compaction::PopulatePenultimateLevelOutputRange() {
  if (!SupportsPerKeyPlacement()) {
+    assert(keep_in_last_level_through_seqno_ == kMaxSequenceNumber);
    return;
  }

@@ -452,6 +453,27 @@ void Compaction::PopulatePenultimateLevelOutputRange() {
  GetBoundaryInternalKeys(input_vstorage_, inputs_,
                          &penultimate_level_smallest_,
                          &penultimate_level_largest_, exclude_level);
+
+  if (penultimate_output_range_type_ !=
+      PenultimateOutputRangeType::kFullRange) {
+    // If not full range in penultimate level, must keep everything already
+    // in the last level there, because moving it back up might cause
+    // overlap/placement issues that are difficult to resolve properly in the
+    // presence of range deletes
+    SequenceNumber max_last_level_seqno = 0;
+    for (const auto& input_lvl : inputs_) {
+      if (input_lvl.level == output_level_) {
+        for (const auto& file : input_lvl.files) {
+          max_last_level_seqno =
+              std::max(max_last_level_seqno, file->fd.largest_seqno);
+        }
+      }
+    }
+
+    keep_in_last_level_through_seqno_ = max_last_level_seqno;
+  } else {
+    keep_in_last_level_through_seqno_ = 0;
+  }
 }

 Compaction::~Compaction() {
@@ -494,22 +516,29 @@ bool Compaction::OverlapPenultimateLevelOutputRange(
 }

 // key includes timestamp if user-defined timestamp is enabled.
-bool Compaction::WithinPenultimateLevelOutputRange(
-    const ParsedInternalKey& ikey) const {
-  if (!SupportsPerKeyPlacement()) {
-    return false;
-  }
+void Compaction::TEST_AssertWithinPenultimateLevelOutputRange(
+    const Slice& user_key, bool expect_failure) const {
+#ifdef NDEBUG
+  (void)user_key;
+  (void)expect_failure;
+#else
+  assert(SupportsPerKeyPlacement());

-  if (penultimate_level_smallest_.size() == 0 ||
-      penultimate_level_largest_.size() == 0) {
-    return false;
-  }
+  assert(penultimate_level_smallest_.size() > 0);
+  assert(penultimate_level_largest_.size() > 0);

-  const InternalKeyComparator* icmp = input_vstorage_->InternalComparator();
+  auto* cmp = input_vstorage_->user_comparator();

  // op_type of a key can change during compaction, e.g. Merge -> Put.
-  return icmp->CompareKeySeq(ikey, penultimate_level_smallest_.Encode()) >= 0 &&
-         icmp->CompareKeySeq(ikey, penultimate_level_largest_.Encode()) <= 0;
+  if (!(cmp->Compare(user_key, penultimate_level_smallest_.user_key()) >= 0)) {
+    assert(expect_failure);
+  } else if (!(cmp->Compare(user_key, penultimate_level_largest_.user_key()) <=
+               0)) {
+    assert(expect_failure);
+  } else {
+    assert(!expect_failure);
+  }
+#endif
 }

 bool Compaction::InputCompressionMatchesOutput() const {
@@ -563,7 +592,7 @@ bool Compaction::IsTrivialMove() const {
  // input files are non overlapping
  if ((mutable_cf_options_.compaction_options_universal.allow_trivial_move) &&
      (output_level_ != 0) &&
-      (cfd_->ioptions()->compaction_style == kCompactionStyleUniversal)) {
+      (cfd_->ioptions().compaction_style == kCompactionStyleUniversal)) {
    return is_trivial_move_;
  }

@@ -621,7 +650,7 @@ bool Compaction::KeyNotExistsBeyondOutputLevel(
  if (bottommost_level_) {
    return true;
  } else if (output_level_ != 0 &&
-             cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
+             cfd_->ioptions().compaction_style == kCompactionStyleLevel) {
    // Maybe use binary search to find right entry instead of linear search?
    const Comparator* user_cmp = cfd_->user_comparator();
    for (int lvl = output_level_ + 1; lvl < number_levels_; lvl++) {
@@ -662,7 +691,7 @@ bool Compaction::KeyRangeNotExistsBeyondOutputLevel(
  if (bottommost_level_) {
    return true /* does not overlap */;
  } else if (output_level_ != 0 &&
-             cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
+             cfd_->ioptions().compaction_style == kCompactionStyleLevel) {
    const Comparator* user_cmp = cfd_->user_comparator();
    for (int lvl = output_level_ + 1; lvl < number_levels_; lvl++) {
      const std::vector<FileMetaData*>& files =
@@ -838,12 +867,12 @@ uint64_t Compaction::OutputFilePreallocationSize() const {
 }

 std::unique_ptr<CompactionFilter> Compaction::CreateCompactionFilter() const {
-  if (!cfd_->ioptions()->compaction_filter_factory) {
+  if (!cfd_->ioptions().compaction_filter_factory) {
    return nullptr;
  }

  if (!cfd_->ioptions()
-           ->compaction_filter_factory->ShouldFilterTableFileCreation(
+           .compaction_filter_factory->ShouldFilterTableFileCreation(
               TableFileCreationReason::kCompaction)) {
    return nullptr;
  }
@@ -862,7 +891,7 @@ std::unique_ptr<CompactionFilter> Compaction::CreateCompactionFilter() const {
        "for compaction.");
  }

-  return cfd_->ioptions()->compaction_filter_factory->CreateCompactionFilter(
+  return cfd_->ioptions().compaction_filter_factory->CreateCompactionFilter(
      context);
 }

@@ -896,8 +925,8 @@ bool Compaction::ShouldFormSubcompactions() const {

  // Round-Robin pri under leveled compaction allows subcompactions by default
  // and the number of subcompactions can be larger than max_subcompactions_
-  if (cfd_->ioptions()->compaction_pri == kRoundRobin &&
-      cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
+  if (cfd_->ioptions().compaction_pri == kRoundRobin &&
+      cfd_->ioptions().compaction_style == kCompactionStyleLevel) {
    return output_level_ > 0;
  }

@@ -905,9 +934,9 @@ bool Compaction::ShouldFormSubcompactions() const {
    return false;
  }

-  if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
+  if (cfd_->ioptions().compaction_style == kCompactionStyleLevel) {
    return (start_level_ == 0 || is_manual_compaction_) && output_level_ > 0;
-  } else if (cfd_->ioptions()->compaction_style == kCompactionStyleUniversal) {
+  } else if (cfd_->ioptions().compaction_style == kCompactionStyleUniversal) {
    return number_levels_ > 1 && output_level_ > 0;
  } else {
    return false;
@@ -288,14 +288,14 @@ class Compaction {

  // Return the ImmutableOptions that should be used throughout the compaction
  // procedure
-  const ImmutableOptions* immutable_options() const {
-    return &immutable_options_;
+  const ImmutableOptions& immutable_options() const {
+    return immutable_options_;
  }

  // Return the MutableCFOptions that should be used throughout the compaction
  // procedure
-  const MutableCFOptions* mutable_cf_options() const {
-    return &mutable_cf_options_;
+  const MutableCFOptions& mutable_cf_options() const {
+    return mutable_cf_options_;
  }

  // Returns the size in bytes that the output file should be preallocated to.
@@ -388,12 +388,12 @@ class Compaction {
  bool OverlapPenultimateLevelOutputRange(const Slice& smallest_key,
                                          const Slice& largest_key) const;

-  // Return true if the key is within penultimate level output range for
-  // per_key_placement feature, which is safe to place the key to the
-  // penultimate level. different compaction strategy has different rules.
-  // If per_key_placement is not supported, always return false.
-  //  key includes timestamp if user-defined timestamp is enabled.
-  bool WithinPenultimateLevelOutputRange(const ParsedInternalKey& ikey) const;
+  // For testing purposes, check that a key is within penultimate level
+  // output range for per_key_placement feature, which is safe to place the key
+  // to the penultimate level. Different compaction strategies have different
+  // rules. `user_key` includes timestamp if user-defined timestamp is enabled.
+  void TEST_AssertWithinPenultimateLevelOutputRange(
+      const Slice& user_key, bool expect_failure = false) const;

  CompactionReason compaction_reason() const { return compaction_reason_; }

@@ -456,11 +456,17 @@ class Compaction {
      const ImmutableOptions& immutable_options, const int start_level,
      const int output_level);

+  // If some data cannot be safely migrated "up" the LSM tree due to a change
+  // in the preclude_last_level_data_seconds setting, this indicates a sequence
+  // number for the newest data that must be kept in the last level.
+  SequenceNumber GetKeepInLastLevelThroughSeqno() const {
+    return keep_in_last_level_through_seqno_;
+  }
+
  // mark (or clear) all files that are being compacted
  void MarkFilesBeingCompacted(bool being_compacted) const;

 private:
-
  Status InitInputTableProperties();

  // get the smallest and largest key present in files to be compacted
@@ -606,6 +612,8 @@ class Compaction {
  // Blob garbage collection age cutoff.
  double blob_garbage_collection_age_cutoff_;

+  SequenceNumber keep_in_last_level_through_seqno_ = kMaxSequenceNumber;
+
  // only set when per_key_placement feature is enabled, -1 (kInvalidLevel)
  // means not supported.
  const int penultimate_level_;
@@ -38,19 +38,16 @@ CompactionIterator::CompactionIterator(
    const std::atomic<bool>* shutting_down,
    const std::shared_ptr<Logger> info_log,
    const std::string* full_history_ts_low,
-    const SequenceNumber preserve_time_min_seqno,
-    const SequenceNumber preclude_last_level_min_seqno)
+    std::optional<SequenceNumber> preserve_seqno_min)
    : CompactionIterator(
          input, cmp, merge_helper, last_sequence, snapshots, earliest_snapshot,
          earliest_write_conflict_snapshot, job_snapshot, snapshot_checker, env,
          report_detailed_time, expect_valid_internal_key, range_del_agg,
          blob_file_builder, allow_data_in_errors, enforce_single_del_contracts,
          manual_compaction_canceled,
-          std::unique_ptr<CompactionProxy>(
-              compaction ? new RealCompaction(compaction) : nullptr),
+          compaction ? std::make_unique<RealCompaction>(compaction) : nullptr,
          must_count_input_entries, compaction_filter, shutting_down, info_log,
-          full_history_ts_low, preserve_time_min_seqno,
-          preclude_last_level_min_seqno) {}
+          full_history_ts_low, preserve_seqno_min) {}

 CompactionIterator::CompactionIterator(
    InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
@@ -68,8 +65,7 @@ CompactionIterator::CompactionIterator(
    const std::atomic<bool>* shutting_down,
    const std::shared_ptr<Logger> info_log,
    const std::string* full_history_ts_low,
-    const SequenceNumber preserve_time_min_seqno,
-    const SequenceNumber preclude_last_level_min_seqno)
+    std::optional<SequenceNumber> preserve_seqno_min)
    : input_(input, cmp, must_count_input_entries),
      cmp_(cmp),
      merge_helper_(merge_helper),
@@ -110,10 +106,9 @@ CompactionIterator::CompactionIterator(
      current_key_committed_(false),
      cmp_with_history_ts_low_(0),
      level_(compaction_ == nullptr ? 0 : compaction_->level()),
-      preserve_time_min_seqno_(preserve_time_min_seqno),
-      preclude_last_level_min_seqno_(preclude_last_level_min_seqno) {
+      preserve_seqno_after_(preserve_seqno_min.value_or(earliest_snapshot)) {
  assert(snapshots_ != nullptr);
-  assert(preserve_time_min_seqno_ <= preclude_last_level_min_seqno_);
+  assert(preserve_seqno_after_ <= earliest_snapshot_);

  if (compaction_ != nullptr) {
    level_ptrs_ = std::vector<size_t>(compaction_->number_levels(), 0);
@@ -1018,7 +1013,6 @@ void CompactionIterator::NextFromInput() {
        } else {
          if (ikey_.sequence != 0) {
            iter_stats_.num_timed_put_swap_preferred_seqno++;
-            saved_seq_for_penul_check_ = ikey_.sequence;
            ikey_.sequence = preferred_seqno;
          }
          ikey_.type = kTypeValue;
@@ -1259,71 +1253,6 @@ void CompactionIterator::GarbageCollectBlobIfNeeded() {
  }
 }

-void CompactionIterator::DecideOutputLevel() {
-  assert(compaction_->SupportsPerKeyPlacement());
-  output_to_penultimate_level_ = false;
-  // if the key is newer than the cutoff sequence or within the earliest
-  // snapshot, it should output to the penultimate level.
-  if (ikey_.sequence > preclude_last_level_min_seqno_ ||
-      ikey_.sequence > earliest_snapshot_) {
-    output_to_penultimate_level_ = true;
-  }
-
-#ifndef NDEBUG
-  // Could be overridden by unittest
-  PerKeyPlacementContext context(level_, ikey_.user_key, value_, ikey_.sequence,
-                                 output_to_penultimate_level_);
-  TEST_SYNC_POINT_CALLBACK("CompactionIterator::PrepareOutput.context",
-                           &context);
-  if (ikey_.sequence > earliest_snapshot_) {
-    output_to_penultimate_level_ = true;
-  }
-#endif  // NDEBUG
-
-  // saved_seq_for_penul_check_ is populated in `NextFromInput` when the
-  // entry's sequence number is non zero and validity context for output this
-  // entry is kSwapPreferredSeqno for use in `DecideOutputLevel`. It should be
-  // cleared out here unconditionally. Otherwise, it may end up getting consumed
-  // incorrectly by a different entry.
-  SequenceNumber seq_for_range_check =
-      (saved_seq_for_penul_check_.has_value() &&
-       saved_seq_for_penul_check_.value() != kMaxSequenceNumber)
-          ? saved_seq_for_penul_check_.value()
-          : ikey_.sequence;
-  saved_seq_for_penul_check_ = std::nullopt;
-  ParsedInternalKey ikey_for_range_check = ikey_;
-  if (seq_for_range_check != ikey_.sequence) {
-    ikey_for_range_check.sequence = seq_for_range_check;
-  }
-  if (output_to_penultimate_level_) {
-    // If it's decided to output to the penultimate level, but unsafe to do so,
-    // still output to the last level. For example, moving the data from a lower
-    // level to a higher level outside of the higher-level input key range is
-    // considered unsafe, because the key may conflict with higher-level SSTs
-    // not from this compaction.
-    // TODO: add statistic for declined output_to_penultimate_level
-    bool safe_to_penultimate_level =
-        compaction_->WithinPenultimateLevelOutputRange(ikey_for_range_check);
-    if (!safe_to_penultimate_level) {
-      output_to_penultimate_level_ = false;
-      // It could happen when disable/enable `last_level_temperature` while
-      // holding a snapshot. When `last_level_temperature` is not set
-      // (==kUnknown), the data newer than any snapshot is pushed to the last
-      // level, but when the per_key_placement feature is enabled on the fly,
-      // the data later than the snapshot has to be moved to the penultimate
-      // level, which may or may not be safe. So the user needs to make sure all
-      // snapshot is released before enabling `last_level_temperature` feature
-      // We will migrate the feature to `last_level_temperature` and maybe make
-      // it not dynamically changeable.
-      if (seq_for_range_check > earliest_snapshot_) {
-        status_ = Status::Corruption(
-            "Unsafe to store Seq later than snapshot in the last level if "
-            "per_key_placement is enabled");
-      }
-    }
-  }
-}
-
 void CompactionIterator::PrepareOutput() {
  if (Valid()) {
    if (LIKELY(!is_range_del_)) {
@@ -1332,13 +1261,6 @@ void CompactionIterator::PrepareOutput() {
      } else if (ikey_.type == kTypeBlobIndex) {
        GarbageCollectBlobIfNeeded();
      }
-
-      // For range del sentinel, we don't use it to cut files for bottommost
-      // compaction. So it should not make a difference which output level we
-      // decide.
-      if (compaction_ != nullptr && compaction_->SupportsPerKeyPlacement()) {
-        DecideOutputLevel();
-      }
    }

    // Zeroing out the sequence number leads to better compression.
@@ -1356,8 +1278,7 @@ void CompactionIterator::PrepareOutput() {
        !compaction_->allow_ingest_behind() && bottommost_level_ &&
        DefinitelyInSnapshot(ikey_.sequence, earliest_snapshot_) &&
        ikey_.type != kTypeMerge && current_key_committed_ &&
-        !output_to_penultimate_level_ &&
-        ikey_.sequence < preserve_time_min_seqno_ && !is_range_del_) {
+        ikey_.sequence <= preserve_seqno_after_ && !is_range_del_) {
      if (ikey_.type == kTypeDeletion ||
          (ikey_.type == kTypeSingleDeletion && timestamp_size_ == 0)) {
        ROCKS_LOG_FATAL(
@@ -117,10 +117,6 @@ class CompactionIterator {
    virtual const Compaction* real_compaction() const = 0;

    virtual bool SupportsPerKeyPlacement() const = 0;
-
-    // `key` includes timestamp if user-defined timestamp is enabled.
-    virtual bool WithinPenultimateLevelOutputRange(
-        const ParsedInternalKey&) const = 0;
  };

  class RealCompaction : public CompactionProxy {
@@ -128,8 +124,6 @@ class CompactionIterator {
    explicit RealCompaction(const Compaction* compaction)
        : compaction_(compaction) {
      assert(compaction_);
-      assert(compaction_->immutable_options());
-      assert(compaction_->mutable_cf_options());
    }

    int level() const override { return compaction_->level(); }
@@ -151,11 +145,11 @@ class CompactionIterator {
    }

    bool allow_ingest_behind() const override {
-      return compaction_->immutable_options()->allow_ingest_behind;
+      return compaction_->immutable_options().allow_ingest_behind;
    }

    bool allow_mmap_reads() const override {
-      return compaction_->immutable_options()->allow_mmap_reads;
+      return compaction_->immutable_options().allow_mmap_reads;
    }

    bool enable_blob_garbage_collection() const override {
@@ -167,7 +161,7 @@ class CompactionIterator {
    }

    uint64_t blob_compaction_readahead_size() const override {
-      return compaction_->mutable_cf_options()->blob_compaction_readahead_size;
+      return compaction_->mutable_cf_options().blob_compaction_readahead_size;
    }

    const Version* input_version() const override {
@@ -184,14 +178,6 @@ class CompactionIterator {
      return compaction_->SupportsPerKeyPlacement();
    }

-    // Check if key is within penultimate level output range, to see if it's
-    // safe to output to the penultimate level for per_key_placement feature.
-    // `key` includes timestamp if user-defined timestamp is enabled.
-    bool WithinPenultimateLevelOutputRange(
-        const ParsedInternalKey& ikey) const override {
-      return compaction_->WithinPenultimateLevelOutputRange(ikey);
-    }
-
   private:
    const Compaction* compaction_;
  };
@@ -216,29 +202,29 @@ class CompactionIterator {
      const std::atomic<bool>* shutting_down = nullptr,
      const std::shared_ptr<Logger> info_log = nullptr,
      const std::string* full_history_ts_low = nullptr,
-      const SequenceNumber preserve_time_min_seqno = kMaxSequenceNumber,
-      const SequenceNumber preclude_last_level_min_seqno = kMaxSequenceNumber);
+      std::optional<SequenceNumber> preserve_seqno_min = {});

  // Constructor with custom CompactionProxy, used for tests.
-  CompactionIterator(
-      InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
-      SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
-      SequenceNumber earliest_snapshot,
-      SequenceNumber earliest_write_conflict_snapshot,
-      SequenceNumber job_snapshot, const SnapshotChecker* snapshot_checker,
-      Env* env, bool report_detailed_time, bool expect_valid_internal_key,
-      CompactionRangeDelAggregator* range_del_agg,
-      BlobFileBuilder* blob_file_builder, bool allow_data_in_errors,
-      bool enforce_single_del_contracts,
-      const std::atomic<bool>& manual_compaction_canceled,
-      std::unique_ptr<CompactionProxy> compaction,
-      bool must_count_input_entries,
-      const CompactionFilter* compaction_filter = nullptr,
-      const std::atomic<bool>* shutting_down = nullptr,
-      const std::shared_ptr<Logger> info_log = nullptr,
-      const std::string* full_history_ts_low = nullptr,
-      const SequenceNumber preserve_time_min_seqno = kMaxSequenceNumber,
-      const SequenceNumber preclude_last_level_min_seqno = kMaxSequenceNumber);
+  CompactionIterator(InternalIterator* input, const Comparator* cmp,
+                     MergeHelper* merge_helper, SequenceNumber last_sequence,
+                     std::vector<SequenceNumber>* snapshots,
+                     SequenceNumber earliest_snapshot,
+                     SequenceNumber earliest_write_conflict_snapshot,
+                     SequenceNumber job_snapshot,
+                     const SnapshotChecker* snapshot_checker, Env* env,
+                     bool report_detailed_time, bool expect_valid_internal_key,
+                     CompactionRangeDelAggregator* range_del_agg,
+                     BlobFileBuilder* blob_file_builder,
+                     bool allow_data_in_errors,
+                     bool enforce_single_del_contracts,
+                     const std::atomic<bool>& manual_compaction_canceled,
+                     std::unique_ptr<CompactionProxy> compaction,
+                     bool must_count_input_entries,
+                     const CompactionFilter* compaction_filter = nullptr,
+                     const std::atomic<bool>* shutting_down = nullptr,
+                     const std::shared_ptr<Logger> info_log = nullptr,
+                     const std::string* full_history_ts_low = nullptr,
+                     std::optional<SequenceNumber> preserve_seqno_min = {});

  ~CompactionIterator();

@@ -269,11 +255,6 @@ class CompactionIterator {
  const CompactionIterationStats& iter_stats() const { return iter_stats_; }
  bool HasNumInputEntryScanned() const { return input_.HasNumItered(); }
  uint64_t NumInputEntryScanned() const { return input_.NumItered(); }
-  // If the current key should be placed on penultimate level, only valid if
-  // per_key_placement is supported
-  bool output_to_penultimate_level() const {
-    return output_to_penultimate_level_;
-  }
  Status InputStatus() const { return input_.status(); }

  bool IsDeleteRangeSentinelKey() const { return is_range_del_; }
@@ -285,10 +266,6 @@ class CompactionIterator {
  // Do final preparations before presenting the output to the callee.
  void PrepareOutput();

-  // Decide the current key should be output to the last level or penultimate
-  // level, only call for compaction supports per key placement
-  void DecideOutputLevel();
-
  // Passes the output value to the blob file builder (if any), and replaces it
  // with the corresponding blob reference if it has been actually written to a
  // blob file (i.e. if it passed the value size check). Returns true if the
@@ -440,16 +417,6 @@ class CompactionIterator {
  // NextFromInput()).
  ParsedInternalKey ikey_;

-  // When a kTypeValuePreferredSeqno entry's preferred seqno is safely swapped
-  // in in this compaction, this field saves its original sequence number for
-  // range checking whether it's safe to be placed on the penultimate level.
-  // This is to ensure when such an entry happens to be the right boundary of
-  // penultimate safe range, it won't get excluded because with the preferred
-  // seqno swapped in, it's now larger than the right boundary (itself before
-  // the swap). This is safe to do, because preferred seqno is swapped in only
-  // when no entries with the same user key exist on lower levels and this entry
-  // is already visible in the earliest snapshot.
-  std::optional<SequenceNumber> saved_seq_for_penul_check_ = kMaxSequenceNumber;
  // Stores whether ikey_.user_key is valid. If set to false, the user key is
  // not compared against the current key in the underlying iterator.
  bool has_current_user_key_ = false;
@@ -507,17 +474,8 @@ class CompactionIterator {
  // just been zeroed out during bottommost compaction.
  bool last_key_seq_zeroed_{false};

-  // True if the current key should be output to the penultimate level if
-  // possible, compaction logic makes the final decision on which level to
-  // output to.
-  bool output_to_penultimate_level_{false};
-
-  // min seqno for preserving the time information.
-  const SequenceNumber preserve_time_min_seqno_ = kMaxSequenceNumber;
-
-  // min seqno to preclude the data from the last level, if the key seqno larger
-  // than this, it will be output to penultimate level
-  const SequenceNumber preclude_last_level_min_seqno_ = kMaxSequenceNumber;
+  // Max seqno that can be zeroed out at last level (various reasons)
+  const SequenceNumber preserve_seqno_after_ = kMaxSequenceNumber;

  void AdvanceInputIter() { input_.Next(); }

@@ -192,11 +192,6 @@ class FakeCompaction : public CompactionIterator::CompactionProxy {
    return supports_per_key_placement;
  }

-  bool WithinPenultimateLevelOutputRange(
-      const ParsedInternalKey& key) const override {
-    return (!key.user_key.starts_with("unsafe_pb"));
-  }
-
  bool key_not_exists_beyond_output_level = false;

  bool is_bottommost_level = false;
@@ -844,123 +839,6 @@ TEST_P(CompactionIteratorTest, ZeroSeqOfKeyAndSnapshot) {
 INSTANTIATE_TEST_CASE_P(CompactionIteratorTestInstance, CompactionIteratorTest,
                        testing::Values(true, false));

-class PerKeyPlacementCompIteratorTest : public CompactionIteratorTest {
- public:
-  bool SupportsPerKeyPlacement() const override { return true; }
-};
-
-TEST_P(PerKeyPlacementCompIteratorTest, SplitLastLevelData) {
-  std::atomic_uint64_t latest_cold_seq = 0;
-
-  SyncPoint::GetInstance()->SetCallBack(
-      "CompactionIterator::PrepareOutput.context", [&](void* arg) {
-        auto context = static_cast<PerKeyPlacementContext*>(arg);
-        context->output_to_penultimate_level =
-            context->seq_num > latest_cold_seq;
-      });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  latest_cold_seq = 5;
-
-  InitIterators(
-      {test::KeyStr("a", 7, kTypeValue), test::KeyStr("b", 6, kTypeValue),
-       test::KeyStr("c", 5, kTypeValue)},
-      {"vala", "valb", "valc"}, {}, {}, kMaxSequenceNumber, kMaxSequenceNumber,
-      nullptr, nullptr, true);
-  c_iter_->SeekToFirst();
-  ASSERT_TRUE(c_iter_->Valid());
-
-  // the first 2 keys are hot, which should has
-  // `output_to_penultimate_level()==true` and seq num not zeroed out
-  ASSERT_EQ(test::KeyStr("a", 7, kTypeValue), c_iter_->key().ToString());
-  ASSERT_TRUE(c_iter_->output_to_penultimate_level());
-  c_iter_->Next();
-  ASSERT_TRUE(c_iter_->Valid());
-  ASSERT_EQ(test::KeyStr("b", 6, kTypeValue), c_iter_->key().ToString());
-  ASSERT_TRUE(c_iter_->output_to_penultimate_level());
-  c_iter_->Next();
-  ASSERT_TRUE(c_iter_->Valid());
-  // `a` is cold data, which should be output to bottommost
-  ASSERT_EQ(test::KeyStr("c", 0, kTypeValue), c_iter_->key().ToString());
-  ASSERT_FALSE(c_iter_->output_to_penultimate_level());
-  c_iter_->Next();
-  ASSERT_OK(c_iter_->status());
-  ASSERT_FALSE(c_iter_->Valid());
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-}
-
-TEST_P(PerKeyPlacementCompIteratorTest, SnapshotData) {
-  AddSnapshot(5);
-
-  InitIterators(
-      {test::KeyStr("a", 7, kTypeValue), test::KeyStr("b", 6, kTypeDeletion),
-       test::KeyStr("b", 5, kTypeValue)},
-      {"vala", "", "valb"}, {}, {}, kMaxSequenceNumber, kMaxSequenceNumber,
-      nullptr, nullptr, true);
-  c_iter_->SeekToFirst();
-  ASSERT_TRUE(c_iter_->Valid());
-
-  // The first key and the tombstone are within snapshot, which should output
-  // to the penultimate level (and seq num cannot be zeroed out).
-  ASSERT_EQ(test::KeyStr("a", 7, kTypeValue), c_iter_->key().ToString());
-  ASSERT_TRUE(c_iter_->output_to_penultimate_level());
-  c_iter_->Next();
-  ASSERT_TRUE(c_iter_->Valid());
-  ASSERT_EQ(test::KeyStr("b", 6, kTypeDeletion), c_iter_->key().ToString());
-  ASSERT_TRUE(c_iter_->output_to_penultimate_level());
-  c_iter_->Next();
-  ASSERT_TRUE(c_iter_->Valid());
-  // `a` is not protected by the snapshot, the sequence number is zero out and
-  // should output bottommost
-  ASSERT_EQ(test::KeyStr("b", 0, kTypeValue), c_iter_->key().ToString());
-  ASSERT_FALSE(c_iter_->output_to_penultimate_level());
-  c_iter_->Next();
-  ASSERT_OK(c_iter_->status());
-  ASSERT_FALSE(c_iter_->Valid());
-}
-
-TEST_P(PerKeyPlacementCompIteratorTest, ConflictWithSnapshot) {
-  std::atomic_uint64_t latest_cold_seq = 0;
-
-  SyncPoint::GetInstance()->SetCallBack(
-      "CompactionIterator::PrepareOutput.context", [&](void* arg) {
-        auto context = static_cast<PerKeyPlacementContext*>(arg);
-        context->output_to_penultimate_level =
-            context->seq_num > latest_cold_seq;
-      });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  latest_cold_seq = 6;
-
-  AddSnapshot(5);
-
-  InitIterators({test::KeyStr("a", 7, kTypeValue),
-                 test::KeyStr("unsafe_pb", 6, kTypeValue),
-                 test::KeyStr("c", 5, kTypeValue)},
-                {"vala", "valb", "valc"}, {}, {}, kMaxSequenceNumber,
-                kMaxSequenceNumber, nullptr, nullptr, true);
-  c_iter_->SeekToFirst();
-  ASSERT_TRUE(c_iter_->Valid());
-
-  ASSERT_EQ(test::KeyStr("a", 7, kTypeValue), c_iter_->key().ToString());
-  ASSERT_TRUE(c_iter_->output_to_penultimate_level());
-  // the 2nd key is unsafe to output_to_penultimate_level, but it's within
-  // snapshot so for per_key_placement feature it has to be outputted to the
-  // penultimate level. which is a corruption. We should never see
-  // such case as the data with seq num (within snapshot) should always come
-  // from higher compaction input level, which makes it safe to
-  // output_to_penultimate_level.
-  c_iter_->Next();
-  ASSERT_TRUE(c_iter_->status().IsCorruption());
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-}
-
-INSTANTIATE_TEST_CASE_P(PerKeyPlacementCompIteratorTest,
-                        PerKeyPlacementCompIteratorTest,
-                        testing::Values(true, false));
-
 // Tests how CompactionIterator work together with SnapshotChecker.
 class CompactionIteratorWithSnapshotCheckerTest
    : public CompactionIteratorTest {
@@ -245,7 +245,10 @@ void CompactionJob::ReportStartedCompaction(Compaction* compaction) {
  compaction_job_stats_->is_full_compaction = compaction->is_full_compaction();
 }

-void CompactionJob::Prepare() {
+void CompactionJob::Prepare(
+    std::optional<std::pair<std::optional<Slice>, std::optional<Slice>>>
+        known_single_subcompact) {
+  db_mutex_->AssertHeld();
  AutoThreadOperationStageUpdater stage_updater(
      ThreadStatus::STAGE_COMPACTION_PREPARE);

@@ -260,11 +263,12 @@ void CompactionJob::Prepare() {
  write_hint_ = storage_info->CalculateSSTWriteHint(c->output_level());
  bottommost_level_ = c->bottommost_level();

-  if (c->ShouldFormSubcompactions()) {
+  if (!known_single_subcompact.has_value() && c->ShouldFormSubcompactions()) {
    StopWatch sw(db_options_.clock, stats_, SUBCOMPACTION_SETUP_TIME);
    GenSubcompactionBoundaries();
  }
  if (boundaries_.size() >= 1) {
+    assert(!known_single_subcompact.has_value());
    for (size_t i = 0; i <= boundaries_.size(); i++) {
      compact_->sub_compact_states.emplace_back(
          c, (i != 0) ? std::optional<Slice>(boundaries_[i - 1]) : std::nullopt,
@@ -280,16 +284,25 @@ void CompactionJob::Prepare() {
    RecordInHistogram(stats_, NUM_SUBCOMPACTIONS_SCHEDULED,
                      compact_->sub_compact_states.size());
  } else {
-    compact_->sub_compact_states.emplace_back(c, std::nullopt, std::nullopt,
+    std::optional<Slice> start_key;
+    std::optional<Slice> end_key;
+    if (known_single_subcompact.has_value()) {
+      start_key = known_single_subcompact.value().first;
+      end_key = known_single_subcompact.value().second;
+    } else {
+      assert(!start_key.has_value() && !end_key.has_value());
+    }
+    compact_->sub_compact_states.emplace_back(c, start_key, end_key,
                                              /*sub_job_id*/ 0);
  }

  // collect all seqno->time information from the input files which will be used
  // to encode seqno->time to the output files.
-
+  SequenceNumber preserve_time_min_seqno = kMaxSequenceNumber;
+  SequenceNumber preclude_last_level_min_seqno = kMaxSequenceNumber;
  uint64_t preserve_time_duration =
-      std::max(c->mutable_cf_options()->preserve_internal_time_seconds,
-               c->mutable_cf_options()->preclude_last_level_data_seconds);
+      std::max(c->mutable_cf_options().preserve_internal_time_seconds,
+               c->mutable_cf_options().preclude_last_level_data_seconds);

  if (preserve_time_duration > 0) {
    const ReadOptions read_options(Env::IOActivity::kCompaction);
@@ -319,16 +332,16 @@ void CompactionJob::Prepare() {
                     "Failed to get current time in compaction: Status: %s",
                     s.ToString().c_str());
      // preserve all time information
-      preserve_time_min_seqno_ = 0;
-      preclude_last_level_min_seqno_ = 0;
+      preserve_time_min_seqno = 0;
+      preclude_last_level_min_seqno = 0;
      seqno_to_time_mapping_.Enforce();
    } else {
      seqno_to_time_mapping_.Enforce(_current_time);
      seqno_to_time_mapping_.GetCurrentTieringCutoffSeqnos(
          static_cast<uint64_t>(_current_time),
-          c->mutable_cf_options()->preserve_internal_time_seconds,
-          c->mutable_cf_options()->preclude_last_level_data_seconds,
-          &preserve_time_min_seqno_, &preclude_last_level_min_seqno_);
+          c->mutable_cf_options().preserve_internal_time_seconds,
+          c->mutable_cf_options().preclude_last_level_data_seconds,
+          &preserve_time_min_seqno, &preclude_last_level_min_seqno);
    }
    // For accuracy of the GetProximalSeqnoBeforeTime queries above, we only
    // limit the capacity after them.
@@ -341,6 +354,43 @@ void CompactionJob::Prepare() {
    // larger than kMaxSeqnoTimePairsPerSST.
    seqno_to_time_mapping_.SetCapacity(kMaxSeqnoToTimeEntries);
  }
+#ifndef NDEBUG
+  assert(preserve_time_min_seqno <= preclude_last_level_min_seqno);
+  TEST_SYNC_POINT_CALLBACK(
+      "CompactionJob::PrepareTimes():preclude_last_level_min_seqno",
+      static_cast<void*>(&preclude_last_level_min_seqno));
+  // Restore the invariant asserted above, in case it was broken under the
+  // callback
+  preserve_time_min_seqno =
+      std::min(preclude_last_level_min_seqno, preserve_time_min_seqno);
+#endif
+
+  // Preserve sequence numbers for preserved write times and snapshots, though
+  // the specific sequence number of the earliest snapshot can be zeroed.
+  preserve_seqno_after_ =
+      std::max(preserve_time_min_seqno, SequenceNumber{1}) - 1;
+  preserve_seqno_after_ = std::min(preserve_seqno_after_, earliest_snapshot_);
+  // If using preclude feature, also preclude snapshots from last level, just
+  // because they are heuristically more likely to be accessed than non-snapshot
+  // data.
+  if (preclude_last_level_min_seqno < kMaxSequenceNumber &&
+      earliest_snapshot_ < preclude_last_level_min_seqno) {
+    preclude_last_level_min_seqno = earliest_snapshot_;
+  }
+  // Now combine what we would like to preclude from last level with what we
+  // can safely support without dangerously moving data back up the LSM tree,
+  // to get the final seqno threshold for penultimate vs. last. In particular,
+  // when the reserved output key range for the penultimate level does not
+  // include the entire last level input key range, we need to keep entries
+  // already in the last level there. (Even allowing within-range entries to
+  // move back up could cause problems with range tombstones. Perhaps it
+  // would be better in some rare cases to keep entries in the last level
+  // one-by-one rather than based on sequence number, but that would add extra
+  // tracking and complexity to CompactionIterator that is probably not
+  // worthwhile overall. Correctness is also more clear when splitting by
+  // seqno threshold.)
+  penultimate_after_seqno_ = std::max(preclude_last_level_min_seqno,
+                                      c->GetKeepInLastLevelThroughSeqno());
 }

 uint64_t CompactionJob::GetSubcompactionsLimit() {
@@ -469,14 +519,14 @@ void CompactionJob::GenSubcompactionBoundaries() {
  ReadOptions read_options(Env::IOActivity::kCompaction);
  read_options.rate_limiter_priority = GetRateLimiterPriority();
  auto* c = compact_->compaction;
-  if (c->mutable_cf_options()->table_factory->Name() ==
+  if (c->mutable_cf_options().table_factory->Name() ==
      TableFactory::kPlainTableName()) {
    return;
  }

  if (c->max_subcompactions() <= 1 &&
-      !(c->immutable_options()->compaction_pri == kRoundRobin &&
-        c->immutable_options()->compaction_style == kCompactionStyleLevel)) {
+      !(c->immutable_options().compaction_pri == kRoundRobin &&
+        c->immutable_options().compaction_style == kCompactionStyleLevel)) {
    return;
  }
  auto* cfd = c->column_family_data();
@@ -506,7 +556,7 @@ void CompactionJob::GenSubcompactionBoundaries() {
        FileMetaData* f = flevel->files[i].file_metadata;
        std::vector<TableReader::Anchor> my_anchors;
        Status s = cfd->table_cache()->ApproximateKeyAnchors(
-            read_options, icomp, *f, *c->mutable_cf_options(), my_anchors);
+            read_options, icomp, *f, c->mutable_cf_options(), my_anchors);
        if (!s.ok() || my_anchors.empty()) {
          my_anchors.emplace_back(f->largest.user_key(), f->fd.GetFileSize());
        }
@@ -545,8 +595,8 @@ void CompactionJob::GenSubcompactionBoundaries() {
  // Get the number of planned subcompactions, may update reserve threads
  // and update extra_num_subcompaction_threads_reserved_ for round-robin
  uint64_t num_planned_subcompactions;
-  if (c->immutable_options()->compaction_pri == kRoundRobin &&
-      c->immutable_options()->compaction_style == kCompactionStyleLevel) {
+  if (c->immutable_options().compaction_pri == kRoundRobin &&
+      c->immutable_options().compaction_style == kCompactionStyleLevel) {
    // For round-robin compaction prioity, we need to employ more
    // subcompactions (may exceed the max_subcompaction limit). The extra
    // subcompactions will be executed using reserved threads and taken into
@@ -585,9 +635,9 @@ void CompactionJob::GenSubcompactionBoundaries() {
  uint64_t target_range_size = std::max(
      total_size / num_planned_subcompactions,
      MaxFileSizeForLevel(
-          *(c->mutable_cf_options()), out_lvl,
-          c->immutable_options()->compaction_style, base_level,
-          c->immutable_options()->level_compaction_dynamic_level_bytes));
+          c->mutable_cf_options(), out_lvl,
+          c->immutable_options().compaction_style, base_level,
+          c->immutable_options().level_compaction_dynamic_level_bytes));

  if (target_range_size >= total_size) {
    return;
@@ -730,14 +780,13 @@ Status CompactionJob::Run() {
            verify_table_read_options, file_options_,
            cfd->internal_comparator(), files_output[file_idx]->meta,
            /*range_del_agg=*/nullptr,
-            *compact_->compaction->mutable_cf_options(),
+            compact_->compaction->mutable_cf_options(),
            /*table_reader_ptr=*/nullptr,
            cfd->internal_stats()->GetFileReadHist(
                compact_->compaction->output_level()),
            TableReaderCaller::kCompactionRefill, /*arena=*/nullptr,
            /*skip_filters=*/false, compact_->compaction->output_level(),
-            MaxFileSizeForL0MetaPin(
-                *compact_->compaction->mutable_cf_options()),
+            MaxFileSizeForL0MetaPin(compact_->compaction->mutable_cf_options()),
            /*smallest_compaction_key=*/nullptr,
            /*largest_compaction_key=*/nullptr,
            /*allow_unprepared_value=*/false);
@@ -794,7 +843,7 @@ Status CompactionJob::Run() {
  for (const auto& state : compact_->sub_compact_states) {
    for (const auto& output : state.GetOutputs()) {
      auto fn =
-          TableFileName(state.compaction->immutable_options()->cf_paths,
+          TableFileName(state.compaction->immutable_options().cf_paths,
                        output.meta.fd.GetNumber(), output.meta.fd.GetPathId());
      compact_->compaction->SetOutputTableProperties(fn,
                                                     output.table_properties);
@@ -853,8 +902,7 @@ Status CompactionJob::Run() {
  return status;
 }

-Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options,
-                              bool* compaction_released) {
+Status CompactionJob::Install(bool* compaction_released) {
  assert(compact_);

  AutoThreadOperationStageUpdater stage_updater(
@@ -870,7 +918,7 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options,
                                            compaction_stats_);

  if (status.ok()) {
-    status = InstallCompactionResults(mutable_cf_options, compaction_released);
+    status = InstallCompactionResults(compaction_released);
  }
  if (!versions_->io_status().ok()) {
    io_status_ = versions_->io_status();
@@ -959,8 +1007,7 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options,
  UpdateCompactionJobStats(stats);

  auto stream = event_logger_->LogToBuffer(log_buffer_, 8192);
-  stream << "job" << job_id_ << "event"
-         << "compaction_finished"
+  stream << "job" << job_id_ << "event" << "compaction_finished"
         << "compaction_time_micros" << stats.micros
         << "compaction_time_cpu_micros" << stats.cpu_micros << "output_level"
         << compact_->compaction->output_level() << "num_output_files"
@@ -1050,12 +1097,10 @@ void CompactionJob::NotifyOnSubcompactionBegin(
    listener->OnSubcompactionBegin(info);
  }
  info.status.PermitUncheckedError();
-
 }

 void CompactionJob::NotifyOnSubcompactionCompleted(
    SubcompactionState* sub_compact) {
-
  if (db_options_.listeners.empty()) {
    return;
  }
@@ -1098,8 +1143,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {

  // Create compaction filter and fail the compaction if
  // IgnoreSnapshots() = false because it is not supported anymore
-  const CompactionFilter* compaction_filter =
-      cfd->ioptions()->compaction_filter;
+  const CompactionFilter* compaction_filter = cfd->ioptions().compaction_filter;
  std::unique_ptr<CompactionFilter> compaction_filter_from_factory = nullptr;
  if (compaction_filter == nullptr) {
    compaction_filter_from_factory =
@@ -1115,9 +1159,12 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {

  NotifyOnSubcompactionBegin(sub_compact);

-  auto range_del_agg = std::make_unique<CompactionRangeDelAggregator>(
-      &cfd->internal_comparator(), existing_snapshots_, &full_history_ts_low_,
-      &trim_ts_);
+  // This is assigned after creation of SubcompactionState to simplify that
+  // creation across both CompactionJob and CompactionServiceCompactionJob
+  sub_compact->AssignRangeDelAggregator(
+      std::make_unique<CompactionRangeDelAggregator>(
+          &cfd->internal_comparator(), existing_snapshots_,
+          &full_history_ts_low_, &trim_ts_));

  // TODO: since we already use C++17, should use
  // std::optional<const Slice> instead.
@@ -1162,7 +1209,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
  // Although the v2 aggregator is what the level iterator(s) know about,
  // the AddTombstones calls will be propagated down to the v1 aggregator.
  std::unique_ptr<InternalIterator> raw_input(versions_->MakeInputIterator(
-      read_options, sub_compact->compaction, range_del_agg.get(),
+      read_options, sub_compact->compaction, sub_compact->RangeDelAgg(),
      file_options_for_read_, start, end));
  InternalIterator* input = raw_input.get();

@@ -1254,28 +1301,27 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
  }

  MergeHelper merge(
-      env_, cfd->user_comparator(), cfd->ioptions()->merge_operator.get(),
+      env_, cfd->user_comparator(), cfd->ioptions().merge_operator.get(),
      compaction_filter, db_options_.info_log.get(),
      false /* internal key corruption is expected */,
      existing_snapshots_.empty() ? 0 : existing_snapshots_.back(),
      snapshot_checker_, compact_->compaction->level(), db_options_.stats);

-  const MutableCFOptions* mutable_cf_options =
+  const auto& mutable_cf_options =
      sub_compact->compaction->mutable_cf_options();
-  assert(mutable_cf_options);

  std::vector<std::string> blob_file_paths;

  // TODO: BlobDB to support output_to_penultimate_level compaction, which needs
  //  2 builders, so may need to move to `CompactionOutputs`
  std::unique_ptr<BlobFileBuilder> blob_file_builder(
-      (mutable_cf_options->enable_blob_files &&
+      (mutable_cf_options.enable_blob_files &&
       sub_compact->compaction->output_level() >=
-           mutable_cf_options->blob_file_starting_level)
+           mutable_cf_options.blob_file_starting_level)
          ? new BlobFileBuilder(
                versions_, fs_.get(),
-                sub_compact->compaction->immutable_options(),
-                mutable_cf_options, &file_options_, &write_options, db_id_,
+                &sub_compact->compaction->immutable_options(),
+                &mutable_cf_options, &file_options_, &write_options, db_id_,
                db_session_id_, job_id_, cfd->GetID(), cfd->GetName(),
                write_hint_, io_tracer_, blob_callback_,
                BlobFileCreationReason::kCompaction, &blob_file_paths,
@@ -1298,20 +1344,15 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
      &existing_snapshots_, earliest_snapshot_,
      earliest_write_conflict_snapshot_, job_snapshot_seq, snapshot_checker_,
      env_, ShouldReportDetailedTime(env_, stats_),
-      /*expect_valid_internal_key=*/true, range_del_agg.get(),
+      /*expect_valid_internal_key=*/true, sub_compact->RangeDelAgg(),
      blob_file_builder.get(), db_options_.allow_data_in_errors,
      db_options_.enforce_single_del_contracts, manual_compaction_canceled_,
      sub_compact->compaction
          ->DoesInputReferenceBlobFiles() /* must_count_input_entries */,
      sub_compact->compaction, compaction_filter, shutting_down_,
-      db_options_.info_log, full_history_ts_low, preserve_time_min_seqno_,
-      preclude_last_level_min_seqno_);
+      db_options_.info_log, full_history_ts_low, preserve_seqno_after_);
  c_iter->SeekToFirst();

-  // Assign range delete aggregator to the target output level, which makes sure
-  // it only output to single level
-  sub_compact->AssignRangeDelAggregator(std::move(range_del_agg));
-
  const auto& c_iter_stats = c_iter->iter_stats();

  // define the open and close functions for the compaction files, which will be
@@ -1355,12 +1396,41 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
      last_cpu_micros = cur_cpu_micros;
    }

+    const auto& ikey = c_iter->ikey();
+    bool use_penultimate_output = ikey.sequence > penultimate_after_seqno_;
+#ifndef NDEBUG
+    if (sub_compact->compaction->SupportsPerKeyPlacement()) {
+      // Could be overridden by unittest
+      PerKeyPlacementContext context(sub_compact->compaction->output_level(),
+                                     ikey.user_key, c_iter->value(),
+                                     ikey.sequence, use_penultimate_output);
+      TEST_SYNC_POINT_CALLBACK("CompactionIterator::PrepareOutput.context",
+                               &context);
+      if (use_penultimate_output) {
+        // Verify that entries sent to the penultimate level are within the
+        // allowed range (because the input key range of the last level could
+        // be larger than the allowed output key range of the penultimate
+        // level). This check uses user keys (ignores sequence numbers) because
+        // compaction boundaries are a "clean cut" between user keys (see
+        // CompactionPicker::ExpandInputsToCleanCut()), which is especially
+        // important when preferred sequence numbers has been swapped in for
+        // kTypeValuePreferredSeqno / TimedPut.
+        sub_compact->compaction->TEST_AssertWithinPenultimateLevelOutputRange(
+            c_iter->user_key());
+      }
+    } else {
+      assert(penultimate_after_seqno_ == kMaxSequenceNumber);
+      assert(!use_penultimate_output);
+    }
+#endif  // NDEBUG
+
    // Add current compaction_iterator key to target compaction output, if the
    // output file needs to be close or open, it will call the `open_file_func`
    // and `close_file_func`.
    // TODO: it would be better to have the compaction file open/close moved
    // into `CompactionOutputs` which has the output file information.
-    status = sub_compact->AddToOutput(*c_iter, open_file_func, close_file_func);
+    status = sub_compact->AddToOutput(*c_iter, use_penultimate_output,
+                                      open_file_func, close_file_func);
    if (!status.ok()) {
      break;
    }
@@ -1567,16 +1637,31 @@ Status CompactionJob::FinishCompactionOutputFile(
    earliest_snapshot = existing_snapshots_[0];
  }
  if (s.ok()) {
+    // Inclusive lower bound, exclusive upper bound
+    std::pair<SequenceNumber, SequenceNumber> keep_seqno_range{
+        0, kMaxSequenceNumber};
+    if (sub_compact->compaction->SupportsPerKeyPlacement()) {
+      if (outputs.IsPenultimateLevel()) {
+        keep_seqno_range.first = penultimate_after_seqno_;
+      } else {
+        keep_seqno_range.second = penultimate_after_seqno_;
+      }
+    }
    CompactionIterationStats range_del_out_stats;
-    // if the compaction supports per_key_placement, only output range dels to
-    // the penultimate level.
-    // Note: Use `bottommost_level_ = true` for both bottommost and
+    // NOTE1: Use `bottommost_level_ = true` for both bottommost and
    // output_to_penultimate_level compaction here, as it's only used to decide
-    // if range dels could be dropped.
-    if (outputs.HasRangeDel()) {
-      s = outputs.AddRangeDels(comp_start_user_key, comp_end_user_key,
-                               range_del_out_stats, bottommost_level_,
-                               cfd->internal_comparator(), earliest_snapshot,
+    // if range dels could be dropped. (Logically, we are taking a single sorted
+    // run returned from CompactionIterator and physically splitting it between
+    // two output levels.)
+    // NOTE2: with per-key placement, range tombstones will be filtered on
+    // each output level based on sequence number (traversed twice). This is
+    // CPU-inefficient for a large number of range tombstones, but that would
+    // be an unusual work load.
+    if (sub_compact->HasRangeDel()) {
+      s = outputs.AddRangeDels(*sub_compact->RangeDelAgg(), comp_start_user_key,
+                               comp_end_user_key, range_del_out_stats,
+                               bottommost_level_, cfd->internal_comparator(),
+                               earliest_snapshot, keep_seqno_range,
                               next_table_min_key, full_history_ts_low_);
    }
    RecordDroppedKeys(range_del_out_stats, &sub_compact->compaction_job_stats);
@@ -1639,7 +1724,7 @@ Status CompactionJob::FinishCompactionOutputFile(
    // This happens when the output level is bottom level, at the same time
    // the sub_compact output nothing.
    std::string fname =
-        TableFileName(sub_compact->compaction->immutable_options()->cf_paths,
+        TableFileName(sub_compact->compaction->immutable_options().cf_paths,
                      meta->fd.GetNumber(), meta->fd.GetPathId());

    // TODO(AR) it is not clear if there are any larger implications if
@@ -1686,7 +1771,7 @@ Status CompactionJob::FinishCompactionOutputFile(
    }
  }
  EventHelpers::LogAndNotifyTableFileCreationFinished(
-      event_logger_, cfd->ioptions()->listeners, dbname_, cfd->GetName(), fname,
+      event_logger_, cfd->ioptions().listeners, dbname_, cfd->GetName(), fname,
      job_id_, output_fd, oldest_blob_file_number, tp,
      TableFileCreationReason::kCompaction, status_for_listener, file_checksum,
      file_checksum_func_name);
@@ -1714,8 +1799,7 @@ Status CompactionJob::FinishCompactionOutputFile(
  return s;
 }

-Status CompactionJob::InstallCompactionResults(
-    const MutableCFOptions& mutable_cf_options, bool* compaction_released) {
+Status CompactionJob::InstallCompactionResults(bool* compaction_released) {
  assert(compact_);

  db_mutex_->AssertHeld();
@@ -1789,7 +1873,7 @@ Status CompactionJob::InstallCompactionResults(
  if ((compaction->compaction_reason() ==
           CompactionReason::kLevelMaxLevelSize ||
       compaction->compaction_reason() == CompactionReason::kRoundRobinTtl) &&
-      compaction->immutable_options()->compaction_pri == kRoundRobin) {
+      compaction->immutable_options().compaction_pri == kRoundRobin) {
    int start_level = compaction->start_level();
    if (start_level > 0) {
      auto vstorage = compaction->input_version()->storage_info();
@@ -1804,11 +1888,11 @@ Status CompactionJob::InstallCompactionResults(
    *compaction_released = true;
  };

-  return versions_->LogAndApply(
-      compaction->column_family_data(), mutable_cf_options, read_options,
-      write_options, edit, db_mutex_, db_directory_,
-      /*new_descriptor_log=*/false,
-      /*column_family_options=*/nullptr, manifest_wcb);
+  return versions_->LogAndApply(compaction->column_family_data(), read_options,
+                                write_options, edit, db_mutex_, db_directory_,
+                                /*new_descriptor_log=*/false,
+                                /*column_family_options=*/nullptr,
+                                manifest_wcb);
 }

 void CompactionJob::RecordCompactionIOStats() {
@@ -1844,7 +1928,7 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
  // Fire events.
  ColumnFamilyData* cfd = sub_compact->compaction->column_family_data();
  EventHelpers::NotifyTableFileCreationStarted(
-      cfd->ioptions()->listeners, dbname_, cfd->GetName(), fname, job_id_,
+      cfd->ioptions().listeners, dbname_, cfd->GetName(), fname, job_id_,
      TableFileCreationReason::kCompaction);
  // Make the output file
  std::unique_ptr<FSWritableFile> writable_file;
@@ -1858,12 +1942,12 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
  FileOptions fo_copy = file_options_;
  Temperature temperature = sub_compact->compaction->output_temperature();
  Temperature last_level_temp =
-      sub_compact->compaction->mutable_cf_options()->last_level_temperature;
+      sub_compact->compaction->mutable_cf_options().last_level_temperature;
  // Here last_level_temperature supersedes default_write_temperature, when
  // enabled and applicable
  if (last_level_temp != Temperature::kUnknown &&
      sub_compact->compaction->is_last_level() &&
-      !sub_compact->IsCurrentPenultimateLevel()) {
+      !outputs.IsPenultimateLevel()) {
    temperature = last_level_temp;
  }
  fo_copy.temperature = temperature;
@@ -1886,7 +1970,7 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
        job_id_, file_number, s.ToString().c_str());
    LogFlush(db_options_.info_log);
    EventHelpers::LogAndNotifyTableFileCreationFinished(
-        event_logger_, cfd->ioptions()->listeners, dbname_, cfd->GetName(),
+        event_logger_, cfd->ioptions().listeners, dbname_, cfd->GetName(),
        fname, job_id_, FileDescriptor(), kInvalidBlobFileNumber,
        TableProperties(), TableFileCreationReason::kCompaction, s,
        kUnknownFileChecksum, kUnknownFileChecksumFuncName);
@@ -1957,7 +2041,7 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
  writable_file->SetPreallocationBlockSize(static_cast<size_t>(
      sub_compact->compaction->OutputFilePreallocationSize()));
  const auto& listeners =
-      sub_compact->compaction->immutable_options()->listeners;
+      sub_compact->compaction->immutable_options().listeners;
  outputs.AssignFileWriter(new WritableFileWriter(
      std::move(writable_file), fname, fo_copy, db_options_.clock, io_tracer_,
      db_options_.stats, Histograms::SST_WRITE_MICROS, listeners,
@@ -1968,7 +2052,7 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
  const ReadOptions read_options(Env::IOActivity::kCompaction);
  const WriteOptions write_options(Env::IOActivity::kCompaction);
  TableBuilderOptions tboptions(
-      *cfd->ioptions(), *(sub_compact->compaction->mutable_cf_options()),
+      cfd->ioptions(), sub_compact->compaction->mutable_cf_options(),
      read_options, write_options, cfd->internal_comparator(),
      cfd->internal_tbl_prop_coll_factories(),
      sub_compact->compaction->output_compression(),
@@ -1977,9 +2061,7 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
      bottommost_level_, TableFileCreationReason::kCompaction,
      0 /* oldest_key_time */, current_time, db_id_, db_session_id_,
      sub_compact->compaction->max_output_file_size(), file_number,
-      preclude_last_level_min_seqno_ == kMaxSequenceNumber
-          ? preclude_last_level_min_seqno_
-          : std::min(earliest_snapshot_, preclude_last_level_min_seqno_));
+      penultimate_after_seqno_ /*last_level_inclusive_max_seqno_threshold*/);

  outputs.NewBuilder(tboptions);

@@ -2036,9 +2118,8 @@ bool CompactionJob::UpdateCompactionStats(uint64_t* num_input_range_del) {
      if (file_input_entries == 0) {
        uint64_t file_number = file_meta->fd.GetNumber();
        // Try getting info from table property
-        std::string fn =
-            TableFileName(compaction->immutable_options()->cf_paths,
-                          file_number, file_meta->fd.GetPathId());
+        std::string fn = TableFileName(compaction->immutable_options().cf_paths,
+                                       file_number, file_meta->fd.GetPathId());
        const auto& tp = input_table_properties.find(fn);
        if (tp != input_table_properties.end()) {
          file_input_entries = tp->second->num_entries;
@@ -2120,7 +2201,6 @@ void CompactionJob::UpdateCompactionJobStats(
 void CompactionJob::LogCompaction() {
  Compaction* compaction = compact_->compaction;
  ColumnFamilyData* cfd = compaction->column_family_data();
-
  // Let's check if anything will get logged. Don't prepare all the info if
  // we're not logging
  if (db_options_.info_log_level <= InfoLogLevel::INFO_LEVEL) {
@@ -2135,8 +2215,7 @@ void CompactionJob::LogCompaction() {
                   cfd->GetName().c_str(), scratch);
    // build event logger report
    auto stream = event_logger_->Log();
-    stream << "job" << job_id_ << "event"
-           << "compaction_started"
+    stream << "job" << job_id_ << "event" << "compaction_started"
           << "compaction_reason"
           << GetCompactionReasonString(compaction->compaction_reason());
    for (size_t i = 0; i < compaction->num_input_levels(); ++i) {
@@ -2153,8 +2232,8 @@ void CompactionJob::LogCompaction() {
                   ? int64_t{-1}  // Use -1 for "none"
                   : static_cast<int64_t>(existing_snapshots_[0]));
    if (compaction->SupportsPerKeyPlacement()) {
-      stream << "preclude_last_level_min_seqno"
-             << preclude_last_level_min_seqno_;
+      stream << "prenultimate_after_seqno" << penultimate_after_seqno_;
+      stream << "preserve_seqno_after" << preserve_seqno_after_;
      stream << "penultimate_output_level" << compaction->GetPenultimateLevel();
      stream << "penultimate_output_range"
             << GetCompactionPenultimateOutputRangeTypeString(
@@ -2173,7 +2252,7 @@ void CompactionJob::LogCompaction() {
 }

 std::string CompactionJob::GetTableFileName(uint64_t file_number) {
-  return TableFileName(compact_->compaction->immutable_options()->cf_paths,
+  return TableFileName(compact_->compaction->immutable_options().cf_paths,
                       file_number, compact_->compaction->output_path_id());
 }

@@ -176,7 +176,13 @@ class CompactionJob {

  // REQUIRED: mutex held
  // Prepare for the compaction by setting up boundaries for each subcompaction
-  void Prepare();
+  // and organizing seqno <-> time info. `known_single_subcompact` is non-null
+  // if we already have a known single subcompaction, with optional key bounds
+  // (currently for executing a remote compaction).
+  void Prepare(
+      std::optional<std::pair<std::optional<Slice>, std::optional<Slice>>>
+          known_single_subcompact);
+
  // REQUIRED mutex not held
  // Launch threads for each subcompaction and wait for them to finish. After
  // that, verify table is usable and finally do bookkeeping to unify
@@ -187,8 +193,7 @@ class CompactionJob {
  // Add compaction input/output to the current version
  // Releases compaction file through Compaction::ReleaseCompactionFiles().
  // Sets *compaction_released to true if compaction is released.
-  Status Install(const MutableCFOptions& mutable_cf_options,
-                 bool* compaction_released);
+  Status Install(bool* compaction_released);

  // Return the IO status
  IOStatus io_status() const { return io_status_; }
@@ -276,8 +281,7 @@ class CompactionJob {
                                    const Slice& next_table_min_key,
                                    const Slice* comp_start_user_key,
                                    const Slice* comp_end_user_key);
-  Status InstallCompactionResults(const MutableCFOptions& mutable_cf_options,
-                                  bool* compaction_released);
+  Status InstallCompactionResults(bool* compaction_released);
  Status OpenCompactionOutputFile(SubcompactionState* sub_compact,
                                  CompactionOutputs& outputs);

@@ -353,15 +357,14 @@ class CompactionJob {
  // it also collects the smallest_seqno -> oldest_ancester_time from the SST.
  SeqnoToTimeMapping seqno_to_time_mapping_;

-  // Minimal sequence number for preserving the time information. The time info
-  // older than this sequence number won't be preserved after the compaction and
-  // if it's bottommost compaction, the seq num will be zeroed out.
-  SequenceNumber preserve_time_min_seqno_ = kMaxSequenceNumber;
+  // Max seqno that can be zeroed out in last level, including for preserving
+  // write times.
+  SequenceNumber preserve_seqno_after_ = kMaxSequenceNumber;

  // Minimal sequence number to preclude the data from the last level. If the
  // key has bigger (newer) sequence number than this, it will be precluded from
  // the last level (output to penultimate level).
-  SequenceNumber preclude_last_level_min_seqno_ = kMaxSequenceNumber;
+  SequenceNumber penultimate_after_seqno_ = kMaxSequenceNumber;

  // Get table file name in where it's outputting to, which should also be in
  // `output_directory_`.
@@ -496,6 +499,10 @@ class CompactionServiceCompactionJob : private CompactionJob {
      const CompactionServiceInput& compaction_service_input,
      CompactionServiceResult* compaction_service_result);

+  // REQUIRED: mutex held
+  // Like CompactionJob::Prepare()
+  void Prepare();
+
  // Run the compaction in current thread and return the result
  Status Run();

@@ -959,7 +959,6 @@ int main(int argc, char** argv) {
  return RUN_ALL_TESTS();
 }

-
 #else

 int main(int /*argc*/, char** /*argv*/) { return 0; }
@@ -3,7 +3,6 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).

-
 #include "db/compaction/compaction_job.h"

 #include <algorithm>
@@ -302,7 +301,7 @@ class CompactionJobTestBase : public testing::Test {
    std::unique_ptr<TableBuilder> table_builder(
        cf_options_.table_factory->NewTableBuilder(
            TableBuilderOptions(
-                *cfd_->ioptions(), mutable_cf_options_, read_options,
+                cfd_->ioptions(), mutable_cf_options_, read_options,
                write_options, cfd_->internal_comparator(),
                cfd_->internal_tbl_prop_coll_factories(),
                CompressionType::kNoCompression, CompressionOptions(),
@@ -398,8 +397,8 @@ class CompactionJobTestBase : public testing::Test {

    mutex_.Lock();
    EXPECT_OK(versions_->LogAndApply(
-        versions_->GetColumnFamilySet()->GetDefault(), mutable_cf_options_,
-        read_options_, write_options_, &edit, &mutex_, nullptr));
+        versions_->GetColumnFamilySet()->GetDefault(), read_options_,
+        write_options_, &edit, &mutex_, nullptr));
    mutex_.Unlock();
  }

@@ -461,7 +460,7 @@ class CompactionJobTestBase : public testing::Test {
      ReadOptions read_opts;
      Status s = cf_options_.table_factory->NewTableReader(
          read_opts,
-          TableReaderOptions(*cfd->ioptions(), nullptr, FileOptions(),
+          TableReaderOptions(cfd->ioptions(), nullptr, FileOptions(),
                             cfd_->internal_comparator(),
                             0 /* block_protection_bytes_per_key */),
          std::move(freader), file_size, &table_reader, false);
@@ -646,12 +645,12 @@ class CompactionJobTestBase : public testing::Test {
    }

    Compaction compaction(
-        cfd->current()->storage_info(), *cfd->ioptions(),
-        *cfd->GetLatestMutableCFOptions(), mutable_db_options_,
+        cfd->current()->storage_info(), cfd->ioptions(),
+        cfd->GetLatestMutableCFOptions(), mutable_db_options_,
        compaction_input_files, output_level,
        mutable_cf_options_.target_file_size_base,
        mutable_cf_options_.max_compaction_bytes, 0, kNoCompression,
-        cfd->GetLatestMutableCFOptions()->compression_opts,
+        cfd->GetLatestMutableCFOptions().compression_opts,
        Temperature::kUnknown, max_subcompactions, grandparents,
        /*earliest_snapshot*/ std::nullopt, /*snapshot_checker*/ nullptr, true);
    compaction.FinalizeInputInfo(cfd->current());
@@ -678,15 +677,14 @@ class CompactionJobTestBase : public testing::Test {
        full_history_ts_low_);
    VerifyInitializationOfCompactionJobStats(compaction_job_stats_);

-    compaction_job.Prepare();
+    compaction_job.Prepare(std::nullopt /*subcompact to be computed*/);
    mutex_.Unlock();
    Status s = compaction_job.Run();
    ASSERT_OK(s);
    ASSERT_OK(compaction_job.io_status());
    mutex_.Lock();
    bool compaction_released = false;
-    ASSERT_OK(compaction_job.Install(*cfd->GetLatestMutableCFOptions(),
-                                     &compaction_released));
+    ASSERT_OK(compaction_job.Install(&compaction_released));
    ASSERT_OK(compaction_job.io_status());
    mutex_.Unlock();
    log_buffer.FlushBufferToLog();
@@ -1536,14 +1534,11 @@ TEST_F(CompactionJobTest, VerifyPenultimateLevelOutput) {
      /*verify_func=*/[&](Compaction& comp) {
        for (char c = 'a'; c <= 'z'; c++) {
          if (c == 'a') {
-            ParsedInternalKey pik("a", 0U, kTypeValue);
-            ASSERT_FALSE(comp.WithinPenultimateLevelOutputRange(pik));
+            comp.TEST_AssertWithinPenultimateLevelOutputRange(
+                "a", true /*expect_failure*/);
          } else {
            std::string c_str{c};
-            // WithinPenultimateLevelOutputRange checks internal key range.
-            // 'z' is the last key, so set seqno properly.
-            ParsedInternalKey pik(c_str, c == 'z' ? 12U : 0U, kTypeValue);
-            ASSERT_TRUE(comp.WithinPenultimateLevelOutputRange(pik));
+            comp.TEST_AssertWithinPenultimateLevelOutputRange(c_str);
          }
        }
      });
@@ -1900,8 +1895,8 @@ TEST_F(CompactionJobTest, CutToSkipGrandparentFile) {
  const std::vector<int> input_levels = {0, 1};
  auto lvl0_files = cfd_->current()->storage_info()->LevelFiles(0);
  auto lvl1_files = cfd_->current()->storage_info()->LevelFiles(1);
-    RunCompaction({lvl0_files, lvl1_files}, input_levels,
-                  {expected_file1, expected_file2});
+  RunCompaction({lvl0_files, lvl1_files}, input_levels,
+                {expected_file1, expected_file2});
 }

 TEST_F(CompactionJobTest, CutToAlignGrandparentBoundary) {
@@ -1975,8 +1970,8 @@ TEST_F(CompactionJobTest, CutToAlignGrandparentBoundary) {
  const std::vector<int> input_levels = {0, 1};
  auto lvl0_files = cfd_->current()->storage_info()->LevelFiles(0);
  auto lvl1_files = cfd_->current()->storage_info()->LevelFiles(1);
-    RunCompaction({lvl0_files, lvl1_files}, input_levels,
-                  {expected_file1, expected_file2});
+  RunCompaction({lvl0_files, lvl1_files}, input_levels,
+                {expected_file1, expected_file2});
 }

 TEST_F(CompactionJobTest, CutToAlignGrandparentBoundarySameKey) {
@@ -2037,8 +2032,8 @@ TEST_F(CompactionJobTest, CutToAlignGrandparentBoundarySameKey) {
  for (int i = 80; i <= 100; i++) {
    snapshots.emplace_back(i);
  }
-    RunCompaction({lvl0_files, lvl1_files}, input_levels,
-                  {expected_file1, expected_file2}, snapshots);
+  RunCompaction({lvl0_files, lvl1_files}, input_levels,
+                {expected_file1, expected_file2}, snapshots);
 }

 TEST_F(CompactionJobTest, CutForMaxCompactionBytesSameKey) {
@@ -320,7 +320,7 @@ bool CompactionOutputs::ShouldStopBefore(const CompactionIterator& c_iter) {
    // More details, check PR #1963
    const size_t num_skippable_boundaries_crossed =
        being_grandparent_gap_ ? 2 : 3;
-    if (compaction_->immutable_options()->compaction_style ==
+    if (compaction_->immutable_options().compaction_style ==
            kCompactionStyleLevel &&
        num_grandparent_boundaries_crossed >=
            num_skippable_boundaries_crossed &&
@@ -341,7 +341,7 @@ bool CompactionOutputs::ShouldStopBefore(const CompactionIterator& c_iter) {
    // target file size. The test shows it can generate larger files than a
    // static threshold like 75% and has a similar write amplification
    // improvement.
-    if (compaction_->immutable_options()->compaction_style ==
+    if (compaction_->immutable_options().compaction_style ==
            kCompactionStyleLevel &&
        current_output_file_size_ >=
            ((compaction_->target_output_file_size() + 99) / 100) *
@@ -455,9 +455,11 @@ void SetMaxSeqAndTs(InternalKey& internal_key, const Slice& user_key,
 }  // namespace

 Status CompactionOutputs::AddRangeDels(
+    CompactionRangeDelAggregator& range_del_agg,
    const Slice* comp_start_user_key, const Slice* comp_end_user_key,
    CompactionIterationStats& range_del_out_stats, bool bottommost_level,
    const InternalKeyComparator& icmp, SequenceNumber earliest_snapshot,
+    std::pair<SequenceNumber, SequenceNumber> keep_seqno_range,
    const Slice& next_table_min_key, const std::string& full_history_ts_low) {
  // The following example does not happen since
  // CompactionOutput::ShouldStopBefore() always return false for the first
@@ -470,7 +472,7 @@ Status CompactionOutputs::AddRangeDels(
  // Then meta.smallest will be set to comp_start_user_key@seqno
  // and meta.largest will be set to comp_start_user_key@kMaxSequenceNumber
  // which violates the assumption that meta.smallest should be <= meta.largest.
-  assert(HasRangeDel());
+  assert(!range_del_agg.IsEmpty());
  FileMetaData& meta = current_output().meta;
  const Comparator* ucmp = icmp.user_comparator();
  InternalKey lower_bound_buf, upper_bound_buf;
@@ -579,13 +581,19 @@ Status CompactionOutputs::AddRangeDels(
  assert(comp_end_user_key == nullptr || upper_bound == nullptr ||
         ucmp->CompareWithoutTimestamp(ExtractUserKey(*upper_bound),
                                       *comp_end_user_key) <= 0);
-  auto it = range_del_agg_->NewIterator(lower_bound, upper_bound);
+  auto it = range_del_agg.NewIterator(lower_bound, upper_bound);
  Slice last_tombstone_start_user_key{};
  bool reached_lower_bound = false;
  const ReadOptions read_options(Env::IOActivity::kCompaction);
  for (it->SeekToFirst(); it->Valid(); it->Next()) {
    auto tombstone = it->Tombstone();
    auto kv = tombstone.Serialize();
+    // Filter out by seqno for per-key placement
+    if (tombstone.seq_ < keep_seqno_range.first ||
+        tombstone.seq_ >= keep_seqno_range.second) {
+      continue;
+    }
+
    InternalKey tombstone_end = tombstone.SerializeEndKey();
    // TODO: the underlying iterator should support clamping the bounds.
    // tombstone_end.Encode is of form user_key@kMaxSeqno
@@ -744,11 +752,10 @@ Status CompactionOutputs::AddRangeDels(
 }

 void CompactionOutputs::FillFilesToCutForTtl() {
-  if (compaction_->immutable_options()->compaction_style !=
+  if (compaction_->immutable_options().compaction_style !=
          kCompactionStyleLevel ||
-      compaction_->immutable_options()->compaction_pri !=
-          kMinOverlappingRatio ||
-      compaction_->mutable_cf_options()->ttl == 0 ||
+      compaction_->immutable_options().compaction_pri != kMinOverlappingRatio ||
+      compaction_->mutable_cf_options().ttl == 0 ||
      compaction_->num_input_levels() < 2 || compaction_->bottommost_level()) {
    return;
  }
@@ -756,20 +763,19 @@ void CompactionOutputs::FillFilesToCutForTtl() {
  // We define new file with the oldest ancestor time to be younger than 1/4
  // TTL, and an old one to be older than 1/2 TTL time.
  int64_t temp_current_time;
-  auto get_time_status =
-      compaction_->immutable_options()->clock->GetCurrentTime(
-          &temp_current_time);
+  auto get_time_status = compaction_->immutable_options().clock->GetCurrentTime(
+      &temp_current_time);
  if (!get_time_status.ok()) {
    return;
  }

  auto current_time = static_cast<uint64_t>(temp_current_time);
-  if (current_time < compaction_->mutable_cf_options()->ttl) {
+  if (current_time < compaction_->mutable_cf_options().ttl) {
    return;
  }

  uint64_t old_age_thres =
-      current_time - compaction_->mutable_cf_options()->ttl / 2;
+      current_time - compaction_->mutable_cf_options().ttl / 2;
  const std::vector<FileMetaData*>& olevel =
      *(compaction_->inputs(compaction_->num_input_levels() - 1));
  for (FileMetaData* file : olevel) {
@@ -779,7 +785,7 @@ void CompactionOutputs::FillFilesToCutForTtl() {
    // of small files.
    if (oldest_ancester_time < old_age_thres &&
        file->fd.GetFileSize() >
-            compaction_->mutable_cf_options()->target_file_size_base / 2) {
+            compaction_->mutable_cf_options().target_file_size_base / 2) {
      files_to_cut_for_ttl_.push_back(file);
    }
  }
@@ -45,6 +45,8 @@ class CompactionOutputs {
  explicit CompactionOutputs(const Compaction* compaction,
                             const bool is_penultimate_level);

+  bool IsPenultimateLevel() const { return is_penultimate_level_; }
+
  // Add generated output to the list
  void AddOutput(FileMetaData&& meta, const InternalKeyComparator& icmp,
                 bool enable_hash, bool finished = false,
@@ -182,18 +184,13 @@ class CompactionOutputs {
  // @param next_table_min_key internal key lower bound for the next compaction
  // output.
  // @param full_history_ts_low used for range tombstone garbage collection.
-  Status AddRangeDels(const Slice* comp_start_user_key,
-                      const Slice* comp_end_user_key,
-                      CompactionIterationStats& range_del_out_stats,
-                      bool bottommost_level, const InternalKeyComparator& icmp,
-                      SequenceNumber earliest_snapshot,
-                      const Slice& next_table_min_key,
-                      const std::string& full_history_ts_low);
-
-  // if the outputs have range delete, range delete is also data
-  bool HasRangeDel() const {
-    return range_del_agg_ && !range_del_agg_->IsEmpty();
-  }
+  Status AddRangeDels(
+      CompactionRangeDelAggregator& range_del_agg,
+      const Slice* comp_start_user_key, const Slice* comp_end_user_key,
+      CompactionIterationStats& range_del_out_stats, bool bottommost_level,
+      const InternalKeyComparator& icmp, SequenceNumber earliest_snapshot,
+      std::pair<SequenceNumber, SequenceNumber> keep_seqno_range,
+      const Slice& next_table_min_key, const std::string& full_history_ts_low);

 private:
  friend class SubcompactionState;
@@ -259,11 +256,15 @@ class CompactionOutputs {
  // Close the current output. `open_file_func` is needed for creating new file
  // for range-dels only output file.
  Status CloseOutput(const Status& curr_status,
+                     CompactionRangeDelAggregator* range_del_agg,
                     const CompactionFileOpenFunc& open_file_func,
                     const CompactionFileCloseFunc& close_file_func) {
    Status status = curr_status;
-    // handle subcompaction containing only range deletions
-    if (status.ok() && !HasBuilder() && !HasOutput() && HasRangeDel()) {
+    // Handle subcompaction containing only range deletions. They could
+    // be dropped or sent to another output level, so this is only an
+    // over-approximate check for whether opening is needed.
+    if (status.ok() && !HasBuilder() && !HasOutput() && range_del_agg &&
+        !range_del_agg->IsEmpty()) {
      status = open_file_func(*this);
    }
    if (HasBuilder()) {
@@ -288,16 +289,6 @@ class CompactionOutputs {
    return outputs_.back();
  }

-  // Assign the range_del_agg to the target output level. There's only one
-  // range-del-aggregator per compaction outputs, for
-  // output_to_penultimate_level compaction it is only assigned to the
-  // penultimate level.
-  void AssignRangeDelAggregator(
-      std::unique_ptr<CompactionRangeDelAggregator>&& range_del_agg) {
-    assert(range_del_agg_ == nullptr);
-    range_del_agg_ = std::move(range_del_agg);
-  }
-
  const Compaction* compaction_;

  // current output builder and writer
@@ -319,7 +310,6 @@ class CompactionOutputs {
  // indicate if this CompactionOutputs obj for penultimate_level, should always
  // be false if per_key_placement feature is not enabled.
  const bool is_penultimate_level_;
-  std::unique_ptr<CompactionRangeDelAggregator> range_del_agg_ = nullptr;

  // partitioner information
  std::string last_key_for_partitioner_;
@@ -721,14 +721,14 @@ Compaction* CompactionPicker::CompactRange(
  // two files overlap.
  if (input_level > 0) {
    const uint64_t limit = mutable_cf_options.max_compaction_bytes;
-    uint64_t input_level_total = 0;
    int hint_index = -1;
-    InternalKey* smallest = nullptr;
+    assert(!inputs.empty());
+    // Always include first file for progress.
+    uint64_t input_level_total = inputs[0]->fd.GetFileSize();
+    InternalKey* smallest = &(inputs[0]->smallest);
    InternalKey* largest = nullptr;
-    for (size_t i = 0; i + 1 < inputs.size(); ++i) {
-      if (!smallest) {
-        smallest = &inputs[i]->smallest;
-      }
+    for (size_t i = 1; i < inputs.size(); ++i) {
+      // Consider whether to include inputs[i]
      largest = &inputs[i]->largest;

      uint64_t input_file_size = inputs[i]->fd.GetFileSize();
@@ -744,13 +744,11 @@ Compaction* CompactionPicker::CompactRange(

      input_level_total += input_file_size;

-      if (input_level_total + output_level_total >= limit) {
+      if (input_level_total + output_level_total > limit) {
+        // To ensure compaction size is <= limit, leave out inputs from
+        // index i onwards.
        covering_the_whole_range = false;
-        // still include the current file, so the compaction could be larger
-        // than max_compaction_bytes, which is also to make sure the compaction
-        // can make progress even `max_compaction_bytes` is small (e.g. smaller
-        // than an SST file).
-        inputs.files.resize(i + 1);
+        inputs.files.resize(i);
        break;
      }
    }
@@ -819,6 +817,9 @@ Compaction* CompactionPicker::CompactRange(
    output_level = vstorage->base_level();
    assert(output_level > 0);
  }
+  for (int i = input_level + 1; i < output_level; i++) {
+    assert(vstorage->NumLevelFiles(i) == 0);
+  }
  output_level_inputs.level = output_level;
  if (input_level != output_level) {
    int parent_index = -1;
@@ -73,6 +73,8 @@ class CompactionPicker {
  // compaction_end will be set to nullptr.
  // Client is responsible for compaction_end storage -- when called,
  // *compaction_end should point to valid InternalKey!
+  // REQUIRES: If not compacting all levels (input_level == kCompactAllLevels),
+  // then levels between input_level and output_level should be empty.
  virtual Compaction* CompactRange(
      const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
      const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage,
@@ -2656,6 +2656,37 @@ TEST_F(CompactionPickerTest, HitCompactionLimitWhenAddFileFromInputLevel) {
  ASSERT_EQ(5U, compaction->input(1, 0)->fd.GetNumber());
 }

+TEST_F(CompactionPickerTest, CompactRangeMaxCompactionBytes) {
+  mutable_cf_options_.max_compaction_bytes = 800000U;
+  NewVersionStorage(6, kCompactionStyleLevel);
+  // We will first pick file 1 and 2 and then stop before file 3.
+  // Since picking file 3 will pull in file 4 and 5 from L2 and
+  // exceed max_compaction_bytes.
+  Add(1, 1U, "100", "110", 10000U);
+  Add(1, 2U, "200", "210", 10000U, 0, 0);
+  Add(1, 3U, "400", "410", 10000U, 0, 0);
+  Add(2, 4U, "300", "310", 400000U);
+  Add(2, 5U, "320", "330", 400000U);
+  UpdateVersionStorageInfo();
+
+  bool manual_conflict = false;
+  InternalKey manual_end;
+  InternalKey* manual_end_ptr = &manual_end;
+  std::unique_ptr<Compaction> compaction(level_compaction_picker.CompactRange(
+      cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(),
+      /*input_level=*/1, /*output_level=*/2,
+      /*compact_range_options*/ {}, /*begin=*/nullptr, /*end=*/nullptr,
+      &manual_end_ptr, &manual_conflict,
+      /*max_file_num_to_ignore=*/std::numeric_limits<uint64_t>::max(),
+      /*trim_ts=*/""));
+  ASSERT_TRUE(compaction.get() != nullptr);
+  ASSERT_EQ(1U, compaction->num_input_levels());
+  ASSERT_EQ(2, compaction->output_level());
+  ASSERT_EQ(2U, compaction->num_input_files(0));
+  ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
+  ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber());
+}
+
 TEST_F(CompactionPickerTest, IsTrivialMoveOn) {
  mutable_cf_options_.max_bytes_for_level_base = 10000u;
  mutable_cf_options_.max_compaction_bytes = 10001u;
@@ -177,7 +177,7 @@ CompactionJob::ProcessKeyValueCompactionWithCompactionService(
  for (const auto& file : compaction_result.output_files) {
    uint64_t file_num = versions_->NewFileNumber();
    auto src_file = compaction_result.output_path + "/" + file.file_name;
-    auto tgt_file = TableFileName(compaction->immutable_options()->cf_paths,
+    auto tgt_file = TableFileName(compaction->immutable_options().cf_paths,
                                  file_num, compaction->output_path_id());
    s = fs_->RenameFile(src_file, tgt_file, IOOptions(), nullptr);
    if (!s.ok()) {
@@ -189,6 +189,10 @@ CompactionJob::ProcessKeyValueCompactionWithCompactionService(

    FileMetaData meta;
    uint64_t file_size;
+    // FIXME: file_size should be part of CompactionServiceOutputFile so that
+    // we don't get DB corruption if the full file size has not been propagated
+    // back to the caller through the file system (which could have metadata
+    // lag or caching bugs).
    s = fs_->GetFileSize(tgt_file, IOOptions(), &file_size, nullptr);
    if (!s.ok()) {
      sub_compact->status = s;
@@ -265,43 +269,38 @@ CompactionServiceCompactionJob::CompactionServiceCompactionJob(
    std::string output_path,
    const CompactionServiceInput& compaction_service_input,
    CompactionServiceResult* compaction_service_result)
-    : CompactionJob(
-          job_id, compaction, db_options, mutable_db_options, file_options,
-          versions, shutting_down, log_buffer, nullptr, output_directory,
-          nullptr, stats, db_mutex, db_error_handler,
-          std::move(existing_snapshots), kMaxSequenceNumber, nullptr, nullptr,
-          std::move(table_cache), event_logger,
-          compaction->mutable_cf_options()->paranoid_file_checks,
-          compaction->mutable_cf_options()->report_bg_io_stats, dbname,
-          &(compaction_service_result->stats), Env::Priority::USER, io_tracer,
-          manual_compaction_canceled, db_id, db_session_id,
-          compaction->column_family_data()->GetFullHistoryTsLow()),
+    : CompactionJob(job_id, compaction, db_options, mutable_db_options,
+                    file_options, versions, shutting_down, log_buffer, nullptr,
+                    output_directory, nullptr, stats, db_mutex,
+                    db_error_handler, std::move(existing_snapshots),
+                    kMaxSequenceNumber, nullptr, nullptr,
+                    std::move(table_cache), event_logger,
+                    compaction->mutable_cf_options().paranoid_file_checks,
+                    compaction->mutable_cf_options().report_bg_io_stats, dbname,
+                    &(compaction_service_result->stats), Env::Priority::USER,
+                    io_tracer, manual_compaction_canceled, db_id, db_session_id,
+                    compaction->column_family_data()->GetFullHistoryTsLow()),
      output_path_(std::move(output_path)),
      compaction_input_(compaction_service_input),
      compaction_result_(compaction_service_result) {}

+void CompactionServiceCompactionJob::Prepare() {
+  std::optional<Slice> begin;
+  if (compaction_input_.has_begin) {
+    begin = compaction_input_.begin;
+  }
+  std::optional<Slice> end;
+  if (compaction_input_.has_end) {
+    end = compaction_input_.end;
+  }
+  CompactionJob::Prepare(std::make_pair(begin, end));
+}
+
 Status CompactionServiceCompactionJob::Run() {
  AutoThreadOperationStageUpdater stage_updater(
      ThreadStatus::STAGE_COMPACTION_RUN);

  auto* c = compact_->compaction;
-  assert(c->column_family_data() != nullptr);
-  const VersionStorageInfo* storage_info = c->input_version()->storage_info();
-  assert(storage_info);
-  assert(storage_info->NumLevelFiles(compact_->compaction->level()) > 0);
-  write_hint_ = storage_info->CalculateSSTWriteHint(c->output_level());
-
-  bottommost_level_ = c->bottommost_level();
-
-  Slice begin = compaction_input_.begin;
-  Slice end = compaction_input_.end;
-  compact_->sub_compact_states.emplace_back(
-      c,
-      compaction_input_.has_begin ? std::optional<Slice>(begin)
-                                  : std::optional<Slice>(),
-      compaction_input_.has_end ? std::optional<Slice>(end)
-                                : std::optional<Slice>(),
-      /*sub_job_id*/ 0);

  log_buffer_->FlushBufferToLog();
  LogCompaction();
@@ -114,6 +114,8 @@ class MyTestCompactionService : public CompactionService {
    }
  }

+  void CancelAwaitingJobs() override { canceled_ = true; }
+
  void OnInstallation(const std::string& /*scheduled_job_id*/,
                      CompactionServiceJobStatus status) override {
    final_updated_status_ = status;
@@ -146,6 +148,7 @@ class MyTestCompactionService : public CompactionService {
  }

  void SetCanceled(bool canceled) { canceled_ = canceled; }
+  bool GetCanceled() { return canceled_; }

  void GetResult(CompactionServiceResult* deserialized) {
    CompactionServiceResult::Read(result_, deserialized).PermitUncheckedError();
@@ -194,12 +197,15 @@ class CompactionServiceTest : public DBTestBase {
    options->statistics = primary_statistics_;
    compactor_statistics_ = CreateDBStatistics();

-    compaction_service_ = std::make_shared<MyTestCompactionService>(
+    auto my_cs = std::make_shared<MyTestCompactionService>(
        dbname_, *options, compactor_statistics_, remote_listeners,
        remote_table_properties_collector_factories);
+
+    compaction_service_ = my_cs;
    options->compaction_service = compaction_service_;
    DestroyAndReopen(*options);
    CreateAndReopenWithCF({"cf_1", "cf_2", "cf_3"}, *options);
+    my_cs->SetCanceled(false);
  }

  Statistics* GetCompactorStatistics() { return compactor_statistics_.get(); }
@@ -339,6 +345,8 @@ TEST_F(CompactionServiceTest, BasicCompactions) {
        assert(*id != kNullUniqueId64x2);
        verify_passed++;
      });
+  Close();
+  my_cs->SetCanceled(false);
  ReopenWithColumnFamilies({kDefaultColumnFamilyName, "cf_1", "cf_2", "cf_3"},
                           options);
  ASSERT_GT(verify_passed, 0);
@@ -487,7 +495,7 @@ TEST_F(CompactionServiceTest, PreservedOptionsLocalCompaction) {
        ASSERT_TRUE(s.IsNotFound());
        // Should be old value
        ASSERT_EQ(2, compaction->mutable_cf_options()
-                         ->level0_file_num_compaction_trigger);
+                         .level0_file_num_compaction_trigger);
        ASSERT_TRUE(dbfull()->min_options_file_numbers_.empty());
      });

@@ -573,7 +581,7 @@ TEST_F(CompactionServiceTest, PreservedOptionsRemoteCompaction) {
      "CompactionJob::ProcessKeyValueCompaction()::Processing", [&](void* arg) {
        auto compaction = static_cast<Compaction*>(arg);
        ASSERT_EQ(2, compaction->mutable_cf_options()
-                         ->level0_file_num_compaction_trigger);
+                         .level0_file_num_compaction_trigger);
      });

  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
@@ -996,6 +1004,41 @@ TEST_F(CompactionServiceTest, CancelCompactionOnRemoteSide) {
  VerifyTestData();
 }

+TEST_F(CompactionServiceTest, CancelCompactionOnPrimarySide) {
+  Options options = CurrentOptions();
+  options.disable_auto_compactions = true;
+  ReopenWithCompactionService(&options);
+  GenerateTestData();
+
+  auto my_cs = GetCompactionService();
+
+  std::string start_str = Key(15);
+  std::string end_str = Key(45);
+  Slice start(start_str);
+  Slice end(end_str);
+  uint64_t comp_num = my_cs->GetCompactionNum();
+
+  ReopenWithCompactionService(&options);
+  GenerateTestData();
+  my_cs = GetCompactionService();
+
+  // Primary DB calls CancelAllBackgroundWork() while the compaction is running
+  SyncPoint::GetInstance()->SetCallBack(
+      "CompactionJob::Run():Inprogress",
+      [&](void* /*arg*/) { CancelAllBackgroundWork(db_, false /*wait*/); });
+
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  Status s = db_->CompactRange(CompactRangeOptions(), &start, &end);
+  ASSERT_TRUE(s.IsIncomplete());
+
+  // Check canceled_ was set to true by CancelAwaitingJobs()
+  ASSERT_TRUE(my_cs->GetCanceled());
+
+  // compaction number is not increased
+  ASSERT_GE(my_cs->GetCompactionNum(), comp_num);
+}
+
 TEST_F(CompactionServiceTest, FailedToStart) {
  Options options = CurrentOptions();
  options.disable_auto_compactions = true;
@@ -1118,6 +1161,63 @@ TEST_F(CompactionServiceTest, Snapshot) {
  db_->ReleaseSnapshot(s1);
 }

+TEST_F(CompactionServiceTest, PrecludeLastLevel) {
+  const int kNumTrigger = 4;
+  const int kNumLevels = 7;
+  const int kNumKeys = 100;
+
+  Options options = CurrentOptions();
+  options.compaction_style = kCompactionStyleUniversal;
+  options.last_level_temperature = Temperature::kCold;
+  options.level0_file_num_compaction_trigger = 4;
+  options.max_subcompactions = 10;
+  options.num_levels = kNumLevels;
+
+  ReopenWithCompactionService(&options);
+  // Alternate for comparison: DestroyAndReopen(options);
+
+  // This is simpler than setting up mock time to make the user option work,
+  // but is not as direct as testing with preclude option itself.
+  SyncPoint::GetInstance()->SetCallBack(
+      "Compaction::SupportsPerKeyPlacement:Enabled",
+      [&](void* arg) { *static_cast<bool*>(arg) = true; });
+  SyncPoint::GetInstance()->SetCallBack(
+      "CompactionJob::PrepareTimes():preclude_last_level_min_seqno",
+      [&](void* arg) { *static_cast<SequenceNumber*>(arg) = 100; });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  for (int i = 0; i < kNumTrigger; i++) {
+    for (int j = 0; j < kNumKeys; j++) {
+      // FIXME: need to assign outputs to levels to allow overlapping ranges:
+      // ASSERT_OK(Put(Key(j * kNumTrigger + i), "v" + std::to_string(i)));
+      // instead of this (too easy):
+      ASSERT_OK(Put(Key(i * kNumKeys + j), "v" + std::to_string(i)));
+    }
+    ASSERT_OK(Flush());
+  }
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+
+  // Data split between penultimate (kUnknown) and last (kCold) levels
+  // FIXME: need to assign outputs to levels to get this:
+  // ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel());
+  // ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0);
+  // ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
+  // instead of this (WRONG but currently expected):
+  ASSERT_EQ("0,0,0,0,0,0,2", FilesPerLevel());
+  // Check manifest temperatures
+  ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0);
+  ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0);
+  // TODO: Check FileSystem temperatures with FileTemperatureTestFS
+
+  for (int i = 0; i < kNumTrigger; i++) {
+    for (int j = 0; j < kNumKeys; j++) {
+      // FIXME
+      // ASSERT_EQ(Get(Key(j * kNumTrigger + i)), "v" + std::to_string(i));
+      ASSERT_EQ(Get(Key(i * kNumKeys + j)), "v" + std::to_string(i));
+    }
+  }
+}
+
 TEST_F(CompactionServiceTest, ConcurrentCompaction) {
  Options options = CurrentOptions();
  options.level0_file_num_compaction_trigger = 100;
@@ -15,8 +15,19 @@
 namespace ROCKSDB_NAMESPACE {
 void SubcompactionState::AggregateCompactionOutputStats(
    InternalStats::CompactionStatsFull& compaction_stats) const {
+  // Outputs should be closed. By extension, any files created just for
+  // range deletes have already been written also.
+  assert(compaction_outputs_.HasBuilder() == false);
+  assert(penultimate_level_outputs_.HasBuilder() == false);
+
+  // FIXME: These stats currently include abandonned output files
+  // assert(compaction_outputs_.stats_.num_output_files ==
+  //        compaction_outputs_.outputs_.size());
+  // assert(penultimate_level_outputs_.stats_.num_output_files ==
+  //        penultimate_level_outputs_.outputs_.size());
+
  compaction_stats.stats.Add(compaction_outputs_.stats_);
-  if (HasPenultimateLevelOutputs()) {
+  if (penultimate_level_outputs_.HasOutput()) {
    compaction_stats.has_penultimate_level_output = true;
    compaction_stats.penultimate_level_stats.Add(
        penultimate_level_outputs_.stats_);
@@ -43,7 +54,7 @@ void SubcompactionState::Cleanup(Cache* cache) {
      // be picked up by scanning the DB directory.
      TableCache::ReleaseObsolete(
          cache, out.meta.fd.GetNumber(), nullptr /*handle*/,
-          compaction->mutable_cf_options()->uncache_aggressiveness);
+          compaction->mutable_cf_options().uncache_aggressiveness);
    }
  }
  // TODO: sub_compact.io_status is not checked like status. Not sure if thats
@@ -52,7 +63,7 @@ void SubcompactionState::Cleanup(Cache* cache) {
 }

 Slice SubcompactionState::SmallestUserKey() const {
-  if (has_penultimate_level_outputs_) {
+  if (penultimate_level_outputs_.HasOutput()) {
    Slice a = compaction_outputs_.SmallestUserKey();
    Slice b = penultimate_level_outputs_.SmallestUserKey();
    if (a.empty()) {
@@ -74,7 +85,7 @@ Slice SubcompactionState::SmallestUserKey() const {
 }

 Slice SubcompactionState::LargestUserKey() const {
-  if (has_penultimate_level_outputs_) {
+  if (penultimate_level_outputs_.HasOutput()) {
    Slice a = compaction_outputs_.LargestUserKey();
    Slice b = penultimate_level_outputs_.LargestUserKey();
    if (a.empty()) {
@@ -96,18 +107,13 @@ Slice SubcompactionState::LargestUserKey() const {
 }

 Status SubcompactionState::AddToOutput(
-    const CompactionIterator& iter,
+    const CompactionIterator& iter, bool use_penultimate_output,
    const CompactionFileOpenFunc& open_file_func,
    const CompactionFileCloseFunc& close_file_func) {
-  // update target output first
-  is_current_penultimate_level_ = iter.output_to_penultimate_level();
-  current_outputs_ = is_current_penultimate_level_ ? &penultimate_level_outputs_
-                                                   : &compaction_outputs_;
-  if (is_current_penultimate_level_) {
-    has_penultimate_level_outputs_ = true;
-  }
-
-  return Current().AddToOutput(iter, open_file_func, close_file_func);
+  // update target output
+  current_outputs_ = use_penultimate_output ? &penultimate_level_outputs_
+                                            : &compaction_outputs_;
+  return current_outputs_->AddToOutput(iter, open_file_func, close_file_func);
 }

 }  // namespace ROCKSDB_NAMESPACE
@@ -81,22 +81,13 @@ class SubcompactionState {
  // it returns both the last level outputs and penultimate level outputs.
  OutputIterator GetOutputs() const;

-  // Assign range dels aggregator, for each range_del, it can only be assigned
-  // to one output level, for per_key_placement, it's going to be the
-  // penultimate level.
-  // TODO: This does not work for per_key_placement + user-defined timestamp +
-  //  DeleteRange() combo. If user-defined timestamp is enabled,
-  //  it is possible for a range tombstone to belong to bottommost level (
-  //  seqno < earliest snapshot) without being dropped (garbage collection
-  //  for user-defined timestamp).
+  // Assign range dels aggregator. The various tombstones will potentially
+  // be filtered to different outputs.
  void AssignRangeDelAggregator(
      std::unique_ptr<CompactionRangeDelAggregator>&& range_del_agg) {
-    if (compaction->SupportsPerKeyPlacement()) {
-      penultimate_level_outputs_.AssignRangeDelAggregator(
-          std::move(range_del_agg));
-    } else {
-      compaction_outputs_.AssignRangeDelAggregator(std::move(range_del_agg));
-    }
+    assert(range_del_agg_ == nullptr);
+    assert(range_del_agg);
+    range_del_agg_ = std::move(range_del_agg);
  }

  void RemoveLastEmptyOutput() {
@@ -151,20 +142,11 @@ class SubcompactionState {
        sub_job_id(state.sub_job_id),
        compaction_outputs_(std::move(state.compaction_outputs_)),
        penultimate_level_outputs_(std::move(state.penultimate_level_outputs_)),
-        is_current_penultimate_level_(state.is_current_penultimate_level_),
-        has_penultimate_level_outputs_(state.has_penultimate_level_outputs_) {
-    current_outputs_ = is_current_penultimate_level_
-                           ? &penultimate_level_outputs_
-                           : &compaction_outputs_;
-  }
-
-  bool HasPenultimateLevelOutputs() const {
-    return has_penultimate_level_outputs_ ||
-           penultimate_level_outputs_.HasRangeDel();
-  }
-
-  bool IsCurrentPenultimateLevel() const {
-    return is_current_penultimate_level_;
+        range_del_agg_(std::move(state.range_del_agg_)) {
+    current_outputs_ =
+        state.current_outputs_ == &state.penultimate_level_outputs_
+            ? &penultimate_level_outputs_
+            : &compaction_outputs_;
  }

  // Add all the new files from this compaction to version_edit
@@ -187,8 +169,18 @@ class SubcompactionState {
    return *current_outputs_;
  }

+  CompactionRangeDelAggregator* RangeDelAgg() const {
+    return range_del_agg_.get();
+  }
+
+  // if the outputs have range delete, range delete is also data
+  bool HasRangeDel() const {
+    return range_del_agg_ && !range_del_agg_->IsEmpty();
+  }
+
  // Add compaction_iterator key/value to the `Current` output group.
  Status AddToOutput(const CompactionIterator& iter,
+                     bool use_penultimate_output,
                     const CompactionFileOpenFunc& open_file_func,
                     const CompactionFileCloseFunc& close_file_func);

@@ -197,14 +189,20 @@ class SubcompactionState {
  Status CloseCompactionFiles(const Status& curr_status,
                              const CompactionFileOpenFunc& open_file_func,
                              const CompactionFileCloseFunc& close_file_func) {
+    auto per_key = compaction->SupportsPerKeyPlacement();
    // Call FinishCompactionOutputFile() even if status is not ok: it needs to
    // close the output file.
    // CloseOutput() may open new compaction output files.
-    is_current_penultimate_level_ = true;
-    Status s = penultimate_level_outputs_.CloseOutput(
-        curr_status, open_file_func, close_file_func);
-    is_current_penultimate_level_ = false;
-    s = compaction_outputs_.CloseOutput(s, open_file_func, close_file_func);
+    Status s = curr_status;
+    if (per_key) {
+      s = penultimate_level_outputs_.CloseOutput(
+          s, range_del_agg_.get(), open_file_func, close_file_func);
+    } else {
+      assert(penultimate_level_outputs_.HasBuilder() == false);
+      assert(penultimate_level_outputs_.HasOutput() == false);
+    }
+    s = compaction_outputs_.CloseOutput(s, range_del_agg_.get(), open_file_func,
+                                        close_file_func);
    return s;
  }

@@ -213,8 +211,7 @@ class SubcompactionState {
  CompactionOutputs compaction_outputs_;
  CompactionOutputs penultimate_level_outputs_;
  CompactionOutputs* current_outputs_ = &compaction_outputs_;
-  bool is_current_penultimate_level_ = false;
-  bool has_penultimate_level_outputs_ = false;
+  std::unique_ptr<CompactionRangeDelAggregator> range_del_agg_;
 };

 }  // namespace ROCKSDB_NAMESPACE
@@ -195,10 +195,9 @@ TEST_F(TieredCompactionTest, SequenceBasedTieredStorageUniversal) {
  std::vector<SequenceNumber> seq_history;

  SyncPoint::GetInstance()->SetCallBack(
-      "CompactionIterator::PrepareOutput.context", [&](void* arg) {
-        auto context = static_cast<PerKeyPlacementContext*>(arg);
-        context->output_to_penultimate_level =
-            context->seq_num > latest_cold_seq;
+      "CompactionJob::PrepareTimes():preclude_last_level_min_seqno",
+      [&](void* arg) {
+        *static_cast<SequenceNumber*>(arg) = latest_cold_seq.load();
      });
  SyncPoint::GetInstance()->EnableProcessing();

@@ -339,7 +338,10 @@ TEST_F(TieredCompactionTest, SequenceBasedTieredStorageUniversal) {
  ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
 }

-TEST_F(TieredCompactionTest, RangeBasedTieredStorageUniversal) {
+// This test was essentially for a hacked-up version on future functionality.
+// It can be resurrected if/when a form of range-based tiering is properly
+// implemented.
+TEST_F(TieredCompactionTest, DISABLED_RangeBasedTieredStorageUniversal) {
  const int kNumTrigger = 4;
  const int kNumLevels = 7;
  const int kNumKeys = 100;
@@ -440,12 +442,12 @@ TEST_F(TieredCompactionTest, RangeBasedTieredStorageUniversal) {
  {
    MutexLock l(&mutex);
    hot_start = Key(1);
-    hot_end = Key(1000);
+    hot_end = Key(300);
  }

  // generate files just enough to trigger compaction
  for (int i = 0; i < kNumTrigger - 1; i++) {
-    for (int j = 0; j < 1000; j++) {
+    for (int j = 0; j < 300; j++) {
      ASSERT_OK(Put(Key(j), "value" + std::to_string(j)));
    }
    ASSERT_OK(Flush());
@@ -494,6 +496,7 @@ TEST_F(TieredCompactionTest, RangeBasedTieredStorageUniversal) {
  // verify data
  std::string value;
  for (int i = 0; i < kNumKeys; i++) {
+    SCOPED_TRACE(Key(i));
    if (i < 70) {
      ASSERT_TRUE(db_->Get(ReadOptions(), Key(i), &value).IsNotFound());
    } else {
@@ -527,13 +530,13 @@ TEST_F(TieredCompactionTest, LevelColdRangeDelete) {
  options.max_subcompactions = 10;
  DestroyAndReopen(options);

-  std::atomic_uint64_t latest_cold_seq = 0;
+  // Initially let everything into cold
+  std::atomic_uint64_t latest_cold_seq = kMaxSequenceNumber;

  SyncPoint::GetInstance()->SetCallBack(
-      "CompactionIterator::PrepareOutput.context", [&](void* arg) {
-        auto context = static_cast<PerKeyPlacementContext*>(arg);
-        context->output_to_penultimate_level =
-            context->seq_num > latest_cold_seq;
+      "CompactionJob::PrepareTimes():preclude_last_level_min_seqno",
+      [&](void* arg) {
+        *static_cast<SequenceNumber*>(arg) = latest_cold_seq.load();
      });
  SyncPoint::GetInstance()->EnableProcessing();

@@ -634,13 +637,13 @@ TEST_F(TieredCompactionTest, LevelOutofBoundaryRangeDelete) {
  options.max_subcompactions = 10;
  DestroyAndReopen(options);

-  std::atomic_uint64_t latest_cold_seq = 0;
+  // Initially let everything into cold
+  std::atomic_uint64_t latest_cold_seq = kMaxSequenceNumber;

  SyncPoint::GetInstance()->SetCallBack(
-      "CompactionIterator::PrepareOutput.context", [&](void* arg) {
-        auto context = static_cast<PerKeyPlacementContext*>(arg);
-        context->output_to_penultimate_level =
-            context->seq_num > latest_cold_seq;
+      "CompactionJob::PrepareTimes():preclude_last_level_min_seqno",
+      [&](void* arg) {
+        *static_cast<SequenceNumber*>(arg) = latest_cold_seq.load();
      });
  SyncPoint::GetInstance()->EnableProcessing();

@@ -654,6 +657,8 @@ TEST_F(TieredCompactionTest, LevelOutofBoundaryRangeDelete) {
  ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
  ASSERT_EQ("0,0,10", FilesPerLevel());

+  // Stop admitting to cold tier
+  latest_cold_seq = dbfull()->GetLatestSequenceNumber();
  auto snap = db_->GetSnapshot();

  // only range delete
@@ -767,10 +772,9 @@ TEST_F(TieredCompactionTest, UniversalRangeDelete) {
  std::atomic_uint64_t latest_cold_seq = 0;

  SyncPoint::GetInstance()->SetCallBack(
-      "CompactionIterator::PrepareOutput.context", [&](void* arg) {
-        auto context = static_cast<PerKeyPlacementContext*>(arg);
-        context->output_to_penultimate_level =
-            context->seq_num > latest_cold_seq;
+      "CompactionJob::PrepareTimes():preclude_last_level_min_seqno",
+      [&](void* arg) {
+        *static_cast<SequenceNumber*>(arg) = latest_cold_seq.load();
      });
  SyncPoint::GetInstance()->EnableProcessing();

@@ -894,14 +898,13 @@ TEST_F(TieredCompactionTest, SequenceBasedTieredStorageLevel) {
  options.max_subcompactions = 10;
  DestroyAndReopen(options);

-  std::atomic_uint64_t latest_cold_seq = 0;
+  std::atomic_uint64_t latest_cold_seq = kMaxSequenceNumber;
  std::vector<SequenceNumber> seq_history;

  SyncPoint::GetInstance()->SetCallBack(
-      "CompactionIterator::PrepareOutput.context", [&](void* arg) {
-        auto context = static_cast<PerKeyPlacementContext*>(arg);
-        context->output_to_penultimate_level =
-            context->seq_num > latest_cold_seq;
+      "CompactionJob::PrepareTimes():preclude_last_level_min_seqno",
+      [&](void* arg) {
+        *static_cast<SequenceNumber*>(arg) = latest_cold_seq.load();
      });
  SyncPoint::GetInstance()->EnableProcessing();

@@ -952,6 +955,7 @@ TEST_F(TieredCompactionTest, SequenceBasedTieredStorageLevel) {
  VerifyCompactionStats(expect_stats, expect_pl_stats);

  // Add new data, which is all hot and overriding all existing data
+  latest_cold_seq = dbfull()->GetLatestSequenceNumber();
  for (int i = 0; i < kNumTrigger; i++) {
    for (int j = 0; j < kNumKeys; j++) {
      ASSERT_OK(Put(Key(i * 10 + j), "value" + std::to_string(i)));
@@ -1104,7 +1108,13 @@ TEST_F(TieredCompactionTest, SequenceBasedTieredStorageLevel) {
  ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
 }

-TEST_F(TieredCompactionTest, RangeBasedTieredStorageLevel) {
+// This test was essentially for a hacked-up version on future functionality.
+// It can be resurrected if/when a form of range-based tiering is properly
+// implemented.
+// FIXME: aside from that, this test reproduces a near-endless compaction
+// cycle that needs to be reproduced independently and fixed before
+// leveled compaction can be used with the preclude feature in production.
+TEST_F(TieredCompactionTest, DISABLED_RangeBasedTieredStorageLevel) {
  const int kNumTrigger = 4;
  const int kNumLevels = 7;
  const int kNumKeys = 100;
@@ -1213,6 +1223,9 @@ TEST_F(TieredCompactionTest, RangeBasedTieredStorageLevel) {

  // Tests that we only compact keys up to penultimate level
  // that are within penultimate level input's internal key range.
+  // UPDATE: this functionality has changed. With penultimate-enabled
+  // compaction, the expanded potential output range in the penultimate
+  // level is reserved so should be safe to use.
  {
    MutexLock l(&mutex);
    hot_start = Key(0);
@@ -1221,7 +1234,6 @@ TEST_F(TieredCompactionTest, RangeBasedTieredStorageLevel) {
  const Snapshot* temp_snap = db_->GetSnapshot();
  // Key(0) and Key(1) here are inserted with higher sequence number
  // than Key(0) and Key(1) inserted above.
-  // Only Key(0) in last level will be compacted up, not Key(1).
  ASSERT_OK(Put(Key(0), "value" + std::to_string(0)));
  ASSERT_OK(Put(Key(1), "value" + std::to_string(100)));
  ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
@@ -1231,14 +1243,17 @@ TEST_F(TieredCompactionTest, RangeBasedTieredStorageLevel) {
    db_->GetLiveFilesMetaData(&metas);
    for (const auto& f : metas) {
      if (f.temperature == Temperature::kUnknown) {
-        // Expect Key(0), Key(0), Key(1)
-        ASSERT_EQ(f.num_entries, 3);
+        // UPDATED (was 3 entries, Key0..Key1)
+        ASSERT_EQ(f.num_entries, 52);
        ASSERT_EQ(f.smallestkey, Key(0));
-        ASSERT_EQ(f.largestkey, Key(1));
+        ASSERT_EQ(f.largestkey, Key(49));
      } else {
        ASSERT_EQ(f.temperature, Temperature::kCold);
-        // Key(2)-Key(49) and Key(100).
-        ASSERT_EQ(f.num_entries, 50);
+        // UPDATED (was 50 entries, Key0..Key49)
+        // Key(100) is outside the hot range
+        ASSERT_EQ(f.num_entries, 1);
+        ASSERT_EQ(f.smallestkey, Key(100));
+        ASSERT_EQ(f.largestkey, Key(100));
      }
    }
  }
@@ -1279,8 +1294,6 @@ class PrecludeLastLevelTestBase : public DBTestBase {
      const std::unordered_map<std::string, std::string>& db_config_change) {
    if (dynamic) {
      if (config_change.size() > 0) {
-        // FIXME: temporary while preserve/preclude options are not user mutable
-        SaveAndRestore<bool> m(&TEST_allowSetOptionsImmutableInMutable, true);
        ASSERT_OK(db_->SetOptions(config_change));
      }
      if (db_config_change.size() > 0) {
@@ -1616,7 +1629,8 @@ TEST_P(PrecludeLastLevelTest, CheckInternalKeyRange) {

  ASSERT_EQ("0,0,0,0,0,2,2", FilesPerLevel());

-  auto VerifyLogicalState = [&]() {
+  auto VerifyLogicalState = [&](int line) {
+    SCOPED_TRACE("Called from line " + std::to_string(line));
    // First with snapshot
    ASSERT_EQ("val2", Get(Key(2), snapshot));
    ASSERT_EQ("val3", Get(Key(3), snapshot));
@@ -1630,7 +1644,7 @@ TEST_P(PrecludeLastLevelTest, CheckInternalKeyRange) {
    ASSERT_EQ("val100", Get(Key(100)));
  };

-  VerifyLogicalState();
+  VerifyLogicalState(__LINE__);

  // Try to compact keys up
  CompactRangeOptions cro;
@@ -1641,24 +1655,40 @@ TEST_P(PrecludeLastLevelTest, CheckInternalKeyRange) {
  // type:1 vs. file #15 smallest key: '6B6579303030303033' seq:104, type:1
  ASSERT_OK(CompactRange(cro, Key(1), Key(2)));

-  VerifyLogicalState();
+  VerifyLogicalState(__LINE__);

  db_->ReleaseSnapshot(snapshot);
  Close();
 }

-TEST_P(PrecludeLastLevelTest, RangeTombstoneSnapshotMigrateFromLast) {
+INSTANTIATE_TEST_CASE_P(PrecludeLastLevelTest, PrecludeLastLevelTest,
+                        ::testing::Bool());
+
+class PrecludeWithCompactStyleTest : public PrecludeLastLevelTestBase,
+                                     public testing::WithParamInterface<bool> {
+ public:
+  void ApplyConfigChange(
+      Options* options,
+      const std::unordered_map<std::string, std::string>& config_change,
+      const std::unordered_map<std::string, std::string>& db_config_change =
+          {}) {
+    // Depends on dynamic config change while holding a snapshot
+    ApplyConfigChangeImpl(true /*dynamic*/, options, config_change,
+                          db_config_change);
+  }
+};
+
+TEST_P(PrecludeWithCompactStyleTest, RangeTombstoneSnapshotMigrateFromLast) {
  // Reproducer for issue originally described in
  // https://github.com/facebook/rocksdb/pull/9964/files#r1024449523
-  if (!UseDynamicConfig()) {
-    // Depends on config change while holding a snapshot
-    return;
-  }
+  const bool universal = GetParam();
  const int kNumLevels = 7;
  auto options = CurrentOptions();
  options.env = mock_env_.get();
  options.last_level_temperature = Temperature::kCold;
-  options.level_compaction_dynamic_level_bytes = true;
+  options.compaction_style =
+      universal ? kCompactionStyleUniversal : kCompactionStyleLevel;
+  options.compaction_options_universal.allow_trivial_move = true;
  options.num_levels = kNumLevels;
  options.statistics = CreateDBStatistics();
  options.max_subcompactions = 10;
@@ -1678,7 +1708,13 @@ TEST_P(PrecludeLastLevelTest, RangeTombstoneSnapshotMigrateFromLast) {
  ASSERT_OK(db_->DeleteRange({}, db_->DefaultColumnFamily(), Key(1), Key(5)));
  ASSERT_OK(Put(Key(1), "val1"));
  ASSERT_OK(Flush());
-  MoveFilesToLevel(6);
+
+  // Send to last level
+  if (universal) {
+    ASSERT_OK(CompactRange({}, {}, {}));
+  } else {
+    MoveFilesToLevel(6);
+  }
  ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel());

  ApplyConfigChange(&options, {{"preclude_last_level_data_seconds", "10000"}});
@@ -1697,10 +1733,18 @@ TEST_P(PrecludeLastLevelTest, RangeTombstoneSnapshotMigrateFromLast) {
      [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(1000)); });

  ASSERT_OK(Flush());
-  MoveFilesToLevel(5);
+
+  // Send three files to next-to-last level (if explicitly needed)
+  if (universal) {
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
+  } else {
+    MoveFilesToLevel(5);
+  }
+
  ASSERT_EQ("0,0,0,0,0,3,1", FilesPerLevel());

-  auto VerifyLogicalState = [&]() {
+  auto VerifyLogicalState = [&](int line) {
+    SCOPED_TRACE("Called from line " + std::to_string(line));
    // First with snapshot
    if (snapshot) {
      ASSERT_EQ("NOT_FOUND", Get(Key(0), snapshot));
@@ -1720,24 +1764,31 @@ TEST_P(PrecludeLastLevelTest, RangeTombstoneSnapshotMigrateFromLast) {
    ASSERT_EQ("val7", Get(Key(7)));
  };

-  VerifyLogicalState();
+  VerifyLogicalState(__LINE__);

  // Try a limited range compaction
-  // FIXME: this currently hits the "Unsafe to store Seq later than snapshot"
-  // error. Needs to work safely for preclude option to be user mutable.
-  // ASSERT_OK(CompactRange({}, Key(3), Key(4)));
+  // (These would previously hit "Unsafe to store Seq later than snapshot")
+  if (universal) {
+    uint64_t middle_l5 = GetLevelFileMetadatas(5)[1]->fd.GetNumber();
+    ASSERT_OK(db_->CompactFiles({}, {MakeTableFileName(middle_l5)}, 6));
+  } else {
+    ASSERT_OK(CompactRange({}, Key(3), Key(4)));
+  }
  EXPECT_EQ("0,0,0,0,0,3,1", FilesPerLevel());
-  VerifyLogicalState();
+  VerifyLogicalState(__LINE__);

  // Compact everything, but some data still goes to both penultimate and last
  // levels. A full-range compaction should be safe to "migrate" data from the
  // last level to penultimate (because of preclude setting change).
  ASSERT_OK(CompactRange({}, {}, {}));
  EXPECT_EQ("0,0,0,0,0,1,1", FilesPerLevel());
-  VerifyLogicalState();
+  VerifyLogicalState(__LINE__);
  // Key1 should have been migrated out of the last level
-  auto& meta = *GetLevelFileMetadatas(6)[0];
-  ASSERT_LT(Key(1), meta.smallest.user_key().ToString());
+  // FIXME: doesn't yet work with leveled compaction
+  if (universal) {
+    auto& meta = *GetLevelFileMetadatas(6)[0];
+    ASSERT_LT(Key(1), meta.smallest.user_key().ToString());
+  }

  // Make data eligible for last level
  db_->ReleaseSnapshot(snapshot);
@@ -1746,13 +1797,13 @@ TEST_P(PrecludeLastLevelTest, RangeTombstoneSnapshotMigrateFromLast) {

  ASSERT_OK(CompactRange({}, {}, {}));
  EXPECT_EQ("0,0,0,0,0,0,1", FilesPerLevel());
-  VerifyLogicalState();
+  VerifyLogicalState(__LINE__);

  Close();
 }

-INSTANTIATE_TEST_CASE_P(PrecludeLastLevelTest, PrecludeLastLevelTest,
-                        ::testing::Bool());
+INSTANTIATE_TEST_CASE_P(PrecludeWithCompactStyleTest,
+                        PrecludeWithCompactStyleTest, ::testing::Bool());

 class TimedPutPrecludeLastLevelTest
    : public PrecludeLastLevelTestBase,
@@ -4,7 +4,6 @@
 //  (found in the LICENSE.Apache file in the root directory).
 //

-
 #include "rocksdb/convenience.h"

 #include "db/convenience_impl.h"
@@ -7,8 +7,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.

-#include "rocksdb/options.h"
-
 #include <fcntl.h>
 #include <sys/stat.h>
 #include <sys/types.h>
@@ -25,6 +23,7 @@
 #include "rocksdb/convenience.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
+#include "rocksdb/options.h"
 #include "rocksdb/table.h"
 #include "rocksdb/utilities/transaction_db.h"
 #include "rocksdb/write_batch.h"
@@ -3,7 +3,6 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).

-
 #include "db/db_impl/db_impl.h"
 #include "db/db_test_util.h"
 #include "rocksdb/db.h"
@@ -349,4 +348,3 @@ int main(int argc, char** argv) {
    return 0;
  }
 }
-
@@ -878,7 +878,6 @@ TEST_F(DBBasicTest, Snapshot) {
  } while (ChangeOptions());
 }

-
 class DBBasicMultiConfigs : public DBBasicTest,
                            public ::testing::WithParamInterface<int> {
 public:
@@ -2604,8 +2603,7 @@ class DBMultiGetAsyncIOTest : public DBBasicTest,
    // Warm up the block cache so we don't need to use the IO uring
    Iterator* iter = dbfull()->NewIterator(ReadOptions());
    for (iter->SeekToFirst(); iter->Valid() && iter->status().ok();
-         iter->Next())
-      ;
+         iter->Next());
    EXPECT_OK(iter->status());
    delete iter;
 #endif  // ROCKSDB_IOURING_PRESENT
@@ -305,7 +305,6 @@ class ReadOnlyCacheWrapper : public CacheWrapper {
 }  // anonymous namespace
 #endif  // SNAPPY

-
 // Make sure that when options.block_cache is set, after a new table is
 // created its index/filter blocks are added to block cache.
 TEST_F(DBBlockCacheTest, IndexAndFilterBlocksOfNewTableAddedToCache) {
@@ -1758,8 +1757,8 @@ class CacheKeyTest : public testing::Test {
    tp_.db_id = std::to_string(db_id_);
    tp_.orig_file_number = file_number;
    bool is_stable;
-    std::string cur_session_id;       // ignored
-    uint64_t cur_file_number = 42;    // ignored
+    std::string cur_session_id;     // ignored
+    uint64_t cur_file_number = 42;  // ignored
    OffsetableCacheKey rv;
    BlockBasedTable::SetupBaseCacheKey(&tp_, cur_session_id, cur_file_number,
                                       &rv, &is_stable);
@@ -6417,10 +6417,13 @@ TEST_P(RoundRobinSubcompactionsAgainstPressureToken, PressureTokenTest) {
  options.level0_file_num_compaction_trigger = 4;
  options.target_file_size_base = kKeysPerBuffer * 1024;
  options.compaction_pri = CompactionPri::kRoundRobin;
-  // Target size is chosen so that filling the level with `kFilesPerLevel` files
-  // will make it oversized by `kNumSubcompactions` files.
+  // Pick a small target level size so that round-robin compaction will pick
+  // more files to compact and trigger subcompactions. Actual number of
+  // subcompactions will be limited by number of bg threads available.
+  // Cannot be too small to cause compaction pressure. See
+  // GetPendingCompactionBytesForCompactionSpeedup().
  options.max_bytes_for_level_base =
-      (kFilesPerLevel - kNumSubcompactions) * kKeysPerBuffer * 1024;
+      (kFilesPerLevel - 10) * kKeysPerBuffer * 1024;
  options.disable_auto_compactions = true;
  // Setup `kNumSubcompactions` threads but limited subcompactions so
  // that RoundRobin requires extra compactions from reserved threads
@@ -6446,21 +6449,34 @@ TEST_P(RoundRobinSubcompactionsAgainstPressureToken, PressureTokenTest) {
    ASSERT_EQ(kFilesPerLevel, NumTableFilesAtLevel(lvl, 0));
  }

+  bool compaction_num_input_file_verified = false;
+  SyncPoint::GetInstance()->SetCallBack(
+      "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) {
+        if (!compaction_num_input_file_verified) {
+          Compaction* compaction = static_cast<Compaction*>(arg);
+          // In Round-Robin, # of subcompactions needed is the # of input files
+          ASSERT_GT(compaction->num_input_files(0), 1);
+          compaction_num_input_file_verified = true;
+        }
+      });
+
  // This is a variable for making sure the following callback is called
  // and the assertions in it are indeed excuted.
  bool num_planned_subcompactions_verified = false;
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "CompactionJob::GenSubcompactionBoundaries:0", [&](void* arg) {
-        uint64_t num_planned_subcompactions = *(static_cast<uint64_t*>(arg));
-        if (grab_pressure_token_) {
-          // `kNumSubcompactions` files are selected for round-robin under auto
-          // compaction. The number of planned subcompaction is restricted by
-          // the limited number of max_background_compactions
-          ASSERT_EQ(num_planned_subcompactions, kNumSubcompactions);
-        } else {
-          ASSERT_EQ(num_planned_subcompactions, 1);
+        if (!num_planned_subcompactions_verified) {
+          uint64_t num_planned_subcompactions = *(static_cast<uint64_t*>(arg));
+          if (grab_pressure_token_) {
+            // `kNumSubcompactions` files are selected for round-robin under
+            // auto compaction. The number of planned subcompaction is
+            // restricted by the limited number of max_background_compactions
+            ASSERT_EQ(num_planned_subcompactions, kNumSubcompactions);
+          } else {
+            ASSERT_EQ(num_planned_subcompactions, 1);
+          }
+          num_planned_subcompactions_verified = true;
        }
-        num_planned_subcompactions_verified = true;
      });

  // The following 3 dependencies have to be added to ensure the auto
@@ -6911,7 +6927,8 @@ TEST_F(DBCompactionTest, ManualCompactionMax) {
  DestroyAndReopen(opts);
  generate_sst_func();
  uint64_t total_size = (l1_avg_size * 10) + (l2_avg_size * 100);
-  opts.max_compaction_bytes = total_size / num_split;
+  // Slightly inflate max_compaction_bytes since it's usually a strict bound.
+  opts.max_compaction_bytes = total_size / num_split + l2_avg_size * 10;
  opts.target_file_size_base = total_size / num_split;
  Reopen(opts);
  num_compactions.store(0);
@@ -6933,8 +6950,10 @@ TEST_F(DBCompactionTest, ManualCompactionMax) {
  DestroyAndReopen(opts);
  generate_sst_func();
  total_size = (l1_avg_size * 10) + (l2_avg_size * 100);
+  // Slightly inflate max_compaction_bytes since it's usually a strict bound.
  Status s = db_->SetOptions(
-      {{"max_compaction_bytes", std::to_string(total_size / num_split)},
+      {{"max_compaction_bytes",
+        std::to_string(total_size / num_split + 10 * l2_avg_size)},
       {"target_file_size_base", std::to_string(total_size / num_split)}});
  ASSERT_OK(s);

@@ -491,7 +491,6 @@ TEST_F(DBTestDynamicLevel, DISABLED_MigrateToDynamicLevelMaxBytesBase) {
 }
 }  // namespace ROCKSDB_NAMESPACE

-
 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
@@ -3,12 +3,13 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
 //
+#include <iostream>
+#include <string>
+
 #include "db/db_test_util.h"
 #include "port/stack_trace.h"
 #include "rocksdb/perf_context.h"
 #include "test_util/sync_point.h"
-#include <iostream>
-#include <string>

 namespace ROCKSDB_NAMESPACE {

@@ -25,7 +26,6 @@ class DBEncryptionTest : public DBTestBase {
  }
 };

-
 TEST_F(DBEncryptionTest, CheckEncrypted) {
  ASSERT_OK(Put("foo567", "v1.fetdq"));
  ASSERT_OK(Put("bar123", "v2.dfgkjdfghsd"));
@@ -116,7 +116,6 @@ TEST_F(DBEncryptionTest, ReadEmptyFile) {
  ASSERT_TRUE(data.empty());
 }

-
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -4,7 +4,6 @@
 //  (found in the LICENSE.Apache file in the root directory).
 //

-
 #include <algorithm>
 #include <cstdint>
 #include <memory>
@@ -266,7 +265,7 @@ Status DBImpl::GetLiveFilesStorageInfo(
      continue;
    }
    VersionStorageInfo& vsi = *cfd->current()->storage_info();
-    auto& cf_paths = cfd->ioptions()->cf_paths;
+    auto& cf_paths = cfd->ioptions().cf_paths;

    auto GetDir = [&](size_t path_id) {
      // Matching TableFileName() behavior
@@ -248,7 +248,7 @@ Status CompactedDBImpl::Init(const Options& options) {
 }

 Status CompactedDBImpl::Open(const Options& options, const std::string& dbname,
-                             DB** dbptr) {
+                             std::unique_ptr<DB>* dbptr) {
  *dbptr = nullptr;

  if (options.max_open_files != -1) {
@@ -267,7 +267,7 @@ Status CompactedDBImpl::Open(const Options& options, const std::string& dbname,
    ROCKS_LOG_INFO(db->immutable_db_options_.info_log,
                   "Opened the db as fully compacted mode");
    LogFlush(db->immutable_db_options_.info_log);
-    *dbptr = db.release();
+    *dbptr = std::move(db);
  }
  return s;
 }
@@ -22,7 +22,7 @@ class CompactedDBImpl : public DBImpl {
  ~CompactedDBImpl() override;

  static Status Open(const Options& options, const std::string& dbname,
-                     DB** dbptr);
+                     std::unique_ptr<DB>* dbptr);

  // Implementations of the DB interface
  using DB::Get;
@@ -386,9 +386,8 @@ Status DBImpl::ResumeImpl(DBRecoverContext context) {
          static_cast_with_check<ColumnFamilyHandleImpl>(default_cf_handle_);
      assert(cfh);
      ColumnFamilyData* cfd = cfh->cfd();
-      const MutableCFOptions& cf_opts = *cfd->GetLatestMutableCFOptions();
-      s = versions_->LogAndApply(cfd, cf_opts, read_options, write_options,
-                                 &edit, &mutex_, directories_.GetDbDir());
+      s = versions_->LogAndApply(cfd, read_options, write_options, &edit,
+                                 &mutex_, directories_.GetDbDir());
      if (!s.ok()) {
        io_s = versions_->io_status();
        if (!io_s.ok()) {
@@ -418,10 +417,19 @@ Status DBImpl::ResumeImpl(DBRecoverContext context) {
    }
  }

+  JobContext job_context(0);
+  FindObsoleteFiles(&job_context, true);
+  mutex_.Unlock();
+  // If DB shutdown initiated here, it will wait for this ongoing recovery.
+  job_context.manifest_file_number = 1;
+  if (job_context.HaveSomethingToDelete()) {
+    PurgeObsoleteFiles(job_context);
+  }
+  job_context.Clean();
+  mutex_.Lock();
+
  if (s.ok()) {
-    // This will notify and unblock threads waiting for error recovery to
-    // finish. Those previouly waiting threads can now proceed, which may
-    // include closing the db.
+    // Will notify and unblock threads waiting for error recovery to finish.
    s = error_handler_.ClearBGError();
  } else {
    // NOTE: this is needed to pass ASSERT_STATUS_CHECKED
@@ -430,15 +438,6 @@ Status DBImpl::ResumeImpl(DBRecoverContext context) {
    error_handler_.GetRecoveryError().PermitUncheckedError();
  }

-  JobContext job_context(0);
-  FindObsoleteFiles(&job_context, true);
-  mutex_.Unlock();
-  job_context.manifest_file_number = 1;
-  if (job_context.HaveSomethingToDelete()) {
-    PurgeObsoleteFiles(job_context);
-  }
-  job_context.Clean();
-
  if (s.ok()) {
    ROCKS_LOG_INFO(immutable_db_options_.info_log, "Successfully resumed DB");
  } else {
@@ -446,7 +445,6 @@ Status DBImpl::ResumeImpl(DBRecoverContext context) {
                   s.ToString().c_str());
  }

-  mutex_.Lock();
  // Check for shutdown again before scheduling further compactions,
  // since we released and re-acquired the lock above
  if (shutdown_initiated_) {
@@ -509,6 +507,11 @@ void DBImpl::CancelAllBackgroundWork(bool wait) {
    s.PermitUncheckedError();  //**TODO: What to do on error?
  }

+  // Cancel awaiting remote compactions
+  if (immutable_db_options_.compaction_service) {
+    immutable_db_options_.compaction_service->CancelAwaitingJobs();
+  }
+
  shutting_down_.store(true, std::memory_order_release);
  bg_cv_.SignalAll();
  if (!wait) {
@@ -540,8 +543,8 @@ Status DBImpl::CloseHelper() {
  // continuing with the shutdown
  mutex_.Lock();
  shutdown_initiated_ = true;
-  error_handler_.CancelErrorRecovery();
-  while (error_handler_.IsRecoveryInProgress()) {
+  error_handler_.CancelErrorRecoveryForShutDown();
+  while (!error_handler_.ReadyForShutdown()) {
    bg_cv_.Wait();
  }
  mutex_.Unlock();
@@ -852,7 +855,7 @@ Status DBImpl::RegisterRecordSeqnoTimeWorker(const ReadOptions& read_options,
    InstrumentedMutexLock l(&mutex_);

    for (auto cfd : *versions_->GetColumnFamilySet()) {
-      auto& mopts = *cfd->GetLatestMutableCFOptions();
+      auto& mopts = cfd->GetLatestMutableCFOptions();
      // preserve time is the max of 2 options.
      uint64_t preserve_seconds =
          std::max(mopts.preserve_internal_time_seconds,
@@ -1167,7 +1170,7 @@ void DBImpl::DumpStats() {
      }

      auto* table_factory =
-          cfd->GetCurrentMutableCFOptions()->table_factory.get();
+          cfd->GetCurrentMutableCFOptions().table_factory.get();
      assert(table_factory != nullptr);
      // FIXME: need to a shared_ptr if/when block_cache is going to be mutable
      Cache* cache =
@@ -1286,7 +1289,7 @@ Status DBImpl::SetOptions(
  }

  InstrumentedMutexLock ol(&options_mutex_);
-  MutableCFOptions new_options;
+  MutableCFOptions new_options_copy;  // For logging outside of DB mutex
  Status s;
  Status persist_options_status;
  SuperVersionContext sv_context(/* create_superversion */ true);
@@ -1295,16 +1298,20 @@ Status DBImpl::SetOptions(
    InstrumentedMutexLock l(&mutex_);
    s = cfd->SetOptions(db_options, options_map);
    if (s.ok()) {
-      new_options = *cfd->GetLatestMutableCFOptions();
+      new_options_copy = cfd->GetLatestMutableCFOptions();
      // Append new version to recompute compaction score.
      VersionEdit dummy_edit;
-      s = versions_->LogAndApply(cfd, new_options, read_options, write_options,
-                                 &dummy_edit, &mutex_, directories_.GetDbDir());
+      s = versions_->LogAndApply(cfd, read_options, write_options, &dummy_edit,
+                                 &mutex_, directories_.GetDbDir());
+      if (!versions_->io_status().ok()) {
+        assert(!s.ok());
+        error_handler_.SetBGError(versions_->io_status(),
+                                  BackgroundErrorReason::kManifestWrite);
+      }
      // Trigger possible flush/compactions. This has to be before we persist
      // options to file, otherwise there will be a deadlock with writer
      // thread.
-      InstallSuperVersionAndScheduleWork(cfd, &sv_context, new_options);
-
+      InstallSuperVersionAndScheduleWork(cfd, &sv_context);
      persist_options_status =
          WriteOptionsFile(write_options, true /*db_mutex_already_held*/);
      bg_cv_.SignalAll();
@@ -1328,7 +1335,7 @@ Status DBImpl::SetOptions(
  if (s.ok()) {
    ROCKS_LOG_INFO(immutable_db_options_.info_log,
                   "[%s] SetOptions() succeeded", cfd->GetName().c_str());
-    new_options.Dump(immutable_db_options_.info_log.get());
+    new_options_copy.Dump(immutable_db_options_.info_log.get());
    if (!persist_options_status.ok()) {
      // NOTE: WriteOptionsFile already logs on failure
      s = persist_options_status;
@@ -2110,7 +2117,7 @@ InternalIterator* DBImpl::NewInternalIterator(
    } else {
      mem_tombstone_iter = std::make_unique<TruncatedRangeDelIterator>(
          std::unique_ptr<FragmentedRangeTombstoneIterator>(range_del_iter),
-          &cfd->ioptions()->internal_comparator, nullptr /* smallest */,
+          &cfd->ioptions().internal_comparator, nullptr /* smallest */,
          nullptr /* largest */);
    }
    merge_iter_builder.AddPointAndTombstoneIterator(
@@ -3643,9 +3650,9 @@ Status DBImpl::CreateColumnFamilyImpl(const ReadOptions& read_options,
      write_thread_.EnterUnbatched(&w, &mutex_);
      // LogAndApply will both write the creation in MANIFEST and create
      // ColumnFamilyData object
-      s = versions_->LogAndApply(nullptr, MutableCFOptions(cf_options),
-                                 read_options, write_options, &edit, &mutex_,
-                                 directories_.GetDbDir(), false, &cf_options);
+      s = versions_->LogAndApply(nullptr, read_options, write_options, &edit,
+                                 &mutex_, directories_.GetDbDir(), false,
+                                 &cf_options);
      write_thread_.ExitUnbatched(&w);
    }
    if (s.ok()) {
@@ -3659,8 +3666,7 @@ Status DBImpl::CreateColumnFamilyImpl(const ReadOptions& read_options,
      auto* cfd =
          versions_->GetColumnFamilySet()->GetColumnFamily(column_family_name);
      assert(cfd != nullptr);
-      InstallSuperVersionAndScheduleWork(cfd, &sv_context,
-                                         *cfd->GetLatestMutableCFOptions());
+      InstallSuperVersionAndScheduleWork(cfd, &sv_context);

      if (!cfd->mem()->IsSnapshotSupported()) {
        is_snapshot_supported_ = false;
@@ -3754,13 +3760,12 @@ Status DBImpl::DropColumnFamilyImpl(ColumnFamilyHandle* column_family) {
      // we drop column family from a single write thread
      WriteThread::Writer w;
      write_thread_.EnterUnbatched(&w, &mutex_);
-      s = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),
-                                 read_options, write_options, &edit, &mutex_,
-                                 directories_.GetDbDir());
+      s = versions_->LogAndApply(cfd, read_options, write_options, &edit,
+                                 &mutex_, directories_.GetDbDir());
      write_thread_.ExitUnbatched(&w);
    }
    if (s.ok()) {
-      auto& moptions = *cfd->GetLatestMutableCFOptions();
+      auto& moptions = cfd->GetLatestMutableCFOptions();
      max_total_in_memory_state_ -=
          moptions.write_buffer_size * moptions.max_write_buffer_number;
      used_preserve_preclude = moptions.preserve_internal_time_seconds > 0 ||
@@ -3891,7 +3896,7 @@ Iterator* DBImpl::NewIterator(const ReadOptions& _read_options,
    auto iter = new ForwardIterator(this, read_options, cfd, sv,
                                    /* allow_unprepared_value */ true);
    result = NewDBIterator(
-        env_, read_options, *cfd->ioptions(), sv->mutable_cf_options,
+        env_, read_options, cfd->ioptions(), sv->mutable_cf_options,
        cfd->user_comparator(), iter, sv->current, kMaxSequenceNumber,
        sv->mutable_cf_options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */, cfh);
@@ -3973,7 +3978,7 @@ ArenaWrappedDBIter* DBImpl::NewIteratorImpl(
  // likely that any iterator pointer is close to the iterator it points to so
  // that they are likely to be in the same cache line and/or page.
  ArenaWrappedDBIter* db_iter = NewArenaWrappedDbIterator(
-      env_, read_options, *cfh->cfd()->ioptions(), sv->mutable_cf_options,
+      env_, read_options, cfh->cfd()->ioptions(), sv->mutable_cf_options,
      sv->current, snapshot,
      sv->mutable_cf_options.max_sequential_skip_in_iterations,
      sv->version_number, read_callback, cfh, expose_blob_index, allow_refresh);
@@ -4108,7 +4113,7 @@ Status DBImpl::NewIterators(
                                      cf_sv_pair.super_version,
                                      /* allow_unprepared_value */ true);
      iterators->push_back(
-          NewDBIterator(env_, read_options, *cf_sv_pair.cfd->ioptions(),
+          NewDBIterator(env_, read_options, cf_sv_pair.cfd->ioptions(),
                        cf_sv_pair.super_version->mutable_cf_options,
                        cf_sv_pair.cfd->user_comparator(), iter,
                        cf_sv_pair.super_version->current, kMaxSequenceNumber,
@@ -4346,7 +4351,7 @@ void DBImpl::ReleaseSnapshot(const Snapshot* s) {
    CfdList cf_scheduled;
    if (oldest_snapshot > bottommost_files_mark_threshold_) {
      for (auto* cfd : *versions_->GetColumnFamilySet()) {
-        if (!cfd->ioptions()->allow_ingest_behind) {
+        if (!cfd->ioptions().allow_ingest_behind) {
          cfd->current()->storage_info()->UpdateOldestSnapshot(
              oldest_snapshot, /*allow_ingest_behind=*/false);
          if (!cfd->current()
@@ -4367,7 +4372,7 @@ void DBImpl::ReleaseSnapshot(const Snapshot* s) {
      SequenceNumber new_bottommost_files_mark_threshold = kMaxSequenceNumber;
      for (auto* cfd : *versions_->GetColumnFamilySet()) {
        if (CfdListContains(cf_scheduled, cfd) ||
-            cfd->ioptions()->allow_ingest_behind) {
+            cfd->ioptions().allow_ingest_behind) {
          continue;
        }
        new_bottommost_files_mark_threshold = std::min(
@@ -4878,7 +4883,7 @@ Status DBImpl::GetUpdatesSince(
  return wal_manager_.GetUpdatesSince(seq, iter, read_options, versions_.get());
 }

-Status DBImpl::DeleteFile(std::string name) {
+Status DBImpl::DEPRECATED_DeleteFile(std::string name) {
  // TODO: plumb Env::IOActivity, Env::IOPriority
  const ReadOptions read_options;
  const WriteOptions write_options;
@@ -4961,13 +4966,11 @@ Status DBImpl::DeleteFile(std::string name) {
    }
    edit.SetColumnFamily(cfd->GetID());
    edit.DeleteFile(level, number);
-    status = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),
-                                    read_options, write_options, &edit, &mutex_,
-                                    directories_.GetDbDir());
+    status = versions_->LogAndApply(cfd, read_options, write_options, &edit,
+                                    &mutex_, directories_.GetDbDir());
    if (status.ok()) {
      InstallSuperVersionAndScheduleWork(
-          cfd, job_context.superversion_contexts.data(),
-          *cfd->GetLatestMutableCFOptions());
+          cfd, job_context.superversion_contexts.data());
    }
    FindObsoleteFiles(&job_context, false);
  }  // lock released here
@@ -5066,21 +5069,19 @@ Status DBImpl::DeleteFilesInRanges(ColumnFamilyHandle* column_family,
      }
    }
    if (!deleted_files.empty()) {
-      vstorage->ComputeCompactionScore(*cfd->ioptions(),
-                                       *cfd->GetLatestMutableCFOptions());
+      vstorage->ComputeCompactionScore(cfd->ioptions(),
+                                       cfd->GetLatestMutableCFOptions());
    }
    if (edit.GetDeletedFiles().empty()) {
      job_context.Clean();
      return status;
    }
    input_version->Ref();
-    status = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),
-                                    read_options, write_options, &edit, &mutex_,
-                                    directories_.GetDbDir());
+    status = versions_->LogAndApply(cfd, read_options, write_options, &edit,
+                                    &mutex_, directories_.GetDbDir());
    if (status.ok()) {
      InstallSuperVersionAndScheduleWork(
-          cfd, job_context.superversion_contexts.data(),
-          *cfd->GetLatestMutableCFOptions());
+          cfd, job_context.superversion_contexts.data());
    }
    for (auto* deleted_file : deleted_files) {
      deleted_file->being_compacted = false;
@@ -5679,7 +5680,7 @@ Status DBImpl::RenameTempFileToOptionsFile(const std::string& file_name,
 void DBImpl::NewThreadStatusCfInfo(ColumnFamilyData* cfd) const {
  if (immutable_db_options_.enable_thread_tracking) {
    ThreadStatusUtil::NewColumnFamilyInfo(this, cfd, cfd->GetName(),
-                                          cfd->ioptions()->env);
+                                          cfd->ioptions().env);
  }
 }

@@ -5897,6 +5898,7 @@ Status DBImpl::IngestExternalFile(

 Status DBImpl::IngestExternalFiles(
    const std::vector<IngestExternalFileArg>& args) {
+  PERF_TIMER_GUARD(file_ingestion_nanos);
  // TODO: plumb Env::IOActivity, Env::IOPriority
  const WriteOptions write_options;

@@ -6041,6 +6043,7 @@ Status DBImpl::IngestExternalFiles(
    if (two_write_queues_) {
      nonmem_write_thread_.EnterUnbatched(&nonmem_w, &mutex_);
    }
+    PERF_TIMER_GUARD(file_ingestion_blocking_live_writes_nanos);

    // When unordered_write is enabled, the keys are writing to memtable in an
    // unordered way. If the ingestion job checks memtable key range before the
@@ -6121,14 +6124,12 @@ Status DBImpl::IngestExternalFiles(
      ReadOptions read_options;
      read_options.fill_cache = args[0].options.fill_cache;
      autovector<ColumnFamilyData*> cfds_to_commit;
-      autovector<const MutableCFOptions*> mutable_cf_options_list;
      autovector<autovector<VersionEdit*>> edit_lists;
      uint32_t num_entries = 0;
      for (size_t i = 0; i != num_cfs; ++i) {
        auto* cfd = ingestion_jobs[i].GetColumnFamilyData();
        assert(!cfd->IsDropped());
        cfds_to_commit.push_back(cfd);
-        mutable_cf_options_list.push_back(cfd->GetLatestMutableCFOptions());
        autovector<VersionEdit*> edit_list;
        edit_list.push_back(ingestion_jobs[i].edit());
        edit_lists.push_back(edit_list);
@@ -6143,10 +6144,10 @@ Status DBImpl::IngestExternalFiles(
        }
        assert(0 == num_entries);
      }
-      status = versions_->LogAndApply(
-          cfds_to_commit, mutable_cf_options_list, read_options, write_options,
+      status =
+          versions_->LogAndApply(cfds_to_commit, read_options, write_options,

-          edit_lists, &mutex_, directories_.GetDbDir());
+                                 edit_lists, &mutex_, directories_.GetDbDir());
      // It is safe to update VersionSet last seqno here after LogAndApply since
      // LogAndApply persists last sequence number from VersionEdits,
      // which are from file's largest seqno and not from VersionSet.
@@ -6178,8 +6179,7 @@ Status DBImpl::IngestExternalFiles(
      for (size_t i = 0; i != num_cfs; ++i) {
        auto* cfd = ingestion_jobs[i].GetColumnFamilyData();
        assert(!cfd->IsDropped());
-        InstallSuperVersionAndScheduleWork(cfd, &sv_ctxs[i],
-                                           *cfd->GetLatestMutableCFOptions());
+        InstallSuperVersionAndScheduleWork(cfd, &sv_ctxs[i]);
 #ifndef NDEBUG
        if (0 == i && num_cfs > 1) {
          TEST_SYNC_POINT("DBImpl::IngestExternalFiles:InstallSVForFirstCF:0");
@@ -6203,6 +6203,7 @@ Status DBImpl::IngestExternalFiles(
      nonmem_write_thread_.ExitUnbatched(&nonmem_w);
    }
    write_thread_.ExitUnbatched(&w);
+    PERF_TIMER_STOP(file_ingestion_blocking_live_writes_nanos);

    if (status.ok()) {
      for (auto& job : ingestion_jobs) {
@@ -6297,12 +6298,11 @@ Status DBImpl::CreateColumnFamilyWithImport(
      // and this will overwrite the external file. To protect the external
      // file, we have to make sure the file number will never being reused.
      next_file_number = versions_->FetchAddFileNumber(total_file_num);
-      auto cf_options = cfd->GetLatestMutableCFOptions();
      status =
-          versions_->LogAndApply(cfd, *cf_options, read_options, write_options,
-                                 &dummy_edit, &mutex_, directories_.GetDbDir());
+          versions_->LogAndApply(cfd, read_options, write_options, &dummy_edit,
+                                 &mutex_, directories_.GetDbDir());
      if (status.ok()) {
-        InstallSuperVersionAndScheduleWork(cfd, &dummy_sv_ctx, *cf_options);
+        InstallSuperVersionAndScheduleWork(cfd, &dummy_sv_ctx);
      }
    }
  }
@@ -6335,12 +6335,11 @@ Status DBImpl::CreateColumnFamilyWithImport(

      // Install job edit [Mutex will be unlocked here]
      if (status.ok()) {
-        auto cf_options = cfd->GetLatestMutableCFOptions();
-        status = versions_->LogAndApply(cfd, *cf_options, read_options,
-                                        write_options, import_job.edit(),
-                                        &mutex_, directories_.GetDbDir());
+        status = versions_->LogAndApply(cfd, read_options, write_options,
+                                        import_job.edit(), &mutex_,
+                                        directories_.GetDbDir());
        if (status.ok()) {
-          InstallSuperVersionAndScheduleWork(cfd, &sv_context, *cf_options);
+          InstallSuperVersionAndScheduleWork(cfd, &sv_context);
        }
      }

@@ -6541,7 +6540,7 @@ Status DBImpl::VerifyChecksumInternal(const ReadOptions& read_options,
        const auto& fd = fd_with_krange.fd;
        const FileMetaData* fmeta = fd_with_krange.file_metadata;
        assert(fmeta);
-        std::string fname = TableFileName(cfd->ioptions()->cf_paths,
+        std::string fname = TableFileName(cfd->ioptions().cf_paths,
                                          fd.GetNumber(), fd.GetPathId());
        if (use_file_checksum) {
          s = VerifyFullFileChecksum(fmeta->file_checksum,
@@ -6565,7 +6564,7 @@ Status DBImpl::VerifyChecksumInternal(const ReadOptions& read_options,
        const uint64_t blob_file_number = meta->GetBlobFileNumber();

        const std::string blob_file_name = BlobFileName(
-            cfd->ioptions()->cf_paths.front().path, blob_file_number);
+            cfd->ioptions().cf_paths.front().path, blob_file_number);
        s = VerifyFullFileChecksum(meta->GetChecksumValue(),
                                   meta->GetChecksumMethod(), blob_file_name,
                                   read_options);
@@ -6758,16 +6757,15 @@ Status DBImpl::ReserveFileNumbersBeforeIngestion(
  pending_output_elem.reset(new std::list<uint64_t>::iterator(
      CaptureCurrentFileNumberInPendingOutputs()));
  *next_file_number = versions_->FetchAddFileNumber(static_cast<uint64_t>(num));
-  auto cf_options = cfd->GetLatestMutableCFOptions();
  VersionEdit dummy_edit;
  // If crash happen after a hard link established, Recover function may
  // reuse the file number that has already assigned to the internal file,
  // and this will overwrite the external file. To protect the external
  // file, we have to make sure the file number will never being reused.
-  s = versions_->LogAndApply(cfd, *cf_options, read_options, write_options,
-                             &dummy_edit, &mutex_, directories_.GetDbDir());
+  s = versions_->LogAndApply(cfd, read_options, write_options, &dummy_edit,
+                             &mutex_, directories_.GetDbDir());
  if (s.ok()) {
-    InstallSuperVersionAndScheduleWork(cfd, &dummy_sv_ctx, *cf_options);
+    InstallSuperVersionAndScheduleWork(cfd, &dummy_sv_ctx);
  }
  dummy_sv_ctx.Clean();
  return s;
@@ -6806,17 +6804,24 @@ void DBImpl::RecordSeqnoToTimeMapping(uint64_t populate_historical_seconds) {
  // for SeqnoToTimeMapping. We don't know how long it has been since the last
  // sequence number was written, so we at least have a one-sided bound by
  // sampling in this order.
-  SequenceNumber seqno = GetLatestSequenceNumber();
-  int64_t unix_time_signed = 0;
-  immutable_db_options_.clock->GetCurrentTime(&unix_time_signed)
-      .PermitUncheckedError();  // Ignore error
-  uint64_t unix_time = static_cast<uint64_t>(unix_time_signed);
-
+  // ALSO, to avoid out-of-order mappings, we need to get the seqno and times
+  // while holding the DB mutex. (This is really to make testing happy because
+  // it's fine to throw out extra close-but-not-quite-consistent mappings in
+  // production.)
  std::vector<SuperVersionContext> sv_contexts;
-  if (populate_historical_seconds > 0) {
-    bool success = true;
-    {
-      InstrumentedMutexLock l(&mutex_);
+  bool success = true;
+  SequenceNumber seqno;
+  uint64_t unix_time;
+  {
+    InstrumentedMutexLock l(&mutex_);
+
+    seqno = GetLatestSequenceNumber();
+    int64_t unix_time_signed = 0;
+    immutable_db_options_.clock->GetCurrentTime(&unix_time_signed)
+        .PermitUncheckedError();  // Ignore error
+    unix_time = static_cast<uint64_t>(unix_time_signed);
+
+    if (populate_historical_seconds > 0) {
      if (seqno > 1 && unix_time > populate_historical_seconds) {
        // seqno=0 is reserved
        SequenceNumber from_seqno = 1;
@@ -6830,7 +6835,20 @@ void DBImpl::RecordSeqnoToTimeMapping(uint64_t populate_historical_seconds) {
        assert(unix_time > populate_historical_seconds);
        success = false;
      }
+    } else {
+      // FIXME: assert(seqno > 0);
+      // Always successful assuming seqno never go backwards
+      seqno_to_time_mapping_.Append(seqno, unix_time);
+      InstallSeqnoToTimeMappingInSV(&sv_contexts);
    }
+  }
+
+  // clean up & report outside db mutex
+  for (SuperVersionContext& sv_context : sv_contexts) {
+    sv_context.Clean();
+  }
+
+  if (populate_historical_seconds > 0) {
    if (success) {
      ROCKS_LOG_INFO(
          immutable_db_options_.info_log,
@@ -6845,16 +6863,7 @@ void DBImpl::RecordSeqnoToTimeMapping(uint64_t populate_historical_seconds) {
          seqno, unix_time - populate_historical_seconds, unix_time);
    }
  } else {
-    InstrumentedMutexLock l(&mutex_);
-    // FIXME: assert(seqno > 0);
-    // Always successful assuming seqno never go backwards
-    seqno_to_time_mapping_.Append(seqno, unix_time);
-    InstallSeqnoToTimeMappingInSV(&sv_contexts);
-  }
-
-  // clean up outside db mutex
-  for (SuperVersionContext& sv_context : sv_contexts) {
-    sv_context.Clean();
+    assert(success);
  }
 }

@@ -6927,7 +6936,7 @@ void DBImpl::InstallSeqnoToTimeMappingInSV(
    sv_contexts->emplace_back(/*create_superversion=*/true);
    sv_contexts->back().new_seqno_to_time_mapping = new_seqno_to_time_mapping;
    cfd->InstallSuperVersion(&sv_contexts->back(),
-                             *(cfd->GetLatestMutableCFOptions()));
+                             cfd->GetLatestMutableCFOptions());
  }
  bg_cv_.SignalAll();
 }
@@ -162,6 +162,17 @@ class Directories {
  std::unique_ptr<FSDirectory> wal_dir_;
 };

+struct DBOpenLogReporter : public log::Reader::Reporter {
+  Env* env;
+  Logger* info_log;
+  const char* fname;
+  Status* status;  // nullptr if immutable_db_options_.paranoid_checks==false
+  bool* old_log_record;
+  void Corruption(size_t bytes, const Status& s) override;
+
+  void OldLogRecord(size_t bytes) override;
+};
+
 // While DB is the public interface of RocksDB, and DBImpl is the actual
 // class implementing it. It's the entrance of the core RocksdB engine.
 // All other DB implementations, e.g. TransactionDB, BlobDB, etc, wrap a
@@ -524,7 +535,7 @@ class DBImpl : public DB {
      SequenceNumber seq_number, std::unique_ptr<TransactionLogIterator>* iter,
      const TransactionLogIterator::ReadOptions& read_options =
          TransactionLogIterator::ReadOptions()) override;
-  Status DeleteFile(std::string name) override;
+  Status DEPRECATED_DeleteFile(std::string name) override;
  Status DeleteFilesInRanges(ColumnFamilyHandle* column_family,
                             const RangePtr* ranges, size_t n,
                             bool include_end = true);
@@ -1081,9 +1092,10 @@ class DBImpl : public DB {
  // This is to be used only by internal rocksdb classes.
  static Status Open(const DBOptions& db_options, const std::string& name,
                     const std::vector<ColumnFamilyDescriptor>& column_families,
-                     std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
-                     const bool seq_per_batch, const bool batch_per_txn,
-                     const bool is_retry, bool* can_retry);
+                     std::vector<ColumnFamilyHandle*>* handles,
+                     std::unique_ptr<DB>* dbptr, const bool seq_per_batch,
+                     const bool batch_per_txn, const bool is_retry,
+                     bool* can_retry);

  static IOStatus CreateAndNewDirectory(
      FileSystem* fs, const std::string& dirname,
@@ -1440,7 +1452,6 @@ class DBImpl : public DB {
        uint32_t size = static_cast<uint32_t>(map_.size());
        map_.emplace(cfd->GetID(), size);
        cfds_.emplace_back(cfd);
-        mutable_cf_opts_.emplace_back(cfd->GetLatestMutableCFOptions());
        edit_lists_.emplace_back(autovector<VersionEdit*>());
      }
      uint32_t i = map_[cfd->GetID()];
@@ -1449,7 +1460,6 @@ class DBImpl : public DB {

    std::unordered_map<uint32_t, uint32_t> map_;  // cf_id to index;
    autovector<ColumnFamilyData*> cfds_;
-    autovector<const MutableCFOptions*> mutable_cf_opts_;
    autovector<autovector<VersionEdit*>> edit_lists_;
    // All existing data files (SST files and Blob files) found during DB::Open.
    std::vector<std::string> existing_data_files_;
@@ -1910,8 +1920,8 @@ class DBImpl : public DB {
    const InternalKey* begin = nullptr;  // nullptr means beginning of key range
    const InternalKey* end = nullptr;    // nullptr means end of key range
    InternalKey* manual_end = nullptr;   // how far we are compacting
-    InternalKey tmp_storage;      // Used to keep track of compaction progress
-    InternalKey tmp_storage1;     // Used to keep track of compaction progress
+    InternalKey tmp_storage;   // Used to keep track of compaction progress
+    InternalKey tmp_storage1;  // Used to keep track of compaction progress

    // When the user provides a canceled pointer in CompactRangeOptions, the
    // above varaibe is the reference of the user-provided
@@ -2043,6 +2053,90 @@ class DBImpl : public DB {
                         bool is_retry, bool* corrupted_log_found,
                         RecoveryContext* recovery_ctx);

+  void SetupLogFilesRecovery(
+      const std::vector<uint64_t>& wal_numbers,
+      std::unordered_map<int, VersionEdit>* version_edits, int* job_id,
+      uint64_t* min_wal_number);
+
+  Status ProcessLogFiles(const std::vector<uint64_t>& wal_numbers,
+                         bool read_only, bool is_retry, uint64_t min_wal_number,
+                         int job_id, SequenceNumber* next_sequence,
+                         std::unordered_map<int, VersionEdit>* version_edits,
+                         bool* corrupted_wal_found,
+                         RecoveryContext* recovery_ctx);
+
+  Status ProcessLogFile(
+      uint64_t wal_number, uint64_t min_wal_number, bool is_retry,
+      bool read_only, int job_id, SequenceNumber* next_sequence,
+      bool* stop_replay_for_corruption, bool* stop_replay_by_wal_filter,
+      uint64_t* corrupted_wal_number, bool* corrupted_wal_found,
+      std::unordered_map<int, VersionEdit>* version_edits, bool* flushed,
+      PredecessorWALInfo& predecessor_wal_info);
+
+  void SetupLogFileProcessing(uint64_t wal_number);
+
+  Status InitializeLogReader(
+      uint64_t wal_number, bool is_retry, std::string& fname,
+
+      bool stop_replay_for_corruption, uint64_t min_wal_number,
+      const PredecessorWALInfo& predecessor_wal_info,
+      bool* const old_log_record, Status* const reporter_status,
+      DBOpenLogReporter* reporter, std::unique_ptr<log::Reader>& reader);
+  Status ProcessLogRecord(
+      Slice record, const std::unique_ptr<log::Reader>& reader,
+      const UnorderedMap<uint32_t, size_t>& running_ts_sz, uint64_t wal_number,
+      const std::string& fname, bool read_only, int job_id,
+      std::function<void()> logFileDropped, DBOpenLogReporter* reporter,
+      uint64_t* record_checksum, SequenceNumber* last_seqno_observed,
+      SequenceNumber* next_sequence, bool* stop_replay_for_corruption,
+      Status* status, bool* stop_replay_by_wal_filter,
+      std::unordered_map<int, VersionEdit>* version_edits, bool* flushed);
+
+  Status InitializeWriteBatchForLogRecord(
+      Slice record, const std::unique_ptr<log::Reader>& reader,
+      const UnorderedMap<uint32_t, size_t>& running_ts_sz, WriteBatch* batch,
+      std::unique_ptr<WriteBatch>& new_batch, WriteBatch*& batch_to_use,
+      uint64_t* record_checksum);
+
+  void MaybeReviseStopReplayForCorruption(
+      SequenceNumber sequence, SequenceNumber const* const next_sequence,
+      bool* stop_replay_for_corruption);
+
+  Status InsertLogRecordToMemtable(WriteBatch* batch_to_use,
+                                   uint64_t wal_number,
+                                   SequenceNumber* next_sequence,
+                                   bool* has_valid_writes);
+
+  Status MaybeWriteLevel0TableForRecovery(
+      bool has_valid_writes, bool read_only, uint64_t wal_number, int job_id,
+      SequenceNumber const* const next_sequence,
+      std::unordered_map<int, VersionEdit>* version_edits, bool* flushed);
+
+  Status HandleNonOkStatusOrOldLogRecord(
+      uint64_t wal_number, SequenceNumber const* const next_sequence,
+      Status log_read_status, bool* old_log_record,
+      bool* stop_replay_for_corruption, uint64_t* corrupted_wal_number,
+      bool* corrupted_wal_found);
+
+  Status UpdatePredecessorWALInfo(uint64_t wal_number,
+                                  const SequenceNumber last_seqno_observed,
+                                  const std::string& fname,
+                                  PredecessorWALInfo& predecessor_wal_info);
+
+  void FinishLogFileProcessing(const Status& status,
+                               const SequenceNumber* next_sequence);
+
+  // Return `Status::Corruption()` when `stop_replay_for_corruption == true` and
+  // exits inconsistency between SST and WAL data
+  Status MaybeHandleStopReplayForCorruptionForInconsistency(
+      bool stop_replay_for_corruption, uint64_t corrupted_wal_number);
+
+  Status MaybeFlushFinalMemtableOrRestoreActiveLogFiles(
+      const std::vector<uint64_t>& wal_numbers, bool read_only, int job_id,
+      bool flushed, std::unordered_map<int, VersionEdit>* version_edits,
+      RecoveryContext* recovery_ctx);
+
+  void FinishLogFilesRecovery(int job_id, const Status& status);
  // The following two methods are used to flush a memtable to
  // storage. The first one is used at database RecoveryTime (when the
  // database is opened) and is heavyweight because it holds the mutex
@@ -2222,7 +2316,8 @@ class DBImpl : public DB {
                      const WriteOptions& write_options,
                      log::Writer* log_writer, uint64_t* log_used,
                      uint64_t* log_size,
-                      LogFileNumberSize& log_file_number_size);
+                      LogFileNumberSize& log_file_number_size,
+                      SequenceNumber sequence);

  IOStatus WriteToWAL(const WriteThread::WriteGroup& write_group,
                      log::Writer* log_writer, uint64_t* log_used,
@@ -2426,9 +2521,8 @@ class DBImpl : public DB {
  // All ColumnFamily state changes go through this function. Here we analyze
  // the new state and we schedule background work if we detect that the new
  // state needs flush or compaction.
-  void InstallSuperVersionAndScheduleWork(
-      ColumnFamilyData* cfd, SuperVersionContext* sv_context,
-      const MutableCFOptions& mutable_cf_options);
+  void InstallSuperVersionAndScheduleWork(ColumnFamilyData* cfd,
+                                          SuperVersionContext* sv_context);

  bool GetIntPropertyInternal(ColumnFamilyData* cfd,
                              const DBPropertyInfo& property_info,
@@ -2467,6 +2561,7 @@ class DBImpl : public DB {

  IOStatus CreateWAL(const WriteOptions& write_options, uint64_t log_file_num,
                     uint64_t recycle_log_number, size_t preallocate_block_size,
+                     const PredecessorWALInfo& predecessor_wal_info,
                     log::Writer** new_log);

  // Validate self-consistency of DB options
@@ -66,7 +66,7 @@ bool DBImpl::RequestCompactionToken(ColumnFamilyData* cfd, bool force,
                                    LogBuffer* log_buffer) {
  assert(*token == nullptr);
  auto limiter = static_cast<ConcurrentTaskLimiterImpl*>(
-      cfd->ioptions()->compaction_thread_limiter.get());
+      cfd->ioptions().compaction_thread_limiter.get());
  if (limiter == nullptr) {
    return true;
  }
@@ -99,7 +99,7 @@ bool DBImpl::ShouldRescheduleFlushRequestToRetainUDT(
  // Check if holding on the flush will cause entering write stall mode.
  // Write stall entered because of the accumulation of write buffers can be
  // alleviated if we continue with the flush instead of postponing it.
-  const auto& mutable_cf_options = *cfd->GetLatestMutableCFOptions();
+  const auto& mutable_cf_options = cfd->GetLatestMutableCFOptions();

  // Use the same criteria as WaitUntilFlushWouldNotStallWrites does w.r.t
  // defining what a write stall is about to happen means. If this uses a
@@ -116,7 +116,7 @@ bool DBImpl::ShouldRescheduleFlushRequestToRetainUDT(
          cfd->GetUnflushedMemTableCountForWriteStallCheck(),
          /*num_l0_files=*/0,
          /*num_compaction_needed_bytes=*/0, mutable_cf_options,
-          *cfd->ioptions())
+          cfd->ioptions())
          .first;
  if (write_stall != WriteStallCondition::kNormal) {
    return false;
@@ -213,7 +213,7 @@ Status DBImpl::FlushMemTableToOutputFile(
      snapshot_seqs, earliest_write_conflict_snapshot, snapshot_checker,
      job_context, flush_reason, log_buffer, directories_.GetDbDir(),
      GetDataDir(cfd, 0U),
-      GetCompressionFlush(*cfd->ioptions(), mutable_cf_options), stats_,
+      GetCompressionFlush(cfd->ioptions(), mutable_cf_options), stats_,
      &event_logger_, mutable_cf_options.report_bg_io_stats,
      true /* sync_output_directory */, true /* write_manifest */, thread_pri,
      io_tracer_, cfd->GetSuperVersion()->ShareSeqnoToTimeMapping(), db_id_,
@@ -299,8 +299,7 @@ Status DBImpl::FlushMemTableToOutputFile(
  }

  if (s.ok()) {
-    InstallSuperVersionAndScheduleWork(cfd, superversion_context,
-                                       mutable_cf_options);
+    InstallSuperVersionAndScheduleWork(cfd, superversion_context);
    if (made_progress) {
      *made_progress = true;
    }
@@ -369,7 +368,7 @@ Status DBImpl::FlushMemTableToOutputFile(
    if (sfm) {
      // Notify sst_file_manager that a new file was added
      std::string file_path = MakeTableFileName(
-          cfd->ioptions()->cf_paths[0].path, file_meta.fd.GetNumber());
+          cfd->ioptions().cf_paths[0].path, file_meta.fd.GetNumber());
      // TODO (PR7798).  We should only add the file to the FileManager if it
      // exists. Otherwise, some tests may fail.  Ignore the error in the
      // interim.
@@ -404,7 +403,8 @@ Status DBImpl::FlushMemTablesToOutputFiles(
  const auto& bg_flush_arg = bg_flush_args[0];
  ColumnFamilyData* cfd = bg_flush_arg.cfd_;
  // intentional infrequent copy for each flush
-  MutableCFOptions mutable_cf_options_copy = *cfd->GetLatestMutableCFOptions();
+  const MutableCFOptions mutable_cf_options_copy =
+      cfd->GetLatestMutableCFOptions();
  SuperVersionContext* superversion_context =
      bg_flush_arg.superversion_context_;
  FlushReason flush_reason = bg_flush_arg.flush_reason_;
@@ -461,7 +461,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
  for (int i = 0; i < num_cfs; ++i) {
    auto cfd = cfds[i];
    FSDirectory* data_dir = GetDataDir(cfd, 0U);
-    const std::string& curr_path = cfd->ioptions()->cf_paths[0].path;
+    const std::string& curr_path = cfd->ioptions().cf_paths[0].path;

    // Add to distinct output directories if eligible. Use linear search. Since
    // the number of elements in the vector is not large, performance should be
@@ -478,7 +478,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
      distinct_output_dirs.emplace_back(data_dir);
    }

-    all_mutable_cf_options.emplace_back(*cfd->GetLatestMutableCFOptions());
+    all_mutable_cf_options.emplace_back(cfd->GetLatestMutableCFOptions());
    const MutableCFOptions& mutable_cf_options = all_mutable_cf_options.back();
    uint64_t max_memtable_id = bg_flush_args[i].max_memtable_id_;
    FlushReason flush_reason = bg_flush_args[i].flush_reason_;
@@ -488,7 +488,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
        &shutting_down_, snapshot_seqs, earliest_write_conflict_snapshot,
        snapshot_checker, job_context, flush_reason, log_buffer,
        directories_.GetDbDir(), data_dir,
-        GetCompressionFlush(*cfd->ioptions(), mutable_cf_options), stats_,
+        GetCompressionFlush(cfd->ioptions(), mutable_cf_options), stats_,
        &event_logger_, mutable_cf_options.report_bg_io_stats,
        false /* sync_output_directory */, false /* write_manifest */,
        thread_pri, io_tracer_,
@@ -754,7 +754,6 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
  if (s.ok()) {
    autovector<ColumnFamilyData*> tmp_cfds;
    autovector<const autovector<ReadOnlyMemTable*>*> mems_list;
-    autovector<const MutableCFOptions*> mutable_cf_options_list;
    autovector<FileMetaData*> tmp_file_meta;
    autovector<std::list<std::unique_ptr<FlushJobInfo>>*>
        committed_flush_jobs_info;
@@ -763,7 +762,6 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
      if (!cfds[i]->IsDropped() && !mems.empty()) {
        tmp_cfds.emplace_back(cfds[i]);
        mems_list.emplace_back(&mems);
-        mutable_cf_options_list.emplace_back(&all_mutable_cf_options[i]);
        tmp_file_meta.emplace_back(&file_meta[i]);
        committed_flush_jobs_info.emplace_back(
            jobs[i]->GetCommittedFlushJobsInfo());
@@ -771,8 +769,8 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
    }

    s = InstallMemtableAtomicFlushResults(
-        nullptr /* imm_lists */, tmp_cfds, mutable_cf_options_list, mems_list,
-        versions_.get(), &logs_with_prep_tracker_, &mutex_, tmp_file_meta,
+        nullptr /* imm_lists */, tmp_cfds, mems_list, versions_.get(),
+        &logs_with_prep_tracker_, &mutex_, tmp_file_meta,
        committed_flush_jobs_info, &job_context->memtables_to_free,
        directories_.GetDbDir(), log_buffer);
  }
@@ -786,9 +784,8 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
      if (cfds[i]->IsDropped()) {
        continue;
      }
-      InstallSuperVersionAndScheduleWork(cfds[i],
-                                         &job_context->superversion_contexts[i],
-                                         all_mutable_cf_options[i]);
+      InstallSuperVersionAndScheduleWork(
+          cfds[i], &job_context->superversion_contexts[i]);

      const std::string& column_family_name = cfds[i]->GetName();

@@ -834,7 +831,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
                             jobs[i]->GetCommittedFlushJobsInfo());
      if (sfm) {
        std::string file_path = MakeTableFileName(
-            cfds[i]->ioptions()->cf_paths[0].path, file_meta[i].fd.GetNumber());
+            cfds[i]->ioptions().cf_paths[0].path, file_meta[i].fd.GetNumber());
        // TODO (PR7798).  We should only add the file to the FileManager if it
        // exists. Otherwise, some tests may fail.  Ignore the error in the
        // interim.
@@ -908,7 +905,7 @@ void DBImpl::NotifyOnFlushBegin(ColumnFamilyData* cfd, FileMetaData* file_meta,
    //                 go to L0 in the future.
    const uint64_t file_number = file_meta->fd.GetNumber();
    info.file_path =
-        MakeTableFileName(cfd->ioptions()->cf_paths[0].path, file_number);
+        MakeTableFileName(cfd->ioptions().cf_paths[0].path, file_number);
    info.file_number = file_number;
    info.thread_id = env_->GetThreadID();
    info.job_id = job_id;
@@ -1040,9 +1037,8 @@ Status DBImpl::IncreaseFullHistoryTsLowImpl(ColumnFamilyData* cfd,
    return Status::InvalidArgument(oss.str());
  }

-  Status s = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),
-                                    read_options, write_options, &edit, &mutex_,
-                                    directories_.GetDbDir());
+  Status s = versions_->LogAndApply(cfd, read_options, write_options, &edit,
+                                    &mutex_, directories_.GetDbDir());
  if (!s.ok()) {
    return s;
  }
@@ -1066,11 +1062,11 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
  auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(column_family);
  auto cfd = cfh->cfd();

-  if (options.target_path_id >= cfd->ioptions()->cf_paths.size()) {
+  if (options.target_path_id >= cfd->ioptions().cf_paths.size()) {
    return Status::InvalidArgument("Invalid target path ID");
  }
  if (options.change_level &&
-      cfd->ioptions()->compaction_style == kCompactionStyleFIFO) {
+      cfd->ioptions().compaction_style == kCompactionStyleFIFO) {
    return Status::NotSupported(
        "FIFO compaction does not support change_level.");
  }
@@ -1122,7 +1118,7 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
  constexpr int kInvalidLevel = -1;
  int final_output_level = kInvalidLevel;
  bool exclusive = options.exclusive_manual_compaction;
-  if (cfd->ioptions()->compaction_style == kCompactionStyleUniversal &&
+  if (cfd->ioptions().compaction_style == kCompactionStyleUniversal &&
      cfd->NumberLevels() > 1) {
    // Always compact all files together.
    final_output_level = cfd->NumberLevels() - 1;
@@ -1134,7 +1130,7 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
                            final_output_level, options, begin, end, exclusive,
                            false /* disable_trivial_move */,
                            std::numeric_limits<uint64_t>::max(), trim_ts);
-  } else if (cfd->ioptions()->compaction_style == kCompactionStyleFIFO) {
+  } else if (cfd->ioptions().compaction_style == kCompactionStyleFIFO) {
    // FIFOCompactionPicker::CompactRange() will ignore the input key range
    // [begin, end] and just try to pick compaction based on the configured
    // option `compaction_options_fifo`. So we skip checking if [begin, end]
@@ -1152,7 +1148,7 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,

      // Might need to query the partitioner
      SstPartitionerFactory* partitioner_factory =
-          current_version->cfd()->ioptions()->sst_partitioner_factory.get();
+          current_version->cfd()->ioptions().sst_partitioner_factory.get();
      std::unique_ptr<SstPartitioner> partitioner;
      if (partitioner_factory && begin != nullptr && end != nullptr) {
        SstPartitioner::Context context;
@@ -1228,7 +1224,7 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
      CleanupSuperVersion(super_version);
    }
    if (s.ok() && first_overlapped_level != kInvalidLevel) {
-      if (cfd->ioptions()->compaction_style == kCompactionStyleUniversal) {
+      if (cfd->ioptions().compaction_style == kCompactionStyleUniversal) {
        assert(first_overlapped_level == 0);
        s = RunManualCompaction(
            cfd, first_overlapped_level, first_overlapped_level, options, begin,
@@ -1237,7 +1233,7 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
            trim_ts);
        final_output_level = first_overlapped_level;
      } else {
-        assert(cfd->ioptions()->compaction_style == kCompactionStyleLevel);
+        assert(cfd->ioptions().compaction_style == kCompactionStyleLevel);
        uint64_t next_file_number = versions_->current_next_file_number();
        // Start compaction from `first_overlapped_level`, one level down at a
        // time, until output level >= max_overlapped_level.
@@ -1250,9 +1246,9 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
        for (;;) {
          // Always allow L0 -> L1 compaction
          if (level > 0) {
-            if (cfd->ioptions()->level_compaction_dynamic_level_bytes) {
-              assert(final_output_level < cfd->ioptions()->num_levels);
-              if (final_output_level + 1 == cfd->ioptions()->num_levels) {
+            if (cfd->ioptions().level_compaction_dynamic_level_bytes) {
+              assert(final_output_level < cfd->ioptions().num_levels);
+              if (final_output_level + 1 == cfd->ioptions().num_levels) {
                break;
              }
            } else {
@@ -1271,7 +1267,7 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
            }
          }
          output_level = level + 1;
-          if (cfd->ioptions()->level_compaction_dynamic_level_bytes &&
+          if (cfd->ioptions().level_compaction_dynamic_level_bytes &&
              level == 0) {
            output_level = ColumnFamilyData::kCompactToBaseLevel;
          }
@@ -1303,8 +1299,8 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
          // bottommost level intra-level compaction
          if ((options.bottommost_level_compaction ==
                   BottommostLevelCompaction::kIfHaveCompactionFilter &&
-               (cfd->ioptions()->compaction_filter != nullptr ||
-                cfd->ioptions()->compaction_filter_factory != nullptr)) ||
+               (cfd->ioptions().compaction_filter != nullptr ||
+                cfd->ioptions().compaction_filter_factory != nullptr)) ||
              options.bottommost_level_compaction ==
                  BottommostLevelCompaction::kForceOptimized ||
              options.bottommost_level_compaction ==
@@ -1458,7 +1454,7 @@ Status DBImpl::CompactFilesImpl(
  }

  if (output_path_id < 0) {
-    if (cfd->ioptions()->cf_paths.size() == 1U) {
+    if (cfd->ioptions().cf_paths.size() == 1U) {
      output_path_id = 0;
    } else {
      return Status::NotSupported(
@@ -1467,12 +1463,12 @@ Status DBImpl::CompactFilesImpl(
    }
  }

-  if (cfd->ioptions()->allow_ingest_behind &&
-      output_level >= cfd->ioptions()->num_levels - 1) {
+  if (cfd->ioptions().allow_ingest_behind &&
+      output_level >= cfd->ioptions().num_levels - 1) {
    return Status::InvalidArgument(
        "Exceed the maximum output level defined by "
        "the current compaction algorithm with ingest_behind --- " +
-        std::to_string(cfd->ioptions()->num_levels - 1));
+        std::to_string(cfd->ioptions().num_levels - 1));
  }

  std::vector<CompactionInputFiles> input_files;
@@ -1509,7 +1505,7 @@ Status DBImpl::CompactFilesImpl(
  assert(cfd->compaction_picker());
  c.reset(cfd->compaction_picker()->CompactFiles(
      compact_options, input_files, output_level, version->storage_info(),
-      *cfd->GetLatestMutableCFOptions(), mutable_db_options_, output_path_id));
+      cfd->GetLatestMutableCFOptions(), mutable_db_options_, output_path_id));
  // we already sanitized the set of input files and checked for conflicts
  // without releasing the lock, so we're guaranteed a compaction can be formed.
  assert(c != nullptr);
@@ -1539,8 +1535,8 @@ Status DBImpl::CompactFilesImpl(
      GetDataDir(c->column_family_data(), 0), stats_, &mutex_, &error_handler_,
      snapshot_seqs, earliest_write_conflict_snapshot, snapshot_checker,
      job_context, table_cache_, &event_logger_,
-      c->mutable_cf_options()->paranoid_file_checks,
-      c->mutable_cf_options()->report_bg_io_stats, dbname_,
+      c->mutable_cf_options().paranoid_file_checks,
+      c->mutable_cf_options().report_bg_io_stats, dbname_,
      &compaction_job_stats, Env::Priority::USER, io_tracer_,
      kManualCompactionCanceledFalse_, db_id_, db_session_id_,
      c->column_family_data()->GetFullHistoryTsLow(), c->trim_ts(),
@@ -1551,10 +1547,10 @@ Status DBImpl::CompactFilesImpl(
  // takes running compactions into account (by skipping files that are already
  // being compacted). Since we just changed compaction score, we recalculate it
  // here.
-  version->storage_info()->ComputeCompactionScore(*cfd->ioptions(),
-                                                  *c->mutable_cf_options());
+  version->storage_info()->ComputeCompactionScore(cfd->ioptions(),
+                                                  c->mutable_cf_options());

-  compaction_job.Prepare();
+  compaction_job.Prepare(std::nullopt /*subcompact to be computed*/);

  std::unique_ptr<std::list<uint64_t>::iterator> min_options_file_number_elem;
  if (immutable_db_options().compaction_service != nullptr) {
@@ -1576,16 +1572,14 @@ Status DBImpl::CompactFilesImpl(
  }

  bool compaction_released = false;
-  Status status =
-      compaction_job.Install(*c->mutable_cf_options(), &compaction_released);
+  Status status = compaction_job.Install(&compaction_released);
  if (!compaction_released) {
    c->ReleaseCompactionFiles(s);
  }
  if (status.ok()) {
    assert(compaction_job.io_status().ok());
    InstallSuperVersionAndScheduleWork(
-        c->column_family_data(), job_context->superversion_contexts.data(),
-        *c->mutable_cf_options());
+        c->column_family_data(), job_context->superversion_contexts.data());
  }
  // status above captures any error during compaction_job.Install, so its ok
  // not check compaction_job.io_status() explicitly if we're not calling
@@ -1633,13 +1627,13 @@ Status DBImpl::CompactFilesImpl(
  if (output_file_names != nullptr) {
    for (const auto& newf : c->edit()->GetNewFiles()) {
      output_file_names->push_back(TableFileName(
-          c->immutable_options()->cf_paths, newf.second.fd.GetNumber(),
+          c->immutable_options().cf_paths, newf.second.fd.GetNumber(),
          newf.second.fd.GetPathId()));
    }

    for (const auto& blob_file : c->edit()->GetBlobFileAdditions()) {
      output_file_names->push_back(
-          BlobFileName(c->immutable_options()->cf_paths.front().path,
+          BlobFileName(c->immutable_options().cf_paths.front().path,
                       blob_file.GetBlobFileNumber()));
    }
  }
@@ -1774,7 +1768,7 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
  }
  refitting_level_ = true;

-  const MutableCFOptions mutable_cf_options = *cfd->GetLatestMutableCFOptions();
+  const auto& mutable_cf_options = cfd->GetLatestMutableCFOptions();
  // move to a smaller level
  int to_level = target_level;
  if (target_level < 0) {
@@ -1845,12 +1839,12 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
                    cfd->current()->DebugString().data());

    std::unique_ptr<Compaction> c(new Compaction(
-        vstorage, *cfd->ioptions(), mutable_cf_options, mutable_db_options_,
+        vstorage, cfd->ioptions(), mutable_cf_options, mutable_db_options_,
        {input}, to_level,
        MaxFileSizeForLevel(
            mutable_cf_options, to_level,
            cfd->ioptions()
-                ->compaction_style) /* output file size limit, not applicable */
+                .compaction_style) /* output file size limit, not applicable */
        ,
        LLONG_MAX /* max compaction bytes, not applicable */,
        0 /* output path ID, not applicable */, mutable_cf_options.compression,
@@ -1883,14 +1877,14 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
                    "[%s] Apply version edit:\n%s", cfd->GetName().c_str(),
                    edit.DebugString().data());

-    Status status = versions_->LogAndApply(cfd, mutable_cf_options,
-                                           read_options, write_options, &edit,
-                                           &mutex_, directories_.GetDbDir());
+    Status status =
+        versions_->LogAndApply(cfd, read_options, write_options, &edit, &mutex_,
+                               directories_.GetDbDir());
    c->MarkFilesBeingCompacted(false);
    cfd->compaction_picker()->UnregisterCompaction(c.get());
    c.reset();

-    InstallSuperVersionAndScheduleWork(cfd, &sv_context, mutable_cf_options);
+    InstallSuperVersionAndScheduleWork(cfd, &sv_context);

    ROCKS_LOG_DEBUG(immutable_db_options_.info_log, "[%s] LogAndApply: %s\n",
                    cfd->GetName().c_str(), status.ToString().data());
@@ -2043,16 +2037,16 @@ Status DBImpl::RunManualCompaction(
  // For universal compaction, we enforce every manual compaction to compact
  // all files.
  if (begin == nullptr ||
-      cfd->ioptions()->compaction_style == kCompactionStyleUniversal ||
-      cfd->ioptions()->compaction_style == kCompactionStyleFIFO) {
+      cfd->ioptions().compaction_style == kCompactionStyleUniversal ||
+      cfd->ioptions().compaction_style == kCompactionStyleFIFO) {
    manual.begin = nullptr;
  } else {
    begin_storage.SetMinPossibleForUserKey(*begin);
    manual.begin = &begin_storage;
  }
  if (end == nullptr ||
-      cfd->ioptions()->compaction_style == kCompactionStyleUniversal ||
-      cfd->ioptions()->compaction_style == kCompactionStyleFIFO) {
+      cfd->ioptions().compaction_style == kCompactionStyleUniversal ||
+      cfd->ioptions().compaction_style == kCompactionStyleFIFO) {
    manual.end = nullptr;
  } else {
    end_storage.SetMaxPossibleForUserKey(*end);
@@ -2133,7 +2127,7 @@ Status DBImpl::RunManualCompaction(
        scheduled ||
        (((manual.manual_end = &manual.tmp_storage1) != nullptr) &&
         ((compaction = manual.cfd->CompactRange(
-               *manual.cfd->GetLatestMutableCFOptions(), mutable_db_options_,
+               manual.cfd->GetLatestMutableCFOptions(), mutable_db_options_,
               manual.input_level, manual.output_level, compact_range_options,
               manual.begin, manual.end, &manual.manual_end, &manual_conflict,
               max_file_num_to_ignore, trim_ts)) == nullptr &&
@@ -2676,7 +2670,7 @@ Status DBImpl::WaitUntilFlushWouldNotStallWrites(ColumnFamilyData* cfd,
        return Status::OK();
      }

-      const auto& mutable_cf_options = *cfd->GetLatestMutableCFOptions();
+      const auto& mutable_cf_options = cfd->GetLatestMutableCFOptions();
      const auto* vstorage = cfd->current()->storage_info();

      // Skip stalling check if we're below auto-flush and auto-compaction
@@ -2684,7 +2678,7 @@ Status DBImpl::WaitUntilFlushWouldNotStallWrites(ColumnFamilyData* cfd,
      // triggers are so low that stalling is needed for any background work. In
      // that case we shouldn't wait since background work won't be scheduled.
      if (cfd->imm()->NumNotFlushed() <
-              cfd->ioptions()->min_write_buffer_number_to_merge &&
+              cfd->ioptions().min_write_buffer_number_to_merge &&
          vstorage->l0_delay_trigger_count() <
              mutable_cf_options.level0_file_num_compaction_trigger) {
        break;
@@ -2700,7 +2694,7 @@ Status DBImpl::WaitUntilFlushWouldNotStallWrites(ColumnFamilyData* cfd,
              cfd->GetUnflushedMemTableCountForWriteStallCheck(),
              vstorage->l0_delay_trigger_count() + 1,
              vstorage->estimated_compaction_needed_bytes(), mutable_cf_options,
-              *cfd->ioptions())
+              cfd->ioptions())
              .first;
    } while (write_stall_condition != WriteStallCondition::kNormal);
  }
@@ -3679,8 +3673,8 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
    // Compaction makes a copy of the latest MutableCFOptions. It should be used
    // throughout the compaction procedure to make sure consistency. It will
    // eventually be installed into SuperVersion
-    auto* mutable_cf_options = cfd->GetLatestMutableCFOptions();
-    if (!mutable_cf_options->disable_auto_compactions && !cfd->IsDropped()) {
+    const auto& mutable_cf_options = cfd->GetLatestMutableCFOptions();
+    if (!mutable_cf_options.disable_auto_compactions && !cfd->IsDropped()) {
      // NOTE: try to avoid unnecessary copy of MutableCFOptions if
      // compaction is not necessary. Need to make sure mutex is held
      // until we make a copy in the following code
@@ -3689,7 +3683,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
      std::vector<SequenceNumber> snapshot_seqs;
      // This info is not useful for other scenarios, so save querying existing
      // snapshots for those cases.
-      if (cfd->ioptions()->compaction_style == kCompactionStyleUniversal &&
+      if (cfd->ioptions().compaction_style == kCompactionStyleUniversal &&
          cfd->user_comparator()->timestamp_size() == 0) {
        SequenceNumber earliest_write_conflict_snapshot;
        GetSnapshotContext(job_context, &snapshot_seqs,
@@ -3697,7 +3691,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
                           &snapshot_checker);
        assert(is_snapshot_supported_ || snapshots_.empty());
      }
-      c.reset(cfd->PickCompaction(*mutable_cf_options, mutable_db_options_,
+      c.reset(cfd->PickCompaction(mutable_cf_options, mutable_db_options_,
                                  snapshot_seqs, snapshot_checker, log_buffer));
      TEST_SYNC_POINT("DBImpl::BackgroundCompaction():AfterPickCompaction");

@@ -3711,8 +3705,8 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
          c->column_family_data()
              ->current()
              ->storage_info()
-              ->ComputeCompactionScore(*(c->immutable_options()),
-                                       *(c->mutable_cf_options()));
+              ->ComputeCompactionScore(c->immutable_options(),
+                                       c->mutable_cf_options());
          AddToCompactionQueue(cfd);

          c.reset();
@@ -3760,7 +3754,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
    TEST_SYNC_POINT_CALLBACK("DBImpl::BackgroundCompaction:BeforeCompaction",
                             c->column_family_data());
    assert(c->num_input_files(1) == 0);
-    assert(c->column_family_data()->ioptions()->compaction_style ==
+    assert(c->column_family_data()->ioptions().compaction_style ==
           kCompactionStyleFIFO);

    compaction_job_stats.num_input_files = c->num_input_files(0);
@@ -3772,8 +3766,8 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
      c->edit()->DeleteFile(c->level(), f->fd.GetNumber());
    }
    status = versions_->LogAndApply(
-        c->column_family_data(), *c->mutable_cf_options(), read_options,
-        write_options, c->edit(), &mutex_, directories_.GetDbDir(),
+        c->column_family_data(), read_options, write_options, c->edit(),
+        &mutex_, directories_.GetDbDir(),
        /*new_descriptor_log=*/false, /*column_family_options=*/nullptr,
        [&c, &compaction_released](const Status& s) {
          c->ReleaseCompactionFiles(s);
@@ -3781,8 +3775,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
        });
    io_s = versions_->io_status();
    InstallSuperVersionAndScheduleWork(
-        c->column_family_data(), job_context->superversion_contexts.data(),
-        *c->mutable_cf_options());
+        c->column_family_data(), job_context->superversion_contexts.data());
    ROCKS_LOG_BUFFER(log_buffer, "[%s] Deleted %d files\n",
                     c->column_family_data()->GetName().c_str(),
                     c->num_input_files(0));
@@ -3838,7 +3831,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
      }
    }
    if (c->compaction_reason() == CompactionReason::kLevelMaxLevelSize &&
-        c->immutable_options()->compaction_pri == kRoundRobin) {
+        c->immutable_options().compaction_pri == kRoundRobin) {
      int start_level = c->start_level();
      if (start_level > 0) {
        auto vstorage = c->input_version()->storage_info();
@@ -3848,26 +3841,23 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
      }
    }
    status = versions_->LogAndApply(
-        c->column_family_data(), *c->mutable_cf_options(), read_options,
-        write_options, c->edit(), &mutex_, directories_.GetDbDir(),
+        c->column_family_data(), read_options, write_options, c->edit(),
+        &mutex_, directories_.GetDbDir(),
        /*new_descriptor_log=*/false, /*column_family_options=*/nullptr,
        [&c, &compaction_released](const Status& s) {
          c->ReleaseCompactionFiles(s);
          compaction_released = true;
        });
    io_s = versions_->io_status();
-    // Use latest MutableCFOptions
    InstallSuperVersionAndScheduleWork(
-        c->column_family_data(), job_context->superversion_contexts.data(),
-        *c->mutable_cf_options());
+        c->column_family_data(), job_context->superversion_contexts.data());

    VersionStorageInfo::LevelSummaryStorage tmp;
    c->column_family_data()->internal_stats()->IncBytesMoved(c->output_level(),
                                                             moved_bytes);
    {
      event_logger_.LogToBuffer(log_buffer)
-          << "job" << job_context->job_id << "event"
-          << "trivial_move"
+          << "job" << job_context->job_id << "event" << "trivial_move"
          << "destination_level" << c->output_level() << "files" << moved_files
          << "total_files_size" << moved_bytes;
    }
@@ -3929,15 +3919,15 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
        GetDataDir(c->column_family_data(), 0), stats_, &mutex_,
        &error_handler_, snapshot_seqs, earliest_write_conflict_snapshot,
        snapshot_checker, job_context, table_cache_, &event_logger_,
-        c->mutable_cf_options()->paranoid_file_checks,
-        c->mutable_cf_options()->report_bg_io_stats, dbname_,
+        c->mutable_cf_options().paranoid_file_checks,
+        c->mutable_cf_options().report_bg_io_stats, dbname_,
        &compaction_job_stats, thread_pri, io_tracer_,
        is_manual ? manual_compaction->canceled
                  : kManualCompactionCanceledFalse_,
        db_id_, db_session_id_, c->column_family_data()->GetFullHistoryTsLow(),
        c->trim_ts(), &blob_callback_, &bg_compaction_scheduled_,
        &bg_bottom_compaction_scheduled_);
-    compaction_job.Prepare();
+    compaction_job.Prepare(std::nullopt /*subcompact to be computed*/);

    std::unique_ptr<std::list<uint64_t>::iterator> min_options_file_number_elem;
    if (immutable_db_options().compaction_service != nullptr) {
@@ -3959,13 +3949,11 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
      ReleaseOptionsFileNumber(min_options_file_number_elem);
    }

-    status =
-        compaction_job.Install(*c->mutable_cf_options(), &compaction_released);
+    status = compaction_job.Install(&compaction_released);
    io_s = compaction_job.io_status();
    if (status.ok()) {
      InstallSuperVersionAndScheduleWork(
-          c->column_family_data(), job_context->superversion_contexts.data(),
-          *c->mutable_cf_options());
+          c->column_family_data(), job_context->superversion_contexts.data());
    }
    *made_progress = true;
    TEST_SYNC_POINT_CALLBACK("DBImpl::BackgroundCompaction:AfterCompaction",
@@ -4043,8 +4031,8 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
      c->column_family_data()
          ->current()
          ->storage_info()
-          ->ComputeCompactionScore(*(c->immutable_options()),
-                                   *(c->mutable_cf_options()));
+          ->ComputeCompactionScore(c->immutable_options(),
+                                   c->mutable_cf_options());
      if (!cfd->queued_for_compaction()) {
        AddToCompactionQueue(cfd);
      }
@@ -4079,10 +4067,9 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
      // We only compacted part of the requested range.  Update *m
      // to the range that is left to be compacted.
      // Universal and FIFO compactions should always compact the whole range
-      assert(m->cfd->ioptions()->compaction_style !=
-                 kCompactionStyleUniversal ||
-             m->cfd->ioptions()->num_levels > 1);
-      assert(m->cfd->ioptions()->compaction_style != kCompactionStyleFIFO);
+      assert(m->cfd->ioptions().compaction_style != kCompactionStyleUniversal ||
+             m->cfd->ioptions().num_levels > 1);
+      assert(m->cfd->ioptions().compaction_style != kCompactionStyleFIFO);
      m->tmp_storage = *m->manual_end;
      m->begin = &m->tmp_storage;
      m->incomplete = true;
@@ -4229,7 +4216,7 @@ void DBImpl::BuildCompactionJobInfo(
    for (const auto fmd : *c->inputs(i)) {
      const FileDescriptor& desc = fmd->fd;
      const uint64_t file_number = desc.GetNumber();
-      auto fn = TableFileName(c->immutable_options()->cf_paths, file_number,
+      auto fn = TableFileName(c->immutable_options().cf_paths, file_number,
                              desc.GetPathId());
      compaction_job_info->input_files.push_back(fn);
      compaction_job_info->input_file_infos.push_back(CompactionFileInfo{
@@ -4242,17 +4229,17 @@ void DBImpl::BuildCompactionJobInfo(
    const FileDescriptor& desc = meta.fd;
    const uint64_t file_number = desc.GetNumber();
    compaction_job_info->output_files.push_back(TableFileName(
-        c->immutable_options()->cf_paths, file_number, desc.GetPathId()));
+        c->immutable_options().cf_paths, file_number, desc.GetPathId()));
    compaction_job_info->output_file_infos.push_back(CompactionFileInfo{
        newf.first, file_number, meta.oldest_blob_file_number});
  }
  compaction_job_info->blob_compression_type =
-      c->mutable_cf_options()->blob_compression_type;
+      c->mutable_cf_options().blob_compression_type;

  // Update BlobFilesInfo.
  for (const auto& blob_file : c->edit()->GetBlobFileAdditions()) {
    BlobFileAdditionInfo blob_file_addition_info(
-        BlobFileName(c->immutable_options()->cf_paths.front().path,
+        BlobFileName(c->immutable_options().cf_paths.front().path,
                     blob_file.GetBlobFileNumber()) /*blob_file_path*/,
        blob_file.GetBlobFileNumber(), blob_file.GetTotalBlobCount(),
        blob_file.GetTotalBlobBytes());
@@ -4263,7 +4250,7 @@ void DBImpl::BuildCompactionJobInfo(
  // Update BlobFilesGarbageInfo.
  for (const auto& blob_file : c->edit()->GetBlobFileGarbages()) {
    BlobFileGarbageInfo blob_file_garbage_info(
-        BlobFileName(c->immutable_options()->cf_paths.front().path,
+        BlobFileName(c->immutable_options().cf_paths.front().path,
                     blob_file.GetBlobFileNumber()) /*blob_file_path*/,
        blob_file.GetBlobFileNumber(), blob_file.GetGarbageBlobCount(),
        blob_file.GetGarbageBlobBytes());
@@ -4285,9 +4272,9 @@ void DBImpl::BuildCompactionJobInfo(
 // for superversion_to_free

 void DBImpl::InstallSuperVersionAndScheduleWork(
-    ColumnFamilyData* cfd, SuperVersionContext* sv_context,
-    const MutableCFOptions& mutable_cf_options) {
+    ColumnFamilyData* cfd, SuperVersionContext* sv_context) {
  mutex_.AssertHeld();
+  const auto& mutable_cf_options = cfd->GetLatestMutableCFOptions();

  // Update max_total_in_memory_state_
  size_t old_memtable_size = 0;
@@ -4310,7 +4297,7 @@ void DBImpl::InstallSuperVersionAndScheduleWork(
  bottommost_files_mark_threshold_ = kMaxSequenceNumber;
  standalone_range_deletion_files_mark_threshold_ = kMaxSequenceNumber;
  for (auto* my_cfd : *versions_->GetColumnFamilySet()) {
-    if (!my_cfd->ioptions()->allow_ingest_behind) {
+    if (!my_cfd->ioptions().allow_ingest_behind) {
      bottommost_files_mark_threshold_ = std::min(
          bottommost_files_mark_threshold_,
          my_cfd->current()->storage_info()->bottommost_files_mark_threshold());
@@ -103,8 +103,8 @@ Status DBImpl::TEST_CompactRange(int level, const Slice* begin,
    cfd = cfh->cfd();
  }
  int output_level =
-      (cfd->ioptions()->compaction_style == kCompactionStyleUniversal ||
-       cfd->ioptions()->compaction_style == kCompactionStyleFIFO)
+      (cfd->ioptions().compaction_style == kCompactionStyleUniversal ||
+       cfd->ioptions().compaction_style == kCompactionStyleFIFO)
          ? level
          : level + 1;
  return RunManualCompaction(
@@ -239,7 +239,7 @@ void DBImpl::TEST_GetAllBlockCaches(
  for (auto cfd : *versions_->GetColumnFamilySet()) {
    if (const auto bbto =
            cfd->GetCurrentMutableCFOptions()
-                ->table_factory->GetOptions<BlockBasedTableOptions>()) {
+                .table_factory->GetOptions<BlockBasedTableOptions>()) {
      cache_set->insert(bbto->block_cache.get());
    }
  }
@@ -267,7 +267,7 @@ Status DBImpl::TEST_GetLatestMutableCFOptions(
  InstrumentedMutexLock l(&mutex_);

  auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(column_family);
-  *mutable_cf_options = *cfh->cfd()->GetLatestMutableCFOptions();
+  *mutable_cf_options = cfh->cfd()->GetLatestMutableCFOptions();
  return Status::OK();
 }

@@ -45,8 +45,8 @@ Status DBImpl::SuggestCompactRange(ColumnFamilyHandle* column_family,
    }
    // Since we have some more files to compact, we should also recompute
    // compaction score
-    vstorage->ComputeCompactionScore(*cfd->ioptions(),
-                                     *cfd->GetLatestMutableCFOptions());
+    vstorage->ComputeCompactionScore(cfd->ioptions(),
+                                     cfd->GetLatestMutableCFOptions());
    EnqueuePendingCompaction(cfd);
    MaybeScheduleFlushOrCompaction();
  }
@@ -146,13 +146,11 @@ Status DBImpl::PromoteL0(ColumnFamilyHandle* column_family, int target_level) {
                   f->user_defined_timestamps_persisted);
    }

-    status = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),
-                                    read_options, write_options, &edit, &mutex_,
-                                    directories_.GetDbDir());
+    status = versions_->LogAndApply(cfd, read_options, write_options, &edit,
+                                    &mutex_, directories_.GetDbDir());
    if (status.ok()) {
      InstallSuperVersionAndScheduleWork(
-          cfd, job_context.superversion_contexts.data(),
-          *cfd->GetLatestMutableCFOptions());
+          cfd, job_context.superversion_contexts.data());
    }
  }  // lock released here
  LogFlush(immutable_db_options_.info_log);
@@ -159,6 +159,7 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,

  // Get obsolete files.  This function will also update the list of
  // pending files in VersionSet().
+  assert(versions_);
  versions_->GetObsoleteFiles(
      &job_context->sst_delete_files, &job_context->blob_delete_files,
      &job_context->manifest_delete_files, job_context->min_pending_output);
@@ -1039,7 +1040,7 @@ std::set<std::string> DBImpl::CollectAllDBPaths() {
    all_db_paths.insert(NormalizePath(db_path.path));
  }
  for (const auto* cfd : *versions_->GetColumnFamilySet()) {
-    for (const auto& cf_path : cfd->ioptions()->cf_paths) {
+    for (const auto& cf_path : cfd->ioptions().cf_paths) {
      all_db_paths.insert(NormalizePath(cf_path.path));
    }
  }
@@ -140,10 +140,8 @@ Status DBImplFollower::TryCatchUpWithLeader() {
          // to super versions in a lock free manner, it checks the earliest
          // sequence number to detect if there was a change in version in
          // the meantime.
-          const MutableCFOptions mutable_cf_options =
-              *cfd->GetLatestMutableCFOptions();
          MemTable* new_mem = cfd->ConstructNewMemtable(
-              mutable_cf_options, versions_->LastSequence());
+              cfd->GetLatestMutableCFOptions(), versions_->LastSequence());
          cfd->mem()->SetNextLogNumber(cfd->GetLogNumber());
          cfd->mem()->ConstructFragmentedRangeTombstones();
          cfd->imm()->Add(cfd->mem(), &job_context.memtables_to_free);
@@ -16,7 +16,6 @@

 namespace ROCKSDB_NAMESPACE {

-
 DBImplReadOnly::DBImplReadOnly(const DBOptions& db_options,
                               const std::string& dbname)
    : DBImpl(db_options, dbname, /*seq_per_batch*/ false,
@@ -173,7 +172,7 @@ Iterator* DBImplReadOnly::NewIterator(const ReadOptions& _read_options,
          : latest_snapshot;
  ReadCallback* read_callback = nullptr;  // No read callback provided.
  auto db_iter = NewArenaWrappedDbIterator(
-      env_, read_options, *cfd->ioptions(), super_version->mutable_cf_options,
+      env_, read_options, cfd->ioptions(), super_version->mutable_cf_options,
      super_version->current, read_seq,
      super_version->mutable_cf_options.max_sequential_skip_in_iterations,
      super_version->version_number, read_callback);
@@ -240,7 +239,7 @@ Status DBImplReadOnly::NewIterators(
  assert(cfd_to_sv.size() == column_families.size());
  for (auto [cfd, sv] : cfd_to_sv) {
    auto* db_iter = NewArenaWrappedDbIterator(
-        env_, read_options, *cfd->ioptions(), sv->mutable_cf_options,
+        env_, read_options, cfd->ioptions(), sv->mutable_cf_options,
        sv->current, read_seq,
        sv->mutable_cf_options.max_sequential_skip_in_iterations,
        sv->version_number, read_callback);
@@ -276,7 +275,8 @@ Status OpenForReadOnlyCheckExistence(const DBOptions& db_options,
 }  // namespace

 Status DB::OpenForReadOnly(const Options& options, const std::string& dbname,
-                           DB** dbptr, bool /*error_if_wal_file_exists*/) {
+                           std::unique_ptr<DB>* dbptr,
+                           bool /*error_if_wal_file_exists*/) {
  Status s = OpenForReadOnlyCheckExistence(options, dbname);
  if (!s.ok()) {
    return s;
@@ -310,7 +310,7 @@ Status DB::OpenForReadOnly(const Options& options, const std::string& dbname,
 Status DB::OpenForReadOnly(
    const DBOptions& db_options, const std::string& dbname,
    const std::vector<ColumnFamilyDescriptor>& column_families,
-    std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
+    std::vector<ColumnFamilyHandle*>* handles, std::unique_ptr<DB>* dbptr,
    bool error_if_wal_file_exists) {
  // If dbname does not exist in the file system, should not do anything
  Status s = OpenForReadOnlyCheckExistence(db_options, dbname);
@@ -326,7 +326,7 @@ Status DB::OpenForReadOnly(
 Status DBImplReadOnly::OpenForReadOnlyWithoutCheck(
    const DBOptions& db_options, const std::string& dbname,
    const std::vector<ColumnFamilyDescriptor>& column_families,
-    std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
+    std::vector<ColumnFamilyHandle*>* handles, std::unique_ptr<DB>* dbptr,
    bool error_if_wal_file_exists) {
  *dbptr = nullptr;
  handles->clear();
@@ -357,7 +357,7 @@ Status DBImplReadOnly::OpenForReadOnlyWithoutCheck(
  impl->mutex_.Unlock();
  sv_context.Clean();
  if (s.ok()) {
-    *dbptr = impl;
+    dbptr->reset(impl);
    for (auto* h : *handles) {
      impl->NewThreadStatusCfInfo(
          static_cast_with_check<ColumnFamilyHandleImpl>(h)->cfd());
@@ -372,5 +372,4 @@ Status DBImplReadOnly::OpenForReadOnlyWithoutCheck(
  return s;
 }

-
 }  // namespace ROCKSDB_NAMESPACE
@@ -5,7 +5,6 @@

 #pragma once

-
 #include <string>
 #include <vector>

@@ -172,7 +171,7 @@ class DBImplReadOnly : public DBImpl {
  static Status OpenForReadOnlyWithoutCheck(
      const DBOptions& db_options, const std::string& dbname,
      const std::vector<ColumnFamilyDescriptor>& column_families,
-      std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
+      std::vector<ColumnFamilyHandle*>* handles, std::unique_ptr<DB>* dbptr,
      bool error_if_wal_file_exists = false);
  friend class DB;
 };
@@ -55,9 +55,9 @@ Status DBImplSecondary::Recover(
  // Initial max_total_in_memory_state_ before recovery logs.
  max_total_in_memory_state_ = 0;
  for (auto cfd : *versions_->GetColumnFamilySet()) {
-    auto* mutable_cf_options = cfd->GetLatestMutableCFOptions();
-    max_total_in_memory_state_ += mutable_cf_options->write_buffer_size *
-                                  mutable_cf_options->max_write_buffer_number;
+    const auto& mutable_cf_options = cfd->GetLatestMutableCFOptions();
+    max_total_in_memory_state_ += mutable_cf_options.write_buffer_size *
+                                  mutable_cf_options.max_write_buffer_number;
  }
  if (s.ok()) {
    default_cf_handle_ = new ColumnFamilyHandleImpl(
@@ -270,10 +270,8 @@ Status DBImplSecondary::RecoverLogFiles(
          if (!cfd->mem()->IsEmpty() &&
              (curr_log_num == std::numeric_limits<uint64_t>::max() ||
               curr_log_num != log_number)) {
-            const MutableCFOptions mutable_cf_options =
-                *cfd->GetLatestMutableCFOptions();
-            MemTable* new_mem =
-                cfd->ConstructNewMemtable(mutable_cf_options, seq_of_batch);
+            MemTable* new_mem = cfd->ConstructNewMemtable(
+                cfd->GetLatestMutableCFOptions(), seq_of_batch);
            cfd->mem()->SetNextLogNumber(log_number);
            cfd->mem()->ConstructFragmentedRangeTombstones();
            cfd->imm()->Add(cfd->mem(), &job_context->memtables_to_free);
@@ -533,7 +531,7 @@ ArenaWrappedDBIter* DBImplSecondary::NewIteratorImpl(
  snapshot = versions_->LastSequence();
  assert(snapshot != kMaxSequenceNumber);
  auto db_iter = NewArenaWrappedDbIterator(
-      env_, read_options, *cfh->cfd()->ioptions(),
+      env_, read_options, cfh->cfd()->ioptions(),
      super_version->mutable_cf_options, super_version->current, snapshot,
      super_version->mutable_cf_options.max_sequential_skip_in_iterations,
      super_version->version_number, read_callback, cfh, expose_blob_index,
@@ -737,7 +735,8 @@ Status DBImplSecondary::TryCatchUpWithPrimary() {
 }

 Status DB::OpenAsSecondary(const Options& options, const std::string& dbname,
-                           const std::string& secondary_path, DB** dbptr) {
+                           const std::string& secondary_path,
+                           std::unique_ptr<DB>* dbptr) {
  *dbptr = nullptr;

  DBOptions db_options(options);
@@ -759,7 +758,7 @@ Status DB::OpenAsSecondary(
    const DBOptions& db_options, const std::string& dbname,
    const std::string& secondary_path,
    const std::vector<ColumnFamilyDescriptor>& column_families,
-    std::vector<ColumnFamilyHandle*>* handles, DB** dbptr) {
+    std::vector<ColumnFamilyHandle*>* handles, std::unique_ptr<DB>* dbptr) {
  *dbptr = nullptr;

  DBOptions tmp_opts(db_options);
@@ -826,7 +825,7 @@ Status DB::OpenAsSecondary(
  impl->mutex_.Unlock();
  sv_context.Clean();
  if (s.ok()) {
-    *dbptr = impl;
+    dbptr->reset(impl);
    for (auto h : *handles) {
      impl->NewThreadStatusCfInfo(
          static_cast_with_check<ColumnFamilyHandleImpl>(h)->cfd());
@@ -864,16 +863,15 @@ Status DBImplSecondary::CompactWithoutInstallation(
  ColumnFamilyMetaData cf_meta;
  version->GetColumnFamilyMetaData(&cf_meta);

-  const MutableCFOptions* mutable_cf_options = cfd->GetLatestMutableCFOptions();
-  ColumnFamilyOptions cf_options = cfd->GetLatestCFOptions();
  VersionStorageInfo* vstorage = version->storage_info();

  // Use comp_options to reuse some CompactFiles functions
  CompactionOptions comp_options;
  comp_options.compression = kDisableCompressionOption;
  comp_options.output_file_size_limit = MaxFileSizeForLevel(
-      *mutable_cf_options, input.output_level, cf_options.compaction_style,
-      vstorage->base_level(), cf_options.level_compaction_dynamic_level_bytes);
+      cfd->GetLatestMutableCFOptions(), input.output_level,
+      cfd->ioptions().compaction_style, vstorage->base_level(),
+      cfd->ioptions().level_compaction_dynamic_level_bytes);

  std::vector<CompactionInputFiles> input_files;
  Status s = cfd->compaction_picker()->GetCompactionInputsFromFileNumbers(
@@ -886,7 +884,7 @@ Status DBImplSecondary::CompactWithoutInstallation(
  assert(cfd->compaction_picker());
  c.reset(cfd->compaction_picker()->CompactFiles(
      comp_options, input_files, input.output_level, vstorage,
-      *mutable_cf_options, mutable_db_options_, 0));
+      cfd->GetLatestMutableCFOptions(), mutable_db_options_, 0));
  assert(c != nullptr);

  c->FinalizeInputInfo(version);
@@ -917,6 +915,8 @@ Status DBImplSecondary::CompactWithoutInstallation(
      options.canceled ? *options.canceled : kManualCompactionCanceledFalse_,
      input.db_id, db_session_id_, secondary_path_, input, result);

+  compaction_job.Prepare();
+
  mutex_.Unlock();
  s = compaction_job.Run();
  mutex_.Lock();
@@ -1057,5 +1057,4 @@ Status DB::OpenAndCompact(
                        output, override_options);
 }

-
 }  // namespace ROCKSDB_NAMESPACE
@@ -5,7 +5,6 @@

 #pragma once

-
 #include <string>
 #include <vector>

@@ -67,7 +67,7 @@ Status DBImpl::Merge(const WriteOptions& o, ColumnFamilyHandle* column_family,
    return s;
  }
  auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(column_family);
-  if (!cfh->cfd()->ioptions()->merge_operator) {
+  if (!cfh->cfd()->ioptions().merge_operator) {
    return Status::NotSupported("Provide a merge_operator when opening DB");
  } else {
    return DB::Merge(o, column_family, key, val);
@@ -232,8 +232,8 @@ Status DBImpl::IngestWBWI(std::shared_ptr<WriteBatchWithIndex> wbwi,
      return s;
    }
    WBWIMemTable* wbwi_memtable =
-        new WBWIMemTable(wbwi, cfd->user_comparator(), cf_id, cfd->ioptions(),
-                         cfd->GetLatestMutableCFOptions(), stat);
+        new WBWIMemTable(wbwi, cfd->user_comparator(), cf_id, &cfd->ioptions(),
+                         &cfd->GetLatestMutableCFOptions(), stat);
    wbwi_memtable->Ref();
    wbwi_memtable->AssignSequenceNumbers(assigned_seqno);
    // This is needed to keep the WAL that contains Prepare alive until
@@ -1558,7 +1558,8 @@ IOStatus DBImpl::WriteToWAL(const WriteBatch& merged_batch,
                            const WriteOptions& write_options,
                            log::Writer* log_writer, uint64_t* log_used,
                            uint64_t* log_size,
-                            LogFileNumberSize& log_file_number_size) {
+                            LogFileNumberSize& log_file_number_size,
+                            SequenceNumber sequence) {
  assert(log_size != nullptr);

  Slice log_entry = WriteBatchInternal::Contents(&merged_batch);
@@ -1584,7 +1585,7 @@ IOStatus DBImpl::WriteToWAL(const WriteBatch& merged_batch,
  if (!io_s.ok()) {
    return io_s;
  }
-  io_s = log_writer->AddRecord(write_options, log_entry);
+  io_s = log_writer->AddRecord(write_options, log_entry, sequence);

  if (UNLIKELY(needs_locking)) {
    log_write_mutex_.Unlock();
@@ -1634,7 +1635,7 @@ IOStatus DBImpl::WriteToWAL(const WriteThread::WriteGroup& write_group,
  write_options.rate_limiter_priority =
      write_group.leader->rate_limiter_priority;
  io_s = WriteToWAL(*merged_batch, write_options, log_writer, log_used,
-                    &log_size, log_file_number_size);
+                    &log_size, log_file_number_size, sequence);
  if (to_be_cached_state) {
    cached_recoverable_state_ = *to_be_cached_state;
    cached_recoverable_state_empty_ = false;
@@ -1760,7 +1761,7 @@ IOStatus DBImpl::ConcurrentWriteToWAL(
  write_options.rate_limiter_priority =
      write_group.leader->rate_limiter_priority;
  io_s = WriteToWAL(*merged_batch, write_options, log_writer, log_used,
-                    &log_size, log_file_number_size);
+                    &log_size, log_file_number_size, sequence);
  if (to_be_cached_state) {
    cached_recoverable_state_ = *to_be_cached_state;
    cached_recoverable_state_empty_ = false;
@@ -2421,7 +2422,9 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
  }
  uint64_t new_log_number =
      creating_new_log ? versions_->NewFileNumber() : logfile_number_;
-  const MutableCFOptions mutable_cf_options = *cfd->GetLatestMutableCFOptions();
+  // For use outside of holding DB mutex
+  const MutableCFOptions mutable_cf_options_copy =
+      cfd->GetLatestMutableCFOptions();

  // Set memtable_info for memtable sealed callback
  // TODO: memtable_info for `new_imm`
@@ -2431,7 +2434,7 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
  memtable_info.earliest_seqno = cfd->mem()->GetEarliestSequenceNumber();
  memtable_info.num_entries = cfd->mem()->NumEntries();
  memtable_info.num_deletes = cfd->mem()->NumDeletion();
-  if (!cfd->ioptions()->persist_user_defined_timestamps &&
+  if (!cfd->ioptions().persist_user_defined_timestamps &&
      cfd->user_comparator()->timestamp_size() > 0) {
    const Slice& newest_udt = cfd->mem()->GetNewestUDT();
    memtable_info.newest_udt.assign(newest_udt.data(), newest_udt.size());
@@ -2440,13 +2443,22 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
  // flush happens before logging, but that should be ok.
  int num_imm_unflushed = cfd->imm()->NumNotFlushed();
  const auto preallocate_block_size =
-      GetWalPreallocateBlockSize(mutable_cf_options.write_buffer_size);
+      GetWalPreallocateBlockSize(mutable_cf_options_copy.write_buffer_size);
  mutex_.Unlock();
  if (creating_new_log) {
+    PredecessorWALInfo info;
+    log_write_mutex_.Lock();
+    if (!logs_.empty()) {
+      log::Writer* cur_log_writer = logs_.back().writer;
+      info = PredecessorWALInfo(cur_log_writer->get_log_number(),
+                                cur_log_writer->file()->GetFileSize(),
+                                cur_log_writer->GetLastSeqnoRecorded());
+    }
+    log_write_mutex_.Unlock();
    // TODO: Write buffer size passed in should be max of all CF's instead
    // of mutable_cf_options.write_buffer_size.
    io_s = CreateWAL(write_options, new_log_number, recycle_log_number,
-                     preallocate_block_size, &new_log);
+                     preallocate_block_size, info, &new_log);
    if (s.ok()) {
      s = io_s;
    }
@@ -2464,8 +2476,8 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
    } else {
      seq = versions_->LastSequence();
    }
-    new_mem =
-        cfd->ConstructNewMemtable(mutable_cf_options, /*earliest_seq=*/seq);
+    new_mem = cfd->ConstructNewMemtable(mutable_cf_options_copy,
+                                        /*earliest_seq=*/seq);
    context->superversion_context.NewSuperVersion();

    ROCKS_LOG_INFO(immutable_db_options_.info_log,
@@ -2620,8 +2632,7 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
  }
  new_mem->Ref();
  cfd->SetMemtable(new_mem);
-  InstallSuperVersionAndScheduleWork(cfd, &context->superversion_context,
-                                     mutable_cf_options);
+  InstallSuperVersionAndScheduleWork(cfd, &context->superversion_context);

  // Notify client that memtable is sealed, now that we have successfully
  // installed a new memtable
@@ -97,12 +97,20 @@ class CorruptionFS : public FileSystemWrapper {
        for (size_t i = 0; i < num_reqs; ++i) {
          FSReadRequest& req = reqs[i];
          if (fs_.fs_buffer_) {
-            FSAllocationPtr buffer(new char[req.len], [](void* ptr) {
-              delete[] static_cast<char*>(ptr);
-            });
-            req.fs_scratch = std::move(buffer);
+            // See https://github.com/facebook/rocksdb/pull/13195 for why we
+            // want to set up our test implementation for FSAllocationPtr this
+            // way.
+            char* internalData = new char[req.len];
            req.status = Read(req.offset, req.len, options, &req.result,
-                              static_cast<char*>(req.fs_scratch.get()), dbg);
+                              internalData, dbg);
+
+            Slice* internalSlice = new Slice(internalData, req.len);
+            FSAllocationPtr internalPtr(internalSlice, [](void* ptr) {
+              delete[] static_cast<const char*>(
+                  static_cast<Slice*>(ptr)->data_);
+              delete static_cast<Slice*>(ptr);
+            });
+            req.fs_scratch = std::move(internalPtr);
          } else {
            req.status = Read(req.offset, req.len, options, &req.result,
                              req.scratch, dbg);
@@ -332,7 +332,6 @@ TEST_F(DBTestXactLogIterator, TransactionLogIteratorBlobs) {
 }
 }  // namespace ROCKSDB_NAMESPACE

-
 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
@@ -5,11 +5,11 @@

 #include "db/db_test_util.h"
 #include "port/stack_trace.h"
+#include "rocksdb/merge_operator.h"
 #include "rocksdb/perf_context.h"
 #include "rocksdb/utilities/debug.h"
 #include "table/block_based/block_builder.h"
 #include "test_util/sync_point.h"
-#include "rocksdb/merge_operator.h"
 #include "utilities/fault_injection_env.h"
 #include "utilities/merge_operators.h"
 #include "utilities/merge_operators/sortlist.h"
@@ -1451,7 +1451,6 @@ TEST_F(DBOptionsTest, ChangeCompression) {
  SyncPoint::GetInstance()->DisableProcessing();
 }

-
 TEST_F(DBOptionsTest, BottommostCompressionOptsWithFallbackType) {
  // Verify the bottommost compression options still take effect even when the
  // bottommost compression type is left at its default value. Verify for both
@@ -226,7 +226,6 @@ TEST_P(DBRateLimiterOnReadTest, Iterator) {
  ASSERT_EQ(expected, options_.rate_limiter->GetTotalRequests(Env::IO_USER));
 }

-
 TEST_P(DBRateLimiterOnReadTest, VerifyChecksum) {
  if (use_direct_io_ && !IsDirectIOSupported()) {
    return;
@@ -271,7 +270,6 @@ TEST_P(DBRateLimiterOnReadTest, VerifyFileChecksums) {
  ASSERT_EQ(expected, options_.rate_limiter->GetTotalRequests(Env::IO_USER));
 }

-
 class DBRateLimiterOnWriteTest : public DBTestBase {
 public:
  explicit DBRateLimiterOnWriteTest()
@@ -1991,7 +1991,6 @@ TEST_F(DBSSTTest, DBWithSFMForBlobFilesAtomicFlush) {
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
 }

-
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -198,7 +198,6 @@ TEST_F(DBStatisticsTest, ExcludeTickers) {
  ASSERT_GT(options.statistics->getTickerCount(BYTES_READ), 0);
 }

-
 TEST_F(DBStatisticsTest, VerifyChecksumReadStat) {
  Options options = CurrentOptions();
  options.file_checksum_gen_factory = GetFileChecksumGenCrc32cFactory();
@@ -25,7 +25,6 @@
 #include "util/atomic.h"
 #include "util/random.h"

-
 namespace ROCKSDB_NAMESPACE {

 // A helper function that ensures the table properties returned in
@@ -744,7 +743,6 @@ INSTANTIATE_TEST_CASE_P(DBTablePropertiesTest, DBTablePropertiesTest,

 }  // namespace ROCKSDB_NAMESPACE

-
 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
@@ -703,7 +703,6 @@ TEST_P(DBTestTailingIterator, SeekToFirstWithUpperBoundBug) {

 }  // namespace ROCKSDB_NAMESPACE

-
 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
@@ -1161,7 +1161,6 @@ class DelayFilterFactory : public CompactionFilterFactory {
 };
 }  // anonymous namespace

-
 static std::string CompressibleString(Random* rnd, int len) {
  std::string r;
  test::CompressibleString(rnd, 0.8, len, &r);
@@ -3410,7 +3409,9 @@ class ModelDB : public DB {
    return Status::NotSupported();
  }

-  Status DeleteFile(std::string /*name*/) override { return Status::OK(); }
+  Status DEPRECATED_DeleteFile(std::string /*name*/) override {
+    return Status::OK();
+  }

  Status GetUpdatesSince(
      ROCKSDB_NAMESPACE::SequenceNumber,
@@ -3715,6 +3716,139 @@ TEST_F(DBTest, BlockBasedTablePrefixIndexTest) {
  ASSERT_EQ("v2", Get("k2"));
 }

+TEST_F(DBTest, SetOptionsEffectiveInSuperVersions) {
+  // Basically, to test the SetOptions take effect with (and only with)
+  // new SuperVersions, and remain in effect through some things like
+  // flush and compaction, we use some queries that depend on the current
+  // prefix extractor.
+  //
+  // Making the semantics of read options dependent on the current state of
+  // mutable options is kind of an anti-pattern that prefix_seek_opt_in_only
+  // is helping to phase out. However, this is useful for rather directly
+  // testing the expected behavior of mutable options handling.
+  ReadOptions ropts;
+  ropts.prefix_same_as_start = true;
+  Options options = CurrentOptions();
+  options.prefix_extractor.reset(NewFixedPrefixTransform(5));
+  options.prefix_seek_opt_in_only = false;
+  Reopen(options);
+
+  ASSERT_OK(Put("goat1", "g1"));
+  ASSERT_OK(Put("goat2", "g2"));
+
+  std::unique_ptr<Iterator> iter(db_->NewIterator(ropts));
+
+  auto VerifyTransform4 = [&](int caller_line) {
+    SCOPED_TRACE("Called from " + std::to_string(caller_line));
+    // Nothing with this prefix
+    iter->Seek("game1");
+    ASSERT_OK(iter->status());
+    ASSERT_FALSE(iter->Valid());
+
+    iter->Seek("goat1");
+    ASSERT_OK(iter->status());
+    ASSERT_TRUE(iter->Valid());
+    ASSERT_EQ("goat1", iter->key());
+    iter->Next();
+    ASSERT_OK(iter->status());
+    ASSERT_TRUE(iter->Valid());
+    ASSERT_EQ("goat2", iter->key());
+  };
+
+  auto VerifyTransform5 = [&](int caller_line) {
+    SCOPED_TRACE("Called from " + std::to_string(caller_line));
+    // Nothing with this prefix
+    iter->Seek("game1");
+    ASSERT_OK(iter->status());
+    ASSERT_FALSE(iter->Valid());
+
+    iter->Seek("goat1");
+    ASSERT_OK(iter->status());
+    ASSERT_TRUE(iter->Valid());
+    ASSERT_EQ("goat1", iter->key());
+    iter->Next();
+    ASSERT_OK(iter->status());
+    ASSERT_FALSE(iter->Valid());
+
+    iter->Seek("goat2");
+    ASSERT_OK(iter->status());
+    ASSERT_TRUE(iter->Valid());
+    ASSERT_EQ("goat2", iter->key());
+    iter->Next();
+    ASSERT_OK(iter->status());
+    ASSERT_FALSE(iter->Valid());
+  };
+
+  for (int i = 0;; ++i) {
+    SCOPED_TRACE("Iteration " + std::to_string(i));
+    // Baseline
+    VerifyTransform5(__LINE__);
+
+    if (i == 0) {
+      // Test a "normal" change with nothing happening in parallel
+      ASSERT_OK(db_->SetOptions({{"prefix_extractor", "fixed:4"}}));
+
+      // Iterator still uses old superversion
+      VerifyTransform5(__LINE__);
+
+      // Refresh updates the SuperVersion
+      ASSERT_OK(iter->Refresh());
+    } else if (i == 1) {
+      // Test a setting change in parallel with flush
+      iter = nullptr;
+      SyncPoint::GetInstance()->DisableProcessing();
+      SyncPoint::GetInstance()->ClearAllCallBacks();
+      SyncPoint::GetInstance()->SetCallBack(
+          "FlushJob::WriteLevel0Table:num_memtables", [&](void*) {
+            // During flush, without DB mutex held
+            ASSERT_OK(db_->SetOptions({{"prefix_extractor", "fixed:4"}}));
+            iter.reset(db_->NewIterator(ropts));
+            VerifyTransform4(__LINE__);
+          });
+      SyncPoint::GetInstance()->EnableProcessing();
+      ASSERT_OK(Flush());
+      SyncPoint::GetInstance()->DisableProcessing();
+      SyncPoint::GetInstance()->ClearAllCallBacks();
+      // Callback was called
+      ASSERT_NE(iter, nullptr);
+    } else if (i == 2) {
+      // Test a setting change in parallel with compaction
+      iter = nullptr;
+      SyncPoint::GetInstance()->DisableProcessing();
+      SyncPoint::GetInstance()->ClearAllCallBacks();
+      SyncPoint::GetInstance()->SetCallBack(
+          "CompactionJob::Run():EndStatusSet", [&](void*) {
+            // During compaction, without DB mutex held
+            ASSERT_OK(db_->SetOptions({{"prefix_extractor", "fixed:4"}}));
+            iter.reset(db_->NewIterator(ropts));
+            VerifyTransform4(__LINE__);
+          });
+      SyncPoint::GetInstance()->EnableProcessing();
+      // Need data overlapping that L0 file to prevent trivial move
+      ASSERT_OK(Put("aaaaa", "a"));
+      ASSERT_OK(Put("zzzzz", "a"));
+      ASSERT_OK(CompactRange({}, {}, {}));
+      SyncPoint::GetInstance()->DisableProcessing();
+      SyncPoint::GetInstance()->ClearAllCallBacks();
+      // Callback was called
+      ASSERT_NE(iter, nullptr);
+    } else {
+      break;
+    }
+    // Change has taken effect
+    VerifyTransform4(__LINE__);
+    // Same after a new iterator (in case a new SuperVersion reverted the
+    // setting)
+    iter.reset(db_->NewIterator(ropts));
+    VerifyTransform4(__LINE__);
+
+    // Back to baseline setting
+    ASSERT_OK(db_->SetOptions({{"prefix_extractor", "fixed:5"}}));
+    // New iterator uses latest SuperVersion
+    iter.reset(db_->NewIterator(ropts));
+  }
+}
+
 TEST_F(DBTest, BlockBasedTablePrefixHashIndexTest) {
  // create a DB with block prefix index
  BlockBasedTableOptions table_options;
@@ -4378,7 +4512,6 @@ TEST_F(DBTest, ConcurrentMemtableNotSupported) {
  ASSERT_NOK(db_->CreateColumnFamily(cf_options, "name", &handle));
 }

-
 TEST_F(DBTest, SanitizeNumThreads) {
  for (int attempt = 0; attempt < 2; attempt++) {
    const size_t kTotalTasks = 8;
@@ -5240,7 +5373,7 @@ TEST_F(DBTest, DynamicLevelCompressionPerLevel) {
  db_->GetColumnFamilyMetaData(&cf_meta);
  for (const auto& file : cf_meta.levels[4].files) {
    listener->SetExpectedFileName(dbname_ + file.name);
-    ASSERT_OK(dbfull()->DeleteFile(file.name));
+    ASSERT_OK(dbfull()->DEPRECATED_DeleteFile(file.name));
  }
  listener->VerifyMatchedCount(cf_meta.levels[4].files.size());

@@ -5788,7 +5921,6 @@ TEST_F(DBTest, FileCreationRandomFailure) {
  }
 }

-
 TEST_F(DBTest, DynamicMiscOptions) {
  // Test max_sequential_skip_in_iterations
  Options options;
@@ -7230,7 +7362,6 @@ TEST_F(DBTest, ReusePinnableSlice) {
            1);
 }

-
 TEST_F(DBTest, DeletingOldWalAfterDrop) {
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
      {{"Test:AllowFlushes", "DBImpl::BGWorkFlush"},
@@ -7355,7 +7486,6 @@ TEST_F(DBTest, LargeBlockSizeTest) {
  ASSERT_NOK(TryReopenWithColumnFamilies({"default", "pikachu"}, options));
 }

-
 TEST_F(DBTest, CreationTimeOfOldestFile) {
  const int kNumKeysPerFile = 32;
  const int kNumLevelFiles = 2;
@@ -135,7 +135,6 @@ TEST_F(DBTest2, PartitionedIndexUserToInternalKey) {
  }
 }

-
 class PrefixFullBloomWithReverseComparator
    : public DBTestBase,
      public ::testing::WithParamInterface<bool> {
@@ -1977,7 +1976,6 @@ TEST_F(DBTest2, CompactionStall) {
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }

-
 TEST_F(DBTest2, FirstSnapshotTest) {
  Options options;
  options.write_buffer_size = 100000;  // Small write buffer
@@ -3600,7 +3598,6 @@ TEST_F(DBTest2, OptimizeForSmallDB) {
  value.Reset();
 }

-
 TEST_F(DBTest2, IterRaceFlush1) {
  ASSERT_OK(Put("foo", "v1"));

@@ -4104,7 +4101,6 @@ TEST_F(DBTest2, ReadCallbackTest) {
  }
 }

-
 TEST_F(DBTest2, LiveFilesOmitObsoleteFiles) {
  // Regression test for race condition where an obsolete file is returned to
  // user as a "live file" but then deleted, all while file deletions are
@@ -4432,7 +4428,7 @@ TEST_F(DBTest2, TraceAndReplay) {
      db2->NewDefaultReplayer(handles, std::move(trace_reader), &replayer));

  TraceExecutionResultHandler res_handler;
-  std::function<void(Status, std::unique_ptr<TraceRecordResult> &&)> res_cb =
+  std::function<void(Status, std::unique_ptr<TraceRecordResult>&&)> res_cb =
      [&res_handler](Status exec_s, std::unique_ptr<TraceRecordResult>&& res) {
        ASSERT_TRUE(exec_s.ok() || exec_s.IsNotSupported());
        if (res != nullptr) {
@@ -5164,7 +5160,6 @@ TEST_F(DBTest2, TraceWithFilter) {
  ASSERT_EQ(count, 6);
 }

-
 TEST_F(DBTest2, PinnableSliceAndMmapReads) {
  Options options = CurrentOptions();
  options.env = env_;
@@ -7693,7 +7688,6 @@ TEST_F(DBTest2, RecoverEpochNumber) {
  }
 }

-
 TEST_F(DBTest2, RenameDirectory) {
  Options options = CurrentOptions();
  DestroyAndReopen(options);
@@ -122,7 +122,6 @@ DBTestBase::~DBTestBase() {
 }

 bool DBTestBase::ShouldSkipOptions(int option_config, int skip_mask) {
-
  if ((skip_mask & kSkipUniversalCompaction) &&
      (option_config == kUniversalCompaction ||
       option_config == kUniversalCompactionMultiLevel ||
@@ -1208,7 +1207,6 @@ std::string DBTestBase::FilesPerLevel(int cf) {
  return result;
 }

-
 std::vector<uint64_t> DBTestBase::GetBlobFileNumbers() {
  VersionSet* const versions = dbfull()->GetVersionSet();
  assert(versions);
@@ -1667,7 +1665,7 @@ void DBTestBase::VerifyDBFromMap(
    data_iter = true_data.begin();
    for (iter->SeekToFirst(); iter->Valid(); iter->Next(), ++data_iter) {
      ASSERT_EQ(iter->key().ToString(), data_iter->first);
-        ASSERT_EQ(iter->value().ToString(), data_iter->second);
+      ASSERT_EQ(iter->value().ToString(), data_iter->second);
      iter_cnt++;
      total_reads++;
    }
@@ -1683,7 +1681,7 @@ void DBTestBase::VerifyDBFromMap(
    auto data_rev = true_data.rbegin();
    for (iter->SeekToLast(); iter->Valid(); iter->Prev(), data_rev++) {
      ASSERT_EQ(iter->key().ToString(), data_rev->first);
-        ASSERT_EQ(iter->value().ToString(), data_rev->second);
+      ASSERT_EQ(iter->value().ToString(), data_rev->second);
      iter_cnt++;
      total_reads++;
    }
@@ -1765,7 +1763,6 @@ void DBTestBase::VerifyDBInternal(
  iter->~InternalIterator();
 }

-
 uint64_t DBTestBase::GetNumberOfSstFilesForColumnFamily(
    DB* db, std::string column_family_name) {
  std::vector<LiveFileMetaData> metadata;
@@ -2369,7 +2369,6 @@ TEST_F(DBTestUniversalCompaction2, PeriodicCompactionOffpeak) {

 }  // namespace ROCKSDB_NAMESPACE

-
 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
@@ -1838,9 +1838,194 @@ class RecoveryTestHelper {
  }
 };

-class DBWALTestWithParams : public DBWALTestBase,
-                            public ::testing::WithParamInterface<
-                                std::tuple<bool, int, int, CompressionType>> {
+TEST_F(DBWALTest, TrackAndVerifyWALsRecycleWAL) {
+  Options options = CurrentOptions();
+  options.avoid_flush_during_shutdown = true;
+  options.wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery;
+  options.recycle_log_file_num = 1;
+  options.track_and_verify_wals = true;
+
+  DestroyAndReopen(options);
+
+  ASSERT_OK(Put("key_ignore", "wal_to_recycle"));
+  ASSERT_OK(Put("key_ignore1", "wal_to_recycle"));
+  ASSERT_OK(Put("key_ignore2", "wal_to_recycle"));
+  ASSERT_OK(Flush());
+
+  ASSERT_OK(Put("key_ignore", "wal_to_recycle"));
+  ASSERT_OK(Put("key_ignore1", "wal_to_recycle"));
+  ASSERT_OK(Put("key_ignore2", "wal_to_recycle"));
+  ASSERT_OK(Flush());
+
+  // Stop background flush to avoid deleting any WAL
+  options.env->SetBackgroundThreads(1, Env::HIGH);
+  test::SleepingBackgroundTask sleeping_task;
+  options.env->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
+                        &sleeping_task, Env::Priority::HIGH);
+
+  // Recycle the first WAL
+  ASSERT_OK(Put("key1", "old_value"));
+  // Recycle the second WAL
+  ASSERT_OK(dbfull()->TEST_SwitchWAL());
+  ASSERT_OK(Put("key1", "new_value"));
+
+  // Create a WAL hole on sequence number by truncating the first WAL to 0 byte
+  VectorWalPtr log_files;
+  ASSERT_OK(db_->GetSortedWalFiles(log_files));
+  ASSERT_EQ(log_files.size(), 2);
+  std::string log_name = LogFileName(dbname_, log_files.front()->LogNumber());
+  Close();
+  // Drop `Put("key1", "old_value")` in the first WAL
+  ASSERT_OK(test::TruncateFile(options.env, log_name, 0 /* new_length */));
+
+  Status s = DB::Open(options, dbname_, &db_);
+
+  ASSERT_OK(s);
+
+  ASSERT_EQ("wal_to_recycle", Get("key_ignore2"));
+  ASSERT_EQ("NOT_FOUND", Get("key1"));
+
+  Close();
+}
+
+class DBWALTrackAndVerifyWALsWithParamsTest
+    : public DBWALTestBase,
+      public ::testing::WithParamInterface<WALRecoveryMode> {
+ public:
+  DBWALTrackAndVerifyWALsWithParamsTest()
+      : DBWALTestBase("/db_wal_track_and_verify_wals_with_params_test") {}
+};
+
+INSTANTIATE_TEST_CASE_P(
+    DBWALTrackAndVerifyWALsWithParamsTest,
+    DBWALTrackAndVerifyWALsWithParamsTest,
+    ::testing::Values(WALRecoveryMode::kTolerateCorruptedTailRecords,
+                      WALRecoveryMode::kAbsoluteConsistency,
+                      WALRecoveryMode::kPointInTimeRecovery,
+                      WALRecoveryMode::kSkipAnyCorruptedRecords));
+
+TEST_P(DBWALTrackAndVerifyWALsWithParamsTest, Basic) {
+  Options options = CurrentOptions();
+  options.avoid_flush_during_shutdown = true;
+  options.track_and_verify_wals = true;
+  options.wal_recovery_mode = GetParam();
+
+  // Stop background flush to avoid deleting any WAL
+  options.env->SetBackgroundThreads(1, Env::HIGH);
+  test::SleepingBackgroundTask sleeping_task;
+  options.env->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
+                        &sleeping_task, Env::Priority::HIGH);
+
+  for (int i = 0; i < 5; i++) {
+    DestroyAndReopen(options);
+
+    ASSERT_OK(Put("key1", "old_value"));
+    SequenceNumber last_seqno_recorded_in_fist_wal =
+        dbfull()->GetLatestSequenceNumber();
+
+    ASSERT_OK(dbfull()->TEST_SwitchWAL());
+    ASSERT_OK(Put("key1", "new_value"));
+
+    VectorWalPtr log_files;
+    ASSERT_OK(db_->GetSortedWalFiles(log_files));
+    ASSERT_EQ(log_files.size(), 2);
+    uint64_t first_log_number = log_files.front()->LogNumber();
+    std::string first_log_name = LogFileName(dbname_, first_log_number);
+    std::string second_log_name =
+        LogFileName(dbname_, log_files.back()->LogNumber());
+
+    if (i == 0) {
+      // Delete the obsolete WAL and verify it will not be seen as a WAL hole
+      sleeping_task.WakeUp();
+      sleeping_task.WaitUntilDone();
+      ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork());
+      // Stop background flush to avoid deleting any WAL
+      sleeping_task.Reset();
+      options.env->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
+                            &sleeping_task, Env::Priority::HIGH);
+      Close();
+    } else if (i == 1) {
+      // Create a WAL hole on WAL number by deleting the first WAL and verify
+      // the hole will be detected
+      Close();
+      ASSERT_OK(options.env->DeleteFile(first_log_name));
+    } else if (i == 2) {
+      // Create a WAL hole on sequence number by truncating the first WAL and
+      // verify the hole will be detected
+      Close();
+      ASSERT_OK(
+          test::TruncateFile(options.env, first_log_name, 0 /* new_length */));
+    } else if (i == 3) {
+      // Create a WAL hole on size difference by truncating the first WAL and
+      // mocking a correct sequence number to force triggering corruption based
+      // on size instead of sequence number and verify the hole will be detected
+      Close();
+      ASSERT_OK(
+          test::TruncateFile(options.env, first_log_name, 0 /* new_length */));
+      SyncPoint::GetInstance()->SetCallBack(
+          "DBImpl::UpdatePredecessorWALInfo", [&](void* arg) {
+            std::pair<uint64_t, SequenceNumber*>* pair =
+                static_cast<std::pair<uint64_t, SequenceNumber*>*>(arg);
+            if (pair->first == first_log_number) {
+              *(pair->second) = last_seqno_recorded_in_fist_wal;
+            }
+          });
+      SyncPoint::GetInstance()->EnableProcessing();
+    } else if (i == 4) {
+      // Delete all wals and verify opening a DB with no WAL will be detected
+      Close();
+      ASSERT_OK(options.env->DeleteFile(first_log_name));
+      ASSERT_OK(options.env->DeleteFile(second_log_name));
+    }
+
+    Status s = DB::Open(options, dbname_, &db_);
+
+    if (i == 0) {
+      ASSERT_OK(s);
+      ASSERT_EQ("new_value", Get("key1"));
+      continue;
+    } else if (i == 3) {
+      SyncPoint::GetInstance()->ClearAllCallBacks();
+      SyncPoint::GetInstance()->DisableProcessing();
+    } else if (i == 4) {
+      ASSERT_TRUE(s.IsCorruption());
+      ASSERT_TRUE(
+          s.ToString().find("Opening an existing DB with no WAL files") !=
+          std::string::npos);
+      Close();
+      continue;
+    }
+
+    if (options.wal_recovery_mode == WALRecoveryMode::kPointInTimeRecovery) {
+      ASSERT_OK(s);
+      ASSERT_EQ("NOT_FOUND", Get("key1"));
+    } else if (options.wal_recovery_mode ==
+                   WALRecoveryMode::kAbsoluteConsistency ||
+               options.wal_recovery_mode ==
+                   WALRecoveryMode::kTolerateCorruptedTailRecords) {
+      ASSERT_TRUE(s.IsCorruption());
+      std::string msg;
+      if (i == 1) {
+        msg = "Missing WAL";
+      } else if (i == 2) {
+        msg = "Mismatched last sequence number recorded in the WAL";
+      } else if (i == 3) {
+        msg = "Mismatched size of the WAL";
+      }
+      ASSERT_TRUE(s.ToString().find(msg) != std::string::npos);
+    } else {
+      ASSERT_OK(s);
+      ASSERT_EQ("new_value", Get("key1"));
+    }
+
+    Close();
+  }
+}
+
+class DBWALTestWithParams
+    : public DBWALTestBase,
+      public ::testing::WithParamInterface<
+          std::tuple<bool, int, int, CompressionType, bool>> {
 public:
  DBWALTestWithParams() : DBWALTestBase("/db_wal_test_with_params") {}
 };
@@ -1853,7 +2038,8 @@ INSTANTIATE_TEST_CASE_P(
                                            RecoveryTestHelper::kWALFilesCount,
                                        1),
                       ::testing::Values(CompressionType::kNoCompression,
-                                         CompressionType::kZSTD)));
+                                         CompressionType::kZSTD),
+                       ::testing::Bool()));

 class DBWALTestWithParamsVaryingRecoveryMode
    : public DBWALTestBase,
@@ -1891,6 +2077,7 @@ TEST_P(DBWALTestWithParams, kTolerateCorruptedTailRecords) {

  // Fill data for testing
  Options options = CurrentOptions();
+  options.track_and_verify_wals = std::get<4>(GetParam());
  const size_t row_count = RecoveryTestHelper::FillData(this, &options);
  // test checksum failure or parsing
  RecoveryTestHelper::CorruptWAL(this, options, corrupt_offset * .3,
@@ -1914,6 +2101,7 @@ TEST_P(DBWALTestWithParams, kTolerateCorruptedTailRecords) {
 TEST_P(DBWALTestWithParams, kAbsoluteConsistency) {
  // Verify clean slate behavior
  Options options = CurrentOptions();
+  options.track_and_verify_wals = std::get<4>(GetParam());
  const size_t row_count = RecoveryTestHelper::FillData(this, &options);
  options.create_if_missing = false;
  ASSERT_OK(TryReopen(options));
@@ -2124,7 +2312,6 @@ TEST_F(DBWALTest, FixSyncWalOnObseletedWalWithNewManifestCausingMissingWAL) {
        wal_synced = true;
      });

-
  SyncPoint::GetInstance()->EnableProcessing();

  ASSERT_OK(Flush());
@@ -2165,6 +2352,7 @@ TEST_P(DBWALTestWithParams, kPointInTimeRecovery) {

  // Fill data for testing
  Options options = CurrentOptions();
+  options.track_and_verify_wals = std::get<4>(GetParam());
  options.wal_compression = compression_type;
  const size_t row_count = RecoveryTestHelper::FillData(this, &options);

@@ -2222,6 +2410,7 @@ TEST_P(DBWALTestWithParams, kSkipAnyCorruptedRecords) {

  // Fill data for testing
  Options options = CurrentOptions();
+  options.track_and_verify_wals = std::get<4>(GetParam());
  options.wal_compression = compression_type;
  const size_t row_count = RecoveryTestHelper::FillData(this, &options);

@@ -2699,12 +2888,12 @@ TEST_F(DBWALTest, ReadOnlyRecoveryNoTruncate) {
  // create DB and close with file truncate disabled
  std::atomic_bool enable_truncate{false};

-  SyncPoint::GetInstance()->SetCallBack(
-      "PosixWritableFile::Close", [&](void* arg) {
-        if (!enable_truncate) {
-          *(static_cast<size_t*>(arg)) = 0;
-        }
-      });
+  SyncPoint::GetInstance()->SetCallBack("PosixWritableFile::Close",
+                                        [&](void* arg) {
+                                          if (!enable_truncate) {
+                                            *(static_cast<size_t*>(arg)) = 0;
+                                          }
+                                        });
  SyncPoint::GetInstance()->EnableProcessing();

  DestroyAndReopen(options);
@@ -2793,7 +2982,6 @@ TEST_F(DBWALTest, WalInManifestButNotInSortedWals) {
  Close();
 }

-
 TEST_F(DBWALTest, WalTermTest) {
  Options options = CurrentOptions();
  options.env = env_;
@@ -3257,7 +3257,6 @@ TEST_F(DBBasicTestWithTimestamp, MultiGetNoReturnTs) {
  Close();
 }

-
 INSTANTIATE_TEST_CASE_P(
    Timestamp, DBBasicTestWithTimestampCompressionSettings,
    ::testing::Combine(
@@ -780,7 +780,6 @@ TEST_P(DBWriteBufferManagerTest, MixedSlowDownOptionsMultipleDB) {
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }

-
 // Tests a `WriteBufferManager` constructed with `allow_stall == false` does not
 // thrash memtable switching when full and a CF receives multiple writes.
 // Instead, we expect to switch a CF's memtable for flush only when that CF does
@@ -498,6 +498,8 @@ class InternalKey {
  // Intended only to be used together with ConvertFromUserKey().
  std::string* rep() { return &rep_; }

+  const std::string* const_rep() const { return &rep_; }
+
  // Assuming that *rep() contains a user key, this method makes internal key
  // out of it in-place. This saves a memcpy compared to Set()/SetFrom().
  void ConvertFromUserKey(SequenceNumber s, ValueType t) {
@@ -1179,4 +1181,64 @@ struct ParsedInternalKeyComparator {
  const InternalKeyComparator* cmp;
 };

+class PredecessorWALInfo {
+ public:
+  PredecessorWALInfo()
+      : log_number_(0),
+        size_bytes_(0),
+        last_seqno_recorded_(0),
+        initialized_(false) {}
+
+  explicit PredecessorWALInfo(uint64_t log_number, uint64_t size_bytes,
+                              SequenceNumber last_seqno_recorded)
+      : log_number_(log_number),
+        size_bytes_(size_bytes),
+        last_seqno_recorded_(last_seqno_recorded),
+        initialized_(true) {}
+
+  uint64_t GetLogNumber() const {
+    assert(initialized_);
+    return log_number_;
+  }
+
+  uint64_t GetSizeBytes() const {
+    assert(initialized_);
+    return size_bytes_;
+  }
+
+  SequenceNumber GetLastSeqnoRecorded() const {
+    assert(initialized_);
+    return last_seqno_recorded_;
+  }
+
+  bool IsInitialized() const { return initialized_; }
+
+  inline void EncodeTo(std::string* dst) const {
+    assert(dst != nullptr);
+    assert(initialized_);
+    PutFixed64(dst, log_number_);
+    PutFixed64(dst, size_bytes_);
+    PutFixed64(dst, last_seqno_recorded_);
+  }
+
+  inline Status DecodeFrom(Slice* src) {
+    if (!GetFixed64(src, &log_number_)) {
+      return Status::Corruption("Error decoding log number");
+    }
+    if (!GetFixed64(src, &size_bytes_)) {
+      return Status::Corruption("Error decoding size bytes");
+    }
+    if (!GetFixed64(src, &last_seqno_recorded_)) {
+      return Status::Corruption("Error decoding last seqno recorded");
+    }
+    initialized_ = true;
+    return Status::OK();
+  }
+
+ private:
+  uint64_t log_number_;
+  uint64_t size_bytes_;
+  SequenceNumber last_seqno_recorded_;
+  bool initialized_;
+};
 }  // namespace ROCKSDB_NAMESPACE
@@ -174,15 +174,15 @@ TEST_F(DeleteFileTest, AddKeysAndQueryLevels) {
  ASSERT_EQ(level1keycount, 50000);
  ASSERT_EQ(level2keycount, 50000);

-  Status status = db_->DeleteFile("0.sst");
+  Status status = db_->DEPRECATED_DeleteFile("0.sst");
  ASSERT_TRUE(status.IsInvalidArgument());

  // intermediate level files cannot be deleted.
-  status = db_->DeleteFile(level1file);
+  status = db_->DEPRECATED_DeleteFile(level1file);
  ASSERT_TRUE(status.IsInvalidArgument());

  // Lowest level file deletion should succeed.
-  status = db_->DeleteFile(level2file);
+  status = db_->DEPRECATED_DeleteFile(level2file);
  ASSERT_OK(status);
 }

@@ -519,7 +519,7 @@ TEST_F(DeleteFileTest, DeleteFileWithIterator) {
    level2file = metadata[0].name;
  }

-  Status status = db_->DeleteFile(level2file);
+  Status status = db_->DEPRECATED_DeleteFile(level2file);
  fprintf(stdout, "Deletion status %s: %s\n", level2file.c_str(),
          status.ToString().c_str());
  ASSERT_OK(status);
@@ -551,7 +551,7 @@ TEST_F(DeleteFileTest, DeleteLogFiles) {
  ASSERT_OK(env_->FileExists(wal_dir_ + "/" + alive_log->PathName()));
  fprintf(stdout, "Deleting alive log file %s\n",
          alive_log->PathName().c_str());
-  ASSERT_NOK(db_->DeleteFile(alive_log->PathName()));
+  ASSERT_NOK(db_->DEPRECATED_DeleteFile(alive_log->PathName()));
  ASSERT_OK(env_->FileExists(wal_dir_ + "/" + alive_log->PathName()));
  logfiles.clear();

@@ -569,7 +569,7 @@ TEST_F(DeleteFileTest, DeleteLogFiles) {
  ASSERT_OK(env_->FileExists(wal_dir_ + "/" + archived_log->PathName()));
  fprintf(stdout, "Deleting archived log file %s\n",
          archived_log->PathName().c_str());
-  ASSERT_OK(db_->DeleteFile(archived_log->PathName()));
+  ASSERT_OK(db_->DEPRECATED_DeleteFile(archived_log->PathName()));
  ASSERT_TRUE(
      env_->FileExists(wal_dir_ + "/" + archived_log->PathName()).IsNotFound());
 }
@@ -605,8 +605,8 @@ TEST_F(DeleteFileTest, DeleteNonDefaultColumnFamily) {
  auto new_file = metadata[0].smallest_seqno > metadata[1].smallest_seqno
                      ? metadata[0].name
                      : metadata[1].name;
-  ASSERT_TRUE(db_->DeleteFile(new_file).IsInvalidArgument());
-  ASSERT_OK(db_->DeleteFile(old_file));
+  ASSERT_TRUE(db_->DEPRECATED_DeleteFile(new_file).IsInvalidArgument());
+  ASSERT_OK(db_->DEPRECATED_DeleteFile(old_file));

  {
    std::unique_ptr<Iterator> itr(db_->NewIterator(ReadOptions(), handles_[1]));
@@ -227,7 +227,7 @@ std::map<std::tuple<BackgroundErrorReason, bool>, Status::Severity>
         Status::Severity::kFatalError},
 };

-void ErrorHandler::CancelErrorRecovery() {
+void ErrorHandler::CancelErrorRecoveryForShutDown() {
  db_mutex_->AssertHeld();

  // We'll release the lock before calling sfm, so make sure no new
@@ -573,6 +573,8 @@ Status ErrorHandler::ClearBGError() {

  // Signal that recovery succeeded
  if (recovery_error_.ok()) {
+    // If this assertion fails, it means likely bg error is not set after a
+    // file is quarantined during MANIFEST write.
    assert(files_to_quarantine_.empty());
    Status old_bg_error = bg_error_;
    // old_bg_error is only for notifying listeners, so may not be checked
@@ -585,8 +587,15 @@ Status ErrorHandler::ClearBGError() {
    recovery_error_.PermitUncheckedError();
    recovery_in_prog_ = false;
    soft_error_no_bg_work_ = false;
-    EventHelpers::NotifyOnErrorRecoveryEnd(db_options_.listeners, old_bg_error,
-                                           bg_error_, db_mutex_);
+    if (!db_->shutdown_initiated_) {
+      // NotifyOnErrorRecoveryEnd() may release and re-acquire db_mutex_.
+      // Prevent DB from being closed while we notify listeners. DB close will
+      // wait until allow_db_shutdown_ = true, see ReadyForShutdown().
+      allow_db_shutdown_ = false;
+      EventHelpers::NotifyOnErrorRecoveryEnd(
+          db_options_.listeners, old_bg_error, bg_error_, db_mutex_);
+      allow_db_shutdown_ = true;
+    }
  }
  return recovery_error_;
 }
@@ -44,6 +44,7 @@ class ErrorHandler {
        auto_recovery_(false),
        recovery_in_prog_(false),
        soft_error_no_bg_work_(false),
+        allow_db_shutdown_(true),
        is_db_stopped_(false),
        bg_error_stats_(db_options.statistics) {
    // Clear the checked flag for uninitialized errors
@@ -63,6 +64,12 @@ class ErrorHandler {

  Status GetRecoveryError() const { return recovery_error_; }

+  // REQUIREs: db mutex held
+  //
+  // Returns non-OK status if encountered error during recovery.
+  // Returns OK if bg error is successfully cleared. May releases and
+  // re-acquire db mutex to notify listeners. However, DB close (if initiated)
+  // will be blocked until db mutex is released after return.
  Status ClearBGError();

  bool IsDBStopped() { return is_db_stopped_.load(std::memory_order_acquire); }
@@ -79,8 +86,14 @@ class ErrorHandler {

  bool IsRecoveryInProgress() { return recovery_in_prog_; }

+  // REQUIRES: db mutex held
+  bool ReadyForShutdown() {
+    db_mutex_->AssertHeld();
+    return !recovery_in_prog_ && allow_db_shutdown_;
+  }
+
  Status RecoverFromBGError(bool is_manual = false);
-  void CancelErrorRecovery();
+  void CancelErrorRecoveryForShutDown();

  void EndAutoRecovery();

@@ -121,6 +134,8 @@ class ErrorHandler {
  // A flag to indicate that for the soft error, we should not allow any
  // background work except the work is from recovery.
  bool soft_error_no_bg_work_;
+  // Used in ClearBGError() to prevent DB from being closed.
+  bool allow_db_shutdown_;

  // Used to store the context for recover, such as flush reason.
  DBRecoverContext recover_context_;
@@ -81,9 +81,8 @@ void EventHelpers::LogAndNotifyTableFileCreationFinished(
    JSONWriter jwriter;
    AppendCurrentTime(&jwriter);
    jwriter << "cf_name" << cf_name << "job" << job_id << "event"
-            << "table_file_creation"
-            << "file_number" << fd.GetNumber() << "file_size"
-            << fd.GetFileSize() << "file_checksum"
+            << "table_file_creation" << "file_number" << fd.GetNumber()
+            << "file_size" << fd.GetFileSize() << "file_checksum"
            << Slice(file_checksum).ToString(true) << "file_checksum_func_name"
            << file_checksum_func_name << "smallest_seqno" << fd.smallest_seqno
            << "largest_seqno" << fd.largest_seqno;
@@ -199,8 +198,7 @@ void EventHelpers::LogAndNotifyTableFileDeletion(
  JSONWriter jwriter;
  AppendCurrentTime(&jwriter);

-  jwriter << "job" << job_id << "event"
-          << "table_file_deletion"
+  jwriter << "job" << job_id << "event" << "table_file_deletion"
          << "file_number" << file_number;
  if (!status.ok()) {
    jwriter << "status" << status.ToString();
@@ -280,11 +278,11 @@ void EventHelpers::LogAndNotifyBlobFileCreationFinished(
    JSONWriter jwriter;
    AppendCurrentTime(&jwriter);
    jwriter << "cf_name" << cf_name << "job" << job_id << "event"
-            << "blob_file_creation"
-            << "file_number" << file_number << "total_blob_count"
-            << total_blob_count << "total_blob_bytes" << total_blob_bytes
-            << "file_checksum" << file_checksum << "file_checksum_func_name"
-            << file_checksum_func_name << "status" << s.ToString();
+            << "blob_file_creation" << "file_number" << file_number
+            << "total_blob_count" << total_blob_count << "total_blob_bytes"
+            << total_blob_bytes << "file_checksum" << file_checksum
+            << "file_checksum_func_name" << file_checksum_func_name << "status"
+            << s.ToString();

    jwriter.EndObject();
    event_logger->Log(jwriter);
@@ -311,8 +309,7 @@ void EventHelpers::LogAndNotifyBlobFileDeletion(
    JSONWriter jwriter;
    AppendCurrentTime(&jwriter);

-    jwriter << "job" << job_id << "event"
-            << "blob_file_deletion"
+    jwriter << "job" << job_id << "event" << "blob_file_deletion"
            << "file_number" << file_number;
    if (!status.ok()) {
      jwriter << "status" << status.ToString();
@@ -37,7 +37,6 @@ Status PromoteL0(DB* db, ColumnFamilyHandle* column_family, int target_level) {
  return db->PromoteL0(column_family, target_level);
 }

-
 Status SuggestCompactRange(DB* db, const Slice* begin, const Slice* end) {
  return SuggestCompactRange(db, db->DefaultColumnFamily(), begin, end);
 }
@@ -11,6 +11,7 @@
 #include "port/stack_trace.h"
 #include "rocksdb/advanced_options.h"
 #include "rocksdb/options.h"
+#include "rocksdb/perf_context.h"
 #include "rocksdb/sst_file_writer.h"
 #include "test_util/testharness.h"
 #include "test_util/testutil.h"
@@ -260,14 +261,54 @@ TEST_F(ExternalSSTFileBasicTest, Basic) {
  ASSERT_NOK(s) << s.ToString();

  DestroyAndReopen(options);
+
+  SyncPoint::GetInstance()->LoadDependency({
+      {"DBImpl::IngestExternalFile:AfterIncIngestFileCounter",
+       "ExternalSSTFileBasicTest.LiveWriteStart"},
+      {"ExternalSSTFileBasicTest.LiveWriteStart",
+       "DBImpl::IngestExternalFiles:InstallSVForFirstCF:0"},
+  });
+  SyncPoint::GetInstance()->EnableProcessing();
+  PerfContext* write_thread_perf_context;
+  std::thread write_thread([&] {
+    TEST_SYNC_POINT("ExternalSSTFileBasicTest.LiveWriteStart");
+    SetPerfLevel(kEnableWait);
+    write_thread_perf_context = get_perf_context();
+    write_thread_perf_context->Reset();
+    ASSERT_OK(db_->Put(WriteOptions(), "bar", "v2"));
+    ASSERT_GT(write_thread_perf_context->write_thread_wait_nanos, 0);
+    // Test sync points were used to make sure this live write enter write
+    // thread after the file ingestion entered write thread. So by the time this
+    // live write finishes, the latest seqno is 1 means file ingestion used
+    // seqno 0.
+    ASSERT_EQ(db_->GetLatestSequenceNumber(), 1U);
+  });
+
  // Add file using file path
+  SetPerfLevel(kEnableTimeExceptForMutex);
+  PerfContext* perf_ctx = get_perf_context();
+  perf_ctx->Reset();
  s = DeprecatedAddFile({file1});
+  ASSERT_GT(perf_context.file_ingestion_nanos, 0);
+  ASSERT_GT(perf_context.file_ingestion_blocking_live_writes_nanos, 0);
  ASSERT_OK(s) << s.ToString();
-  ASSERT_EQ(db_->GetLatestSequenceNumber(), 0U);
  for (int k = 0; k < 100; k++) {
    ASSERT_EQ(Get(Key(k)), Key(k) + "_val");
  }

+  write_thread.join();
+  SyncPoint::GetInstance()->DisableProcessing();
+
+  // Re-ingest the file just to check the perf context not enabled at and below
+  // kEnableWait.
+  SetPerfLevel(kEnableWait);
+  perf_ctx->Reset();
+  IngestExternalFileOptions opts;
+  opts.allow_global_seqno = true;
+  opts.allow_blocking_flush = true;
+  ASSERT_OK(db_->IngestExternalFile({file1}, opts));
+  ASSERT_EQ(perf_context.file_ingestion_nanos, 0);
+  ASSERT_EQ(perf_context.file_ingestion_blocking_live_writes_nanos, 0);
  DestroyAndRecreateExternalSSTFilesDir();
 }

@@ -2878,7 +2919,6 @@ INSTANTIATE_TEST_CASE_P(ExternalSSTFileBasicTest, ExternalSSTFileBasicTest,
                                        std::make_tuple(false, true),
                                        std::make_tuple(false, false)));

-
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -115,7 +115,7 @@ Status ExternalSstFileIngestionJob::Prepare(
    f.copy_file = false;
    const std::string path_outside_db = f.external_file_path;
    const std::string path_inside_db = TableFileName(
-        cfd_->ioptions()->cf_paths, f.fd.GetNumber(), f.fd.GetPathId());
+        cfd_->ioptions().cf_paths, f.fd.GetNumber(), f.fd.GetPathId());
    if (ingestion_options_.move_files || ingestion_options_.link_files) {
      status =
          fs_->LinkFile(path_outside_db, path_inside_db, IOOptions(), nullptr);
@@ -463,7 +463,7 @@ Status ExternalSstFileIngestionJob::AssignLevelsForOneBatch(
      status = CheckLevelForIngestedBehindFile(file);
    } else {
      status = AssignLevelAndSeqnoForIngestedFile(
-          super_version, force_global_seqno, cfd_->ioptions()->compaction_style,
+          super_version, force_global_seqno, cfd_->ioptions().compaction_style,
          *last_seqno, file, &assigned_seqno, prev_batch_uppermost_level);
    }

@@ -584,13 +584,13 @@ void ExternalSstFileIngestionJob::CreateEquivalentFileIngestingCompactions() {
    int output_level = pair.first;
    const CompactionInputFiles& input = pair.second;

-    const auto& mutable_cf_options = *(cfd_->GetLatestMutableCFOptions());
+    const auto& mutable_cf_options = cfd_->GetLatestMutableCFOptions();
    file_ingesting_compactions_.push_back(new Compaction(
-        cfd_->current()->storage_info(), *cfd_->ioptions(), mutable_cf_options,
+        cfd_->current()->storage_info(), cfd_->ioptions(), mutable_cf_options,
        mutable_db_options_, {input}, output_level,
        /* output file size limit not applicable */
        MaxFileSizeForLevel(mutable_cf_options, output_level,
-                            cfd_->ioptions()->compaction_style),
+                            cfd_->ioptions().compaction_style),
        LLONG_MAX /* max compaction bytes, not applicable */,
        0 /* output path ID, not applicable */, mutable_cf_options.compression,
        mutable_cf_options.compression_opts,
@@ -631,8 +631,7 @@ void ExternalSstFileIngestionJob::UpdateStats() {
  uint64_t total_time = clock_->NowMicros() - job_start_time_;

  EventLoggerStream stream = event_logger_->Log();
-  stream << "event"
-         << "ingest_finished";
+  stream << "event" << "ingest_finished";
  stream << "files_ingested";
  stream.StartArray();

@@ -750,7 +749,7 @@ Status ExternalSstFileIngestionJob::ResetTableReader(
  status = sv->mutable_cf_options.table_factory->NewTableReader(
      ro,
      TableReaderOptions(
-          *cfd_->ioptions(), sv->mutable_cf_options.prefix_extractor,
+          cfd_->ioptions(), sv->mutable_cf_options.prefix_extractor,
          env_options_, cfd_->internal_comparator(),
          sv->mutable_cf_options.block_protection_bytes_per_key,
          /*skip_filters*/ false, /*immortal*/ false,
@@ -844,7 +843,7 @@ Status ExternalSstFileIngestionJob::SanityCheckTableProperties(
  bool mark_sst_file_has_no_udt = false;
  Status s = ValidateUserDefinedTimestampsOptions(
      cfd_->user_comparator(), props->comparator_name,
-      cfd_->ioptions()->persist_user_defined_timestamps,
+      cfd_->ioptions().persist_user_defined_timestamps,
      file_to_ingest->user_defined_timestamps_persisted,
      &mark_sst_file_has_no_udt);
  if (s.ok() && mark_sst_file_has_no_udt) {
@@ -329,7 +329,7 @@ Status FlushJob::Run(LogsWithPrepTracker* prep_tracker, FileMetaData* file_meta,
      TEST_SYNC_POINT("FlushJob::InstallResults");
      // Replace immutable memtable with the generated Table
      s = cfd_->imm()->TryInstallMemtableFlushResults(
-              cfd_, mutable_cf_options_, mems_, prep_tracker, versions_, db_mutex_,
+              cfd_, mems_, prep_tracker, versions_, db_mutex_,
              meta_.fd.GetNumber(), &job_context_->memtables_to_free, db_directory_,
              log_buffer_, &committed_flush_jobs_info_,
              !(mempurge_s.ok()) /* write_edit : true if no mempurge happened (or if aborted),
@@ -345,8 +345,7 @@ Status FlushJob::Run(LogsWithPrepTracker* prep_tracker, FileMetaData* file_meta,

  // When measure_io_stats_ is true, the default 512 bytes is not enough.
  auto stream = event_logger_->LogToBuffer(log_buffer_, 1024);
-  stream << "job" << job_context_->job_id << "event"
-         << "flush_finished";
+  stream << "job" << job_context_->job_id << "event" << "flush_finished";
  stream << "output_compression"
         << CompressionTypeToString(output_compression_);
  stream << "lsm_state";
@@ -449,7 +448,7 @@ Status FlushJob::MemPurge() {
      NewMergingIterator(&(cfd_->internal_comparator()), memtables.data(),
                         static_cast<int>(memtables.size()), &arena));

-  auto* ioptions = cfd_->ioptions();
+  const auto& ioptions = cfd_->ioptions();

  // Place iterator at the First (meaning most recent) key node.
  iter->SeekToFirst();
@@ -470,8 +469,8 @@ Status FlushJob::MemPurge() {
    // MaxSize is the size of a memtable.
    size_t maxSize = mutable_cf_options_.write_buffer_size;
    std::unique_ptr<CompactionFilter> compaction_filter;
-    if (ioptions->compaction_filter_factory != nullptr &&
-        ioptions->compaction_filter_factory->ShouldFilterTableFileCreation(
+    if (ioptions.compaction_filter_factory != nullptr &&
+        ioptions.compaction_filter_factory->ShouldFilterTableFileCreation(
            TableFileCreationReason::kFlush)) {
      CompactionFilter::Context ctx;
      ctx.is_full_compaction = false;
@@ -479,7 +478,7 @@ Status FlushJob::MemPurge() {
      ctx.column_family_id = cfd_->GetID();
      ctx.reason = TableFileCreationReason::kFlush;
      compaction_filter =
-          ioptions->compaction_filter_factory->CreateCompactionFilter(ctx);
+          ioptions.compaction_filter_factory->CreateCompactionFilter(ctx);
      if (compaction_filter != nullptr &&
          !compaction_filter->IgnoreSnapshots()) {
        s = Status::NotSupported(
@@ -489,7 +488,7 @@ Status FlushJob::MemPurge() {
      }
    }

-    new_mem = new MemTable((cfd_->internal_comparator()), *(cfd_->ioptions()),
+    new_mem = new MemTable(cfd_->internal_comparator(), cfd_->ioptions(),
                           mutable_cf_options_, cfd_->write_buffer_mgr(),
                           earliest_seqno, cfd_->GetID());
    assert(new_mem != nullptr);
@@ -498,8 +497,8 @@ Status FlushJob::MemPurge() {
    assert(env);
    MergeHelper merge(
        env, (cfd_->internal_comparator()).user_comparator(),
-        (ioptions->merge_operator).get(), compaction_filter.get(),
-        ioptions->logger, true /* internal key corruption is not ok */,
+        (ioptions.merge_operator).get(), compaction_filter.get(),
+        ioptions.logger, true /* internal key corruption is not ok */,
        existing_snapshots_.empty() ? 0 : existing_snapshots_.back(),
        snapshot_checker_);
    assert(job_context_);
@@ -509,14 +508,14 @@ Status FlushJob::MemPurge() {
        iter.get(), (cfd_->internal_comparator()).user_comparator(), &merge,
        kMaxSequenceNumber, &existing_snapshots_, earliest_snapshot_,
        earliest_write_conflict_snapshot_, job_snapshot_seq, snapshot_checker_,
-        env, ShouldReportDetailedTime(env, ioptions->stats),
+        env, ShouldReportDetailedTime(env, ioptions.stats),
        true /* internal key corruption is not ok */, range_del_agg.get(),
-        nullptr, ioptions->allow_data_in_errors,
-        ioptions->enforce_single_del_contracts,
+        nullptr, ioptions.allow_data_in_errors,
+        ioptions.enforce_single_del_contracts,
        /*manual_compaction_canceled=*/kManualCompactionCanceledFalse,
        false /* must_count_input_entries */,
        /*compaction=*/nullptr, compaction_filter.get(),
-        /*shutting_down=*/nullptr, ioptions->info_log, full_history_ts_low);
+        /*shutting_down=*/nullptr, ioptions.info_log, full_history_ts_low);

    // Set earliest sequence number in the new memtable
    // to be equal to the earliest sequence number of the
@@ -866,7 +865,7 @@ Status FlushJob::WriteLevel0Table() {
  assert(ucmp);
  const size_t ts_sz = ucmp->timestamp_size();
  const bool logical_strip_timestamp =
-      ts_sz > 0 && !cfd_->ioptions()->persist_user_defined_timestamps;
+      ts_sz > 0 && !cfd_->ioptions().persist_user_defined_timestamps;

  std::vector<BlobFileAddition> blob_file_additions;

@@ -933,9 +932,8 @@ Status FlushJob::WriteLevel0Table() {
    //  hitting limit memtable_max_range_deletions, flush_reason_ is still
    //  "Write Buffer Full", should make update flush_reason_ accordingly.
    event_logger_->Log() << "job" << job_context_->job_id << "event"
-                         << "flush_started"
-                         << "num_memtables" << mems_.size() << "num_entries"
-                         << total_num_entries << "num_deletes"
+                         << "flush_started" << "num_memtables" << mems_.size()
+                         << "num_entries" << total_num_entries << "num_deletes"
                         << total_num_deletes << "total_data_size"
                         << total_data_size << "memory_usage"
                         << total_memory_usage << "num_range_deletes"
@@ -988,7 +986,7 @@ Status FlushJob::WriteLevel0Table() {
      read_options.rate_limiter_priority = io_priority;
      const WriteOptions write_options(io_priority, Env::IOActivity::kFlush);
      TableBuilderOptions tboptions(
-          *cfd_->ioptions(), mutable_cf_options_, read_options, write_options,
+          cfd_->ioptions(), mutable_cf_options_, read_options, write_options,
          cfd_->internal_comparator(), cfd_->internal_tbl_prop_coll_factories(),
          output_compression_, mutable_cf_options_.compression_opts,
          cfd_->GetID(), cfd_->GetName(), 0 /* level */,
@@ -1142,7 +1140,7 @@ std::unique_ptr<FlushJobInfo> FlushJob::GetFlushJobInfo() const {

  const uint64_t file_number = meta_.fd.GetNumber();
  info->file_path =
-      MakeTableFileName(cfd_->ioptions()->cf_paths[0].path, file_number);
+      MakeTableFileName(cfd_->ioptions().cf_paths[0].path, file_number);
  info->file_number = file_number;
  info->oldest_blob_file_number = meta_.oldest_blob_file_number;
  info->thread_id = db_options_.env->GetThreadID();
@@ -1156,7 +1154,7 @@ std::unique_ptr<FlushJobInfo> FlushJob::GetFlushJobInfo() const {
  // Update BlobFilesInfo.
  for (const auto& blob_file : edit_->GetBlobFileAdditions()) {
    BlobFileAdditionInfo blob_file_addition_info(
-        BlobFileName(cfd_->ioptions()->cf_paths.front().path,
+        BlobFileName(cfd_->ioptions().cf_paths.front().path,
                     blob_file.GetBlobFileNumber()) /*blob_file_path*/,
        blob_file.GetBlobFileNumber(), blob_file.GetTotalBlobCount(),
        blob_file.GetTotalBlobBytes());
@@ -1173,7 +1171,7 @@ void FlushJob::GetEffectiveCutoffUDTForPickedMemTables() {
  assert(ucmp);
  const size_t ts_sz = ucmp->timestamp_size();
  if (db_options_.atomic_flush || ts_sz == 0 ||
-      cfd_->ioptions()->persist_user_defined_timestamps) {
+      cfd_->ioptions().persist_user_defined_timestamps) {
    return;
  }
  // Find the newest user-defined timestamps from all the flushed memtables.
@@ -1195,7 +1193,11 @@ void FlushJob::GetEffectiveCutoffUDTForPickedMemTables() {
 }

 void FlushJob::GetPrecludeLastLevelMinSeqno() {
-  if (mutable_cf_options_.preclude_last_level_data_seconds == 0) {
+  if (mutable_cf_options_.preclude_last_level_data_seconds == 0 ||
+      // FIXME: create FlushJob and build SuperVersions such that
+      // preclude_last_level_data_seconds > 0 implies
+      // seqno_to_time_mapping_ != nullptr
+      seqno_to_time_mapping_ == nullptr) {
    return;
  }
  int64_t current_time = 0;
@@ -1236,8 +1238,7 @@ Status FlushJob::MaybeIncreaseFullHistoryTsLowToAboveCutoffUDT() {
  VersionEdit edit;
  edit.SetColumnFamily(cfd_->GetID());
  edit.SetFullHistoryTsLow(new_full_history_ts_low);
-  return versions_->LogAndApply(cfd_, *cfd_->GetLatestMutableCFOptions(),
-                                ReadOptions(Env::IOActivity::kFlush),
+  return versions_->LogAndApply(cfd_, ReadOptions(Env::IOActivity::kFlush),
                                WriteOptions(Env::IOActivity::kFlush), &edit,
                                db_mutex_, output_file_directory_);
 }
@@ -189,7 +189,7 @@ TEST_F(FlushJobTest, Empty) {
  SnapshotChecker* snapshot_checker = nullptr;  // not relavant
  FlushJob flush_job(
      dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_,
-      *cfd->GetLatestMutableCFOptions(),
+      cfd->GetLatestMutableCFOptions(),
      std::numeric_limits<uint64_t>::max() /* memtable_id */, env_options_,
      versions_.get(), &mutex_, &shutting_down_, {}, kMaxSequenceNumber,
      snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr,
@@ -207,7 +207,7 @@ TEST_F(FlushJobTest, Empty) {
 TEST_F(FlushJobTest, NonEmpty) {
  JobContext job_context(0);
  auto cfd = versions_->GetColumnFamilySet()->GetDefault();
-  auto new_mem = cfd->ConstructNewMemtable(*cfd->GetLatestMutableCFOptions(),
+  auto new_mem = cfd->ConstructNewMemtable(cfd->GetLatestMutableCFOptions(),
                                           kMaxSequenceNumber);
  new_mem->Ref();
  auto inserted_keys = mock::MakeMockFile();
@@ -275,7 +275,7 @@ TEST_F(FlushJobTest, NonEmpty) {
  SnapshotChecker* snapshot_checker = nullptr;  // not relavant
  FlushJob flush_job(
      dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_,
-      *cfd->GetLatestMutableCFOptions(),
+      cfd->GetLatestMutableCFOptions(),
      std::numeric_limits<uint64_t>::max() /* memtable_id */, env_options_,
      versions_.get(), &mutex_, &shutting_down_, {}, kMaxSequenceNumber,
      snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr,
@@ -310,7 +310,7 @@ TEST_F(FlushJobTest, FlushMemTablesSingleColumnFamily) {
  std::vector<uint64_t> memtable_ids;
  std::vector<MemTable*> new_mems;
  for (size_t i = 0; i != num_mems; ++i) {
-    MemTable* mem = cfd->ConstructNewMemtable(*cfd->GetLatestMutableCFOptions(),
+    MemTable* mem = cfd->ConstructNewMemtable(cfd->GetLatestMutableCFOptions(),
                                              kMaxSequenceNumber);
    mem->SetID(i);
    mem->Ref();
@@ -339,7 +339,7 @@ TEST_F(FlushJobTest, FlushMemTablesSingleColumnFamily) {
  uint64_t flush_memtable_id = smallest_memtable_id + num_mems_to_flush - 1;
  FlushJob flush_job(
      dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_,
-      *cfd->GetLatestMutableCFOptions(), flush_memtable_id, env_options_,
+      cfd->GetLatestMutableCFOptions(), flush_memtable_id, env_options_,
      versions_.get(), &mutex_, &shutting_down_, {}, kMaxSequenceNumber,
      snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr,
      nullptr, kNoCompression, db_options_.statistics.get(), &event_logger,
@@ -387,7 +387,7 @@ TEST_F(FlushJobTest, FlushMemtablesMultipleColumnFamilies) {
    smallest_seqs.push_back(curr_seqno);
    for (size_t i = 0; i != num_memtables[k]; ++i) {
      MemTable* mem = cfd->ConstructNewMemtable(
-          *cfd->GetLatestMutableCFOptions(), kMaxSequenceNumber);
+          cfd->GetLatestMutableCFOptions(), kMaxSequenceNumber);
      mem->SetID(i);
      mem->Ref();

@@ -411,7 +411,7 @@ TEST_F(FlushJobTest, FlushMemtablesMultipleColumnFamilies) {
  for (auto cfd : all_cfds) {
    std::vector<SequenceNumber> snapshot_seqs;
    flush_jobs.emplace_back(new FlushJob(
-        dbname_, cfd, db_options_, *cfd->GetLatestMutableCFOptions(),
+        dbname_, cfd, db_options_, cfd->GetLatestMutableCFOptions(),
        memtable_ids[k], env_options_, versions_.get(), &mutex_,
        &shutting_down_, snapshot_seqs, kMaxSequenceNumber, snapshot_checker,
        &job_context, FlushReason::kTest, nullptr, nullptr, nullptr,
@@ -444,10 +444,6 @@ TEST_F(FlushJobTest, FlushMemtablesMultipleColumnFamilies) {
    const auto& mems = flush_jobs[i]->GetMemTables();
    mems_list.push_back(&mems);
  }
-  autovector<const MutableCFOptions*> mutable_cf_options_list;
-  for (auto cfd : all_cfds) {
-    mutable_cf_options_list.push_back(cfd->GetLatestMutableCFOptions());
-  }
  autovector<std::list<std::unique_ptr<FlushJobInfo>>*>
      committed_flush_jobs_info;
  for (auto& job : flush_jobs) {
@@ -455,8 +451,8 @@ TEST_F(FlushJobTest, FlushMemtablesMultipleColumnFamilies) {
  }

  Status s = InstallMemtableAtomicFlushResults(
-      nullptr /* imm_lists */, all_cfds, mutable_cf_options_list, mems_list,
-      versions_.get(), nullptr /* prep_tracker */, &mutex_, file_meta_ptrs,
+      nullptr /* imm_lists */, all_cfds, mems_list, versions_.get(),
+      nullptr /* prep_tracker */, &mutex_, file_meta_ptrs,
      committed_flush_jobs_info, &job_context.memtables_to_free,
      nullptr /* db_directory */, nullptr /* log_buffer */);
  ASSERT_OK(s);
@@ -489,7 +485,7 @@ TEST_F(FlushJobTest, FlushMemtablesMultipleColumnFamilies) {
 TEST_F(FlushJobTest, Snapshots) {
  JobContext job_context(0);
  auto cfd = versions_->GetColumnFamilySet()->GetDefault();
-  auto new_mem = cfd->ConstructNewMemtable(*cfd->GetLatestMutableCFOptions(),
+  auto new_mem = cfd->ConstructNewMemtable(cfd->GetLatestMutableCFOptions(),
                                           kMaxSequenceNumber);

  std::set<SequenceNumber> snapshots_set;
@@ -539,7 +535,7 @@ TEST_F(FlushJobTest, Snapshots) {
  SnapshotChecker* snapshot_checker = nullptr;  // not relavant
  FlushJob flush_job(
      dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_,
-      *cfd->GetLatestMutableCFOptions(),
+      cfd->GetLatestMutableCFOptions(),
      std::numeric_limits<uint64_t>::max() /* memtable_id */, env_options_,
      versions_.get(), &mutex_, &shutting_down_, snapshots, kMaxSequenceNumber,
      snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr,
@@ -567,7 +563,7 @@ TEST_F(FlushJobTest, GetRateLimiterPriorityForWrite) {
  std::vector<uint64_t> memtable_ids;
  std::vector<MemTable*> new_mems;
  for (size_t i = 0; i != num_mems; ++i) {
-    MemTable* mem = cfd->ConstructNewMemtable(*cfd->GetLatestMutableCFOptions(),
+    MemTable* mem = cfd->ConstructNewMemtable(cfd->GetLatestMutableCFOptions(),
                                              kMaxSequenceNumber);
    mem->SetID(i);
    mem->Ref();
@@ -596,7 +592,7 @@ TEST_F(FlushJobTest, GetRateLimiterPriorityForWrite) {
  uint64_t flush_memtable_id = smallest_memtable_id + num_mems_to_flush - 1;
  FlushJob flush_job(
      dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_,
-      *cfd->GetLatestMutableCFOptions(), flush_memtable_id, env_options_,
+      cfd->GetLatestMutableCFOptions(), flush_memtable_id, env_options_,
      versions_.get(), &mutex_, &shutting_down_, {}, kMaxSequenceNumber,
      snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr,
      nullptr, kNoCompression, db_options_.statistics.get(), &event_logger,
@@ -627,7 +623,7 @@ TEST_F(FlushJobTest, GetRateLimiterPriorityForWrite) {
 TEST_F(FlushJobTest, ReplaceTimedPutWriteTimeWithPreferredSeqno) {
  JobContext job_context(0);
  auto cfd = versions_->GetColumnFamilySet()->GetDefault();
-  auto new_mem = cfd->ConstructNewMemtable(*cfd->GetLatestMutableCFOptions(),
+  auto new_mem = cfd->ConstructNewMemtable(cfd->GetLatestMutableCFOptions(),
                                           kMaxSequenceNumber);
  new_mem->Ref();
  std::shared_ptr<SeqnoToTimeMapping> seqno_to_time_mapping =
@@ -665,7 +661,7 @@ TEST_F(FlushJobTest, ReplaceTimedPutWriteTimeWithPreferredSeqno) {
  SnapshotChecker* snapshot_checker = nullptr;  // not relevant
  FlushJob flush_job(
      dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_,
-      *cfd->GetLatestMutableCFOptions(),
+      cfd->GetLatestMutableCFOptions(),
      std::numeric_limits<uint64_t>::max() /* memtable_id */, env_options_,
      versions_.get(), &mutex_, &shutting_down_, {}, kMaxSequenceNumber,
      snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr,
@@ -748,7 +744,7 @@ TEST_P(FlushJobTimestampTest, AllKeysExpired) {

  {
    MemTable* new_mem = cfd->ConstructNewMemtable(
-        *cfd->GetLatestMutableCFOptions(), kMaxSequenceNumber);
+        cfd->GetLatestMutableCFOptions(), kMaxSequenceNumber);
    new_mem->Ref();
    for (int i = 0; i < 100; ++i) {
      uint64_t ts = curr_ts_.fetch_add(1);
@@ -772,7 +768,7 @@ TEST_P(FlushJobTimestampTest, AllKeysExpired) {
  PutFixed64(&full_history_ts_low, std::numeric_limits<uint64_t>::max());
  cfd->SetFullHistoryTsLow(full_history_ts_low);
  FlushJob flush_job(
-      dbname_, cfd, db_options_, *cfd->GetLatestMutableCFOptions(),
+      dbname_, cfd, db_options_, cfd->GetLatestMutableCFOptions(),
      std::numeric_limits<uint64_t>::max() /* memtable_id */, env_options_,
      versions_.get(), &mutex_, &shutting_down_, snapshots, kMaxSequenceNumber,
      snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr,
@@ -814,7 +810,7 @@ TEST_P(FlushJobTimestampTest, NoKeyExpired) {

  {
    MemTable* new_mem = cfd->ConstructNewMemtable(
-        *cfd->GetLatestMutableCFOptions(), kMaxSequenceNumber);
+        cfd->GetLatestMutableCFOptions(), kMaxSequenceNumber);
    new_mem->Ref();
    for (int i = 0; i < 100; ++i) {
      uint64_t ts = curr_ts_.fetch_add(1);
@@ -834,7 +830,7 @@ TEST_P(FlushJobTimestampTest, NoKeyExpired) {
  PutFixed64(&full_history_ts_low, 0);
  cfd->SetFullHistoryTsLow(full_history_ts_low);
  FlushJob flush_job(
-      dbname_, cfd, db_options_, *cfd->GetLatestMutableCFOptions(),
+      dbname_, cfd, db_options_, cfd->GetLatestMutableCFOptions(),
      std::numeric_limits<uint64_t>::max() /* memtable_id */, env_options_,
      versions_.get(), &mutex_, &shutting_down_, snapshots, kMaxSequenceNumber,
      snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr,
@@ -242,7 +242,7 @@ ForwardIterator::ForwardIterator(DBImpl* db, const ReadOptions& read_options,
  if (sv_) {
    RebuildIterators(false);
  }
-  if (!CheckFSFeatureSupport(cfd_->ioptions()->env->GetFileSystem().get(),
+  if (!CheckFSFeatureSupport(cfd_->ioptions().env->GetFileSystem().get(),
                             FSSupportedOps::kAsyncIO)) {
    read_options_.async_io = false;
  }
@@ -4,13 +4,12 @@
 //  (found in the LICENSE.Apache file in the root directory).
 #pragma once

-#include "rocksdb/comparator.h"
-
 #include <queue>
 #include <string>
 #include <vector>

 #include "memory/arena.h"
+#include "rocksdb/comparator.h"
 #include "rocksdb/db.h"
 #include "rocksdb/iterator.h"
 #include "rocksdb/options.h"
@@ -4,8 +4,6 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).

-#include "db/version_builder.h"
-
 #include "db/import_column_family_job.h"

 #include <algorithm>
@@ -13,6 +11,7 @@
 #include <string>
 #include <vector>

+#include "db/version_builder.h"
 #include "db/version_edit.h"
 #include "file/file_util.h"
 #include "file/random_access_file_reader.h"
@@ -109,7 +108,7 @@ Status ImportColumnFamilyJob::Prepare(uint64_t next_file_number,
    for (auto& f : files_to_import_per_cf) {
      const auto path_outside_db = f.external_file_path;
      const auto path_inside_db = TableFileName(
-          cfd_->ioptions()->cf_paths, f.fd.GetNumber(), f.fd.GetPathId());
+          cfd_->ioptions().cf_paths, f.fd.GetNumber(), f.fd.GetPathId());

      if (hardlink_files) {
        status = fs_->LinkFile(path_outside_db, path_inside_db, IOOptions(),
@@ -187,16 +186,16 @@ Status ImportColumnFamilyJob::Run() {
  // RecoverEpochNumbers() below.
  for (size_t i = 0; s.ok() && i < files_to_import_.size(); ++i) {
    VersionBuilder dummy_version_builder(
-        cfd_->current()->version_set()->file_options(), cfd_->ioptions(),
+        cfd_->current()->version_set()->file_options(), &cfd_->ioptions(),
        cfd_->table_cache(), cfd_->current()->storage_info(),
        cfd_->current()->version_set(),
        cfd_->GetFileMetadataCacheReservationManager());
    VersionStorageInfo dummy_vstorage(
        &cfd_->internal_comparator(), cfd_->user_comparator(),
-        cfd_->NumberLevels(), cfd_->ioptions()->compaction_style,
-        nullptr /* src_vstorage */, cfd_->ioptions()->force_consistency_checks,
-        EpochNumberRequirement::kMightMissing, cfd_->ioptions()->clock,
-        cfd_->GetLatestMutableCFOptions()->bottommost_file_compaction_delay,
+        cfd_->NumberLevels(), cfd_->ioptions().compaction_style,
+        nullptr /* src_vstorage */, cfd_->ioptions().force_consistency_checks,
+        EpochNumberRequirement::kMightMissing, cfd_->ioptions().clock,
+        cfd_->GetLatestMutableCFOptions().bottommost_file_compaction_delay,
        cfd_->current()->version_set()->offpeak_time_option());
    for (size_t j = 0; s.ok() && j < files_to_import_[i].size(); ++j) {
      const auto& f = files_to_import_[i][j];
@@ -331,7 +330,7 @@ Status ImportColumnFamilyJob::GetIngestedFileInfo(
  //  creating `TableReaderOptions` when the support is there.
  status = sv->mutable_cf_options.table_factory->NewTableReader(
      TableReaderOptions(
-          *cfd_->ioptions(), sv->mutable_cf_options.prefix_extractor,
+          cfd_->ioptions(), sv->mutable_cf_options.prefix_extractor,
          env_options_, cfd_->internal_comparator(),
          sv->mutable_cf_options.block_protection_bytes_per_key,
          /*skip_filters*/ false, /*immortal*/ false,
@@ -4,7 +4,6 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).

-
 #include <functional>

 #include "db/db_test_util.h"
@@ -1057,4 +1056,3 @@ int main(int argc, char** argv) {
  ::testing::InitGoogleTest(&argc, argv);
  return RUN_ALL_TESTS();
 }
-
@@ -924,7 +924,7 @@ bool InternalStats::HandleLiveBlobFileGarbageSize(uint64_t* value,
 }

 Cache* InternalStats::GetBlobCacheForStats() {
-  return cfd_->ioptions()->blob_cache.get();
+  return cfd_->ioptions().blob_cache.get();
 }

 bool InternalStats::HandleBlobCacheCapacity(uint64_t* value, DBImpl* /*db*/,
@@ -1468,9 +1468,9 @@ bool InternalStats::HandleEstimateOldestKeyTime(uint64_t* value, DBImpl* /*db*/,
  // TODO(yiwu): The property is currently available for fifo compaction
  // with allow_compaction = false. This is because we don't propagate
  // oldest_key_time on compaction.
-  if (cfd_->ioptions()->compaction_style != kCompactionStyleFIFO ||
+  if (cfd_->ioptions().compaction_style != kCompactionStyleFIFO ||
      cfd_->GetCurrentMutableCFOptions()
-          ->compaction_options_fifo.allow_compaction) {
+          .compaction_options_fifo.allow_compaction) {
    return false;
  }
  // TODO: plumb Env::IOActivity, Env::IOPriority
@@ -1496,7 +1496,7 @@ bool InternalStats::HandleEstimateOldestKeyTime(uint64_t* value, DBImpl* /*db*/,

 Cache* InternalStats::GetBlockCacheForStats() {
  // NOTE: called in startup before GetCurrentMutableCFOptions() is ready
-  auto* table_factory = cfd_->GetLatestMutableCFOptions()->table_factory.get();
+  auto* table_factory = cfd_->GetLatestMutableCFOptions().table_factory.get();
  assert(table_factory != nullptr);
  // FIXME: need to a shared_ptr if/when block_cache is going to be mutable
  return table_factory->GetOptions<Cache>(TableFactory::kBlockCacheOpts());
@@ -1752,7 +1752,7 @@ void InternalStats::DumpCFMapStats(
  assert(vstorage);

  int num_levels_to_check =
-      (cfd_->ioptions()->compaction_style == kCompactionStyleLevel)
+      (cfd_->ioptions().compaction_style == kCompactionStyleLevel)
          ? vstorage->num_levels() - 1
          : 1;

@@ -2163,8 +2163,7 @@ class BlockCachePropertyAggregator : public IntPropertyAggregator {
  virtual ~BlockCachePropertyAggregator() override = default;

  void Add(ColumnFamilyData* cfd, uint64_t value) override {
-    auto* table_factory =
-        cfd->GetCurrentMutableCFOptions()->table_factory.get();
+    auto* table_factory = cfd->GetCurrentMutableCFOptions().table_factory.get();
    assert(table_factory != nullptr);
    Cache* cache =
        table_factory->GetOptions<Cache>(TableFactory::kBlockCacheOpts());
@@ -33,7 +33,6 @@
 #include "util/string_util.h"
 #include "utilities/merge_operators.h"

-
 namespace ROCKSDB_NAMESPACE {

 class EventListenerTest : public DBTestBase {
@@ -1591,7 +1590,6 @@ TEST_F(EventListenerTest, BlobDBFileTest) {

 }  // namespace ROCKSDB_NAMESPACE

-
 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
@@ -12,12 +12,14 @@

 #pragma once

+#include <cstdint>
+
 #include "rocksdb/rocksdb_namespace.h"

 namespace ROCKSDB_NAMESPACE {
 namespace log {

-enum RecordType {
+enum RecordType : uint8_t {
  // Zero is reserved for preallocated files
  kZeroType = 0,
  kFullType = 1,
@@ -36,11 +38,18 @@ enum RecordType {
  // Compression Type
  kSetCompressionType = 9,

+  // For all the values >= 10, the 1 bit indicates whether it's recyclable
  // User-defined timestamp sizes
  kUserDefinedTimestampSizeType = 10,
  kRecyclableUserDefinedTimestampSizeType = 11,
+
+  // For WAL verification
+  kPredecessorWALInfoType = 130,
+  kRecyclePredecessorWALInfoType = 131,
 };
-constexpr int kMaxRecordType = kRecyclableUserDefinedTimestampSizeType;
+// Unknown type of value with the 8-th bit set will be ignored
+constexpr uint8_t kRecordTypeSafeIgnoreMask = 1 << 7;
+constexpr uint8_t kMaxRecordType = kRecyclePredecessorWALInfoType;

 constexpr unsigned int kBlockSize = 32768;

@@ -24,7 +24,10 @@ Reader::Reporter::~Reporter() = default;

 Reader::Reader(std::shared_ptr<Logger> info_log,
               std::unique_ptr<SequentialFileReader>&& _file,
-               Reporter* reporter, bool checksum, uint64_t log_num)
+               Reporter* reporter, bool checksum, uint64_t log_num,
+               bool track_and_verify_wals, bool stop_replay_for_corruption,
+               uint64_t min_wal_number_to_keep,
+               const PredecessorWALInfo& observed_predecessor_wal_info)
    : info_log_(info_log),
      file_(std::move(_file)),
      reporter_(reporter),
@@ -37,13 +40,17 @@ Reader::Reader(std::shared_ptr<Logger> info_log,
      last_record_offset_(0),
      end_of_buffer_offset_(0),
      log_number_(log_num),
+      track_and_verify_wals_(track_and_verify_wals),
+      stop_replay_for_corruption_(stop_replay_for_corruption),
+      min_wal_number_to_keep_(min_wal_number_to_keep),
+      observed_predecessor_wal_info_(observed_predecessor_wal_info),
      recycled_(false),
      first_record_read_(false),
      compression_type_(kNoCompression),
      compression_type_record_read_(false),
      uncompress_(nullptr),
      hash_state_(nullptr),
-      uncompress_hash_state_(nullptr){}
+      uncompress_hash_state_(nullptr) {}

 Reader::~Reader() {
  delete[] backing_store_;
@@ -65,6 +72,9 @@ Reader::~Reader() {
 //
 // TODO krad: Evaluate if we need to move to a more strict mode where we
 // restrict the inconsistency to only the last log
+// TODO (hx235): move `wal_recovery_mode` to be a member data like other
+// information (e.g, `stop_replay_for_corruption`) to decide whether to
+// check for and surface corruption in `ReadRecord()`
 bool Reader::ReadRecord(Slice* record, std::string* scratch,
                        WALRecoveryMode wal_recovery_mode,
                        uint64_t* record_checksum) {
@@ -88,7 +98,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
  while (true) {
    uint64_t physical_record_offset = end_of_buffer_offset_ - buffer_.size();
    size_t drop_size = 0;
-    const unsigned int record_type =
+    const uint8_t record_type =
        ReadPhysicalRecord(&fragment, &drop_size, record_checksum);
    switch (record_type) {
      case kFullType:
@@ -185,6 +195,23 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
        }
        break;
      }
+      case kPredecessorWALInfoType:
+      case kRecyclePredecessorWALInfoType: {
+        prospective_record_offset = physical_record_offset;
+        scratch->clear();
+        last_record_offset_ = prospective_record_offset;
+
+        PredecessorWALInfo recorded_predecessor_wal_info;
+        Status s = recorded_predecessor_wal_info.DecodeFrom(&fragment);
+        if (!s.ok()) {
+          ReportCorruption(fragment.size(),
+                           "could not decode PredecessorWALInfoType record");
+        } else {
+          MaybeVerifyPredecessorWALInfo(wal_recovery_mode, fragment,
+                                        recorded_predecessor_wal_info);
+        }
+        break;
+      }
      case kUserDefinedTimestampSizeType:
      case kRecyclableUserDefinedTimestampSizeType: {
        if (in_fragmented_record && !scratch->empty()) {
@@ -313,11 +340,13 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
        break;

      default: {
-        std::string reason =
-            "unknown record type " + std::to_string(record_type);
-        ReportCorruption(
-            (fragment.size() + (in_fragmented_record ? scratch->size() : 0)),
-            reason.c_str());
+        if ((record_type & kRecordTypeSafeIgnoreMask) == 0) {
+          std::string reason =
+              "unknown record type " + std::to_string(record_type);
+          ReportCorruption(
+              (fragment.size() + (in_fragmented_record ? scratch->size() : 0)),
+              reason.c_str());
+        }
        in_fragmented_record = false;
        scratch->clear();
        break;
@@ -327,6 +356,57 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
  return false;
 }

+void Reader::MaybeVerifyPredecessorWALInfo(
+    WALRecoveryMode wal_recovery_mode, Slice fragment,
+    const PredecessorWALInfo& recorded_predecessor_wal_info) {
+  if (!track_and_verify_wals_ ||
+      wal_recovery_mode == WALRecoveryMode::kSkipAnyCorruptedRecords ||
+      stop_replay_for_corruption_) {
+    return;
+  }
+  assert(recorded_predecessor_wal_info.IsInitialized());
+  uint64_t recorded_predecessor_log_number =
+      recorded_predecessor_wal_info.GetLogNumber();
+
+  // This is the first WAL recovered thus with no predecessor WAL info has been
+  // initialized
+  if (!observed_predecessor_wal_info_.IsInitialized()) {
+    if (recorded_predecessor_log_number >= min_wal_number_to_keep_) {
+      std::string reason = "Missing WAL of log number " +
+                           std::to_string(recorded_predecessor_log_number);
+      ReportCorruption(fragment.size(), reason.c_str());
+    }
+  } else {
+    if (observed_predecessor_wal_info_.GetLogNumber() !=
+        recorded_predecessor_log_number) {
+      std::string reason = "Missing WAL of log number " +
+                           std::to_string(recorded_predecessor_log_number);
+      ReportCorruption(fragment.size(), reason.c_str());
+    } else if (observed_predecessor_wal_info_.GetLastSeqnoRecorded() !=
+               recorded_predecessor_wal_info.GetLastSeqnoRecorded()) {
+      std::string reason =
+          "Mismatched last sequence number recorded in the WAL of log number " +
+          std::to_string(recorded_predecessor_log_number) + ". Recorded " +
+          std::to_string(recorded_predecessor_wal_info.GetLastSeqnoRecorded()) +
+          ". Observed " +
+          std::to_string(
+              observed_predecessor_wal_info_.GetLastSeqnoRecorded()) +
+          ". (Last sequence number equal to 0 indicates no WAL records)";
+      ReportCorruption(fragment.size(), reason.c_str());
+    } else if (observed_predecessor_wal_info_.GetSizeBytes() !=
+               recorded_predecessor_wal_info.GetSizeBytes()) {
+      std::string reason =
+          "Mismatched size of the WAL of log number " +
+          std::to_string(recorded_predecessor_log_number) + ". Recorded " +
+          std::to_string(recorded_predecessor_wal_info.GetSizeBytes()) +
+          " bytes. Observed " +
+          std::to_string(observed_predecessor_wal_info_.GetSizeBytes()) +
+          " bytes.";
+      ReportCorruption(fragment.size(), reason.c_str());
+    }
+  }
+}
+
 uint64_t Reader::LastRecordOffset() { return last_record_offset_; }

 uint64_t Reader::LastRecordEnd() {
@@ -419,7 +499,7 @@ void Reader::ReportOldLogRecord(size_t bytes) {
  }
 }

-bool Reader::ReadMore(size_t* drop_size, int* error) {
+bool Reader::ReadMore(size_t* drop_size, uint8_t* error) {
  if (!eof_ && !read_error_) {
    // Last read was a full read, so this is a trailer to skip
    buffer_.clear();
@@ -460,15 +540,15 @@ bool Reader::ReadMore(size_t* drop_size, int* error) {
  }
 }

-unsigned int Reader::ReadPhysicalRecord(Slice* result, size_t* drop_size,
-                                        uint64_t* fragment_checksum) {
+uint8_t Reader::ReadPhysicalRecord(Slice* result, size_t* drop_size,
+                                   uint64_t* fragment_checksum) {
  while (true) {
    // We need at least the minimum header size
    if (buffer_.size() < static_cast<size_t>(kHeaderSize)) {
      // the default value of r is meaningless because ReadMore will overwrite
      // it if it returns false; in case it returns true, the return value will
      // not be used anyway
-      int r = kEof;
+      uint8_t r = kEof;
      if (!ReadMore(drop_size, &r)) {
        return r;
      }
@@ -479,12 +559,13 @@ unsigned int Reader::ReadPhysicalRecord(Slice* result, size_t* drop_size,
    const char* header = buffer_.data();
    const uint32_t a = static_cast<uint32_t>(header[4]) & 0xff;
    const uint32_t b = static_cast<uint32_t>(header[5]) & 0xff;
-    const unsigned int type = header[6];
+    const uint8_t type = static_cast<uint8_t>(header[6]);
    const uint32_t length = a | (b << 8);
    int header_size = kHeaderSize;
    const bool is_recyclable_type =
        ((type >= kRecyclableFullType && type <= kRecyclableLastType) ||
-         type == kRecyclableUserDefinedTimestampSizeType);
+         type == kRecyclableUserDefinedTimestampSizeType ||
+         type == kRecyclePredecessorWALInfoType);
    if (is_recyclable_type) {
      header_size = kRecyclableHeaderSize;
      if (first_record_read_ && !recycled_) {
@@ -494,7 +575,7 @@ unsigned int Reader::ReadPhysicalRecord(Slice* result, size_t* drop_size,
      recycled_ = true;
      // We need enough for the larger header
      if (buffer_.size() < static_cast<size_t>(kRecyclableHeaderSize)) {
-        int r = kEof;
+        uint8_t r = kEof;
        if (!ReadMore(drop_size, &r)) {
          return r;
        }
@@ -549,6 +630,8 @@ unsigned int Reader::ReadPhysicalRecord(Slice* result, size_t* drop_size,
    buffer_.remove_prefix(header_size + length);

    if (!uncompress_ || type == kSetCompressionType ||
+        type == kPredecessorWALInfoType ||
+        type == kRecyclePredecessorWALInfoType ||
        type == kUserDefinedTimestampSizeType ||
        type == kRecyclableUserDefinedTimestampSizeType) {
      *result = Slice(header + header_size, length);
@@ -638,7 +721,9 @@ Status Reader::UpdateRecordedTimestampSize(
 }

 bool FragmentBufferedReader::ReadRecord(Slice* record, std::string* scratch,
-                                        WALRecoveryMode /*unused*/,
+                                        WALRecoveryMode wal_recovery_mode
+
+                                        ,
                                        uint64_t* /* checksum */) {
  assert(record != nullptr);
  assert(scratch != nullptr);
@@ -651,7 +736,7 @@ bool FragmentBufferedReader::ReadRecord(Slice* record, std::string* scratch,
  uint64_t prospective_record_offset = 0;
  uint64_t physical_record_offset = end_of_buffer_offset_ - buffer_.size();
  size_t drop_size = 0;
-  unsigned int fragment_type_or_err = 0;  // Initialize to make compiler happy
+  uint8_t fragment_type_or_err = 0;  // Initialize to make compiler happy
  Slice fragment;
  while (TryReadFragment(&fragment, &drop_size, &fragment_type_or_err)) {
    switch (fragment_type_or_err) {
@@ -728,7 +813,24 @@ bool FragmentBufferedReader::ReadRecord(Slice* record, std::string* scratch,
        }
        break;
      }
+      case kPredecessorWALInfoType:
+      case kRecyclePredecessorWALInfoType: {
+        fragments_.clear();
+        prospective_record_offset = physical_record_offset;
+        last_record_offset_ = prospective_record_offset;
+        in_fragmented_record_ = false;

+        PredecessorWALInfo recorded_predecessor_wal_info;
+        Status s = recorded_predecessor_wal_info.DecodeFrom(&fragment);
+        if (!s.ok()) {
+          ReportCorruption(fragment.size(),
+                           "could not decode PredecessorWALInfoType record");
+        } else {
+          MaybeVerifyPredecessorWALInfo(wal_recovery_mode, fragment,
+                                        recorded_predecessor_wal_info);
+        }
+        break;
+      }
      case kUserDefinedTimestampSizeType:
      case kRecyclableUserDefinedTimestampSizeType: {
        if (in_fragmented_record_ && !scratch->empty()) {
@@ -781,11 +883,13 @@ bool FragmentBufferedReader::ReadRecord(Slice* record, std::string* scratch,
        break;

      default: {
-        std::string reason =
-            "unknown record type " + std::to_string(fragment_type_or_err);
-        ReportCorruption(
-            fragment.size() + (in_fragmented_record_ ? fragments_.size() : 0),
-            reason.c_str());
+        if ((fragment_type_or_err & kRecordTypeSafeIgnoreMask) == 0) {
+          std::string reason =
+              "unknown record type " + std::to_string(fragment_type_or_err);
+          ReportCorruption(
+              fragment.size() + (in_fragmented_record_ ? fragments_.size() : 0),
+              reason.c_str());
+        }
        in_fragmented_record_ = false;
        fragments_.clear();
        break;
@@ -803,7 +907,7 @@ void FragmentBufferedReader::UnmarkEOF() {
  UnmarkEOFInternal();
 }

-bool FragmentBufferedReader::TryReadMore(size_t* drop_size, int* error) {
+bool FragmentBufferedReader::TryReadMore(size_t* drop_size, uint8_t* error) {
  if (!eof_ && !read_error_) {
    // Last read was a full read, so this is a trailer to skip
    buffer_.clear();
@@ -844,15 +948,15 @@ bool FragmentBufferedReader::TryReadMore(size_t* drop_size, int* error) {
 }

 // return true if the caller should process the fragment_type_or_err.
-bool FragmentBufferedReader::TryReadFragment(
-    Slice* fragment, size_t* drop_size, unsigned int* fragment_type_or_err) {
+bool FragmentBufferedReader::TryReadFragment(Slice* fragment, size_t* drop_size,
+                                             uint8_t* fragment_type_or_err) {
  assert(fragment != nullptr);
  assert(drop_size != nullptr);
  assert(fragment_type_or_err != nullptr);

  while (buffer_.size() < static_cast<size_t>(kHeaderSize)) {
    size_t old_size = buffer_.size();
-    int error = kEof;
+    uint8_t error = kEof;
    if (!TryReadMore(drop_size, &error)) {
      *fragment_type_or_err = error;
      return false;
@@ -863,11 +967,12 @@ bool FragmentBufferedReader::TryReadFragment(
  const char* header = buffer_.data();
  const uint32_t a = static_cast<uint32_t>(header[4]) & 0xff;
  const uint32_t b = static_cast<uint32_t>(header[5]) & 0xff;
-  const unsigned int type = header[6];
+  const uint8_t type = static_cast<uint8_t>(header[6]);
  const uint32_t length = a | (b << 8);
  int header_size = kHeaderSize;
  if ((type >= kRecyclableFullType && type <= kRecyclableLastType) ||
-      type == kRecyclableUserDefinedTimestampSizeType) {
+      type == kRecyclableUserDefinedTimestampSizeType ||
+      type == kRecyclePredecessorWALInfoType) {
    if (first_record_read_ && !recycled_) {
      // A recycled log should have started with a recycled record
      *fragment_type_or_err = kBadRecord;
@@ -877,7 +982,7 @@ bool FragmentBufferedReader::TryReadFragment(
    header_size = kRecyclableHeaderSize;
    while (buffer_.size() < static_cast<size_t>(kRecyclableHeaderSize)) {
      size_t old_size = buffer_.size();
-      int error = kEof;
+      uint8_t error = kEof;
      if (!TryReadMore(drop_size, &error)) {
        *fragment_type_or_err = error;
        return false;
@@ -894,7 +999,7 @@ bool FragmentBufferedReader::TryReadFragment(

  while (header_size + length > buffer_.size()) {
    size_t old_size = buffer_.size();
-    int error = kEof;
+    uint8_t error = kEof;
    if (!TryReadMore(drop_size, &error)) {
      *fragment_type_or_err = error;
      return false;
@@ -923,6 +1028,8 @@ bool FragmentBufferedReader::TryReadFragment(
  buffer_.remove_prefix(header_size + length);

  if (!uncompress_ || type == kSetCompressionType ||
+      type == kPredecessorWALInfoType ||
+      type == kRecyclePredecessorWALInfoType ||
      type == kUserDefinedTimestampSizeType ||
      type == kRecyclableUserDefinedTimestampSizeType) {
    *fragment = Slice(header + header_size, length);
@@ -10,6 +10,7 @@
 #pragma once
 #include <stdint.h>

+#include <cstdint>
 #include <memory>
 #include <unordered_map>
 #include <vector>
@@ -57,9 +58,15 @@ class Reader {
  // live while this Reader is in use.
  //
  // If "checksum" is true, verify checksums if available.
+  // TODO(hx235): seperate WAL related parameters from general `Reader`
+  // parameters
  Reader(std::shared_ptr<Logger> info_log,
         std::unique_ptr<SequentialFileReader>&& file, Reporter* reporter,
-         bool checksum, uint64_t log_num);
+         bool checksum, uint64_t log_num, bool track_and_verify_wals = false,
+         bool stop_replay_for_corruption = false,
+         uint64_t min_wal_number_to_keep = std::numeric_limits<uint64_t>::max(),
+         const PredecessorWALInfo& observed_predecessor_wal_info =
+             PredecessorWALInfo());
  // No copying allowed
  Reader(const Reader&) = delete;
  void operator=(const Reader&) = delete;
@@ -147,6 +154,17 @@ class Reader {
  // which log number this is
  uint64_t const log_number_;

+  // See `Optinos::track_and_verify_wals`
+  bool track_and_verify_wals_;
+  // Below variables are used for WAL verification
+  // TODO(hx235): To revise `stop_replay_for_corruption_` inside `LogReader`
+  // since we have `observed_predecessor_wal_info_` to verify against the
+  // `recorded_predecessor_wal_info_` recorded in current WAL. If there is no
+  // WAL hole, we can revise `stop_replay_for_corruption_` to be false.
+  bool stop_replay_for_corruption_;
+  uint64_t min_wal_number_to_keep_;
+  PredecessorWALInfo observed_predecessor_wal_info_;
+
  // Whether this is a recycled log file
  bool recycled_;

@@ -171,7 +189,7 @@ class Reader {
  UnorderedMap<uint32_t, size_t> recorded_cf_to_ts_sz_;

  // Extend record types with the following special values
-  enum {
+  enum : uint8_t {
    kEof = kMaxRecordType + 1,
    // Returned whenever we find an invalid physical record.
    // Currently there are three situations in which this happens:
@@ -192,11 +210,11 @@ class Reader {
  // If WAL compressioned is enabled, fragment_checksum is the checksum of the
  // fragment computed from the orginal buffer containinng uncompressed
  // fragment.
-  unsigned int ReadPhysicalRecord(Slice* result, size_t* drop_size,
-                                  uint64_t* fragment_checksum = nullptr);
+  uint8_t ReadPhysicalRecord(Slice* result, size_t* drop_size,
+                             uint64_t* fragment_checksum = nullptr);

  // Read some more
-  bool ReadMore(size_t* drop_size, int* error);
+  bool ReadMore(size_t* drop_size, uint8_t* error);

  void UnmarkEOFInternal();

@@ -210,14 +228,25 @@ class Reader {

  Status UpdateRecordedTimestampSize(
      const std::vector<std::pair<uint32_t, size_t>>& cf_to_ts_sz);
+
+  void MaybeVerifyPredecessorWALInfo(
+      WALRecoveryMode wal_recovery_mode, Slice fragment,
+      const PredecessorWALInfo& recorded_predecessor_wal_info);
 };

 class FragmentBufferedReader : public Reader {
 public:
-  FragmentBufferedReader(std::shared_ptr<Logger> info_log,
-                         std::unique_ptr<SequentialFileReader>&& _file,
-                         Reporter* reporter, bool checksum, uint64_t log_num)
-      : Reader(info_log, std::move(_file), reporter, checksum, log_num),
+  FragmentBufferedReader(
+      std::shared_ptr<Logger> info_log,
+      std::unique_ptr<SequentialFileReader>&& _file, Reporter* reporter,
+      bool checksum, uint64_t log_num, bool verify_and_track_wals = false,
+      bool stop_replay_for_corruption = false,
+      uint64_t min_wal_number_to_keep = std::numeric_limits<uint64_t>::max(),
+      const PredecessorWALInfo& observed_predecessor_wal_info =
+          PredecessorWALInfo())
+      : Reader(info_log, std::move(_file), reporter, checksum, log_num,
+               verify_and_track_wals, stop_replay_for_corruption,
+               min_wal_number_to_keep, observed_predecessor_wal_info),
        fragments_(),
        in_fragmented_record_(false) {}
  ~FragmentBufferedReader() override {}
@@ -232,9 +261,9 @@ class FragmentBufferedReader : public Reader {
  bool in_fragmented_record_;

  bool TryReadFragment(Slice* result, size_t* drop_size,
-                       unsigned int* fragment_type_or_err);
+                       uint8_t* fragment_type_or_err);

-  bool TryReadMore(size_t* drop_size, int* error);
+  bool TryReadMore(size_t* drop_size, uint8_t* error);

  // No copy allowed
  FragmentBufferedReader(const FragmentBufferedReader&);